From 77b25662b42968eb19cac6c43b1db1f46a62614c Mon Sep 17 00:00:00 2001 From: 134 Date: Sat, 4 Jul 2026 13:40:02 +0800 Subject: [PATCH 01/70] fix: send layout drafts to fal scale edits --- app/api/canvas/local-edit/route.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 004fa0011..28be69d65 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -3884,18 +3884,19 @@ export async function POST(request: NextRequest) { isScaleAdjustOperation && resolvedScaleAdjustModelEntry?.provider?.type === "alibaba-image"; const useScaleAdjustCoordinatePrompt = useScaleAdjustCoordinateGuide; + const useScaleAdjustFullFrameFalEdit = + isScaleAdjustOperation && + isScaleAdjustFalNanoBananaEditModel(resolvedScaleAdjustModelEntry); const useFalScaleAdjustCoordinatePrompt = useScaleAdjustCoordinatePrompt && - resolvedScaleAdjustModelEntry?.provider?.type === "fal-image"; + resolvedScaleAdjustModelEntry?.provider?.type === "fal-image" && + !useScaleAdjustFullFrameFalEdit; const useScaleAdjustVisualLayoutGuide = useScaleAdjustCoordinatePrompt && !useFalScaleAdjustCoordinatePrompt && !useProviderTextOnlyCoordinatePrompt; const useScaleAdjustMarkedPhotoGuide = useScaleAdjustCoordinatePrompt && !useScaleAdjustVisualLayoutGuide; - const useScaleAdjustFullFrameFalEdit = - isScaleAdjustOperation && - isScaleAdjustFalNanoBananaEditModel(resolvedScaleAdjustModelEntry); const useScaleAdjustStructureLockedPatchWorkflow = isScaleAdjustOperation && !useScaleAdjustFullFrameFalEdit && From 28080924f8b2ee08aea4b7c8d5a568da783a8ac8 Mon Sep 17 00:00:00 2001 From: 134 Date: Sun, 5 Jul 2026 14:07:49 +0800 Subject: [PATCH 02/70] fix: use source plate for scale target guide --- app/api/canvas/local-edit/route.ts | 173 ++++++++++++++++------------- 1 file changed, 98 insertions(+), 75 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 868acc9a7..e6e3488e0 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -139,7 +139,7 @@ type ScaleAdjustPatchReference = { }; type ScaleAdjustStructureLockedComposite = { compositeClip?: { input: Buffer; left: number; top: number }; - method?: "rmbg-cutout" | "pixel-mask"; + method?: "pixel-mask" | "rect-crop-guide" | "rmbg-cutout"; overlay: { input: Buffer; left: number; top: number }; targetClip?: { input: Buffer; left: number; top: number }; wash: { input: Buffer; left: number; top: number }; @@ -2644,18 +2644,27 @@ function suppressScaleAdjustProtectedFurnitureDarkAlpha(params: { const isProtectedSideArea = (sourceX: number, sourceY: number) => { if (protectedPersonSide === "right") { return ( - (sourceX >= sourceBox.width * 0.54 && sourceY >= sourceBox.height * 0.25) || - (sourceX >= sourceBox.width * 0.68 && sourceY >= sourceBox.height * 0.12) + (sourceX >= sourceBox.width * 0.72 && sourceY >= sourceBox.height * 0.22) || + (sourceX >= sourceBox.width * 0.58 && sourceY >= sourceBox.height * 0.68) ); } if (protectedPersonSide === "left") { return ( - (sourceX <= sourceBox.width * 0.46 && sourceY >= sourceBox.height * 0.25) || - (sourceX <= sourceBox.width * 0.32 && sourceY >= sourceBox.height * 0.12) + (sourceX <= sourceBox.width * 0.28 && sourceY >= sourceBox.height * 0.22) || + (sourceX <= sourceBox.width * 0.42 && sourceY >= sourceBox.height * 0.68) ); } return sourceY >= sourceBox.height * 0.3; }; + const isFarProtectedSideArea = (sourceX: number, sourceY: number) => { + if (protectedPersonSide === "right") { + return sourceX >= sourceBox.width * 0.84 && sourceY >= sourceBox.height * 0.3; + } + if (protectedPersonSide === "left") { + return sourceX <= sourceBox.width * 0.16 && sourceY >= sourceBox.height * 0.3; + } + return sourceY >= sourceBox.height * 0.58; + }; let clearedPixels = 0; let markedPixels = 0; @@ -2674,16 +2683,22 @@ function suppressScaleAdjustProtectedFurnitureDarkAlpha(params: { const sourceX = bbox.left + localX; const sourceY = bbox.top + localY; const inProtectedSideArea = isProtectedSideArea(sourceX, sourceY); + const farProtectedSideArea = isFarProtectedSideArea(sourceX, sourceY); + const likelyHumanPixel = + isScaleAdjustLikelySkinPixel(r, g, b) || + (isScaleAdjustLikelyWhiteFabricPixel(r, g, b, luma, saturation) && + luma < 246 && + saturation < 0.24); const neutralDark = luma < 150 && (saturation < 0.55 || (r < 110 && g < 110 && b < 110)); const lowContrastDark = luma < 132 && saturation < 0.32; const shouldClear = - inProtectedSideArea || - neutralDark || - lowContrastDark || - luma < 100 || - (luma < 132 && saturation < 0.22); + (inProtectedSideArea && likelyHumanPixel) || + (farProtectedSideArea && + (likelyHumanPixel || + (neutralDark && luma < 72) || + (lowContrastDark && saturation < 0.18))); if (!shouldClear) continue; output[alphaIndex] = 0; @@ -2821,10 +2836,26 @@ async function buildScaleAdjustStructureLockedComposite(params: { }); const sourceWashMask = dilateScaleAdjustStructureMask({ height: placement.sourceBox.height, - mask: subjectMask, + mask: protectedPersonSide + ? dilateScaleAdjustStructureMask({ + height: placement.sourceBox.height, + mask, + radius: Math.max( + 3, + Math.round( + Math.min(placement.sourceBox.width, placement.sourceBox.height) * + 0.034 + ) + ), + width: placement.sourceBox.width, + }) + : subjectMask, radius: Math.max( 2, - Math.round(Math.min(placement.sourceBox.width, placement.sourceBox.height) * 0.012) + Math.round( + Math.min(placement.sourceBox.width, placement.sourceBox.height) * + (protectedPersonSide ? 0.02 : 0.012) + ) ), width: placement.sourceBox.width, }); @@ -2900,16 +2931,42 @@ async function buildScaleAdjustStructureLockedComposite(params: { ((sourceCenterY - placement.sourceFootprintBox.top) / Math.max(1, placement.sourceFootprintBox.height)) * placement.targetBox.height; - const structureOverlay = await sharp(subjectCrop, { failOn: "none" }) - .joinChannel(subjectAlpha) - .resize(placedWidth, placedHeight, { - fit: "fill", - kernel: sharp.kernel.lanczos3, - }) - .png({ compressionLevel: 6 }) - .toBuffer(); + const useRectCropTargetGuide = preferPixelStructure && Boolean(protectedPersonSide); + const structureOverlay = useRectCropTargetGuide + ? await sharp(sourceBuffer, { failOn: "none" }) + .extract(placement.sourceBox) + .resize(placement.placedWidth, placement.placedHeight, { + fit: "fill", + kernel: sharp.kernel.lanczos3, + }) + .ensureAlpha() + .composite([ + { + input: buildScaleAdjustPatchMaskSvg({ + height: placement.placedHeight, + width: placement.placedWidth, + }), + blend: "dest-in", + }, + ]) + .png({ compressionLevel: 6 }) + .toBuffer() + : await sharp(subjectCrop, { failOn: "none" }) + .joinChannel(subjectAlpha) + .resize(placedWidth, placedHeight, { + fit: "fill", + kernel: sharp.kernel.lanczos3, + }) + .png({ compressionLevel: 6 }) + .toBuffer(); const targetClipMaskPatch = await buildScaleAdjustOverlayClipMaskPatch(structureOverlay); + const overlayLeft = useRectCropTargetGuide + ? placement.placedLeft + : Math.round(targetCenterX - placedWidth / 2); + const overlayTop = useRectCropTargetGuide + ? placement.placedTop + : Math.round(targetCenterY - placedHeight / 2); return { compositeClip: { @@ -2917,16 +2974,16 @@ async function buildScaleAdjustStructureLockedComposite(params: { left: placement.sourceBox.left, top: placement.sourceBox.top, }, - method: "pixel-mask", + method: useRectCropTargetGuide ? "rect-crop-guide" : "pixel-mask", overlay: { input: structureOverlay, - left: Math.round(targetCenterX - placedWidth / 2), - top: Math.round(targetCenterY - placedHeight / 2), + left: overlayLeft, + top: overlayTop, }, targetClip: { input: targetClipMaskPatch, - left: Math.round(targetCenterX - placedWidth / 2), - top: Math.round(targetCenterY - placedHeight / 2), + left: overlayLeft, + top: overlayTop, }, wash: { input: sourceWashPatch, @@ -3131,43 +3188,6 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { if (structureLockAppliedRef) { structureLockAppliedRef.value = true; } - if (protectedPersonSide && sourceRaw) { - const washRight = - protectedPersonSide === "right" - ? Math.round(placement.sourceFootprintBox.width * 0.74) - : placement.sourceFootprintBox.width; - const washLeft = - protectedPersonSide === "left" - ? placement.sourceFootprintBox.left + - Math.round(placement.sourceFootprintBox.width * 0.26) - : placement.sourceFootprintBox.left; - const washWidth = - protectedPersonSide === "left" - ? Math.max( - 1, - placement.sourceFootprintBox.width - - Math.round(placement.sourceFootprintBox.width * 0.26) - ) - : Math.max(1, washRight); - sourceFootprintBoxes.push({ - left: washLeft, - top: placement.sourceFootprintBox.top, - width: washWidth, - height: placement.sourceFootprintBox.height, - background: estimateScaleAdjustBoxBackground({ - box: { - left: washLeft, - top: placement.sourceFootprintBox.top, - width: washWidth, - height: placement.sourceFootprintBox.height, - }, - raw: sourceRaw, - width: sourceWidth, - }), - innerOpacity: 0.5, - opacity: 0.72, - }); - } subjectWashComposites.push(structureLockedComposite.wash); if (structureLockedComposite.compositeClip) { compositeClipComposites.push(structureLockedComposite.compositeClip); @@ -4351,22 +4371,23 @@ function buildScaleAdjustFullFramePatchPrompt( : widthFactor > 1.08 || heightFactor > 1.08 ? "make the same selected furniture/product visibly larger" : "move the same selected furniture/product to the final footprint"; - return `Region ${region.index}: original selected search area in the full photo is x=${region.pixelRect.left}, y=${region.pixelRect.top}, w=${region.pixelRect.width}, h=${region.pixelRect.height}; final target footprint in input image 1 is x=${targetPixelRect.left}, y=${targetPixelRect.top}, w=${targetPixelRect.width}, h=${targetPixelRect.height}; requested scale width=${widthFactor.toFixed(2)}, height=${heightFactor.toFixed(2)}. User request: ${region.prompt || action}. Main action: ${action}.`; + return `Region ${region.index}: original selected search area in input image 1 is x=${region.pixelRect.left}, y=${region.pixelRect.top}, w=${region.pixelRect.width}, h=${region.pixelRect.height}; final target footprint shown by input image 2 is x=${targetPixelRect.left}, y=${targetPixelRect.top}, w=${targetPixelRect.width}, h=${targetPixelRect.height}; requested scale width=${widthFactor.toFixed(2)}, height=${heightFactor.toFixed(2)}. User request: ${region.prompt || action}. Main action: ${action}.`; }) .join("\n"); return [ "Full-frame local image edit for furniture/product scale adjustment.", - "Return exactly one clean realistic full image with the same framing, camera, aspect ratio, and canvas extent as input image 2. Do not crop, zoom, pan, or resize the whole photo.", - "Input image 1 is a full-photo target draft. The selected furniture/product has already been moved and scaled with source-photo pixels to the required final footprint. Treat the size, position, silhouette, product structure, material, perspective, wheels, rods, panels, hooks, weave, and visible details in input image 1 as the hard target.", - "Input image 2 is the original clean full photo. Use it only for clean background texture, lighting, perspective, non-target people, non-target objects, and untouched areas. Do not restore the selected furniture/product to its old size or old position from input image 2.", - "AI work is limited to natural background completion, boundary blending, occlusion, contact shadows, and removing rough draft artifacts. Do not repaint, redesign, replace, stretch, duplicate, or distort the selected furniture/product interior.", - "The target draft can contain translucent pasted pixels, faint old-footprint washes, clipped dark clothing fragments, suppressed dark details, or partial mask silhouettes. These are draft artifacts only. Use input image 2 to recover clean product details at the target size, but do not copy any dark blob, black hole, ghost fragment, transparent residue, pasted silhouette, or old-size placement as final photo content.", + "Return exactly one clean realistic full image with the same framing, camera, aspect ratio, and canvas extent as input image 1. Do not crop, zoom, pan, or resize the whole photo.", + "Input image 1 is the original clean full photo and the locked background plate for the final output. Preserve its framing, lighting, perspective, non-target people, non-target objects, and untouched areas.", + "Input image 2 is a full-photo target draft guide. It only shows where the selected furniture/product should end up and how large its final visual footprint should be. Treat the target draft size, center, silhouette, product structure, material, perspective, wheels, rods, panels, hooks, weave, and visible details as the hard target instruction.", + "The selected furniture/product in input image 1 is still at the old size and old position. Do not restore, keep, or prefer that old footprint. Use input image 1 for clean pixels and identity, then move/scale only the selected furniture/product to match input image 2.", + "AI work is limited to moving/scaling the locked selected furniture/product, natural background completion, boundary blending, occlusion, and contact shadows. Do not repaint, redesign, replace, stretch, duplicate, or distort the selected furniture/product interior.", + "The target draft can contain translucent pasted pixels, faint old-footprint washes, clipped fragments, suppressed dark details, partial mask silhouettes, or rectangular crop-guide edges. These are draft artifacts only. Use input image 1 to recover clean product details at the target size, but do not copy any dark blob, black hole, ghost fragment, transparent residue, pasted silhouette, crop rectangle, or old-size placement as final photo content.", "If the target footprint intersects a non-target person, preserve that person as the foreground occluder exactly as in the original photo: do not move, cover, deform, redraw, or change their face, body, clothes, hands, legs, feet, or pose. Blend the furniture/product behind the person naturally where needed.", "Erase the old larger furniture/product footprint outside the final target footprint and rebuild newly visible wall, curtain, floor, rug, shadow, and background naturally. Do not leave ghost copies, old-size remnants, rectangular haze, pale wash blocks, patch edges, black holes, dirty speckles, or visible seams.", "Keep all non-target content stable: person identity and anatomy, sofa, window, curtain, cabinet, lamp, wall art, floor, rug, lighting, textural grain, and global color must remain consistent.", "Do not output boxes, masks, labels, coordinate marks, UI marks, arrows, guide overlays, crop borders, or any annotation artifacts.", - "中文硬约束：输入图1是完整画面的目标草稿，目标家具/商品已经在最终大小和最终位置；输入图2只是原始干净画面参考，绝对不能恢复旧尺寸。只做背景补全、遮挡融合、阴影和边缘清理。草稿里的半透明贴片、暗色残块、黑洞、旧尺寸残影都只是错误遮罩提示，最终必须删除。非目标人物必须保持原样，不能被目标家具盖住、变形、换脸、改姿势或改腿脚。最终必须是一张完整自然照片，不能有矩形雾块、贴片边、黑洞、残影、标注框。", + "中文硬约束：输入图1是原始干净图，也是最终输出的锁定底板；输入图2只是完整画面的目标草稿，用来说明目标家具/商品的最终大小和最终位置。绝对不能把输入图1里的旧尺寸家具保留下来，也不能把输入图2的贴片痕迹当成最终内容。只移动/等比缩放目标家具并做背景补全、遮挡融合、阴影和边缘清理。草稿里的半透明贴片、暗色残块、黑洞、旧尺寸残影、矩形裁切边都只是错误遮罩提示，最终必须删除。非目标人物必须保持原样，不能被目标家具盖住、变形、换脸、改姿势或改腿脚。最终必须是一张完整自然照片，不能有矩形雾块、贴片边、黑洞、残影、标注框。", "", "Region instructions:", regionLines, @@ -4797,8 +4818,8 @@ export async function POST(request: NextRequest) { ? scaleAdjustPatchReference ? useScaleAdjustFullFrameStructurePatch ? [ - scaleAdjustPatchReference.targetDraftFullUrl as string, sourceImageUrl, + scaleAdjustPatchReference.targetDraftFullUrl as string, ...scaleAdjustCropReferenceImageUrls, ...manualRegionReferenceImageUrls, ] @@ -4910,7 +4931,9 @@ export async function POST(request: NextRequest) { ? resolvedScaleAdjustModelEntry?.provider?.type || null : null, cleanSourceReferenceIndex: - scaleAdjustPatchReference && + scaleAdjustPatchReference && useScaleAdjustFullFrameStructurePatch + ? 1 + : scaleAdjustPatchReference && useScaleAdjustPatchTargetDraftPrimary && !includeScaleAdjustPatchSourceReference ? null @@ -4937,7 +4960,7 @@ export async function POST(request: NextRequest) { isScaleAdjustFalNanoBananaEdit && hasScaleAdjustShrink, scaleAdjustPatchReferenceOrder: scaleAdjustPatchReference && useScaleAdjustFullFrameStructurePatch - ? "full-target-draft-full-source" + ? "full-source-full-target-draft" : scaleAdjustPatchReference && useScaleAdjustPatchTargetDraftPrimary ? includeScaleAdjustPatchSourceReference ? "target-draft-source-identity" @@ -5004,9 +5027,9 @@ export async function POST(request: NextRequest) { scaleAdjustTargetRegionCount: regions.filter((region) => region.targetRect) .length, scaleAdjustPromptStrategy: isScaleAdjustOperation - ? scaleAdjustPatchReference - ? useScaleAdjustFullFrameStructurePatch - ? "full-frame-structure-locked-draft-v7-protected-side-cleared-one-shot" + ? scaleAdjustPatchReference + ? useScaleAdjustFullFrameStructurePatch + ? "full-frame-source-locked-target-guide-v8-one-shot" : scaleAdjustPatchReference.structureLockedDraft ? "ai-patch-workspace-structure-locked-draft-v87-subject-alpha-clipped-one-shot" : "ai-patch-workspace-draft-v75-soft-wipe-erase-ghost-one-shot" From c0f376c4cffc1ea320a983c120fb01e1f57cd448 Mon Sep 17 00:00:00 2001 From: 134 Date: Sun, 5 Jul 2026 14:23:41 +0800 Subject: [PATCH 03/70] fix: erase old footprint in scale guide --- app/api/canvas/local-edit/route.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index e6e3488e0..e4c06ba9d 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -3188,6 +3188,18 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { if (structureLockAppliedRef) { structureLockAppliedRef.value = true; } + if (structureLockedComposite.method === "rect-crop-guide" && sourceRaw) { + sourceFootprintBoxes.push({ + ...placement.sourceFootprintBox, + background: estimateScaleAdjustBoxBackground({ + box: placement.sourceFootprintBox, + raw: sourceRaw, + width: sourceWidth, + }), + innerOpacity: 0.7, + opacity: 0.88, + }); + } subjectWashComposites.push(structureLockedComposite.wash); if (structureLockedComposite.compositeClip) { compositeClipComposites.push(structureLockedComposite.compositeClip); @@ -5029,7 +5041,7 @@ export async function POST(request: NextRequest) { scaleAdjustPromptStrategy: isScaleAdjustOperation ? scaleAdjustPatchReference ? useScaleAdjustFullFrameStructurePatch - ? "full-frame-source-locked-target-guide-v8-one-shot" + ? "full-frame-source-locked-target-guide-v9-erased-old-footprint-one-shot" : scaleAdjustPatchReference.structureLockedDraft ? "ai-patch-workspace-structure-locked-draft-v87-subject-alpha-clipped-one-shot" : "ai-patch-workspace-draft-v75-soft-wipe-erase-ghost-one-shot" From c3d8c5d56c96dcd738e471dc6e83202b2d4cdf9a Mon Sep 17 00:00:00 2001 From: 134 Date: Sun, 5 Jul 2026 14:44:34 +0800 Subject: [PATCH 04/70] fix: keep scale guide source-first --- app/api/canvas/local-edit/route.ts | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index e4c06ba9d..e6e3488e0 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -3188,18 +3188,6 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { if (structureLockAppliedRef) { structureLockAppliedRef.value = true; } - if (structureLockedComposite.method === "rect-crop-guide" && sourceRaw) { - sourceFootprintBoxes.push({ - ...placement.sourceFootprintBox, - background: estimateScaleAdjustBoxBackground({ - box: placement.sourceFootprintBox, - raw: sourceRaw, - width: sourceWidth, - }), - innerOpacity: 0.7, - opacity: 0.88, - }); - } subjectWashComposites.push(structureLockedComposite.wash); if (structureLockedComposite.compositeClip) { compositeClipComposites.push(structureLockedComposite.compositeClip); @@ -5041,7 +5029,7 @@ export async function POST(request: NextRequest) { scaleAdjustPromptStrategy: isScaleAdjustOperation ? scaleAdjustPatchReference ? useScaleAdjustFullFrameStructurePatch - ? "full-frame-source-locked-target-guide-v9-erased-old-footprint-one-shot" + ? "full-frame-source-locked-target-guide-v8-one-shot" : scaleAdjustPatchReference.structureLockedDraft ? "ai-patch-workspace-structure-locked-draft-v87-subject-alpha-clipped-one-shot" : "ai-patch-workspace-draft-v75-soft-wipe-erase-ghost-one-shot" From 14e75197cfd91dc3bb55c931cfe8766f26850296 Mon Sep 17 00:00:00 2001 From: 134 Date: Sun, 5 Jul 2026 21:52:48 +0800 Subject: [PATCH 05/70] fix: composite full-frame scale edits --- app/api/canvas/local-edit/route.ts | 26 ++++++++++++++++++-------- lib/canvas/generate-image.ts | 30 ++++++++++++++++++++++-------- 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index e6e3488e0..8137722e3 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -3357,13 +3357,21 @@ async function createScaleAdjustPatchReference(params: { const targetDraftBuffer = targetDraftReference.buffer; const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; if (params.fullFrameReferenceOnly && structureLockAppliedRef.value) { - const targetDraftFullUpload = await uploadFile( - `canvas-scale-adjust-full-target-draft-${suffix}.png`, - toArrayBuffer(targetDraftBuffer) - ); + const [targetDraftFullUpload, compositeClipMaskUpload] = await Promise.all([ + uploadFile( + `canvas-scale-adjust-full-target-draft-${suffix}.png`, + toArrayBuffer(targetDraftBuffer) + ), + targetDraftReference.compositeClipMaskBuffer + ? uploadFile( + `canvas-scale-adjust-composite-clip-mask-${suffix}.png`, + toArrayBuffer(targetDraftReference.compositeClipMaskBuffer) + ) + : Promise.resolve(null), + ]); return { - compositeClipMaskUrl: null, + compositeClipMaskUrl: compositeClipMaskUpload?.url || null, sourceCropUrl: sourceImageUrl || targetDraftFullUpload.url, structureLockedDraft: structureLockAppliedRef.value, targetDraftFullUrl: targetDraftFullUpload.url, @@ -5048,8 +5056,7 @@ export async function POST(request: NextRequest) { : "locked-plate-semantic-target-map-no-old-crop-v15" : undefined, scaleAdjustPostCompositeApplied: - Boolean(scaleAdjustPatchReference) && - !useScaleAdjustFullFrameStructurePatch, + Boolean(scaleAdjustPatchReference), scaleAdjustUsesFullFrameOutput: isScaleAdjustOperation && (!scaleAdjustPatchReference || useScaleAdjustFullFrameStructurePatch), @@ -5087,7 +5094,7 @@ export async function POST(request: NextRequest) { ? { maskImageUrl: scaleAdjustEditMaskUpload.url } : undefined, referenceImages, - scaleAdjustPatchComposite: scaleAdjustPatchReference && !useScaleAdjustFullFrameStructurePatch + scaleAdjustPatchComposite: scaleAdjustPatchReference ? { diffAlphaCutoff: scaleAdjustPatchReference.structureLockedDraft ? 24 : 8, @@ -5103,6 +5110,9 @@ export async function POST(request: NextRequest) { scaleAdjustPatchReference.structureLockedDraft ? 36 : 42, featherMinPx: 10, featherRatio: 0.018, + generatedFrame: useScaleAdjustFullFrameStructurePatch + ? "full" + : "workspace", maskMode: "diff", regions: scaleAdjustPatchCompositeRegions, sourceImageUrl, diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index 77cf52646..acd2a2a09 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -56,6 +56,7 @@ export interface CanvasLocalEditCompositeConfig { export interface CanvasScaleAdjustPatchCompositeConfig { clipMaskImageUrl?: string; + generatedFrame?: "workspace" | "full"; sourceImageUrl: string; workspaceRegion: CanvasLocalEditCompositeRegion; regions?: CanvasLocalEditCompositeRegion[]; @@ -536,6 +537,7 @@ function normalizeScaleAdjustPatchCompositeConfig( max: 0.12, min: 0, }), + generatedFrame: value.generatedFrame === "full" ? "full" : "workspace", }; } @@ -3039,14 +3041,25 @@ async function applyScaleAdjustPatchCompositeToImages(params: { for (const [index, image] of images.entries()) { const generatedUrl = image.originalUrl || image.thumbnailUrl; const generatedBuffer = await fetchImageBufferFromUrl(generatedUrl); - const resizedGeneratedBuffer = await sharp(generatedBuffer, { failOn: "none" }) - .rotate() - .resize(workspaceBox.width, workspaceBox.height, { - fit: "cover", - position: "center", - }) - .png({ compressionLevel: 6 }) - .toBuffer(); + const resizedGeneratedBuffer = + composite.generatedFrame === "full" + ? await sharp(generatedBuffer, { failOn: "none" }) + .rotate() + .resize(sourceWidth, sourceHeight, { + fit: "cover", + position: "center", + }) + .extract(workspaceBox) + .png({ compressionLevel: 6 }) + .toBuffer() + : await sharp(generatedBuffer, { failOn: "none" }) + .rotate() + .resize(workspaceBox.width, workspaceBox.height, { + fit: "cover", + position: "center", + }) + .png({ compressionLevel: 6 }) + .toBuffer(); const diffPatchMask = composite.maskMode === "diff" ? await buildLocalEditDiffCompositeMask({ @@ -3139,6 +3152,7 @@ async function applyScaleAdjustPatchCompositeToImages(params: { imageCount: mergedImages.length, customClipMaskApplied: Boolean(customClipPatchMaskRaw), diffRegionClipApplied: Boolean(regionClipPatchMask), + generatedFrame: composite.generatedFrame || "workspace", maskMode: composite.maskMode || "workspace", maskRegionCount: maskRegionBoxes?.length || 0, sourceHeight, From fa5212a33e68d3a582cbf3aa55da4f5a643333e2 Mon Sep 17 00:00:00 2001 From: 134 Date: Sun, 5 Jul 2026 22:25:56 +0800 Subject: [PATCH 06/70] fix: keep furniture scale drafts mask based --- app/api/canvas/local-edit/route.ts | 46 +++++++----------------------- 1 file changed, 11 insertions(+), 35 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 8137722e3..3f0dc7147 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -2931,42 +2931,18 @@ async function buildScaleAdjustStructureLockedComposite(params: { ((sourceCenterY - placement.sourceFootprintBox.top) / Math.max(1, placement.sourceFootprintBox.height)) * placement.targetBox.height; - const useRectCropTargetGuide = preferPixelStructure && Boolean(protectedPersonSide); - const structureOverlay = useRectCropTargetGuide - ? await sharp(sourceBuffer, { failOn: "none" }) - .extract(placement.sourceBox) - .resize(placement.placedWidth, placement.placedHeight, { - fit: "fill", - kernel: sharp.kernel.lanczos3, - }) - .ensureAlpha() - .composite([ - { - input: buildScaleAdjustPatchMaskSvg({ - height: placement.placedHeight, - width: placement.placedWidth, - }), - blend: "dest-in", - }, - ]) - .png({ compressionLevel: 6 }) - .toBuffer() - : await sharp(subjectCrop, { failOn: "none" }) - .joinChannel(subjectAlpha) - .resize(placedWidth, placedHeight, { - fit: "fill", - kernel: sharp.kernel.lanczos3, - }) - .png({ compressionLevel: 6 }) - .toBuffer(); + const structureOverlay = await sharp(subjectCrop, { failOn: "none" }) + .joinChannel(subjectAlpha) + .resize(placedWidth, placedHeight, { + fit: "fill", + kernel: sharp.kernel.lanczos3, + }) + .png({ compressionLevel: 6 }) + .toBuffer(); const targetClipMaskPatch = await buildScaleAdjustOverlayClipMaskPatch(structureOverlay); - const overlayLeft = useRectCropTargetGuide - ? placement.placedLeft - : Math.round(targetCenterX - placedWidth / 2); - const overlayTop = useRectCropTargetGuide - ? placement.placedTop - : Math.round(targetCenterY - placedHeight / 2); + const overlayLeft = Math.round(targetCenterX - placedWidth / 2); + const overlayTop = Math.round(targetCenterY - placedHeight / 2); return { compositeClip: { @@ -2974,7 +2950,7 @@ async function buildScaleAdjustStructureLockedComposite(params: { left: placement.sourceBox.left, top: placement.sourceBox.top, }, - method: useRectCropTargetGuide ? "rect-crop-guide" : "pixel-mask", + method: "pixel-mask", overlay: { input: structureOverlay, left: overlayLeft, From deee79c5c2fde86aa296557678bbccb65add91ec Mon Sep 17 00:00:00 2001 From: 134 Date: Sun, 5 Jul 2026 22:53:11 +0800 Subject: [PATCH 07/70] fix: avoid compositing dirty full-frame scale drafts --- app/api/canvas/local-edit/route.ts | 54 +++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 3f0dc7147..038dcfacc 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -2931,18 +2931,42 @@ async function buildScaleAdjustStructureLockedComposite(params: { ((sourceCenterY - placement.sourceFootprintBox.top) / Math.max(1, placement.sourceFootprintBox.height)) * placement.targetBox.height; - const structureOverlay = await sharp(subjectCrop, { failOn: "none" }) - .joinChannel(subjectAlpha) - .resize(placedWidth, placedHeight, { - fit: "fill", - kernel: sharp.kernel.lanczos3, - }) - .png({ compressionLevel: 6 }) - .toBuffer(); + const useRectCropTargetGuide = preferPixelStructure && Boolean(protectedPersonSide); + const structureOverlay = useRectCropTargetGuide + ? await sharp(sourceBuffer, { failOn: "none" }) + .extract(placement.sourceBox) + .resize(placement.placedWidth, placement.placedHeight, { + fit: "fill", + kernel: sharp.kernel.lanczos3, + }) + .ensureAlpha() + .composite([ + { + input: buildScaleAdjustPatchMaskSvg({ + height: placement.placedHeight, + width: placement.placedWidth, + }), + blend: "dest-in", + }, + ]) + .png({ compressionLevel: 6 }) + .toBuffer() + : await sharp(subjectCrop, { failOn: "none" }) + .joinChannel(subjectAlpha) + .resize(placedWidth, placedHeight, { + fit: "fill", + kernel: sharp.kernel.lanczos3, + }) + .png({ compressionLevel: 6 }) + .toBuffer(); const targetClipMaskPatch = await buildScaleAdjustOverlayClipMaskPatch(structureOverlay); - const overlayLeft = Math.round(targetCenterX - placedWidth / 2); - const overlayTop = Math.round(targetCenterY - placedHeight / 2); + const overlayLeft = useRectCropTargetGuide + ? placement.placedLeft + : Math.round(targetCenterX - placedWidth / 2); + const overlayTop = useRectCropTargetGuide + ? placement.placedTop + : Math.round(targetCenterY - placedHeight / 2); return { compositeClip: { @@ -2950,7 +2974,7 @@ async function buildScaleAdjustStructureLockedComposite(params: { left: placement.sourceBox.left, top: placement.sourceBox.top, }, - method: "pixel-mask", + method: useRectCropTargetGuide ? "rect-crop-guide" : "pixel-mask", overlay: { input: structureOverlay, left: overlayLeft, @@ -5032,7 +5056,8 @@ export async function POST(request: NextRequest) { : "locked-plate-semantic-target-map-no-old-crop-v15" : undefined, scaleAdjustPostCompositeApplied: - Boolean(scaleAdjustPatchReference), + Boolean(scaleAdjustPatchReference) && + !useScaleAdjustFullFrameStructurePatch, scaleAdjustUsesFullFrameOutput: isScaleAdjustOperation && (!scaleAdjustPatchReference || useScaleAdjustFullFrameStructurePatch), @@ -5070,7 +5095,7 @@ export async function POST(request: NextRequest) { ? { maskImageUrl: scaleAdjustEditMaskUpload.url } : undefined, referenceImages, - scaleAdjustPatchComposite: scaleAdjustPatchReference + scaleAdjustPatchComposite: scaleAdjustPatchReference && !useScaleAdjustFullFrameStructurePatch ? { diffAlphaCutoff: scaleAdjustPatchReference.structureLockedDraft ? 24 : 8, @@ -5086,9 +5111,6 @@ export async function POST(request: NextRequest) { scaleAdjustPatchReference.structureLockedDraft ? 36 : 42, featherMinPx: 10, featherRatio: 0.018, - generatedFrame: useScaleAdjustFullFrameStructurePatch - ? "full" - : "workspace", maskMode: "diff", regions: scaleAdjustPatchCompositeRegions, sourceImageUrl, From 26aa4fb12f956eff71b1e8aa70e6fb6906456545 Mon Sep 17 00:00:00 2001 From: 134 Date: Mon, 6 Jul 2026 01:08:37 +0800 Subject: [PATCH 08/70] fix: lock scale adjust product over repaired background --- app/api/canvas/local-edit/route.ts | 248 ++++++++++++++++++++++++++--- lib/canvas/generate-image.ts | 129 ++++++++++++++- 2 files changed, 354 insertions(+), 23 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 038dcfacc..259fec9d6 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -124,7 +124,10 @@ type ScaleAdjustCropReference = { url: string; }; type ScaleAdjustPatchReference = { + backgroundPlateFullUrl?: string | null; compositeClipMaskUrl?: string | null; + productLockMethod?: string | null; + productLockOverlayUrl?: string | null; sourceCropUrl: string; structureLockedDraft?: boolean; targetDraftFullUrl?: string | null; @@ -139,8 +142,9 @@ type ScaleAdjustPatchReference = { }; type ScaleAdjustStructureLockedComposite = { compositeClip?: { input: Buffer; left: number; top: number }; - method?: "pixel-mask" | "rect-crop-guide" | "rmbg-cutout"; + method?: "pixel-mask" | "rect-crop-guide" | "rect-crop-guide-rmbg-lock" | "rmbg-cutout"; overlay: { input: Buffer; left: number; top: number }; + productLock?: { input: Buffer; left: number; top: number }; targetClip?: { input: Buffer; left: number; top: number }; wash: { input: Buffer; left: number; top: number }; }; @@ -2312,6 +2316,7 @@ async function resizeScaleAdjustAlphaMask(params: { async function buildScaleAdjustRmbgStructureLockedComposite(params: { placement: ReturnType; + protectedPersonSide?: ScaleAdjustProtectedPersonSide | null; raw: Buffer; sourceBuffer: Buffer; sourceImageUrl?: string; @@ -2320,6 +2325,7 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: { }): Promise { const { placement, + protectedPersonSide = null, raw, sourceBuffer, sourceImageUrl, @@ -2372,8 +2378,24 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: { rawRgba: cutoutRaw, width: cutoutWidth, }); + const protectedCleanedAlpha = + protectedPersonSide && sameSizeCutout && sourceCropRaw + ? suppressScaleAdjustProtectedFurnitureDarkAlpha({ + alphaMask: Buffer.from(cleanedAlpha), + bbox: { left: 0, top: 0, width: cutoutWidth, height: cutoutHeight }, + clearProtectedOcclusionZones: true, + protectedPersonSide, + rawRgb: sourceCropRaw, + sourceBox: placement.sourceBox, + }) + : cleanedAlpha; + if (protectedCleanedAlpha !== cleanedAlpha) { + for (let index = 0; index < protectedCleanedAlpha.length; index += 1) { + cleanedRaw[index * 4 + 3] = protectedCleanedAlpha[index] || 0; + } + } const highConfidenceCutoutMask = thresholdScaleAdjustAlphaMask({ - maskBuffer: cleanedAlpha, + maskBuffer: protectedCleanedAlpha, threshold: 96, }); const highConfidenceCutoutBbox = getScaleAdjustMaskBoundingBox({ @@ -2562,6 +2584,7 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: { cutoutWidth, placedHeight, placedWidth, + protectedPersonSide, sameSizeCutout, alphaBandingStats, sourceHeight: placement.sourceBox.height, @@ -2580,6 +2603,11 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: { left: Math.round(targetCenterX - placedWidth / 2), top: Math.round(targetCenterY - placedHeight / 2), }, + productLock: { + input: structureOverlay, + left: Math.round(targetCenterX - placedWidth / 2), + top: Math.round(targetCenterY - placedHeight / 2), + }, targetClip: { input: targetClipMaskPatch, left: Math.round(targetCenterX - placedWidth / 2), @@ -2633,11 +2661,19 @@ function validateScaleAdjustProtectedPersonMask(params: { function suppressScaleAdjustProtectedFurnitureDarkAlpha(params: { alphaMask: Buffer; bbox: { left: number; top: number; width: number; height: number }; + clearProtectedOcclusionZones?: boolean; protectedPersonSide: ScaleAdjustProtectedPersonSide | null; rawRgb: Buffer; sourceBox: { width: number; height: number }; }) { - const { alphaMask, bbox, protectedPersonSide, rawRgb, sourceBox } = params; + const { + alphaMask, + bbox, + clearProtectedOcclusionZones = false, + protectedPersonSide, + rawRgb, + sourceBox, + } = params; if (!protectedPersonSide) return alphaMask; const output = Buffer.from(alphaMask); @@ -2665,6 +2701,26 @@ function suppressScaleAdjustProtectedFurnitureDarkAlpha(params: { } return sourceY >= sourceBox.height * 0.58; }; + const isProtectedOcclusionArea = (sourceX: number, sourceY: number) => { + if (!clearProtectedOcclusionZones) return false; + if (protectedPersonSide === "right") { + return ( + (sourceX >= sourceBox.width * 0.69 && sourceY >= sourceBox.height * 0.54) || + (sourceX >= sourceBox.width * 0.58 && sourceY >= sourceBox.height * 0.7) || + (sourceX >= sourceBox.width * 0.78 && sourceY >= sourceBox.height * 0.4) || + (sourceX >= sourceBox.width * 0.91 && sourceY >= sourceBox.height * 0.26) + ); + } + if (protectedPersonSide === "left") { + return ( + (sourceX <= sourceBox.width * 0.31 && sourceY >= sourceBox.height * 0.54) || + (sourceX <= sourceBox.width * 0.42 && sourceY >= sourceBox.height * 0.7) || + (sourceX <= sourceBox.width * 0.22 && sourceY >= sourceBox.height * 0.4) || + (sourceX <= sourceBox.width * 0.09 && sourceY >= sourceBox.height * 0.26) + ); + } + return sourceY >= sourceBox.height * 0.68; + }; let clearedPixels = 0; let markedPixels = 0; @@ -2694,6 +2750,7 @@ function suppressScaleAdjustProtectedFurnitureDarkAlpha(params: { (saturation < 0.55 || (r < 110 && g < 110 && b < 110)); const lowContrastDark = luma < 132 && saturation < 0.32; const shouldClear = + isProtectedOcclusionArea(sourceX, sourceY) || (inProtectedSideArea && likelyHumanPixel) || (farProtectedSideArea && (likelyHumanPixel || @@ -2743,16 +2800,17 @@ async function buildScaleAdjustStructureLockedComposite(params: { sourceWidth, userId, } = params; - // The current RMBG provider can bake horizontal matte stripes into furniture cutouts. const allowRmbgStructureLock = - !preferPixelStructure && - (process.env.CANVAS_SCALE_ADJUST_ENABLE_RMBG_STRUCTURE === "1" || + process.env.CANVAS_SCALE_ADJUST_ENABLE_RMBG_PRODUCT_LOCK !== "0" && + ((preferPixelStructure && Boolean(protectedPersonSide)) || + process.env.CANVAS_SCALE_ADJUST_ENABLE_RMBG_STRUCTURE === "1" || (includePersonPixels && process.env.CANVAS_SCALE_ADJUST_DISABLE_PERSON_RMBG !== "1")); const rmbgComposite = !allowRmbgStructureLock ? null : await buildScaleAdjustRmbgStructureLockedComposite({ placement, + protectedPersonSide, raw, sourceBuffer, sourceImageUrl, @@ -3133,6 +3191,11 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { left: number; top: number; }> = []; + const productLockComposites: Array<{ + input: Buffer; + left: number; + top: number; + }> = []; for (const region of regions) { const placement = getScaleAdjustVisualGuidePlacement({ @@ -3195,6 +3258,9 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { if (structureLockedComposite.targetClip) { targetClipComposites.push(structureLockedComposite.targetClip); } + if (structureLockedComposite.productLock) { + productLockComposites.push(structureLockedComposite.productLock); + } composites.push(structureLockedComposite.overlay); continue; } @@ -3208,6 +3274,7 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { const shrinkCutoutComposite = shouldUseShrinkCutoutDraft ? await buildScaleAdjustRmbgStructureLockedComposite({ placement, + protectedPersonSide, raw: sourceRaw, sourceBuffer, sourceImageUrl, @@ -3224,6 +3291,9 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { if (shrinkCutoutComposite.targetClip) { targetClipComposites.push(shrinkCutoutComposite.targetClip); } + if (shrinkCutoutComposite.productLock) { + productLockComposites.push(shrinkCutoutComposite.productLock); + } composites.push(shrinkCutoutComposite.overlay); continue; } @@ -3304,6 +3374,28 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { ]) .png({ compressionLevel: 6 }) .toBuffer(); + const backgroundPlateBuffer = productLockComposites.length + ? await sharp(sourceBuffer, { failOn: "none" }) + .composite([ + ...washComposite, + ...subjectWashComposites, + ]) + .png({ compressionLevel: 6 }) + .toBuffer() + : null; + const productLockOverlayBuffer = productLockComposites.length + ? await sharp({ + create: { + background: { alpha: 0, b: 0, g: 0, r: 0 }, + channels: 4, + height: sourceHeight, + width: sourceWidth, + }, + }) + .composite(productLockComposites) + .png({ compressionLevel: 6 }) + .toBuffer() + : null; const compositeClipMaskBuffer = compositeClipComposites.length || targetClipComposites.length ? await buildScaleAdjustCompositeClipMaskBuffer({ @@ -3314,7 +3406,7 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { }) : null; - return { buffer, compositeClipMaskBuffer }; + return { backgroundPlateBuffer, buffer, compositeClipMaskBuffer, productLockOverlayBuffer }; } async function createScaleAdjustPatchReference(params: { @@ -3357,11 +3449,28 @@ async function createScaleAdjustPatchReference(params: { const targetDraftBuffer = targetDraftReference.buffer; const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; if (params.fullFrameReferenceOnly && structureLockAppliedRef.value) { - const [targetDraftFullUpload, compositeClipMaskUpload] = await Promise.all([ + const [ + targetDraftFullUpload, + backgroundPlateFullUpload, + productLockOverlayUpload, + compositeClipMaskUpload, + ] = await Promise.all([ uploadFile( `canvas-scale-adjust-full-target-draft-${suffix}.png`, toArrayBuffer(targetDraftBuffer) ), + targetDraftReference.backgroundPlateBuffer + ? uploadFile( + `canvas-scale-adjust-background-plate-${suffix}.png`, + toArrayBuffer(targetDraftReference.backgroundPlateBuffer) + ) + : Promise.resolve(null), + targetDraftReference.productLockOverlayBuffer + ? uploadFile( + `canvas-scale-adjust-product-lock-overlay-${suffix}.png`, + toArrayBuffer(targetDraftReference.productLockOverlayBuffer) + ) + : Promise.resolve(null), targetDraftReference.compositeClipMaskBuffer ? uploadFile( `canvas-scale-adjust-composite-clip-mask-${suffix}.png`, @@ -3371,7 +3480,10 @@ async function createScaleAdjustPatchReference(params: { ]); return { + backgroundPlateFullUrl: backgroundPlateFullUpload?.url || null, compositeClipMaskUrl: compositeClipMaskUpload?.url || null, + productLockMethod: productLockOverlayUpload ? "rmbg-cutout" : null, + productLockOverlayUrl: productLockOverlayUpload?.url || null, sourceCropUrl: sourceImageUrl || targetDraftFullUpload.url, structureLockedDraft: structureLockAppliedRef.value, targetDraftFullUrl: targetDraftFullUpload.url, @@ -3395,6 +3507,8 @@ async function createScaleAdjustPatchReference(params: { sourceCropUpload, targetDraftUpload, targetDraftFullUpload, + backgroundPlateFullUpload, + productLockOverlayUpload, compositeClipMaskUpload, ] = await Promise.all([ uploadFile( @@ -3409,6 +3523,18 @@ async function createScaleAdjustPatchReference(params: { `canvas-scale-adjust-full-target-draft-${suffix}.png`, toArrayBuffer(targetDraftBuffer) ), + targetDraftReference.backgroundPlateBuffer + ? uploadFile( + `canvas-scale-adjust-background-plate-${suffix}.png`, + toArrayBuffer(targetDraftReference.backgroundPlateBuffer) + ) + : Promise.resolve(null), + targetDraftReference.productLockOverlayBuffer + ? uploadFile( + `canvas-scale-adjust-product-lock-overlay-${suffix}.png`, + toArrayBuffer(targetDraftReference.productLockOverlayBuffer) + ) + : Promise.resolve(null), targetDraftReference.compositeClipMaskBuffer ? uploadFile( `canvas-scale-adjust-composite-clip-mask-${suffix}.png`, @@ -3418,7 +3544,10 @@ async function createScaleAdjustPatchReference(params: { ]); return { + backgroundPlateFullUrl: backgroundPlateFullUpload?.url || null, compositeClipMaskUrl: compositeClipMaskUpload?.url || null, + productLockMethod: productLockOverlayUpload ? "rmbg-cutout" : null, + productLockOverlayUrl: productLockOverlayUpload?.url || null, sourceCropUrl: sourceCropUpload.url, structureLockedDraft: structureLockAppliedRef.value, targetDraftFullUrl: targetDraftFullUpload.url, @@ -4404,6 +4533,40 @@ function buildScaleAdjustFullFramePatchPrompt( ].join("\n"); } +function buildScaleAdjustProductLockBackgroundPrompt( + regions: NormalizedLocalEditRegion[], + scalePercent: ScaleAdjustPercent | null | undefined, + sourceSize: { width: number; height: number } +) { + const regionLines = regions + .map((region) => { + const targetRect = getScaleAdjustTargetRect(region, scalePercent); + const targetPixelRect = + region.targetPixelRect || + rectToPixels(targetRect, sourceSize.width, sourceSize.height); + return `Region ${region.index}: remove the selected furniture/product from the original area x=${region.pixelRect.left}, y=${region.pixelRect.top}, w=${region.pixelRect.width}, h=${region.pixelRect.height}; keep the final target footprint x=${targetPixelRect.left}, y=${targetPixelRect.top}, w=${targetPixelRect.width}, h=${targetPixelRect.height} visually clean and empty because the locked product pixels will be composited there after generation. User request: ${region.prompt}.`; + }) + .join("\n"); + + return [ + "Full-frame background repair for product-locked scale adjustment.", + "Return exactly one clean realistic full image with the same framing, camera, aspect ratio, and canvas extent as input image 1. Do not crop, zoom, pan, or resize the whole photo.", + "Input image 1 is the original clean full photo. Input image 2 is a background plate reference that removes the selected furniture/product footprint and keeps the room/person/background layout stable.", + "Important: do not draw, recreate, move, scale, duplicate, or redesign the selected furniture/product in the final image. The system will composite the locked original product pixels into the final target footprint after your background image is returned.", + "Your job is only to erase the old selected furniture/product footprint, complete newly exposed wall, curtain, floor, rug, shadow, sofa edge, and background naturally, and keep the target footprint clean enough for the locked product layer to sit there.", + "Preserve all non-target people exactly as foreground/background context: do not move, cover, deform, redraw, change face, change body, change hands, change legs, change clothes, or change pose.", + "Keep all non-target content stable: sofa, window, curtain, cabinet, lamp, wall art, floor, rug, lighting, textural grain, and global color must remain consistent.", + "Remove draft artifacts from input image 2: pale wash blocks, rough masks, missing patches, hard rectangular edges, black holes, ghost fragments, pasted silhouettes, and old-size residues. They are only edit guides, not final content.", + "Do not output boxes, masks, labels, coordinate marks, UI marks, arrows, guide overlays, crop borders, or any annotation artifacts.", + "中文硬约束：这是产品结构锁定流程。你只生成背景底图，不要生成、移动、缩放或重绘目标家具/商品；目标家具/商品会在你输出后由系统用原图锁定像素叠回目标位置。必须擦除旧位置的目标家具/商品并自然补全墙面、窗帘、地面、阴影和背景。非目标人物必须保持原样，不能变脸、变形、改变衣服、手、腿、脚或姿势。最终背景图不能有矩形雾块、贴片边、黑洞、残影、标注框，也不能出现第二个目标家具/商品。", + "", + "Region instructions:", + regionLines, + "", + "Success check before returning: the selected furniture/product is absent from the generated background plate, the old footprint is naturally repaired, the target footprint is clean, and the full image still looks like one natural photograph.", + ].join("\n"); +} + export async function POST(request: NextRequest) { try { const session = await auth(); @@ -4749,6 +4912,10 @@ export async function POST(request: NextRequest) { regions.some((region) => isScaleAdjustFurnitureProductLikePrompt(region.prompt) ); + const useScaleAdjustProductLockComposite = + useScaleAdjustFullFrameStructurePatch && + Boolean(scaleAdjustPatchReference?.backgroundPlateFullUrl) && + Boolean(scaleAdjustPatchReference?.productLockOverlayUrl); const useScaleAdjustPatchTargetDraftPrimary = Boolean(scaleAdjustPatchReference); const includeScaleAdjustPatchSourceReference = Boolean(scaleAdjustPatchReference); @@ -4759,11 +4926,17 @@ export async function POST(request: NextRequest) { const prompt = isScaleAdjustOperation ? scaleAdjustPatchReference ? useScaleAdjustFullFrameStructurePatch - ? buildScaleAdjustFullFramePatchPrompt( - regions, - scalePercent, - { width: sourceWidth, height: sourceHeight } - ) + ? useScaleAdjustProductLockComposite + ? buildScaleAdjustProductLockBackgroundPrompt( + regions, + scalePercent, + { width: sourceWidth, height: sourceHeight } + ) + : buildScaleAdjustFullFramePatchPrompt( + regions, + scalePercent, + { width: sourceWidth, height: sourceHeight } + ) : buildScaleAdjustPatchPrompt( regions, scalePercent, @@ -4816,6 +4989,8 @@ export async function POST(request: NextRequest) { "full room image, boxes, labels, UI marks, arrows, text, overlays, seams, rectangular patch edge, haze block, ghost copy, old-size subject, unchanged subject, duplicate subject, stretched subject, distorted product, changed product structure"; const scaleAdjustFullFramePatchNegativePrompt = "boxes, labels, UI marks, arrows, text, overlays, seams, rectangular patch edge, haze block, ghost copy, old-size subject, unchanged subject, duplicate subject, stretched subject, distorted product, changed product structure, changed person, changed face, deformed person, extra limbs, changed hands, changed feet, whole-image zoom, crop, pan"; + const scaleAdjustProductLockBackgroundNegativePrompt = + "selected furniture, selected product, moved furniture, scaled furniture, recreated product, duplicate furniture, duplicate product, old-size furniture, target object visible, product ghost, product remnant, boxes, labels, UI marks, arrows, text, overlays, seams, rectangular patch edge, haze block, black hole, changed person, changed face, deformed person, extra limbs, changed hands, changed feet, whole-image zoom, crop, pan"; const manualRegionReferenceImageUrls = regions.flatMap((region) => region.referenceImages.map((reference) => reference.url) ); @@ -4825,12 +5000,19 @@ export async function POST(request: NextRequest) { const referenceImages = isScaleAdjustOperation ? scaleAdjustPatchReference ? useScaleAdjustFullFrameStructurePatch - ? [ - sourceImageUrl, - scaleAdjustPatchReference.targetDraftFullUrl as string, - ...scaleAdjustCropReferenceImageUrls, - ...manualRegionReferenceImageUrls, - ] + ? useScaleAdjustProductLockComposite + ? [ + sourceImageUrl, + scaleAdjustPatchReference.backgroundPlateFullUrl as string, + ...scaleAdjustCropReferenceImageUrls, + ...manualRegionReferenceImageUrls, + ] + : [ + sourceImageUrl, + scaleAdjustPatchReference.targetDraftFullUrl as string, + ...scaleAdjustCropReferenceImageUrls, + ...manualRegionReferenceImageUrls, + ] : useScaleAdjustPatchTargetDraftPrimary ? [ scaleAdjustPatchReference.targetDraftUrl, @@ -4921,7 +5103,9 @@ export async function POST(request: NextRequest) { annotatedReferenceSentToModel: sendScaleAdjustAnnotatedReferenceToModel, annotatedReferenceType: isScaleAdjustOperation ? useScaleAdjustFullFrameStructurePatch - ? "scale-adjust-full-frame-structure-target-draft" + ? useScaleAdjustProductLockComposite + ? "scale-adjust-product-lock-background-plate" + : "scale-adjust-full-frame-structure-target-draft" : useProviderTextOnlyCoordinatePrompt ? "scale-adjust-text-coordinate-only-debug-map" : useScaleAdjustCoordinatePrompt @@ -4982,6 +5166,14 @@ export async function POST(request: NextRequest) { scaleAdjustPatchReference?.targetDraftUrl || null, scaleAdjustPatchTargetDraftFullUrl: scaleAdjustPatchReference?.targetDraftFullUrl || null, + scaleAdjustBackgroundPlateFullUrl: + scaleAdjustPatchReference?.backgroundPlateFullUrl || null, + scaleAdjustProductLockApplied: + useScaleAdjustProductLockComposite, + scaleAdjustProductLockMethod: + scaleAdjustPatchReference?.productLockMethod || null, + scaleAdjustProductLockOverlayUrl: + scaleAdjustPatchReference?.productLockOverlayUrl || null, scaleAdjustPatchCompositeClipMaskUrl: scaleAdjustPatchReference?.compositeClipMaskUrl || null, scaleAdjustPatchWorkspacePixelRect: @@ -5036,7 +5228,9 @@ export async function POST(request: NextRequest) { .length, scaleAdjustPromptStrategy: isScaleAdjustOperation ? scaleAdjustPatchReference - ? useScaleAdjustFullFrameStructurePatch + ? useScaleAdjustProductLockComposite + ? "product-lock-background-plate-rmbg-v1-one-shot" + : useScaleAdjustFullFrameStructurePatch ? "full-frame-source-locked-target-guide-v8-one-shot" : scaleAdjustPatchReference.structureLockedDraft ? "ai-patch-workspace-structure-locked-draft-v87-subject-alpha-clipped-one-shot" @@ -5079,7 +5273,9 @@ export async function POST(request: NextRequest) { isScaleAdjustOperation ? scaleAdjustPatchReference ? useScaleAdjustFullFrameStructurePatch - ? scaleAdjustFullFramePatchNegativePrompt + ? useScaleAdjustProductLockComposite + ? scaleAdjustProductLockBackgroundNegativePrompt + : scaleAdjustFullFramePatchNegativePrompt : useFalScaleAdjustPatchPrompt ? scaleAdjustFalPatchNegativePrompt : scaleAdjustPatchNegativePrompt @@ -5119,6 +5315,14 @@ export async function POST(request: NextRequest) { workspaceRegion: scaleAdjustPatchReference.workspaceRect, } : undefined, + scaleAdjustProductLockComposite: useScaleAdjustProductLockComposite + ? { + overlayImageUrl: + scaleAdjustPatchReference?.productLockOverlayUrl as string, + sourceHeight, + sourceWidth, + } + : undefined, scaleAdjustVerification: scaleAdjustVerificationConfig, localEditComposite: isScaleAdjustOperation ? undefined diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index acd2a2a09..baf12f257 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -71,6 +71,12 @@ export interface CanvasScaleAdjustPatchCompositeConfig { featherRatio?: number; } +export interface CanvasScaleAdjustProductLockCompositeConfig { + overlayImageUrl: string; + sourceHeight: number; + sourceWidth: number; +} + export interface CanvasScaleAdjustVerificationConfig { allowDeterministicFallback?: boolean; allowOutsideChangedWhenChangedInside?: boolean; @@ -99,6 +105,7 @@ export interface CanvasGenerateImageRetryVariant { referenceImage?: string; referenceImages?: string[]; scaleAdjustPatchComposite?: CanvasScaleAdjustPatchCompositeConfig; + scaleAdjustProductLockComposite?: CanvasScaleAdjustProductLockCompositeConfig; scaleAdjustVerification?: CanvasScaleAdjustVerificationConfig; } @@ -123,6 +130,7 @@ export interface CanvasGenerateImageRequest { debugMeta?: Record; localEditComposite?: CanvasLocalEditCompositeConfig; scaleAdjustPatchComposite?: CanvasScaleAdjustPatchCompositeConfig; + scaleAdjustProductLockComposite?: CanvasScaleAdjustProductLockCompositeConfig; scaleAdjustVerification?: CanvasScaleAdjustVerificationConfig; retryVariants?: CanvasGenerateImageRetryVariant[]; internalRetryVariantIndex?: number; @@ -541,6 +549,23 @@ function normalizeScaleAdjustPatchCompositeConfig( }; } +function normalizeScaleAdjustProductLockCompositeConfig( + value: CanvasGenerateImageRequest["scaleAdjustProductLockComposite"] +): CanvasScaleAdjustProductLockCompositeConfig | null { + if (!value || typeof value !== "object") return null; + const overlayImageUrl = String(value.overlayImageUrl || "").trim(); + if (!/^https?:\/\//i.test(overlayImageUrl)) return null; + const sourceWidth = Math.round(Number(value.sourceWidth || 0)); + const sourceHeight = Math.round(Number(value.sourceHeight || 0)); + if (sourceWidth <= 0 || sourceHeight <= 0) return null; + + return { + overlayImageUrl, + sourceHeight, + sourceWidth, + }; +} + function normalizeCanvasRegion(region: unknown): CanvasLocalEditCompositeRegion | null { if (!region || typeof region !== "object") return null; const raw = region as Partial; @@ -3163,6 +3188,96 @@ async function applyScaleAdjustPatchCompositeToImages(params: { return mergedImages; } +async function applyScaleAdjustProductLockCompositeToImages(params: { + images: ParsedImage[]; + composite: CanvasScaleAdjustProductLockCompositeConfig | null; + debugLabel: string; +}) { + const { images, composite, debugLabel } = params; + if (!composite || images.length === 0) return images; + + const overlayBuffer = await sharp( + await fetchImageBufferFromUrl(composite.overlayImageUrl), + { failOn: "none" } + ) + .rotate() + .ensureAlpha() + .png({ compressionLevel: 6 }) + .toBuffer(); + const overlayMetadata = await sharp(overlayBuffer, { failOn: "none" }).metadata(); + const overlayWidth = Math.round(Number(overlayMetadata.width || 0)); + const overlayHeight = Math.round(Number(overlayMetadata.height || 0)); + if (overlayWidth <= 0 || overlayHeight <= 0) return images; + + const mergedImages: ParsedImage[] = []; + for (const [index, image] of images.entries()) { + const generatedUrl = image.originalUrl || image.thumbnailUrl; + const generatedBuffer = await fetchImageBufferFromUrl(generatedUrl); + const normalizedGeneratedBuffer = await sharp(generatedBuffer, { + failOn: "none", + }) + .rotate() + .ensureAlpha() + .png({ compressionLevel: 6 }) + .toBuffer(); + const generatedMetadata = await sharp(normalizedGeneratedBuffer, { + failOn: "none", + }).metadata(); + const generatedWidth = Math.round(Number(generatedMetadata.width || 0)); + const generatedHeight = Math.round(Number(generatedMetadata.height || 0)); + if (generatedWidth <= 0 || generatedHeight <= 0) { + mergedImages.push(image); + continue; + } + + const resizedOverlayBuffer = await sharp(overlayBuffer, { failOn: "none" }) + .resize(generatedWidth, generatedHeight, { + fit: "fill", + kernel: sharp.kernel.lanczos3, + }) + .png({ compressionLevel: 6 }) + .toBuffer(); + const mergedBuffer = await sharp(normalizedGeneratedBuffer, { + failOn: "none", + }) + .composite([{ input: resizedOverlayBuffer, left: 0, top: 0 }]) + .png({ compressionLevel: 6 }) + .toBuffer(); + const thumbnailBuffer = await sharp(mergedBuffer, { failOn: "none" }) + .resize(512, 512, { fit: "inside", withoutEnlargement: true }) + .png({ compressionLevel: 6 }) + .toBuffer(); + const suffix = `${Date.now()}-${index + 1}-${Math.random() + .toString(36) + .slice(2, 8)}`; + const [originalUpload, thumbnailUpload] = await Promise.all([ + uploadFile( + `canvas-scale-adjust-product-lock-${suffix}.png`, + bufferToArrayBuffer(mergedBuffer) + ), + uploadFile( + `canvas-scale-adjust-product-lock-thumb-${suffix}.png`, + bufferToArrayBuffer(thumbnailBuffer) + ), + ]); + mergedImages.push({ + originalUrl: originalUpload.url, + thumbnailUrl: thumbnailUpload.url, + }); + } + + console.info("[Canvas Generate Image] Scale adjust product lock composite applied:", { + debugLabel, + imageCount: mergedImages.length, + overlayHeight, + overlayWidth, + sourceHeight: composite.sourceHeight, + sourceWidth: composite.sourceWidth, + }); + + return mergedImages; +} + function getProviderBatchLimit(providerType: string) { switch (providerType) { case "ruxa-image": @@ -3661,6 +3776,10 @@ export async function generateCanvasImage( const scaleAdjustPatchComposite = normalizeScaleAdjustPatchCompositeConfig( request.scaleAdjustPatchComposite ); + const scaleAdjustProductLockComposite = + normalizeScaleAdjustProductLockCompositeConfig( + request.scaleAdjustProductLockComposite + ); const scaleAdjustVerification = normalizeScaleAdjustVerificationConfig( request.scaleAdjustVerification ); @@ -3887,6 +4006,7 @@ export async function generateCanvasImage( ...debugMeta, hasLocalEditComposite: Boolean(localEditComposite), hasScaleAdjustPatchComposite: Boolean(scaleAdjustPatchComposite), + hasScaleAdjustProductLockComposite: Boolean(scaleAdjustProductLockComposite), hasScaleAdjustVerification: Boolean(scaleAdjustVerification), }); @@ -4240,9 +4360,16 @@ export async function generateCanvasImage( images: compositedImages, }); + const scaleAdjustProductLockedImages = + await applyScaleAdjustProductLockCompositeToImages({ + composite: scaleAdjustProductLockComposite, + debugLabel, + images: scaleAdjustPatchCompositedImages, + }); + const finalImages = await applyScaleAdjustVerificationToImages({ debugLabel, - images: scaleAdjustPatchCompositedImages, + images: scaleAdjustProductLockedImages, verification: scaleAdjustVerification, }); console.info("[Canvas Generate Image] Completed:", { From f6faf6bb44c372edf7a5a732f1b819508e4b33cf Mon Sep 17 00:00:00 2001 From: 134 Date: Mon, 6 Jul 2026 01:30:01 +0800 Subject: [PATCH 09/70] fix: constrain product lock scale adjust to source patch --- app/api/canvas/local-edit/route.ts | 112 +++++++++++++++++++++-------- 1 file changed, 81 insertions(+), 31 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 259fec9d6..aa3d169a6 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -2404,34 +2404,6 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: { width: cutoutWidth, }); if (!highConfidenceCutoutBbox) return null; - const refinedCutoutRaw = await applyScaleAdjustAlphaMaskToRgba({ - alphaMask: highConfidenceCutoutMask, - height: cutoutHeight, - rawRgba: cleanedRaw, - width: cutoutWidth, - }); - const cleanedCutout = await sharp(refinedCutoutRaw, { - raw: { channels: 4, height: cutoutHeight, width: cutoutWidth }, - }) - .png({ compressionLevel: 6 }) - .toBuffer(); - const alphaBbox = getScaleAdjustAlphaBoundingBox({ - height: cutoutHeight, - rawRgba: refinedCutoutRaw, - width: cutoutWidth, - }); - if (!alphaBbox) return null; - - const alphaRatio = alphaBbox.count / Math.max(1, cutoutWidth * cutoutHeight); - const bboxCoversCanvas = - alphaBbox.width >= cutoutWidth * 0.97 && - alphaBbox.height >= cutoutHeight * 0.97; - if ( - alphaRatio < SCALE_ADJUST_RMBG_MIN_ALPHA_RATIO || - (alphaRatio > SCALE_ADJUST_RMBG_MAX_SOLID_ALPHA_RATIO && bboxCoversCanvas) - ) { - return null; - } const fullAlphaMask = Buffer.from(highConfidenceCutoutMask); const sourceMaskBuffer = sameSizeCutout @@ -2464,15 +2436,68 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: { maskBuffer: sourceMaskBuffer, threshold: 96, }); + const protectedPersonMask = + protectedPersonSide && sameSizeCutout && sourceCropRaw + ? validateScaleAdjustProtectedPersonMask({ + box: { width: cutoutWidth, height: cutoutHeight }, + mask: buildScaleAdjustProtectedPersonSeedMask({ + box: { left: 0, top: 0, width: cutoutWidth, height: cutoutHeight }, + raw: sourceCropRaw, + side: protectedPersonSide, + width: cutoutWidth, + }), + }) + : null; const filteredSourceMask = filterScaleAdjustStructureComponents({ box: placement.sourceBox, mask: highConfidenceSourceMask, + protectedPersonMask, + protectedPersonSide, }); const filteredSourceBbox = getScaleAdjustMaskBoundingBox({ height: placement.sourceBox.height, mask: filteredSourceMask, width: placement.sourceBox.width, }); + const overlayAlphaMask = + sameSizeCutout && filteredSourceBbox + ? filteredSourceMask + : highConfidenceCutoutMask; + const overlayAlphaBbox = getScaleAdjustMaskBoundingBox({ + height: cutoutHeight, + mask: overlayAlphaMask, + width: cutoutWidth, + }); + if (!overlayAlphaBbox) return null; + const refinedCutoutRaw = await applyScaleAdjustAlphaMaskToRgba({ + alphaMask: overlayAlphaMask, + height: cutoutHeight, + rawRgba: cleanedRaw, + width: cutoutWidth, + }); + const cleanedCutout = await sharp(refinedCutoutRaw, { + raw: { channels: 4, height: cutoutHeight, width: cutoutWidth }, + }) + .png({ compressionLevel: 6 }) + .toBuffer(); + const alphaBbox = getScaleAdjustAlphaBoundingBox({ + height: cutoutHeight, + rawRgba: refinedCutoutRaw, + width: cutoutWidth, + }); + if (!alphaBbox) return null; + + const alphaRatio = + overlayAlphaBbox.count / Math.max(1, cutoutWidth * cutoutHeight); + const bboxCoversCanvas = + overlayAlphaBbox.width >= cutoutWidth * 0.97 && + overlayAlphaBbox.height >= cutoutHeight * 0.97; + if ( + alphaRatio < SCALE_ADJUST_RMBG_MIN_ALPHA_RATIO || + (alphaRatio > SCALE_ADJUST_RMBG_MAX_SOLID_ALPHA_RATIO && bboxCoversCanvas) + ) { + return null; + } const sourceWashBaseMask = filteredSourceBbox ? solidifyScaleAdjustStructureMask({ box: placement.sourceBox, @@ -2582,9 +2607,13 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: { alphaRatio: Number(alphaRatio.toFixed(3)), cutoutHeight, cutoutWidth, + filteredAlphaRatio: Number( + (overlayAlphaBbox.count / Math.max(1, cutoutWidth * cutoutHeight)).toFixed(3) + ), placedHeight, placedWidth, protectedPersonSide, + protectedPersonMaskApplied: Boolean(protectedPersonMask), sameSizeCutout, alphaBandingStats, sourceHeight: placement.sourceBox.height, @@ -5229,7 +5258,7 @@ export async function POST(request: NextRequest) { scaleAdjustPromptStrategy: isScaleAdjustOperation ? scaleAdjustPatchReference ? useScaleAdjustProductLockComposite - ? "product-lock-background-plate-rmbg-v1-one-shot" + ? "product-lock-background-plate-rmbg-v2-source-patch-one-shot" : useScaleAdjustFullFrameStructurePatch ? "full-frame-source-locked-target-guide-v8-one-shot" : scaleAdjustPatchReference.structureLockedDraft @@ -5251,7 +5280,8 @@ export async function POST(request: NextRequest) { : undefined, scaleAdjustPostCompositeApplied: Boolean(scaleAdjustPatchReference) && - !useScaleAdjustFullFrameStructurePatch, + (!useScaleAdjustFullFrameStructurePatch || + useScaleAdjustProductLockComposite), scaleAdjustUsesFullFrameOutput: isScaleAdjustOperation && (!scaleAdjustPatchReference || useScaleAdjustFullFrameStructurePatch), @@ -5291,7 +5321,27 @@ export async function POST(request: NextRequest) { ? { maskImageUrl: scaleAdjustEditMaskUpload.url } : undefined, referenceImages, - scaleAdjustPatchComposite: scaleAdjustPatchReference && !useScaleAdjustFullFrameStructurePatch + scaleAdjustPatchComposite: useScaleAdjustProductLockComposite + ? { + diffAlphaCutoff: 18, + diffAlphaMultiplier: 2.6, + diffCompareBlurPx: 1.1, + diffHighThreshold: 64, + diffLowThreshold: 20, + featherMaxPx: 34, + featherMinPx: 8, + featherRatio: 0.015, + generatedFrame: "full", + maskMode: "diff", + regions: scaleAdjustPatchCompositeRegions, + sourceImageUrl, + clipMaskImageUrl: + scaleAdjustPatchReference?.compositeClipMaskUrl || undefined, + workspaceRegion: + (scaleAdjustPatchReference as ScaleAdjustPatchReference) + .workspaceRect, + } + : scaleAdjustPatchReference && !useScaleAdjustFullFrameStructurePatch ? { diffAlphaCutoff: scaleAdjustPatchReference.structureLockedDraft ? 24 : 8, From f7a5accb61a610f638a3983ed052d88f0197db57 Mon Sep 17 00:00:00 2001 From: 134 Date: Mon, 6 Jul 2026 02:29:49 +0800 Subject: [PATCH 10/70] fix: protect people in product locked scale adjust --- app/api/canvas/local-edit/route.ts | 420 +++++++++++++++++++++-------- lib/canvas/generate-image.ts | 57 +++- 2 files changed, 362 insertions(+), 115 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index aa3d169a6..a76b245da 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -128,6 +128,7 @@ type ScaleAdjustPatchReference = { compositeClipMaskUrl?: string | null; productLockMethod?: string | null; productLockOverlayUrl?: string | null; + productLockProtectMaskUrl?: string | null; sourceCropUrl: string; structureLockedDraft?: boolean; targetDraftFullUrl?: string | null; @@ -145,6 +146,7 @@ type ScaleAdjustStructureLockedComposite = { method?: "pixel-mask" | "rect-crop-guide" | "rect-crop-guide-rmbg-lock" | "rmbg-cutout"; overlay: { input: Buffer; left: number; top: number }; productLock?: { input: Buffer; left: number; top: number }; + productLockProtectMask?: { input: Buffer; left: number; top: number }; targetClip?: { input: Buffer; left: number; top: number }; wash: { input: Buffer; left: number; top: number }; }; @@ -1586,14 +1588,17 @@ function buildScaleAdjustStructureInitialMask(params: { } function buildScaleAdjustProtectedPersonSeedMask(params: { + background?: { b: number; g: number; luma: number; r: number } | null; box: { left: number; top: number; width: number; height: number }; raw: Buffer; side: ScaleAdjustProtectedPersonSide; width: number; }) { - const { box, raw, side, width } = params; + const { background = null, box, raw, side, width } = params; + const candidate = new Uint8Array(box.width * box.height); + const anchor = new Uint8Array(box.width * box.height); + const visited = new Uint8Array(box.width * box.height); const seed = new Uint8Array(box.width * box.height); - let seedCount = 0; const sideAllowsX = (localX: number) => { if (side === "right") return localX >= box.width * 0.52; if (side === "left") return localX <= box.width * 0.48; @@ -1611,17 +1616,106 @@ function buildScaleAdjustProtectedPersonSeedMask(params: { const b = raw[rawIndex + 2] || 0; const { luma, saturation } = getScaleAdjustPixelStats(r, g, b); const likelySkin = isScaleAdjustLikelySkinPixel(r, g, b); + const colorDistance = background + ? Math.sqrt( + (r - background.r) ** 2 + + (g - background.g) ** 2 + + (b - background.b) ** 2 + ) + : 999; + const backgroundLikeLightPixel = + Boolean(background) && + luma >= (background?.luma || 0) - 3 && + colorDistance < 18; const likelyWhiteFabric = isScaleAdjustLikelyWhiteFabricPixel(r, g, b, luma, saturation) && luma < 246 && - saturation < 0.22; + saturation < 0.22 && + !backgroundLikeLightPixel && + (colorDistance > 16 || !background || luma < background.luma - 5); if (!likelySkin && !likelyWhiteFabric) continue; - seed[localY * box.width + localX] = 255; - seedCount += 1; + const index = localY * box.width + localX; + candidate[index] = 255; + if (likelySkin) { + anchor[index] = 255; + } + } + } + + const queueX: number[] = []; + const queueY: number[] = []; + let seedCount = 0; + let anchorCount = 0; + for (let y = 0; y < box.height; y += 1) { + for (let x = 0; x < box.width; x += 1) { + const startIndex = y * box.width + x; + if (candidate[startIndex] === 0 || visited[startIndex]) continue; + + queueX.length = 0; + queueY.length = 0; + queueX.push(x); + queueY.push(y); + visited[startIndex] = 1; + let head = 0; + let area = 0; + let anchors = 0; + let minX = x; + let minY = y; + let maxX = x; + let maxY = y; + + while (head < queueX.length) { + const cx = queueX[head] || 0; + const cy = queueY[head] || 0; + head += 1; + area += 1; + if (anchor[cy * box.width + cx]) anchors += 1; + minX = Math.min(minX, cx); + minY = Math.min(minY, cy); + maxX = Math.max(maxX, cx); + maxY = Math.max(maxY, cy); + + for (let dy = -1; dy <= 1; dy += 1) { + for (let dx = -1; dx <= 1; dx += 1) { + if (dx === 0 && dy === 0) continue; + const nx = cx + dx; + const ny = cy + dy; + if (nx < 0 || ny < 0 || nx >= box.width || ny >= box.height) continue; + const nextIndex = ny * box.width + nx; + if (candidate[nextIndex] === 0 || visited[nextIndex]) continue; + visited[nextIndex] = 1; + queueX.push(nx); + queueY.push(ny); + } + } + } + + const componentWidth = maxX - minX + 1; + const componentHeight = maxY - minY + 1; + const broadBackgroundStripe = + anchors === 0 && + componentHeight >= box.height * 0.48 && + componentWidth >= box.width * 0.16; + const tinyUnanchoredPatch = anchors === 0 && area < box.width * box.height * 0.004; + if (anchors === 0 || broadBackgroundStripe || tinyUnanchoredPatch) { + continue; + } + + for (let index = 0; index < queueX.length; index += 1) { + const pixelX = queueX[index] || 0; + const pixelY = queueY[index] || 0; + const seedIndex = pixelY * box.width + pixelX; + if (seed[seedIndex]) continue; + seed[seedIndex] = 255; + seedCount += 1; + } + anchorCount += anchors; } } const seedRatio = seedCount / Math.max(1, box.width * box.height); + const anchorRatio = anchorCount / Math.max(1, box.width * box.height); + if (anchorCount < 12 || anchorRatio < 0.00012) return null; if (seedCount < 24 || seedRatio < 0.0006) return null; return dilateScaleAdjustStructureMask({ @@ -2148,6 +2242,19 @@ function thresholdScaleAdjustAlphaMask(params: { return output; } +function removeScaleAdjustMaskFromAlpha(params: { + alphaMask: Buffer | Uint8Array; + removeMask: Uint8Array | null; +}) { + const { alphaMask, removeMask } = params; + if (!removeMask || removeMask.length !== alphaMask.length) return alphaMask; + const output = Buffer.from(alphaMask); + for (let index = 0; index < output.length; index += 1) { + if (removeMask[index]) output[index] = 0; + } + return output; +} + async function applyScaleAdjustAlphaMaskToRgba(params: { alphaMask: Uint8Array; height: number; @@ -2343,7 +2450,9 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: { userId, imageBase64: sourceCropBuffer.toString("base64"), imageBuffer: sourceCropBuffer, - prompt: "比例调整主体抠图：只保留框选的目标人物/家具/商品主体，输出透明背景 PNG", + prompt: protectedPersonSide + ? "比例调整家具/商品抠图：只保留框选区域内需要缩放的目标家具/商品主体和它本身的配件结构，明确排除非目标人物、手、脚、腿、衣服、书本、身体遮挡物和背景，输出透明背景 PNG" + : "比例调整主体抠图：只保留框选的目标人物/家具/商品主体，输出透明背景 PNG", skipGallerySave: true, sourceImageUrl, }); @@ -2378,7 +2487,25 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: { rawRgba: cutoutRaw, width: cutoutWidth, }); - const protectedCleanedAlpha = + const background = estimateScaleAdjustBoxBackground({ + box: placement.sourceBox, + raw, + width: sourceWidth, + }); + const protectedPersonMask = + protectedPersonSide && sameSizeCutout && sourceCropRaw + ? validateScaleAdjustProtectedPersonMask({ + box: { width: cutoutWidth, height: cutoutHeight }, + mask: buildScaleAdjustProtectedPersonSeedMask({ + background, + box: { left: 0, top: 0, width: cutoutWidth, height: cutoutHeight }, + raw: sourceCropRaw, + side: protectedPersonSide, + width: cutoutWidth, + }), + }) + : null; + const sideProtectedCleanedAlpha = protectedPersonSide && sameSizeCutout && sourceCropRaw ? suppressScaleAdjustProtectedFurnitureDarkAlpha({ alphaMask: Buffer.from(cleanedAlpha), @@ -2389,10 +2516,12 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: { sourceBox: placement.sourceBox, }) : cleanedAlpha; - if (protectedCleanedAlpha !== cleanedAlpha) { - for (let index = 0; index < protectedCleanedAlpha.length; index += 1) { - cleanedRaw[index * 4 + 3] = protectedCleanedAlpha[index] || 0; - } + const protectedCleanedAlpha = removeScaleAdjustMaskFromAlpha({ + alphaMask: sideProtectedCleanedAlpha, + removeMask: protectedPersonMask, + }); + for (let index = 0; index < protectedCleanedAlpha.length; index += 1) { + cleanedRaw[index * 4 + 3] = protectedCleanedAlpha[index] || 0; } const highConfidenceCutoutMask = thresholdScaleAdjustAlphaMask({ maskBuffer: protectedCleanedAlpha, @@ -2427,27 +2556,10 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: { ); return null; } - const background = estimateScaleAdjustBoxBackground({ - box: placement.sourceBox, - raw, - width: sourceWidth, - }); const highConfidenceSourceMask = thresholdScaleAdjustAlphaMask({ maskBuffer: sourceMaskBuffer, threshold: 96, }); - const protectedPersonMask = - protectedPersonSide && sameSizeCutout && sourceCropRaw - ? validateScaleAdjustProtectedPersonMask({ - box: { width: cutoutWidth, height: cutoutHeight }, - mask: buildScaleAdjustProtectedPersonSeedMask({ - box: { left: 0, top: 0, width: cutoutWidth, height: cutoutHeight }, - raw: sourceCropRaw, - side: protectedPersonSide, - width: cutoutWidth, - }), - }) - : null; const filteredSourceMask = filterScaleAdjustStructureComponents({ box: placement.sourceBox, mask: highConfidenceSourceMask, @@ -2517,7 +2629,7 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: { }); const sourceCompositeClipMask = dilateScaleAdjustStructureMask({ height: placement.sourceBox.height, - mask: filteredSourceBbox ? filteredSourceMask : highConfidenceSourceMask, + mask: sourceWashBaseMask, radius: Math.max( 1, Math.round( @@ -2602,6 +2714,23 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: { .toBuffer(); const targetClipMaskPatch = await buildScaleAdjustOverlayClipMaskPatch(structureOverlay); + const productLockProtectMaskPatch = protectedPersonMask + ? await buildScaleAdjustCompositeClipMaskPatch({ + box: placement.sourceBox, + maskBuffer: dilateScaleAdjustStructureMask({ + height: placement.sourceBox.height, + mask: protectedPersonMask, + radius: Math.max( + 2, + Math.round( + Math.min(placement.sourceBox.width, placement.sourceBox.height) * + 0.012 + ) + ), + width: placement.sourceBox.width, + }), + }) + : null; console.info("[Canvas Local Edit] Scale adjust RMBG structure draft applied:", { alphaRatio: Number(alphaRatio.toFixed(3)), @@ -2637,6 +2766,15 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: { left: Math.round(targetCenterX - placedWidth / 2), top: Math.round(targetCenterY - placedHeight / 2), }, + ...(productLockProtectMaskPatch + ? { + productLockProtectMask: { + input: productLockProtectMaskPatch, + left: placement.sourceBox.left, + top: placement.sourceBox.top, + }, + } + : {}), targetClip: { input: targetClipMaskPatch, left: Math.round(targetCenterX - placedWidth / 2), @@ -3220,11 +3358,16 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { left: number; top: number; }> = []; - const productLockComposites: Array<{ - input: Buffer; - left: number; - top: number; - }> = []; + const productLockComposites: Array<{ + input: Buffer; + left: number; + top: number; + }> = []; + const productLockProtectMaskComposites: Array<{ + input: Buffer; + left: number; + top: number; + }> = []; for (const region of regions) { const placement = getScaleAdjustVisualGuidePlacement({ @@ -3284,15 +3427,20 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { if (structureLockedComposite.compositeClip) { compositeClipComposites.push(structureLockedComposite.compositeClip); } - if (structureLockedComposite.targetClip) { - targetClipComposites.push(structureLockedComposite.targetClip); + if (structureLockedComposite.targetClip && !structureLockedComposite.productLock) { + targetClipComposites.push(structureLockedComposite.targetClip); + } + if (structureLockedComposite.productLock) { + productLockComposites.push(structureLockedComposite.productLock); + } + if (structureLockedComposite.productLockProtectMask) { + productLockProtectMaskComposites.push( + structureLockedComposite.productLockProtectMask + ); + } + composites.push(structureLockedComposite.overlay); + continue; } - if (structureLockedComposite.productLock) { - productLockComposites.push(structureLockedComposite.productLock); - } - composites.push(structureLockedComposite.overlay); - continue; - } const shouldUseShrinkCutoutDraft = process.env.CANVAS_SCALE_ADJUST_ENABLE_RMBG_SHRINK_CUTOUT === "1" && @@ -3317,15 +3465,20 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { if (shrinkCutoutComposite.compositeClip) { compositeClipComposites.push(shrinkCutoutComposite.compositeClip); } - if (shrinkCutoutComposite.targetClip) { - targetClipComposites.push(shrinkCutoutComposite.targetClip); + if (shrinkCutoutComposite.targetClip && !shrinkCutoutComposite.productLock) { + targetClipComposites.push(shrinkCutoutComposite.targetClip); + } + if (shrinkCutoutComposite.productLock) { + productLockComposites.push(shrinkCutoutComposite.productLock); + } + if (shrinkCutoutComposite.productLockProtectMask) { + productLockProtectMaskComposites.push( + shrinkCutoutComposite.productLockProtectMask + ); + } + composites.push(shrinkCutoutComposite.overlay); + continue; } - if (shrinkCutoutComposite.productLock) { - productLockComposites.push(shrinkCutoutComposite.productLock); - } - composites.push(shrinkCutoutComposite.overlay); - continue; - } sourceFootprintBoxes.push({ ...placement.sourceFootprintBox, @@ -3412,31 +3565,50 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { .png({ compressionLevel: 6 }) .toBuffer() : null; - const productLockOverlayBuffer = productLockComposites.length - ? await sharp({ - create: { - background: { alpha: 0, b: 0, g: 0, r: 0 }, + const productLockOverlayBuffer = productLockComposites.length + ? await sharp({ + create: { + background: { alpha: 0, b: 0, g: 0, r: 0 }, channels: 4, height: sourceHeight, width: sourceWidth, }, }) .composite(productLockComposites) - .png({ compressionLevel: 6 }) - .toBuffer() - : null; - const compositeClipMaskBuffer = - compositeClipComposites.length || targetClipComposites.length - ? await buildScaleAdjustCompositeClipMaskBuffer({ - height: sourceHeight, + .png({ compressionLevel: 6 }) + .toBuffer() + : null; + const productLockProtectMaskBuffer = productLockProtectMaskComposites.length + ? await sharp({ + create: { + background: { alpha: 0, b: 255, g: 255, r: 255 }, + channels: 4, + height: sourceHeight, + width: sourceWidth, + }, + }) + .composite(productLockProtectMaskComposites) + .png({ compressionLevel: 6 }) + .toBuffer() + : null; + const compositeClipMaskBuffer = + compositeClipComposites.length || targetClipComposites.length + ? await buildScaleAdjustCompositeClipMaskBuffer({ + height: sourceHeight, sourceMaskComposites: compositeClipComposites, targetMaskComposites: targetClipComposites, width: sourceWidth, }) : null; - return { backgroundPlateBuffer, buffer, compositeClipMaskBuffer, productLockOverlayBuffer }; -} + return { + backgroundPlateBuffer, + buffer, + compositeClipMaskBuffer, + productLockOverlayBuffer, + productLockProtectMaskBuffer, + }; + } async function createScaleAdjustPatchReference(params: { fullFrameReferenceOnly?: boolean; @@ -3478,12 +3650,13 @@ async function createScaleAdjustPatchReference(params: { const targetDraftBuffer = targetDraftReference.buffer; const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; if (params.fullFrameReferenceOnly && structureLockAppliedRef.value) { - const [ - targetDraftFullUpload, - backgroundPlateFullUpload, - productLockOverlayUpload, - compositeClipMaskUpload, - ] = await Promise.all([ + const [ + targetDraftFullUpload, + backgroundPlateFullUpload, + productLockOverlayUpload, + productLockProtectMaskUpload, + compositeClipMaskUpload, + ] = await Promise.all([ uploadFile( `canvas-scale-adjust-full-target-draft-${suffix}.png`, toArrayBuffer(targetDraftBuffer) @@ -3494,15 +3667,21 @@ async function createScaleAdjustPatchReference(params: { toArrayBuffer(targetDraftReference.backgroundPlateBuffer) ) : Promise.resolve(null), - targetDraftReference.productLockOverlayBuffer - ? uploadFile( - `canvas-scale-adjust-product-lock-overlay-${suffix}.png`, - toArrayBuffer(targetDraftReference.productLockOverlayBuffer) - ) - : Promise.resolve(null), - targetDraftReference.compositeClipMaskBuffer - ? uploadFile( - `canvas-scale-adjust-composite-clip-mask-${suffix}.png`, + targetDraftReference.productLockOverlayBuffer + ? uploadFile( + `canvas-scale-adjust-product-lock-overlay-${suffix}.png`, + toArrayBuffer(targetDraftReference.productLockOverlayBuffer) + ) + : Promise.resolve(null), + targetDraftReference.productLockProtectMaskBuffer + ? uploadFile( + `canvas-scale-adjust-product-lock-protect-mask-${suffix}.png`, + toArrayBuffer(targetDraftReference.productLockProtectMaskBuffer) + ) + : Promise.resolve(null), + targetDraftReference.compositeClipMaskBuffer + ? uploadFile( + `canvas-scale-adjust-composite-clip-mask-${suffix}.png`, toArrayBuffer(targetDraftReference.compositeClipMaskBuffer) ) : Promise.resolve(null), @@ -3510,10 +3689,11 @@ async function createScaleAdjustPatchReference(params: { return { backgroundPlateFullUrl: backgroundPlateFullUpload?.url || null, - compositeClipMaskUrl: compositeClipMaskUpload?.url || null, - productLockMethod: productLockOverlayUpload ? "rmbg-cutout" : null, - productLockOverlayUrl: productLockOverlayUpload?.url || null, - sourceCropUrl: sourceImageUrl || targetDraftFullUpload.url, + compositeClipMaskUrl: compositeClipMaskUpload?.url || null, + productLockMethod: productLockOverlayUpload ? "rmbg-cutout" : null, + productLockOverlayUrl: productLockOverlayUpload?.url || null, + productLockProtectMaskUrl: productLockProtectMaskUpload?.url || null, + sourceCropUrl: sourceImageUrl || targetDraftFullUpload.url, structureLockedDraft: structureLockAppliedRef.value, targetDraftFullUrl: targetDraftFullUpload.url, targetDraftUrl: targetDraftFullUpload.url, @@ -3532,14 +3712,15 @@ async function createScaleAdjustPatchReference(params: { .png({ compressionLevel: 6 }) .toBuffer(), ]); - const [ - sourceCropUpload, - targetDraftUpload, - targetDraftFullUpload, - backgroundPlateFullUpload, - productLockOverlayUpload, - compositeClipMaskUpload, - ] = await Promise.all([ + const [ + sourceCropUpload, + targetDraftUpload, + targetDraftFullUpload, + backgroundPlateFullUpload, + productLockOverlayUpload, + productLockProtectMaskUpload, + compositeClipMaskUpload, + ] = await Promise.all([ uploadFile( `canvas-scale-adjust-workspace-source-${suffix}.png`, toArrayBuffer(sourceCropBuffer) @@ -3558,15 +3739,21 @@ async function createScaleAdjustPatchReference(params: { toArrayBuffer(targetDraftReference.backgroundPlateBuffer) ) : Promise.resolve(null), - targetDraftReference.productLockOverlayBuffer - ? uploadFile( - `canvas-scale-adjust-product-lock-overlay-${suffix}.png`, - toArrayBuffer(targetDraftReference.productLockOverlayBuffer) - ) - : Promise.resolve(null), - targetDraftReference.compositeClipMaskBuffer - ? uploadFile( - `canvas-scale-adjust-composite-clip-mask-${suffix}.png`, + targetDraftReference.productLockOverlayBuffer + ? uploadFile( + `canvas-scale-adjust-product-lock-overlay-${suffix}.png`, + toArrayBuffer(targetDraftReference.productLockOverlayBuffer) + ) + : Promise.resolve(null), + targetDraftReference.productLockProtectMaskBuffer + ? uploadFile( + `canvas-scale-adjust-product-lock-protect-mask-${suffix}.png`, + toArrayBuffer(targetDraftReference.productLockProtectMaskBuffer) + ) + : Promise.resolve(null), + targetDraftReference.compositeClipMaskBuffer + ? uploadFile( + `canvas-scale-adjust-composite-clip-mask-${suffix}.png`, toArrayBuffer(targetDraftReference.compositeClipMaskBuffer) ) : Promise.resolve(null), @@ -3574,10 +3761,11 @@ async function createScaleAdjustPatchReference(params: { return { backgroundPlateFullUrl: backgroundPlateFullUpload?.url || null, - compositeClipMaskUrl: compositeClipMaskUpload?.url || null, - productLockMethod: productLockOverlayUpload ? "rmbg-cutout" : null, - productLockOverlayUrl: productLockOverlayUpload?.url || null, - sourceCropUrl: sourceCropUpload.url, + compositeClipMaskUrl: compositeClipMaskUpload?.url || null, + productLockMethod: productLockOverlayUpload ? "rmbg-cutout" : null, + productLockOverlayUrl: productLockOverlayUpload?.url || null, + productLockProtectMaskUrl: productLockProtectMaskUpload?.url || null, + sourceCropUrl: sourceCropUpload.url, structureLockedDraft: structureLockAppliedRef.value, targetDraftFullUrl: targetDraftFullUpload.url, targetDraftUrl: targetDraftUpload.url, @@ -5201,10 +5389,12 @@ export async function POST(request: NextRequest) { useScaleAdjustProductLockComposite, scaleAdjustProductLockMethod: scaleAdjustPatchReference?.productLockMethod || null, - scaleAdjustProductLockOverlayUrl: - scaleAdjustPatchReference?.productLockOverlayUrl || null, - scaleAdjustPatchCompositeClipMaskUrl: - scaleAdjustPatchReference?.compositeClipMaskUrl || null, + scaleAdjustProductLockOverlayUrl: + scaleAdjustPatchReference?.productLockOverlayUrl || null, + scaleAdjustProductLockProtectMaskUrl: + scaleAdjustPatchReference?.productLockProtectMaskUrl || null, + scaleAdjustPatchCompositeClipMaskUrl: + scaleAdjustPatchReference?.compositeClipMaskUrl || null, scaleAdjustPatchWorkspacePixelRect: scaleAdjustPatchReference?.workspacePixelRect || null, scaleAdjustPatchWorkspaceRect: @@ -5365,13 +5555,15 @@ export async function POST(request: NextRequest) { workspaceRegion: scaleAdjustPatchReference.workspaceRect, } : undefined, - scaleAdjustProductLockComposite: useScaleAdjustProductLockComposite - ? { - overlayImageUrl: - scaleAdjustPatchReference?.productLockOverlayUrl as string, - sourceHeight, - sourceWidth, - } + scaleAdjustProductLockComposite: useScaleAdjustProductLockComposite + ? { + overlayImageUrl: + scaleAdjustPatchReference?.productLockOverlayUrl as string, + protectMaskImageUrl: + scaleAdjustPatchReference?.productLockProtectMaskUrl || undefined, + sourceHeight, + sourceWidth, + } : undefined, scaleAdjustVerification: scaleAdjustVerificationConfig, localEditComposite: isScaleAdjustOperation diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index baf12f257..e56eab833 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -73,6 +73,7 @@ export interface CanvasScaleAdjustPatchCompositeConfig { export interface CanvasScaleAdjustProductLockCompositeConfig { overlayImageUrl: string; + protectMaskImageUrl?: string; sourceHeight: number; sourceWidth: number; } @@ -555,12 +556,16 @@ function normalizeScaleAdjustProductLockCompositeConfig( if (!value || typeof value !== "object") return null; const overlayImageUrl = String(value.overlayImageUrl || "").trim(); if (!/^https?:\/\//i.test(overlayImageUrl)) return null; + const protectMaskImageUrl = String(value.protectMaskImageUrl || "").trim(); const sourceWidth = Math.round(Number(value.sourceWidth || 0)); const sourceHeight = Math.round(Number(value.sourceHeight || 0)); if (sourceWidth <= 0 || sourceHeight <= 0) return null; return { overlayImageUrl, + ...(protectMaskImageUrl && /^https?:\/\//i.test(protectMaskImageUrl) + ? { protectMaskImageUrl } + : {}), sourceHeight, sourceWidth, }; @@ -2807,6 +2812,44 @@ async function applyRawAlphaToImage(params: { .toBuffer(); } +async function applyScaleAdjustProductProtectMaskToOverlay(params: { + height: number; + overlayBuffer: Buffer; + protectMaskBuffer: Buffer; + width: number; +}) { + const { height, overlayBuffer, protectMaskBuffer, width } = params; + const overlayRaw = await sharp(overlayBuffer, { failOn: "none" }) + .resize(width, height, { fit: "fill" }) + .ensureAlpha() + .raw() + .toBuffer(); + const protectRaw = await sharp(protectMaskBuffer, { failOn: "none" }) + .rotate() + .resize(width, height, { fit: "fill" }) + .ensureAlpha() + .extractChannel(3) + .raw() + .toBuffer(); + const pixelCount = Math.min( + protectRaw.length, + Math.floor(overlayRaw.length / 4) + ); + for (let index = 0; index < pixelCount; index += 1) { + const protect = protectRaw[index] || 0; + if (protect <= 0) continue; + const alphaIndex = index * 4 + 3; + const alpha = overlayRaw[alphaIndex] || 0; + overlayRaw[alphaIndex] = Math.round((alpha * Math.max(0, 255 - protect)) / 255); + } + + return sharp(overlayRaw, { + raw: { channels: 4, height, width }, + }) + .png({ compressionLevel: 6 }) + .toBuffer(); +} + async function applyLocalEditCompositeToImages(params: { images: ParsedImage[]; composite: CanvasLocalEditCompositeConfig | null; @@ -3204,6 +3247,9 @@ async function applyScaleAdjustProductLockCompositeToImages(params: { .ensureAlpha() .png({ compressionLevel: 6 }) .toBuffer(); + const protectMaskBuffer = composite.protectMaskImageUrl + ? await fetchImageBufferFromUrl(composite.protectMaskImageUrl) + : null; const overlayMetadata = await sharp(overlayBuffer, { failOn: "none" }).metadata(); const overlayWidth = Math.round(Number(overlayMetadata.width || 0)); const overlayHeight = Math.round(Number(overlayMetadata.height || 0)); @@ -3230,13 +3276,21 @@ async function applyScaleAdjustProductLockCompositeToImages(params: { continue; } - const resizedOverlayBuffer = await sharp(overlayBuffer, { failOn: "none" }) + const resizedOverlayBaseBuffer = await sharp(overlayBuffer, { failOn: "none" }) .resize(generatedWidth, generatedHeight, { fit: "fill", kernel: sharp.kernel.lanczos3, }) .png({ compressionLevel: 6 }) .toBuffer(); + const resizedOverlayBuffer = protectMaskBuffer + ? await applyScaleAdjustProductProtectMaskToOverlay({ + height: generatedHeight, + overlayBuffer: resizedOverlayBaseBuffer, + protectMaskBuffer, + width: generatedWidth, + }) + : resizedOverlayBaseBuffer; const mergedBuffer = await sharp(normalizedGeneratedBuffer, { failOn: "none", }) @@ -3271,6 +3325,7 @@ async function applyScaleAdjustProductLockCompositeToImages(params: { imageCount: mergedImages.length, overlayHeight, overlayWidth, + protectMaskApplied: Boolean(protectMaskBuffer), sourceHeight: composite.sourceHeight, sourceWidth: composite.sourceWidth, }); From 6694846c58bc43d6b32fa7b4a6c7cc4667190f22 Mon Sep 17 00:00:00 2001 From: 134 Date: Mon, 6 Jul 2026 03:00:28 +0800 Subject: [PATCH 11/70] fix: harden scale adjust person occlusion masks --- app/api/canvas/local-edit/route.ts | 58 +++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 5 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index a76b245da..cc187948a 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -1726,6 +1726,34 @@ function buildScaleAdjustProtectedPersonSeedMask(params: { }); } +function buildScaleAdjustProtectedPersonGeometryMask(params: { + box: { width: number; height: number }; + side: ScaleAdjustProtectedPersonSide | null; +}) { + const { box, side } = params; + if (!side) return null; + const mask = new Uint8Array(box.width * box.height); + for (let y = 0; y < box.height; y += 1) { + for (let x = 0; x < box.width; x += 1) { + const xr = x / Math.max(1, box.width); + const yr = y / Math.max(1, box.height); + const protect = + side === "right" + ? (xr >= 0.72 && yr >= 0.26) || + (xr >= 0.58 && yr >= 0.56) || + (xr >= 0.48 && yr >= 0.7) + : side === "left" + ? (xr <= 0.28 && yr >= 0.26) || + (xr <= 0.42 && yr >= 0.56) || + (xr <= 0.52 && yr >= 0.7) + : (yr >= 0.62 && (xr <= 0.2 || xr >= 0.8)) || + (yr >= 0.72 && (xr <= 0.34 || xr >= 0.66)); + if (protect) mask[y * box.width + x] = 255; + } + } + return mask; +} + function filterScaleAdjustStructureComponents(params: { box: { width: number; height: number }; mask: Uint8Array; @@ -1821,13 +1849,15 @@ function filterScaleAdjustStructureComponents(params: { if (protectedPersonSide === "right") { return ( (pixelX >= width * 0.7 && pixelY >= height * 0.38) || - (pixelX >= width * 0.62 && pixelY >= height * 0.56) + (pixelX >= width * 0.62 && pixelY >= height * 0.56) || + (pixelX >= width * 0.48 && pixelY >= height * 0.7) ); } if (protectedPersonSide === "left") { return ( (pixelX <= width * 0.3 && pixelY >= height * 0.38) || - (pixelX <= width * 0.38 && pixelY >= height * 0.56) + (pixelX <= width * 0.38 && pixelY >= height * 0.56) || + (pixelX <= width * 0.52 && pixelY >= height * 0.7) ); } if (protectedPersonSide === "auto") { @@ -2110,7 +2140,7 @@ async function buildScaleAdjustMaskWashPatch(params: { rgba[offset] = Math.round(background.r); rgba[offset + 1] = Math.round(background.g); rgba[offset + 2] = Math.round(background.b); - rgba[offset + 3] = Math.round((alpha[index] || 0) * 0.9); + rgba[offset + 3] = Math.min(255, Math.round((alpha[index] || 0) * 0.985)); } return sharp(rgba, { raw: { channels: 4, height: box.height, width: box.width }, @@ -2714,12 +2744,20 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: { .toBuffer(); const targetClipMaskPatch = await buildScaleAdjustOverlayClipMaskPatch(structureOverlay); - const productLockProtectMaskPatch = protectedPersonMask + const productLockProtectMask = + protectedPersonMask || + (protectedPersonSide + ? buildScaleAdjustProtectedPersonGeometryMask({ + box: placement.sourceBox, + side: protectedPersonSide, + }) + : null); + const productLockProtectMaskPatch = productLockProtectMask ? await buildScaleAdjustCompositeClipMaskPatch({ box: placement.sourceBox, maskBuffer: dilateScaleAdjustStructureMask({ height: placement.sourceBox.height, - mask: protectedPersonMask, + mask: productLockProtectMask, radius: Math.max( 2, Math.round( @@ -2742,6 +2780,8 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: { placedHeight, placedWidth, protectedPersonSide, + protectedPersonGeometryMaskApplied: + Boolean(productLockProtectMask) && !protectedPersonMask, protectedPersonMaskApplied: Boolean(protectedPersonMask), sameSizeCutout, alphaBandingStats, @@ -2874,6 +2914,10 @@ function suppressScaleAdjustProtectedFurnitureDarkAlpha(params: { return ( (sourceX >= sourceBox.width * 0.69 && sourceY >= sourceBox.height * 0.54) || (sourceX >= sourceBox.width * 0.58 && sourceY >= sourceBox.height * 0.7) || + (sourceX >= sourceBox.width * 0.5 && sourceY >= sourceBox.height * 0.74) || + (sourceX >= sourceBox.width * 0.56 && + sourceY >= sourceBox.height * 0.58 && + sourceY <= sourceBox.height * 0.78) || (sourceX >= sourceBox.width * 0.78 && sourceY >= sourceBox.height * 0.4) || (sourceX >= sourceBox.width * 0.91 && sourceY >= sourceBox.height * 0.26) ); @@ -2882,6 +2926,10 @@ function suppressScaleAdjustProtectedFurnitureDarkAlpha(params: { return ( (sourceX <= sourceBox.width * 0.31 && sourceY >= sourceBox.height * 0.54) || (sourceX <= sourceBox.width * 0.42 && sourceY >= sourceBox.height * 0.7) || + (sourceX <= sourceBox.width * 0.5 && sourceY >= sourceBox.height * 0.74) || + (sourceX <= sourceBox.width * 0.44 && + sourceY >= sourceBox.height * 0.58 && + sourceY <= sourceBox.height * 0.78) || (sourceX <= sourceBox.width * 0.22 && sourceY >= sourceBox.height * 0.4) || (sourceX <= sourceBox.width * 0.09 && sourceY >= sourceBox.height * 0.26) ); From 7640e6b5ca249392cd9a4a1ed4c76bd03abb4a13 Mon Sep 17 00:00:00 2001 From: 134 Date: Mon, 6 Jul 2026 03:22:59 +0800 Subject: [PATCH 12/70] fix: clean product lock masks before scale adjust overlay --- app/api/canvas/local-edit/route.ts | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index cc187948a..78ec735f9 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -1944,8 +1944,9 @@ function filterScaleAdjustStructureComponents(params: { function solidifyScaleAdjustStructureMask(params: { box: { width: number; height: number }; mask: Uint8Array; + maxPanelSpanRatio?: number; }) { - const { box, mask } = params; + const { box, mask, maxPanelSpanRatio = 0.86 } = params; const width = box.width; const height = box.height; const bridgeGap = Math.max(2, Math.round(Math.min(width, height) * 0.006)); @@ -2033,7 +2034,7 @@ function solidifyScaleAdjustStructureMask(params: { const density = marked / Math.max(1, span * bandHeight); if ( span < width * 0.16 || - span > width * 0.86 || + span > width * maxPanelSpanRatio || density < 0.025 || density > 0.78 ) { @@ -2535,6 +2536,14 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: { }), }) : null; + const protectedPersonRemovalMask = + protectedPersonMask || + (protectedPersonSide && sameSizeCutout + ? buildScaleAdjustProtectedPersonGeometryMask({ + box: { width: cutoutWidth, height: cutoutHeight }, + side: protectedPersonSide, + }) + : null); const sideProtectedCleanedAlpha = protectedPersonSide && sameSizeCutout && sourceCropRaw ? suppressScaleAdjustProtectedFurnitureDarkAlpha({ @@ -2548,7 +2557,7 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: { : cleanedAlpha; const protectedCleanedAlpha = removeScaleAdjustMaskFromAlpha({ alphaMask: sideProtectedCleanedAlpha, - removeMask: protectedPersonMask, + removeMask: protectedPersonRemovalMask, }); for (let index = 0; index < protectedCleanedAlpha.length; index += 1) { cleanedRaw[index * 4 + 3] = protectedCleanedAlpha[index] || 0; @@ -2644,6 +2653,7 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: { ? solidifyScaleAdjustStructureMask({ box: placement.sourceBox, mask: filteredSourceMask, + maxPanelSpanRatio: protectedPersonSide ? 0.96 : 0.9, }) : highConfidenceSourceMask; const sourceWashMask = dilateScaleAdjustStructureMask({ @@ -2745,7 +2755,7 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: { const targetClipMaskPatch = await buildScaleAdjustOverlayClipMaskPatch(structureOverlay); const productLockProtectMask = - protectedPersonMask || + protectedPersonRemovalMask || (protectedPersonSide ? buildScaleAdjustProtectedPersonGeometryMask({ box: placement.sourceBox, @@ -2781,7 +2791,7 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: { placedWidth, protectedPersonSide, protectedPersonGeometryMaskApplied: - Boolean(productLockProtectMask) && !protectedPersonMask, + Boolean(protectedPersonRemovalMask) && !protectedPersonMask, protectedPersonMaskApplied: Boolean(protectedPersonMask), sameSizeCutout, alphaBandingStats, From c79da3e1d086bb1a7910134b74ce74a1bf0cef2b Mon Sep 17 00:00:00 2001 From: 134 Date: Mon, 6 Jul 2026 03:41:56 +0800 Subject: [PATCH 13/70] fix: preserve single-channel scale adjust masks after blur --- app/api/canvas/local-edit/route.ts | 10 ++++++++++ lib/canvas/generate-image.ts | 2 ++ 2 files changed, 12 insertions(+) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 78ec735f9..dec126da7 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -2132,7 +2132,9 @@ async function buildScaleAdjustMaskWashPatch(params: { const alpha = await sharp(maskBuffer, { raw: { channels: 1, height: box.height, width: box.width }, }) + .greyscale() .blur(Math.max(1.2, Math.min(box.width, box.height) * 0.018)) + .greyscale() .raw() .toBuffer(); const rgba = Buffer.alloc(box.width * box.height * 4); @@ -2158,7 +2160,9 @@ async function buildScaleAdjustCompositeClipMaskPatch(params: { const alpha = await sharp(Buffer.from(maskBuffer), { raw: { channels: 1, height: box.height, width: box.width }, }) + .greyscale() .blur(Math.max(0.8, Math.min(box.width, box.height) * 0.006)) + .greyscale() .raw() .toBuffer(); const rgba = Buffer.alloc(box.width * box.height * 4); @@ -2183,7 +2187,9 @@ async function buildScaleAdjustOverlayClipMaskPatch(imageBuffer: Buffer) { const alpha = await sharp(imageBuffer, { failOn: "none" }) .ensureAlpha() .extractChannel(3) + .greyscale() .blur(Math.max(0.8, Math.min(width, height) * 0.006)) + .greyscale() .raw() .toBuffer(); const rgba = Buffer.alloc(width * height * 4); @@ -2297,7 +2303,9 @@ async function applyScaleAdjustAlphaMaskToRgba(params: { const softenedAlpha = await sharp(Buffer.from(alphaMask), { raw: { channels: 1, height, width }, }) + .greyscale() .blur(blur) + .greyscale() .raw() .toBuffer(); const output = Buffer.from(rawRgba); @@ -3184,7 +3192,9 @@ async function buildScaleAdjustStructureLockedComposite(params: { const subjectAlpha = await sharp(alphaMaskForSubject, { raw: { channels: 1, height: bbox.height, width: bbox.width }, }) + .greyscale() .blur(Math.max(0.7, Math.min(bbox.width, bbox.height) * 0.006)) + .greyscale() .png({ compressionLevel: 6 }) .toBuffer(); const widthFactor = diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index e56eab833..aca38923c 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -2314,7 +2314,9 @@ async function buildScaleAdjustFallbackObjectPng(params: { const alphaBuffer = await sharp(mask, { raw: { channels: 1, height: box.height, width: box.width }, }) + .greyscale() .blur(Math.max(0.7, Math.min(box.width, box.height) * 0.012)) + .greyscale() .png({ compressionLevel: 6 }) .toBuffer(); From 831ce10a5baa97bd4cefb00c0e4bc2332c91da0e Mon Sep 17 00:00:00 2001 From: 134 Date: Mon, 6 Jul 2026 04:28:56 +0800 Subject: [PATCH 14/70] fix: use mask edit for product lock scale adjust --- app/api/canvas/local-edit/route.ts | 307 +++++++++++++++++++---------- 1 file changed, 198 insertions(+), 109 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index dec126da7..7dbc354a7 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -127,6 +127,7 @@ type ScaleAdjustPatchReference = { backgroundPlateFullUrl?: string | null; compositeClipMaskUrl?: string | null; productLockMethod?: string | null; + productLockEditMaskUrl?: string | null; productLockOverlayUrl?: string | null; productLockProtectMaskUrl?: string | null; sourceCropUrl: string; @@ -466,7 +467,7 @@ function resolveScaleAdjustPreferredModelId(params: { if (params.preferMaskCapableGpt) { const maskCapableGptImage2 = gptImage2Candidates .filter(isScaleAdjustMaskCapableModel) - .filter(isScaleAdjustHealthyModel); + .filter((entry) => entry.model.healthStatus !== "auto_disabled"); if (maskCapableGptImage2[0]) { return maskCapableGptImage2[0].model.id; } @@ -2232,6 +2233,35 @@ async function buildScaleAdjustCompositeClipMaskBuffer(params: { .toBuffer(); } +async function buildScaleAdjustTransparentEditMaskFromOpacityMask(params: { + height: number; + maskBuffer: Buffer; + width: number; +}) { + const { height, maskBuffer, width } = params; + const editAlpha = await sharp(maskBuffer, { failOn: "none" }) + .rotate() + .resize(width, height, { fit: "fill" }) + .ensureAlpha() + .extractChannel(3) + .greyscale() + .raw() + .toBuffer(); + const rgba = Buffer.alloc(width * height * 4); + for (let index = 0; index < width * height; index += 1) { + const offset = index * 4; + rgba[offset] = 0; + rgba[offset + 1] = 0; + rgba[offset + 2] = 0; + rgba[offset + 3] = clamp(255 - (editAlpha[index] || 0), 0, 255); + } + return sharp(rgba, { + raw: { channels: 4, height, width }, + }) + .png({ compressionLevel: 9 }) + .toBuffer(); +} + function getScaleAdjustAlphaBoundingBox(params: { height: number; rawRgba: Buffer; @@ -3633,50 +3663,59 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { .png({ compressionLevel: 6 }) .toBuffer() : null; - const productLockOverlayBuffer = productLockComposites.length - ? await sharp({ - create: { - background: { alpha: 0, b: 0, g: 0, r: 0 }, + const productLockOverlayBuffer = productLockComposites.length + ? await sharp({ + create: { + background: { alpha: 0, b: 0, g: 0, r: 0 }, channels: 4, height: sourceHeight, width: sourceWidth, }, }) .composite(productLockComposites) - .png({ compressionLevel: 6 }) - .toBuffer() - : null; - const productLockProtectMaskBuffer = productLockProtectMaskComposites.length - ? await sharp({ - create: { - background: { alpha: 0, b: 255, g: 255, r: 255 }, - channels: 4, - height: sourceHeight, - width: sourceWidth, - }, - }) - .composite(productLockProtectMaskComposites) - .png({ compressionLevel: 6 }) - .toBuffer() - : null; - const compositeClipMaskBuffer = - compositeClipComposites.length || targetClipComposites.length + .png({ compressionLevel: 6 }) + .toBuffer() + : null; + const productLockProtectMaskBuffer = productLockProtectMaskComposites.length + ? await sharp({ + create: { + background: { alpha: 0, b: 255, g: 255, r: 255 }, + channels: 4, + height: sourceHeight, + width: sourceWidth, + }, + }) + .composite(productLockProtectMaskComposites) + .png({ compressionLevel: 6 }) + .toBuffer() + : null; + const compositeClipMaskBuffer = + compositeClipComposites.length || targetClipComposites.length ? await buildScaleAdjustCompositeClipMaskBuffer({ height: sourceHeight, - sourceMaskComposites: compositeClipComposites, - targetMaskComposites: targetClipComposites, - width: sourceWidth, - }) - : null; + sourceMaskComposites: compositeClipComposites, + targetMaskComposites: targetClipComposites, + width: sourceWidth, + }) + : null; + const productLockEditMaskBuffer = + productLockComposites.length && compositeClipMaskBuffer + ? await buildScaleAdjustTransparentEditMaskFromOpacityMask({ + height: sourceHeight, + maskBuffer: compositeClipMaskBuffer, + width: sourceWidth, + }) + : null; - return { - backgroundPlateBuffer, - buffer, - compositeClipMaskBuffer, - productLockOverlayBuffer, - productLockProtectMaskBuffer, - }; - } + return { + backgroundPlateBuffer, + buffer, + compositeClipMaskBuffer, + productLockEditMaskBuffer, + productLockOverlayBuffer, + productLockProtectMaskBuffer, + }; +} async function createScaleAdjustPatchReference(params: { fullFrameReferenceOnly?: boolean; @@ -3718,13 +3757,14 @@ async function createScaleAdjustPatchReference(params: { const targetDraftBuffer = targetDraftReference.buffer; const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; if (params.fullFrameReferenceOnly && structureLockAppliedRef.value) { - const [ - targetDraftFullUpload, - backgroundPlateFullUpload, - productLockOverlayUpload, - productLockProtectMaskUpload, - compositeClipMaskUpload, - ] = await Promise.all([ + const [ + targetDraftFullUpload, + backgroundPlateFullUpload, + productLockOverlayUpload, + productLockProtectMaskUpload, + compositeClipMaskUpload, + productLockEditMaskUpload, + ] = await Promise.all([ uploadFile( `canvas-scale-adjust-full-target-draft-${suffix}.png`, toArrayBuffer(targetDraftBuffer) @@ -3735,33 +3775,40 @@ async function createScaleAdjustPatchReference(params: { toArrayBuffer(targetDraftReference.backgroundPlateBuffer) ) : Promise.resolve(null), - targetDraftReference.productLockOverlayBuffer - ? uploadFile( - `canvas-scale-adjust-product-lock-overlay-${suffix}.png`, - toArrayBuffer(targetDraftReference.productLockOverlayBuffer) - ) - : Promise.resolve(null), - targetDraftReference.productLockProtectMaskBuffer - ? uploadFile( - `canvas-scale-adjust-product-lock-protect-mask-${suffix}.png`, - toArrayBuffer(targetDraftReference.productLockProtectMaskBuffer) - ) - : Promise.resolve(null), - targetDraftReference.compositeClipMaskBuffer - ? uploadFile( - `canvas-scale-adjust-composite-clip-mask-${suffix}.png`, + targetDraftReference.productLockOverlayBuffer + ? uploadFile( + `canvas-scale-adjust-product-lock-overlay-${suffix}.png`, + toArrayBuffer(targetDraftReference.productLockOverlayBuffer) + ) + : Promise.resolve(null), + targetDraftReference.productLockProtectMaskBuffer + ? uploadFile( + `canvas-scale-adjust-product-lock-protect-mask-${suffix}.png`, + toArrayBuffer(targetDraftReference.productLockProtectMaskBuffer) + ) + : Promise.resolve(null), + targetDraftReference.compositeClipMaskBuffer + ? uploadFile( + `canvas-scale-adjust-composite-clip-mask-${suffix}.png`, toArrayBuffer(targetDraftReference.compositeClipMaskBuffer) ) : Promise.resolve(null), + targetDraftReference.productLockEditMaskBuffer + ? uploadFile( + `canvas-scale-adjust-product-lock-edit-mask-${suffix}.png`, + toArrayBuffer(targetDraftReference.productLockEditMaskBuffer) + ) + : Promise.resolve(null), ]); return { backgroundPlateFullUrl: backgroundPlateFullUpload?.url || null, - compositeClipMaskUrl: compositeClipMaskUpload?.url || null, - productLockMethod: productLockOverlayUpload ? "rmbg-cutout" : null, - productLockOverlayUrl: productLockOverlayUpload?.url || null, - productLockProtectMaskUrl: productLockProtectMaskUpload?.url || null, - sourceCropUrl: sourceImageUrl || targetDraftFullUpload.url, + compositeClipMaskUrl: compositeClipMaskUpload?.url || null, + productLockMethod: productLockOverlayUpload ? "rmbg-cutout" : null, + productLockEditMaskUrl: productLockEditMaskUpload?.url || null, + productLockOverlayUrl: productLockOverlayUpload?.url || null, + productLockProtectMaskUrl: productLockProtectMaskUpload?.url || null, + sourceCropUrl: sourceImageUrl || targetDraftFullUpload.url, structureLockedDraft: structureLockAppliedRef.value, targetDraftFullUrl: targetDraftFullUpload.url, targetDraftUrl: targetDraftFullUpload.url, @@ -3780,15 +3827,16 @@ async function createScaleAdjustPatchReference(params: { .png({ compressionLevel: 6 }) .toBuffer(), ]); - const [ - sourceCropUpload, - targetDraftUpload, - targetDraftFullUpload, - backgroundPlateFullUpload, - productLockOverlayUpload, - productLockProtectMaskUpload, - compositeClipMaskUpload, - ] = await Promise.all([ + const [ + sourceCropUpload, + targetDraftUpload, + targetDraftFullUpload, + backgroundPlateFullUpload, + productLockOverlayUpload, + productLockProtectMaskUpload, + compositeClipMaskUpload, + productLockEditMaskUpload, + ] = await Promise.all([ uploadFile( `canvas-scale-adjust-workspace-source-${suffix}.png`, toArrayBuffer(sourceCropBuffer) @@ -3807,33 +3855,40 @@ async function createScaleAdjustPatchReference(params: { toArrayBuffer(targetDraftReference.backgroundPlateBuffer) ) : Promise.resolve(null), - targetDraftReference.productLockOverlayBuffer - ? uploadFile( - `canvas-scale-adjust-product-lock-overlay-${suffix}.png`, - toArrayBuffer(targetDraftReference.productLockOverlayBuffer) - ) - : Promise.resolve(null), - targetDraftReference.productLockProtectMaskBuffer - ? uploadFile( - `canvas-scale-adjust-product-lock-protect-mask-${suffix}.png`, - toArrayBuffer(targetDraftReference.productLockProtectMaskBuffer) - ) - : Promise.resolve(null), - targetDraftReference.compositeClipMaskBuffer - ? uploadFile( - `canvas-scale-adjust-composite-clip-mask-${suffix}.png`, + targetDraftReference.productLockOverlayBuffer + ? uploadFile( + `canvas-scale-adjust-product-lock-overlay-${suffix}.png`, + toArrayBuffer(targetDraftReference.productLockOverlayBuffer) + ) + : Promise.resolve(null), + targetDraftReference.productLockProtectMaskBuffer + ? uploadFile( + `canvas-scale-adjust-product-lock-protect-mask-${suffix}.png`, + toArrayBuffer(targetDraftReference.productLockProtectMaskBuffer) + ) + : Promise.resolve(null), + targetDraftReference.compositeClipMaskBuffer + ? uploadFile( + `canvas-scale-adjust-composite-clip-mask-${suffix}.png`, toArrayBuffer(targetDraftReference.compositeClipMaskBuffer) ) : Promise.resolve(null), + targetDraftReference.productLockEditMaskBuffer + ? uploadFile( + `canvas-scale-adjust-product-lock-edit-mask-${suffix}.png`, + toArrayBuffer(targetDraftReference.productLockEditMaskBuffer) + ) + : Promise.resolve(null), ]); return { backgroundPlateFullUrl: backgroundPlateFullUpload?.url || null, - compositeClipMaskUrl: compositeClipMaskUpload?.url || null, - productLockMethod: productLockOverlayUpload ? "rmbg-cutout" : null, - productLockOverlayUrl: productLockOverlayUpload?.url || null, - productLockProtectMaskUrl: productLockProtectMaskUpload?.url || null, - sourceCropUrl: sourceCropUpload.url, + compositeClipMaskUrl: compositeClipMaskUpload?.url || null, + productLockMethod: productLockOverlayUpload ? "rmbg-cutout" : null, + productLockEditMaskUrl: productLockEditMaskUpload?.url || null, + productLockOverlayUrl: productLockOverlayUpload?.url || null, + productLockProtectMaskUrl: productLockProtectMaskUpload?.url || null, + sourceCropUrl: sourceCropUpload.url, structureLockedDraft: structureLockAppliedRef.value, targetDraftFullUrl: targetDraftFullUpload.url, targetDraftUrl: targetDraftUpload.url, @@ -4821,7 +4876,8 @@ function buildScaleAdjustFullFramePatchPrompt( function buildScaleAdjustProductLockBackgroundPrompt( regions: NormalizedLocalEditRegion[], scalePercent: ScaleAdjustPercent | null | undefined, - sourceSize: { width: number; height: number } + sourceSize: { width: number; height: number }, + options: { maskEdit?: boolean } = {} ) { const regionLines = regions .map((region) => { @@ -4836,12 +4892,16 @@ function buildScaleAdjustProductLockBackgroundPrompt( return [ "Full-frame background repair for product-locked scale adjustment.", "Return exactly one clean realistic full image with the same framing, camera, aspect ratio, and canvas extent as input image 1. Do not crop, zoom, pan, or resize the whole photo.", - "Input image 1 is the original clean full photo. Input image 2 is a background plate reference that removes the selected furniture/product footprint and keeps the room/person/background layout stable.", + options.maskEdit + ? "Input image 1 is the original clean full photo. The supplied edit mask marks the old selected furniture/product footprint that must be repaired as natural background; do not edit outside the masked area." + : "Input image 1 is the original clean full photo. Input image 2 is a background plate reference that removes the selected furniture/product footprint and keeps the room/person/background layout stable.", "Important: do not draw, recreate, move, scale, duplicate, or redesign the selected furniture/product in the final image. The system will composite the locked original product pixels into the final target footprint after your background image is returned.", "Your job is only to erase the old selected furniture/product footprint, complete newly exposed wall, curtain, floor, rug, shadow, sofa edge, and background naturally, and keep the target footprint clean enough for the locked product layer to sit there.", "Preserve all non-target people exactly as foreground/background context: do not move, cover, deform, redraw, change face, change body, change hands, change legs, change clothes, or change pose.", "Keep all non-target content stable: sofa, window, curtain, cabinet, lamp, wall art, floor, rug, lighting, textural grain, and global color must remain consistent.", - "Remove draft artifacts from input image 2: pale wash blocks, rough masks, missing patches, hard rectangular edges, black holes, ghost fragments, pasted silhouettes, and old-size residues. They are only edit guides, not final content.", + options.maskEdit + ? "The edit mask is only an invisible repair boundary. Do not output pale wash blocks, rough mask shapes, hard rectangular edges, black holes, ghost fragments, pasted silhouettes, or old-size residues." + : "Remove draft artifacts from input image 2: pale wash blocks, rough masks, missing patches, hard rectangular edges, black holes, ghost fragments, pasted silhouettes, and old-size residues. They are only edit guides, not final content.", "Do not output boxes, masks, labels, coordinate marks, UI marks, arrows, guide overlays, crop borders, or any annotation artifacts.", "中文硬约束：这是产品结构锁定流程。你只生成背景底图，不要生成、移动、缩放或重绘目标家具/商品；目标家具/商品会在你输出后由系统用原图锁定像素叠回目标位置。必须擦除旧位置的目标家具/商品并自然补全墙面、窗帘、地面、阴影和背景。非目标人物必须保持原样，不能变脸、变形、改变衣服、手、腿、脚或姿势。最终背景图不能有矩形雾块、贴片边、黑洞、残影、标注框，也不能出现第二个目标家具/商品。", "", @@ -5201,6 +5261,10 @@ export async function POST(request: NextRequest) { useScaleAdjustFullFrameStructurePatch && Boolean(scaleAdjustPatchReference?.backgroundPlateFullUrl) && Boolean(scaleAdjustPatchReference?.productLockOverlayUrl); + const useScaleAdjustProductLockMaskEdit = + useScaleAdjustProductLockComposite && + isScaleAdjustMaskCapableModel(resolvedScaleAdjustModelEntry) && + Boolean(scaleAdjustPatchReference?.productLockEditMaskUrl); const useScaleAdjustPatchTargetDraftPrimary = Boolean(scaleAdjustPatchReference); const includeScaleAdjustPatchSourceReference = Boolean(scaleAdjustPatchReference); @@ -5215,7 +5279,8 @@ export async function POST(request: NextRequest) { ? buildScaleAdjustProductLockBackgroundPrompt( regions, scalePercent, - { width: sourceWidth, height: sourceHeight } + { width: sourceWidth, height: sourceHeight }, + { maskEdit: useScaleAdjustProductLockMaskEdit } ) : buildScaleAdjustFullFramePatchPrompt( regions, @@ -5286,7 +5351,13 @@ export async function POST(request: NextRequest) { ? scaleAdjustPatchReference ? useScaleAdjustFullFrameStructurePatch ? useScaleAdjustProductLockComposite - ? [ + ? useScaleAdjustProductLockMaskEdit + ? [ + sourceImageUrl, + ...scaleAdjustCropReferenceImageUrls, + ...manualRegionReferenceImageUrls, + ] + : [ sourceImageUrl, scaleAdjustPatchReference.backgroundPlateFullUrl as string, ...scaleAdjustCropReferenceImageUrls, @@ -5389,7 +5460,9 @@ export async function POST(request: NextRequest) { annotatedReferenceType: isScaleAdjustOperation ? useScaleAdjustFullFrameStructurePatch ? useScaleAdjustProductLockComposite - ? "scale-adjust-product-lock-background-plate" + ? useScaleAdjustProductLockMaskEdit + ? "scale-adjust-product-lock-mask-edit" + : "scale-adjust-product-lock-background-plate" : "scale-adjust-full-frame-structure-target-draft" : useProviderTextOnlyCoordinatePrompt ? "scale-adjust-text-coordinate-only-debug-map" @@ -5437,7 +5510,9 @@ export async function POST(request: NextRequest) { isScaleAdjustFalNanoBananaEdit && hasScaleAdjustShrink, scaleAdjustPatchReferenceOrder: scaleAdjustPatchReference && useScaleAdjustFullFrameStructurePatch - ? "full-source-full-target-draft" + ? useScaleAdjustProductLockMaskEdit + ? "full-source-product-lock-edit-mask" + : "full-source-full-target-draft" : scaleAdjustPatchReference && useScaleAdjustPatchTargetDraftPrimary ? includeScaleAdjustPatchSourceReference ? "target-draft-source-identity" @@ -5457,12 +5532,19 @@ export async function POST(request: NextRequest) { useScaleAdjustProductLockComposite, scaleAdjustProductLockMethod: scaleAdjustPatchReference?.productLockMethod || null, - scaleAdjustProductLockOverlayUrl: - scaleAdjustPatchReference?.productLockOverlayUrl || null, - scaleAdjustProductLockProtectMaskUrl: - scaleAdjustPatchReference?.productLockProtectMaskUrl || null, - scaleAdjustPatchCompositeClipMaskUrl: - scaleAdjustPatchReference?.compositeClipMaskUrl || null, + scaleAdjustProductLockMaskEditApplied: + useScaleAdjustProductLockMaskEdit, + scaleAdjustProductLockEditMaskUrl: + scaleAdjustPatchReference?.productLockEditMaskUrl || null, + scaleAdjustProductLockOverlayUrl: + scaleAdjustPatchReference?.productLockOverlayUrl || null, + scaleAdjustProductLockProtectMaskUrl: + scaleAdjustPatchReference?.productLockProtectMaskUrl || null, + scaleAdjustPatchCompositeClipMaskUrl: + scaleAdjustPatchReference?.compositeClipMaskUrl || null, + scaleAdjustProviderMaskImageUrl: useScaleAdjustProductLockMaskEdit + ? scaleAdjustPatchReference?.productLockEditMaskUrl || null + : scaleAdjustEditMaskUpload?.url || null, scaleAdjustPatchWorkspacePixelRect: scaleAdjustPatchReference?.workspacePixelRect || null, scaleAdjustPatchWorkspaceRect: @@ -5516,7 +5598,9 @@ export async function POST(request: NextRequest) { scaleAdjustPromptStrategy: isScaleAdjustOperation ? scaleAdjustPatchReference ? useScaleAdjustProductLockComposite - ? "product-lock-background-plate-rmbg-v2-source-patch-one-shot" + ? useScaleAdjustProductLockMaskEdit + ? "product-lock-mask-edit-rmbg-source-patch-one-shot" + : "product-lock-background-plate-rmbg-v2-source-patch-one-shot" : useScaleAdjustFullFrameStructurePatch ? "full-frame-source-locked-target-guide-v8-one-shot" : scaleAdjustPatchReference.structureLockedDraft @@ -5575,9 +5659,14 @@ export async function POST(request: NextRequest) { : "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、选区边框、白边、画幅压扁、画面拉伸、主体截断、动物截断、缺头、缺尾巴、缺四肢、贴边裁切、矩形贴片、硬边、拼接线、补丁感、色差边缘、背景错位、地面纹理断裂、椅脚错位", prompt, promptExtend: false, - providerImageParams: scaleAdjustEditMaskUpload - ? { maskImageUrl: scaleAdjustEditMaskUpload.url } - : undefined, + providerImageParams: useScaleAdjustProductLockMaskEdit + ? { + maskImageUrl: + scaleAdjustPatchReference?.productLockEditMaskUrl as string, + } + : scaleAdjustEditMaskUpload + ? { maskImageUrl: scaleAdjustEditMaskUpload.url } + : undefined, referenceImages, scaleAdjustPatchComposite: useScaleAdjustProductLockComposite ? { From fb9f0541fa85f66c0f41145dc4810f9f270b01a0 Mon Sep 17 00:00:00 2001 From: 134 Date: Mon, 6 Jul 2026 04:49:56 +0800 Subject: [PATCH 15/70] fix: restrict scale adjust mask failover --- app/api/canvas/local-edit/route.ts | 2 ++ lib/canvas/generate-image.ts | 27 +++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 7dbc354a7..552373197 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -5534,6 +5534,8 @@ export async function POST(request: NextRequest) { scaleAdjustPatchReference?.productLockMethod || null, scaleAdjustProductLockMaskEditApplied: useScaleAdjustProductLockMaskEdit, + scaleAdjustRequiresProviderMaskInput: + useScaleAdjustProductLockMaskEdit, scaleAdjustProductLockEditMaskUrl: scaleAdjustPatchReference?.productLockEditMaskUrl || null, scaleAdjustProductLockOverlayUrl: diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index aca38923c..79d9b70a0 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -3421,6 +3421,19 @@ function isCanvasScaleAdjustEmergencyFalEditModel(item: CanvasModelWithProvider) ); } +function isCanvasScaleAdjustProviderMaskEditModel(item: CanvasModelWithProvider) { + if (getCanvasScaleAdjustModelFamily(item) !== "gpt-image-2") return false; + if (item.provider.type === "openai-image") return true; + if (item.provider.type !== "fal-image") return false; + const modelId = String(item.model.modelId || "").trim().toLowerCase(); + return ( + modelId === "openai/gpt-image-2/edit" || + modelId === "fal-ai/gpt-image-2/edit" || + modelId.endsWith("/gpt-image-2/edit") || + modelId.includes("gpt-image-2/edit") + ); +} + interface CanvasImageGenConfig { maxImages?: number; supports1K?: boolean; @@ -3617,6 +3630,7 @@ function findCanvasFailoverModel(params: { failedModelRefs: string[]; failedProviderTypes?: string[]; preferLowCostSameNameProviders?: boolean; + requiresProviderMaskInput?: boolean; requiresImageInput?: boolean; requiresTextToImage?: boolean; scaleAdjustModelFamily?: string | null; @@ -3675,6 +3689,12 @@ function findCanvasFailoverModel(params: { } if (failedProviderTypes.has(item.provider.type)) return false; if (!item.model.supportsImageGen) return false; + if ( + params.requiresProviderMaskInput && + !isCanvasScaleAdjustProviderMaskEditModel(item) + ) { + return false; + } if ( params.requiresImageInput && modelLacksReferenceImageInputSupport(item.model, item.provider) @@ -3827,6 +3847,10 @@ export async function generateCanvasImage( typeof debugMeta?.scaleAdjustModelFamily === "string" ? debugMeta.scaleAdjustModelFamily.trim() : ""; + const requiresProviderMaskInput = Boolean( + debugMeta?.scaleAdjustProductLockMaskEditApplied && + debugMeta?.scaleAdjustProviderMaskImageUrl + ); const localEditComposite = normalizeLocalEditCompositeConfig( request.localEditComposite ); @@ -3886,6 +3910,7 @@ export async function generateCanvasImage( preferLowCostSameNameProviders: Boolean( debugMeta?.scaleAdjustPreferSameNameProviderFailover ), + requiresProviderMaskInput, requiresImageInput, scaleAdjustModelFamily, }); @@ -3928,6 +3953,7 @@ export async function generateCanvasImage( preferLowCostSameNameProviders: Boolean( debugMeta?.scaleAdjustPreferSameNameProviderFailover ), + requiresProviderMaskInput, requiresTextToImage, scaleAdjustModelFamily, }); @@ -4594,6 +4620,7 @@ export async function generateCanvasImage( preferLowCostSameNameProviders: Boolean( debugMeta?.scaleAdjustPreferSameNameProviderFailover ), + requiresProviderMaskInput, requiresImageInput: normalizedReferenceImages.length > 0, requiresTextToImage: normalizedReferenceImages.length === 0, scaleAdjustModelFamily, From 00a6ac08df45b327e5ab3f351ed0e828d8fc89f2 Mon Sep 17 00:00:00 2001 From: 134 Date: Mon, 6 Jul 2026 05:11:23 +0800 Subject: [PATCH 16/70] fix: tighten product lock scale adjust clip mask --- app/api/canvas/local-edit/route.ts | 44 ++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 552373197..5d94baec2 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -2687,6 +2687,15 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: { ) { return null; } + const sourceEditBaseMask = filteredSourceBbox + ? filteredSourceMask + : highConfidenceSourceMask; + const sourceEditMaskBbox = getScaleAdjustMaskBoundingBox({ + height: placement.sourceBox.height, + mask: sourceEditBaseMask, + width: placement.sourceBox.width, + }); + if (!sourceEditMaskBbox) return null; const sourceWashBaseMask = filteredSourceBbox ? solidifyScaleAdjustStructureMask({ box: placement.sourceBox, @@ -2707,15 +2716,26 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: { }); const sourceCompositeClipMask = dilateScaleAdjustStructureMask({ height: placement.sourceBox.height, - mask: sourceWashBaseMask, + mask: sourceEditBaseMask, radius: Math.max( - 1, + 4, Math.round( - Math.min(placement.sourceBox.width, placement.sourceBox.height) * 0.003 + Math.min(placement.sourceBox.width, placement.sourceBox.height) * + (protectedPersonSide ? 0.018 : 0.014) ) ), width: placement.sourceBox.width, }); + const sourceWashMaskBbox = getScaleAdjustMaskBoundingBox({ + height: placement.sourceBox.height, + mask: sourceWashMask, + width: placement.sourceBox.width, + }); + const sourceCompositeClipBbox = getScaleAdjustMaskBoundingBox({ + height: placement.sourceBox.height, + mask: sourceCompositeClipMask, + width: placement.sourceBox.width, + }); const sourceWashPatch = await buildScaleAdjustMaskWashPatch({ background, box: placement.sourceBox, @@ -2833,7 +2853,25 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: { protectedPersonMaskApplied: Boolean(protectedPersonMask), sameSizeCutout, alphaBandingStats, + sourceClipAlphaRatio: Number( + ( + (sourceCompositeClipBbox?.count || 0) / + Math.max(1, placement.sourceBox.width * placement.sourceBox.height) + ).toFixed(3) + ), + sourceEditAlphaRatio: Number( + ( + sourceEditMaskBbox.count / + Math.max(1, placement.sourceBox.width * placement.sourceBox.height) + ).toFixed(3) + ), sourceHeight: placement.sourceBox.height, + sourceWashAlphaRatio: Number( + ( + (sourceWashMaskBbox?.count || 0) / + Math.max(1, placement.sourceBox.width * placement.sourceBox.height) + ).toFixed(3) + ), sourceWidth: placement.sourceBox.width, }); From aad838a8e0de845517bae6c7a869edea0e321e0b Mon Sep 17 00:00:00 2001 From: 134 Date: Mon, 6 Jul 2026 05:29:57 +0800 Subject: [PATCH 17/70] fix: guide product lock background repair with tight mask --- app/api/canvas/local-edit/route.ts | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 5d94baec2..7ee9fb4b8 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -2696,13 +2696,7 @@ async function buildScaleAdjustRmbgStructureLockedComposite(params: { width: placement.sourceBox.width, }); if (!sourceEditMaskBbox) return null; - const sourceWashBaseMask = filteredSourceBbox - ? solidifyScaleAdjustStructureMask({ - box: placement.sourceBox, - mask: filteredSourceMask, - maxPanelSpanRatio: protectedPersonSide ? 0.96 : 0.9, - }) - : highConfidenceSourceMask; + const sourceWashBaseMask = sourceEditBaseMask; const sourceWashMask = dilateScaleAdjustStructureMask({ height: placement.sourceBox.height, mask: sourceWashBaseMask, @@ -4915,7 +4909,7 @@ function buildScaleAdjustProductLockBackgroundPrompt( regions: NormalizedLocalEditRegion[], scalePercent: ScaleAdjustPercent | null | undefined, sourceSize: { width: number; height: number }, - options: { maskEdit?: boolean } = {} + options: { backgroundReference?: boolean; maskEdit?: boolean } = {} ) { const regionLines = regions .map((region) => { @@ -4933,6 +4927,9 @@ function buildScaleAdjustProductLockBackgroundPrompt( options.maskEdit ? "Input image 1 is the original clean full photo. The supplied edit mask marks the old selected furniture/product footprint that must be repaired as natural background; do not edit outside the masked area." : "Input image 1 is the original clean full photo. Input image 2 is a background plate reference that removes the selected furniture/product footprint and keeps the room/person/background layout stable.", + options.maskEdit && options.backgroundReference + ? "Input image 2 is a rough background-only repair reference from the same photo. Use it only to understand that the old selected product must disappear and to infer nearby wall, curtain, floor, rug, shadow, and lighting continuity. Do not copy any flat wash, haze, patch edge, mask silhouette, or rough repaired texture from input image 2." + : "", "Important: do not draw, recreate, move, scale, duplicate, or redesign the selected furniture/product in the final image. The system will composite the locked original product pixels into the final target footprint after your background image is returned.", "Your job is only to erase the old selected furniture/product footprint, complete newly exposed wall, curtain, floor, rug, shadow, sofa edge, and background naturally, and keep the target footprint clean enough for the locked product layer to sit there.", "Preserve all non-target people exactly as foreground/background context: do not move, cover, deform, redraw, change face, change body, change hands, change legs, change clothes, or change pose.", @@ -5318,7 +5315,12 @@ export async function POST(request: NextRequest) { regions, scalePercent, { width: sourceWidth, height: sourceHeight }, - { maskEdit: useScaleAdjustProductLockMaskEdit } + { + backgroundReference: + useScaleAdjustProductLockMaskEdit && + Boolean(scaleAdjustPatchReference?.backgroundPlateFullUrl), + maskEdit: useScaleAdjustProductLockMaskEdit, + } ) : buildScaleAdjustFullFramePatchPrompt( regions, @@ -5392,6 +5394,9 @@ export async function POST(request: NextRequest) { ? useScaleAdjustProductLockMaskEdit ? [ sourceImageUrl, + ...(scaleAdjustPatchReference.backgroundPlateFullUrl + ? [scaleAdjustPatchReference.backgroundPlateFullUrl] + : []), ...scaleAdjustCropReferenceImageUrls, ...manualRegionReferenceImageUrls, ] From d94f6dea8c2d0be81b1cba93867f0e6dd0f9c522 Mon Sep 17 00:00:00 2001 From: 134 Date: Mon, 6 Jul 2026 05:54:52 +0800 Subject: [PATCH 18/70] fix: send opaque scale adjust masks to fal --- app/api/canvas/local-edit/route.ts | 67 +++++++++++++++++++++++++++--- lib/ai/providers/fal-image.ts | 5 ++- 2 files changed, 65 insertions(+), 7 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 7ee9fb4b8..6e4ad5593 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -128,6 +128,7 @@ type ScaleAdjustPatchReference = { compositeClipMaskUrl?: string | null; productLockMethod?: string | null; productLockEditMaskUrl?: string | null; + productLockFalEditMaskUrl?: string | null; productLockOverlayUrl?: string | null; productLockProtectMaskUrl?: string | null; sourceCropUrl: string; @@ -2262,6 +2263,36 @@ async function buildScaleAdjustTransparentEditMaskFromOpacityMask(params: { .toBuffer(); } +async function buildScaleAdjustOpaqueEditMaskFromOpacityMask(params: { + height: number; + maskBuffer: Buffer; + width: number; +}) { + const { height, maskBuffer, width } = params; + const editAlpha = await sharp(maskBuffer, { failOn: "none" }) + .rotate() + .resize(width, height, { fit: "fill" }) + .ensureAlpha() + .extractChannel(3) + .greyscale() + .raw() + .toBuffer(); + const rgba = Buffer.alloc(width * height * 4); + for (let index = 0; index < width * height; index += 1) { + const offset = index * 4; + const value = editAlpha[index] || 0; + rgba[offset] = value; + rgba[offset + 1] = value; + rgba[offset + 2] = value; + rgba[offset + 3] = 255; + } + return sharp(rgba, { + raw: { channels: 4, height, width }, + }) + .png({ compressionLevel: 9 }) + .toBuffer(); +} + function getScaleAdjustAlphaBoundingBox(params: { height: number; rawRgba: Buffer; @@ -3738,12 +3769,21 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { width: sourceWidth, }) : null; + const productLockFalEditMaskBuffer = + productLockComposites.length && compositeClipMaskBuffer + ? await buildScaleAdjustOpaqueEditMaskFromOpacityMask({ + height: sourceHeight, + maskBuffer: compositeClipMaskBuffer, + width: sourceWidth, + }) + : null; return { backgroundPlateBuffer, buffer, compositeClipMaskBuffer, productLockEditMaskBuffer, + productLockFalEditMaskBuffer, productLockOverlayBuffer, productLockProtectMaskBuffer, }; @@ -3796,6 +3836,7 @@ async function createScaleAdjustPatchReference(params: { productLockProtectMaskUpload, compositeClipMaskUpload, productLockEditMaskUpload, + productLockFalEditMaskUpload, ] = await Promise.all([ uploadFile( `canvas-scale-adjust-full-target-draft-${suffix}.png`, @@ -3831,6 +3872,12 @@ async function createScaleAdjustPatchReference(params: { toArrayBuffer(targetDraftReference.productLockEditMaskBuffer) ) : Promise.resolve(null), + targetDraftReference.productLockFalEditMaskBuffer + ? uploadFile( + `canvas-scale-adjust-product-lock-fal-edit-mask-${suffix}.png`, + toArrayBuffer(targetDraftReference.productLockFalEditMaskBuffer) + ) + : Promise.resolve(null), ]); return { @@ -3838,6 +3885,7 @@ async function createScaleAdjustPatchReference(params: { compositeClipMaskUrl: compositeClipMaskUpload?.url || null, productLockMethod: productLockOverlayUpload ? "rmbg-cutout" : null, productLockEditMaskUrl: productLockEditMaskUpload?.url || null, + productLockFalEditMaskUrl: productLockFalEditMaskUpload?.url || null, productLockOverlayUrl: productLockOverlayUpload?.url || null, productLockProtectMaskUrl: productLockProtectMaskUpload?.url || null, sourceCropUrl: sourceImageUrl || targetDraftFullUpload.url, @@ -3868,6 +3916,7 @@ async function createScaleAdjustPatchReference(params: { productLockProtectMaskUpload, compositeClipMaskUpload, productLockEditMaskUpload, + productLockFalEditMaskUpload, ] = await Promise.all([ uploadFile( `canvas-scale-adjust-workspace-source-${suffix}.png`, @@ -3911,6 +3960,12 @@ async function createScaleAdjustPatchReference(params: { toArrayBuffer(targetDraftReference.productLockEditMaskBuffer) ) : Promise.resolve(null), + targetDraftReference.productLockFalEditMaskBuffer + ? uploadFile( + `canvas-scale-adjust-product-lock-fal-edit-mask-${suffix}.png`, + toArrayBuffer(targetDraftReference.productLockFalEditMaskBuffer) + ) + : Promise.resolve(null), ]); return { @@ -3918,6 +3973,7 @@ async function createScaleAdjustPatchReference(params: { compositeClipMaskUrl: compositeClipMaskUpload?.url || null, productLockMethod: productLockOverlayUpload ? "rmbg-cutout" : null, productLockEditMaskUrl: productLockEditMaskUpload?.url || null, + productLockFalEditMaskUrl: productLockFalEditMaskUpload?.url || null, productLockOverlayUrl: productLockOverlayUpload?.url || null, productLockProtectMaskUrl: productLockProtectMaskUpload?.url || null, sourceCropUrl: sourceCropUpload.url, @@ -5316,9 +5372,6 @@ export async function POST(request: NextRequest) { scalePercent, { width: sourceWidth, height: sourceHeight }, { - backgroundReference: - useScaleAdjustProductLockMaskEdit && - Boolean(scaleAdjustPatchReference?.backgroundPlateFullUrl), maskEdit: useScaleAdjustProductLockMaskEdit, } ) @@ -5394,9 +5447,6 @@ export async function POST(request: NextRequest) { ? useScaleAdjustProductLockMaskEdit ? [ sourceImageUrl, - ...(scaleAdjustPatchReference.backgroundPlateFullUrl - ? [scaleAdjustPatchReference.backgroundPlateFullUrl] - : []), ...scaleAdjustCropReferenceImageUrls, ...manualRegionReferenceImageUrls, ] @@ -5581,6 +5631,8 @@ export async function POST(request: NextRequest) { useScaleAdjustProductLockMaskEdit, scaleAdjustProductLockEditMaskUrl: scaleAdjustPatchReference?.productLockEditMaskUrl || null, + scaleAdjustProductLockFalEditMaskUrl: + scaleAdjustPatchReference?.productLockFalEditMaskUrl || null, scaleAdjustProductLockOverlayUrl: scaleAdjustPatchReference?.productLockOverlayUrl || null, scaleAdjustProductLockProtectMaskUrl: @@ -5706,6 +5758,9 @@ export async function POST(request: NextRequest) { promptExtend: false, providerImageParams: useScaleAdjustProductLockMaskEdit ? { + falMaskImageUrl: + scaleAdjustPatchReference?.productLockFalEditMaskUrl || + (scaleAdjustPatchReference?.productLockEditMaskUrl as string), maskImageUrl: scaleAdjustPatchReference?.productLockEditMaskUrl as string, } diff --git a/lib/ai/providers/fal-image.ts b/lib/ai/providers/fal-image.ts index b4e8f37fc..930815fe8 100644 --- a/lib/ai/providers/fal-image.ts +++ b/lib/ai/providers/fal-image.ts @@ -21,6 +21,7 @@ interface FalImageParams { safetyTolerance?: string | number; seed?: string | number; enableWebSearch?: boolean; + falMaskImageUrl?: string; quality?: string; maskImageUrl?: string; openaiApiKey?: string; @@ -690,7 +691,9 @@ function buildFalRequestPayload(params: { output_format: outputFormat, }; - const maskImageUrl = normalizeOptionalHttpUrl(params.imageParams?.maskImageUrl); + const maskImageUrl = + normalizeOptionalHttpUrl(params.imageParams?.falMaskImageUrl) || + normalizeOptionalHttpUrl(params.imageParams?.maskImageUrl); if (maskImageUrl) { input.mask_url = maskImageUrl; } From 819bb9248816f350ef2153434459c308561c4ffc Mon Sep 17 00:00:00 2001 From: 134 Date: Mon, 6 Jul 2026 06:12:27 +0800 Subject: [PATCH 19/70] fix: use product lock draft as fal repair input --- app/api/canvas/local-edit/route.ts | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 6e4ad5593..760f49063 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -4965,7 +4965,7 @@ function buildScaleAdjustProductLockBackgroundPrompt( regions: NormalizedLocalEditRegion[], scalePercent: ScaleAdjustPercent | null | undefined, sourceSize: { width: number; height: number }, - options: { backgroundReference?: boolean; maskEdit?: boolean } = {} + options: { draftInput?: boolean; maskEdit?: boolean } = {} ) { const regionLines = regions .map((region) => { @@ -4981,13 +4981,14 @@ function buildScaleAdjustProductLockBackgroundPrompt( "Full-frame background repair for product-locked scale adjustment.", "Return exactly one clean realistic full image with the same framing, camera, aspect ratio, and canvas extent as input image 1. Do not crop, zoom, pan, or resize the whole photo.", options.maskEdit - ? "Input image 1 is the original clean full photo. The supplied edit mask marks the old selected furniture/product footprint that must be repaired as natural background; do not edit outside the masked area." + ? options.draftInput + ? "Input image 1 is a full-photo repair draft from the original photo. The old selected furniture/product footprint has been roughly wiped and the locked product may already appear near its final target footprint. The supplied edit mask marks the rough/old footprint area that must be repaired into natural background and seamless integration; do not edit outside the masked area." + : "Input image 1 is the original clean full photo. The supplied edit mask marks the old selected furniture/product footprint that must be repaired as natural background; do not edit outside the masked area." : "Input image 1 is the original clean full photo. Input image 2 is a background plate reference that removes the selected furniture/product footprint and keeps the room/person/background layout stable.", - options.maskEdit && options.backgroundReference - ? "Input image 2 is a rough background-only repair reference from the same photo. Use it only to understand that the old selected product must disappear and to infer nearby wall, curtain, floor, rug, shadow, and lighting continuity. Do not copy any flat wash, haze, patch edge, mask silhouette, or rough repaired texture from input image 2." - : "", "Important: do not draw, recreate, move, scale, duplicate, or redesign the selected furniture/product in the final image. The system will composite the locked original product pixels into the final target footprint after your background image is returned.", - "Your job is only to erase the old selected furniture/product footprint, complete newly exposed wall, curtain, floor, rug, shadow, sofa edge, and background naturally, and keep the target footprint clean enough for the locked product layer to sit there.", + options.draftInput + ? "Your job is only to turn the rough wiped footprint and any visible draft haze, gray wash, ghost product silhouette, or edge patch into natural wall, curtain, floor, rug, shadow, sofa edge, and background. If the locked product is visible in the draft, keep the surrounding background and contact shadows natural, but do not invent a second product or preserve old-size remnants." + : "Your job is only to erase the old selected furniture/product footprint, complete newly exposed wall, curtain, floor, rug, shadow, sofa edge, and background naturally, and keep the target footprint clean enough for the locked product layer to sit there.", "Preserve all non-target people exactly as foreground/background context: do not move, cover, deform, redraw, change face, change body, change hands, change legs, change clothes, or change pose.", "Keep all non-target content stable: sofa, window, curtain, cabinet, lamp, wall art, floor, rug, lighting, textural grain, and global color must remain consistent.", options.maskEdit @@ -5372,6 +5373,7 @@ export async function POST(request: NextRequest) { scalePercent, { width: sourceWidth, height: sourceHeight }, { + draftInput: useScaleAdjustProductLockMaskEdit, maskEdit: useScaleAdjustProductLockMaskEdit, } ) @@ -5446,7 +5448,8 @@ export async function POST(request: NextRequest) { ? useScaleAdjustProductLockComposite ? useScaleAdjustProductLockMaskEdit ? [ - sourceImageUrl, + scaleAdjustPatchReference.targetDraftFullUrl || + sourceImageUrl, ...scaleAdjustCropReferenceImageUrls, ...manualRegionReferenceImageUrls, ] From 690d04ec712878f466359dfc52c5c8e8bd33ec19 Mon Sep 17 00:00:00 2001 From: 134 Date: Tue, 7 Jul 2026 14:16:09 +0800 Subject: [PATCH 20/70] feat: add downloads for style pro and ai wash results --- components/canvas/AiWashWorkspace.tsx | 53 +++++++++++++++ components/style-pro/StyleProWorkspace.tsx | 76 +++++++++++++++++++++- 2 files changed, 128 insertions(+), 1 deletion(-) diff --git a/components/canvas/AiWashWorkspace.tsx b/components/canvas/AiWashWorkspace.tsx index 5e878afe9..10690b970 100644 --- a/components/canvas/AiWashWorkspace.tsx +++ b/components/canvas/AiWashWorkspace.tsx @@ -3,6 +3,7 @@ import { ChevronDown, Clock3, + Download, History as HistoryIcon, ImageIcon, Loader2, @@ -45,6 +46,7 @@ import { type AiWashControlLevel, type AiWashPreset, } from "@/lib/ai-wash/config"; +import { downloadCanvasImageAsPng } from "@/lib/client/canvas-image-download"; import { buildImageProxyUrl } from "@/lib/image-delivery"; import { cn } from "@/lib/utils"; import { imageGenerationService } from "@/services/canvas"; @@ -368,6 +370,11 @@ function getAiWashStatusLabel(status?: AiWashResultStatus) { } } +function buildAiWashDownloadName(result: AiWashResultCard) { + const createdAt = result.createdAt || Date.now(); + return `ai-wash-${result.title || result.presetId || "result"}-${createdAt}`; +} + function composePresetPrompt(params: { preset: AiWashPreset; selectedLevelIndexByControl: Record; @@ -456,6 +463,9 @@ export function AiWashWorkspace({ useState>({}); const [historySourceOverride, setHistorySourceOverride] = useState(""); const [isSubmitting, setIsSubmitting] = useState(false); + const [downloadingResultId, setDownloadingResultId] = useState( + null + ); const [isHistoryExpanded, setIsHistoryExpanded] = useState(false); const [lastErrorMessage, setLastErrorMessage] = useState(""); const [comparePositionById, setComparePositionById] = useState< @@ -977,6 +987,30 @@ export function AiWashWorkspace({ }); }, [compareSourceUrl, onApplyResult, selectedResult]); + const handleDownloadSelectedResult = useCallback(async () => { + if (!selectedResult?.imageUrl) { + toast.error("请先选择一张洗图结果"); + return; + } + + const downloadId = selectedResult.presetId || selectedResult.imageUrl; + setDownloadingResultId(downloadId); + try { + await downloadCanvasImageAsPng({ + url: selectedResult.imageUrl, + name: buildAiWashDownloadName(selectedResult), + candidates: [selectedResult.imageUrl], + }); + toast.success("图片已开始下载"); + } catch (error) { + toast.error(error instanceof Error ? error.message : "下载失败"); + } finally { + setDownloadingResultId((current) => + current === downloadId ? null : current + ); + } + }, [selectedResult]); + const handleLoadHistoryBatch = useCallback((entry: AiWashHistoryBatch) => { const loadedPresetId = (entry.items.find((item) => item.isDefault) || entry.items[0])?.presetId || ""; @@ -1430,6 +1464,25 @@ export function AiWashWorkspace({ + {selectedResult?.imageUrl ? ( + + ) : null} + + ) : null} ); } @@ -1021,6 +1055,9 @@ export function StyleProWorkspace({ useState("style"); const [selectedModelId, setSelectedModelId] = useState(""); const [uploading, setUploading] = useState(false); + const [downloadingImageId, setDownloadingImageId] = useState( + null + ); const [generationTaskState, setGenerationTaskState] = useState(null); const generating = @@ -1458,6 +1495,34 @@ export function StyleProWorkspace({ [config] ); + const handleDownloadResult = useCallback( + async (item?: StyleProHistoryItem | null) => { + const target = item || currentResult; + if (!target?.resultImageUrl) { + toast.error("暂无可下载的结果图"); + return; + } + + const downloadId = target.id || target.resultImageUrl; + setDownloadingImageId(downloadId); + try { + await downloadCanvasImageAsPng({ + url: target.resultImageUrl, + name: buildStyleProDownloadName(target), + candidates: [target.resultImageUrl], + }); + toast.success("图片已开始下载"); + } catch (error) { + toast.error(error instanceof Error ? error.message : "下载失败"); + } finally { + setDownloadingImageId((current) => + current === downloadId ? null : current + ); + } + }, + [currentResult] + ); + const disabledReason = !sourceImageUrl ? "请先上传图片" : generationMode === "realistic" && config?.realisticModeAvailable === false @@ -1875,6 +1940,15 @@ export function StyleProWorkspace({ { + void handleDownloadResult(currentResult); + }} taskState={generationTaskState} /> From 57a4840df24af9d29ba76706f87bd1fe4002befa Mon Sep 17 00:00:00 2001 From: 134 Date: Tue, 7 Jul 2026 14:27:48 +0800 Subject: [PATCH 21/70] fix: keep canvas chat messages ordered --- .../canvas/chat-panel/canvas-chat-media.ts | 12 +++-- tests/routes/canvas-chat-helpers.test.ts | 49 +++++++++++++++++++ 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/components/canvas/chat-panel/canvas-chat-media.ts b/components/canvas/chat-panel/canvas-chat-media.ts index 0d022f2d6..88d2e2269 100644 --- a/components/canvas/chat-panel/canvas-chat-media.ts +++ b/components/canvas/chat-panel/canvas-chat-media.ts @@ -381,11 +381,15 @@ export function sortCanvasMessagesByCreatedAt(messages: ChatMessage[]): ChatMess .sort((a, b) => { const messageA = a.message; const messageB = b.message; - const timeA = getMessageTimestamp(messageA) ?? 0; - const timeB = getMessageTimestamp(messageB) ?? 0; - if (timeA !== timeB) return timeA - timeB; + const timeA = getMessageTimestamp(messageA); + const timeB = getMessageTimestamp(messageB); + const hasTimeA = timeA !== null; + const hasTimeB = timeB !== null; - if (messageA.role !== messageB.role) { + if (hasTimeA && hasTimeB && timeA !== timeB) return timeA - timeB; + if (hasTimeA !== hasTimeB) return hasTimeA ? -1 : 1; + + if (hasTimeA && hasTimeB && messageA.role !== messageB.role) { if (messageA.role === "user") return -1; if (messageB.role === "user") return 1; } diff --git a/tests/routes/canvas-chat-helpers.test.ts b/tests/routes/canvas-chat-helpers.test.ts index d51b0caec..6edd96b14 100644 --- a/tests/routes/canvas-chat-helpers.test.ts +++ b/tests/routes/canvas-chat-helpers.test.ts @@ -4,6 +4,7 @@ import { extractAssistantImageUrls, extractVideoUrls, normalizeMediaUrl, + sortCanvasMessagesByCreatedAt, stripMediaMarkup, } from "@/components/canvas/chat-panel/canvas-chat-media"; import { @@ -211,6 +212,54 @@ import type { CanvasAgentContext } from "@/types/canvas/agent-protocol"; import type { ChatMessage } from "@/lib/types"; test.describe("canvas chat pure helpers", () => { + test("keeps optimistic messages without timestamps after persisted history", () => { + const messages = [ + { + id: "optimistic-latest-user", + role: "user", + parts: [{ type: "text", text: "你能为我做什么？列出表给我" }], + }, + { + id: "persisted-first-user", + role: "user", + createdAt: "2026-07-07T01:00:00.000Z", + parts: [{ type: "text", text: "你好" }], + }, + { + id: "persisted-first-assistant", + role: "assistant", + createdAt: "2026-07-07T01:00:01.000Z", + parts: [{ type: "text", text: "你好，我在。" }], + }, + ] as ChatMessage[]; + + expect(sortCanvasMessagesByCreatedAt(messages).map((message) => message.id)).toEqual([ + "persisted-first-user", + "persisted-first-assistant", + "optimistic-latest-user", + ]); + }); + + test("preserves relative order among messages without timestamps", () => { + const messages = [ + { + id: "pending-user", + role: "user", + parts: [{ type: "text", text: "生成一张图" }], + }, + { + id: "pending-assistant", + role: "assistant", + parts: [{ type: "text", text: "分析中" }], + }, + ] as ChatMessage[]; + + expect(sortCanvasMessagesByCreatedAt(messages).map((message) => message.id)).toEqual([ + "pending-user", + "pending-assistant", + ]); + }); + test("formats uploaded spreadsheet rows as safe markdown tables for OpenClaw", () => { const markdown = rowsToOpenClawMarkdownTable([ ["名称", "备注"], From ee7d104734b43865f29a75ec8f68ff833af8a01c Mon Sep 17 00:00:00 2001 From: 134 Date: Tue, 7 Jul 2026 14:38:26 +0800 Subject: [PATCH 22/70] fix: add agent preface before generation --- app/(chat)/api/chat/route.ts | 86 +++++++++++++++++++++++++++ components/canvas/CanvasChatPanel.tsx | 5 +- 2 files changed, 87 insertions(+), 4 deletions(-) diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index 3dd0b80fc..df38dbc43 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -541,6 +541,65 @@ function writeCanvasOperationShadowChunk(params: { ); } +const AGENT_EXECUTION_PREFACE_INTERNAL_PATTERN = + /(?:^|\n)\s*(?:#{1,6}\s*)?(?:需求理解|图片分析|执行计划|输出设置|优化后提示词|使用建议)(?:\s|[:：]|$)|【\s*(?:需求理解|图片分析|执行计划|输出设置|优化后提示词|使用建议)\s*】|提示词|自动优化|模型|提交生成|回写|流程|步骤/; + +function buildAgentExecutionPrefaceText(params: { + friendlyMessage?: string | null; + fallback: string; +}) { + const fallback = String(params.fallback || "").replace(/\s+/g, " ").trim(); + const rawMessage = String(params.friendlyMessage || "") + .replace(//g, "") + .trim(); + + if (!rawMessage || AGENT_EXECUTION_PREFACE_INTERNAL_PATTERN.test(rawMessage)) { + return fallback; + } + + const candidate = rawMessage + .split(/\r?\n/) + .map((line) => line.replace(/^[-*]\s*/, "").trim()) + .find(Boolean); + + if (!candidate || AGENT_EXECUTION_PREFACE_INTERNAL_PATTERN.test(candidate)) { + return fallback; + } + + const firstSentence = candidate.match(/^(.{1,80}?[。！？!?~～])/u)?.[1] || candidate; + const compact = firstSentence.replace(/\s+/g, " ").trim().slice(0, 80); + return /[。！？!?~～]$/.test(compact) ? compact : `${compact}。`; +} + +async function writeAgentExecutionPrefaceMessage(params: { + writer: StreamWriter; + chatId: string; + label: string; + friendlyMessage?: string | null; + fallback: string; +}) { + const text = buildAgentExecutionPrefaceText({ + friendlyMessage: params.friendlyMessage, + fallback: params.fallback, + }); + if (!text) return null; + + const messageId = generateUUID(); + writeAssistantTextChunk(params.writer, text, params.label, messageId); + + try { + await saveAssistantTextOnlyMessage({ + chatId: params.chatId, + messageId, + text, + }); + } catch (error) { + chatApiLogger.warn("[Chat API] Failed to persist agent execution preface:", error); + } + + return messageId; +} + async function replayAssistantMessagesForDuplicateRequest({ chatId, userMessageId, @@ -7715,6 +7774,16 @@ export async function POST(request: Request) { ) { const directImageStream = createUIMessageStream({ execute: async ({ writer }) => { + await writeAgentExecutionPrefaceMessage({ + writer, + chatId: id, + label: "agent:generate_image:preface", + friendlyMessage: smartAutoFriendlyMessage, + fallback: + directAgentToolTargetImageUrls.length > 0 + ? "收到，我会参考你选的图片来生成新画面，稍等一下。" + : "好的，我按你的描述来做一版画面，稍等我处理一下。", + }); writeAgentProgressChunk( writer, buildDirectAgentImageContextProgress({ @@ -7917,6 +7986,16 @@ export async function POST(request: Request) { ) { const directVideoStream = createUIMessageStream({ execute: async ({ writer }) => { + await writeAgentExecutionPrefaceMessage({ + writer, + chatId: id, + label: "agent:generate_video:preface", + friendlyMessage: smartAutoFriendlyMessage, + fallback: + directAgentToolTargetImageUrls.length > 0 + ? "收到，我会参考你选的画面来生成视频，稍等一下。" + : "好的，我按你的需求来做这段视频，稍等我处理一下。", + }); const assistantMessageId = generateUUID(); writeAgentProgressChunk( writer, @@ -8123,6 +8202,13 @@ export async function POST(request: Request) { const directToolLabel = getDirectAgentToolLabel(directAgentToolCapability); const directToolStream = createUIMessageStream({ execute: async ({ writer }) => { + await writeAgentExecutionPrefaceMessage({ + writer, + chatId: id, + label: `agent:${directAgentToolCapability}:preface`, + friendlyMessage: smartAutoFriendlyMessage, + fallback: `好的，我来帮你${directToolLabel}，稍等我处理一下。`, + }); writeAgentProgressChunk( writer, buildDirectAgentImageContextProgress({ diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index a482b60f6..181ea13f9 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -12225,10 +12225,7 @@ export function CanvasChatPanel({ const isAgentTargetClarificationActive = shouldShowAgentTargetLockCard && activeAgentTargetLock?.stage === "clarify"; const shouldShowAgentProgressCard = - activeComposerMode === "agent" && - Boolean(activeAgentProgress) && - activeAgentProgress?.stage !== "completed" && - activeAgentProgress?.stage !== "failed"; + false; const agentTargetLockPreviewItems = useMemo(() => { if (!activeAgentTargetLock) return [] as AgentTargetLockRefPreview[]; const targets = Array.isArray(activeAgentTargetLock.targets) ? activeAgentTargetLock.targets : []; From 15f9e1910abea11a04a7c1338ebeb9a8648428d5 Mon Sep 17 00:00:00 2001 From: 134 Date: Tue, 7 Jul 2026 14:54:58 +0800 Subject: [PATCH 23/70] fix: stabilize canvas agent chat feedback --- components/canvas/CanvasChatPanel.tsx | 72 ++++++++++++-- .../runtime/direct-agent-response-builders.ts | 94 ++++++++++++++----- tests/routes/canvas-chat-helpers.test.ts | 26 +++++ 3 files changed, 162 insertions(+), 30 deletions(-) diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index 181ea13f9..b00a2649e 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -792,6 +792,8 @@ const CANVAS_AGENT_SECTION_TITLES = new Set([ "需求理解", CANVAS_IMAGE_ANALYSIS_TITLE, "执行计划", + "执行结果", + "后续操作", "优化后提示词", "输出设置", ]); @@ -2303,6 +2305,7 @@ export function CanvasChatPanel({ const [analysisCollapseState, setAnalysisCollapseState] = useState>({}); const [typedAssistantSections, setTypedAssistantSections] = useState>({}); const [activeTypingSections, setActiveTypingSections] = useState>({}); + const [agentTypewriterMessageIds, setAgentTypewriterMessageIds] = useState>({}); const [clawdTypewriterMessageIds, setClawdTypewriterMessageIds] = useState>({}); const [insufficientPointsState, setInsufficientPointsState] = useState<{ balance: number; @@ -6466,6 +6469,18 @@ export function CanvasChatPanel({ const normalizedAttachments = Array.isArray(messageData.attachments) ? messageData.attachments : []; + const appendedMessageId = String(messageData.id || "").trim(); + const appendedAssistantText = normalizedParts + .filter( + (part: any) => + part?.type === "text" && typeof part?.text === "string" + ) + .map((part: any) => String(part.text || "")) + .join("\n\n"); + const appendedAssistantVisibleText = normalizeCanvasAgentText( + stripCanvasModelRoutingLines(stripMediaMarkup(appendedAssistantText)) + .replace(//g, "") + ); const emptyNearestUserImageKeys = new Set(); const incomingImageUrls = normalizedParts.flatMap((part: any) => part?.type === "text" && typeof part?.text === "string" @@ -6525,6 +6540,18 @@ export function CanvasChatPanel({ if (!isEventForActiveChat) { return; } + + if ( + messageData.role === "assistant" && + appendedMessageId && + appendedAssistantVisibleText + ) { + setAgentTypewriterMessageIds((previousIds) => + previousIds[appendedMessageId] + ? previousIds + : { ...previousIds, [appendedMessageId]: true } + ); + } setMessages((prevMessages) => { const existingIndex = prevMessages.findIndex( @@ -6993,11 +7020,20 @@ export function CanvasChatPanel({ currentModelIdRef.current = selectedModelId; inFlightChatIdRef.current = activeChatId; inFlightRawChatIdRef.current = rawChatId; + const launchUserMessageId = generateUUID(); + const launchCreatedAt = new Date(); sendMessage({ + id: launchUserMessageId, role: "user", parts, - }); + createdAt: launchCreatedAt, + metadata: { + createdAt: launchCreatedAt.toISOString(), + composerMode: activeComposerModeRef.current || activeComposerMode, + modelId: selectedModelId, + }, + } as any); hasHydratedHistoryRef.current = true; pendingLaunchRef.current = null; @@ -7010,7 +7046,7 @@ export function CanvasChatPanel({ if (editableDiv) { editableDiv.innerHTML = ""; } - }, [activeChatId, rawChatId, selectedModelId, status, sendMessage, clearSelection, launchAutoSendTick]); + }, [activeChatId, rawChatId, selectedModelId, status, sendMessage, clearSelection, launchAutoSendTick, activeComposerMode]); // 当历史消息加载完成后，更新消息列表（支持同一会话的增量刷新） const loadedHistorySignatureRef = useRef(""); @@ -10821,6 +10857,20 @@ export function CanvasChatPanel({ !shouldTreatAsCanvasAgentControlIntent && effectiveComposerMode === "image" && Boolean(requestModel && isCanvasImageModel(requestModel)); + const userMessageId = generateUUID(); + const latestMessageTimestamp = messages.reduce( + (latest, message) => Math.max(latest, getMessageTimestamp(message) ?? 0), + 0 + ); + const optimisticCreatedAt = new Date( + Math.max(Date.now(), latestMessageTimestamp + 1) + ); + const optimisticUserMetadata = { + createdAt: optimisticCreatedAt.toISOString(), + composerMode: effectiveComposerMode, + agentEntryMode: effectiveAgentEntryMode, + modelId: effectiveModelId, + }; // 🆕 隐藏提示词（不在 UI 中展示） const hiddenSegments: string[] = []; @@ -10933,9 +10983,12 @@ export function CanvasChatPanel({ // 使用与 multimodal-input 相同的格式发送消息 try { sendMessage({ + id: userMessageId, role: "user", parts: parts, - }); + createdAt: optimisticCreatedAt, + metadata: optimisticUserMetadata, + } as any); } catch (error) { releaseSendGuard(activeChatId); if (shouldShowOptimisticImageGeneration) { @@ -11019,6 +11072,7 @@ export function CanvasChatPanel({ markSessionClawdSkillTagPromptApplied, buildHistoryMessagesUrl, workflowContext, + messages, ]); const handleOpenClawQuickAction = useCallback( @@ -12953,6 +13007,11 @@ export function CanvasChatPanel({ const shouldTypeClawAssistantMessage = message.role === "assistant" && Boolean(clawdTypewriterMessageIds[message.id]); + const shouldTypeAgentAssistantMessage = + message.role === "assistant" && + Boolean(agentTypewriterMessageIds[message.id]); + const shouldForceTypeAssistantMessage = + shouldTypeClawAssistantMessage || shouldTypeAgentAssistantMessage; const isClawAssistantMessage = message.role === "assistant" && (isClawdMode || isClawdChatMessage(message)); @@ -13161,16 +13220,16 @@ export function CanvasChatPanel({ !typedAssistantSections[analysisTypingKey] && ((shouldTypeAssistantMessage && !hasCanvasRenderableMarkdown(analysisSection?.analysis || "")) || - shouldTypeClawAssistantMessage); + shouldForceTypeAssistantMessage); const shouldAnimatePlainText = !analysisSection && Boolean(plainText) && !typedAssistantSections[plainTextTypingKey] && ((shouldTypeAssistantMessage && !hasCanvasRenderableMarkdown(plainText)) || - shouldTypeClawAssistantMessage); + shouldForceTypeAssistantMessage); const shouldDeferAssistantMedia = - isClawAssistantMessage && + (isClawAssistantMessage || shouldTypeAgentAssistantMessage) && (shouldAnimateAnalysis || shouldAnimatePlainText); const visibleImageUrls = shouldDeferAssistantMedia ? [] @@ -13182,6 +13241,7 @@ export function CanvasChatPanel({ !isDetailGenerationMode && !isClawAssistantMessage && visibleImageUrls.length > 0 && + !plainText.trim() && !plainText.includes(CANVAS_IMAGE_DONE_SUMMARY) && !isLatestAssistantDuringGeneration; const renderPlainText = (text: string, key: string) => ( diff --git a/lib/ai/runtime/direct-agent-response-builders.ts b/lib/ai/runtime/direct-agent-response-builders.ts index c1b349c86..0576f56d1 100644 --- a/lib/ai/runtime/direct-agent-response-builders.ts +++ b/lib/ai/runtime/direct-agent-response-builders.ts @@ -10,6 +10,56 @@ import { type DirectAgentToolCapability, } from "@/lib/ai/runtime/direct-agent-execution"; +function normalizeDirectAgentResultSentence(value: string) { + const normalized = String(value || "") + .replace(/结果如下[：:]?\s*$/u, "") + .replace(/[，,。:：]\s*$/u, "") + .trim(); + return normalized || "已完成处理"; +} + +function buildDirectAgentResultCard(params: { + result: string; + mediaMarkdown?: string | null; + followUp: string; +}) { + return [ + "### 执行结果", + `- ${normalizeDirectAgentResultSentence(params.result)}，结果如下。`, + params.mediaMarkdown, + "### 后续操作", + `- ${params.followUp}`, + ] + .filter(Boolean) + .join("\n\n"); +} + +function getDirectAgentFollowUpMessage(capability: DirectAgentToolCapability) { + switch (capability) { + case "remove_background": + return "如需微调边缘、替换背景或继续合成，可以直接告诉我。"; + case "remove_watermark": + return "如需继续修补局部痕迹或统一画面质感，可以直接告诉我。"; + case "enhance": + return "如需继续调整清晰度、色彩或细节强度，可以直接告诉我。"; + case "resize": + return "如需继续改尺寸、比例或输出规格，可以直接告诉我。"; + case "extend": + return "如需继续扩展画面、调整构图或补充边缘内容，可以直接告诉我。"; + case "angle_variant": + return "如需继续换角度、换机位或统一系列视角，可以直接告诉我。"; + case "local_edit": + case "box_cutout": + return "如需继续修改局部、替换元素或细化边缘，可以直接告诉我。"; + case "generate_video": + return "如需继续调整镜头、节奏、比例或时长，可以直接告诉我。"; + case "generate_image": + case "generic_edit": + default: + return "如需继续调整风格、构图、主体细节或尺寸，可以直接告诉我。"; + } +} + export function buildDirectAgentAnalysisAssistantText(params: { summary: string; answer?: string | null; @@ -32,12 +82,11 @@ export function buildDirectAgentImageGenerationAssistantText(params: { modelDisplayName?: string | null; }) { return prependImageModelBadge( - [ - getDirectAgentToolSuccessMessage("generate_image"), - params.imageMarkdown, - ] - .filter(Boolean) - .join("\n\n"), + buildDirectAgentResultCard({ + result: getDirectAgentToolSuccessMessage("generate_image"), + mediaMarkdown: params.imageMarkdown, + followUp: getDirectAgentFollowUpMessage("generate_image"), + }), params.modelDisplayName, ); } @@ -47,12 +96,11 @@ export function buildDirectAgentVideoGenerationAssistantText(params: { modelDisplayName?: string | null; }) { return prependImageModelBadge( - [ - getDirectAgentToolSuccessMessage("generate_video"), - buildAgentToolVideoMarkdown(params.videoUrl), - ] - .filter(Boolean) - .join("\n\n"), + buildDirectAgentResultCard({ + result: getDirectAgentToolSuccessMessage("generate_video"), + mediaMarkdown: buildAgentToolVideoMarkdown(params.videoUrl), + followUp: getDirectAgentFollowUpMessage("generate_video"), + }), params.modelDisplayName, ); } @@ -64,16 +112,15 @@ export function buildDirectAgentImageEditAssistantText(params: { imageSize?: string; }) { return prependImageModelBadge( - [ - getDirectAgentToolSuccessMessage(params.capability), - buildAgentToolImageMarkdown({ + buildDirectAgentResultCard({ + result: getDirectAgentToolSuccessMessage(params.capability), + mediaMarkdown: buildAgentToolImageMarkdown({ originalUrl: params.originalUrl, thumbnailUrl: params.thumbnailUrl, imageSize: params.imageSize, }), - ] - .filter(Boolean) - .join("\n\n"), + followUp: getDirectAgentFollowUpMessage(params.capability), + }), getDirectAgentToolLabel(params.capability), ); } @@ -87,12 +134,11 @@ export function buildDirectAgentBatchImageEditAssistantText(params: { }>; }) { return prependImageModelBadge( - [ - getDirectAgentToolSuccessMessage(params.capability), - buildAgentToolImagesMarkdown(params.results), - ] - .filter(Boolean) - .join("\n\n"), + buildDirectAgentResultCard({ + result: getDirectAgentToolSuccessMessage(params.capability), + mediaMarkdown: buildAgentToolImagesMarkdown(params.results), + followUp: getDirectAgentFollowUpMessage(params.capability), + }), getDirectAgentToolLabel(params.capability), ); } diff --git a/tests/routes/canvas-chat-helpers.test.ts b/tests/routes/canvas-chat-helpers.test.ts index 6edd96b14..c4d257b53 100644 --- a/tests/routes/canvas-chat-helpers.test.ts +++ b/tests/routes/canvas-chat-helpers.test.ts @@ -131,6 +131,10 @@ import { resolveDirectAgentToolExecution, shouldRunDirectAgentBatchImageEdit, } from "@/lib/ai/runtime/direct-agent-execution"; +import { + buildDirectAgentImageEditAssistantText, + buildDirectAgentImageGenerationAssistantText, +} from "@/lib/ai/runtime/direct-agent-response-builders"; import { inferAngleEditOptionsFromText } from "@/lib/ai/runtime/angle-edit-options"; import { hasDirectAgentLocalEditIntent, @@ -1179,6 +1183,28 @@ test.describe("canvas chat pure helpers", () => { ); }); + test("formats direct agent image results as structured markdown cards", () => { + const generatedText = buildDirectAgentImageGenerationAssistantText({ + imageMarkdown: "![生成结果](https://cdn.example.com/generated.png)", + modelDisplayName: "Test Image Model", + }); + + expect(generatedText).toContain("### 执行结果"); + expect(generatedText).toContain("- 已完成图片生成，结果如下。"); + expect(generatedText).toContain("![生成结果](https://cdn.example.com/generated.png)"); + expect(generatedText).toContain("### 后续操作"); + + const editText = buildDirectAgentImageEditAssistantText({ + capability: "remove_background", + originalUrl: "https://cdn.example.com/cutout.png", + }); + + expect(editText).toContain("### 执行结果"); + expect(editText).toContain("- 已完成抠图处理，结果如下。"); + expect(editText).toContain("https://cdn.example.com/cutout.png"); + expect(editText).toContain("替换背景"); + }); + test("only runs direct image edit batches for explicit safe batch requests", () => { expect( shouldRunDirectAgentBatchImageEdit({ From 610d2aed9b35ce8b9c9c6b3630f877b7a6dc37b2 Mon Sep 17 00:00:00 2001 From: 134 Date: Tue, 7 Jul 2026 15:17:43 +0800 Subject: [PATCH 24/70] feat: add canvas agent orchestration kernel --- app/(chat)/api/chat/route.ts | 14 +- components/canvas/CanvasChatPanel.tsx | 28 ++-- lib/ai/context-memory/target-resolution.ts | 53 +++++-- lib/ai/runtime/canvas-agent-kernel.ts | 156 +++++++++++++++++++++ lib/ai/runtime/intent-arbiter.ts | 28 ++-- tests/routes/canvas-chat-helpers.test.ts | 94 +++++++++++++ 6 files changed, 338 insertions(+), 35 deletions(-) create mode 100644 lib/ai/runtime/canvas-agent-kernel.ts diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index df38dbc43..c3be7a608 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -262,6 +262,7 @@ import { recordProviderHealthSuccess, shouldAvoidProvider, } from "@/lib/ai/runtime/provider-health"; +import { resolveCanvasAgentRequestPolicy } from "@/lib/ai/runtime/canvas-agent-kernel"; import { arbitrateAgentIntent } from "@/lib/ai/runtime/intent-arbiter"; import { buildAgentRoleIdentityInstruction } from "@/lib/ai/runtime/agent-role-profile"; import { getVideoStoryboardConfig } from "@/lib/video-agent/config"; @@ -1095,8 +1096,19 @@ export async function POST(request: Request) { const shouldRespectDirectComposerMode = composerMode === "image" || composerMode === "video" || composerMode === "detail"; const shouldIsolateDirectComposerContext = shouldRespectDirectComposerMode; - let shouldRunAgentConversation = finalModelId === "auto"; + const requestPolicy = resolveCanvasAgentRequestPolicy({ + composerMode, + agentEntryMode: agentPresetId ? "preset" : "auto", + selectedChatModel: finalModelId, + hasCanvasControlIntentFastPath, + }); + finalModelId = requestPolicy.effectiveSelectedChatModel; + let shouldRunAgentConversation = + finalModelId === "auto" || requestPolicy.shouldForceToolOrchestration; let shouldKeepSelectedModel = false; + if (requestPolicy.reason === "agent_auto_composer") { + chatApiLogger.info("[Smart Auto Mode] 🧭 Agent composer forces tool orchestration, ignoring stale selected model"); + } if (shouldRespectDirectComposerMode && shouldRunAgentConversation) { shouldRunAgentConversation = false; chatApiLogger.info(`[Smart Auto Mode] ⛳ Direct composer mode (${composerMode}) disables initial auto-agent routing`); diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index b00a2649e..0c1439f39 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -77,6 +77,7 @@ import { getImageCreditCostFromConfig, getImageModelCreditDiscountLabel, } from "@/lib/ai/image-model-pricing"; +import { resolveCanvasAgentRequestPolicy } from "@/lib/ai/runtime/canvas-agent-kernel"; import { getVideoToolOptionsForMode } from "@/lib/ai/video-tool-capability"; import type { ChatMessage } from "@/lib/types"; import type { @@ -5352,14 +5353,8 @@ export function CanvasChatPanel({ } if (mode === "agent") { - const preferredAgentModelId = - allModels.find( - (model) => - model.id === lastConversationModelId && - isCanvasConversationOnlyModel(model) - )?.id || "auto"; - currentModelIdRef.current = preferredAgentModelId; - setSelectedModelId(preferredAgentModelId); + currentModelIdRef.current = "auto"; + setSelectedModelId("auto"); setIsDetailGenerationMode(false); return targetChatSessionId; } @@ -6160,7 +6155,15 @@ export function CanvasChatPanel({ const hiddenPrompt = hiddenPromptRef.current || undefined; const autoContext = autoContextRef.current || undefined; const contextPayload = capturedContextPayload; - const requestModel = allModels.find((model) => model.id === currentModelIdRef.current); + const requestPolicy = resolveCanvasAgentRequestPolicy({ + composerMode: effectiveComposerMode, + agentEntryMode: effectiveAgentEntryMode, + selectedChatModel: currentModelIdRef.current, + }); + const effectiveSelectedChatModel = requestPolicy.effectiveSelectedChatModel; + const requestModel = allModels.find( + (model) => model.id === effectiveSelectedChatModel + ); const requestIsVideoModel = isCanvasVideoModel(requestModel); const requestIsConversationModel = isCanvasConversationOnlyModel(requestModel); hiddenPromptRef.current = null; @@ -6177,10 +6180,13 @@ export function CanvasChatPanel({ selectedAgentPreset?.id ? selectedAgentPreset.id : undefined, - selectedChatModel: currentModelIdRef.current, + selectedChatModel: effectiveSelectedChatModel, composerMode: effectiveComposerMode, selectedVisibilityType: "private", - deepThinkingEnabled: requestIsConversationModel + deepThinkingEnabled: + effectiveComposerMode === "agent" + ? undefined + : requestIsConversationModel ? false : requestIsVideoModel || currentModelIdRef.current === "auto" ? undefined diff --git a/lib/ai/context-memory/target-resolution.ts b/lib/ai/context-memory/target-resolution.ts index e0d8f8064..6c69dd7ef 100644 --- a/lib/ai/context-memory/target-resolution.ts +++ b/lib/ai/context-memory/target-resolution.ts @@ -5,16 +5,17 @@ import type { ResolvedImageRef, ResolvedTargetAssets, } from "@/lib/ai/context-memory/types"; +import { + hasCanvasAgentActionableImageEditIntent, + hasCanvasAgentDeicticTarget, + resolveCanvasAgentTargetPreference, +} from "@/lib/ai/runtime/canvas-agent-kernel"; const ORIGINAL_IMAGE_REGEX = /(原图|上传图|参考图|最初那张|第一张图|素材图|源图|原始图片|原始那张|我发的那张|上传的那张|原来的图)/i; const LATEST_RESULT_REGEX = /(上一张|上一轮|刚生成|最新生成|这个结果|该结果|按这个结果|继续在这张|基于这张结果|延续上一次结果|上一版|刚才那张结果|生成的图|生成图|结果图|出图结果|刚出的图|上一张生成图|上一轮结果图|现在这张生成图)/i; -const DEICTIC_TARGET_REGEX = - /(这张图|这只|这个|这款|这套|当前这张|当前这个|当前这只|这一张|这一只|这个产品|这个物体|这只猫|这只狗)/i; const SELECTION_REGEX = /(选中的|当前这张|我选的|这几张|这些图|选中图)/i; -const ACTIONABLE_EDIT_REGEX = - /(改|调整|优化|换|替换|去掉|抠|重绘|增强|修|处理|按照|按|保留|统一|色差|背景|清晰|风格|亮一点|暗一点|白底|透明底)/i; const ACTIONABLE_GENERATION_REGEX = /(生图|生成|画一张|出图|来一张|再来一张|做一张|做图|做视频|视频生成|生成视频|动起来)/i; const IMPLICIT_HISTORY_TARGET_REGEX = @@ -255,6 +256,7 @@ export function resolveTargetAssets( const focusPolicy = params.contextPayload?.focusPolicy || "auto"; const workingStateGeneratedUrls = normalizeStringArray(workingState?.lastGeneratedImageUrls); const workingStateUploadedUrls = normalizeStringArray(workingState?.lastUploadedImageUrls); + const workingStateResolvedUrls = normalizeStringArray(workingState?.lastResolvedTargetImageUrls); const generatedRefs = workingStateGeneratedUrls.length > 0 ? toWorkingMemoryRefs(workingStateGeneratedUrls) : assistantRefs; const originalRefs = @@ -384,7 +386,39 @@ export function resolveTargetAssets( }); } - if (DEICTIC_TARGET_REGEX.test(text)) { + const canvasAgentTargetPreference = resolveCanvasAgentTargetPreference({ + text, + selectedImageCount: selectedRefs.length, + latestResultImageCount: generatedRefs.length, + workingMemoryTargetImageCount: workingStateResolvedUrls.length, + }); + if (canvasAgentTargetPreference.shouldPreferLatestResult) { + return buildResolvedTargetAssets({ + targetRefs: generatedRefs, + candidateRefs, + resolutionMode: + workingStateGeneratedUrls.length > 0 ? "working_memory" : "latest_result", + confidence: canvasAgentTargetPreference.confidence, + needsClarification: false, + reason: "用户使用指代表达并提出图片编辑意图，默认续接最近生成结果", + fromHistory: true, + }); + } + + if (canvasAgentTargetPreference.shouldPreferWorkingMemoryTarget) { + const refs = toWorkingMemoryRefs(workingStateResolvedUrls); + return buildResolvedTargetAssets({ + targetRefs: refs, + candidateRefs, + resolutionMode: "working_memory", + confidence: canvasAgentTargetPreference.confidence, + needsClarification: false, + reason: "用户使用指代表达并提出图片编辑意图，使用工作记忆中的上一轮目标图", + fromHistory: true, + }); + } + + if (hasCanvasAgentDeicticTarget(text)) { if (selectedRefs.length > 0) { return buildResolvedTargetAssets({ targetRefs: selectedRefs, @@ -421,7 +455,7 @@ export function resolveTargetAssets( } if ( - ACTIONABLE_EDIT_REGEX.test(text) && + hasCanvasAgentActionableImageEditIntent(text) && currentRefs.length === 0 && generatedRefs.length > 0 && selectedLooksLikeOriginal && @@ -463,14 +497,13 @@ export function resolveTargetAssets( }); } - const workingStateResolvedUrls = normalizeStringArray(workingState?.lastResolvedTargetImageUrls); const allowsImplicitHistoryTarget = IMPLICIT_HISTORY_TARGET_REGEX.test(text) && - (ACTIONABLE_EDIT_REGEX.test(text) || + (hasCanvasAgentActionableImageEditIntent(text) || ACTIONABLE_GENERATION_REGEX.test(text) || LATEST_RESULT_REGEX.test(text) || ORIGINAL_IMAGE_REGEX.test(text) || - DEICTIC_TARGET_REGEX.test(text) || + hasCanvasAgentDeicticTarget(text) || SELECTION_REGEX.test(text)); if (allowsImplicitHistoryTarget) { @@ -525,7 +558,7 @@ export function resolveTargetAssets( } } - if (ACTIONABLE_EDIT_REGEX.test(text)) { + if (hasCanvasAgentActionableImageEditIntent(text)) { return buildResolvedTargetAssets({ targetRefs: [], candidateRefs, diff --git a/lib/ai/runtime/canvas-agent-kernel.ts b/lib/ai/runtime/canvas-agent-kernel.ts new file mode 100644 index 000000000..68467384f --- /dev/null +++ b/lib/ai/runtime/canvas-agent-kernel.ts @@ -0,0 +1,156 @@ +import type { DirectAgentToolCapability } from "@/lib/ai/runtime/direct-agent-execution"; + +export type CanvasAgentComposerMode = "agent" | "image" | "video" | "detail" | "clawd"; +export type CanvasAgentEntryMode = "auto" | "preset"; + +export interface CanvasAgentRequestPolicy { + effectiveSelectedChatModel: string; + shouldForceToolOrchestration: boolean; + reason: string | null; +} + +export interface CanvasAgentTargetPreference { + shouldPreferLatestResult: boolean; + shouldPreferWorkingMemoryTarget: boolean; + confidence: number; + reason: string | null; +} + +const DEICTIC_TARGET_REGEX = + /(这张图|这只|这个|这款|这套|当前这张|当前这个|当前这只|这一张|这一只|这个产品|这个物体|这只猫|这只狗)/i; + +const ACTIONABLE_IMAGE_EDIT_REGEX = + /(抠图|去背景|透明背景|去背|扣图|移除背景|去水印|水印|去logo|去\s*logo|移除logo|去除logo|换背景|改背景|背景换成|背景改成|白底|纯色背景|替换.*背景|变清晰|高清|锐化|降噪|提升质量|修复|清晰化|扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|多视角|换视角|改视角|视角变体|新角度|换角度|改角度|去除|消除|擦掉|删除|移除|p掉|重绘|风格化|卡通化|动漫风|油画风|调色|调整颜色|亮度|对比度|饱和度|缩放|放大|缩小|调整尺寸|裁剪|resize|crop|修图|p图|改图|调图|编辑.*图|优化.*图|重做|再调整|继续改|继续调|处理)/i; + +export function normalizeCanvasAgentRequestText(text: string) { + return String(text || "").toLowerCase().replace(/\s+/g, "").trim(); +} + +export function hasCanvasAgentDeicticTarget(text: string) { + return DEICTIC_TARGET_REGEX.test(String(text || "")); +} + +export function hasCanvasAgentActionableImageEditIntent(text: string) { + return ACTIONABLE_IMAGE_EDIT_REGEX.test(String(text || "")); +} + +export function inferCanvasAgentImageEditCapability( + text: string, +): DirectAgentToolCapability | null { + const normalizedText = normalizeCanvasAgentRequestText(text); + if (!normalizedText) return null; + if (/去水印|水印|去logo|移除logo|去除logo/.test(normalizedText)) { + return "remove_watermark"; + } + if (/抠图|去背景|透明背景|去背|扣图|移除背景/.test(normalizedText)) { + return "remove_background"; + } + if (/多视角|换视角|改视角|视角变体|新角度|换角度|改角度|侧面|侧视|背面|背后|俯视|俯拍|仰视|仰拍|低机位|高机位/.test(normalizedText)) { + return "angle_variant"; + } + if (/扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|比例|画幅|竖版|横版|竖图|横图|9:16|16:9|1:1/.test(normalizedText)) { + return "extend"; + } + if (/调整尺寸|裁剪|resize|crop|缩放|放大|缩小/.test(normalizedText)) { + return "resize"; + } + if (/局部|框选|选区|区域|涂抹|蒙版/.test(normalizedText)) { + return "local_edit"; + } + if (/高清|清晰|锐化|降噪|提升质量|修复|清晰化/.test(normalizedText)) { + return "enhance"; + } + if (hasCanvasAgentActionableImageEditIntent(text)) { + return "generic_edit"; + } + return null; +} + +export function resolveCanvasAgentRequestPolicy(params: { + composerMode?: string | null; + agentEntryMode?: string | null; + selectedChatModel?: string | null; + hasCanvasControlIntentFastPath?: boolean; +}): CanvasAgentRequestPolicy { + const composerMode = String(params.composerMode || "").trim(); + const agentEntryMode = String(params.agentEntryMode || "").trim(); + const selectedChatModel = String(params.selectedChatModel || "").trim() || "auto"; + const isAgentAutoComposer = composerMode === "agent" && agentEntryMode !== "preset"; + const shouldForceToolOrchestration = + isAgentAutoComposer || Boolean(params.hasCanvasControlIntentFastPath); + + if (isAgentAutoComposer) { + return { + effectiveSelectedChatModel: "auto", + shouldForceToolOrchestration: true, + reason: "agent_auto_composer", + }; + } + + if (params.hasCanvasControlIntentFastPath) { + return { + effectiveSelectedChatModel: selectedChatModel, + shouldForceToolOrchestration: true, + reason: "canvas_control_fast_path", + }; + } + + return { + effectiveSelectedChatModel: selectedChatModel, + shouldForceToolOrchestration, + reason: null, + }; +} + +export function resolveCanvasAgentTargetPreference(params: { + text: string; + selectedImageCount: number; + latestResultImageCount: number; + workingMemoryTargetImageCount: number; +}): CanvasAgentTargetPreference { + const hasDeicticTarget = hasCanvasAgentDeicticTarget(params.text); + const hasEditIntent = hasCanvasAgentActionableImageEditIntent(params.text); + const selectedImageCount = Math.max(0, Math.round(params.selectedImageCount || 0)); + const latestResultImageCount = Math.max( + 0, + Math.round(params.latestResultImageCount || 0), + ); + const workingMemoryTargetImageCount = Math.max( + 0, + Math.round(params.workingMemoryTargetImageCount || 0), + ); + + if (!hasDeicticTarget || !hasEditIntent || selectedImageCount > 0) { + return { + shouldPreferLatestResult: false, + shouldPreferWorkingMemoryTarget: false, + confidence: 0, + reason: null, + }; + } + + if (latestResultImageCount > 0) { + return { + shouldPreferLatestResult: true, + shouldPreferWorkingMemoryTarget: false, + confidence: 90, + reason: "deictic_edit_prefers_latest_result", + }; + } + + if (workingMemoryTargetImageCount > 0) { + return { + shouldPreferLatestResult: false, + shouldPreferWorkingMemoryTarget: true, + confidence: 86, + reason: "deictic_edit_uses_working_memory_target", + }; + } + + return { + shouldPreferLatestResult: false, + shouldPreferWorkingMemoryTarget: false, + confidence: 0, + reason: null, + }; +} diff --git a/lib/ai/runtime/intent-arbiter.ts b/lib/ai/runtime/intent-arbiter.ts index cfd463d3b..ce1081ef1 100644 --- a/lib/ai/runtime/intent-arbiter.ts +++ b/lib/ai/runtime/intent-arbiter.ts @@ -5,6 +5,10 @@ import type { ResolvedTargetAssets } from "@/lib/ai/context-memory/types"; import type { AnalysisResult } from "@/lib/ai/smart-analyzer"; import type { TaskBoundaryDecision } from "@/lib/ai/runtime/task-boundary-classifier"; import { hasDirectAgentLocalEditIntent } from "@/lib/ai/runtime/local-edit-options"; +import { + hasCanvasAgentActionableImageEditIntent, + inferCanvasAgentImageEditCapability, +} from "@/lib/ai/runtime/canvas-agent-kernel"; export type IntentArbiterMode = | "respond" @@ -116,23 +120,21 @@ function isExplicitImageGenerationRequest(text: string): boolean { } function isLikelyImageEditRequest(text: string): boolean { - const normalizedText = String(text || "").toLowerCase(); - return hasDirectAgentLocalEditIntent(text) || /抠图|去背景|透明背景|去背|扣图|移除背景|去水印|水印|去logo|去\s*logo|移除logo|去除logo|换背景|改背景|背景换成|背景改成|白底|纯色背景|替换.*背景|变清晰|高清|锐化|降噪|提升质量|修复|清晰化|扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|换成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|改成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|比例|画幅|竖版|横版|多视角|换视角|改视角|视角变体|新角度|换角度|改角度|侧面|侧视|背面|背后|俯视|俯拍|仰视|仰拍|低机位|高机位|去除|消除|擦掉|删除|移除|p掉|重绘|风格化|卡通化|动漫风|油画风|转.*风格|变.*风格|改.*风格|调色|调整颜色|亮度|对比度|饱和度|色调|曝光|滤镜|改成.*色|换成.*色|变成.*色|颜色改成|颜色换成|颜色变成|缩放|放大|缩小|调整尺寸|裁剪|resize|crop|修图|p图|改图|调图|编辑.*图|优化.*图|重做|再调整|继续改|继续调/.test( - normalizedText, - ); + return hasDirectAgentLocalEditIntent(text) || hasCanvasAgentActionableImageEditIntent(text); } function inferDefaultImageEditSubIntent(text: string): string { - const lowerText = String(text || "").toLowerCase(); - if (/去水印|水印|去logo|去\s*logo|移除logo|去除logo/.test(lowerText)) return "remove_watermark"; - if (/抠图|去背景|透明背景|去背|扣图|移除背景/.test(lowerText)) return "remove_background"; - if (/换背景|改背景|背景换成|背景改成|白底|透明底/.test(lowerText)) return "replace_background"; - if (/多视角|换视角|改视角|视角变体|新角度|换角度|改角度|侧面|侧视|背面|背后|俯视|俯拍|仰视|仰拍|低机位|高机位/.test(lowerText)) return "angle_variant"; + const inferredCapability = inferCanvasAgentImageEditCapability(text); + if ( + inferredCapability && + inferredCapability !== "generic_edit" && + inferredCapability !== "generate_image" && + inferredCapability !== "generate_video" && + inferredCapability !== "analyze" + ) { + return inferredCapability; + } if (hasDirectAgentLocalEditIntent(text)) return "local_edit"; - if (/扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|比例|画幅|竖版|横版|竖图|横图|9:16|16:9|1:1/.test(lowerText)) return "extend"; - if (/风格|卡通|动漫|油画|像素风|写实|插画/.test(lowerText)) return "style_transfer"; - if (/颜色|调色|亮度|对比度|饱和度|改成.*色|换成.*色/.test(lowerText)) return "color_adjust"; - if (/去除|消除|擦掉|删除|移除|p掉|重绘/.test(lowerText)) return "inpaint"; return "enhance"; } diff --git a/tests/routes/canvas-chat-helpers.test.ts b/tests/routes/canvas-chat-helpers.test.ts index c4d257b53..1ed8f6a5a 100644 --- a/tests/routes/canvas-chat-helpers.test.ts +++ b/tests/routes/canvas-chat-helpers.test.ts @@ -45,6 +45,10 @@ import { } from "@/lib/canvas-toolbar-config"; import { shouldBypassCanvasAgentMissingTargetClarification } from "@/lib/ai/runtime/canvas-agent-clarification"; import { hasCanvasAgentRoutingContext } from "@/lib/ai/canvas-agent-routing"; +import { + resolveCanvasAgentRequestPolicy, + resolveCanvasAgentTargetPreference, +} from "@/lib/ai/runtime/canvas-agent-kernel"; import { inferAgentImageCountFromText } from "@/lib/ai/runtime/image-count-intent"; import { isAgentModeScopedChatId, @@ -128,6 +132,7 @@ import { getDirectAgentToolSuccessMessage, hasExplicitDirectAgentBatchImageEditIntent, isDirectAgentBatchImageEditCapability, + resolveDirectAgentToolContext, resolveDirectAgentToolExecution, shouldRunDirectAgentBatchImageEdit, } from "@/lib/ai/runtime/direct-agent-execution"; @@ -1205,6 +1210,95 @@ test.describe("canvas chat pure helpers", () => { expect(editText).toContain("替换背景"); }); + test("resolves deictic canvas edit requests to latest results and direct tools", () => { + const latestResultUrl = "https://cdn.example.com/latest-cat-result.png"; + const requestPolicy = resolveCanvasAgentRequestPolicy({ + composerMode: "agent", + selectedChatModel: "conversation-model-id", + }); + + expect(requestPolicy.effectiveSelectedChatModel).toBe("auto"); + expect(requestPolicy.shouldForceToolOrchestration).toBe(true); + + const targetPreference = resolveCanvasAgentTargetPreference({ + text: "将这只猫从图中扣图扣出来", + selectedImageCount: 0, + latestResultImageCount: 1, + workingMemoryTargetImageCount: 1, + }); + + expect(targetPreference.shouldPreferLatestResult).toBe(true); + + const mediaInput = { + imageRefs: [], + currentMessageImageRefs: [], + selectedImageRefs: [], + canvasImageRefs: [], + historyUserImageRefs: [], + historyAssistantImageRefs: [ + { + url: latestResultUrl, + normalizedUrl: latestResultUrl, + source: "history_assistant_result", + role: "history_result", + }, + ], + resolvedImageRefs: [], + currentMessageImageUrls: [], + selectedImageUrls: [], + historyUserImageUrls: [], + historyAssistantImageUrls: [latestResultUrl], + resolvedImageUrls: [latestResultUrl], + selectedAssetIds: [], + candidateAssetIds: [], + uploadedAssetIds: [], + hasImage: true, + source: "history", + fromHistory: true, + } as any; + const workingState = { + lastGeneratedImageUrls: [latestResultUrl], + lastResolvedTargetImageUrls: [latestResultUrl], + }; + const targetResolution = { + targetRefs: [], + candidateRefs: [], + targetImageUrls: [latestResultUrl], + targetAssetIds: [], + candidateImageUrls: [], + candidateAssetIds: [], + resolutionMode: "working_memory", + confidence: targetPreference.confidence, + needsClarification: false, + reason: targetPreference.reason, + fromHistory: true, + } as any; + + const directContext = resolveDirectAgentToolContext({ + execution: { + mode: "edit_image", + routeIntent: "image_edit", + routeSubIntent: "remove_background", + targetSource: targetResolution.resolutionMode, + preferredCapability: "remove_background", + shouldUseStructuredReply: true, + shouldOptimizePrompt: true, + shouldApplyImagePipeline: true, + shouldApplyVideoPipeline: false, + shouldStayConversational: false, + shouldPublishToCanvas: false, + toolCalls: [], + availableTools: ["removeBackground"], + } as any, + requestResolvedMediaInput: mediaInput, + requestTargetResolution: targetResolution, + requestWorkingState: workingState, + }); + + expect(directContext?.execution.toolName).toBe("removeBackground"); + expect(directContext?.targetImageUrl).toBe(latestResultUrl); + }); + test("only runs direct image edit batches for explicit safe batch requests", () => { expect( shouldRunDirectAgentBatchImageEdit({ From f88ada378e79a29dedb1c17b40a72ea491f1bbfb Mon Sep 17 00:00:00 2001 From: 134 Date: Tue, 7 Jul 2026 15:42:54 +0800 Subject: [PATCH 25/70] feat: add canvas agent progress timeline --- components/canvas/CanvasChatPanel.tsx | 165 ++++++++++++++---- .../canvas/chat-panel/canvas-agent-stream.ts | 111 +++++++++++- tests/routes/canvas-chat-helpers.test.ts | 40 +++++ 3 files changed, 282 insertions(+), 34 deletions(-) diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index 0c1439f39..d5e82e3d0 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -144,15 +144,16 @@ import { type CanvasToolbarItemKey, } from "@/lib/canvas-toolbar-config"; import { - AGENT_PROGRESS_STEPS, type AgentProgressStreamPayload, type AgentTargetLockRefPreview, type AgentTargetLockStreamPayload, + buildAgentProgressNodeSteps, type CanvasOperationStreamPayload, type CanvasViewOperationStreamPayload, type CanvasMediaStreamPayload, filterCanvasMediaPayloadForUnhandledItems, getAgentProgressFallbackTitle, + getAgentProgressStepLabel, getAgentTargetLockTitle, getAgentTargetResolutionLabel, getCanvasMediaPayloadHandledItemIds, @@ -1623,6 +1624,130 @@ function CanvasAssistantTypingPlaceholder({ hint }: { hint?: string | null }) { ); } +function CanvasAgentProgressTimeline({ + progress, +}: { + progress?: AgentProgressStreamPayload | null; +}) { + const steps = buildAgentProgressNodeSteps(progress); + const title = progress?.title?.trim() || getAgentProgressFallbackTitle(progress); + const isFailed = progress?.stage === "failed"; + const isCompleted = progress?.stage === "completed"; + const isClarify = progress?.stage === "clarify"; + + return ( +
+
+
+ + {isFailed ? ( + + ) : isCompleted ? ( + + ) : ( + + )} + + + {title} + +
+ +
+
+ {steps.map((step, index) => { + const isStepCompleted = step.state === "completed"; + const isStepCurrent = step.state === "current"; + const isStepFailed = step.state === "failed"; + const lineIsActive = + isStepCompleted || isStepCurrent || isStepFailed; + + return ( +
+
+ + {isStepCompleted ? ( + + ) : isStepFailed ? ( + + ) : isStepCurrent ? ( + + ) : ( + + )} + {isStepCurrent ? ( + + ) : null} + + + {step.label} + +
+ {index < steps.length - 1 ? ( + + ) : null} +
+ ); + })} +
+
+
+
+ ); +} + function CanvasComposerUploadProgressDialog({ progress, }: { @@ -12318,26 +12443,10 @@ export function CanvasChatPanel({ activeAgentProgress?.title?.trim() || getAgentProgressFallbackTitle(activeAgentProgress); const agentProgressMessage = activeAgentProgress?.message?.trim() || "我正在根据当前上下文持续推进这一步。"; - const agentProgressSteps = useMemo(() => { - const currentStepIndex = Math.max( - 0, - AGENT_PROGRESS_STEPS.findIndex((step) => step.key === activeAgentProgress?.currentStep) - ); - return AGENT_PROGRESS_STEPS.map((step, index) => { - let state: "upcoming" | "current" | "completed" | "failed" = "upcoming"; - if (activeAgentProgress?.stage === "failed") { - state = index < currentStepIndex ? "completed" : index === currentStepIndex ? "failed" : "upcoming"; - } else if (activeAgentProgress?.stage === "completed") { - state = "completed"; - } else { - state = index < currentStepIndex ? "completed" : index === currentStepIndex ? "current" : "upcoming"; - } - return { - ...step, - state, - }; - }); - }, [activeAgentProgress]); + const agentProgressSteps = useMemo( + () => buildAgentProgressNodeSteps(activeAgentProgress), + [activeAgentProgress] + ); const renderModelSelectContent = () => ( -
- - 分析中 -
- + )}
@@ -14660,9 +14764,10 @@ export function CanvasChatPanel({ {agentProgressTitle} - {AGENT_PROGRESS_STEPS.find( - (step) => step.key === activeAgentProgress.currentStep - )?.label || "处理中"} + {getAgentProgressStepLabel( + activeAgentProgress.currentStep || "lock_target", + activeAgentProgress + )}
diff --git a/components/canvas/chat-panel/canvas-agent-stream.ts b/components/canvas/chat-panel/canvas-agent-stream.ts index 3e2480894..e7a6246a2 100644 --- a/components/canvas/chat-panel/canvas-agent-stream.ts +++ b/components/canvas/chat-panel/canvas-agent-stream.ts @@ -206,14 +206,26 @@ export type AgentProgressStreamPayload = { capability?: string; }; +export type AgentProgressStepState = + | "upcoming" + | "current" + | "completed" + | "failed"; + +export type AgentProgressNodeStep = { + key: NonNullable; + label: string; + state: AgentProgressStepState; +}; + export const AGENT_PROGRESS_STEPS: Array<{ key: NonNullable; label: string; }> = [ - { key: "lock_target", label: "整理素材" }, - { key: "analyze", label: "理解需求" }, - { key: "select_tool", label: "优化提示" }, - { key: "execute", label: "提交生成" }, + { key: "lock_target", label: "理解用户需求" }, + { key: "analyze", label: "分析使用工具" }, + { key: "select_tool", label: "确认执行工具" }, + { key: "execute", label: "执行中" }, { key: "finalize", label: "整理结果" }, ]; @@ -260,3 +272,94 @@ export function getAgentProgressFallbackTitle(payload?: AgentProgressStreamPaylo return "正在理解需求"; } } + +export function getAgentProgressActionLabel( + payload?: AgentProgressStreamPayload | null +) { + switch (payload?.capability) { + case "analyze": + return "图片分析"; + case "generic_edit": + return "修图"; + case "remove_background": + return "抠图"; + case "remove_watermark": + return "去水印"; + case "enhance": + return "图像增强"; + case "resize": + return "尺寸调整"; + case "extend": + return "扩图"; + case "angle_variant": + return "多视角"; + case "local_edit": + return "局部编辑"; + case "box_cutout": + return "框选抠图"; + case "generate_image": + return "生图"; + case "generate_video": + return "视频生成"; + default: + return "工具"; + } +} + +export function getAgentProgressStepLabel( + stepKey: NonNullable, + payload?: AgentProgressStreamPayload | null +) { + const actionLabel = getAgentProgressActionLabel(payload); + switch (stepKey) { + case "lock_target": + return "理解用户需求"; + case "analyze": + return payload?.capability === "analyze" ? "分析图片" : "分析使用工具"; + case "select_tool": + return actionLabel === "工具" ? "确认执行工具" : `确认${actionLabel}工具`; + case "execute": + return actionLabel === "工具" ? "执行中" : `${actionLabel}中`; + case "finalize": + return "整理结果"; + default: + return "处理中"; + } +} + +export function buildAgentProgressNodeSteps( + payload?: AgentProgressStreamPayload | null +): AgentProgressNodeStep[] { + const currentStep = payload?.currentStep || "lock_target"; + const currentStepIndex = Math.max( + 0, + AGENT_PROGRESS_STEPS.findIndex((step) => step.key === currentStep) + ); + + return AGENT_PROGRESS_STEPS.map((step, index) => { + let state: AgentProgressStepState = "upcoming"; + if (payload?.stage === "failed") { + state = + index < currentStepIndex + ? "completed" + : index === currentStepIndex + ? "failed" + : "upcoming"; + } else if (payload?.stage === "completed") { + state = "completed"; + } else { + state = + index < currentStepIndex + ? "completed" + : index === currentStepIndex + ? "current" + : "upcoming"; + } + + return { + key: step.key, + label: getAgentProgressStepLabel(step.key, payload), + state, + }; + }); +} diff --git a/tests/routes/canvas-chat-helpers.test.ts b/tests/routes/canvas-chat-helpers.test.ts index 1ed8f6a5a..2504c581c 100644 --- a/tests/routes/canvas-chat-helpers.test.ts +++ b/tests/routes/canvas-chat-helpers.test.ts @@ -9,8 +9,10 @@ import { } from "@/components/canvas/chat-panel/canvas-chat-media"; import { filterCanvasMediaPayloadForUnhandledItems, + buildAgentProgressNodeSteps, getCanvasMediaPayloadHandledItemIds, getCanvasOperationHandledMediaItemIds, + getAgentProgressStepLabel, isCanvasAgentViewOperationClientEnabled, normalizeCanvasAgentClientBoolean, shouldRetryCanvasViewOperationAfterRefresh, @@ -1210,6 +1212,44 @@ test.describe("canvas chat pure helpers", () => { expect(editText).toContain("替换背景"); }); + test("builds capability-aware agent progress node labels", () => { + const selectingSteps = buildAgentProgressNodeSteps({ + stage: "planning", + currentStep: "select_tool", + capability: "remove_background", + }); + + expect(selectingSteps.map((step) => step.label)).toEqual([ + "理解用户需求", + "分析使用工具", + "确认抠图工具", + "抠图中", + "整理结果", + ]); + expect(selectingSteps.map((step) => step.state)).toEqual([ + "completed", + "completed", + "current", + "upcoming", + "upcoming", + ]); + + const executingSteps = buildAgentProgressNodeSteps({ + stage: "executing", + currentStep: "execute", + capability: "remove_background", + }); + + expect(executingSteps[2]?.state).toBe("completed"); + expect(executingSteps[3]).toMatchObject({ + label: "抠图中", + state: "current", + }); + expect(getAgentProgressStepLabel("execute", { capability: "generate_video" })).toBe( + "视频生成中" + ); + }); + test("resolves deictic canvas edit requests to latest results and direct tools", () => { const latestResultUrl = "https://cdn.example.com/latest-cat-result.png"; const requestPolicy = resolveCanvasAgentRequestPolicy({ From 4b9f3a25c9b1aa428e6f2d13da8252e6de80cf5e Mon Sep 17 00:00:00 2001 From: 134 Date: Tue, 7 Jul 2026 16:02:33 +0800 Subject: [PATCH 26/70] fix: resolve unselected canvas image targets --- components/canvas/CanvasChatPanel.tsx | 22 ++++++-- lib/ai/context-memory/media-input.ts | 1 + lib/ai/context-memory/target-resolution.ts | 27 ++++++++++ lib/ai/context-memory/types.ts | 1 + lib/ai/runtime/direct-agent-execution.ts | 1 + lib/canvas/local-upload-state.ts | 9 +++- tests/routes/canvas-chat-helpers.test.ts | 60 ++++++++++++++++++++++ 7 files changed, 115 insertions(+), 6 deletions(-) diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index d5e82e3d0..cdd55e1e2 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -857,6 +857,18 @@ function isInternalCanvasImageGenerationFailureText(parts: Array) { ); } +function getCanvasImageNodeUrl(node: CanvasNode | null | undefined) { + if (!node || node.type !== "image") return ""; + const data = (node.data || {}) as Record; + return String( + data.src || + data.previewImage || + data.imageUrl || + data.renderArtifacts?.screenshotUrl || + "" + ).trim(); +} + function findCanvasImageAnchorNode( nodes: CanvasNode[], params: { @@ -873,7 +885,7 @@ function findCanvasImageAnchorNode( if (sourceKeys.size > 0) { const matchedNode = nodes.find((node) => { if (node.type !== "image") return false; - const src = String((node.data as any)?.src || "").trim(); + const src = getCanvasImageNodeUrl(node); return sourceKeys.has(normalizeImageIdentityUrl(src)); }); if (matchedNode) return matchedNode; @@ -7871,7 +7883,7 @@ export function CanvasChatPanel({ pendingTarget: pendingCanvasAgentAnnotationEditTargetRef.current, isEligibleNode: (node) => node.type === "image" && - isValidChatHttpUrl(String((node.data as any)?.src || "").trim()), + isValidChatHttpUrl(getCanvasImageNodeUrl(node)), }); if ( pendingCanvasAgentAnnotationEditTargetRef.current && @@ -7888,7 +7900,7 @@ export function CanvasChatPanel({ const imageNodes = nodes.filter( (node) => node.type === "image" && - isValidChatHttpUrl(String((node.data as any)?.src || "").trim()), + isValidChatHttpUrl(getCanvasImageNodeUrl(node)), ); const effectiveComposerMode = activeComposerModeRef.current || activeComposerMode; @@ -7923,7 +7935,7 @@ export function CanvasChatPanel({ new Map( [ ...selectedImageNodes.filter((node) => - isValidChatHttpUrl(String((node.data as any)?.src || "").trim()) + isValidChatHttpUrl(getCanvasImageNodeUrl(node)) ), ...(pendingAnnotationTargetNode ? [pendingAnnotationTargetNode] : []), ].map((node) => [node.id, node]) @@ -8201,7 +8213,7 @@ export function CanvasChatPanel({ assetId: isValidChatUuid((node.data as any)?.assetId) ? String((node.data as any).assetId).trim() : undefined, - url: String((node.data as any)?.src || "").trim(), + url: getCanvasImageNodeUrl(node), title: typeof (node.data as any)?.title === "string" ? String((node.data as any).title).slice(0, 256) diff --git a/lib/ai/context-memory/media-input.ts b/lib/ai/context-memory/media-input.ts index 6d06e1697..228c24391 100644 --- a/lib/ai/context-memory/media-input.ts +++ b/lib/ai/context-memory/media-input.ts @@ -308,6 +308,7 @@ export function resolveChatMediaInput( resolvedImageRefs, currentMessageImageUrls: currentMessageImageRefs.map((ref) => ref.url), selectedImageUrls: selectedImageRefs.map((ref) => ref.url), + canvasImageUrls: canvasImageRefs.map((ref) => ref.url), historyUserImageUrls: history.latestUser.map((ref) => ref.url), historyAssistantImageUrls: history.latestAssistant.map((ref) => ref.url), resolvedImageUrls: resolvedImageRefs.map((ref) => ref.url), diff --git a/lib/ai/context-memory/target-resolution.ts b/lib/ai/context-memory/target-resolution.ts index 6c69dd7ef..fb253758b 100644 --- a/lib/ai/context-memory/target-resolution.ts +++ b/lib/ai/context-memory/target-resolution.ts @@ -497,6 +497,21 @@ export function resolveTargetAssets( }); } + if ( + canvasRefs.length === 1 && + (hasCanvasAgentActionableImageEditIntent(text) || focusPolicy === "auto") + ) { + return buildResolvedTargetAssets({ + targetRefs: [canvasRefs[0]], + candidateRefs, + resolutionMode: "text_reference", + confidence: 84, + needsClarification: false, + reason: "用户使用画布/图片指代表达，且当前画布仅有一张可处理图片", + fromHistory: true, + }); + } + const allowsImplicitHistoryTarget = IMPLICIT_HISTORY_TARGET_REGEX.test(text) && (hasCanvasAgentActionableImageEditIntent(text) || @@ -559,6 +574,18 @@ export function resolveTargetAssets( } if (hasCanvasAgentActionableImageEditIntent(text)) { + if (canvasRefs.length === 1) { + return buildResolvedTargetAssets({ + targetRefs: [canvasRefs[0]], + candidateRefs, + resolutionMode: "text_reference", + confidence: 82, + needsClarification: false, + reason: "用户表达了图片编辑意图，且当前画布仅有一张可处理图片", + fromHistory: true, + }); + } + return buildResolvedTargetAssets({ targetRefs: [], candidateRefs, diff --git a/lib/ai/context-memory/types.ts b/lib/ai/context-memory/types.ts index d183534fe..bc6cafdb4 100644 --- a/lib/ai/context-memory/types.ts +++ b/lib/ai/context-memory/types.ts @@ -57,6 +57,7 @@ export interface ResolvedChatMediaInput { resolvedImageRefs: ResolvedImageRef[]; currentMessageImageUrls: string[]; selectedImageUrls: string[]; + canvasImageUrls: string[]; historyUserImageUrls: string[]; historyAssistantImageUrls: string[]; resolvedImageUrls: string[]; diff --git a/lib/ai/runtime/direct-agent-execution.ts b/lib/ai/runtime/direct-agent-execution.ts index bd507cb55..79a6f6642 100644 --- a/lib/ai/runtime/direct-agent-execution.ts +++ b/lib/ai/runtime/direct-agent-execution.ts @@ -653,6 +653,7 @@ export function pickAgentSourceImageUrls(params: { params.requestTargetResolution?.targetImageUrls || [], params.requestResolvedMediaInput?.selectedImageUrls || [], params.requestResolvedMediaInput?.currentMessageImageUrls || [], + params.requestResolvedMediaInput?.canvasImageUrls || [], params.requestWorkingState?.lastResolvedTargetImageUrls || [], params.requestWorkingState?.lastGeneratedImageUrls || [], params.requestResolvedMediaInput?.historyAssistantImageUrls || [], diff --git a/lib/canvas/local-upload-state.ts b/lib/canvas/local-upload-state.ts index 7845f4766..0f1cf4b6c 100644 --- a/lib/canvas/local-upload-state.ts +++ b/lib/canvas/local-upload-state.ts @@ -29,6 +29,13 @@ export function isCanvasImageReadyForAi(node: CanvasNode | null | undefined) { return false; } - const src = String(((node.data as Record)?.src || "")).trim(); + const data = (node.data || {}) as Record; + const src = String( + data.src || + data.previewImage || + data.imageUrl || + data.renderArtifacts?.screenshotUrl || + "" + ).trim(); return /^https?:\/\//i.test(src); } diff --git a/tests/routes/canvas-chat-helpers.test.ts b/tests/routes/canvas-chat-helpers.test.ts index 2504c581c..1888fa834 100644 --- a/tests/routes/canvas-chat-helpers.test.ts +++ b/tests/routes/canvas-chat-helpers.test.ts @@ -1339,6 +1339,66 @@ test.describe("canvas chat pure helpers", () => { expect(directContext?.targetImageUrl).toBe(latestResultUrl); }); + test("falls back to canvas images for direct agent edit tools", () => { + const canvasImageUrl = "https://cdn.example.com/canvas-cat.png"; + const mediaInput = { + imageRefs: [], + currentMessageImageRefs: [], + selectedImageRefs: [], + canvasImageRefs: [ + { + url: canvasImageUrl, + normalizedUrl: canvasImageUrl, + nodeId: "cat-node", + source: "canvas_context", + role: "reference", + x: 120, + y: 80, + width: 640, + height: 640, + isSelected: false, + }, + ], + historyUserImageRefs: [], + historyAssistantImageRefs: [], + resolvedImageRefs: [], + currentMessageImageUrls: [], + selectedImageUrls: [], + canvasImageUrls: [canvasImageUrl], + historyUserImageUrls: [], + historyAssistantImageUrls: [], + resolvedImageUrls: [], + selectedAssetIds: [], + candidateAssetIds: [], + uploadedAssetIds: [], + hasImage: false, + source: "none", + fromHistory: false, + } as any; + + const directContext = resolveDirectAgentToolContext({ + execution: { + mode: "edit_image", + routeIntent: "image_edit", + routeSubIntent: "remove_background", + targetSource: "text_reference", + preferredCapability: "remove_background", + shouldUseStructuredReply: true, + shouldOptimizePrompt: true, + shouldApplyImagePipeline: true, + shouldApplyVideoPipeline: false, + shouldStayConversational: false, + shouldPublishToCanvas: false, + toolCalls: [], + availableTools: ["removeBackground"], + } as any, + requestResolvedMediaInput: mediaInput, + }); + + expect(directContext?.execution.toolName).toBe("removeBackground"); + expect(directContext?.targetImageUrl).toBe(canvasImageUrl); + }); + test("only runs direct image edit batches for explicit safe batch requests", () => { expect( shouldRunDirectAgentBatchImageEdit({ From 8c159a3828fa922934bd33b5e7d59d7a35318427 Mon Sep 17 00:00:00 2001 From: 134 Date: Tue, 7 Jul 2026 16:30:03 +0800 Subject: [PATCH 27/70] fix: remove agent progress bubble and add dialogue memory --- app/(chat)/api/chat/route.ts | 91 ++++++++++---- components/canvas/CanvasChatPanel.tsx | 128 -------------------- lib/ai/runtime/recent-dialogue-memory.ts | 146 +++++++++++++++++++++++ tests/routes/canvas-chat-helpers.test.ts | 56 +++++++++ 4 files changed, 268 insertions(+), 153 deletions(-) create mode 100644 lib/ai/runtime/recent-dialogue-memory.ts diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index c3be7a608..5b901905b 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -250,6 +250,7 @@ import { pruneTaskBoundaryContext, pruneTaskBoundaryHistoryMessages, } from "@/lib/ai/runtime/task-boundary-context-pruner"; +import { buildRecentDialogueMemory } from "@/lib/ai/runtime/recent-dialogue-memory"; import { resolveAgentModelSlotDescriptor, upsertAgentModelSlot, @@ -1085,6 +1086,10 @@ export async function POST(request: Request) { let confirmedPromptForConversationState: string | undefined; let smartAutoHistoryMessagesFromDb: DBMessage[] = []; let smartAutoMediaHistoryMessagesFromDb: DBMessage[] = []; + let recentDialogueMemory = buildRecentDialogueMemory({ + messages: [], + currentText: requestTextContent, + }); let preferredAgentModelId: string | undefined; let configuredAgentConversationModelId: string | undefined; let agentModelSlots: AgentModelSlots = {}; @@ -1489,6 +1494,17 @@ export async function POST(request: Request) { taskType: taskBoundaryDecision.taskType, taskMode: taskBoundaryDecision.taskMode, }); + recentDialogueMemory = buildRecentDialogueMemory({ + messages: loadedHistoryMessages, + currentText: smartAutoInputText, + maxEntries: taskBoundaryDecision.taskType === "chat" ? 6 : 4, + maxTotalChars: taskBoundaryDecision.taskType === "chat" ? 1000 : 700, + }); + chatApiLogger.info("[RecentDialogueMemory] Smart auto recall:", { + entries: recentDialogueMemory.entries.length, + isRecallRequest: recentDialogueMemory.isRecallRequest, + taskType: taskBoundaryDecision.taskType, + }); autoTextForFallback = smartAutoInputText; if (!requestWorkingState && !shouldIsolateDirectComposerContext) { @@ -1563,23 +1579,12 @@ export async function POST(request: Request) { resolutionConfidence: requestTargetResolution.confidence, }); - // 🆕 从历史消息中提取对话文本（用于上下文理解） - const historyMessages: { role: "user" | "assistant"; content: string }[] = []; - for (const msg of smartAutoHistoryMessagesFromDb.slice(-10)) { // 最多取最近10条 - const parts = msg.parts as any[]; - if (parts && parts.length > 0) { - // 提取文本内容 - const textParts = parts.filter((p: any) => p.type === 'text'); - const content = textParts.map((p: any) => p.text || '').join(' ').trim(); - if (content) { - historyMessages.push({ - role: msg.role as "user" | "assistant", - content: content.substring(0, 300), // 限制长度 - }); - } - } - } - chatApiLogger.info("[Smart Auto Mode] History context:", historyMessages.length, "messages"); + const historyMessages: { role: "user" | "assistant"; content: string }[] = + recentDialogueMemory.entries.map((entry) => ({ + role: entry.role, + content: entry.content, + })); + chatApiLogger.info("[Smart Auto Mode] History context:", historyMessages.length, "memory entries"); // 2. 使用 LangGraph 驱动的自动模式工作流 const { runAutoAgentGraph } = await import("@/lib/ai/auto-agent-graph"); @@ -1612,8 +1617,15 @@ export async function POST(request: Request) { workflowState: composedConversationState.threadState.workflowState, memoryHints: composedConversationState.threadState.memoryHints, contextSummary: composedConversationState.threadState.contextSummary, + recentDialogueMemory: recentDialogueMemory.contextSummary, }, }); + const agentGraphContext = [ + composedConversationState.composedConversationMemory.contextSummary, + recentDialogueMemory.contextSummary, + ] + .filter(Boolean) + .join("\n"); const autoGraphResult = await withOperationTimeout( runAutoAgentGraph({ text: smartAutoInputText, @@ -1641,10 +1653,7 @@ export async function POST(request: Request) { requestResolvedMediaInput.historyUserImageUrls.length, currentUploadImageCount: requestResolvedMediaInput.currentMessageImageUrls.length, historyMessages, - context: - composedConversationState.composedConversationMemory.contextSummary || - effectiveAutoContextText || - undefined, + context: agentGraphContext || effectiveAutoContextText || undefined, preferredModelId: preferredAgentModelId, threadState: composedConversationState.threadState, }), @@ -2747,6 +2756,21 @@ export async function POST(request: Request) { purpose: "media", hasCurrentImageInput: hasImageInputForConversationAction, }); + recentDialogueMemory = shouldIsolateDirectComposerContext + ? recentDialogueMemory + : buildRecentDialogueMemory({ + messages: loadedMessagesForBoundary, + currentText: requestTextContent, + maxEntries: taskBoundaryDecision.taskType === "chat" ? 6 : 4, + maxTotalChars: taskBoundaryDecision.taskType === "chat" ? 1000 : 700, + }); + if (recentDialogueMemory.entries.length > 0) { + chatApiLogger.info("[RecentDialogueMemory] Request recall:", { + entries: recentDialogueMemory.entries.length, + isRecallRequest: recentDialogueMemory.isRecallRequest, + taskType: taskBoundaryDecision.taskType, + }); + } if (!requestWorkingState && !shouldIsolateDirectComposerContext) { requestWorkingState = await getConversationWorkingStateByChatId({ @@ -8591,16 +8615,33 @@ export async function POST(request: Request) { resolvedRuntimeModelLookupId = actualModelId; } - const effectiveConversationSystemPrompt = shouldRunAgentConversation - ? `${await buildAgentRoleIdentityInstruction({ + const recentDialogueSystemContext = recentDialogueMemory.contextSummary + ? [ + recentDialogueMemory.contextSummary, + ...recentDialogueMemory.memoryHints, + ].join("\n") + : ""; + const baseConversationSystemPrompt = systemPrompt({ + selectedChatModel: finalModelId, + requestHints, + }); + const agentIdentityInstruction = shouldRunAgentConversation + ? await buildAgentRoleIdentityInstruction({ allowVideoGeneration: allowAgentVideoGeneration, hasCanvasContext: Boolean( effectiveContextPayload?.source === "canvas" || effectiveContextPayload?.selectedNodeIds?.length || effectiveContextPayload?.canvasImageNodes?.length, ), - })}\n\n${systemPrompt({ selectedChatModel: finalModelId, requestHints })}` - : systemPrompt({ selectedChatModel: finalModelId, requestHints }); + }) + : ""; + const effectiveConversationSystemPrompt = [ + agentIdentityInstruction, + recentDialogueSystemContext, + baseConversationSystemPrompt, + ] + .filter(Boolean) + .join("\n\n"); const activeRuntimeModelDescriptor = await resolveAgentModelSlotDescriptor( resolvedRuntimeModelLookupId, diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index cdd55e1e2..ac5ffeb94 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -1636,130 +1636,6 @@ function CanvasAssistantTypingPlaceholder({ hint }: { hint?: string | null }) { ); } -function CanvasAgentProgressTimeline({ - progress, -}: { - progress?: AgentProgressStreamPayload | null; -}) { - const steps = buildAgentProgressNodeSteps(progress); - const title = progress?.title?.trim() || getAgentProgressFallbackTitle(progress); - const isFailed = progress?.stage === "failed"; - const isCompleted = progress?.stage === "completed"; - const isClarify = progress?.stage === "clarify"; - - return ( -
-
-
- - {isFailed ? ( - - ) : isCompleted ? ( - - ) : ( - - )} - - - {title} - -
- -
-
- {steps.map((step, index) => { - const isStepCompleted = step.state === "completed"; - const isStepCurrent = step.state === "current"; - const isStepFailed = step.state === "failed"; - const lineIsActive = - isStepCompleted || isStepCurrent || isStepFailed; - - return ( -
-
- - {isStepCompleted ? ( - - ) : isStepFailed ? ( - - ) : isStepCurrent ? ( - - ) : ( - - )} - {isStepCurrent ? ( - - ) : null} - - - {step.label} - -
- {index < steps.length - 1 ? ( - - ) : null} -
- ); - })} -
-
-
-
- ); -} - function CanvasComposerUploadProgressDialog({ progress, }: { @@ -13672,10 +13548,6 @@ export function CanvasChatPanel({ ) : null} - {isLoading && !shouldRenderImageLoading && ( - - )} -
diff --git a/lib/ai/runtime/recent-dialogue-memory.ts b/lib/ai/runtime/recent-dialogue-memory.ts new file mode 100644 index 000000000..fa8dbd113 --- /dev/null +++ b/lib/ai/runtime/recent-dialogue-memory.ts @@ -0,0 +1,146 @@ +import type { DBMessage } from "@/lib/db/schema"; + +export interface RecentDialogueMemoryEntry { + role: "user" | "assistant"; + content: string; + messageId?: string; +} + +export interface RecentDialogueMemory { + entries: RecentDialogueMemoryEntry[]; + contextSummary: string; + memoryHints: string[]; + isRecallRequest: boolean; +} + +const USER_RECALL_PATTERN = + /(?:我|用户)?(?:刚才|刚刚|前面|之前|上面|上一(?:句|条|轮)|最近).{0,12}(?:说|发|问|讲|提|输入|发送).{0,12}(?:什么|啥|哪句|哪些|内容)|(?:我说了什么|我刚才说啥|我前面说啥|我之前说啥|我发了什么|我问了什么|记得我.{0,12}说|没有记忆)/i; + +const ASSISTANT_RECALL_PATTERN = + /(?:你|助手|ai|AI).{0,12}(?:刚才|刚刚|前面|之前|上面|上一(?:句|条|轮)|最近).{0,12}(?:说|回复|回答|讲).{0,12}(?:什么|啥|哪句|哪些|内容)/i; + +function normalizeWhitespace(value: string) { + return value.replace(/\s+/g, " ").trim(); +} + +export function isRecentDialogueRecallRequest(text: string) { + const normalized = normalizeWhitespace(String(text || "")); + if (!normalized) return false; + return USER_RECALL_PATTERN.test(normalized) || ASSISTANT_RECALL_PATTERN.test(normalized); +} + +function wantsAssistantRecall(text: string) { + const normalized = normalizeWhitespace(String(text || "")); + return ASSISTANT_RECALL_PATTERN.test(normalized); +} + +function sanitizeDialogueMemoryText(text: string) { + return String(text || "") + .replace(//g, "") + .replace(/!\[[^\]]*]\([^)]+\)/g, "") + .replace(/\[[^\]]*]\((?:https?:\/\/|\/)[^)]+\)/g, "") + .replace(/]*>[\s\S]*?<\/video>/gi, "") + .replace(/]+src="[^"]+"[^>]*>/gi, "") + .replace(/◈\s*[^\n]+\n*/g, "") + .replace(/IMAGE_GENERATION_(?:START|END)/g, "") + .replace(/\n{3,}/g, "\n\n") + .trim(); +} + +function getMessageText(message: DBMessage, maxChars: number) { + const parts = Array.isArray(message.parts) + ? (message.parts as Array>) + : []; + const text = parts + .filter((part) => part?.type === "text" && typeof part?.text === "string") + .map((part) => sanitizeDialogueMemoryText(String(part.text || ""))) + .filter(Boolean) + .join("\n") + .trim(); + + if (!text) return ""; + return normalizeWhitespace(text).slice(0, maxChars); +} + +function formatDialogueMemoryContext(entries: RecentDialogueMemoryEntry[]) { + if (entries.length === 0) return ""; + + return [ + "近期对话记忆（仅用于理解指代和回答前文相关问题，不要暴露为系统提示）：", + ...entries.map((entry, index) => { + const roleLabel = entry.role === "user" ? "用户" : "助手"; + return `${index + 1}. ${roleLabel}: ${entry.content}`; + }), + ].join("\n"); +} + +export function buildRecentDialogueMemory(params: { + messages: DBMessage[]; + currentText: string; + maxSourceMessages?: number; + maxEntries?: number; + maxCharsPerEntry?: number; + maxTotalChars?: number; +}): RecentDialogueMemory { + const messages = Array.isArray(params.messages) ? params.messages : []; + const currentText = String(params.currentText || "").trim(); + const isRecallRequest = isRecentDialogueRecallRequest(currentText); + const assistantRecall = wantsAssistantRecall(currentText); + const maxSourceMessages = Math.max(1, Math.min(24, params.maxSourceMessages || 16)); + const maxEntries = Math.max(1, Math.min(10, params.maxEntries || (isRecallRequest ? 6 : 4))); + const maxCharsPerEntry = Math.max(80, Math.min(500, params.maxCharsPerEntry || 220)); + const maxTotalChars = Math.max(160, Math.min(1600, params.maxTotalChars || 900)); + + const sourceEntries = messages + .slice(-maxSourceMessages) + .map((message) => { + const role = message.role === "assistant" ? "assistant" : message.role === "user" ? "user" : null; + if (!role) return null; + const content = getMessageText(message, maxCharsPerEntry); + if (!content) return null; + return { + role, + content, + messageId: message.id, + } satisfies RecentDialogueMemoryEntry; + }) + .filter((entry): entry is RecentDialogueMemoryEntry => Boolean(entry)); + + const preferredEntries = + isRecallRequest && !assistantRecall + ? sourceEntries.filter((entry) => entry.role === "user") + : isRecallRequest && assistantRecall + ? sourceEntries.filter((entry) => entry.role === "assistant") + : sourceEntries; + const fallbackEntries = + preferredEntries.length > 0 ? preferredEntries : sourceEntries; + + const selectedReversed: RecentDialogueMemoryEntry[] = []; + let totalChars = 0; + for (let index = fallbackEntries.length - 1; index >= 0; index -= 1) { + const entry = fallbackEntries[index]; + if (!entry) continue; + if (selectedReversed.length >= maxEntries) break; + if (totalChars + entry.content.length > maxTotalChars && selectedReversed.length > 0) { + continue; + } + selectedReversed.push(entry); + totalChars += entry.content.length; + } + + const entries = selectedReversed.reverse(); + const memoryHints = entries.length + ? [ + isRecallRequest + ? "用户正在询问前文或记忆，应优先根据近期对话记忆回答，不要声称没有上下文。" + : "近期对话记忆只用于理解指代和延续语境，不要逐字复述，除非用户明确询问前文。", + ] + : []; + + return { + entries, + contextSummary: formatDialogueMemoryContext(entries), + memoryHints, + isRecallRequest, + }; +} diff --git a/tests/routes/canvas-chat-helpers.test.ts b/tests/routes/canvas-chat-helpers.test.ts index 1888fa834..6b0027ab4 100644 --- a/tests/routes/canvas-chat-helpers.test.ts +++ b/tests/routes/canvas-chat-helpers.test.ts @@ -52,6 +52,10 @@ import { resolveCanvasAgentTargetPreference, } from "@/lib/ai/runtime/canvas-agent-kernel"; import { inferAgentImageCountFromText } from "@/lib/ai/runtime/image-count-intent"; +import { + buildRecentDialogueMemory, + isRecentDialogueRecallRequest, +} from "@/lib/ai/runtime/recent-dialogue-memory"; import { isAgentModeScopedChatId, isScopedEmptyHistoryRequest, @@ -221,6 +225,19 @@ import { import type { PickedObjectAttachment } from "@/components/canvas/chat-panel/canvas-chat-session"; import type { CanvasAgentContext } from "@/types/canvas/agent-protocol"; import type { ChatMessage } from "@/lib/types"; +import type { DBMessage } from "@/lib/db/schema"; + +function dbTextMessage( + id: string, + role: "user" | "assistant", + text: string, +): DBMessage { + return { + id, + role, + parts: [{ type: "text", text }], + } as unknown as DBMessage; +} test.describe("canvas chat pure helpers", () => { test("keeps optimistic messages without timestamps after persisted history", () => { @@ -309,6 +326,45 @@ test.describe("canvas chat pure helpers", () => { expect(upstream).toContain("| SKU | 销量 |"); }); + test("builds bounded recent dialogue memory for user recall requests", () => { + const memory = buildRecentDialogueMemory({ + messages: [ + dbTextMessage("m1", "user", "你好"), + dbTextMessage("m2", "assistant", "你好，我在。"), + dbTextMessage("m3", "user", "你是一个小可爱"), + dbTextMessage("m4", "assistant", "嘿嘿，谢谢你的夸奖。"), + ], + currentText: "我前面说了什么？", + }); + + expect(isRecentDialogueRecallRequest("我前面说了什么？")).toBe(true); + expect(memory.isRecallRequest).toBe(true); + expect(memory.entries.map((entry) => entry.role)).toEqual(["user", "user"]); + expect(memory.entries.at(-1)?.content).toBe("你是一个小可爱"); + expect(memory.contextSummary).toContain("用户: 你是一个小可爱"); + expect(memory.memoryHints.join(" ")).toContain("不要声称没有上下文"); + }); + + test("sanitizes media markup from recent dialogue memory", () => { + const memory = buildRecentDialogueMemory({ + messages: [ + dbTextMessage("m1", "user", "生成一只小猫"), + dbTextMessage( + "m2", + "assistant", + "\n![原图](https://example.com/cat.png)\n已完成本次生成" + ), + ], + currentText: "你刚才回复了什么？", + }); + + expect(memory.entries).toHaveLength(1); + expect(memory.entries[0].role).toBe("assistant"); + expect(memory.entries[0].content).toBe("已完成本次生成"); + expect(memory.contextSummary).not.toContain("https://example.com"); + expect(memory.contextSummary).not.toContain("IMAGE_GENERATION_START"); + }); + test("infers explicit agent image counts from user text", () => { expect(inferAgentImageCountFromText("再出三版")).toBe(3); expect(inferAgentImageCountFromText("基于这张图再来 3 张")).toBe(3); From 8c719c44acf04d8f91040767f9a82f328d503957 Mon Sep 17 00:00:00 2001 From: 134 Date: Tue, 7 Jul 2026 16:33:03 +0800 Subject: [PATCH 28/70] fix: stabilize recent dialogue memory typing --- lib/ai/runtime/recent-dialogue-memory.ts | 28 ++++++++++++------------ 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/ai/runtime/recent-dialogue-memory.ts b/lib/ai/runtime/recent-dialogue-memory.ts index fa8dbd113..b6b5640fd 100644 --- a/lib/ai/runtime/recent-dialogue-memory.ts +++ b/lib/ai/runtime/recent-dialogue-memory.ts @@ -91,20 +91,20 @@ export function buildRecentDialogueMemory(params: { const maxCharsPerEntry = Math.max(80, Math.min(500, params.maxCharsPerEntry || 220)); const maxTotalChars = Math.max(160, Math.min(1600, params.maxTotalChars || 900)); - const sourceEntries = messages - .slice(-maxSourceMessages) - .map((message) => { - const role = message.role === "assistant" ? "assistant" : message.role === "user" ? "user" : null; - if (!role) return null; - const content = getMessageText(message, maxCharsPerEntry); - if (!content) return null; - return { - role, - content, - messageId: message.id, - } satisfies RecentDialogueMemoryEntry; - }) - .filter((entry): entry is RecentDialogueMemoryEntry => Boolean(entry)); + const sourceEntries: RecentDialogueMemoryEntry[] = []; + for (const message of messages.slice(-maxSourceMessages)) { + const role = message.role === "assistant" ? "assistant" : message.role === "user" ? "user" : null; + if (!role) continue; + + const content = getMessageText(message, maxCharsPerEntry); + if (!content) continue; + + sourceEntries.push({ + role, + content, + messageId: message.id, + }); + } const preferredEntries = isRecallRequest && !assistantRecall From 2e4cb0cf7310243c2182f7c21cb91e58cd3973b4 Mon Sep 17 00:00:00 2001 From: 134 Date: Tue, 7 Jul 2026 16:54:43 +0800 Subject: [PATCH 29/70] fix: keep canvas user message visible while pending --- components/canvas/CanvasChatPanel.tsx | 49 +++++++++++++++++++-------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index ac5ffeb94..40964a6df 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -7041,8 +7041,7 @@ export function CanvasChatPanel({ inFlightRawChatIdRef.current = rawChatId; const launchUserMessageId = generateUUID(); const launchCreatedAt = new Date(); - - sendMessage({ + const launchUserMessage = { id: launchUserMessageId, role: "user", parts, @@ -7052,7 +7051,15 @@ export function CanvasChatPanel({ composerMode: activeComposerModeRef.current || activeComposerMode, modelId: selectedModelId, }, - } as any); + } as ChatMessage; + + sendMessage(launchUserMessage as any); + setMessages((previousMessages) => + sortCanvasMessagesByCreatedAt([ + ...previousMessages.filter((message) => message.id !== launchUserMessageId), + launchUserMessage, + ]) + ); hasHydratedHistoryRef.current = true; pendingLaunchRef.current = null; @@ -7065,7 +7072,7 @@ export function CanvasChatPanel({ if (editableDiv) { editableDiv.innerHTML = ""; } - }, [activeChatId, rawChatId, selectedModelId, status, sendMessage, clearSelection, launchAutoSendTick, activeComposerMode]); + }, [activeChatId, rawChatId, selectedModelId, status, sendMessage, setMessages, clearSelection, launchAutoSendTick, activeComposerMode]); // 当历史消息加载完成后，更新消息列表（支持同一会话的增量刷新） const loadedHistorySignatureRef = useRef(""); @@ -7083,6 +7090,12 @@ export function CanvasChatPanel({ ) { return; } + const hasActiveRegularSubmission = + inFlightChatIdRef.current === activeChatId || + inFlightRawChatIdRef.current === rawChatId; + if (hasActiveRegularSubmission && !shouldForceHydrate) { + return; + } const historyMessages = historyData.messages || []; const signature = buildCanvasHistoryMessagesSignature(rawChatId, historyMessages); @@ -7112,7 +7125,7 @@ export function CanvasChatPanel({ } }; } - }, [historyData, isClawdSubmitting, rawChatId, setMessages, status]); + }, [activeChatId, historyData, isClawdSubmitting, rawChatId, setMessages, status]); useEffect(() => { const streamState = streamImageGenerationStateByChatId[activeChatId]; @@ -10999,15 +11012,23 @@ export function CanvasChatPanel({ }); } - // 使用与 multimodal-input 相同的格式发送消息 - try { - sendMessage({ - id: userMessageId, - role: "user", - parts: parts, - createdAt: optimisticCreatedAt, - metadata: optimisticUserMetadata, - } as any); + const optimisticUserMessage: ChatMessage = { + id: userMessageId, + role: "user", + parts, + createdAt: optimisticCreatedAt, + metadata: optimisticUserMetadata, + } as ChatMessage; + + // 使用与 multimodal-input 相同的格式发送消息 + try { + sendMessage(optimisticUserMessage as any); + setMessages((previousMessages) => + sortCanvasMessagesByCreatedAt([ + ...previousMessages.filter((message) => message.id !== userMessageId), + optimisticUserMessage, + ]) + ); } catch (error) { releaseSendGuard(activeChatId); if (shouldShowOptimisticImageGeneration) { From 5923149dd63649d1a27e5e0b5324e09a65fd3244 Mon Sep 17 00:00:00 2001 From: 134 Date: Tue, 7 Jul 2026 20:36:18 +0800 Subject: [PATCH 30/70] fix: restore canvas pending feedback and scoped capabilities --- app/(chat)/api/chat/route.ts | 47 +++++++++++++--- components/canvas/CanvasChatPanel.tsx | 62 +++++++++++----------- lib/ai/runtime/task-boundary-classifier.ts | 5 +- lib/ai/runtime/task-boundary-text.ts | 12 +++++ tests/routes/canvas-chat-helpers.test.ts | 8 +++ 5 files changed, 93 insertions(+), 41 deletions(-) create mode 100644 lib/ai/runtime/task-boundary-text.ts diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index 5b901905b..67e892f9b 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -257,6 +257,7 @@ import { type AgentModelSlots, } from "@/lib/ai/runtime/agent-model-slots"; import { isPureGreetingTurnRequest } from "@/lib/ai/runtime/task-boundary-classifier"; +import { isCapabilityQuestionTurnRequest } from "@/lib/ai/runtime/task-boundary-text"; import { getProviderHealthSnapshot, recordProviderHealthFailure, @@ -846,6 +847,19 @@ const DEFAULT_STREAM_KEEPALIVE_INTERVAL_MS = 12_000; const CHAT_CREATION_LOOKUP_RETRY_DELAYS_MS = [60, 180, 360] as const; const DETAIL_REPLACEMENT_PROMPT_PACK_REQUIRED_MESSAGE = "请先在「风格化提示词配置」里选择提示词组合，再执行详情场景替换；缺少提示词组时系统不会处理。"; +const CANVAS_AGENT_GREETING_MESSAGE = + "你好，我在。你可以直接告诉我想做什么：生图、修图、生成视频、分析图片都可以。"; +const CANVAS_AGENT_CAPABILITY_MESSAGE = [ + "我主要帮你完成画布里的视觉创作任务：", + "", + "- 生图：按文字、参考图或画布选中内容生成图片。", + "- 修图：换背景、去元素、局部重绘、扩图、增强、抠图、改尺寸。", + "- 视频：文生视频、图生视频、分镜/镜头视频需求整理与生成。", + "- 图片分析：识别主体、风格、构图、材质、光线，并整理可执行提示词。", + "- 画布协作：读取当前画布选择、槽位、最近结果，继续生成、修改或写回画布。", + "", + "你可以直接说：把这张图换成白底、参考左边图片生成 3 张主图、让这张图动起来。", +].join("\n"); const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); @@ -1234,9 +1248,20 @@ export async function POST(request: Request) { const hasCurrentMessageMediaInput = message.parts.some((part: any) => part?.type === "file" || part?.type === "image" || part?.type === "image_url" ); + const isGreetingTurnRequest = isPureGreetingTurnRequest(requestTextContent); + const isCapabilityQuestionRequest = + isCapabilityQuestionTurnRequest(requestTextContent); + const canvasFastPathMessage = isCapabilityQuestionRequest + ? CANVAS_AGENT_CAPABILITY_MESSAGE + : isGreetingTurnRequest + ? CANVAS_AGENT_GREETING_MESSAGE + : ""; if ( - shouldRunAgentConversation && - isPureGreetingTurnRequest(requestTextContent) && + (shouldRunAgentConversation || + isCanvasPageRequest || + contextPayload?.source === "canvas" || + Boolean(requestCanvasAgentContext)) && + canvasFastPathMessage && !hasCurrentMessageMediaInput ) { const session = await auth(); @@ -1245,7 +1270,6 @@ export async function POST(request: Request) { return new ChatSDKError("unauthorized:chat").toResponse(); } - const greetingMessage = "你好，我在。你可以直接告诉我想做什么：生图、修图、生成视频、分析图片都可以。"; const fastPathTitle = requestTextContent.trim().slice(0, 80) || "新对话"; await saveChat({ id, @@ -1266,7 +1290,12 @@ export async function POST(request: Request) { const greetingStream = createUIMessageStream({ execute: async ({ writer }) => { const messageId = generateUUID(); - writeAssistantTextChunk(writer, greetingMessage, "greeting-fast-path", messageId); + writeAssistantTextChunk( + writer, + canvasFastPathMessage, + isCapabilityQuestionRequest ? "capability-fast-path" : "greeting-fast-path", + messageId + ); await saveMessages({ messages: [ @@ -1287,7 +1316,7 @@ export async function POST(request: Request) { id: messageId, chatId: id, role: "assistant", - parts: [{ type: "text", text: greetingMessage }], + parts: [{ type: "text", text: canvasFastPathMessage }], attachments: [], createdAt: new Date(), videoStorageStatus: null, @@ -8625,10 +8654,16 @@ export async function POST(request: Request) { selectedChatModel: finalModelId, requestHints, }); - const agentIdentityInstruction = shouldRunAgentConversation + const shouldUseProductIdentityInstruction = + shouldRunAgentConversation || + isCanvasPageRequest || + contextPayload?.source === "canvas" || + Boolean(requestCanvasAgentContext); + const agentIdentityInstruction = shouldUseProductIdentityInstruction ? await buildAgentRoleIdentityInstruction({ allowVideoGeneration: allowAgentVideoGeneration, hasCanvasContext: Boolean( + isCanvasPageRequest || effectiveContextPayload?.source === "canvas" || effectiveContextPayload?.selectedNodeIds?.length || effectiveContextPayload?.canvasImageNodes?.length, diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index 40964a6df..8b7e3f299 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -1595,21 +1595,15 @@ function OpenClawQuickActions({ ); } -function CanvasAssistantTypingPlaceholder({ hint }: { hint?: string | null }) { +function CanvasAssistantTypingPlaceholder({ + hint, + label = "分析中", +}: { + hint?: string | null; + label?: string; +}) { return (
-
{hint ? (
@@ -1617,19 +1611,11 @@ function CanvasAssistantTypingPlaceholder({ hint }: { hint?: string | null }) {
) : null}
- {[0, 1, 2].map((index) => ( - - ))} + + {label}
@@ -11910,9 +11896,11 @@ export function CanvasChatPanel({ ); const isLoading = status === 'streaming' || status === 'submitted'; + const activeChatSendGuard = getSendGuard(activeChatId); + const isActiveChatPending = isLoading || activeChatSendGuard.inFlight; const isComposerUploading = Boolean(composerUploadProgress); const isSubmitting = - isLoading || isDetailGenerating || isClawdSubmitting || isComposerUploading; + isActiveChatPending || isDetailGenerating || isClawdSubmitting || isComposerUploading; const hasInputOrAttachments = Boolean( input.trim() || attachments.length > 0 || (!isDetailGenerationMode && selectedSkillTag) ); @@ -12287,15 +12275,23 @@ export function CanvasChatPanel({ const isDetailEmptyState = isDetailGenerationMode && renderableVisibleMessages.length === 0 && - !isLoading && + !isActiveChatPending && !shouldShowImageLoading; const isPlainCanvasEmptyState = !isWorkspacePresentation && !isDetailGenerationMode && !isVideoWorkspace && renderableVisibleMessages.length === 0 && - !isLoading && + !isActiveChatPending && !shouldShowImageLoading; + const latestRenderableMessage = renderableVisibleMessages.at(-1); + const shouldRenderAssistantTypingPlaceholder = + !isClawdMode && + !isDetailGenerationMode && + !shouldRenderImageLoading && + !hasActiveTypewriter && + isActiveChatPending && + latestRenderableMessage?.role === "user"; const workspaceEmptyTitle = isClawdMode ? `开始 ${CLAW_MODE_LABEL} 对话` @@ -13490,9 +13486,7 @@ export function CanvasChatPanel({ shouldAnimateAnalysis || shouldAnimatePlainText || isClawdSubmitting || - getSendGuard(activeChatId).inFlight || - status === "streaming" || - status === "submitted" + isActiveChatPending } onAction={handleOpenClawQuickAction} /> @@ -13564,9 +13558,13 @@ export function CanvasChatPanel({ estimatedWaitMs={loadingEstimatedWaitMs} /> )} + + {shouldRenderAssistantTypingPlaceholder ? ( + + ) : null} {isClawdMode && isClawdSubmitting ? ( - + ) : null}
diff --git a/lib/ai/runtime/task-boundary-classifier.ts b/lib/ai/runtime/task-boundary-classifier.ts index a4dddf4a2..fe6991ed2 100644 --- a/lib/ai/runtime/task-boundary-classifier.ts +++ b/lib/ai/runtime/task-boundary-classifier.ts @@ -5,6 +5,7 @@ import type { PendingGridConfirmation, } from "@/lib/ai/conversation-action-resolver"; import { isExplicitPromptOptimizationRequest } from "@/lib/ai/smart-analyzer"; +import { isCapabilityQuestionTurnRequest } from "@/lib/ai/runtime/task-boundary-text"; import { detectStoryboardCommandIntent } from "@/lib/video-agent/storyboard-command-interpreter"; export type TaskBoundaryType = @@ -88,9 +89,7 @@ function isLikelyChatOnlyRequest(text: string): boolean { const normalized = normalizeCompactText(text); if (!normalized) return false; - return /你能做什么|你会什么|你可以做什么|你能帮我做什么|你能干什么|你可以帮我做什么|介绍一下你自己|介绍一下自己|说说你的能力|你有哪些功能|你有什么功能|怎么用|怎么玩|如何使用|帮助说明|帮助手册|功能介绍/.test( - normalized, - ); + return isCapabilityQuestionTurnRequest(normalized); } function isLikelyAnalysisOnlyRequest(text: string, hasImageContext: boolean): boolean { diff --git a/lib/ai/runtime/task-boundary-text.ts b/lib/ai/runtime/task-boundary-text.ts new file mode 100644 index 000000000..dad761f7f --- /dev/null +++ b/lib/ai/runtime/task-boundary-text.ts @@ -0,0 +1,12 @@ +export function normalizeCompactTaskBoundaryText(text: string) { + return String(text || "").toLowerCase().replace(/\s+/g, "").trim(); +} + +export function isCapabilityQuestionTurnRequest(text: string): boolean { + const normalized = normalizeCompactTaskBoundaryText(text); + if (!normalized) return false; + + return /你.*(?:能|会|可以|可|嫩)?.*(?:为我做什么|帮我做什么|帮我什么|做什么|干什么)|你.*(?:有哪些|有什么).*(?:功能|能力)|(?:介绍|说说).*(?:功能|能力|你自己|自己)|怎么用|怎么玩|如何使用|帮助说明|帮助手册|功能介绍/.test( + normalized, + ); +} diff --git a/tests/routes/canvas-chat-helpers.test.ts b/tests/routes/canvas-chat-helpers.test.ts index 6b0027ab4..bb5d878e7 100644 --- a/tests/routes/canvas-chat-helpers.test.ts +++ b/tests/routes/canvas-chat-helpers.test.ts @@ -56,6 +56,7 @@ import { buildRecentDialogueMemory, isRecentDialogueRecallRequest, } from "@/lib/ai/runtime/recent-dialogue-memory"; +import { isCapabilityQuestionTurnRequest } from "@/lib/ai/runtime/task-boundary-text"; import { isAgentModeScopedChatId, isScopedEmptyHistoryRequest, @@ -326,6 +327,13 @@ test.describe("canvas chat pure helpers", () => { expect(upstream).toContain("| SKU | 销量 |"); }); + test("recognizes canvas product capability questions with typo tolerance", () => { + expect(isCapabilityQuestionTurnRequest("你能为我做什么？")).toBe(true); + expect(isCapabilityQuestionTurnRequest("你嫩为我做什么？")).toBe(true); + expect(isCapabilityQuestionTurnRequest("你有哪些功能")).toBe(true); + expect(isCapabilityQuestionTurnRequest("帮我把这张图换成白底")).toBe(false); + }); + test("builds bounded recent dialogue memory for user recall requests", () => { const memory = buildRecentDialogueMemory({ messages: [ From 4ac4ac26cfa5f146e6d914a69e37d4382145101a Mon Sep 17 00:00:00 2001 From: 134 Date: Tue, 7 Jul 2026 21:07:18 +0800 Subject: [PATCH 31/70] fix canvas agent image generation status --- app/(chat)/api/chat/route.ts | 31 +++++++++++++++++++++++++++ components/canvas/CanvasChatPanel.tsx | 22 ++++++++++++------- 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index 67e892f9b..fcbfd0b3d 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -7896,6 +7896,32 @@ export async function POST(request: Request) { operationLabel: "agent-generate-image", resultType: "image", }); + const directImageStatusPayload = { + providerType: requestedModelProviderType, + modelName: finalModelId, + imageCount: + effectiveImageParams?.imageCount || + defaultImageQualityParams.imageCount || + 1, + }; + const stopDirectImageKeepalive = startStreamKeepalive( + writer, + "agent:generate_image", + DEFAULT_STREAM_KEEPALIVE_INTERVAL_MS, + ); + const stopDirectImageStatus = + contextPayload?.source === "canvas" + ? setInterval(() => { + writeImageGenerationStatusChunk( + writer, + { + phase: "processing", + ...directImageStatusPayload, + }, + "agent:generate_image:status-processing-keepalive", + ); + }, DEFAULT_STREAM_KEEPALIVE_INTERVAL_MS) + : null; try { const directImageResult = await executeDirectAgentImageGeneration({ requestTextContent, @@ -8037,6 +8063,11 @@ export async function POST(request: Request) { "", "agent:generate_image:end-error", ); + } finally { + stopDirectImageKeepalive(); + if (stopDirectImageStatus) { + clearInterval(stopDirectImageStatus); + } } }, }); diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index 8b7e3f299..c16e2d7a5 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -11006,16 +11006,20 @@ export function CanvasChatPanel({ metadata: optimisticUserMetadata, } as ChatMessage; + setMessages((previousMessages) => + sortCanvasMessagesByCreatedAt([ + ...previousMessages.filter((message) => message.id !== userMessageId), + optimisticUserMessage, + ]) + ); + // 使用与 multimodal-input 相同的格式发送消息 try { sendMessage(optimisticUserMessage as any); - setMessages((previousMessages) => - sortCanvasMessagesByCreatedAt([ - ...previousMessages.filter((message) => message.id !== userMessageId), - optimisticUserMessage, - ]) - ); } catch (error) { + setMessages((previousMessages) => + previousMessages.filter((message) => message.id !== userMessageId) + ); releaseSendGuard(activeChatId); if (shouldShowOptimisticImageGeneration) { clearStreamImageGenerationStateForChat(activeChatId); @@ -12010,8 +12014,10 @@ export function CanvasChatPanel({ ) ); const shouldShowImageLoading = - ((hasGenerationStart && !hasGenerationEnd) || Boolean(streamImageGenerationState)) && - !!currentModel; + Boolean(streamImageGenerationState) || + (hasGenerationStart && + !hasGenerationEnd && + Boolean(currentModel || activeComposerMode === "agent")); const hasActiveTypewriter = Object.keys(activeTypingSections).length > 0; const shouldRenderImageLoading = shouldShowImageLoading && !hasActiveTypewriter; const loadingImageCount = streamImageGenerationState?.imageCount || imageParams.imageCount || 1; From e2e373ce08a6ac9337fd1cc18c8b6556a25e3123 Mon Sep 17 00:00:00 2001 From: 134 Date: Tue, 7 Jul 2026 21:21:12 +0800 Subject: [PATCH 32/70] fix canvas agent duplicate user message --- components/canvas/CanvasChatPanel.tsx | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index c16e2d7a5..77179b7d1 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -11006,20 +11006,16 @@ export function CanvasChatPanel({ metadata: optimisticUserMetadata, } as ChatMessage; - setMessages((previousMessages) => - sortCanvasMessagesByCreatedAt([ - ...previousMessages.filter((message) => message.id !== userMessageId), - optimisticUserMessage, - ]) - ); - // 使用与 multimodal-input 相同的格式发送消息 try { sendMessage(optimisticUserMessage as any); - } catch (error) { setMessages((previousMessages) => - previousMessages.filter((message) => message.id !== userMessageId) + sortCanvasMessagesByCreatedAt([ + ...previousMessages.filter((message) => message.id !== userMessageId), + optimisticUserMessage, + ]) ); + } catch (error) { releaseSendGuard(activeChatId); if (shouldShowOptimisticImageGeneration) { clearStreamImageGenerationStateForChat(activeChatId); From 51283b81fb85523a524da4f468d353abc8424a64 Mon Sep 17 00:00:00 2001 From: 134 Date: Tue, 7 Jul 2026 21:38:24 +0800 Subject: [PATCH 33/70] fix canvas cutout intent routing --- lib/ai/runtime/canvas-agent-kernel.ts | 4 +- lib/ai/runtime/intent-arbiter.ts | 32 ++++++------- lib/ai/smart-analyzer.ts | 10 ++-- lib/ai/specialists/index.ts | 2 +- tests/routes/canvas-chat-helpers.test.ts | 60 ++++++++++++++++++++++++ 5 files changed, 82 insertions(+), 26 deletions(-) diff --git a/lib/ai/runtime/canvas-agent-kernel.ts b/lib/ai/runtime/canvas-agent-kernel.ts index 68467384f..6c5ae36f2 100644 --- a/lib/ai/runtime/canvas-agent-kernel.ts +++ b/lib/ai/runtime/canvas-agent-kernel.ts @@ -20,7 +20,7 @@ const DEICTIC_TARGET_REGEX = /(这张图|这只|这个|这款|这套|当前这张|当前这个|当前这只|这一张|这一只|这个产品|这个物体|这只猫|这只狗)/i; const ACTIONABLE_IMAGE_EDIT_REGEX = - /(抠图|去背景|透明背景|去背|扣图|移除背景|去水印|水印|去logo|去\s*logo|移除logo|去除logo|换背景|改背景|背景换成|背景改成|白底|纯色背景|替换.*背景|变清晰|高清|锐化|降噪|提升质量|修复|清晰化|扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|多视角|换视角|改视角|视角变体|新角度|换角度|改角度|去除|消除|擦掉|删除|移除|p掉|重绘|风格化|卡通化|动漫风|油画风|调色|调整颜色|亮度|对比度|饱和度|缩放|放大|缩小|调整尺寸|裁剪|resize|crop|修图|p图|改图|调图|编辑.*图|优化.*图|重做|再调整|继续改|继续调|处理)/i; + /(抠图|去背景|透明背景|去背|扣图|抠出|扣出|抠出来|扣出来|移除背景|去水印|水印|去logo|去\s*logo|移除logo|去除logo|换背景|改背景|背景换成|背景改成|白底|纯色背景|替换.*背景|变清晰|高清|锐化|降噪|提升质量|修复|清晰化|扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|多视角|换视角|改视角|视角变体|新角度|换角度|改角度|去除|消除|擦掉|删除|移除|p掉|重绘|风格化|卡通化|动漫风|油画风|调色|调整颜色|亮度|对比度|饱和度|缩放|放大|缩小|调整尺寸|裁剪|resize|crop|修图|p图|改图|调图|编辑.*图|优化.*图|重做|再调整|继续改|继续调|处理)/i; export function normalizeCanvasAgentRequestText(text: string) { return String(text || "").toLowerCase().replace(/\s+/g, "").trim(); @@ -42,7 +42,7 @@ export function inferCanvasAgentImageEditCapability( if (/去水印|水印|去logo|移除logo|去除logo/.test(normalizedText)) { return "remove_watermark"; } - if (/抠图|去背景|透明背景|去背|扣图|移除背景/.test(normalizedText)) { + if (/抠图|去背景|透明背景|去背|扣图|抠出|扣出|抠出来|扣出来|移除背景/.test(normalizedText)) { return "remove_background"; } if (/多视角|换视角|改视角|视角变体|新角度|换角度|改角度|侧面|侧视|背面|背后|俯视|俯拍|仰视|仰拍|低机位|高机位/.test(normalizedText)) { diff --git a/lib/ai/runtime/intent-arbiter.ts b/lib/ai/runtime/intent-arbiter.ts index ce1081ef1..4cb6141b0 100644 --- a/lib/ai/runtime/intent-arbiter.ts +++ b/lib/ai/runtime/intent-arbiter.ts @@ -94,10 +94,10 @@ function isExplicitImageGenerationRequest(text: string): boolean { const visualTarget = "(?:图|图片|海报|头像|插画|场景|主图|产品图|商品图|电商图|壁纸|封面|logo|图标)"; const createVisualTarget = new RegExp( - `(?:生成|创作|创建|制作|设计|做|出|画|绘制)${countToken}?张?.{0,32}${visualTarget}`, + `(?:生成|创作|创建|制作|设计|做|出|画(?!布)|绘制)${countToken}?张?.{0,32}${visualTarget}`, ); const drawSubject = new RegExp( - `(?:画|绘制|生成|创作|创建)${countToken}?(?:只|个|位|幅|张)?.{0,24}(?:小猫|猫咪|猫|狗|人物|角色|ip|头像|logo|图标|插画)`, + `(?:画(?!布)|绘制|生成|创作|创建)${countToken}?(?:只|个|位|幅|张)?.{0,24}(?:小猫|猫咪|猫|狗|人物|角色|ip|头像|logo|图标|插画)`, ); const continueVariants = new RegExp( `(?:再来|再出|再做|再给我|补|多来|继续生成|继续出)${countToken}?张`, @@ -197,6 +197,9 @@ function buildExecutionFromAnalysis(params: { targetSource: ResolvedTargetAssets["resolutionMode"]; }): IntentArbiterExecution { const mode = mapIntentToMode(params.analysis.intent, params.analysis.needsClarification); + const shouldKeepPlannedToolCalls = + params.execution?.mode === mode && + params.execution?.routeIntent === params.analysis.intent; return { mode, @@ -218,7 +221,7 @@ function buildExecutionFromAnalysis(params: { shouldStayConversational: mode === "respond" || mode === "analyze_image" || mode === "clarify", shouldPublishToCanvas: params.execution?.shouldPublishToCanvas || false, - toolCalls: params.execution?.toolCalls || [], + toolCalls: shouldKeepPlannedToolCalls ? params.execution?.toolCalls || [] : [], availableTools: params.execution?.availableTools || [], }; } @@ -262,6 +265,14 @@ function resolvePolicyIntent(params: { }; } + if (hasUsableImage && likelyEdit) { + return { + intent: "image_edit", + subIntent: params.analysis.subIntent || inferDefaultImageEditSubIntent(params.text), + reason: "image_edit_policy_override", + }; + } + switch (params.actionResolution.kind) { case "image_edit_followup": return { @@ -292,13 +303,6 @@ function resolvePolicyIntent(params: { reason: "task_boundary_execution_video", }; case "execution_image": - if (hasUsableImage && likelyEdit && !explicitImageGen) { - return { - intent: "image_edit", - subIntent: params.analysis.subIntent || inferDefaultImageEditSubIntent(params.text), - reason: "task_boundary_execution_image_edit", - }; - } return { intent: "image_gen", reason: "task_boundary_execution_image_generate", @@ -329,14 +333,6 @@ function resolvePolicyIntent(params: { }; } - if (hasUsableImage && likelyEdit && params.analysis.intent === "chat") { - return { - intent: "image_edit", - subIntent: params.analysis.subIntent || inferDefaultImageEditSubIntent(params.text), - reason: "image_edit_policy_override", - }; - } - return { intent: params.analysis.intent, subIntent: diff --git a/lib/ai/smart-analyzer.ts b/lib/ai/smart-analyzer.ts index 3bde4b6ee..c22451481 100644 --- a/lib/ai/smart-analyzer.ts +++ b/lib/ai/smart-analyzer.ts @@ -33,7 +33,7 @@ export interface AnalysisResult { function inferDefaultImageEditSubIntent(text: string): AnalysisResult["subIntent"] { const lowerText = text.toLowerCase(); if (/去水印|水印|去logo|去\s*logo|移除logo|去除logo/.test(lowerText)) return "remove_watermark"; - if (/抠图|去背景|透明背景|去背|扣图|移除背景/.test(lowerText)) return "remove_background"; + if (/抠图|去背景|透明背景|去背|扣图|抠出|扣出|抠出来|扣出来|移除背景/.test(lowerText)) return "remove_background"; if (/换背景|改背景|背景换成|背景改成|白底|透明底/.test(lowerText)) return "replace_background"; if (/多视角|换视角|改视角|视角变体|新角度|换角度|改角度|侧面|侧视|背面|背后|俯视|俯拍|仰视|仰拍|低机位|高机位/.test(lowerText)) return "angle_variant"; if (hasDirectAgentLocalEditIntent(text)) return "local_edit"; @@ -112,10 +112,10 @@ export function isExplicitImageGenerationRequest(text: string): boolean { const visualTarget = "(?:图|图片|海报|头像|插画|场景|主图|产品图|商品图|电商图|壁纸|封面|logo|图标)"; const createVisualTarget = new RegExp( - `(?:生成|创作|创建|制作|设计|做|出|画|绘制)${countToken}?张?.{0,32}${visualTarget}`, + `(?:生成|创作|创建|制作|设计|做|出|画(?!布)|绘制)${countToken}?张?.{0,32}${visualTarget}`, ); const drawSubject = new RegExp( - `(?:画|绘制|生成|创作|创建)${countToken}?(?:只|个|位|幅|张)?.{0,24}(?:小猫|猫咪|猫|狗|人物|角色|ip|头像|logo|图标|插画)`, + `(?:画(?!布)|绘制|生成|创作|创建)${countToken}?(?:只|个|位|幅|张)?.{0,24}(?:小猫|猫咪|猫|狗|人物|角色|ip|头像|logo|图标|插画)`, ); const continueVariants = new RegExp( `(?:再来|再出|再做|再给我|补|多来|继续生成|继续出)${countToken}?张`, @@ -141,7 +141,7 @@ export function isExplicitImageGenerationRequest(text: string): boolean { function isLikelyImageEditRequest(text: string): boolean { const normalizedText = text.toLowerCase(); - return hasDirectAgentLocalEditIntent(text) || /抠图|去背景|透明背景|去背|扣图|移除背景|去水印|水印|去logo|去\s*logo|移除logo|去除logo|换背景|改背景|背景换成|背景改成|白底|纯色背景|替换.*背景|变清晰|高清|锐化|降噪|提升质量|修复|清晰化|扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|换成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|改成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|比例|画幅|竖版|横版|多视角|换视角|改视角|视角变体|新角度|换角度|改角度|侧面|侧视|背面|背后|俯视|俯拍|仰视|仰拍|低机位|高机位|去除|消除|擦掉|删除|移除|p掉|重绘|风格化|卡通化|动漫风|油画风|转.*风格|变.*风格|改.*风格|调色|调整颜色|亮度|对比度|饱和度|色调|曝光|滤镜|改成.*色|换成.*色|变成.*色|颜色改成|颜色换成|颜色变成|缩放|放大|缩小|调整尺寸|裁剪|resize|crop|修图|p图|改图|调图|编辑.*图|优化.*图|重做|再调整|继续改|继续调/.test( + return hasDirectAgentLocalEditIntent(text) || /抠图|去背景|透明背景|去背|扣图|抠出|扣出|抠出来|扣出来|移除背景|去水印|水印|去logo|去\s*logo|移除logo|去除logo|换背景|改背景|背景换成|背景改成|白底|纯色背景|替换.*背景|变清晰|高清|锐化|降噪|提升质量|修复|清晰化|扩展|拓展|延伸|扩图|outpainting|拓宽|扩大|换成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|改成.*(?:比例|竖版|横版|竖图|横图|9:16|16:9|1:1)|比例|画幅|竖版|横版|多视角|换视角|改视角|视角变体|新角度|换角度|改角度|侧面|侧视|背面|背后|俯视|俯拍|仰视|仰拍|低机位|高机位|去除|消除|擦掉|删除|移除|p掉|重绘|风格化|卡通化|动漫风|油画风|转.*风格|变.*风格|改.*风格|调色|调整颜色|亮度|对比度|饱和度|色调|曝光|滤镜|改成.*色|换成.*色|变成.*色|颜色改成|颜色换成|颜色变成|缩放|放大|缩小|调整尺寸|裁剪|resize|crop|修图|p图|改图|调图|编辑.*图|优化.*图|重做|再调整|继续改|继续调/.test( normalizedText, ); } @@ -2331,7 +2331,7 @@ export function fallbackKeywordAnalysis( } // 抠图/去背景 - if (/抠图|去背景|透明背景|去背|扣图|移除背景/.test(lowerText)) { + if (/抠图|去背景|透明背景|去背|扣图|抠出|扣出|抠出来|扣出来|移除背景/.test(lowerText)) { return { intent: "image_edit", subIntent: "remove_background", diff --git a/lib/ai/specialists/index.ts b/lib/ai/specialists/index.ts index f91793f61..75827ad67 100644 --- a/lib/ai/specialists/index.ts +++ b/lib/ai/specialists/index.ts @@ -250,7 +250,7 @@ function normalizeEditCapability( subIntent: "remove_watermark", }; } - if (subIntent === "remove_background" || /抠图|去背|去背景|透明底/.test(normalizedText)) { + if (subIntent === "remove_background" || /抠图|去背|去背景|透明底|抠出|扣出|抠出来|扣出来/.test(normalizedText)) { return { capability: "remove_background", toolName: "removeBackground", diff --git a/tests/routes/canvas-chat-helpers.test.ts b/tests/routes/canvas-chat-helpers.test.ts index bb5d878e7..ee23944a0 100644 --- a/tests/routes/canvas-chat-helpers.test.ts +++ b/tests/routes/canvas-chat-helpers.test.ts @@ -48,9 +48,11 @@ import { import { shouldBypassCanvasAgentMissingTargetClarification } from "@/lib/ai/runtime/canvas-agent-clarification"; import { hasCanvasAgentRoutingContext } from "@/lib/ai/canvas-agent-routing"; import { + inferCanvasAgentImageEditCapability, resolveCanvasAgentRequestPolicy, resolveCanvasAgentTargetPreference, } from "@/lib/ai/runtime/canvas-agent-kernel"; +import { isExplicitImageGenerationRequest } from "@/lib/ai/smart-analyzer"; import { inferAgentImageCountFromText } from "@/lib/ai/runtime/image-count-intent"; import { buildRecentDialogueMemory, @@ -1463,6 +1465,64 @@ test.describe("canvas chat pure helpers", () => { expect(directContext?.targetImageUrl).toBe(canvasImageUrl); }); + test("routes canvas subject cutout wording to remove background instead of image generation", () => { + const canvasImageUrl = "https://cdn.example.com/canvas-dog.png"; + const userText = "帮我将画布中的狗的图扣出来"; + + expect(isExplicitImageGenerationRequest(userText)).toBe(false); + expect(inferCanvasAgentImageEditCapability(userText)).toBe("remove_background"); + + const directContext = resolveDirectAgentToolContext({ + execution: { + mode: "edit_image", + routeIntent: "image_edit", + routeSubIntent: "remove_background", + targetSource: "text_reference", + preferredCapability: "remove_background", + shouldUseStructuredReply: true, + shouldOptimizePrompt: true, + shouldApplyImagePipeline: true, + shouldApplyVideoPipeline: false, + shouldStayConversational: false, + shouldPublishToCanvas: false, + toolCalls: [], + availableTools: ["resolveTargetImage", "removeBackground"], + } as any, + requestResolvedMediaInput: { + imageRefs: [], + currentMessageImageRefs: [], + selectedImageRefs: [], + canvasImageRefs: [ + { + url: canvasImageUrl, + normalizedUrl: canvasImageUrl, + nodeId: "dog-node", + source: "canvas_context", + role: "reference", + }, + ], + historyUserImageRefs: [], + historyAssistantImageRefs: [], + resolvedImageRefs: [], + currentMessageImageUrls: [], + selectedImageUrls: [], + canvasImageUrls: [canvasImageUrl], + historyUserImageUrls: [], + historyAssistantImageUrls: [], + resolvedImageUrls: [], + selectedAssetIds: [], + candidateAssetIds: [], + uploadedAssetIds: [], + hasImage: true, + source: "canvas_context", + fromHistory: false, + } as any, + }); + + expect(directContext?.execution.toolName).toBe("removeBackground"); + expect(directContext?.targetImageUrl).toBe(canvasImageUrl); + }); + test("only runs direct image edit batches for explicit safe batch requests", () => { expect( shouldRunDirectAgentBatchImageEdit({ From a03ea2cb479dd5bf98df61aea0256e68c851c5aa Mon Sep 17 00:00:00 2001 From: 134 Date: Tue, 7 Jul 2026 22:13:39 +0800 Subject: [PATCH 34/70] fix: persist claw chat sessions --- components/canvas/CanvasChatPanel.tsx | 137 +++++++++++++++++++++++--- 1 file changed, 123 insertions(+), 14 deletions(-) diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index 77179b7d1..292dc228d 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -2432,6 +2432,33 @@ export function CanvasChatPanel({ ); }, [getSendGuard]); + const persistChatSessions = useCallback( + (sessions: CanvasChatSession[]) => { + const sessionStorageKey = getSessionStorageKey(canvasId, chatScope); + try { + localStorage.setItem(sessionStorageKey, JSON.stringify(sessions)); + setLoadedChatSessionStorageKey(sessionStorageKey); + } catch (error) { + console.error("[CanvasChat] Failed to save chat sessions:", error); + } + }, + [canvasId, chatScope] + ); + + const updateChatSessions = useCallback( + (updater: (previousSessions: CanvasChatSession[]) => CanvasChatSession[]) => { + setChatSessions((previousSessions) => { + const nextSessions = updater(previousSessions); + if (nextSessions === previousSessions) { + return previousSessions; + } + persistChatSessions(nextSessions); + return nextSessions; + }); + }, + [persistChatSessions] + ); + const startNewChatSession = useCallback( (options?: { mode?: CanvasComposerMode; @@ -2462,13 +2489,19 @@ export function CanvasChatPanel({ forceHistoryHydrationRef.current = null; releaseSendGuard(activeChatId); - setChatSessions((prevSessions) => [newSession, ...prevSessions]); + updateChatSessions((prevSessions) => [newSession, ...prevSessions]); setActiveChatId(newChatSessionId); replaceChatSessionQuery(newChatSessionId); toast.success(options?.toastMessage || "已新建聊天窗口"); return newChatSessionId; }, - [activeChatId, chatSessions, releaseSendGuard, replaceChatSessionQuery] + [ + activeChatId, + chatSessions, + releaseSendGuard, + replaceChatSessionQuery, + updateChatSessions, + ] ); const setSessionClawdSkillTag = useCallback( @@ -2479,11 +2512,12 @@ export function CanvasChatPanel({ ) => { if (!chatSessionId) return; const normalizedTagId = String(tagId || "").trim() || null; - setChatSessions((prevSessions) => + updateChatSessions((prevSessions) => prevSessions.map((session) => session.id === chatSessionId ? { ...session, + mode: "clawd", clawdSkillTagId: normalizedTagId, clawdSkillTagPromptApplied: normalizedTagId === null @@ -2495,17 +2529,18 @@ export function CanvasChatPanel({ ) ); }, - [] + [updateChatSessions] ); const markSessionClawdSkillTagPromptApplied = useCallback( (chatSessionId: string, tagId: string) => { if (!chatSessionId || !tagId) return; - setChatSessions((prevSessions) => + updateChatSessions((prevSessions) => prevSessions.map((session) => session.id === chatSessionId && session.clawdSkillTagId === tagId ? { ...session, + mode: "clawd", clawdSkillTagPromptApplied: true, updatedAt: Date.now(), } @@ -2513,7 +2548,7 @@ export function CanvasChatPanel({ ) ); }, - [] + [updateChatSessions] ); const handleTypewriterActiveChange = useCallback((typingKey: string, active: boolean) => { @@ -4510,12 +4545,84 @@ export function CanvasChatPanel({ activeChatSession, activeComposerMode ); + useEffect(() => { + if (activeChatSessionMode !== "clawd") return; + if (!isClawdModeEnabled) return; + if (activeComposerMode !== "clawd") { + setComposerModeOverride("clawd"); + } + if (isDetailGenerationMode) { + setIsDetailGenerationMode(false); + } + }, [ + activeChatSessionMode, + activeComposerMode, + isClawdModeEnabled, + isDetailGenerationMode, + ]); + + useEffect(() => { + if (!activeChatId) return; + const historyMessages = historyData?.messages || []; + if (historyMessages.length === 0) return; + if (!historyMessages.some(isClawdChatMessage)) return; + + updateChatSessions((prevSessions) => { + const currentIndex = prevSessions.findIndex( + (session) => session.id === activeChatId + ); + const now = Date.now(); + + if (currentIndex < 0) { + const title = + deriveSessionTitleFromMessages(historyMessages) || `${CLAW_MODE_LABEL} 对话`; + return [ + { + id: activeChatId, + title, + mode: "clawd", + createdAt: now, + updatedAt: now, + }, + ...prevSessions, + ]; + } + + const current = prevSessions[currentIndex]; + if (getCanvasChatSessionMode(current) === "clawd") { + return prevSessions; + } + + const nextSessions = [...prevSessions]; + nextSessions[currentIndex] = { + ...current, + mode: "clawd", + updatedAt: Math.max(Number(current.updatedAt) || 0, now), + }; + nextSessions.sort((a, b) => b.updatedAt - a.updatedAt); + return nextSessions; + }); + + if (isClawdModeEnabled && activeComposerMode !== "clawd") { + setComposerModeOverride("clawd"); + if (isDetailGenerationMode) { + setIsDetailGenerationMode(false); + } + } + }, [ + activeChatId, + activeComposerMode, + historyData?.messages, + isClawdModeEnabled, + isDetailGenerationMode, + updateChatSessions, + ]); const currentModeChatSessions = useMemo( () => chatSessions.filter( - (session) => getCanvasChatSessionMode(session) === activeComposerMode + (session) => getCanvasChatSessionMode(session) === activeChatSessionMode ), - [activeComposerMode, chatSessions] + [activeChatSessionMode, chatSessions] ); const agentSelectableModels = useMemo( () => @@ -5319,7 +5426,7 @@ export function CanvasChatPanel({ if (canReuseActiveEmptySession) { forceHistoryHydrationRef.current = null; targetChatSessionId = activeChatId; - setChatSessions((prevSessions) => + updateChatSessions((prevSessions) => prevSessions.map((session) => session.id === activeChatId ? { @@ -5404,6 +5511,7 @@ export function CanvasChatPanel({ releaseSendGuard, replaceChatSessionQuery, startNewChatSession, + updateChatSessions, ] ); @@ -7216,7 +7324,7 @@ export function CanvasChatPanel({ const derivedTitle = deriveSessionTitleFromMessages(messages); const now = Date.now(); - setChatSessions((prevSessions) => { + updateChatSessions((prevSessions) => { const currentIndex = prevSessions.findIndex( (session) => session.id === activeChatId ); @@ -7252,7 +7360,7 @@ export function CanvasChatPanel({ nextSessions.sort((a, b) => b.updatedAt - a.updatedAt); return nextSessions; }); - }, [activeChatId, messages]); + }, [activeChatId, messages, updateChatSessions]); useEffect(() => { if (activeChatSessionMode === "clawd") return; @@ -10550,7 +10658,7 @@ export function CanvasChatPanel({ }, } as ChatMessage; - setChatSessions((prevSessions) => + updateChatSessions((prevSessions) => prevSessions.map((session) => session.id === activeChatId ? { @@ -11096,6 +11204,7 @@ export function CanvasChatPanel({ status, mutate, markSessionClawdSkillTagPromptApplied, + updateChatSessions, buildHistoryMessagesUrl, workflowContext, messages, @@ -12473,8 +12582,8 @@ export function CanvasChatPanel({ ); const createNewChatSession = useCallback(() => { - startNewChatSession({ mode: activeComposerMode }); - }, [activeComposerMode, startNewChatSession]); + startNewChatSession({ mode: activeChatSessionMode }); + }, [activeChatSessionMode, startNewChatSession]); const switchToSession = useCallback((sessionId: string) => { if (!sessionId || sessionId === activeChatId) { From 6bc43ffc3f924a23541b3a015cc4f3c080be84ce Mon Sep 17 00:00:00 2001 From: 134 Date: Tue, 7 Jul 2026 22:16:07 +0800 Subject: [PATCH 35/70] perf: improve home and canvas initial load --- .../[id]/CanvasEditorLegacyPageClient.tsx | 12 +++-- components/home/lovart-home.tsx | 46 +++++++++++++++++-- .../home/lovart/lovart-featured-gallery.tsx | 2 + .../home/lovart/lovart-hero-composer.tsx | 15 +++++- .../home/lovart/lovart-projects-section.tsx | 2 + 5 files changed, 68 insertions(+), 9 deletions(-) diff --git a/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx b/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx index 8e501117e..2a48e250c 100644 --- a/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx +++ b/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx @@ -1976,7 +1976,7 @@ export default function CanvasEditorLegacyPage({ ); const [activeWorkflowExecutionTaskKey, setActiveWorkflowExecutionTaskKey] = useState(null); - const [isPanelOpen, setIsPanelOpen] = useState(() => !isVideoMode); + const [isPanelOpen, setIsPanelOpen] = useState(() => requestedAgentMode); const [floatingPanelWidth, setFloatingPanelWidth] = useState(FLOATING_PANEL_WIDTH); const [ @@ -3280,11 +3280,15 @@ export default function CanvasEditorLegacyPage({ }, [checkCanvasAuthSession]); useEffect(() => { + if (!isPanelOpen && !isVideoCanvasFileListOpen) { + return; + } + const timeoutId = window.setTimeout(() => { setAreSecondaryPanelsReady(true); - }, 800); + }, 120); return () => window.clearTimeout(timeoutId); - }, []); + }, [isPanelOpen, isVideoCanvasFileListOpen]); const hydrateCanvasProject = useCallback( (project: CanvasInitialProject) => { @@ -10844,7 +10848,7 @@ export default function CanvasEditorLegacyPage({
) : null}
- {!areSecondaryPanelsReady ? ( + {!isPanelOpen ? null : !areSecondaryPanelsReady ? ( ) : panelMode === "chat" ? ( // AI 对话内容 - 使用 key 强制在 id 变化时重新挂载 diff --git a/components/home/lovart-home.tsx b/components/home/lovart-home.tsx index 5c9565254..6751449a0 100644 --- a/components/home/lovart-home.tsx +++ b/components/home/lovart-home.tsx @@ -2,7 +2,6 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useRouter } from "next/navigation"; -import { signIn, signOut } from "next-auth/react"; import { useTheme } from "next-themes"; import useSWR from "swr"; import useSWRInfinite from "swr/infinite"; @@ -60,7 +59,6 @@ import { getCanvasLaunchStorageKey, } from "@/lib/canvas-launch"; import type { MembershipConfig } from "@/lib/membership/config"; -import { fetchUploadWithRetry } from "@/lib/client/upload-with-retry"; import { normalizeLovartHomeSidebarConfig, type LovartHomeSidebarItemKey, @@ -465,6 +463,7 @@ export function LovartHome({ const [previewCursorVisible, setPreviewCursorVisible] = useState(true); const [isSecondaryDataReady, setIsSecondaryDataReady] = useState(false); const [isFeaturedDataReady, setIsFeaturedDataReady] = useState(false); + const [isHomeAnimationReady, setIsHomeAnimationReady] = useState(false); const projectSectionRef = useRef(null); const featuredSectionRef = useRef(null); const featuredLoadMoreRef = useRef(null); @@ -535,6 +534,31 @@ export function LovartHome({ }; }, [isFeaturedDataReady]); + useEffect(() => { + if (typeof window === "undefined" || !isSecondaryDataReady) { + return; + } + + const loadHomeAnimation = () => setIsHomeAnimationReady(true); + const idleWindow = window as Window & { + cancelIdleCallback?: (handle: number) => void; + requestIdleCallback?: ( + callback: IdleRequestCallback, + options?: IdleRequestOptions + ) => number; + }; + + if (typeof idleWindow.requestIdleCallback === "function") { + const idleId = idleWindow.requestIdleCallback(loadHomeAnimation, { + timeout: 3200, + }); + return () => idleWindow.cancelIdleCallback?.(idleId); + } + + const timer = window.setTimeout(loadHomeAnimation, 1800); + return () => window.clearTimeout(timer); + }, [isSecondaryDataReady]); + const { data: canvasData, mutate: mutateCanvas } = useSWR<{ success: boolean; data: CanvasProject[]; @@ -671,7 +695,10 @@ export function LovartHome({ enabled?: boolean; htmlContent?: string; fileName?: string; - }>(isSecondaryDataReady ? "/api/home-animation" : null, fetcher); + }>( + isSecondaryDataReady && isHomeAnimationReady ? "/api/home-animation" : null, + fetcher + ); const membershipConfig = membershipData?.config; const membershipPlans = membershipConfig?.plans || []; @@ -896,6 +923,7 @@ export function LovartHome({ } catch (_error) {} try { + const { signOut } = await import("next-auth/react"); await signOut({ callbackUrl: "/", redirect: false, @@ -1109,6 +1137,9 @@ export function LovartHome({ }; const uploadSelectedFiles = async (): Promise => { + const { fetchUploadWithRetry } = await import( + "@/lib/client/upload-with-retry" + ); const result: CanvasLaunchAttachment[] = []; for (const item of files) { @@ -1146,6 +1177,7 @@ export function LovartHome({ setIsLoggingIn(true); try { + const { signIn } = await import("next-auth/react"); const result = await signIn("credentials", { email, password: loginPassword, @@ -1171,6 +1203,7 @@ export function LovartHome({ toast.error("Google 登录未启用"); return; } + const { signIn } = await import("next-auth/react"); await signIn("google", { callbackUrl: "/" }); }; @@ -1226,6 +1259,9 @@ export function LovartHome({ const blob = await response.blob(); const formData = new FormData(); formData.append("file", blob, `edited-${Date.now()}.png`); + const { fetchUploadWithRetry } = await import( + "@/lib/client/upload-with-retry" + ); const uploadResponse = await fetchUploadWithRetry(formData); if (!uploadResponse.ok) { @@ -1802,7 +1838,9 @@ export function LovartHome({ heroPromoText={heroPromoText} heroSubtitleText={heroSubtitleText} heroTitleText={heroTitleText} - homeAnimationData={homeAnimationData} + homeAnimationData={ + isHomeAnimationReady ? homeAnimationData : undefined + } isAgentModeActive={isComposerAgentModeActive} isMembershipEnabled={isMembershipEnabled} isSubmitting={isSubmitting} diff --git a/components/home/lovart/lovart-featured-gallery.tsx b/components/home/lovart/lovart-featured-gallery.tsx index f038c1dd1..456f594d6 100644 --- a/components/home/lovart/lovart-featured-gallery.tsx +++ b/components/home/lovart/lovart-featured-gallery.tsx @@ -62,6 +62,8 @@ export function LovartFeaturedGallery({ {item.prompt
diff --git a/components/home/lovart/lovart-hero-composer.tsx b/components/home/lovart/lovart-hero-composer.tsx index c5f491c59..9f5e73568 100644 --- a/components/home/lovart/lovart-hero-composer.tsx +++ b/components/home/lovart/lovart-hero-composer.tsx @@ -1,6 +1,6 @@ +import dynamic from "next/dynamic"; import { ArrowUp, Box, Lightbulb, Loader2, Plus, Sparkles } from "lucide-react"; import type { ChangeEvent } from "react"; -import { HomeAnimationEmbed } from "@/components/home/home-animation-embed"; import { Button } from "@/components/ui/button"; import { Select, @@ -12,6 +12,19 @@ import { Textarea } from "@/components/ui/textarea"; import { cn } from "@/lib/utils"; import type { ModelOption, UploadFileItem } from "./types"; +const HomeAnimationEmbed = dynamic( + () => + import("@/components/home/home-animation-embed").then( + (module) => module.HomeAnimationEmbed + ), + { + loading: () => ( +
+ ), + ssr: false, + } +); + type LovartHeroComposerProps = { composerPlaceholder: string; displayLogo: string; diff --git a/components/home/lovart/lovart-projects-section.tsx b/components/home/lovart/lovart-projects-section.tsx index 36ab66f87..28dad8ecf 100644 --- a/components/home/lovart/lovart-projects-section.tsx +++ b/components/home/lovart/lovart-projects-section.tsx @@ -76,6 +76,8 @@ export function LovartProjectsSection({ {item.title} ) : ( From ebfc7589a99b8db89770b8bec727c00b63fdc065 Mon Sep 17 00:00:00 2001 From: 134 Date: Tue, 7 Jul 2026 22:26:04 +0800 Subject: [PATCH 36/70] feat: add global claw chat history --- app/api/canvas/clawd-chat/history/route.ts | 42 +++ components/canvas/CanvasChatPanel.tsx | 321 +++++++++++++++++---- lib/db/queries.ts | 38 +++ 3 files changed, 341 insertions(+), 60 deletions(-) create mode 100644 app/api/canvas/clawd-chat/history/route.ts diff --git a/app/api/canvas/clawd-chat/history/route.ts b/app/api/canvas/clawd-chat/history/route.ts new file mode 100644 index 000000000..822587749 --- /dev/null +++ b/app/api/canvas/clawd-chat/history/route.ts @@ -0,0 +1,42 @@ +import { NextResponse } from "next/server"; +import { auth } from "@/app/(auth)/auth"; +import { getClawdChatsByUserId } from "@/lib/db/queries"; + +export const dynamic = "force-dynamic"; +export const revalidate = 0; + +export async function GET(request: Request) { + const session = await auth().catch(() => null); + + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const url = new URL(request.url); + const limit = Math.max( + 1, + Math.min(100, Math.round(Number(url.searchParams.get("limit") || 50))) + ); + + const chats = await getClawdChatsByUserId({ + userId: session.user.id, + limit, + }); + + return NextResponse.json( + { + chats: chats.map((chat) => ({ + id: chat.id, + title: chat.title, + createdAt: chat.createdAt?.toISOString?.() || chat.createdAt, + updatedAt: chat.updatedAt?.toISOString?.() || chat.updatedAt, + messageCount: Number(chat.messageCount || 0), + })), + }, + { + headers: { + "Cache-Control": "no-store, max-age=0", + }, + } + ); +} diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index 292dc228d..24e6b11e8 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -1971,6 +1971,14 @@ interface CanvasChatSession { clawdSkillTagPromptApplied?: boolean; } +interface ClawdHistoryItem { + id: string; + title: string; + createdAt: string | Date; + updatedAt: string | Date; + messageCount: number; +} + const CANVAS_CHAT_QUERY_PARAM = "chat"; const CANVAS_IMAGE_NODE_MANUAL_UPLOAD_EVENT = "canvas:image-node-manual-upload"; @@ -2194,6 +2202,7 @@ export function CanvasChatPanel({ const inFlightRawChatIdRef = useRef(null); const [chatSessions, setChatSessions] = useState([]); const [isSessionMenuOpen, setIsSessionMenuOpen] = useState(false); + const [isClawdHistoryMenuOpen, setIsClawdHistoryMenuOpen] = useState(false); const [isComposerModeMenuOpen, setIsComposerModeMenuOpen] = useState(false); const [selectedSkillTagId, setSelectedSkillTagId] = useState(null); const [areCanvasSkillTagsExpanded, setAreCanvasSkillTagsExpanded] = useState(false); @@ -2239,6 +2248,7 @@ export function CanvasChatPanel({ ); const hasInitializedVideoWorkspaceRef = useRef(false); const sessionMenuRef = useRef(null); + const clawdHistoryMenuRef = useRef(null); const historyScrollTimerRef = useRef | null>(null); const forceHistoryHydrationRef = useRef(null); const copiedUserMessageRef = useRef(null); @@ -2963,6 +2973,17 @@ export function CanvasChatPanel({ return () => document.removeEventListener("mousedown", handleClickOutside); }, [isSessionMenuOpen]); + useEffect(() => { + if (!isClawdHistoryMenuOpen) return; + const handleClickOutside = (event: MouseEvent) => { + if (!clawdHistoryMenuRef.current?.contains(event.target as Node)) { + setIsClawdHistoryMenuOpen(false); + } + }; + document.addEventListener("mousedown", handleClickOutside); + return () => document.removeEventListener("mousedown", handleClickOutside); + }, [isClawdHistoryMenuOpen]); + useEffect(() => { const sessionStorageKey = getSessionStorageKey(canvasId, chatScope); if (loadedChatSessionStorageKey !== sessionStorageKey) return; @@ -4545,6 +4566,19 @@ export function CanvasChatPanel({ activeChatSession, activeComposerMode ); + const clawdHistorySWRKey = + isClawdMode && isClawdHistoryMenuOpen + ? "/api/canvas/clawd-chat/history?limit=80" + : null; + const { + data: clawdHistoryData, + isLoading: isClawdHistoryLoading, + mutate: mutateClawdHistory, + } = useSWR<{ chats: ClawdHistoryItem[] }>(clawdHistorySWRKey, fetcher, { + revalidateOnFocus: false, + revalidateOnReconnect: false, + dedupingInterval: 5000, + }); useEffect(() => { if (activeChatSessionMode !== "clawd") return; if (!isClawdModeEnabled) return; @@ -10785,6 +10819,7 @@ export function CanvasChatPanel({ if (rawChatId) { void mutate(buildHistoryMessagesUrl(rawChatId, { cacheBust: true })); } + void mutateClawdHistory(); hasHydratedHistoryRef.current = true; } catch (error) { console.error("[CanvasChat] Claw request failed:", error); @@ -11203,6 +11238,7 @@ export function CanvasChatPanel({ shouldBlockDuplicateSend, status, mutate, + mutateClawdHistory, markSessionClawdSkillTagPromptApplied, updateChatSessions, buildHistoryMessagesUrl, @@ -12598,6 +12634,70 @@ export function CanvasChatPanel({ toast.success("已切换聊天窗口"); }, [activeChatId, releaseSendGuard, replaceChatSessionQuery]); + const switchToClawdHistorySession = useCallback( + (historyItem: ClawdHistoryItem) => { + const sessionId = normalizeCanvasChatSessionId(historyItem.id); + if (!sessionId) return; + + const updatedAt = Date.parse(String(historyItem.updatedAt || "")) || Date.now(); + const createdAt = Date.parse(String(historyItem.createdAt || "")) || updatedAt; + const title = String(historyItem.title || "").trim() || `${CLAW_MODE_LABEL} 对话`; + + updateChatSessions((prevSessions) => { + const currentIndex = prevSessions.findIndex( + (session) => session.id === sessionId + ); + + if (currentIndex < 0) { + return [ + { + id: sessionId, + title, + mode: "clawd", + createdAt, + updatedAt, + }, + ...prevSessions, + ]; + } + + const nextSessions = [...prevSessions]; + nextSessions[currentIndex] = { + ...nextSessions[currentIndex], + title, + mode: "clawd", + updatedAt: Math.max( + Number(nextSessions[currentIndex]?.updatedAt) || 0, + updatedAt + ), + }; + nextSessions.sort((a, b) => b.updatedAt - a.updatedAt); + return nextSessions; + }); + + forceHistoryHydrationRef.current = toRawCanvasChatId(sessionId); + releaseSendGuard(activeChatId); + setActiveChatId(sessionId); + setComposerModeOverride("clawd"); + setIsDetailGenerationMode(false); + replaceChatSessionQuery(sessionId); + setIsClawdHistoryMenuOpen(false); + toast.success("已打开 Claw 历史对话"); + }, + [activeChatId, releaseSendGuard, replaceChatSessionQuery, updateChatSessions] + ); + + const formatClawdHistoryTime = useCallback((value: string | Date) => { + const timestamp = Date.parse(String(value || "")); + if (!Number.isFinite(timestamp)) return ""; + return new Date(timestamp).toLocaleString("zh-CN", { + month: "2-digit", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + }); + }, []); + const activeSessionTitle = useMemo(() => { const current = chatSessions.find((session) => session.id === activeChatId); return String(current?.title || "").trim(); @@ -12726,72 +12826,173 @@ export function CanvasChatPanel({ > -
- - {isSessionMenuOpen && ( -
+ - ); - }) + + Claw + + {isClawdHistoryMenuOpen && ( +
+
+
+ Claw 对话历史 +
+ +
+
+ {isClawdHistoryLoading ? ( +
+ + 正在加载 Claw 历史 +
+ ) : (clawdHistoryData?.chats || []).length === 0 ? ( +
+ 暂无 Claw 历史 +
+ ) : ( + (clawdHistoryData?.chats || []).map((item) => { + const sessionId = normalizeCanvasChatSessionId(item.id); + const isActive = sessionId === activeChatId; + const timeLabel = formatClawdHistoryTime(item.updatedAt); + return ( + + ); + }) + )} +
-
- )} -
+ )} +
+ ) : ( +
+ + {isSessionMenuOpen && ( +
+
聊天窗口记录
+
+ {currentModeChatSessions.length === 0 ? ( +
暂无记录
+ ) : ( + currentModeChatSessions.map((session, index) => { + const isActive = session.id === activeChatId; + const timeLabel = new Date(session.updatedAt).toLocaleTimeString([], { + hour: "2-digit", + minute: "2-digit", + }); + return ( + + ); + }) + )} +
+
+ )} +
+ )}
) : null}
- {!areSecondaryPanelsReady ? ( + {!isPanelOpen ? null : !areSecondaryPanelsReady ? ( ) : panelMode === "chat" ? ( // AI 对话内容 - 使用 key 强制在 id 变化时重新挂载 diff --git a/components/home/lovart-home.tsx b/components/home/lovart-home.tsx index 5c9565254..6751449a0 100644 --- a/components/home/lovart-home.tsx +++ b/components/home/lovart-home.tsx @@ -2,7 +2,6 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useRouter } from "next/navigation"; -import { signIn, signOut } from "next-auth/react"; import { useTheme } from "next-themes"; import useSWR from "swr"; import useSWRInfinite from "swr/infinite"; @@ -60,7 +59,6 @@ import { getCanvasLaunchStorageKey, } from "@/lib/canvas-launch"; import type { MembershipConfig } from "@/lib/membership/config"; -import { fetchUploadWithRetry } from "@/lib/client/upload-with-retry"; import { normalizeLovartHomeSidebarConfig, type LovartHomeSidebarItemKey, @@ -465,6 +463,7 @@ export function LovartHome({ const [previewCursorVisible, setPreviewCursorVisible] = useState(true); const [isSecondaryDataReady, setIsSecondaryDataReady] = useState(false); const [isFeaturedDataReady, setIsFeaturedDataReady] = useState(false); + const [isHomeAnimationReady, setIsHomeAnimationReady] = useState(false); const projectSectionRef = useRef(null); const featuredSectionRef = useRef(null); const featuredLoadMoreRef = useRef(null); @@ -535,6 +534,31 @@ export function LovartHome({ }; }, [isFeaturedDataReady]); + useEffect(() => { + if (typeof window === "undefined" || !isSecondaryDataReady) { + return; + } + + const loadHomeAnimation = () => setIsHomeAnimationReady(true); + const idleWindow = window as Window & { + cancelIdleCallback?: (handle: number) => void; + requestIdleCallback?: ( + callback: IdleRequestCallback, + options?: IdleRequestOptions + ) => number; + }; + + if (typeof idleWindow.requestIdleCallback === "function") { + const idleId = idleWindow.requestIdleCallback(loadHomeAnimation, { + timeout: 3200, + }); + return () => idleWindow.cancelIdleCallback?.(idleId); + } + + const timer = window.setTimeout(loadHomeAnimation, 1800); + return () => window.clearTimeout(timer); + }, [isSecondaryDataReady]); + const { data: canvasData, mutate: mutateCanvas } = useSWR<{ success: boolean; data: CanvasProject[]; @@ -671,7 +695,10 @@ export function LovartHome({ enabled?: boolean; htmlContent?: string; fileName?: string; - }>(isSecondaryDataReady ? "/api/home-animation" : null, fetcher); + }>( + isSecondaryDataReady && isHomeAnimationReady ? "/api/home-animation" : null, + fetcher + ); const membershipConfig = membershipData?.config; const membershipPlans = membershipConfig?.plans || []; @@ -896,6 +923,7 @@ export function LovartHome({ } catch (_error) {} try { + const { signOut } = await import("next-auth/react"); await signOut({ callbackUrl: "/", redirect: false, @@ -1109,6 +1137,9 @@ export function LovartHome({ }; const uploadSelectedFiles = async (): Promise => { + const { fetchUploadWithRetry } = await import( + "@/lib/client/upload-with-retry" + ); const result: CanvasLaunchAttachment[] = []; for (const item of files) { @@ -1146,6 +1177,7 @@ export function LovartHome({ setIsLoggingIn(true); try { + const { signIn } = await import("next-auth/react"); const result = await signIn("credentials", { email, password: loginPassword, @@ -1171,6 +1203,7 @@ export function LovartHome({ toast.error("Google 登录未启用"); return; } + const { signIn } = await import("next-auth/react"); await signIn("google", { callbackUrl: "/" }); }; @@ -1226,6 +1259,9 @@ export function LovartHome({ const blob = await response.blob(); const formData = new FormData(); formData.append("file", blob, `edited-${Date.now()}.png`); + const { fetchUploadWithRetry } = await import( + "@/lib/client/upload-with-retry" + ); const uploadResponse = await fetchUploadWithRetry(formData); if (!uploadResponse.ok) { @@ -1802,7 +1838,9 @@ export function LovartHome({ heroPromoText={heroPromoText} heroSubtitleText={heroSubtitleText} heroTitleText={heroTitleText} - homeAnimationData={homeAnimationData} + homeAnimationData={ + isHomeAnimationReady ? homeAnimationData : undefined + } isAgentModeActive={isComposerAgentModeActive} isMembershipEnabled={isMembershipEnabled} isSubmitting={isSubmitting} diff --git a/components/home/lovart/lovart-featured-gallery.tsx b/components/home/lovart/lovart-featured-gallery.tsx index f038c1dd1..456f594d6 100644 --- a/components/home/lovart/lovart-featured-gallery.tsx +++ b/components/home/lovart/lovart-featured-gallery.tsx @@ -62,6 +62,8 @@ export function LovartFeaturedGallery({ {item.prompt
diff --git a/components/home/lovart/lovart-hero-composer.tsx b/components/home/lovart/lovart-hero-composer.tsx index c5f491c59..9f5e73568 100644 --- a/components/home/lovart/lovart-hero-composer.tsx +++ b/components/home/lovart/lovart-hero-composer.tsx @@ -1,6 +1,6 @@ +import dynamic from "next/dynamic"; import { ArrowUp, Box, Lightbulb, Loader2, Plus, Sparkles } from "lucide-react"; import type { ChangeEvent } from "react"; -import { HomeAnimationEmbed } from "@/components/home/home-animation-embed"; import { Button } from "@/components/ui/button"; import { Select, @@ -12,6 +12,19 @@ import { Textarea } from "@/components/ui/textarea"; import { cn } from "@/lib/utils"; import type { ModelOption, UploadFileItem } from "./types"; +const HomeAnimationEmbed = dynamic( + () => + import("@/components/home/home-animation-embed").then( + (module) => module.HomeAnimationEmbed + ), + { + loading: () => ( +
+ ), + ssr: false, + } +); + type LovartHeroComposerProps = { composerPlaceholder: string; displayLogo: string; diff --git a/components/home/lovart/lovart-projects-section.tsx b/components/home/lovart/lovart-projects-section.tsx index 36ab66f87..28dad8ecf 100644 --- a/components/home/lovart/lovart-projects-section.tsx +++ b/components/home/lovart/lovart-projects-section.tsx @@ -76,6 +76,8 @@ export function LovartProjectsSection({ {item.title} ) : ( From 0405113d521a685495c5b7c5fe39f99cdd3fdca8 Mon Sep 17 00:00:00 2001 From: 134 Date: Tue, 7 Jul 2026 22:50:06 +0800 Subject: [PATCH 38/70] fix: stabilize scale adjust fal timeout and uploads --- lib/ai/providers/fal-image.ts | 12 ++++++- lib/canvas/generate-image.ts | 66 ++++++++++++++++++++++++++-------- lib/canvas/image-task-queue.ts | 2 +- 3 files changed, 63 insertions(+), 17 deletions(-) diff --git a/lib/ai/providers/fal-image.ts b/lib/ai/providers/fal-image.ts index 930815fe8..1c2b28bdf 100644 --- a/lib/ai/providers/fal-image.ts +++ b/lib/ai/providers/fal-image.ts @@ -28,6 +28,7 @@ interface FalImageParams { systemPrompt?: string; thinkingLevel?: string; syncMode?: boolean; + timeoutMs?: number; } interface UploadedImage { @@ -50,6 +51,8 @@ const DEFAULT_RESOLUTION = "1K"; const DEFAULT_OUTPUT_FORMAT = "png"; const DEFAULT_SAFETY_TOLERANCE = "4"; const DEFAULT_TIMEOUT_MS = 600000; +const MIN_TIMEOUT_MS = 30_000; +const MAX_TIMEOUT_MS = 600_000; const FAL_GENERIC_MAX_EDGE = 4096; const FAL_GENERIC_MAX_PIXELS = 4096 * 4096; const GPT_IMAGE_2_MAX_EDGE = 3840; @@ -434,6 +437,12 @@ function normalizeOptionalHttpUrl(value?: string) { return /^https?:\/\//i.test(normalized) ? normalized : ""; } +function normalizeTimeoutMs(value: unknown) { + const numeric = Number(value); + if (!Number.isFinite(numeric) || numeric <= 0) return DEFAULT_TIMEOUT_MS; + return Math.max(MIN_TIMEOUT_MS, Math.min(MAX_TIMEOUT_MS, Math.round(numeric))); +} + function detectImageExtension(imageUrl: string, contentType?: string | null) { const type = (contentType || "").toLowerCase(); if (type.includes("jpeg") || type.includes("jpg")) return "jpg"; @@ -911,13 +920,14 @@ export function createFalImage({ apiKey, baseURL }: FalImageConfig) { }); const fal = await createFalClientFromApiKey(apiKey); + const timeoutMs = normalizeTimeoutMs(options.imageParams?.timeoutMs); let result: any; try { result = await fal.subscribe(request.falEndpointId, { input: request.input, logs: true, - timeout: DEFAULT_TIMEOUT_MS, + timeout: timeoutMs, onQueueUpdate: (update: any) => { if (update?.status === "IN_PROGRESS" && Array.isArray(update.logs)) { for (const log of update.logs) { diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index 79d9b70a0..51f07b37a 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -292,6 +292,15 @@ function getCanvasProviderRuntimeParams(params: { }; } + if ( + params.providerType === "fal-image" && + isCanvasScaleAdjustDebugRequest(params.debugLabel, params.debugMeta) + ) { + return { + timeoutMs: CANVAS_SCALE_ADJUST_FINAL_PROVIDER_BATCH_TIMEOUT_MS, + }; + } + return {}; } @@ -3059,6 +3068,13 @@ async function applyScaleAdjustPatchCompositeToImages(params: { touchesTop, width: workspaceBox.width, }); + const staticPatchMaskRaw = staticPatchMask + ? await normalizeOpacityMaskToRaw({ + height: workspaceBox.height, + maskBuffer: staticPatchMask, + width: workspaceBox.width, + }) + : null; const regionClipPatchMask = maskRegionBoxes?.length ? buildScaleAdjustPatchCompositeMaskSvg({ featherPx, @@ -3177,12 +3193,22 @@ async function applyScaleAdjustPatchCompositeToImages(params: { imageBuffer: resizedGeneratedBuffer, width: workspaceBox.width, }) - : await sharp(resizedGeneratedBuffer, { failOn: "none" }) - .ensureAlpha() - .composite([{ input: patchMask, blend: "dest-in" }]) - .png({ compressionLevel: 6 }) - .toBuffer(); - const mergedBuffer = await sharp(sourceBuffer, { failOn: "none" }) + : activeClipPatchMaskRaw || staticPatchMaskRaw + ? await applyRawAlphaToImage({ + alphaMaskRaw: + activeClipPatchMaskRaw && staticPatchMaskRaw + ? multiplyAlphaMasks(staticPatchMaskRaw, activeClipPatchMaskRaw) + : activeClipPatchMaskRaw || staticPatchMaskRaw!, + height: workspaceBox.height, + imageBuffer: resizedGeneratedBuffer, + width: workspaceBox.width, + }) + : await sharp(resizedGeneratedBuffer, { failOn: "none" }) + .ensureAlpha() + .composite([{ input: patchMask, blend: "dest-in" }]) + .png({ compressionLevel: 6 }) + .toBuffer(); + const mergedPngBuffer = await sharp(sourceBuffer, { failOn: "none" }) .ensureAlpha() .composite([ { @@ -3193,20 +3219,25 @@ async function applyScaleAdjustPatchCompositeToImages(params: { ]) .png({ compressionLevel: 6 }) .toBuffer(); - const thumbnailBuffer = await sharp(mergedBuffer, { failOn: "none" }) + const mergedBuffer = await sharp(mergedPngBuffer, { failOn: "none" }) + .flatten({ background: "#ffffff" }) + .jpeg({ quality: 94, mozjpeg: true }) + .toBuffer(); + const thumbnailBuffer = await sharp(mergedPngBuffer, { failOn: "none" }) + .flatten({ background: "#ffffff" }) .resize(512, 512, { fit: "inside", withoutEnlargement: true }) - .png({ compressionLevel: 6 }) + .jpeg({ quality: 88, mozjpeg: true }) .toBuffer(); const suffix = `${Date.now()}-${index + 1}-${Math.random() .toString(36) .slice(2, 8)}`; const [originalUpload, thumbnailUpload] = await Promise.all([ uploadFile( - `canvas-scale-adjust-ai-patch-composited-${suffix}.png`, + `canvas-scale-adjust-ai-patch-composited-${suffix}.jpg`, bufferToArrayBuffer(mergedBuffer) ), uploadFile( - `canvas-scale-adjust-ai-patch-composited-thumb-${suffix}.png`, + `canvas-scale-adjust-ai-patch-composited-thumb-${suffix}.jpg`, bufferToArrayBuffer(thumbnailBuffer) ), ]); @@ -3293,26 +3324,31 @@ async function applyScaleAdjustProductLockCompositeToImages(params: { width: generatedWidth, }) : resizedOverlayBaseBuffer; - const mergedBuffer = await sharp(normalizedGeneratedBuffer, { + const mergedPngBuffer = await sharp(normalizedGeneratedBuffer, { failOn: "none", }) .composite([{ input: resizedOverlayBuffer, left: 0, top: 0 }]) .png({ compressionLevel: 6 }) .toBuffer(); - const thumbnailBuffer = await sharp(mergedBuffer, { failOn: "none" }) + const mergedBuffer = await sharp(mergedPngBuffer, { failOn: "none" }) + .flatten({ background: "#ffffff" }) + .jpeg({ quality: 94, mozjpeg: true }) + .toBuffer(); + const thumbnailBuffer = await sharp(mergedPngBuffer, { failOn: "none" }) + .flatten({ background: "#ffffff" }) .resize(512, 512, { fit: "inside", withoutEnlargement: true }) - .png({ compressionLevel: 6 }) + .jpeg({ quality: 88, mozjpeg: true }) .toBuffer(); const suffix = `${Date.now()}-${index + 1}-${Math.random() .toString(36) .slice(2, 8)}`; const [originalUpload, thumbnailUpload] = await Promise.all([ uploadFile( - `canvas-scale-adjust-product-lock-${suffix}.png`, + `canvas-scale-adjust-product-lock-${suffix}.jpg`, bufferToArrayBuffer(mergedBuffer) ), uploadFile( - `canvas-scale-adjust-product-lock-thumb-${suffix}.png`, + `canvas-scale-adjust-product-lock-thumb-${suffix}.jpg`, bufferToArrayBuffer(thumbnailBuffer) ), ]); diff --git a/lib/canvas/image-task-queue.ts b/lib/canvas/image-task-queue.ts index 21fbd0b0d..02ea5b804 100644 --- a/lib/canvas/image-task-queue.ts +++ b/lib/canvas/image-task-queue.ts @@ -59,7 +59,7 @@ interface QueueState { } const DEFAULT_RUNNING_TIMEOUT_MS = 10 * 60 * 1000; -const SCALE_ADJUST_RUNNING_TIMEOUT_MS = 15 * 60 * 1000; +const SCALE_ADJUST_RUNNING_TIMEOUT_MS = 8 * 60 * 1000; const LOCAL_ACTIVE_RUNNING_HARD_TIMEOUT_MS = 30 * 60 * 1000; const PENDING_LOOKAHEAD_LIMIT = 100; const QUEUE_STATUS_PENDING_LOOKAHEAD_LIMIT = 500; From 30eef41dc4248b58f474201adffe4fa3fe140189 Mon Sep 17 00:00:00 2001 From: 134 Date: Tue, 7 Jul 2026 23:09:38 +0800 Subject: [PATCH 39/70] fix: restore longer s3 upload timeout --- lib/storage/service.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/storage/service.ts b/lib/storage/service.ts index 6e3b130c9..96ae55bea 100644 --- a/lib/storage/service.ts +++ b/lib/storage/service.ts @@ -45,7 +45,7 @@ export type StorageUploadFunction = ( let lastKnownGoodDbS3Config: S3UploadConfig | null = null; const STORAGE_FAILOVER_CONFIG_KEY = "storage_failover_config"; -const DEFAULT_S3_REQUEST_TIMEOUT_MS = 30_000; +const DEFAULT_S3_REQUEST_TIMEOUT_MS = 180_000; function getStorageTimeoutMs(name: string, fallback: number, min: number, max: number) { const parsed = Number(process.env[name]); @@ -262,7 +262,7 @@ async function uploadToS3( "S3_UPLOAD_REQUEST_TIMEOUT_MS", DEFAULT_S3_REQUEST_TIMEOUT_MS, 5_000, - 120_000 + 300_000 ), "S3 upload" ); From 17f54855226d41b5ec98d2b70faf86a6839b1299 Mon Sep 17 00:00:00 2001 From: 134 Date: Tue, 7 Jul 2026 23:40:34 +0800 Subject: [PATCH 40/70] fix: harden scale adjust product lock uploads --- app/api/canvas/local-edit/route.ts | 205 +++++++++++++++-------------- lib/storage/service.ts | 86 ++++++++++++ 2 files changed, 189 insertions(+), 102 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 760f49063..1aa59b6db 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -3828,57 +3828,66 @@ async function createScaleAdjustPatchReference(params: { }); const targetDraftBuffer = targetDraftReference.buffer; const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + const targetDraftUploadBuffer = await sharp(targetDraftBuffer, { failOn: "none" }) + .flatten({ background: "#ffffff" }) + .jpeg({ quality: 90, mozjpeg: true }) + .toBuffer(); + const backgroundPlateUploadBuffer = targetDraftReference.backgroundPlateBuffer + ? await sharp(targetDraftReference.backgroundPlateBuffer, { failOn: "none" }) + .flatten({ background: "#ffffff" }) + .jpeg({ quality: 90, mozjpeg: true }) + .toBuffer() + : null; + const uploadOptionalScaleAdjustAsset = async ( + filename: string, + buffer: Buffer | null | undefined + ) => { + if (!buffer) return null; + try { + return await uploadFile(filename, toArrayBuffer(buffer)); + } catch (error) { + console.warn("[Canvas Local Edit] Optional scale adjust asset upload failed:", { + error: error instanceof Error ? error.message : String(error), + filename, + }); + return null; + } + }; if (params.fullFrameReferenceOnly && structureLockAppliedRef.value) { - const [ - targetDraftFullUpload, - backgroundPlateFullUpload, - productLockOverlayUpload, - productLockProtectMaskUpload, - compositeClipMaskUpload, - productLockEditMaskUpload, - productLockFalEditMaskUpload, - ] = await Promise.all([ - uploadFile( - `canvas-scale-adjust-full-target-draft-${suffix}.png`, - toArrayBuffer(targetDraftBuffer) - ), - targetDraftReference.backgroundPlateBuffer - ? uploadFile( - `canvas-scale-adjust-background-plate-${suffix}.png`, - toArrayBuffer(targetDraftReference.backgroundPlateBuffer) - ) - : Promise.resolve(null), + const targetDraftFullUpload = await uploadFile( + `canvas-scale-adjust-full-target-draft-${suffix}.jpg`, + toArrayBuffer(targetDraftUploadBuffer) + ); + const productLockOverlayUpload = await uploadOptionalScaleAdjustAsset( + `canvas-scale-adjust-product-lock-overlay-${suffix}.png`, targetDraftReference.productLockOverlayBuffer - ? uploadFile( - `canvas-scale-adjust-product-lock-overlay-${suffix}.png`, - toArrayBuffer(targetDraftReference.productLockOverlayBuffer) - ) - : Promise.resolve(null), - targetDraftReference.productLockProtectMaskBuffer - ? uploadFile( - `canvas-scale-adjust-product-lock-protect-mask-${suffix}.png`, - toArrayBuffer(targetDraftReference.productLockProtectMaskBuffer) - ) - : Promise.resolve(null), - targetDraftReference.compositeClipMaskBuffer - ? uploadFile( - `canvas-scale-adjust-composite-clip-mask-${suffix}.png`, - toArrayBuffer(targetDraftReference.compositeClipMaskBuffer) - ) - : Promise.resolve(null), - targetDraftReference.productLockEditMaskBuffer - ? uploadFile( - `canvas-scale-adjust-product-lock-edit-mask-${suffix}.png`, - toArrayBuffer(targetDraftReference.productLockEditMaskBuffer) - ) - : Promise.resolve(null), - targetDraftReference.productLockFalEditMaskBuffer - ? uploadFile( - `canvas-scale-adjust-product-lock-fal-edit-mask-${suffix}.png`, - toArrayBuffer(targetDraftReference.productLockFalEditMaskBuffer) - ) - : Promise.resolve(null), - ]); + ); + const [productLockEditMaskUpload, productLockFalEditMaskUpload] = + await Promise.all([ + uploadOptionalScaleAdjustAsset( + `canvas-scale-adjust-product-lock-edit-mask-${suffix}.png`, + targetDraftReference.productLockEditMaskBuffer + ), + uploadOptionalScaleAdjustAsset( + `canvas-scale-adjust-product-lock-fal-edit-mask-${suffix}.png`, + targetDraftReference.productLockFalEditMaskBuffer + ), + ]); + const [backgroundPlateFullUpload, productLockProtectMaskUpload, compositeClipMaskUpload] = + await Promise.all([ + uploadOptionalScaleAdjustAsset( + `canvas-scale-adjust-background-plate-${suffix}.jpg`, + backgroundPlateUploadBuffer + ), + uploadOptionalScaleAdjustAsset( + `canvas-scale-adjust-product-lock-protect-mask-${suffix}.png`, + targetDraftReference.productLockProtectMaskBuffer + ), + uploadOptionalScaleAdjustAsset( + `canvas-scale-adjust-composite-clip-mask-${suffix}.png`, + targetDraftReference.compositeClipMaskBuffer + ), + ]); return { backgroundPlateFullUrl: backgroundPlateFullUpload?.url || null, @@ -3907,10 +3916,22 @@ async function createScaleAdjustPatchReference(params: { .png({ compressionLevel: 6 }) .toBuffer(), ]); + const [sourceCropUpload, targetDraftUpload, targetDraftFullUpload] = + await Promise.all([ + uploadFile( + `canvas-scale-adjust-workspace-source-${suffix}.png`, + toArrayBuffer(sourceCropBuffer) + ), + uploadFile( + `canvas-scale-adjust-workspace-target-draft-${suffix}.png`, + toArrayBuffer(targetDraftCropBuffer) + ), + uploadFile( + `canvas-scale-adjust-full-target-draft-${suffix}.jpg`, + toArrayBuffer(targetDraftUploadBuffer) + ), + ]); const [ - sourceCropUpload, - targetDraftUpload, - targetDraftFullUpload, backgroundPlateFullUpload, productLockOverlayUpload, productLockProtectMaskUpload, @@ -3918,54 +3939,30 @@ async function createScaleAdjustPatchReference(params: { productLockEditMaskUpload, productLockFalEditMaskUpload, ] = await Promise.all([ - uploadFile( - `canvas-scale-adjust-workspace-source-${suffix}.png`, - toArrayBuffer(sourceCropBuffer) + uploadOptionalScaleAdjustAsset( + `canvas-scale-adjust-background-plate-${suffix}.jpg`, + backgroundPlateUploadBuffer ), - uploadFile( - `canvas-scale-adjust-workspace-target-draft-${suffix}.png`, - toArrayBuffer(targetDraftCropBuffer) + uploadOptionalScaleAdjustAsset( + `canvas-scale-adjust-product-lock-overlay-${suffix}.png`, + targetDraftReference.productLockOverlayBuffer ), - uploadFile( - `canvas-scale-adjust-full-target-draft-${suffix}.png`, - toArrayBuffer(targetDraftBuffer) + uploadOptionalScaleAdjustAsset( + `canvas-scale-adjust-product-lock-protect-mask-${suffix}.png`, + targetDraftReference.productLockProtectMaskBuffer + ), + uploadOptionalScaleAdjustAsset( + `canvas-scale-adjust-composite-clip-mask-${suffix}.png`, + targetDraftReference.compositeClipMaskBuffer + ), + uploadOptionalScaleAdjustAsset( + `canvas-scale-adjust-product-lock-edit-mask-${suffix}.png`, + targetDraftReference.productLockEditMaskBuffer + ), + uploadOptionalScaleAdjustAsset( + `canvas-scale-adjust-product-lock-fal-edit-mask-${suffix}.png`, + targetDraftReference.productLockFalEditMaskBuffer ), - targetDraftReference.backgroundPlateBuffer - ? uploadFile( - `canvas-scale-adjust-background-plate-${suffix}.png`, - toArrayBuffer(targetDraftReference.backgroundPlateBuffer) - ) - : Promise.resolve(null), - targetDraftReference.productLockOverlayBuffer - ? uploadFile( - `canvas-scale-adjust-product-lock-overlay-${suffix}.png`, - toArrayBuffer(targetDraftReference.productLockOverlayBuffer) - ) - : Promise.resolve(null), - targetDraftReference.productLockProtectMaskBuffer - ? uploadFile( - `canvas-scale-adjust-product-lock-protect-mask-${suffix}.png`, - toArrayBuffer(targetDraftReference.productLockProtectMaskBuffer) - ) - : Promise.resolve(null), - targetDraftReference.compositeClipMaskBuffer - ? uploadFile( - `canvas-scale-adjust-composite-clip-mask-${suffix}.png`, - toArrayBuffer(targetDraftReference.compositeClipMaskBuffer) - ) - : Promise.resolve(null), - targetDraftReference.productLockEditMaskBuffer - ? uploadFile( - `canvas-scale-adjust-product-lock-edit-mask-${suffix}.png`, - toArrayBuffer(targetDraftReference.productLockEditMaskBuffer) - ) - : Promise.resolve(null), - targetDraftReference.productLockFalEditMaskBuffer - ? uploadFile( - `canvas-scale-adjust-product-lock-fal-edit-mask-${suffix}.png`, - toArrayBuffer(targetDraftReference.productLockFalEditMaskBuffer) - ) - : Promise.resolve(null), ]); return { @@ -5349,14 +5346,18 @@ export async function POST(request: NextRequest) { regions.some((region) => isScaleAdjustFurnitureProductLikePrompt(region.prompt) ); - const useScaleAdjustProductLockComposite = + const canUseScaleAdjustProductLockMaskEdit = useScaleAdjustFullFrameStructurePatch && - Boolean(scaleAdjustPatchReference?.backgroundPlateFullUrl) && - Boolean(scaleAdjustPatchReference?.productLockOverlayUrl); - const useScaleAdjustProductLockMaskEdit = - useScaleAdjustProductLockComposite && isScaleAdjustMaskCapableModel(resolvedScaleAdjustModelEntry) && Boolean(scaleAdjustPatchReference?.productLockEditMaskUrl); + const useScaleAdjustProductLockComposite = + useScaleAdjustFullFrameStructurePatch && + Boolean(scaleAdjustPatchReference?.productLockOverlayUrl) && + (Boolean(scaleAdjustPatchReference?.backgroundPlateFullUrl) || + canUseScaleAdjustProductLockMaskEdit); + const useScaleAdjustProductLockMaskEdit = + useScaleAdjustProductLockComposite && + canUseScaleAdjustProductLockMaskEdit; const useScaleAdjustPatchTargetDraftPrimary = Boolean(scaleAdjustPatchReference); const includeScaleAdjustPatchSourceReference = Boolean(scaleAdjustPatchReference); diff --git a/lib/storage/service.ts b/lib/storage/service.ts index 96ae55bea..b10224424 100644 --- a/lib/storage/service.ts +++ b/lib/storage/service.ts @@ -9,6 +9,7 @@ import { } from "@/lib/db/queries"; import { enqueueStorageMigrationTask, + getEnabledS3StorageConfigs, getPreferredAliyunStorageConfig, isAliyunStorageConfig, } from "@/lib/storage/migration-db"; @@ -491,6 +492,43 @@ async function uploadWithBackupStorageConfig( } } +async function uploadWithAnyEnabledStorageConfig( + filename: string, + buffer: ArrayBuffer, + skipStorageConfigIds: Array +): Promise< + | { + uploaded: { url: string; pathname: string; contentType: string }; + storageConfig?: StorageConfig; + } + | null +> { + const skipped = new Set(skipStorageConfigIds.filter(Boolean) as string[]); + try { + const configs = await getEnabledS3StorageConfigs(); + for (const storageConfig of configs) { + if (skipped.has(storageConfig.id)) continue; + const s3Config = toDbS3UploadConfig(storageConfig); + if (!s3Config) continue; + + try { + logger.debug( + `[Upload] Retrying with enabled S3 config: ${storageConfig.name ?? storageConfig.id}` + ); + const uploaded = await uploadToS3(filename, buffer, s3Config); + lastKnownGoodDbS3Config = s3Config; + return { uploaded, storageConfig }; + } catch (error) { + logger.error("[Upload] Enabled S3 config retry failed:", error); + } + } + } catch (error) { + logger.error("[Upload] Failed to list enabled S3 configs for retry:", error); + } + + return null; +} + // 上传文件到存储服务 export async function uploadFile( filename: string, @@ -546,6 +584,22 @@ export async function uploadFile( return uploadedFromBackup.uploaded; } + const uploadedFromEnabledConfig = await uploadWithAnyEnabledStorageConfig( + filename, + buffer, + [primaryStorageConfigId] + ); + if (uploadedFromEnabledConfig) { + await maybeEnqueueAliyunMigrationTask({ + uploaded: uploadedFromEnabledConfig.uploaded, + storageConfig: uploadedFromEnabledConfig.storageConfig, + migrationConfig, + }).catch((enqueueError) => { + logger.error("[Upload] Failed to enqueue enabled S3 OSS migration task:", enqueueError); + }); + return uploadedFromEnabledConfig.uploaded; + } + if (lastKnownGoodDbS3Config) { try { logger.debug("[Upload] Retrying with last known good database S3 config"); @@ -572,6 +626,22 @@ export async function uploadFile( }); return uploadedFromBackup.uploaded; } + + const uploadedFromEnabledConfig = await uploadWithAnyEnabledStorageConfig( + filename, + buffer, + [primaryStorageConfigId] + ); + if (uploadedFromEnabledConfig) { + await maybeEnqueueAliyunMigrationTask({ + uploaded: uploadedFromEnabledConfig.uploaded, + storageConfig: uploadedFromEnabledConfig.storageConfig, + migrationConfig, + }).catch((enqueueError) => { + logger.error("[Upload] Failed to enqueue enabled S3 OSS migration task:", enqueueError); + }); + return uploadedFromEnabledConfig.uploaded; + } } const envS3Config = buildEnvS3Config(); @@ -651,6 +721,22 @@ export async function createStorageUploadSession(): Promise { + logger.error("[Upload] Failed to enqueue enabled S3 OSS migration task:", migrationError); + }); + return uploadedFromEnabledConfig.uploaded; + } + throw error; } }; From 7b73a9c7f58ba3c0711170220eb5d89ded6c504a Mon Sep 17 00:00:00 2001 From: 134 Date: Wed, 8 Jul 2026 00:09:11 +0800 Subject: [PATCH 41/70] fix: constrain product lock scale repair to source footprint --- app/api/canvas/local-edit/route.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 1aa59b6db..88f6a0432 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -5374,7 +5374,7 @@ export async function POST(request: NextRequest) { scalePercent, { width: sourceWidth, height: sourceHeight }, { - draftInput: useScaleAdjustProductLockMaskEdit, + draftInput: false, maskEdit: useScaleAdjustProductLockMaskEdit, } ) @@ -5449,8 +5449,7 @@ export async function POST(request: NextRequest) { ? useScaleAdjustProductLockComposite ? useScaleAdjustProductLockMaskEdit ? [ - scaleAdjustPatchReference.targetDraftFullUrl || - sourceImageUrl, + sourceImageUrl, ...scaleAdjustCropReferenceImageUrls, ...manualRegionReferenceImageUrls, ] @@ -5784,7 +5783,7 @@ export async function POST(request: NextRequest) { featherRatio: 0.015, generatedFrame: "full", maskMode: "diff", - regions: scaleAdjustPatchCompositeRegions, + regions: regions.map((region) => region.rect), sourceImageUrl, clipMaskImageUrl: scaleAdjustPatchReference?.compositeClipMaskUrl || undefined, From 67310e9ae551f46e2632846f41ac6ce41a1b3820 Mon Sep 17 00:00:00 2001 From: 134 Date: Wed, 8 Jul 2026 00:25:48 +0800 Subject: [PATCH 42/70] fix: timeout scale adjust prep uploads --- app/api/canvas/local-edit/route.ts | 93 ++++++++++++++++++++++-------- 1 file changed, 69 insertions(+), 24 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 88f6a0432..69d076539 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -230,6 +230,50 @@ function toArrayBuffer(buffer: Buffer) { ) as ArrayBuffer; } +function getCanvasScaleAdjustPrepUploadTimeoutMs() { + const parsed = Number(process.env.CANVAS_SCALE_ADJUST_PREP_UPLOAD_TIMEOUT_MS); + if (!Number.isFinite(parsed)) return 25_000; + return Math.max(5_000, Math.min(60_000, Math.round(parsed))); +} + +async function uploadScaleAdjustPrepAsset( + filename: string, + buffer: Buffer | null | undefined +): Promise> | null> { + if (!buffer) return null; + const upload = uploadFile(filename, toArrayBuffer(buffer)); + upload.catch((error) => { + console.warn("[Canvas Local Edit] Scale adjust prep asset upload finished after timeout:", { + error: error instanceof Error ? error.message : String(error), + filename, + }); + }); + + let timeoutId: ReturnType | undefined; + const timeout = new Promise((resolve) => { + timeoutId = setTimeout(() => resolve(null), getCanvasScaleAdjustPrepUploadTimeoutMs()); + }); + + try { + const result = await Promise.race([upload, timeout]); + if (!result) { + console.warn("[Canvas Local Edit] Scale adjust prep asset upload timed out; continuing without it:", { + filename, + timeoutMs: getCanvasScaleAdjustPrepUploadTimeoutMs(), + }); + } + return result; + } catch (error) { + console.warn("[Canvas Local Edit] Scale adjust prep asset upload failed:", { + error: error instanceof Error ? error.message : String(error), + filename, + }); + return null; + } finally { + if (timeoutId) clearTimeout(timeoutId); + } +} + function escapeXml(value: string) { return value .replace(/&/g, "&") @@ -3842,22 +3886,17 @@ async function createScaleAdjustPatchReference(params: { filename: string, buffer: Buffer | null | undefined ) => { - if (!buffer) return null; - try { - return await uploadFile(filename, toArrayBuffer(buffer)); - } catch (error) { - console.warn("[Canvas Local Edit] Optional scale adjust asset upload failed:", { - error: error instanceof Error ? error.message : String(error), - filename, - }); - return null; - } + return uploadScaleAdjustPrepAsset(filename, buffer); }; if (params.fullFrameReferenceOnly && structureLockAppliedRef.value) { - const targetDraftFullUpload = await uploadFile( + const targetDraftFullUpload = await uploadScaleAdjustPrepAsset( `canvas-scale-adjust-full-target-draft-${suffix}.jpg`, - toArrayBuffer(targetDraftUploadBuffer) + targetDraftUploadBuffer ); + const fullFrameFallbackUrl = targetDraftFullUpload?.url || sourceImageUrl; + if (!fullFrameFallbackUrl) { + throw new Error("比例调整预处理参考图上传失败，请稍后重试"); + } const productLockOverlayUpload = await uploadOptionalScaleAdjustAsset( `canvas-scale-adjust-product-lock-overlay-${suffix}.png`, targetDraftReference.productLockOverlayBuffer @@ -3897,10 +3936,10 @@ async function createScaleAdjustPatchReference(params: { productLockFalEditMaskUrl: productLockFalEditMaskUpload?.url || null, productLockOverlayUrl: productLockOverlayUpload?.url || null, productLockProtectMaskUrl: productLockProtectMaskUpload?.url || null, - sourceCropUrl: sourceImageUrl || targetDraftFullUpload.url, + sourceCropUrl: sourceImageUrl || fullFrameFallbackUrl, structureLockedDraft: structureLockAppliedRef.value, - targetDraftFullUrl: targetDraftFullUpload.url, - targetDraftUrl: targetDraftFullUpload.url, + targetDraftFullUrl: fullFrameFallbackUrl, + targetDraftUrl: fullFrameFallbackUrl, workspacePixelRect, workspaceRect, }; @@ -3918,17 +3957,17 @@ async function createScaleAdjustPatchReference(params: { ]); const [sourceCropUpload, targetDraftUpload, targetDraftFullUpload] = await Promise.all([ - uploadFile( + uploadScaleAdjustPrepAsset( `canvas-scale-adjust-workspace-source-${suffix}.png`, - toArrayBuffer(sourceCropBuffer) + sourceCropBuffer ), - uploadFile( + uploadScaleAdjustPrepAsset( `canvas-scale-adjust-workspace-target-draft-${suffix}.png`, - toArrayBuffer(targetDraftCropBuffer) + targetDraftCropBuffer ), - uploadFile( + uploadScaleAdjustPrepAsset( `canvas-scale-adjust-full-target-draft-${suffix}.jpg`, - toArrayBuffer(targetDraftUploadBuffer) + targetDraftUploadBuffer ), ]); const [ @@ -3964,6 +4003,12 @@ async function createScaleAdjustPatchReference(params: { targetDraftReference.productLockFalEditMaskBuffer ), ]); + const sourceCropUrl = sourceCropUpload?.url || sourceImageUrl || targetDraftFullUpload?.url; + const targetDraftUrl = + targetDraftUpload?.url || targetDraftFullUpload?.url || sourceImageUrl; + if (!sourceCropUrl || !targetDraftUrl) { + throw new Error("比例调整预处理参考图上传失败，请稍后重试"); + } return { backgroundPlateFullUrl: backgroundPlateFullUpload?.url || null, @@ -3973,10 +4018,10 @@ async function createScaleAdjustPatchReference(params: { productLockFalEditMaskUrl: productLockFalEditMaskUpload?.url || null, productLockOverlayUrl: productLockOverlayUpload?.url || null, productLockProtectMaskUrl: productLockProtectMaskUpload?.url || null, - sourceCropUrl: sourceCropUpload.url, + sourceCropUrl, structureLockedDraft: structureLockAppliedRef.value, - targetDraftFullUrl: targetDraftFullUpload.url, - targetDraftUrl: targetDraftUpload.url, + targetDraftFullUrl: targetDraftFullUpload?.url || sourceImageUrl || null, + targetDraftUrl, workspacePixelRect, workspaceRect, }; From e67aab87c54563a49f493afd9e70195bc0583ee0 Mon Sep 17 00:00:00 2001 From: 134 Date: Wed, 8 Jul 2026 00:37:28 +0800 Subject: [PATCH 43/70] fix: rebase product lock scale composite on source --- app/api/canvas/local-edit/route.ts | 3 ++ lib/canvas/generate-image.ts | 60 +++++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 69d076539..fa9893a85 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -5866,6 +5866,9 @@ export async function POST(request: NextRequest) { scaleAdjustPatchReference?.productLockOverlayUrl as string, protectMaskImageUrl: scaleAdjustPatchReference?.productLockProtectMaskUrl || undefined, + repairMaskImageUrl: + scaleAdjustPatchReference?.compositeClipMaskUrl || undefined, + sourceImageUrl, sourceHeight, sourceWidth, } diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index 51f07b37a..fe8b00185 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -74,6 +74,8 @@ export interface CanvasScaleAdjustPatchCompositeConfig { export interface CanvasScaleAdjustProductLockCompositeConfig { overlayImageUrl: string; protectMaskImageUrl?: string; + repairMaskImageUrl?: string; + sourceImageUrl?: string; sourceHeight: number; sourceWidth: number; } @@ -566,6 +568,8 @@ function normalizeScaleAdjustProductLockCompositeConfig( const overlayImageUrl = String(value.overlayImageUrl || "").trim(); if (!/^https?:\/\//i.test(overlayImageUrl)) return null; const protectMaskImageUrl = String(value.protectMaskImageUrl || "").trim(); + const repairMaskImageUrl = String(value.repairMaskImageUrl || "").trim(); + const sourceImageUrl = String(value.sourceImageUrl || "").trim(); const sourceWidth = Math.round(Number(value.sourceWidth || 0)); const sourceHeight = Math.round(Number(value.sourceHeight || 0)); if (sourceWidth <= 0 || sourceHeight <= 0) return null; @@ -575,6 +579,12 @@ function normalizeScaleAdjustProductLockCompositeConfig( ...(protectMaskImageUrl && /^https?:\/\//i.test(protectMaskImageUrl) ? { protectMaskImageUrl } : {}), + ...(repairMaskImageUrl && /^https?:\/\//i.test(repairMaskImageUrl) + ? { repairMaskImageUrl } + : {}), + ...(sourceImageUrl && /^https?:\/\//i.test(sourceImageUrl) + ? { sourceImageUrl } + : {}), sourceHeight, sourceWidth, }; @@ -3283,6 +3293,18 @@ async function applyScaleAdjustProductLockCompositeToImages(params: { const protectMaskBuffer = composite.protectMaskImageUrl ? await fetchImageBufferFromUrl(composite.protectMaskImageUrl) : null; + const repairMaskBuffer = composite.repairMaskImageUrl + ? await fetchImageBufferFromUrl(composite.repairMaskImageUrl) + : null; + const sourceBaseBuffer = composite.sourceImageUrl + ? await sharp(await fetchImageBufferFromUrl(composite.sourceImageUrl), { + failOn: "none", + }) + .rotate() + .ensureAlpha() + .png({ compressionLevel: 6 }) + .toBuffer() + : null; const overlayMetadata = await sharp(overlayBuffer, { failOn: "none" }).metadata(); const overlayWidth = Math.round(Number(overlayMetadata.width || 0)); const overlayHeight = Math.round(Number(overlayMetadata.height || 0)); @@ -3324,10 +3346,42 @@ async function applyScaleAdjustProductLockCompositeToImages(params: { width: generatedWidth, }) : resizedOverlayBaseBuffer; - const mergedPngBuffer = await sharp(normalizedGeneratedBuffer, { + const baseForProductLock = sourceBaseBuffer && repairMaskBuffer + ? await sharp(sourceBaseBuffer, { failOn: "none" }) + .resize(generatedWidth, generatedHeight, { + fit: "fill", + kernel: sharp.kernel.lanczos3, + }) + .ensureAlpha() + .png({ compressionLevel: 6 }) + .toBuffer() + : normalizedGeneratedBuffer; + const generatedRepairOverlay = + sourceBaseBuffer && repairMaskBuffer + ? await applyRawAlphaToImage({ + alphaMaskRaw: await sharp(repairMaskBuffer, { failOn: "none" }) + .resize(generatedWidth, generatedHeight, { fit: "fill" }) + .removeAlpha() + .extractChannel(0) + .blur(Math.max(4, Math.round(Math.min(generatedWidth, generatedHeight) * 0.006))) + .raw() + .toBuffer(), + height: generatedHeight, + imageBuffer: normalizedGeneratedBuffer, + width: generatedWidth, + }) + : null; + const mergedPngBuffer = await sharp(baseForProductLock, { failOn: "none", }) - .composite([{ input: resizedOverlayBuffer, left: 0, top: 0 }]) + .composite( + generatedRepairOverlay + ? [ + { input: generatedRepairOverlay, left: 0, top: 0 }, + { input: resizedOverlayBuffer, left: 0, top: 0 }, + ] + : [{ input: resizedOverlayBuffer, left: 0, top: 0 }] + ) .png({ compressionLevel: 6 }) .toBuffer(); const mergedBuffer = await sharp(mergedPngBuffer, { failOn: "none" }) @@ -3364,6 +3418,8 @@ async function applyScaleAdjustProductLockCompositeToImages(params: { overlayHeight, overlayWidth, protectMaskApplied: Boolean(protectMaskBuffer), + repairMaskApplied: Boolean(repairMaskBuffer && sourceBaseBuffer), + sourceBaseApplied: Boolean(sourceBaseBuffer), sourceHeight: composite.sourceHeight, sourceWidth: composite.sourceWidth, }); From 0add599427b5c5caa8ea25cb86046134c888f973 Mon Sep 17 00:00:00 2001 From: 134 Date: Wed, 8 Jul 2026 00:55:36 +0800 Subject: [PATCH 44/70] fix: use alpha channel for product lock repair mask --- lib/canvas/generate-image.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index fe8b00185..4e28a8204 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -3361,8 +3361,8 @@ async function applyScaleAdjustProductLockCompositeToImages(params: { ? await applyRawAlphaToImage({ alphaMaskRaw: await sharp(repairMaskBuffer, { failOn: "none" }) .resize(generatedWidth, generatedHeight, { fit: "fill" }) - .removeAlpha() - .extractChannel(0) + .ensureAlpha() + .extractChannel(3) .blur(Math.max(4, Math.round(Math.min(generatedWidth, generatedHeight) * 0.006))) .raw() .toBuffer(), From 8fad43c5978630441d573d4e50f3bc65180b4e9a Mon Sep 17 00:00:00 2001 From: 134 Date: Wed, 8 Jul 2026 01:11:14 +0800 Subject: [PATCH 45/70] fix: refine scale adjust product lock repair mask --- app/api/canvas/local-edit/route.ts | 130 ++++++++++++++++++++++++++--- lib/canvas/generate-image.ts | 107 ++++++++++++++++++++++-- 2 files changed, 217 insertions(+), 20 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index fa9893a85..ae2224bdf 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -126,6 +126,7 @@ type ScaleAdjustCropReference = { type ScaleAdjustPatchReference = { backgroundPlateFullUrl?: string | null; compositeClipMaskUrl?: string | null; + repairBlendMaskUrl?: string | null; productLockMethod?: string | null; productLockEditMaskUrl?: string | null; productLockFalEditMaskUrl?: string | null; @@ -509,14 +510,6 @@ function resolveScaleAdjustPreferredModelId(params: { params.requestedModelId || params.configuredModelId ) ); - if (params.preferMaskCapableGpt) { - const maskCapableGptImage2 = gptImage2Candidates - .filter(isScaleAdjustMaskCapableModel) - .filter((entry) => entry.model.healthStatus !== "auto_disabled"); - if (maskCapableGptImage2[0]) { - return maskCapableGptImage2[0].model.id; - } - } for (const providerType of params.preferProviderTypes || []) { const providerCandidate = gptImage2Candidates.find( (entry) => @@ -526,6 +519,14 @@ function resolveScaleAdjustPreferredModelId(params: { return providerCandidate.model.id; } } + if (params.preferMaskCapableGpt) { + const maskCapableGptImage2 = gptImage2Candidates + .filter(isScaleAdjustMaskCapableModel) + .filter((entry) => entry.model.healthStatus !== "auto_disabled"); + if (maskCapableGptImage2[0]) { + return maskCapableGptImage2[0].model.id; + } + } if (params.preferFalNanoBananaEdit) { const falEditCandidate = params.models .filter((entry) => isScaleAdjustUsableImageEditModel(entry)) @@ -2278,6 +2279,81 @@ async function buildScaleAdjustCompositeClipMaskBuffer(params: { .toBuffer(); } +async function buildScaleAdjustProductLockRepairBlendMask(params: { + height: number; + productLockOverlayBuffer: Buffer | null; + productLockProtectMaskBuffer: Buffer | null; + sourceRepairMaskBuffer: Buffer | null; + width: number; +}) { + const { + height, + productLockOverlayBuffer, + productLockProtectMaskBuffer, + sourceRepairMaskBuffer, + width, + } = params; + if (!sourceRepairMaskBuffer) return null; + + const repairAlpha = await sharp(sourceRepairMaskBuffer, { failOn: "none" }) + .rotate() + .resize(width, height, { fit: "fill" }) + .ensureAlpha() + .extractChannel(3) + .greyscale() + .raw() + .toBuffer(); + const overlayAlpha = productLockOverlayBuffer + ? await sharp(productLockOverlayBuffer, { failOn: "none" }) + .rotate() + .resize(width, height, { fit: "fill" }) + .ensureAlpha() + .extractChannel(3) + .greyscale() + .threshold(8) + .blur(Math.max(6, Math.round(Math.min(width, height) * 0.012))) + .greyscale() + .raw() + .toBuffer() + : null; + const protectAlpha = productLockProtectMaskBuffer + ? await sharp(productLockProtectMaskBuffer, { failOn: "none" }) + .rotate() + .resize(width, height, { fit: "fill" }) + .ensureAlpha() + .extractChannel(3) + .greyscale() + .threshold(8) + .blur(Math.max(4, Math.round(Math.min(width, height) * 0.008))) + .greyscale() + .raw() + .toBuffer() + : null; + + const rgba = Buffer.alloc(width * height * 4); + let markedPixels = 0; + for (let index = 0; index < width * height; index += 1) { + const offset = index * 4; + const protectedValue = Math.max( + overlayAlpha?.[index] || 0, + protectAlpha?.[index] || 0 + ); + const alpha = clamp((repairAlpha[index] || 0) - protectedValue, 0, 255); + if (alpha > 8) markedPixels += 1; + rgba[offset] = 255; + rgba[offset + 1] = 255; + rgba[offset + 2] = 255; + rgba[offset + 3] = alpha; + } + if (markedPixels === 0) return sourceRepairMaskBuffer; + + return sharp(rgba, { + raw: { channels: 4, height, width }, + }) + .png({ compressionLevel: 6 }) + .toBuffer(); +} + async function buildScaleAdjustTransparentEditMaskFromOpacityMask(params: { height: number; maskBuffer: Buffer; @@ -3805,6 +3881,15 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { width: sourceWidth, }) : null; + const repairBlendMaskBuffer = productLockComposites.length + ? await buildScaleAdjustProductLockRepairBlendMask({ + height: sourceHeight, + productLockOverlayBuffer, + productLockProtectMaskBuffer, + sourceRepairMaskBuffer: compositeClipMaskBuffer, + width: sourceWidth, + }) + : compositeClipMaskBuffer; const productLockEditMaskBuffer = productLockComposites.length && compositeClipMaskBuffer ? await buildScaleAdjustTransparentEditMaskFromOpacityMask({ @@ -3826,6 +3911,7 @@ async function buildScaleAdjustVisualLayoutGuideBuffer(params: { backgroundPlateBuffer, buffer, compositeClipMaskBuffer, + repairBlendMaskBuffer, productLockEditMaskBuffer, productLockFalEditMaskBuffer, productLockOverlayBuffer, @@ -3912,7 +3998,12 @@ async function createScaleAdjustPatchReference(params: { targetDraftReference.productLockFalEditMaskBuffer ), ]); - const [backgroundPlateFullUpload, productLockProtectMaskUpload, compositeClipMaskUpload] = + const [ + backgroundPlateFullUpload, + productLockProtectMaskUpload, + compositeClipMaskUpload, + repairBlendMaskUpload, + ] = await Promise.all([ uploadOptionalScaleAdjustAsset( `canvas-scale-adjust-background-plate-${suffix}.jpg`, @@ -3926,11 +4017,16 @@ async function createScaleAdjustPatchReference(params: { `canvas-scale-adjust-composite-clip-mask-${suffix}.png`, targetDraftReference.compositeClipMaskBuffer ), + uploadOptionalScaleAdjustAsset( + `canvas-scale-adjust-repair-blend-mask-${suffix}.png`, + targetDraftReference.repairBlendMaskBuffer + ), ]); return { backgroundPlateFullUrl: backgroundPlateFullUpload?.url || null, compositeClipMaskUrl: compositeClipMaskUpload?.url || null, + repairBlendMaskUrl: repairBlendMaskUpload?.url || null, productLockMethod: productLockOverlayUpload ? "rmbg-cutout" : null, productLockEditMaskUrl: productLockEditMaskUpload?.url || null, productLockFalEditMaskUrl: productLockFalEditMaskUpload?.url || null, @@ -3975,6 +4071,7 @@ async function createScaleAdjustPatchReference(params: { productLockOverlayUpload, productLockProtectMaskUpload, compositeClipMaskUpload, + repairBlendMaskUpload, productLockEditMaskUpload, productLockFalEditMaskUpload, ] = await Promise.all([ @@ -3994,6 +4091,10 @@ async function createScaleAdjustPatchReference(params: { `canvas-scale-adjust-composite-clip-mask-${suffix}.png`, targetDraftReference.compositeClipMaskBuffer ), + uploadOptionalScaleAdjustAsset( + `canvas-scale-adjust-repair-blend-mask-${suffix}.png`, + targetDraftReference.repairBlendMaskBuffer + ), uploadOptionalScaleAdjustAsset( `canvas-scale-adjust-product-lock-edit-mask-${suffix}.png`, targetDraftReference.productLockEditMaskBuffer @@ -4013,6 +4114,7 @@ async function createScaleAdjustPatchReference(params: { return { backgroundPlateFullUrl: backgroundPlateFullUpload?.url || null, compositeClipMaskUrl: compositeClipMaskUpload?.url || null, + repairBlendMaskUrl: repairBlendMaskUpload?.url || null, productLockMethod: productLockOverlayUpload ? "rmbg-cutout" : null, productLockEditMaskUrl: productLockEditMaskUpload?.url || null, productLockFalEditMaskUrl: productLockFalEditMaskUpload?.url || null, @@ -5178,7 +5280,9 @@ export async function POST(request: NextRequest) { const preferScaleAdjustFalNanoBananaEdit = hasScaleAdjustShrink && !hasScaleAdjustEnlarge; const preferScaleAdjustProviderTypes = - hasScaleAdjustShrink && !hasScaleAdjustEnlarge ? ["grsai-gpt-image"] : []; + hasScaleAdjustShrink && !hasScaleAdjustEnlarge + ? ["snapgen-image", "grsai-gpt-image"] + : []; const displayPrompt = buildLocalEditDisplayPrompt({ operation, scalePercent }); const resolvedAspectRatio = resolveLocalEditAspectRatio( body.aspectRatio, @@ -5687,6 +5791,8 @@ export async function POST(request: NextRequest) { scaleAdjustPatchReference?.productLockProtectMaskUrl || null, scaleAdjustPatchCompositeClipMaskUrl: scaleAdjustPatchReference?.compositeClipMaskUrl || null, + scaleAdjustRepairBlendMaskUrl: + scaleAdjustPatchReference?.repairBlendMaskUrl || null, scaleAdjustProviderMaskImageUrl: useScaleAdjustProductLockMaskEdit ? scaleAdjustPatchReference?.productLockEditMaskUrl || null : scaleAdjustEditMaskUpload?.url || null, @@ -5867,7 +5973,9 @@ export async function POST(request: NextRequest) { protectMaskImageUrl: scaleAdjustPatchReference?.productLockProtectMaskUrl || undefined, repairMaskImageUrl: - scaleAdjustPatchReference?.compositeClipMaskUrl || undefined, + scaleAdjustPatchReference?.repairBlendMaskUrl || + scaleAdjustPatchReference?.compositeClipMaskUrl || + undefined, sourceImageUrl, sourceHeight, sourceWidth, diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index 4e28a8204..3e31bef1d 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -2833,6 +2833,81 @@ async function applyRawAlphaToImage(params: { .toBuffer(); } +async function matchImageColorToSourceNearAlphaMask(params: { + alphaMaskRaw: Buffer; + height: number; + imageBuffer: Buffer; + sourceBuffer: Buffer; + width: number; +}) { + const { alphaMaskRaw, height, imageBuffer, sourceBuffer, width } = params; + if (alphaMaskRaw.length !== width * height) return imageBuffer; + + const [imageRaw, sourceRaw] = await Promise.all([ + sharp(imageBuffer, { failOn: "none" }) + .resize(width, height, { fit: "fill" }) + .removeAlpha() + .toColorspace("srgb") + .raw() + .toBuffer(), + sharp(sourceBuffer, { failOn: "none" }) + .resize(width, height, { fit: "fill" }) + .removeAlpha() + .toColorspace("srgb") + .raw() + .toBuffer(), + ]); + + const sourceSum = [0, 0, 0]; + const imageSum = [0, 0, 0]; + let count = 0; + for (let index = 0; index < alphaMaskRaw.length; index += 1) { + const alpha = alphaMaskRaw[index] || 0; + if (alpha < 4 || alpha > 110) continue; + const offset = index * 3; + const sourceLuma = + (sourceRaw[offset] || 0) * 0.299 + + (sourceRaw[offset + 1] || 0) * 0.587 + + (sourceRaw[offset + 2] || 0) * 0.114; + const imageLuma = + (imageRaw[offset] || 0) * 0.299 + + (imageRaw[offset + 1] || 0) * 0.587 + + (imageRaw[offset + 2] || 0) * 0.114; + if (sourceLuma < 24 || sourceLuma > 248 || imageLuma < 24 || imageLuma > 248) { + continue; + } + for (let channel = 0; channel < 3; channel += 1) { + sourceSum[channel] += sourceRaw[offset + channel] || 0; + imageSum[channel] += imageRaw[offset + channel] || 0; + } + count += 1; + } + + if (count < 500) return imageBuffer; + const offsets = sourceSum.map((sum, channel) => + Math.max(-32, Math.min(32, sum / count - imageSum[channel] / count)) + ); + const correctedRaw = Buffer.from(imageRaw); + for (let index = 0; index < correctedRaw.length; index += 3) { + correctedRaw[index] = Math.max( + 0, + Math.min(255, Math.round((correctedRaw[index] || 0) + offsets[0])) + ); + correctedRaw[index + 1] = Math.max( + 0, + Math.min(255, Math.round((correctedRaw[index + 1] || 0) + offsets[1])) + ); + correctedRaw[index + 2] = Math.max( + 0, + Math.min(255, Math.round((correctedRaw[index + 2] || 0) + offsets[2])) + ); + } + + return sharp(correctedRaw, { raw: { channels: 3, height, width } }) + .png({ compressionLevel: 6 }) + .toBuffer(); +} + async function applyScaleAdjustProductProtectMaskToOverlay(params: { height: number; overlayBuffer: Buffer; @@ -3356,18 +3431,32 @@ async function applyScaleAdjustProductLockCompositeToImages(params: { .png({ compressionLevel: 6 }) .toBuffer() : normalizedGeneratedBuffer; - const generatedRepairOverlay = + const repairAlphaMaskRaw = sourceBaseBuffer && repairMaskBuffer - ? await applyRawAlphaToImage({ - alphaMaskRaw: await sharp(repairMaskBuffer, { failOn: "none" }) - .resize(generatedWidth, generatedHeight, { fit: "fill" }) - .ensureAlpha() - .extractChannel(3) - .blur(Math.max(4, Math.round(Math.min(generatedWidth, generatedHeight) * 0.006))) - .raw() - .toBuffer(), + ? await sharp(repairMaskBuffer, { failOn: "none" }) + .resize(generatedWidth, generatedHeight, { fit: "fill" }) + .ensureAlpha() + .extractChannel(3) + .blur(Math.max(4, Math.round(Math.min(generatedWidth, generatedHeight) * 0.006))) + .raw() + .toBuffer() + : null; + const colorMatchedGeneratedBuffer = + sourceBaseBuffer && repairAlphaMaskRaw + ? await matchImageColorToSourceNearAlphaMask({ + alphaMaskRaw: repairAlphaMaskRaw, height: generatedHeight, imageBuffer: normalizedGeneratedBuffer, + sourceBuffer: baseForProductLock, + width: generatedWidth, + }) + : normalizedGeneratedBuffer; + const generatedRepairOverlay = + repairAlphaMaskRaw + ? await applyRawAlphaToImage({ + alphaMaskRaw: repairAlphaMaskRaw, + height: generatedHeight, + imageBuffer: colorMatchedGeneratedBuffer, width: generatedWidth, }) : null; From 2cb6f51a63cee09f39301fad9b50149c5a32bd52 Mon Sep 17 00:00:00 2001 From: 134 Date: Wed, 8 Jul 2026 01:23:46 +0800 Subject: [PATCH 46/70] fix: repair product locked scale adjust in crop workspace --- app/api/canvas/local-edit/route.ts | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index ae2224bdf..64ad2f432 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -5455,11 +5455,7 @@ export async function POST(request: NextRequest) { sourceHeight, sourceImageUrl, sourceWidth, - fullFrameReferenceOnly: - useScaleAdjustStructureLockedPatchWorkflow && - regions.some((region) => - isScaleAdjustFurnitureProductLikePrompt(region.prompt) - ), + fullFrameReferenceOnly: false, userId: session.user.id, }); } catch (error) { @@ -5492,6 +5488,8 @@ export async function POST(request: NextRequest) { const useScaleAdjustFullFrameStructurePatch = Boolean(scaleAdjustPatchReference?.structureLockedDraft) && Boolean(scaleAdjustPatchReference?.targetDraftFullUrl) && + scaleAdjustPatchReference?.targetDraftUrl === + scaleAdjustPatchReference?.targetDraftFullUrl && regions.some((region) => isScaleAdjustFurnitureProductLikePrompt(region.prompt) ); @@ -5500,10 +5498,11 @@ export async function POST(request: NextRequest) { isScaleAdjustMaskCapableModel(resolvedScaleAdjustModelEntry) && Boolean(scaleAdjustPatchReference?.productLockEditMaskUrl); const useScaleAdjustProductLockComposite = - useScaleAdjustFullFrameStructurePatch && Boolean(scaleAdjustPatchReference?.productLockOverlayUrl) && - (Boolean(scaleAdjustPatchReference?.backgroundPlateFullUrl) || - canUseScaleAdjustProductLockMaskEdit); + (useScaleAdjustFullFrameStructurePatch + ? Boolean(scaleAdjustPatchReference?.backgroundPlateFullUrl) || + canUseScaleAdjustProductLockMaskEdit + : Boolean(scaleAdjustPatchReference)); const useScaleAdjustProductLockMaskEdit = useScaleAdjustProductLockComposite && canUseScaleAdjustProductLockMaskEdit; @@ -5922,7 +5921,8 @@ export async function POST(request: NextRequest) { ? { maskImageUrl: scaleAdjustEditMaskUpload.url } : undefined, referenceImages, - scaleAdjustPatchComposite: useScaleAdjustProductLockComposite + scaleAdjustPatchComposite: + useScaleAdjustProductLockComposite && useScaleAdjustFullFrameStructurePatch ? { diffAlphaCutoff: 18, diffAlphaMultiplier: 2.6, @@ -5973,10 +5973,14 @@ export async function POST(request: NextRequest) { protectMaskImageUrl: scaleAdjustPatchReference?.productLockProtectMaskUrl || undefined, repairMaskImageUrl: - scaleAdjustPatchReference?.repairBlendMaskUrl || - scaleAdjustPatchReference?.compositeClipMaskUrl || - undefined, - sourceImageUrl, + useScaleAdjustFullFrameStructurePatch + ? scaleAdjustPatchReference?.repairBlendMaskUrl || + scaleAdjustPatchReference?.compositeClipMaskUrl || + undefined + : undefined, + sourceImageUrl: useScaleAdjustFullFrameStructurePatch + ? sourceImageUrl + : undefined, sourceHeight, sourceWidth, } From d69346de429a994f2d368f899542b34d4c9b6697 Mon Sep 17 00:00:00 2001 From: 134 Date: Wed, 8 Jul 2026 01:38:51 +0800 Subject: [PATCH 47/70] fix: keep product out of scale repair crop prompt --- app/api/canvas/local-edit/route.ts | 80 ++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 64ad2f432..9d2755bfa 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -124,6 +124,7 @@ type ScaleAdjustCropReference = { url: string; }; type ScaleAdjustPatchReference = { + backgroundPlateCropUrl?: string | null; backgroundPlateFullUrl?: string | null; compositeClipMaskUrl?: string | null; repairBlendMaskUrl?: string | null; @@ -4024,6 +4025,7 @@ async function createScaleAdjustPatchReference(params: { ]); return { + backgroundPlateCropUrl: null, backgroundPlateFullUrl: backgroundPlateFullUpload?.url || null, compositeClipMaskUrl: compositeClipMaskUpload?.url || null, repairBlendMaskUrl: repairBlendMaskUpload?.url || null, @@ -4051,6 +4053,12 @@ async function createScaleAdjustPatchReference(params: { .png({ compressionLevel: 6 }) .toBuffer(), ]); + const backgroundPlateCropBuffer = targetDraftReference.backgroundPlateBuffer + ? await sharp(targetDraftReference.backgroundPlateBuffer, { failOn: "none" }) + .extract(workspacePixelRect) + .png({ compressionLevel: 6 }) + .toBuffer() + : null; const [sourceCropUpload, targetDraftUpload, targetDraftFullUpload] = await Promise.all([ uploadScaleAdjustPrepAsset( @@ -4068,6 +4076,7 @@ async function createScaleAdjustPatchReference(params: { ]); const [ backgroundPlateFullUpload, + backgroundPlateCropUpload, productLockOverlayUpload, productLockProtectMaskUpload, compositeClipMaskUpload, @@ -4079,6 +4088,10 @@ async function createScaleAdjustPatchReference(params: { `canvas-scale-adjust-background-plate-${suffix}.jpg`, backgroundPlateUploadBuffer ), + uploadOptionalScaleAdjustAsset( + `canvas-scale-adjust-background-plate-crop-${suffix}.png`, + backgroundPlateCropBuffer + ), uploadOptionalScaleAdjustAsset( `canvas-scale-adjust-product-lock-overlay-${suffix}.png`, targetDraftReference.productLockOverlayBuffer @@ -4112,6 +4125,7 @@ async function createScaleAdjustPatchReference(params: { } return { + backgroundPlateCropUrl: backgroundPlateCropUpload?.url || null, backgroundPlateFullUrl: backgroundPlateFullUpload?.url || null, compositeClipMaskUrl: compositeClipMaskUpload?.url || null, repairBlendMaskUrl: repairBlendMaskUpload?.url || null, @@ -5148,6 +5162,54 @@ function buildScaleAdjustProductLockBackgroundPrompt( ].join("\n"); } +function buildScaleAdjustProductLockBackgroundCropPrompt( + regions: NormalizedLocalEditRegion[], + scalePercent: ScaleAdjustPercent | null | undefined, + patchReference: ScaleAdjustPatchReference, + sourceSize: { width: number; height: number } +) { + const workspace = patchReference.workspacePixelRect; + const regionLines = regions + .map((region) => { + const targetRect = getScaleAdjustTargetRect(region, scalePercent); + const targetPixelRect = + region.targetPixelRect || + rectToPixels(targetRect, sourceSize.width, sourceSize.height); + const sourceInCrop = { + left: Math.max(0, region.pixelRect.left - workspace.left), + top: Math.max(0, region.pixelRect.top - workspace.top), + width: region.pixelRect.width, + height: region.pixelRect.height, + }; + const targetInCrop = { + left: Math.max(0, targetPixelRect.left - workspace.left), + top: Math.max(0, targetPixelRect.top - workspace.top), + width: targetPixelRect.width, + height: targetPixelRect.height, + }; + return `Region ${region.index}: in this crop, erase the old selected furniture/product footprint around x=${sourceInCrop.left}, y=${sourceInCrop.top}, w=${sourceInCrop.width}, h=${sourceInCrop.height}. Keep the future target footprint x=${targetInCrop.left}, y=${targetInCrop.top}, w=${targetInCrop.width}, h=${targetInCrop.height} empty and clean; the locked product pixels will be composited there after generation. User request: ${region.prompt}.`; + }) + .join("\n"); + + return [ + "Cropped background repair for product-locked scale adjustment.", + `Return exactly one clean realistic crop with the same framing, aspect ratio, and pixel extent as input image 1. This crop maps to full-photo pixels left=${workspace.left}, top=${workspace.top}, width=${workspace.width}, height=${workspace.height}.`, + "Input image 1 is the original crop from the source photo. Input image 2 is a rough empty-background repair guide crop where the selected furniture/product footprint has been wiped.", + "Important: do not draw, recreate, move, scale, duplicate, or redesign the selected furniture/product in your output. The system will composite the locked original product pixels into the target footprint after your crop is returned.", + "Your only job is to repair the old footprint into natural wall, curtain, floor, rug, shadow, sofa edge, and background, while keeping non-target people and non-target objects stable.", + "If input image 2 contains haze, gray wash, ghost silhouettes, crop edges, black holes, rough mask shapes, or pasted fragments, treat them as repair hints only. Do not preserve them as visible content.", + "Preserve any non-target person exactly: do not change face, body, hands, legs, clothes, pose, hair, or book. Do not cover the person with furniture.", + "Keep the target footprint empty and natural; do not place the selected furniture/product there.", + "Do not output boxes, masks, labels, coordinate marks, UI marks, arrows, guide overlays, text, seams, rectangular patch edges, haze blocks, ghost copies, or old-size remnants.", + "中文硬约束：这是产品结构锁定的局部背景修补。你只输出干净背景裁切，不要生成目标家具/商品；旧位置的目标家具/商品必须擦除并补成自然墙面、窗帘、地面、阴影和背景。目标家具/商品会在你输出后由系统用原图锁定像素叠回。最终不能有旧尺寸残影、矩形雾块、贴片边、黑洞、标注框，也不能改变人物。", + "", + "Region instructions:", + regionLines, + "", + "Success check before returning: the selected furniture/product is absent from the crop, the old footprint is naturally repaired, the future target footprint is clean, and the crop looks like one natural photograph.", + ].join("\n"); +} + export async function POST(request: NextRequest) { try { const session = await auth(); @@ -5506,6 +5568,10 @@ export async function POST(request: NextRequest) { const useScaleAdjustProductLockMaskEdit = useScaleAdjustProductLockComposite && canUseScaleAdjustProductLockMaskEdit; + const useScaleAdjustProductLockCropBackground = + useScaleAdjustProductLockComposite && + !useScaleAdjustFullFrameStructurePatch && + Boolean(scaleAdjustPatchReference?.backgroundPlateCropUrl); const useScaleAdjustPatchTargetDraftPrimary = Boolean(scaleAdjustPatchReference); const includeScaleAdjustPatchSourceReference = Boolean(scaleAdjustPatchReference); @@ -5531,6 +5597,13 @@ export async function POST(request: NextRequest) { scalePercent, { width: sourceWidth, height: sourceHeight } ) + : useScaleAdjustProductLockCropBackground + ? buildScaleAdjustProductLockBackgroundCropPrompt( + regions, + scalePercent, + scaleAdjustPatchReference, + { width: sourceWidth, height: sourceHeight } + ) : buildScaleAdjustPatchPrompt( regions, scalePercent, @@ -5613,6 +5686,13 @@ export async function POST(request: NextRequest) { ...scaleAdjustCropReferenceImageUrls, ...manualRegionReferenceImageUrls, ] + : useScaleAdjustProductLockCropBackground + ? [ + scaleAdjustPatchReference.sourceCropUrl, + scaleAdjustPatchReference.backgroundPlateCropUrl as string, + ...scaleAdjustCropReferenceImageUrls, + ...manualRegionReferenceImageUrls, + ] : useScaleAdjustPatchTargetDraftPrimary ? [ scaleAdjustPatchReference.targetDraftUrl, From e0c277c76a8120b9764879cb700023704f3edaaa Mon Sep 17 00:00:00 2001 From: 134 Date: Wed, 8 Jul 2026 14:23:38 +0800 Subject: [PATCH 48/70] improve agent planning brief --- app/(chat)/api/chat/_lib/agent-output.ts | 2 + app/(chat)/api/chat/_lib/image-media.ts | 17 +- app/(chat)/api/chat/route.ts | 2 +- components/canvas/CanvasChatPanel.tsx | 2 + components/elements/response.tsx | 2 + components/message.tsx | 2 +- lib/ai/auto-agent-graph.ts | 44 +++- lib/ai/runtime/agent-planning-brief.ts | 302 +++++++++++++++++++++++ tests/routes/canvas-chat-helpers.test.ts | 38 +++ 9 files changed, 403 insertions(+), 8 deletions(-) create mode 100644 lib/ai/runtime/agent-planning-brief.ts diff --git a/app/(chat)/api/chat/_lib/agent-output.ts b/app/(chat)/api/chat/_lib/agent-output.ts index 9b04ebc34..9d9334c47 100644 --- a/app/(chat)/api/chat/_lib/agent-output.ts +++ b/app/(chat)/api/chat/_lib/agent-output.ts @@ -52,6 +52,8 @@ export function normalizeAgentReportHeadingLine(line: string): string { .trim(); const knownTitles = [ + "意图分析", + "任务规划", "需求理解", "图片分析", "执行计划", diff --git a/app/(chat)/api/chat/_lib/image-media.ts b/app/(chat)/api/chat/_lib/image-media.ts index 5b5a8bfac..844f6aa92 100644 --- a/app/(chat)/api/chat/_lib/image-media.ts +++ b/app/(chat)/api/chat/_lib/image-media.ts @@ -138,14 +138,14 @@ export function buildAutoModeFallbackMessage(params: { : "基于当前上传图片处理。"; return [ - "### 需求理解", + "### 意图分析", `- ${demand}`, "### 图片分析", params.imageSummary ? `- ${params.imageSummary.replace(/\s+/g, " ").trim()}` : `- ${sourceLine}`, "- 我会优先保持主体、构图和整体风格一致，只处理你指定的改动。", - "### 执行计划", + "### 任务规划", "- 先完成本轮核心改动，再检查边缘与光影一致性。", "- 若结果偏差，我会按“保留A，修改B”的方式继续迭代。", "### 输出设置", @@ -154,9 +154,9 @@ export function buildAutoModeFallbackMessage(params: { } return [ - "### 需求理解", + "### 意图分析", `- ${demand}`, - "### 执行计划", + "### 任务规划", "- 先锁定主体与场景，再细化风格、材质与光影。", "### 输出设置", "- 默认单图输出，优先稳定命中。", @@ -222,7 +222,7 @@ export function buildPromptOnlyReply(params: { export function hasStructuredAgentSections(message?: string): boolean { if (!message) return false; - return /(?:###\s*需求理解|【\s*需求理解\s*】|###\s*图片分析|【\s*图片分析\s*】|###\s*执行计划|【\s*执行计划\s*】|###\s*优化后提示词|【\s*优化后提示词\s*】)/.test( + return /(?:###\s*意图分析|【\s*意图分析\s*】|###\s*任务规划|【\s*任务规划\s*】|###\s*需求理解|【\s*需求理解\s*】|###\s*图片分析|【\s*图片分析\s*】|###\s*执行计划|【\s*执行计划\s*】|###\s*优化后提示词|【\s*优化后提示词\s*】)/.test( message, ); } @@ -253,6 +253,13 @@ export function ensureStructuredAgentFriendlyMessage(params: { return fallback; } + if (fallback.includes("### 任务规划\n")) { + return fallback.replace( + "### 任务规划\n", + `### 任务规划\n- 补充说明：${normalizedSupplement}\n`, + ); + } + return fallback.replace( "### 执行计划\n", `### 执行计划\n- 补充说明：${normalizedSupplement}\n`, diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index fcbfd0b3d..93fd9f99c 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -545,7 +545,7 @@ function writeCanvasOperationShadowChunk(params: { } const AGENT_EXECUTION_PREFACE_INTERNAL_PATTERN = - /(?:^|\n)\s*(?:#{1,6}\s*)?(?:需求理解|图片分析|执行计划|输出设置|优化后提示词|使用建议)(?:\s|[:：]|$)|【\s*(?:需求理解|图片分析|执行计划|输出设置|优化后提示词|使用建议)\s*】|提示词|自动优化|模型|提交生成|回写|流程|步骤/; + /(?:^|\n)\s*(?:#{1,6}\s*)?(?:意图分析|任务规划|需求理解|图片分析|执行计划|输出设置|优化后提示词|使用建议)(?:\s|[:：]|$)|【\s*(?:意图分析|任务规划|需求理解|图片分析|执行计划|输出设置|优化后提示词|使用建议)\s*】|提示词|自动优化|模型|提交生成|回写|流程|步骤/; function buildAgentExecutionPrefaceText(params: { friendlyMessage?: string | null; diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index 24e6b11e8..3100f7a9b 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -791,6 +791,8 @@ function buildPreviewImageUrl(url: string, width: number): string { const CANVAS_IMAGE_ANALYSIS_TITLE = "图片分析"; const CANVAS_IMAGE_DONE_SUMMARY = "已完成本次生成，如需微调可继续告诉我。"; const CANVAS_AGENT_SECTION_TITLES = new Set([ + "意图分析", + "任务规划", "需求理解", CANVAS_IMAGE_ANALYSIS_TITLE, "执行计划", diff --git a/components/elements/response.tsx b/components/elements/response.tsx index 5ce92c208..b908c4b7e 100644 --- a/components/elements/response.tsx +++ b/components/elements/response.tsx @@ -47,6 +47,8 @@ type ResponseProps = ComponentProps & { const IMAGE_ANALYSIS_TITLE = "图片分析"; const AGENT_SECTION_TITLES = new Set([ + "意图分析", + "任务规划", "需求理解", IMAGE_ANALYSIS_TITLE, "执行计划", diff --git a/components/message.tsx b/components/message.tsx index 4ff575742..6aa4d0c1b 100644 --- a/components/message.tsx +++ b/components/message.tsx @@ -174,7 +174,7 @@ function filterAgentRoutingNoise(text: string): string { const IMAGE_DONE_SUMMARY = "已完成本次生成，如需微调可继续告诉我。"; const ASSISTANT_TYPEWRITER_SECTION_REGEX = - /(?:###\s*需求理解|【\s*需求理解\s*】|###\s*图片分析|【\s*图片分析\s*】|###\s*执行计划|【\s*执行计划\s*】|###\s*优化后提示词|【\s*优化后提示词\s*】|###\s*输出设置|【\s*输出设置\s*】)/; + /(?:###\s*意图分析|【\s*意图分析\s*】|###\s*任务规划|【\s*任务规划\s*】|###\s*需求理解|【\s*需求理解\s*】|###\s*图片分析|【\s*图片分析\s*】|###\s*执行计划|【\s*执行计划\s*】|###\s*优化后提示词|【\s*优化后提示词\s*】|###\s*输出设置|【\s*输出设置\s*】)/; const ASSISTANT_TYPEWRITER_STORAGE_KEY = "assistant-typewriter-completed-keys-v1"; const MODEL_BADGE_LINE_REGEX = /^\s*[◈◇◆🔹⬡✦•·]\s*([^\n]+?)\s*$/im; diff --git a/lib/ai/auto-agent-graph.ts b/lib/ai/auto-agent-graph.ts index bf3b9fd64..ed723f6c3 100644 --- a/lib/ai/auto-agent-graph.ts +++ b/lib/ai/auto-agent-graph.ts @@ -19,6 +19,10 @@ import { normalizeAgentThreadState, type AgentThreadState, } from "@/lib/ai/runtime/agent-thread-state"; +import { + buildAgentPlanningBriefMarkdown, + isAgentPlanningBriefMarkdown, +} from "@/lib/ai/runtime/agent-planning-brief"; import { getAiModelAgentCodingModelId, isAiModelMultimodal, @@ -1153,17 +1157,55 @@ function buildStrategicFriendlyMessage( if (existingReply) { const isStructuredReply = + isAgentPlanningBriefMarkdown(existingReply) || + /###\s*意图分析/.test(existingReply) || + /###\s*任务规划/.test(existingReply) || /###\s*需求理解/.test(existingReply) || /###\s*图片分析/.test(existingReply) || /###\s*执行计划/.test(existingReply) || + /【\s*意图分析\s*】/.test(existingReply) || + /【\s*任务规划\s*】/.test(existingReply) || /【\s*需求理解\s*】/.test(existingReply) || /【\s*图片分析\s*】/.test(existingReply) || /【\s*执行计划\s*】/.test(existingReply); - if (isStructuredReply || existingReply.length >= 24) { + if (isStructuredReply) { return Promise.resolve(existingReply); } } + if ( + analysis.intent === "image_edit" || + analysis.intent === "image_gen" || + analysis.intent === "video_gen" + ) { + const imageParams = heuristicImageParams(state.text, analysis); + const planningBrief = buildAgentPlanningBriefMarkdown({ + userText: state.text, + intent: analysis.intent, + subIntent: analysis.subIntent, + nextAction: state.plan?.nextAction, + preferredCapability: state.plan?.preferredCapability, + targetSource: state.plan?.targetSource || state.targetResolutionMode, + hasImage: state.hasImage, + imageFromHistory: state.imageFromHistory, + selectedImageCount: state.selectedImageCount, + currentUploadImageCount: state.currentUploadImageCount, + historyImageCount: state.historyImageCount, + imageCount: imageParams?.imageCount, + aspectRatio: imageParams?.aspectRatio, + imageSize: imageParams?.imageSize, + planGoal: state.plan?.goal, + }); + if (process.env.AGENT_LLM_PLANNING_BRIEF_ENABLED !== "true") { + return Promise.resolve(planningBrief); + } + return withTimeoutFallback( + generateAgentConversation(analysis, state), + AGENT_OPERATION_TIMEOUT_MS, + planningBrief, + ); + } + return withTimeoutFallback( generateAgentConversation(analysis, state), AGENT_OPERATION_TIMEOUT_MS, diff --git a/lib/ai/runtime/agent-planning-brief.ts b/lib/ai/runtime/agent-planning-brief.ts new file mode 100644 index 000000000..943a6db17 --- /dev/null +++ b/lib/ai/runtime/agent-planning-brief.ts @@ -0,0 +1,302 @@ +export type AgentPlanningBriefIntent = + | "image_edit" + | "vision" + | "image_gen" + | "video_gen" + | "chat"; + +export type AgentPlanningBriefAction = + | "respond" + | "clarify" + | "analyze_image" + | "generate_image" + | "edit_image" + | "generate_video"; + +export type AgentPlanningBriefTargetSource = + | "explicit_selection" + | "current_upload" + | "text_reference" + | "working_memory" + | "latest_result" + | "clarification" + | "none"; + +export type AgentPlanningBriefCapability = + | "respond" + | "analyze" + | "generic_edit" + | "remove_background" + | "remove_watermark" + | "enhance" + | "extend" + | "resize" + | "angle_variant" + | "local_edit" + | "box_cutout" + | "generate_image" + | "generate_video" + | "publish_canvas"; + +export interface AgentPlanningBriefInput { + userText: string; + intent: AgentPlanningBriefIntent; + subIntent?: string; + nextAction?: AgentPlanningBriefAction; + preferredCapability?: AgentPlanningBriefCapability; + targetSource?: AgentPlanningBriefTargetSource; + hasImage?: boolean; + imageFromHistory?: boolean; + selectedImageCount?: number; + currentUploadImageCount?: number; + historyImageCount?: number; + imageCount?: number; + aspectRatio?: string; + imageSize?: string; + planGoal?: string; +} + +export interface AgentTaskGroup { + label: string; + count?: number; + description: string; +} + +const CHINESE_NUMBER_VALUES: Record = { + 一: 1, + 二: 2, + 两: 2, + 三: 3, + 四: 4, + 五: 5, + 六: 6, + 七: 7, + 八: 8, + 九: 9, + 十: 10, +}; + +function compactLine(value: string, maxLength: number) { + const normalized = String(value || "").replace(/\s+/g, " ").trim(); + if (normalized.length <= maxLength) return normalized; + return `${normalized.slice(0, maxLength)}...`; +} + +function parseCountToken(token?: string | null) { + const value = String(token || "").trim(); + if (!value) return undefined; + if (/^\d+$/.test(value)) { + const numeric = Number(value); + return Number.isFinite(numeric) && numeric > 0 ? Math.round(numeric) : undefined; + } + return CHINESE_NUMBER_VALUES[value]; +} + +function countFromText(text: string) { + const match = String(text || "").match(/([1-9]\d?|一|二|两|三|四|五|六|七|八|九|十)\s*张/); + return parseCountToken(match?.[1]); +} + +function normalizeGroupDescription(description: string) { + return String(description || "") + .replace(/^[为是:：，,\s]+/, "") + .replace(/[。；;，,]\s*$/, "") + .trim(); +} + +export function extractAgentTaskGroups(text: string): AgentTaskGroup[] { + const normalized = String(text || "").replace(/\s+/g, " ").trim(); + if (!normalized) return []; + + const groups: AgentTaskGroup[] = []; + const groupPattern = + /第([一二两三四五六七八九十]|\d{1,2})组(?:为|是|:|：)?([\s\S]*?)(?=(?:第[一二两三四五六七八九十\d]{1,2}组)|[。；;]|$)/g; + let match: RegExpExecArray | null; + while ((match = groupPattern.exec(normalized)) !== null) { + const labelToken = match[1]; + const description = normalizeGroupDescription(match[2] || ""); + if (!description) continue; + const labelNumber = parseCountToken(labelToken); + groups.push({ + label: `第${labelNumber || labelToken}组`, + count: countFromText(description), + description: compactLine(description, 96), + }); + } + + return groups.slice(0, 8); +} + +function getTargetSourceLabel(source?: AgentPlanningBriefTargetSource) { + switch (source) { + case "explicit_selection": + return "画布选中图片"; + case "current_upload": + return "当前上传图片"; + case "text_reference": + return "文本指定图片"; + case "working_memory": + return "会话工作记忆图片"; + case "latest_result": + return "最近生成结果"; + case "clarification": + return "待确认目标"; + case "none": + default: + return "无需现有图片"; + } +} + +function getCapabilityLabel(capability?: AgentPlanningBriefCapability, subIntent?: string) { + switch (capability || subIntent) { + case "remove_background": + return "主体抠图/去背景"; + case "remove_watermark": + return "去水印"; + case "enhance": + return "清晰度增强"; + case "extend": + return "扩图/画幅延展"; + case "resize": + return "尺寸调整"; + case "angle_variant": + case "generate_angle_variant": + return "多视角变体"; + case "local_edit": + case "inpaint_region": + return "局部编辑"; + case "box_cutout": + return "框选抠图"; + case "generate_video": + return "视频生成"; + case "generate_image": + return "图片生成"; + case "analyze": + return "图片分析"; + default: + return "图片编辑"; + } +} + +function inferTotalCount(params: AgentPlanningBriefInput, groups: AgentTaskGroup[]) { + if (typeof params.imageCount === "number" && Number.isFinite(params.imageCount)) { + return Math.max(1, Math.round(params.imageCount)); + } + const groupedTotal = groups.reduce((sum, group) => sum + (group.count || 0), 0); + return groupedTotal > 0 ? groupedTotal : undefined; +} + +function buildIntentLine(params: AgentPlanningBriefInput, groups: AgentTaskGroup[]) { + const target = getTargetSourceLabel(params.targetSource); + const capability = getCapabilityLabel(params.preferredCapability, params.subIntent); + const totalCount = inferTotalCount(params, groups); + const demand = compactLine(params.planGoal || params.userText || "", 120); + + if (params.intent === "image_gen") { + const countText = totalCount ? `，共 ${totalCount} 张` : ""; + const groupText = groups.length > 0 ? `，拆成 ${groups.length} 组` : ""; + const referenceText = params.hasImage ? "，并保持参考图主体核心特征一致" : ""; + return `用户需要生成新图片${groupText}${countText}${referenceText}；核心目标是：${demand || "完成本轮生图任务"}。`; + } + + if (params.intent === "image_edit") { + return `用户需要基于${target}执行${capability}；核心目标是：${demand || "完成本轮图片处理"}。`; + } + + if (params.intent === "video_gen") { + return `用户需要生成视频；先拆解镜头、主体连续性和运动节奏，再提交视频生成。`; + } + + if (params.intent === "vision") { + return `用户需要理解或分析图片内容；先观察主体、场景、构图和潜在问题，再给出结论。`; + } + + return `用户当前主要是在对话或咨询；先回答问题，不主动触发生图或修图工具。`; +} + +function buildImageGenerationPlan(params: AgentPlanningBriefInput, groups: AgentTaskGroup[]) { + if (groups.length > 0) { + return groups.map((group, index) => { + const countText = group.count ? `${group.count}张` : "一组"; + return `${index + 1}. ${group.label}：生成${countText}${group.description.replace(/^\d+\s*张/, "")}，保持主体识别一致。`; + }); + } + + const count = inferTotalCount(params, groups) || 1; + return [ + `1. 锁定主体、场景、风格和构图约束，整理成可执行提示词。`, + `2. 生成 ${count} 张候选图，优先保证主体一致性和画面完成度。`, + `3. 输出结果并保留可继续追改的上下文。`, + ]; +} + +function buildImageEditPlan(params: AgentPlanningBriefInput) { + const target = getTargetSourceLabel(params.targetSource); + const capability = getCapabilityLabel(params.preferredCapability, params.subIntent); + return [ + `1. 先锁定${target}作为本轮处理目标，避免误处理其他图片。`, + `2. 调用${capability}能力执行核心改动，同时尽量保留主体结构和画质。`, + `3. 输出处理结果，并保留原图用于对比或继续微调。`, + ]; +} + +function buildGenericPlan(params: AgentPlanningBriefInput) { + if (params.intent === "video_gen") { + return [ + "1. 拆解视频主体、场景、动作节奏和镜头关系。", + "2. 生成脚本与分镜，并整理可直接提交的视频提示词。", + "3. 按规划提交视频生成，返回结果后支持继续调整。", + ]; + } + if (params.intent === "vision") { + return [ + "1. 先识别图片主体、背景、构图和关键信息。", + "2. 结合用户问题输出判断、建议或可执行修改方向。", + ]; + } + return ["1. 先直接回答用户问题；如果需要执行工具，再根据明确指令进入下一步。"]; +} + +function buildOutputSettings(params: AgentPlanningBriefInput, groups: AgentTaskGroup[]) { + const lines: string[] = []; + const totalCount = inferTotalCount(params, groups); + if (params.intent === "image_gen" || params.intent === "image_edit") { + lines.push(`- 输出数量：${params.intent === "image_edit" ? "1 张处理结果" : `${totalCount || 1} 张`}`); + lines.push(`- 图片比例：${params.aspectRatio && params.aspectRatio !== "auto" ? params.aspectRatio : "按需求自适应"}`); + lines.push(`- 清晰度：${params.imageSize || (params.intent === "image_edit" ? "4K优先" : "2K优先")}`); + if (params.intent === "image_edit") { + lines.push(`- 目标来源：${getTargetSourceLabel(params.targetSource)}`); + } + } else if (params.intent === "video_gen") { + lines.push("- 输出形式：视频结果 + 可继续追改的分镜上下文"); + } else { + lines.push("- 输出形式：直接回答，不主动触发生成工具"); + } + return lines; +} + +export function buildAgentPlanningBriefMarkdown(params: AgentPlanningBriefInput) { + const groups = extractAgentTaskGroups(params.userText); + const planLines = + params.intent === "image_gen" + ? buildImageGenerationPlan(params, groups) + : params.intent === "image_edit" + ? buildImageEditPlan(params) + : buildGenericPlan(params); + const outputSettings = buildOutputSettings(params, groups); + + return [ + "### 意图分析", + `- ${buildIntentLine(params, groups)}`, + "### 任务规划", + ...planLines.map((line) => (line.startsWith("- ") ? line : `- ${line}`)), + "### 输出设置", + ...outputSettings, + ].join("\n"); +} + +export function isAgentPlanningBriefMarkdown(text: string) { + const normalized = String(text || ""); + return /###\s*意图分析|【\s*意图分析\s*】/.test(normalized) && + /###\s*任务规划|【\s*任务规划\s*】/.test(normalized); +} diff --git a/tests/routes/canvas-chat-helpers.test.ts b/tests/routes/canvas-chat-helpers.test.ts index ee23944a0..d2175bb3d 100644 --- a/tests/routes/canvas-chat-helpers.test.ts +++ b/tests/routes/canvas-chat-helpers.test.ts @@ -54,6 +54,11 @@ import { } from "@/lib/ai/runtime/canvas-agent-kernel"; import { isExplicitImageGenerationRequest } from "@/lib/ai/smart-analyzer"; import { inferAgentImageCountFromText } from "@/lib/ai/runtime/image-count-intent"; +import { + buildAgentPlanningBriefMarkdown, + extractAgentTaskGroups, + isAgentPlanningBriefMarkdown, +} from "@/lib/ai/runtime/agent-planning-brief"; import { buildRecentDialogueMemory, isRecentDialogueRecallRequest, @@ -385,6 +390,39 @@ test.describe("canvas chat pure helpers", () => { expect(inferAgentImageCountFromText("再出30张", 12)).toBe(12); }); + test("builds market-grade planning brief for grouped image generation requests", () => { + const request = + "基于上传的潮玩角色图片，生成3组共12张图片，要求保持角色核心特征一致。第一组为4张动作表情夸张的半身表情包；第二组为4张不同场景的角色活动照；第三组为4张服装发型差异巨大的造型照，所有图片比例保持3:4。"; + + const groups = extractAgentTaskGroups(request); + expect(groups).toHaveLength(3); + expect(groups.map((group) => group.count)).toEqual([4, 4, 4]); + expect(groups[0].description).toContain("半身表情包"); + expect(groups[1].description).toContain("不同场景"); + expect(groups[2].description).toContain("服装发型"); + + const markdown = buildAgentPlanningBriefMarkdown({ + userText: request, + intent: "image_gen", + nextAction: "generate_image", + preferredCapability: "generate_image", + targetSource: "current_upload", + hasImage: true, + imageCount: 12, + aspectRatio: "3:4", + imageSize: "2K", + }); + + expect(isAgentPlanningBriefMarkdown(markdown)).toBe(true); + expect(markdown).toContain("### 意图分析"); + expect(markdown).toContain("拆成 3 组，共 12 张"); + expect(markdown).toContain("### 任务规划"); + expect(markdown).toContain("第1组：生成4张"); + expect(markdown).toContain("第2组：生成4张"); + expect(markdown).toContain("第3组：生成4张"); + expect(markdown).toContain("图片比例：3:4"); + }); + test("keeps canvas message empty-history access scoped and explicit", () => { const canvasChatId = "11111111-1111-4111-8111-111111111111"; const agentScopedChatId = "11111111-1111-4111-8111-a93e17a60001"; From 599abc5021ca59fdf913c925f42cc699f0fc5a95 Mon Sep 17 00:00:00 2001 From: 134 Date: Wed, 8 Jul 2026 14:37:07 +0800 Subject: [PATCH 49/70] allow agent multi image generation --- components/elements/response.tsx | 80 ++++++++++++++++--- components/image-preview.tsx | 24 ++++-- .../runtime/direct-agent-media-executors.ts | 8 ++ lib/canvas/generate-image.ts | 6 +- 4 files changed, 102 insertions(+), 16 deletions(-) diff --git a/components/elements/response.tsx b/components/elements/response.tsx index b908c4b7e..82deb547e 100644 --- a/components/elements/response.tsx +++ b/components/elements/response.tsx @@ -905,11 +905,17 @@ export const Response = ({ ), }; - const renderImagePreview = (src: string, alt: string | undefined, key: string) => ( + const renderImagePreview = ( + src: string, + alt: string | undefined, + key: string, + compactGrid = false + ) => ( ); + const renderImageGrid = ( + images: Array<{ alt?: string; src: string }>, + keyPrefix: string + ) => { + if (images.length === 0) return null; + if (images.length === 1) { + const image = images[0]; + return renderImagePreview( + image.src, + image.alt, + `${image.src}-${processingKey}-${keyPrefix}-single` + ); + } + + const gridClassName = + images.length === 2 + ? "grid-cols-2 sm:max-w-[640px]" + : images.length === 3 + ? "grid-cols-2 sm:grid-cols-3 sm:max-w-[780px]" + : "grid-cols-2 sm:grid-cols-4 sm:max-w-[920px]"; + + return ( +
+ {images.map((image, index) => ( +
+ {renderImagePreview( + image.src, + image.alt, + `${image.src}-${processingKey}-${keyPrefix}-grid-${index}`, + true + )} +
+ ))} +
+ ); + }; + const renderTextSegment = (text: string, keyPrefix: string): ReactNode => { if (!text.trim()) return null; @@ -937,20 +986,31 @@ export const Response = ({ const hasImageSegments = segments.some((segment) => segment.type === "image"); if (hasImageSegments) { - return segments.map((segment, index) => { + const nodes: ReactNode[] = []; + let pendingImages: Array<{ alt?: string; src: string }> = []; + const flushImages = (index: number) => { + if (pendingImages.length === 0) return; + nodes.push( + renderImageGrid(pendingImages, `${keyPrefix}-images-${index}`) + ); + pendingImages = []; + }; + + segments.forEach((segment, index) => { if (segment.type === "image") { - return renderImagePreview( - segment.src, - segment.alt, - `${segment.src}-${processingKey}-${keyPrefix}-${index}` - ); + pendingImages.push({ + alt: segment.alt, + src: segment.src, + }); + return; } if (!segment.value.trim()) { - return null; + return; } - return ( + flushImages(index); + nodes.push( ); }); + flushImages(segments.length); + return nodes; } } diff --git a/components/image-preview.tsx b/components/image-preview.tsx index 6908df6fd..1750c9332 100644 --- a/components/image-preview.tsx +++ b/components/image-preview.tsx @@ -42,7 +42,8 @@ export function ImagePreview({ isProcessingExtend, isProcessingAngleEdit, isProcessingWatermarkRemove, - imageIndex + imageIndex, + compactGrid = false }: { src: string; alt?: string; @@ -64,6 +65,7 @@ export function ImagePreview({ isProcessingAngleEdit?: boolean; isProcessingWatermarkRemove?: boolean; imageIndex?: number; + compactGrid?: boolean; }) { const [isOpen, setIsOpen] = useState(false); const [isAnnotationMode, setIsAnnotationMode] = useState(false); @@ -293,16 +295,26 @@ export function ImagePreview({ return ( <> {/* 缩略图（显示压缩后的缩略图）*/} -
+
Generated Image 1; const result = await generateCanvasImage({ prompt, negativePrompt: params.imageParams?.negativePrompt, model: params.finalModelId, ownerKey: params.userId, + debugLabel: "direct-agent-image-generation", + debugMeta: { + agentIgnoreModelMaxImages: shouldIgnoreModelMaxImages, + requestedAgentImageCount: imageCount ?? null, + }, aspectRatio: params.imageParams?.aspectRatio || params.defaultImageQualityParams?.aspectRatio, imageSize, imageCount, + ignoreModelMaxImages: shouldIgnoreModelMaxImages, referenceImages: params.referenceImageUrls, }); diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index e29df494e..6a82b853e 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -116,6 +116,7 @@ export interface CanvasGenerateImageRequest { ownerKey?: string; debugLabel?: string; debugMeta?: Record; + ignoreModelMaxImages?: boolean; localEditComposite?: CanvasLocalEditCompositeConfig; scaleAdjustPatchComposite?: CanvasScaleAdjustPatchCompositeConfig; scaleAdjustVerification?: CanvasScaleAdjustVerificationConfig; @@ -3167,12 +3168,14 @@ function normalizeCanvasImageParamsByModel(params: { imageCount: number; imageSize: string; imageGenConfig: CanvasImageGenConfig | null | undefined; + ignoreModelMaxImages?: boolean; }) { - const maxImages = + const configuredMaxImages = typeof params.imageGenConfig?.maxImages === "number" && Number.isFinite(params.imageGenConfig.maxImages) ? Math.max(1, Math.floor(params.imageGenConfig.maxImages)) : 12; + const maxImages = params.ignoreModelMaxImages ? 12 : configuredMaxImages; const imageCount = Math.max(1, Math.min(maxImages, Math.floor(params.imageCount))); const requestedSize = normalizeCanvasImageSizeValue(params.imageSize); @@ -3635,6 +3638,7 @@ export async function generateCanvasImage( imageCount: requestedImageCount, imageSize: requestedImageSize, imageGenConfig: selectedModel.model.imageGenConfig as CanvasImageGenConfig | null | undefined, + ignoreModelMaxImages: request.ignoreModelMaxImages === true, }); const imageSize = normalizedImageParams.imageSize; const imageCount = From 2bbdd0aa618a223fe84eb98f8adec2bc12cad352 Mon Sep 17 00:00:00 2001 From: 134 Date: Wed, 8 Jul 2026 15:22:17 +0800 Subject: [PATCH 50/70] preserve colorful brand logos --- app/(auth)/login/page.tsx | 7 +- app/api/favicon/route.ts | 122 ++---------------- .../[id]/CanvasEditorLegacyPageClient.tsx | 10 +- app/layout.tsx | 9 -- components/agents/AgentCanvasTopNavDark.tsx | 6 +- .../canvas/LiblibVideoCanvasReplica.tsx | 6 +- components/canvas/canvas-logo-loading.tsx | 4 +- components/favicon-updater.tsx | 29 +---- components/home/lovart-home.tsx | 7 +- .../home/lovart/lovart-hero-composer.tsx | 4 +- components/home/lovart/lovart-sidebar.tsx | 9 +- components/layout/brand-top-bar.tsx | 7 +- .../pascal-editor/pascal-editor-frame.tsx | 7 +- components/top-navbar.tsx | 8 +- 14 files changed, 27 insertions(+), 208 deletions(-) diff --git a/app/(auth)/login/page.tsx b/app/(auth)/login/page.tsx index 62b4069fd..8e98180aa 100644 --- a/app/(auth)/login/page.tsx +++ b/app/(auth)/login/page.tsx @@ -501,11 +501,6 @@ export default function Page() { const displayLogo = mounted ? (resolvedTheme === "dark" && siteLogoDark ? siteLogoDark : siteLogo) : siteLogo; - const shouldInvertLogo = - mounted && - resolvedTheme === "dark" && - !siteLogoDark?.trim() && - Boolean(siteLogo?.trim()); const hasThirdPartyLogin = (isDingTalkEnabled && (Boolean(dingTalkAppKey) || dingTalkOrganizations.length > 0)) || (isGoogleEnabled && isGoogleConfigured); @@ -545,7 +540,7 @@ export default function Page() { {siteName} ) : null}

登录 {siteName}

diff --git a/app/api/favicon/route.ts b/app/api/favicon/route.ts index 0382b454f..092c6190d 100644 --- a/app/api/favicon/route.ts +++ b/app/api/favicon/route.ts @@ -8,7 +8,6 @@ export const dynamic = "force-dynamic"; export const revalidate = 0; const DEFAULT_CONTENT_TYPE = "image/x-icon"; -const DARK_FAVICON_CONTENT_TYPE = "image/png"; const FETCH_TIMEOUT_MS = 10_000; function noStoreHeaders(contentType: string) { @@ -71,86 +70,16 @@ function uniqueNonEmptyValues(values: string[]) { return result; } -async function readLocalFallbackFavicon(preferDark = false) { +async function readLocalFallbackFavicon() { const file = await readFile( join(process.cwd(), "public", "default-favicon.ico") ); - if (preferDark) { - const transformed = await transformFaviconForDark(file).catch(() => null); - const fallback = - transformed || (await renderDarkFallbackFavicon().catch(() => null)); - if (fallback) { - return new NextResponse(fallback, { - status: 200, - headers: noStoreHeaders(DARK_FAVICON_CONTENT_TYPE), - }); - } - } return new NextResponse(file, { status: 200, headers: noStoreHeaders(DEFAULT_CONTENT_TYPE), }); } -async function renderDarkFallbackFavicon() { - const sharp = (await import("sharp")).default; - const svg = `\ - - - - -`; - return await sharp(Buffer.from(svg)).png().toBuffer(); -} - -async function transformFaviconForDark(buffer: Buffer | ArrayBuffer) { - const sharp = (await import("sharp")).default; - const inputBuffer = Buffer.isBuffer(buffer) - ? buffer - : Buffer.from(new Uint8Array(buffer)); - const createBaseImage = () => - sharp(inputBuffer, { - animated: false, - failOn: "none", - }).resize({ - width: 64, - height: 64, - fit: "contain", - background: { r: 0, g: 0, b: 0, alpha: 0 }, - }); - - const { data, info } = await createBaseImage() - .ensureAlpha() - .raw() - .toBuffer({ resolveWithObject: true }); - let visiblePixelCount = 0; - let luminanceTotal = 0; - - for (let index = 0; index < data.length; index += info.channels) { - const alpha = data[index + 3] ?? 255; - if (alpha < 24) continue; - - const red = data[index] ?? 0; - const green = data[index + 1] ?? 0; - const blue = data[index + 2] ?? 0; - luminanceTotal += 0.2126 * red + 0.7152 * green + 0.0722 * blue; - visiblePixelCount += 1; - } - - const averageLuminance = - visiblePixelCount > 0 ? luminanceTotal / visiblePixelCount : 0; - const needsInvertForDark = averageLuminance < 150; - const output = createBaseImage(); - - if (needsInvertForDark) { - output.negate({ alpha: false }); - } - - return await output - .png() - .toBuffer(); -} - export async function GET(request: Request) { try { const requestUrl = new URL(request.url); @@ -159,20 +88,14 @@ export async function GET(request: Request) { const configuredFavicon = String(settings?.favicon || "").trim(); const configuredLightLogo = String(settings?.siteLogo || "").trim(); const configuredDarkLogo = String(settings?.siteLogoDark || "").trim(); - const rawFaviconCandidates = preferDark - ? uniqueNonEmptyValues([ - configuredFavicon, - configuredDarkLogo, - configuredLightLogo, - ]) - : uniqueNonEmptyValues([ - configuredFavicon, - configuredLightLogo, - configuredDarkLogo, - ]); + const rawFaviconCandidates = configuredFavicon + ? [configuredFavicon] + : preferDark + ? uniqueNonEmptyValues([configuredDarkLogo, configuredLightLogo]) + : uniqueNonEmptyValues([configuredLightLogo, configuredDarkLogo]); if (rawFaviconCandidates.length === 0) { - return await readLocalFallbackFavicon(preferDark); + return await readLocalFallbackFavicon(); } for (const rawFaviconUrl of rawFaviconCandidates) { @@ -220,42 +143,15 @@ export async function GET(request: Request) { const contentType = upstream.headers.get("content-type") || inferContentType(faviconUrl); - if (preferDark) { - const transformed = await transformFaviconForDark(buffer).catch((error) => { - console.error("[favicon] Failed to transform dark favicon candidate:", { - targetUrl, - error, - }); - return null; - }); - if (!transformed) continue; - - return new NextResponse(transformed, { - status: 200, - headers: noStoreHeaders(DARK_FAVICON_CONTENT_TYPE), - }); - } - return new NextResponse(buffer, { status: 200, headers: noStoreHeaders(contentType), }); } - if (preferDark) { - const fallback = await renderDarkFallbackFavicon().catch(() => null); - if (fallback) { - return new NextResponse(fallback, { - status: 200, - headers: noStoreHeaders(DARK_FAVICON_CONTENT_TYPE), - }); - } - } - - return await readLocalFallbackFavicon(preferDark); + return await readLocalFallbackFavicon(); } catch (error) { console.error("[favicon] Failed to proxy favicon:", error); - const preferDark = new URL(request.url).searchParams.get("theme") === "dark"; - return await readLocalFallbackFavicon(preferDark); + return await readLocalFallbackFavicon(); } } diff --git a/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx b/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx index 2a48e250c..f55b3e953 100644 --- a/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx +++ b/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx @@ -9640,10 +9640,6 @@ export default function CanvasEditorLegacyPage({ : lightCanvasBrandLogo || darkCanvasBrandLogo : lightCanvasBrandLogo || darkCanvasBrandLogo ).trim(); - const shouldInvertCanvasBrandLogo = - isDarkCanvasTheme && - Boolean(canvasBrandLogo) && - !hasSeparateDarkCanvasBrandLogo; const brandEntry = (
diff --git a/app/layout.tsx b/app/layout.tsx index f26b5bee7..fcee64a6a 100644 --- a/app/layout.tsx +++ b/app/layout.tsx @@ -24,19 +24,10 @@ export async function generateMetadata(): Promise { "AI design workspace for Canvas, Agent workflows, image editing, 3D/CAD, and local AI operations.", icons: { icon: [ - { - url: "/api/favicon?theme=dark", - media: "(prefers-color-scheme: dark)", - sizes: "any", - type: "image/png", - }, { url: "/api/favicon", - media: "(prefers-color-scheme: light)", sizes: "any", }, - { url: "/api/favicon?theme=dark", sizes: "any", type: "image/png" }, - { url: "/api/favicon", type: "image/x-icon" }, ], apple: "/api/favicon", shortcut: "/api/favicon", diff --git a/components/agents/AgentCanvasTopNavDark.tsx b/components/agents/AgentCanvasTopNavDark.tsx index ebb5a0613..6e97e2794 100644 --- a/components/agents/AgentCanvasTopNavDark.tsx +++ b/components/agents/AgentCanvasTopNavDark.tsx @@ -53,7 +53,6 @@ export function AgentCanvasTopNavDark({ }: AgentCanvasTopNavDarkProps) { const resolvedSiteName = String(siteName || "AI Chatbot").trim() || "AI Chatbot"; const displayLogo = siteLogoDark?.trim() || siteLogo?.trim() || ""; - const shouldInvertLogo = !siteLogoDark?.trim() && Boolean(siteLogo?.trim()); return (
@@ -80,10 +79,7 @@ export function AgentCanvasTopNavDark({ {resolvedSiteName} ) : (
diff --git a/components/canvas/LiblibVideoCanvasReplica.tsx b/components/canvas/LiblibVideoCanvasReplica.tsx index 227473f3e..6e48ad0e1 100644 --- a/components/canvas/LiblibVideoCanvasReplica.tsx +++ b/components/canvas/LiblibVideoCanvasReplica.tsx @@ -6695,17 +6695,13 @@ function StylizedBrandLogo({ }) { const resolvedSiteName = resolveSiteName(siteName); const displayLogo = siteLogoDark?.trim() || siteLogo?.trim() || ""; - const shouldInvertLogo = !siteLogoDark?.trim() && Boolean(siteLogo?.trim()); return ( {displayLogo ? ( {resolvedSiteName} ) : ( diff --git a/components/canvas/canvas-logo-loading.tsx b/components/canvas/canvas-logo-loading.tsx index 531fd401f..73f273231 100644 --- a/components/canvas/canvas-logo-loading.tsx +++ b/components/canvas/canvas-logo-loading.tsx @@ -33,9 +33,7 @@ export function CanvasLogoLoadingView({ alt={resolvedSiteName} className={cn( "h-[34px] w-[34px] object-contain", - hasSeparateDarkLogo - ? "dark:hidden" - : "dark:brightness-0 dark:invert" + hasSeparateDarkLogo && "dark:hidden" )} src={lightLogo} /> diff --git a/components/favicon-updater.tsx b/components/favicon-updater.tsx index 9c41c507b..6c9c5a0d4 100644 --- a/components/favicon-updater.tsx +++ b/components/favicon-updater.tsx @@ -5,24 +5,12 @@ import { useEffect } from "react"; export function FaviconUpdater() { useEffect(() => { let isDisposed = false; - const mediaQuery = window.matchMedia?.("(prefers-color-scheme: dark)"); - - const getFaviconUrl = () => { - const html = document.documentElement; - const hasDarkClass = html.classList.contains("dark"); - const hasLightClass = html.classList.contains("light"); - const prefersDark = Boolean(mediaQuery?.matches); - const isDark = hasDarkClass || (!hasLightClass && prefersDark); - return isDark ? "/api/favicon?theme=dark" : "/api/favicon"; - }; + const getFaviconUrl = () => "/api/favicon"; const updateFavicon = () => { if (isDisposed) return; const urlWithTheme = getFaviconUrl(); - const iconType = urlWithTheme.includes("theme=dark") - ? "image/png" - : "image/x-icon"; const href = `${urlWithTheme}${urlWithTheme.includes("?") ? "&" : "?"}v=${Date.now()}`; const managedSelector = @@ -35,12 +23,12 @@ export function FaviconUpdater() { link.href = href; link.setAttribute("data-managed-favicon", "true"); if (link.rel === "icon" || link.rel === "shortcut icon") { - link.type = iconType; + link.removeAttribute("type"); link.setAttribute("sizes", "any"); } }); - const ensureLink = (rel: string, type?: string) => { + const ensureLink = (rel: string) => { let link = document.querySelector( `link[rel="${rel}"]` ) as HTMLLinkElement | null; @@ -51,17 +39,15 @@ export function FaviconUpdater() { } link.href = href; link.setAttribute("data-managed-favicon", "true"); - if (type) { - link.type = type; - } + link.removeAttribute("type"); if (rel === "icon") { link.setAttribute("sizes", "any"); } }; - ensureLink("icon", iconType); + ensureLink("icon"); ensureLink("apple-touch-icon"); - ensureLink("shortcut icon", iconType); + ensureLink("shortcut icon"); }; const observer = new MutationObserver(updateFavicon); @@ -69,8 +55,6 @@ export function FaviconUpdater() { attributes: true, attributeFilter: ["class"], }); - mediaQuery?.addEventListener?.("change", updateFavicon); - Promise.resolve().then(() => { updateFavicon(); }); @@ -78,7 +62,6 @@ export function FaviconUpdater() { return () => { isDisposed = true; observer.disconnect(); - mediaQuery?.removeEventListener?.("change", updateFavicon); }; }, []); diff --git a/components/home/lovart-home.tsx b/components/home/lovart-home.tsx index 6751449a0..fe1a95412 100644 --- a/components/home/lovart-home.tsx +++ b/components/home/lovart-home.tsx @@ -677,8 +677,6 @@ export function LovartHome({ resolvedTheme === "dark" ? siteLogoDark || siteLogo || "" : siteLogo || siteLogoDark || ""; - const shouldInvertLogo = - resolvedTheme === "dark" && !siteLogoDark && Boolean(siteLogo); const { data: tutorialVideosData } = useSWR<{ videos: TutorialVideo[] }>( isSecondaryDataReady && siteSettingsData?.tutorialEnabled @@ -1564,7 +1562,6 @@ export function LovartHome({ contentClassName="pl-[98px]" logo={displayLogo} onLogoClick={() => window.scrollTo({ top: 0, behavior: "smooth" })} - shouldInvertLogo={shouldInvertLogo} showBrand={false} siteName={siteName || "Lovart"} > @@ -1814,7 +1811,6 @@ export function LovartHome({ ? "/settings/agent-workflow" : null } - shouldInvertLogo={shouldInvertLogo} siteLogo={displayLogo} siteName={siteName || "Lovart"} toolbox1Enabled={siteSettingsData?.toolbox1Enabled !== false} @@ -1852,7 +1848,6 @@ export function LovartHome({ prompt={prompt} removeFile={removeFile} setPrompt={setPrompt} - shouldInvertLogo={shouldInvertLogo} siteName={siteName} selectedModelId={selectedComposerModel.id} /> @@ -1990,7 +1985,7 @@ export function LovartHome({ {siteName} ) : (
diff --git a/components/home/lovart/lovart-hero-composer.tsx b/components/home/lovart/lovart-hero-composer.tsx index 9f5e73568..62ae0c66c 100644 --- a/components/home/lovart/lovart-hero-composer.tsx +++ b/components/home/lovart/lovart-hero-composer.tsx @@ -47,7 +47,6 @@ type LovartHeroComposerProps = { prompt: string; removeFile: (id: string) => void; setPrompt: (value: string) => void; - shouldInvertLogo: boolean; siteName: string; modelOptions: ModelOption[]; onModelChange: (modelId: string) => void; @@ -73,7 +72,6 @@ export function LovartHeroComposer({ prompt, removeFile, setPrompt, - shouldInvertLogo, siteName, modelOptions, onModelChange, @@ -104,7 +102,7 @@ export function LovartHeroComposer({ {siteName} ) : (
diff --git a/components/home/lovart/lovart-sidebar.tsx b/components/home/lovart/lovart-sidebar.tsx index e00a52458..cc9fdc394 100644 --- a/components/home/lovart/lovart-sidebar.tsx +++ b/components/home/lovart/lovart-sidebar.tsx @@ -42,7 +42,6 @@ type LovartSidebarProps = { } ) => ReactNode; settingsHref: string | null; - shouldInvertLogo?: boolean; siteLogo?: string | null; siteName: string; toolbox1Enabled: boolean; @@ -71,7 +70,6 @@ export function LovartSidebar({ productPlanningEnabled, renderIcon, settingsHref, - shouldInvertLogo = false, siteLogo, siteName, toolbox1Enabled, @@ -184,12 +182,7 @@ export function LovartSidebar({ {siteName} ) : ( diff --git a/components/layout/brand-top-bar.tsx b/components/layout/brand-top-bar.tsx index c8d79afa5..32af236b0 100644 --- a/components/layout/brand-top-bar.tsx +++ b/components/layout/brand-top-bar.tsx @@ -12,7 +12,6 @@ type BrandTopBarProps = { logo?: string | null; onLogoClick?: () => void; position?: "fixed" | "sticky" | "static"; - shouldInvertLogo?: boolean; showBrand?: boolean; siteName: string; }; @@ -25,7 +24,6 @@ export function BrandTopBar({ logo, onLogoClick, position = "fixed", - shouldInvertLogo = false, showBrand = true, siteName, }: BrandTopBarProps) { @@ -61,10 +59,7 @@ export function BrandTopBar({ {logo ? ( {resolvedSiteName} ) : ( diff --git a/components/pascal-editor/pascal-editor-frame.tsx b/components/pascal-editor/pascal-editor-frame.tsx index d2479a71e..cf9f93f47 100644 --- a/components/pascal-editor/pascal-editor-frame.tsx +++ b/components/pascal-editor/pascal-editor-frame.tsx @@ -59,8 +59,6 @@ export function PascalEditorFrame({ const lightLogo = String(siteLogo || "").trim(); return darkLogo || lightLogo; }, [siteLogo, siteLogoDark]); - const shouldInvertLogo = - !String(siteLogoDark || "").trim() && Boolean(String(siteLogo || "").trim()); const userEmail = String(user?.email || "").trim(); const userLabel = String(user?.name || userEmail || "已登录用户").trim(); const isGuest = guestRegex.test(userEmail); @@ -91,10 +89,7 @@ export function PascalEditorFrame({ {displayLogo ? ( {resolvedSiteName} ) : ( diff --git a/components/top-navbar.tsx b/components/top-navbar.tsx index 9e9ede5a1..0b944d4ab 100644 --- a/components/top-navbar.tsx +++ b/components/top-navbar.tsx @@ -150,7 +150,7 @@ export function TopNavbar({ } }; - // 决定显示哪个 LOGO（浅色用普通，深色用暗色） + // 彩色 logo 不能用滤镜自动反白；暗色主题只在配置了独立 dark logo 时切换。 const displayLogo = resolvedTheme === "dark" ? siteLogoDark && siteLogoDark.trim() !== "" @@ -159,10 +159,6 @@ export function TopNavbar({ : siteLogo && siteLogo.trim() !== "" ? siteLogo : siteLogoDark; - const shouldInvertLogo = - resolvedTheme === "dark" && - (!siteLogoDark || siteLogoDark.trim() === "") && - Boolean(siteLogo && siteLogo.trim() !== ""); const getShortLabel = (label: string) => { const trimmed = label.trim(); @@ -287,7 +283,7 @@ export function TopNavbar({ {displayLogo && ( {siteName} )} From fd5b483badd0be15f39dd8714cec64834b51a20e Mon Sep 17 00:00:00 2001 From: 134 Date: Wed, 8 Jul 2026 15:51:08 +0800 Subject: [PATCH 51/70] fix agent user-facing image edit preface --- app/(chat)/api/chat/route.ts | 13 ++-- lib/ai/auto-agent-graph.ts | 4 ++ lib/ai/runtime/direct-agent-execution.ts | 66 +++++++++++++++++++ .../runtime/direct-agent-response-builders.ts | 8 +-- tests/routes/canvas-chat-helpers.test.ts | 13 ++++ 5 files changed, 96 insertions(+), 8 deletions(-) diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index 93fd9f99c..e4015b6e6 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -239,6 +239,8 @@ import { safePersistDirectAgentVideoFailedTurn, } from "@/lib/ai/runtime/direct-agent-working-state"; import { + getDirectAgentToolDisplayName, + getDirectAgentToolPrefaceFallback, getDirectAgentToolErrorMessage, getDirectAgentToolLabel, getDirectAgentToolTargetMissingMessage, @@ -8296,6 +8298,9 @@ export async function POST(request: Request) { const assistantMessageId = generateUUID(); const directToolLabel = getDirectAgentToolLabel(directAgentToolCapability); + const directToolDisplayName = getDirectAgentToolDisplayName( + directAgentToolCapability, + ); const directToolStream = createUIMessageStream({ execute: async ({ writer }) => { await writeAgentExecutionPrefaceMessage({ @@ -8303,7 +8308,7 @@ export async function POST(request: Request) { chatId: id, label: `agent:${directAgentToolCapability}:preface`, friendlyMessage: smartAutoFriendlyMessage, - fallback: `好的，我来帮你${directToolLabel}，稍等我处理一下。`, + fallback: getDirectAgentToolPrefaceFallback(directAgentToolCapability), }); writeAgentProgressChunk( writer, @@ -8318,7 +8323,7 @@ export async function POST(request: Request) { buildDirectAgentImageEditSelectProgress({ toolName: directAgentExecution.toolName, capability: directAgentToolCapability, - label: directToolLabel, + label: directToolDisplayName, }), `agent:${directAgentToolCapability}:progress-select-tool`, ); @@ -8331,7 +8336,7 @@ export async function POST(request: Request) { buildDirectAgentImageEditExecutingProgress({ toolName: directAgentExecution.toolName, capability: directAgentToolCapability, - label: directToolLabel, + label: directToolDisplayName, }), `agent:${directAgentToolCapability}:progress-executing`, ); @@ -8480,7 +8485,7 @@ export async function POST(request: Request) { buildDirectAgentImageEditCompletedProgress({ toolName: directAgentExecution.toolName, capability: directAgentToolCapability, - label: directToolLabel, + label: directToolDisplayName, }), `agent:${directAgentToolCapability}:progress-completed`, ); diff --git a/lib/ai/auto-agent-graph.ts b/lib/ai/auto-agent-graph.ts index ed723f6c3..bc722c0b7 100644 --- a/lib/ai/auto-agent-graph.ts +++ b/lib/ai/auto-agent-graph.ts @@ -1178,6 +1178,10 @@ function buildStrategicFriendlyMessage( analysis.intent === "image_gen" || analysis.intent === "video_gen" ) { + if (existingReply) { + return Promise.resolve(existingReply); + } + const imageParams = heuristicImageParams(state.text, analysis); const planningBrief = buildAgentPlanningBriefMarkdown({ userText: state.text, diff --git a/lib/ai/runtime/direct-agent-execution.ts b/lib/ai/runtime/direct-agent-execution.ts index 79a6f6642..517da2047 100644 --- a/lib/ai/runtime/direct-agent-execution.ts +++ b/lib/ai/runtime/direct-agent-execution.ts @@ -503,6 +503,72 @@ export function getDirectAgentToolLabel(capability: DirectAgentToolCapability): } } +export function getDirectAgentToolDisplayName( + capability: DirectAgentToolCapability, +): string { + switch (capability) { + case "analyze": + return "图片分析"; + case "generic_edit": + return "图片编辑"; + case "remove_background": + return "抠图"; + case "remove_watermark": + return "去水印"; + case "enhance": + return "图像增强"; + case "resize": + return "尺寸调整"; + case "extend": + return "图像拓展"; + case "angle_variant": + return "多视角"; + case "local_edit": + return "局部编辑"; + case "box_cutout": + return "框选抠图"; + case "generate_image": + return "图片生成"; + case "generate_video": + return "视频生成"; + default: + return "图像处理"; + } +} + +export function getDirectAgentToolPrefaceFallback( + capability: DirectAgentToolCapability, +): string { + switch (capability) { + case "analyze": + return "好的，我先帮你看看这张图，稍等我一下。"; + case "generic_edit": + return "好的，我来帮你修改这张图，稍等我一下。"; + case "remove_background": + return "好的，我来帮你抠出主体，稍等我一下。"; + case "remove_watermark": + return "好的，我来帮你去掉水印，稍等我一下。"; + case "enhance": + return "好的，我来帮你增强这张图，稍等我一下。"; + case "resize": + return "好的，我来帮你调整图片尺寸，稍等我一下。"; + case "extend": + return "好的，我来帮你扩展这张图，稍等我一下。"; + case "angle_variant": + return "好的，我来帮你生成新的视角，稍等我一下。"; + case "local_edit": + return "好的，我来帮你修改这个局部区域，稍等我一下。"; + case "box_cutout": + return "好的，我来帮你把框选区域抠出来，稍等我一下。"; + case "generate_image": + return "好的，我按你的描述来做一版画面，稍等我处理一下。"; + case "generate_video": + return "好的，我按你的需求来做这段视频，稍等我处理一下。"; + default: + return "好的，我来继续处理，稍等我一下。"; + } +} + export function findAgentToolCallArgs( toolName: AgentPlannedToolCall["name"], execution?: AgentPreparedExecution, diff --git a/lib/ai/runtime/direct-agent-response-builders.ts b/lib/ai/runtime/direct-agent-response-builders.ts index 0576f56d1..6a867f3e7 100644 --- a/lib/ai/runtime/direct-agent-response-builders.ts +++ b/lib/ai/runtime/direct-agent-response-builders.ts @@ -5,7 +5,7 @@ import { prependImageModelBadge, } from "@/lib/ai/runtime/agent-media-markdown"; import { - getDirectAgentToolLabel, + getDirectAgentToolDisplayName, getDirectAgentToolSuccessMessage, type DirectAgentToolCapability, } from "@/lib/ai/runtime/direct-agent-execution"; @@ -73,7 +73,7 @@ export function buildDirectAgentAnalysisAssistantText(params: { ] .filter(Boolean) .join("\n\n"), - getDirectAgentToolLabel("analyze"), + getDirectAgentToolDisplayName("analyze"), ); } @@ -121,7 +121,7 @@ export function buildDirectAgentImageEditAssistantText(params: { }), followUp: getDirectAgentFollowUpMessage(params.capability), }), - getDirectAgentToolLabel(params.capability), + getDirectAgentToolDisplayName(params.capability), ); } @@ -139,6 +139,6 @@ export function buildDirectAgentBatchImageEditAssistantText(params: { mediaMarkdown: buildAgentToolImagesMarkdown(params.results), followUp: getDirectAgentFollowUpMessage(params.capability), }), - getDirectAgentToolLabel(params.capability), + getDirectAgentToolDisplayName(params.capability), ); } diff --git a/tests/routes/canvas-chat-helpers.test.ts b/tests/routes/canvas-chat-helpers.test.ts index d2175bb3d..c4b94d227 100644 --- a/tests/routes/canvas-chat-helpers.test.ts +++ b/tests/routes/canvas-chat-helpers.test.ts @@ -142,7 +142,9 @@ import { shouldBypassCanvasImagePointsPrecheckForAgentControl, } from "@/lib/canvas/agent-client-intent"; import { + getDirectAgentToolDisplayName, getDirectAgentToolLabel, + getDirectAgentToolPrefaceFallback, getDirectAgentToolSuccessMessage, hasExplicitDirectAgentBatchImageEditIntent, isDirectAgentBatchImageEditCapability, @@ -1313,9 +1315,20 @@ test.describe("canvas chat pure helpers", () => { expect(editText).toContain("### 执行结果"); expect(editText).toContain("- 已完成抠图处理，结果如下。"); expect(editText).toContain("https://cdn.example.com/cutout.png"); + expect(editText).toContain("◈ 抠图"); + expect(editText).not.toContain("Agent工具："); expect(editText).toContain("替换背景"); }); + test("uses user-facing direct agent tool labels in fallbacks", () => { + expect(getDirectAgentToolDisplayName("generic_edit")).toBe("图片编辑"); + expect(getDirectAgentToolDisplayName("local_edit")).toBe("局部编辑"); + expect(getDirectAgentToolPrefaceFallback("generic_edit")).toBe( + "好的，我来帮你修改这张图，稍等我一下。" + ); + expect(getDirectAgentToolPrefaceFallback("generic_edit")).not.toContain("Agent工具："); + }); + test("builds capability-aware agent progress node labels", () => { const selectingSteps = buildAgentProgressNodeSteps({ stage: "planning", From 7ac4a25a5f1e5d19da055f4663ba0d78929cae5f Mon Sep 17 00:00:00 2001 From: 134 Date: Wed, 8 Jul 2026 20:13:36 +0800 Subject: [PATCH 52/70] fix: honor requested fal scale adjust model --- app/api/canvas/local-edit/route.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 9d2755bfa..5d7d36d7d 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -493,6 +493,13 @@ function resolveScaleAdjustPreferredModelId(params: { params.models, params.configuredModelId ); + if ( + requestedModel?.provider.type === "fal-image" && + requestedModel.model.healthStatus !== "auto_disabled" && + isScaleAdjustUsableImageEditModel(requestedModel) + ) { + return requestedModel.model.id || params.requestedModelId; + } if ( params.honorRequestedModelBeforeGpt && requestedModel && From ff71e3935c4f3b2af96ec48cf743b23bc5fa2608 Mon Sep 17 00:00:00 2001 From: 134 Date: Wed, 8 Jul 2026 20:33:10 +0800 Subject: [PATCH 53/70] fix: reject scale adjust old footprint remnants --- app/api/canvas/local-edit/route.ts | 33 +++- lib/canvas/generate-image.ts | 267 ++++++++++++++++++++++++++++- 2 files changed, 293 insertions(+), 7 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 5d7d36d7d..ba0919323 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -126,6 +126,7 @@ type ScaleAdjustCropReference = { type ScaleAdjustPatchReference = { backgroundPlateCropUrl?: string | null; backgroundPlateFullUrl?: string | null; + compositeClipMaskCropUrl?: string | null; compositeClipMaskUrl?: string | null; repairBlendMaskUrl?: string | null; productLockMethod?: string | null; @@ -4066,6 +4067,12 @@ async function createScaleAdjustPatchReference(params: { .png({ compressionLevel: 6 }) .toBuffer() : null; + const compositeClipMaskCropBuffer = targetDraftReference.compositeClipMaskBuffer + ? await sharp(targetDraftReference.compositeClipMaskBuffer, { failOn: "none" }) + .extract(workspacePixelRect) + .png({ compressionLevel: 6 }) + .toBuffer() + : null; const [sourceCropUpload, targetDraftUpload, targetDraftFullUpload] = await Promise.all([ uploadScaleAdjustPrepAsset( @@ -4087,6 +4094,7 @@ async function createScaleAdjustPatchReference(params: { productLockOverlayUpload, productLockProtectMaskUpload, compositeClipMaskUpload, + compositeClipMaskCropUpload, repairBlendMaskUpload, productLockEditMaskUpload, productLockFalEditMaskUpload, @@ -4111,6 +4119,10 @@ async function createScaleAdjustPatchReference(params: { `canvas-scale-adjust-composite-clip-mask-${suffix}.png`, targetDraftReference.compositeClipMaskBuffer ), + uploadOptionalScaleAdjustAsset( + `canvas-scale-adjust-composite-clip-mask-crop-${suffix}.png`, + compositeClipMaskCropBuffer + ), uploadOptionalScaleAdjustAsset( `canvas-scale-adjust-repair-blend-mask-${suffix}.png`, targetDraftReference.repairBlendMaskBuffer @@ -4134,6 +4146,7 @@ async function createScaleAdjustPatchReference(params: { return { backgroundPlateCropUrl: backgroundPlateCropUpload?.url || null, backgroundPlateFullUrl: backgroundPlateFullUpload?.url || null, + compositeClipMaskCropUrl: compositeClipMaskCropUpload?.url || null, compositeClipMaskUrl: compositeClipMaskUpload?.url || null, repairBlendMaskUrl: repairBlendMaskUpload?.url || null, productLockMethod: productLockOverlayUpload ? "rmbg-cutout" : null, @@ -5201,7 +5214,9 @@ function buildScaleAdjustProductLockBackgroundCropPrompt( return [ "Cropped background repair for product-locked scale adjustment.", `Return exactly one clean realistic crop with the same framing, aspect ratio, and pixel extent as input image 1. This crop maps to full-photo pixels left=${workspace.left}, top=${workspace.top}, width=${workspace.width}, height=${workspace.height}.`, - "Input image 1 is the original crop from the source photo. Input image 2 is a rough empty-background repair guide crop where the selected furniture/product footprint has been wiped.", + patchReference.compositeClipMaskCropUrl + ? "Input image 1 is the original crop from the source photo. Input image 2 is a black/white alpha-mask guide only: white/bright pixels mark the old selected product footprint and target repair boundary to erase or keep empty; black pixels are context to preserve. Do not render the mask itself." + : "Input image 1 is the original crop from the source photo. Input image 2 is a rough empty-background repair guide crop where the selected furniture/product footprint has been wiped.", "Important: do not draw, recreate, move, scale, duplicate, or redesign the selected furniture/product in your output. The system will composite the locked original product pixels into the target footprint after your crop is returned.", "Your only job is to repair the old footprint into natural wall, curtain, floor, rug, shadow, sofa edge, and background, while keeping non-target people and non-target objects stable.", "If input image 2 contains haze, gray wash, ghost silhouettes, crop edges, black holes, rough mask shapes, or pasted fragments, treat them as repair hints only. Do not preserve them as visible content.", @@ -5578,7 +5593,10 @@ export async function POST(request: NextRequest) { const useScaleAdjustProductLockCropBackground = useScaleAdjustProductLockComposite && !useScaleAdjustFullFrameStructurePatch && - Boolean(scaleAdjustPatchReference?.backgroundPlateCropUrl); + Boolean( + scaleAdjustPatchReference?.compositeClipMaskCropUrl || + scaleAdjustPatchReference?.backgroundPlateCropUrl + ); const useScaleAdjustPatchTargetDraftPrimary = Boolean(scaleAdjustPatchReference); const includeScaleAdjustPatchSourceReference = Boolean(scaleAdjustPatchReference); @@ -5696,7 +5714,8 @@ export async function POST(request: NextRequest) { : useScaleAdjustProductLockCropBackground ? [ scaleAdjustPatchReference.sourceCropUrl, - scaleAdjustPatchReference.backgroundPlateCropUrl as string, + (scaleAdjustPatchReference.compositeClipMaskCropUrl || + scaleAdjustPatchReference.backgroundPlateCropUrl) as string, ...scaleAdjustCropReferenceImageUrls, ...manualRegionReferenceImageUrls, ] @@ -5762,6 +5781,12 @@ export async function POST(request: NextRequest) { ), } : {}), + ...(scaleAdjustPatchReference?.compositeClipMaskUrl + ? { + oldFootprintMaskImageUrl: + scaleAdjustPatchReference.compositeClipMaskUrl, + } + : {}), } : undefined; const requestedImageSize = @@ -5877,6 +5902,8 @@ export async function POST(request: NextRequest) { scaleAdjustPatchReference?.productLockProtectMaskUrl || null, scaleAdjustPatchCompositeClipMaskUrl: scaleAdjustPatchReference?.compositeClipMaskUrl || null, + scaleAdjustPatchCompositeClipMaskCropUrl: + scaleAdjustPatchReference?.compositeClipMaskCropUrl || null, scaleAdjustRepairBlendMaskUrl: scaleAdjustPatchReference?.repairBlendMaskUrl || null, scaleAdjustProviderMaskImageUrl: useScaleAdjustProductLockMaskEdit diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index 3e31bef1d..f7fa5656f 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -83,6 +83,7 @@ export interface CanvasScaleAdjustProductLockCompositeConfig { export interface CanvasScaleAdjustVerificationConfig { allowDeterministicFallback?: boolean; allowOutsideChangedWhenChangedInside?: boolean; + oldFootprintMaskImageUrl?: string; protectedRegions?: CanvasLocalEditCompositeRegion[]; rejectFailedCandidates?: boolean; regionPrompts?: string[]; @@ -177,6 +178,8 @@ class ScaleAdjustOutputRejectedError extends Error { ? "比例调整结果改变了框选区域外的画面，已拒绝该模型结果" : params.reason === "rectangular-seam-detected" ? "比例调整结果出现明显矩形拼接硬边，已拒绝该模型结果" + : params.reason === "old-footprint-remnant-detected" + ? "比例调整结果保留了旧尺寸残影，已拒绝该模型结果" : "比例调整结果没有产生有效变化，已拒绝该模型结果"; super(message); this.name = "ScaleAdjustOutputRejectedError"; @@ -215,6 +218,10 @@ const SCALE_ADJUST_SHRINK_OUTSIDE_FOOTPRINT_RELAXED_MAX_RATIO = 0.84; const SCALE_ADJUST_SHRINK_OUTSIDE_FOOTPRINT_MIN_REMOVED_RATIO = 0.14; const SCALE_ADJUST_SHRINK_MIN_SOURCE_OUTSIDE_PIXELS = 60; const SCALE_ADJUST_SHRINK_TARGET_FOOTPRINT_MARGIN = 0.018; +const SCALE_ADJUST_OLD_FOOTPRINT_SIMILAR_DIFF_THRESHOLD = 56; +const SCALE_ADJUST_OLD_FOOTPRINT_MIN_PIXELS = 80; +const SCALE_ADJUST_OLD_FOOTPRINT_SIMILAR_RATIO_THRESHOLD = 0.18; +const SCALE_ADJUST_OLD_FOOTPRINT_DARK_RATIO_THRESHOLD = 0.22; const SCALE_ADJUST_RECTANGULAR_SEAM_CHANGE_THRESHOLD = 18; const SCALE_ADJUST_RECTANGULAR_SEAM_MIN_EDGE_DIFF = 5; const SCALE_ADJUST_RECTANGULAR_SEAM_MAX_EDGE_DIFF = 54; @@ -612,6 +619,10 @@ function normalizeScaleAdjustVerificationConfig( if (!value || typeof value !== "object") return null; const sourceImageUrl = String(value.sourceImageUrl || "").trim(); if (!/^https?:\/\//i.test(sourceImageUrl)) return null; + const oldFootprintMaskImageUrl = String( + (value as { oldFootprintMaskImageUrl?: unknown }).oldFootprintMaskImageUrl || + "" + ).trim(); if (!Array.isArray(value.regions) || value.regions.length === 0) return null; const rawScalePercent = Number(value.scalePercent); const scalePercent = @@ -651,6 +662,9 @@ function normalizeScaleAdjustVerificationConfig( allowOutsideChangedWhenChangedInside: value.allowOutsideChangedWhenChangedInside === true, rejectFailedCandidates: value.rejectFailedCandidates === true, + ...(/^https?:\/\//i.test(oldFootprintMaskImageUrl) + ? { oldFootprintMaskImageUrl } + : {}), ...(regionPrompts.length > 0 ? { regionPrompts } : {}), ...(targetRegions.length > 0 ? { targetRegions } : {}), ...(typeof scalePercent === "number" ? { scalePercent } : {}), @@ -1466,6 +1480,196 @@ async function getScaleAdjustFootprintChangeStats(params: { }; } +async function loadScaleAdjustOldFootprintMask(params: { + height: number; + url?: string; + width: number; +}) { + const { height, url, width } = params; + if (!url) return null; + try { + return await sharp(await fetchImageBufferFromUrl(url), { failOn: "none" }) + .rotate() + .resize(width, height, { fit: "fill" }) + .ensureAlpha() + .extractChannel(3) + .greyscale() + .raw() + .toBuffer(); + } catch (error) { + console.warn("[Canvas Generate Image] Failed to load scale adjust old footprint mask:", { + error: error instanceof Error ? error.message : String(error), + url, + }); + return null; + } +} + +async function getScaleAdjustOldFootprintGhostStats(params: { + generatedBuffer: Buffer; + sourceBuffer: Buffer; + sourceWidth: number; + sourceHeight: number; + verification: CanvasScaleAdjustVerificationConfig; +}) { + const { generatedBuffer, sourceBuffer, sourceHeight, sourceWidth, verification } = + params; + const normalizeToRaw = (buffer: Buffer) => + sharp(buffer, { failOn: "none" }) + .rotate() + .resize(sourceWidth, sourceHeight, { + fit: "cover", + position: "center", + }) + .removeAlpha() + .toColorspace("srgb") + .raw() + .toBuffer(); + const [sourceRaw, generatedRaw, explicitMaskRaw] = await Promise.all([ + normalizeToRaw(sourceBuffer), + normalizeToRaw(generatedBuffer), + loadScaleAdjustOldFootprintMask({ + height: sourceHeight, + url: verification.oldFootprintMaskImageUrl, + width: sourceWidth, + }), + ]); + + const shrinkStats: Array<{ + detected: boolean; + darkResidualPixels: number; + darkResidualRatio: number; + maskSource: "explicit" | "fallback"; + regionIndex: number; + similarPixels: number; + similarRatio: number; + sourceOutsidePixels: number; + }> = []; + + for (const [index, region] of verification.regions.entries()) { + const targetRegion = + verification.targetRegions?.[index] || + (typeof verification.scalePercent === "number" + ? scaleRegionAroundCenter(region, verification.scalePercent) + : null); + if (!targetRegion) continue; + const widthFactor = region.width > 0 ? targetRegion.width / region.width : 1; + const heightFactor = region.height > 0 ? targetRegion.height / region.height : 1; + const isShrink = widthFactor < 0.92 || heightFactor < 0.92; + if (!isShrink) continue; + + const sourceBox = rectToPixelBox({ + height: sourceHeight, + region, + width: sourceWidth, + }); + const targetBox = rectToPixelBox({ + height: sourceHeight, + region: expandCanvasRegion( + targetRegion, + SCALE_ADJUST_SHRINK_TARGET_FOOTPRINT_MARGIN * 1.35 + ), + width: sourceWidth, + }); + const fallbackMask = buildScaleAdjustFallbackSubjectMask({ + box: sourceBox, + raw: sourceRaw, + width: sourceWidth, + }); + const background = estimateScaleAdjustFallbackBackground({ + box: sourceBox, + raw: sourceRaw, + width: sourceWidth, + }); + let sourceOutsidePixels = 0; + let similarPixels = 0; + let darkResidualPixels = 0; + let explicitPixels = 0; + + for (let localY = 0; localY < sourceBox.height; localY += 1) { + const y = sourceBox.top + localY; + for (let localX = 0; localX < sourceBox.width; localX += 1) { + const x = sourceBox.left + localX; + const insideTarget = + x >= targetBox.left && + x < targetBox.left + targetBox.width && + y >= targetBox.top && + y < targetBox.top + targetBox.height; + if (insideTarget) continue; + const localIndex = localY * sourceBox.width + localX; + const absoluteIndex = y * sourceWidth + x; + const explicitMaskValue = explicitMaskRaw?.[absoluteIndex] || 0; + const maskHit = explicitMaskValue > 18 || fallbackMask[localIndex] > 0; + if (!maskHit) continue; + if (explicitMaskValue > 18) explicitPixels += 1; + + const rawIndex = getRawPixelIndex(sourceWidth, x, y); + const sourceR = sourceRaw[rawIndex] || 0; + const sourceG = sourceRaw[rawIndex + 1] || 0; + const sourceB = sourceRaw[rawIndex + 2] || 0; + const generatedR = generatedRaw[rawIndex] || 0; + const generatedG = generatedRaw[rawIndex + 1] || 0; + const generatedB = generatedRaw[rawIndex + 2] || 0; + const sourceLuma = getLuma(sourceR, sourceG, sourceB); + const generatedLuma = getLuma(generatedR, generatedG, generatedB); + const sourceBackgroundDistance = Math.sqrt( + (sourceR - background.r) ** 2 + + (sourceG - background.g) ** 2 + + (sourceB - background.b) ** 2 + ); + if (sourceBackgroundDistance < 24 && explicitMaskValue <= 18) continue; + + sourceOutsidePixels += 1; + const sourceGeneratedDiff = Math.sqrt( + (sourceR - generatedR) ** 2 + + (sourceG - generatedG) ** 2 + + (sourceB - generatedB) ** 2 + ); + if ( + sourceGeneratedDiff <= + SCALE_ADJUST_OLD_FOOTPRINT_SIMILAR_DIFF_THRESHOLD + ) { + similarPixels += 1; + } + if ( + generatedLuma <= background.luma - 32 && + sourceLuma <= background.luma - 18 + ) { + darkResidualPixels += 1; + } + } + } + + const similarRatio = + sourceOutsidePixels > 0 ? similarPixels / sourceOutsidePixels : 0; + const darkResidualRatio = + sourceOutsidePixels > 0 ? darkResidualPixels / sourceOutsidePixels : 0; + const enoughPixels = + sourceOutsidePixels >= SCALE_ADJUST_OLD_FOOTPRINT_MIN_PIXELS; + const detected = + enoughPixels && + (similarRatio >= SCALE_ADJUST_OLD_FOOTPRINT_SIMILAR_RATIO_THRESHOLD || + darkResidualRatio >= SCALE_ADJUST_OLD_FOOTPRINT_DARK_RATIO_THRESHOLD || + (similarRatio >= 0.12 && darkResidualRatio >= 0.14)); + shrinkStats.push({ + detected, + darkResidualPixels, + darkResidualRatio, + maskSource: explicitPixels > 0 ? "explicit" : "fallback", + regionIndex: index, + similarPixels, + similarRatio, + sourceOutsidePixels, + }); + } + + return { + detected: shrinkStats.some((item) => item.detected), + shrinkRegionCount: shrinkStats.length, + shrinkStats, + }; +} + async function loadScaleAdjustSourceImage( verification: CanvasScaleAdjustVerificationConfig ) { @@ -1690,6 +1894,10 @@ async function applyScaleAdjustVerificationToImages(params: { cleanedImage?: ParsedImage; diffScores: number[]; footprintStats: Awaited>; + oldFootprintGhostDetected: boolean; + oldFootprintGhostStats: Awaited< + ReturnType + >; image: ParsedImage; index: number; outsideChanged: boolean; @@ -1712,6 +1920,7 @@ async function applyScaleAdjustVerificationToImages(params: { outsideDiffScores, outsidePixelStats, footprintStats, + oldFootprintGhostStats, rectangularSeamStats, ] = await Promise.all([ getScaleAdjustDiffScores({ @@ -1749,6 +1958,13 @@ async function applyScaleAdjustVerificationToImages(params: { sourceWidth: sourceImage.sourceWidth, verification, }), + getScaleAdjustOldFootprintGhostStats({ + generatedBuffer, + sourceBuffer: sourceImage.sourceBuffer, + sourceHeight: sourceImage.sourceHeight, + sourceWidth: sourceImage.sourceWidth, + verification, + }), getScaleAdjustRectangularSeamStats({ generatedBuffer, sourceBuffer: sourceImage.sourceBuffer, @@ -1772,6 +1988,7 @@ async function applyScaleAdjustVerificationToImages(params: { const outsideChanged = isScaleAdjustOutsideChanged(outsideDiffScores) || isScaleAdjustOutsidePixelChanged(outsidePixelStats); + const oldFootprintGhostDetected = oldFootprintGhostStats.detected; const rectangularSeamDetected = rectangularSeamStats.detected; console.info("[Canvas Generate Image] Scale adjust result guard:", { @@ -1795,6 +2012,18 @@ async function applyScaleAdjustVerificationToImages(params: { }, imageIndex: index, outsideChanged, + oldFootprintGhostDetected, + oldFootprintGhostStats: { + shrinkRegionCount: oldFootprintGhostStats.shrinkRegionCount, + shrinkStats: oldFootprintGhostStats.shrinkStats.map((item) => ({ + darkResidualRatio: Number(item.darkResidualRatio.toFixed(3)), + detected: item.detected, + maskSource: item.maskSource, + regionIndex: item.regionIndex, + similarRatio: Number(item.similarRatio.toFixed(3)), + sourceOutsidePixels: item.sourceOutsidePixels, + })), + }, outsideDiffScores: outsideDiffScores.map((score) => Number(score.toFixed(2))), outsidePixelStats: { changedPixelCount: outsidePixelStats.changedPixelCount, @@ -1849,6 +2078,7 @@ async function applyScaleAdjustVerificationToImages(params: { ...(outsideChanged ? ["outside-frame-changed"] : []), ...(!changedInside ? ["no-effective-scale-change"] : []), ...(rectangularSeamDetected ? ["rectangular-seam-detected"] : []), + ...(oldFootprintGhostDetected ? ["old-footprint-remnant-detected"] : []), ]; if (guardWarnings.length > 0) { console.warn("[Canvas Generate Image] Scale adjust result guard warning; keeping result for preview:", { @@ -1873,6 +2103,18 @@ async function applyScaleAdjustVerificationToImages(params: { })), }, outsideChanged, + oldFootprintGhostDetected, + oldFootprintGhostStats: { + shrinkRegionCount: oldFootprintGhostStats.shrinkRegionCount, + shrinkStats: oldFootprintGhostStats.shrinkStats.map((item) => ({ + darkResidualRatio: Number(item.darkResidualRatio.toFixed(3)), + detected: item.detected, + maskSource: item.maskSource, + regionIndex: item.regionIndex, + similarRatio: Number(item.similarRatio.toFixed(3)), + sourceOutsidePixels: item.sourceOutsidePixels, + })), + }, outsideDiffScores: outsideDiffScores.map((score) => Number(score.toFixed(2))), outsidePixelStats: { changedPixelCount: outsidePixelStats.changedPixelCount, @@ -1939,6 +2181,8 @@ async function applyScaleAdjustVerificationToImages(params: { cleanedImage, diffScores, footprintStats, + oldFootprintGhostDetected, + oldFootprintGhostStats, image, index, outsideChanged, @@ -1963,6 +2207,8 @@ async function applyScaleAdjustVerificationToImages(params: { cleanedImage, diffScores, footprintStats, + oldFootprintGhostDetected, + oldFootprintGhostStats, image, index, outsideChanged, @@ -1981,19 +2227,27 @@ async function applyScaleAdjustVerificationToImages(params: { const outsidePenalty = item.outsideChanged ? 100 : 0; const noChangePenalty = item.changedInside ? 0 : 60; const seamPenalty = item.rectangularSeamDetected ? 140 : 0; + const ghostPenalty = item.oldFootprintGhostDetected ? 160 : 0; return ( insideScore - outsideMean * 0.35 - outsidePenalty - noChangePenalty - - seamPenalty + seamPenalty - + ghostPenalty ); }; const rankedImages = [...evaluatedImages].sort((left, right) => { const leftPassed = - left.changedInside && !left.outsideChanged && !left.rectangularSeamDetected; + left.changedInside && + !left.outsideChanged && + !left.rectangularSeamDetected && + !left.oldFootprintGhostDetected; const rightPassed = - right.changedInside && !right.outsideChanged && !right.rectangularSeamDetected; + right.changedInside && + !right.outsideChanged && + !right.rectangularSeamDetected && + !right.oldFootprintGhostDetected; if (leftPassed !== rightPassed) return leftPassed ? -1 : 1; return scoreEvaluation(right) - scoreEvaluation(left); }); @@ -2006,8 +2260,10 @@ async function applyScaleAdjustVerificationToImages(params: { passed: selected.changedInside && !selected.outsideChanged && - !selected.rectangularSeamDetected, + !selected.rectangularSeamDetected && + !selected.oldFootprintGhostDetected, changedInside: selected.changedInside, + oldFootprintGhostDetected: selected.oldFootprintGhostDetected, outsideChanged: selected.outsideChanged, rectangularSeamDetected: selected.rectangularSeamDetected, changeRegionDiffScores: selected.changeDiffScores.map((score) => @@ -2044,11 +2300,14 @@ async function applyScaleAdjustVerificationToImages(params: { verification.rejectFailedCandidates && (!selected.changedInside || selected.rectangularSeamDetected || + selected.oldFootprintGhostDetected || (selected.outsideChanged && !outsideChangeAccepted)) ) { throw new ScaleAdjustOutputRejectedError({ reason: selected.rectangularSeamDetected ? "rectangular-seam-detected" + : selected.oldFootprintGhostDetected + ? "old-footprint-remnant-detected" : selected.outsideChanged ? "outside-frame-changed" : "no-effective-scale-change", From 3aa031a4e4f986b3c44be844a6ce67f1a7689d20 Mon Sep 17 00:00:00 2001 From: 134 Date: Wed, 8 Jul 2026 20:41:19 +0800 Subject: [PATCH 54/70] fix: upload normalized source for fal scale edit --- app/api/canvas/local-edit/route.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index ba0919323..e238c1341 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -3984,6 +3984,15 @@ async function createScaleAdjustPatchReference(params: { return uploadScaleAdjustPrepAsset(filename, buffer); }; if (params.fullFrameReferenceOnly && structureLockAppliedRef.value) { + const sourceFullUploadBuffer = await sharp(sourceBuffer, { failOn: "none" }) + .rotate() + .flatten({ background: "#ffffff" }) + .jpeg({ quality: 94, mozjpeg: true }) + .toBuffer(); + const sourceFullUpload = await uploadScaleAdjustPrepAsset( + `canvas-scale-adjust-full-source-${suffix}.jpg`, + sourceFullUploadBuffer + ); const targetDraftFullUpload = await uploadScaleAdjustPrepAsset( `canvas-scale-adjust-full-target-draft-${suffix}.jpg`, targetDraftUploadBuffer @@ -4042,7 +4051,7 @@ async function createScaleAdjustPatchReference(params: { productLockFalEditMaskUrl: productLockFalEditMaskUpload?.url || null, productLockOverlayUrl: productLockOverlayUpload?.url || null, productLockProtectMaskUrl: productLockProtectMaskUpload?.url || null, - sourceCropUrl: sourceImageUrl || fullFrameFallbackUrl, + sourceCropUrl: sourceFullUpload?.url || sourceImageUrl || fullFrameFallbackUrl, structureLockedDraft: structureLockAppliedRef.value, targetDraftFullUrl: fullFrameFallbackUrl, targetDraftUrl: fullFrameFallbackUrl, @@ -5695,12 +5704,12 @@ export async function POST(request: NextRequest) { ? useScaleAdjustProductLockComposite ? useScaleAdjustProductLockMaskEdit ? [ - sourceImageUrl, + scaleAdjustPatchReference.sourceCropUrl, ...scaleAdjustCropReferenceImageUrls, ...manualRegionReferenceImageUrls, ] : [ - sourceImageUrl, + scaleAdjustPatchReference.sourceCropUrl, scaleAdjustPatchReference.backgroundPlateFullUrl as string, ...scaleAdjustCropReferenceImageUrls, ...manualRegionReferenceImageUrls, From 6fe69de8de1febfef8d291a0cae0dfc02b5ec6b3 Mon Sep 17 00:00:00 2001 From: 134 Date: Wed, 8 Jul 2026 20:56:03 +0800 Subject: [PATCH 55/70] fix: upload fal edit inputs to fal storage --- lib/ai/providers/fal-image.ts | 83 ++++++++++++++++++++++++++++++++--- 1 file changed, 77 insertions(+), 6 deletions(-) diff --git a/lib/ai/providers/fal-image.ts b/lib/ai/providers/fal-image.ts index 1c2b28bdf..b9c517ccd 100644 --- a/lib/ai/providers/fal-image.ts +++ b/lib/ai/providers/fal-image.ts @@ -654,6 +654,64 @@ function createFalClientFromApiKey(apiKey: string) { ); } +async function uploadImageUrlToFalStorage(params: { + fal: Awaited>; + url: string; +}) { + const { fal, url } = params; + if (/fal\.(?:media|run)\//i.test(url) || /fal-ai/i.test(url)) return url; + + const response = await fetch(url, { signal: AbortSignal.timeout(60_000) }); + if (!response.ok) { + throw new Error(`FAL input image download failed before storage upload: ${response.status}`); + } + + const contentType = response.headers.get("content-type") || "image/png"; + const buffer = Buffer.from(await response.arrayBuffer()); + const blob = new Blob([buffer], { type: contentType }); + return fal.storage.upload(blob); +} + +async function prepareFalInputImageUrls(params: { + fal: Awaited>; + input: Record; + sourceImageUrls: string[]; +}) { + const { fal, input, sourceImageUrls } = params; + const cache = new Map>(); + const upload = (url: string) => { + const normalized = String(url || "").trim(); + if (!/^https?:\/\//i.test(normalized)) return Promise.resolve(normalized); + const existing = cache.get(normalized); + if (existing) return existing; + const uploaded = uploadImageUrlToFalStorage({ fal, url: normalized }); + cache.set(normalized, uploaded); + return uploaded; + }; + + const preparedInput = { ...input }; + if (typeof preparedInput.image_url === "string") { + preparedInput.image_url = await upload(preparedInput.image_url); + } + if (Array.isArray(preparedInput.image_urls)) { + preparedInput.image_urls = await Promise.all( + preparedInput.image_urls.map((url) => upload(String(url || ""))) + ); + } + if (typeof preparedInput.mask_url === "string") { + preparedInput.mask_url = await upload(preparedInput.mask_url); + } + + const preparedSourceImageUrls = await Promise.all( + sourceImageUrls.map((url) => upload(url)) + ); + + return { + input: preparedInput, + sourceImageUrls: preparedSourceImageUrls, + }; +} + function buildFalRequestPayload(params: { modelId: string; overrideModelId?: string; @@ -834,18 +892,26 @@ export async function submitFalImageTask(params: { }): Promise { const request = buildFalRequestPayload(params); const fal = await createFalClientFromApiKey(params.apiKey); + const prepared = await prepareFalInputImageUrls({ + fal, + input: request.input, + sourceImageUrls: request.imageUrls, + }); console.log("[Fal Image] Queue submit", { model: request.activeModelId, endpoint: request.falEndpointId, imageCount: request.imageCount, - hasInputImages: request.imageUrls.length > 0, + hasInputImages: prepared.sourceImageUrls.length > 0, resolution: request.outputSizeLabel, - inputKeys: Object.keys(request.input), + inputKeys: Object.keys(prepared.input), + storagePrepared: prepared.sourceImageUrls.some( + (url, index) => url !== request.imageUrls[index] + ), }); const enqueueResult = await fal.queue.submit(request.falEndpointId, { - input: request.input, + input: prepared.input, }); return { @@ -855,7 +921,7 @@ export async function submitFalImageTask(params: { imageCount: request.imageCount, outputSizeLabel: request.outputSizeLabel, prompt: request.prompt, - sourceImageUrls: request.imageUrls, + sourceImageUrls: prepared.sourceImageUrls, }; } @@ -920,12 +986,17 @@ export function createFalImage({ apiKey, baseURL }: FalImageConfig) { }); const fal = await createFalClientFromApiKey(apiKey); + const prepared = await prepareFalInputImageUrls({ + fal, + input: request.input, + sourceImageUrls: request.imageUrls, + }); const timeoutMs = normalizeTimeoutMs(options.imageParams?.timeoutMs); let result: any; try { result = await fal.subscribe(request.falEndpointId, { - input: request.input, + input: prepared.input, logs: true, timeout: timeoutMs, onQueueUpdate: (update: any) => { @@ -945,7 +1016,7 @@ export function createFalImage({ apiKey, baseURL }: FalImageConfig) { const text = await normalizeFalResultToMarkdown({ payload: result, - sourceImageUrls: request.imageUrls, + sourceImageUrls: prepared.sourceImageUrls, imageCount: request.imageCount, outputSizeLabel: request.outputSizeLabel, }); From e1f514536b7494e2b1cb11dfca75fe20cf274681 Mon Sep 17 00:00:00 2001 From: 134 Date: Wed, 8 Jul 2026 22:14:20 +0800 Subject: [PATCH 56/70] feat: allow canceling claw submissions --- app/api/canvas/clawd-chat/cancel/route.ts | 40 +++++ app/api/canvas/clawd-chat/route.ts | 62 +++++++- components/canvas/CanvasChatPanel.tsx | 178 +++++++++++++++++++++- lib/openclaw-chat/cancel-registry.ts | 76 +++++++++ 4 files changed, 345 insertions(+), 11 deletions(-) create mode 100644 app/api/canvas/clawd-chat/cancel/route.ts create mode 100644 lib/openclaw-chat/cancel-registry.ts diff --git a/app/api/canvas/clawd-chat/cancel/route.ts b/app/api/canvas/clawd-chat/cancel/route.ts new file mode 100644 index 000000000..138a372e1 --- /dev/null +++ b/app/api/canvas/clawd-chat/cancel/route.ts @@ -0,0 +1,40 @@ +import { NextResponse } from "next/server"; +import { z } from "zod"; +import { auth } from "@/app/(auth)/auth"; +import { getChatById } from "@/lib/db/queries"; +import { + buildClawdCancelKey, + cancelClawdRequest, +} from "@/lib/openclaw-chat/cancel-registry"; + +const cancelSchema = z.object({ + id: z.string().uuid(), + userMessageId: z.string().uuid(), +}); + +export async function POST(request: Request) { + const session = await auth().catch(() => null); + if (!session?.user?.id) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const parsed = cancelSchema.safeParse(await request.json().catch(() => null)); + if (!parsed.success) { + return NextResponse.json({ error: "请求参数不正确" }, { status: 400 }); + } + + const existingChat = await getChatById({ id: parsed.data.id }).catch(() => null); + if (existingChat && existingChat.userId !== session.user.id) { + return NextResponse.json({ error: "Forbidden" }, { status: 403 }); + } + + const active = cancelClawdRequest( + buildClawdCancelKey({ + userId: session.user.id, + chatId: parsed.data.id, + userMessageId: parsed.data.userMessageId, + }) + ); + + return NextResponse.json({ canceled: true, active }); +} diff --git a/app/api/canvas/clawd-chat/route.ts b/app/api/canvas/clawd-chat/route.ts index 354dd11d7..01479493a 100644 --- a/app/api/canvas/clawd-chat/route.ts +++ b/app/api/canvas/clawd-chat/route.ts @@ -13,6 +13,14 @@ import { normalizeOpenClawChatConfig, OPENCLAW_CHAT_CONFIG_KEY, } from "@/lib/openclaw-chat/config"; +import { + buildClawdCancelKey, + cancelClawdRequest, + clearClawdRequestCancellation, + isClawdRequestCanceled, + registerActiveClawdRequest, + unregisterActiveClawdRequest, +} from "@/lib/openclaw-chat/cancel-registry"; import { buildOpenClawDisplayMessage, buildOpenClawMessageWithSpreadsheetMarkdown, @@ -141,6 +149,10 @@ function getMetadataString( return typeof value === "string" ? value.trim() : ""; } +function createCanceledResponse() { + return NextResponse.json({ canceled: true }, { status: 499 }); +} + export async function POST(request: Request) { const session = await auth().catch(() => null); @@ -172,6 +184,16 @@ export async function POST(request: Request) { return NextResponse.json({ error: "Forbidden" }, { status: 403 }); } + const userMessageId = parsed.userMessageId || generateUUID(); + const cancelKey = buildClawdCancelKey({ + userId: session.user.id, + chatId: parsed.id, + userMessageId, + }); + if (isClawdRequestCanceled(cancelKey)) { + return createCanceledResponse(); + } + let spreadsheetMarkdownResult: Awaited< ReturnType > | null = null; @@ -200,6 +222,10 @@ export async function POST(request: Request) { spreadsheetMarkdown: spreadsheetMarkdownResult?.markdown || "", }); + if (isClawdRequestCanceled(cancelKey)) { + return createCanceledResponse(); + } + if (!existingChat) { await saveChat({ id: parsed.id, @@ -209,7 +235,6 @@ export async function POST(request: Request) { }); } - const userMessageId = parsed.userMessageId || generateUUID(); const assistantMessageId = generateUUID(); const conversationId = createConversationId(session.user.id, parsed.id); const userProfile = await getUserById({ id: session.user.id }).catch(() => null); @@ -226,9 +251,18 @@ export async function POST(request: Request) { let upstreamPayload: unknown = null; let upstreamStatus = 502; + if (isClawdRequestCanceled(cancelKey)) { + return createCanceledResponse(); + } + try { const controller = new AbortController(); + registerActiveClawdRequest(cancelKey, controller, config.timeoutMs); const timeout = setTimeout(() => controller.abort(), config.timeoutMs); + const handleRequestAbort = () => { + cancelClawdRequest(cancelKey); + }; + request.signal.addEventListener("abort", handleRequestAbort, { once: true }); const response = await fetch(endpoint, { method: "POST", headers: { @@ -266,7 +300,11 @@ export async function POST(request: Request) { }, }), signal: controller.signal, - }).finally(() => clearTimeout(timeout)); + }).finally(() => { + clearTimeout(timeout); + request.signal.removeEventListener("abort", handleRequestAbort); + unregisterActiveClawdRequest(cancelKey); + }); upstreamStatus = response.status; upstreamPayload = await response.json().catch(() => null); @@ -287,8 +325,19 @@ export async function POST(request: Request) { ); } } catch (error) { - const isAbortError = - error instanceof Error && error.name === "AbortError"; + unregisterActiveClawdRequest(cancelKey); + const errorName = + error && typeof error === "object" && "name" in error + ? String((error as { name?: unknown }).name || "") + : ""; + const isAbortError = errorName === "AbortError"; + if (isClawdRequestCanceled(cancelKey)) { + console.info("[ClawdChat API] OpenClaw request canceled:", { + chatId: parsed.id, + userMessageId, + }); + return createCanceledResponse(); + } console.error("[ClawdChat API] OpenClaw request failed:", error); return NextResponse.json( { @@ -311,6 +360,10 @@ export async function POST(request: Request) { ); } + if (isClawdRequestCanceled(cancelKey)) { + return createCanceledResponse(); + } + const existingMessages = await getMessagesByChatId({ id: parsed.id }).catch( () => [] ); @@ -366,6 +419,7 @@ export async function POST(request: Request) { }; await saveMessages({ messages: [userMessage, assistantMessage] }); + clearClawdRequestCancellation(cancelKey); return NextResponse.json({ conversationId, diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index 3100f7a9b..a20bc46e5 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -25,7 +25,7 @@ import { type UiSurfaceSnapshot, } from "@/lib/ui-surface-protocol"; import useSWR, { useSWRConfig } from "swr"; -import { Loader2, ImagePlus, X, Plus, History, LayoutGrid, ArrowRightToLine, RefreshCw, Globe, ArrowUp, Video, Sparkles, Check, ChevronDown, ChevronUp, AlertCircle, CheckCircle2, Brain, ThumbsUp, ThumbsDown, Eye, Trash2, Download, MessageCircle, Clapperboard, Bot, FileSpreadsheet, Paperclip, Copy } from "lucide-react"; +import { Loader2, ImagePlus, X, Plus, History, LayoutGrid, ArrowRightToLine, RefreshCw, Globe, ArrowUp, Video, Sparkles, Check, ChevronDown, ChevronUp, AlertCircle, CheckCircle2, Brain, ThumbsUp, ThumbsDown, Eye, Trash2, Download, MessageCircle, Clapperboard, Bot, FileSpreadsheet, Paperclip, Copy, Square } from "lucide-react"; import { Button } from "@/components/ui/button"; import { Collapsible, @@ -458,6 +458,16 @@ type ClawdComposerSubmissionSnapshot = { submittedTexts: string[]; }; +type PendingClawdRequestRestoreSnapshot = { + chatId: string; + userMessageId: string; + textContent: string; + tableFiles: File[]; + attachments: PickedObjectAttachment[]; + skillTag: CanvasSkillTag | null; + shouldRestoreSkillTag: boolean; +}; + function isClawdSubmittedComposerText( snapshot: ClawdComposerSubmissionSnapshot, value: string @@ -1600,9 +1610,13 @@ function OpenClawQuickActions({ function CanvasAssistantTypingPlaceholder({ hint, label = "分析中", + onCancel, + cancelLabel = "停止", }: { hint?: string | null; label?: string; + onCancel?: () => void; + cancelLabel?: string; }) { return (
@@ -1618,6 +1632,19 @@ function CanvasAssistantTypingPlaceholder({ > {label} + {onCancel ? ( + + ) : null}
@@ -2259,6 +2286,10 @@ export function CanvasChatPanel({ useRef(null); const recentClawdSubmissionRef = useRef(null); + const clawdAbortControllerRef = useRef(null); + const pendingClawdRequestRestoreRef = + useRef(null); + const canceledClawdMessageIdsRef = useRef>(new Set()); const replaceChatSessionQuery = useCallback( (sessionId: string) => { @@ -7120,6 +7151,76 @@ export function CanvasChatPanel({ }, }); + const restorePendingClawdRequestToComposer = useCallback( + (snapshot: PendingClawdRequestRestoreSnapshot) => { + setMessages((previousMessages) => + previousMessages.filter((message) => message.id !== snapshot.userMessageId) + ); + setClawdTableFiles(snapshot.tableFiles); + setAttachments(snapshot.attachments); + + if (snapshot.skillTag && snapshot.shouldRestoreSkillTag) { + setSessionClawdSkillTag(snapshot.chatId, snapshot.skillTag.id, { + promptApplied: false, + }); + setSelectedSkillTagId(snapshot.skillTag.id); + pendingSelectedClawdSkillTagRef.current = snapshot.skillTag; + } + + composerUserEditedAtRef.current = Date.now(); + inputRef.current = snapshot.textContent; + setInput(snapshot.textContent); + + const editableDiv = textareaRef.current as HTMLDivElement | null; + if (editableDiv) { + editableDiv.innerHTML = ""; + editableDiv.textContent = snapshot.textContent; + window.setTimeout(() => { + snapshot.attachments.forEach((attachment, index) => { + if (!attachment.url) return; + insertImageTagInEditor( + attachment.url, + normalizeChatAttachmentName(attachment.name), + index + ); + }); + editableDiv.focus(); + }, 0); + } + }, + [insertImageTagInEditor, setMessages, setSessionClawdSkillTag] + ); + + const handleCancelClawdSubmission = useCallback(() => { + const snapshot = pendingClawdRequestRestoreRef.current; + if (!snapshot) return; + + canceledClawdMessageIdsRef.current.add(snapshot.userMessageId); + pendingClawdRequestRestoreRef.current = null; + pendingClawdSubmissionRef.current = null; + recentClawdSubmissionRef.current = null; + writeStoredClawdComposerSubmissionSnapshot(null); + + void fetch("/api/canvas/clawd-chat/cancel", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + id: snapshot.chatId, + userMessageId: snapshot.userMessageId, + }), + }).catch((error) => { + console.warn("[CanvasChat] Claw cancel request failed:", error); + }); + + clawdAbortControllerRef.current?.abort(); + clawdAbortControllerRef.current = null; + setIsClawdSubmitting(false); + setClawdProcessingHint(null); + releaseSendGuard(snapshot.chatId); + restorePendingClawdRequestToComposer(snapshot); + toast.message("已停止处理，可以重新编辑后再发送"); + }, [releaseSendGuard, restorePendingClawdRequestToComposer]); + useEffect(() => { activeMessagesRef.current = messages; }, [messages]); @@ -10731,6 +10832,19 @@ export function CanvasChatPanel({ pendingClawdSubmissionRef.current = clawdSubmissionSnapshot; recentClawdSubmissionRef.current = clawdSubmissionSnapshot; writeStoredClawdComposerSubmissionSnapshot(clawdSubmissionSnapshot); + const clawdAbortController = new AbortController(); + const clawdRequestRestoreSnapshot: PendingClawdRequestRestoreSnapshot = { + chatId: activeChatId, + userMessageId, + textContent, + tableFiles: tableFilesSnapshot, + attachments: attachmentsSnapshot, + skillTag: clawdSkillTagForRequest || null, + shouldRestoreSkillTag: hasPendingClawdSkillTagPrompt, + }; + canceledClawdMessageIdsRef.current.delete(userMessageId); + clawdAbortControllerRef.current = clawdAbortController; + pendingClawdRequestRestoreRef.current = clawdRequestRestoreSnapshot; setClawdProcessingHint(nextProcessingHint); setIsClawdSubmitting(true); setMessages((previousMessages) => @@ -10773,8 +10887,13 @@ export function CanvasChatPanel({ const response = await fetch("/api/canvas/clawd-chat", { method: "POST", body: formData, + signal: clawdAbortController.signal, }); const responseData = await response.json().catch(() => null); + if (response.status === 499 || responseData?.canceled === true) { + canceledClawdMessageIdsRef.current.add(userMessageId); + throw new DOMException("Claw request canceled", "AbortError"); + } if (!response.ok) { console.error("[CanvasChat] Claw API returned error:", { status: response.status, @@ -10824,6 +10943,17 @@ export function CanvasChatPanel({ void mutateClawdHistory(); hasHydratedHistoryRef.current = true; } catch (error) { + const wasCanceled = + canceledClawdMessageIdsRef.current.has(userMessageId) || + (error instanceof Error && error.name === "AbortError"); + if (wasCanceled) { + if (pendingClawdRequestRestoreRef.current === clawdRequestRestoreSnapshot) { + pendingClawdRequestRestoreRef.current = null; + restorePendingClawdRequestToComposer(clawdRequestRestoreSnapshot); + toast.message("已停止处理，可以重新编辑后再发送"); + } + return; + } console.error("[CanvasChat] Claw request failed:", error); pendingClawdSubmissionRef.current = null; if (recentClawdSubmissionRef.current === clawdSubmissionSnapshot) { @@ -10865,13 +10995,25 @@ export function CanvasChatPanel({ } } finally { pendingClawdSubmissionRef.current = null; + if (pendingClawdRequestRestoreRef.current === clawdRequestRestoreSnapshot) { + pendingClawdRequestRestoreRef.current = null; + } + if (clawdAbortControllerRef.current === clawdAbortController) { + clawdAbortControllerRef.current = null; + } + const wasCanceled = canceledClawdMessageIdsRef.current.delete(userMessageId); if (recentClawdSubmissionRef.current === clawdSubmissionSnapshot) { + if (wasCanceled) { + recentClawdSubmissionRef.current = null; + writeStoredClawdComposerSubmissionSnapshot(null); + } else { const refreshedSubmissionSnapshot = { ...clawdSubmissionSnapshot, expiresAt: Date.now() + CLAWD_SUBMITTED_COMPOSER_RECOVERY_TTL_MS, }; recentClawdSubmissionRef.current = refreshedSubmissionSnapshot; writeStoredClawdComposerSubmissionSnapshot(refreshedSubmissionSnapshot); + } } setIsClawdSubmitting(false); setClawdProcessingHint(null); @@ -11244,6 +11386,7 @@ export function CanvasChatPanel({ markSessionClawdSkillTagPromptApplied, updateChatSessions, buildHistoryMessagesUrl, + restorePendingClawdRequestToComposer, workflowContext, messages, ]); @@ -12092,6 +12235,7 @@ export function CanvasChatPanel({ isClawdMode && activeClawdSkillTag ? activeClawdSkillTag.placeholder?.trim() || "" : ""; + const isClawdStopAvailable = isClawdMode && isClawdSubmitting; const handleClawdPresetPlaceholderDoubleClick = useCallback( (event: ReactMouseEvent) => { if (isSubmitting || !isClawdMode || !clawdEditablePresetPlaceholder) return; @@ -13878,7 +14022,12 @@ export function CanvasChatPanel({ ) : null} {isClawdMode && isClawdSubmitting ? ( - + ) : null}
@@ -15756,7 +15905,9 @@ export function CanvasChatPanel({ isWorkspacePresentation ? "h-10 rounded-full px-4 transition-all duration-200 flex-shrink-0" : "h-8 w-8 rounded-full transition-all duration-200 flex-shrink-0", - isVideoToolbarDark + isClawdStopAvailable + ? "bg-rose-500 text-white shadow-sm hover:bg-rose-600 dark:bg-rose-500 dark:text-white dark:hover:bg-rose-400" + : isVideoToolbarDark ? canSend ? "border border-[#e7ff1a]/20 bg-[#e7ff1a] text-zinc-950 shadow-sm hover:bg-[#dff40f]" : "border border-white/10 bg-[#242529] text-zinc-500 hover:bg-[#2c2d33]" @@ -15768,15 +15919,28 @@ export function CanvasChatPanel({ ? "bg-[#eef0f3] text-[#20242a] shadow-sm ring-1 ring-inset ring-[#d5dae2] hover:bg-[#e4e7ec] dark:bg-white dark:text-zinc-950 dark:ring-white/15 dark:hover:bg-zinc-100" : "bg-[#f1f1f2] text-[#b6bac2] hover:bg-[#ececee] dark:bg-[#242529] dark:text-[#777d88] dark:hover:bg-[#2c2d33]" )} - onClick={() => void handleSend()} - disabled={isSubmitting || !canSend} + onClick={() => { + if (isClawdStopAvailable) { + handleCancelClawdSubmission(); + return; + } + void handleSend(); + }} + disabled={isClawdStopAvailable ? false : isSubmitting || !canSend} title={ - isWorkspacePresentation || isDetailGenerationMode + isClawdStopAvailable + ? "停止处理并重新编辑" + : isWorkspacePresentation || isDetailGenerationMode ? getToolbarItem("chatStartGeneration").label : getToolbarItem("chatSend").label } > - {isSubmitting ? ( + {isClawdStopAvailable ? ( + <> + + {isWorkspacePresentation ? 停止 : null} + + ) : isSubmitting ? ( <> {isWorkspacePresentation ? 生成中 : null} diff --git a/lib/openclaw-chat/cancel-registry.ts b/lib/openclaw-chat/cancel-registry.ts new file mode 100644 index 000000000..8896d37be --- /dev/null +++ b/lib/openclaw-chat/cancel-registry.ts @@ -0,0 +1,76 @@ +const CANCELED_REQUEST_TTL_MS = 10 * 60 * 1000; + +type ActiveClawdRequest = { + controller: AbortController; + expiresAt: number; +}; + +const activeRequests = new Map(); +const canceledRequests = new Map(); + +function cleanupExpiredClawdRequests(now = Date.now()) { + for (const [key, request] of activeRequests.entries()) { + if (request.expiresAt <= now) { + activeRequests.delete(key); + } + } + + for (const [key, expiresAt] of canceledRequests.entries()) { + if (expiresAt <= now) { + canceledRequests.delete(key); + } + } +} + +export function buildClawdCancelKey({ + userId, + chatId, + userMessageId, +}: { + userId: string; + chatId: string; + userMessageId: string; +}) { + return `${userId}:${chatId}:${userMessageId}`; +} + +export function registerActiveClawdRequest( + key: string, + controller: AbortController, + timeoutMs: number +) { + cleanupExpiredClawdRequests(); + activeRequests.set(key, { + controller, + expiresAt: Date.now() + Math.max(timeoutMs, CANCELED_REQUEST_TTL_MS), + }); +} + +export function unregisterActiveClawdRequest(key: string) { + activeRequests.delete(key); +} + +export function cancelClawdRequest(key: string) { + cleanupExpiredClawdRequests(); + canceledRequests.set(key, Date.now() + CANCELED_REQUEST_TTL_MS); + const activeRequest = activeRequests.get(key); + if (activeRequest && !activeRequest.controller.signal.aborted) { + activeRequest.controller.abort(); + } + return Boolean(activeRequest); +} + +export function isClawdRequestCanceled(key: string) { + cleanupExpiredClawdRequests(); + const expiresAt = canceledRequests.get(key); + if (!expiresAt) return false; + if (expiresAt <= Date.now()) { + canceledRequests.delete(key); + return false; + } + return true; +} + +export function clearClawdRequestCancellation(key: string) { + canceledRequests.delete(key); +} From accaaad8f1bbfa7ac0a765d46fe195aa1bd9a3c2 Mon Sep 17 00:00:00 2001 From: 134 Date: Wed, 8 Jul 2026 22:42:41 +0800 Subject: [PATCH 57/70] fix: avoid route refresh when switching claw history --- components/canvas/CanvasChatPanel.tsx | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index a20bc46e5..a95e017c3 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -4,7 +4,7 @@ import { useChat } from "@ai-sdk/react"; import { DefaultChatTransport } from "ai"; import { useEffect, useRef, useState, useCallback, useMemo, type ClipboardEvent as ReactClipboardEvent, type DragEvent as ReactDragEvent, type MouseEvent as ReactMouseEvent } from "react"; import { useSession } from "next-auth/react"; -import { usePathname, useRouter, useSearchParams } from "next/navigation"; +import { useSearchParams } from "next/navigation"; import { createPortal } from "react-dom"; import type { DetailUiProtocol } from "@/lib/detail-ui-protocol"; import type { DetailUiScreenReviewPatch } from "@/lib/detail-ui-protocol"; @@ -2148,8 +2148,6 @@ export function CanvasChatPanel({ ); const effectiveAgentEnabled = agentEnabled && !isVideoWorkspace; const { data: session } = useSession(); - const router = useRouter(); - const pathname = usePathname(); const searchParams = useSearchParams(); const requestedChatSessionId = useMemo( () => normalizeCanvasChatSessionId(searchParams.get(CANVAS_CHAT_QUERY_PARAM)), @@ -2296,15 +2294,11 @@ export function CanvasChatPanel({ const normalized = normalizeCanvasChatSessionId(sessionId); if (!normalized || typeof window === "undefined") return; - const params = new URLSearchParams(window.location.search); - params.set(CANVAS_CHAT_QUERY_PARAM, normalized); - const query = params.toString(); - const hash = window.location.hash || ""; - router.replace(`${pathname}${query ? `?${query}` : ""}${hash}`, { - scroll: false, - }); + const url = new URL(window.location.href); + url.searchParams.set(CANVAS_CHAT_QUERY_PARAM, normalized); + window.history.replaceState(window.history.state, "", url); }, - [pathname, router] + [] ); // 🆕 标签悬停预览状态 From c9ed6628097d51d0dfe9853754ce7dbc7deeb05d Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 9 Jul 2026 12:00:42 +0800 Subject: [PATCH 58/70] Fix canvas scale adjust local edit build --- app/api/canvas/local-edit/route.ts | 143 ++++++++++++++++++++++++++++- 1 file changed, 139 insertions(+), 4 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 28be69d65..24cc075ed 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -199,6 +199,32 @@ function inferClosestAspectRatio(width: number, height: number) { return closest; } +function inferScaleAdjustPatchAspectRatio(width: number, height: number) { + if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) { + return "1:1"; + } + const sourceRatio = width / height; + const providerSafeRatios = [ + "1:1", + "2:3", + "3:2", + "3:4", + "4:3", + "9:16", + "16:9", + ] as const; + let closest: (typeof providerSafeRatios)[number] = "1:1"; + let closestDiff = Number.POSITIVE_INFINITY; + for (const candidate of providerSafeRatios) { + const diff = Math.abs(parseAspectRatioValue(candidate) - sourceRatio); + if (diff < closestDiff) { + closest = candidate; + closestDiff = diff; + } + } + return closest; +} + function resolveLocalEditAspectRatio(input: unknown, sourceWidth: number, sourceHeight: number) { const requested = String(input || "").trim(); if ( @@ -339,7 +365,26 @@ function isScaleAdjustFalNanoBananaEditModel( const label = `${entry.model.displayName || ""} ${entry.model.modelId || ""}` .toLowerCase() .replace(/\s+/g, " "); - return label.includes("/edit") || /\bedit\b/.test(label); + const capabilities = + entry.model.capabilities && typeof entry.model.capabilities === "object" + ? (entry.model.capabilities as Record) + : {}; + const defaultParams = + entry.model.defaultParams && typeof entry.model.defaultParams === "object" + ? (entry.model.defaultParams as Record) + : {}; + const editEndpoint = + typeof capabilities.editEndpoint === "string" + ? capabilities.editEndpoint + : typeof defaultParams.editEndpoint === "string" + ? defaultParams.editEndpoint + : ""; + return ( + label.includes("/edit") || + /\bedit\b/.test(label) || + capabilities.autoRouteEditEndpoint === true || + editEndpoint.toLowerCase().includes("/edit") + ); } function isScaleAdjustHealthyModel( @@ -761,6 +806,51 @@ function expandRectWithMargins( }; } +function expandRectToProviderAspectLimit( + rect: NormalizedLocalEditRegion["rect"], + sourceWidth: number, + sourceHeight: number, + maxAspectRatio = 3 +) { + if ( + !Number.isFinite(sourceWidth) || + !Number.isFinite(sourceHeight) || + sourceWidth <= 0 || + sourceHeight <= 0 || + maxAspectRatio <= 1 + ) { + return rect; + } + + const pixelWidth = rect.width * sourceWidth; + const pixelHeight = rect.height * sourceHeight; + if (pixelWidth <= 0 || pixelHeight <= 0) return rect; + + const ratio = pixelWidth / pixelHeight; + let nextWidth = rect.width; + let nextHeight = rect.height; + + if (ratio > maxAspectRatio) { + nextHeight = Math.min(1, pixelWidth / maxAspectRatio / sourceHeight); + } else if (ratio < 1 / maxAspectRatio) { + nextWidth = Math.min(1, (pixelHeight / maxAspectRatio) / sourceWidth); + } else { + return rect; + } + + const centerX = rect.x + rect.width / 2; + const centerY = rect.y + rect.height / 2; + const x = clamp(centerX - nextWidth / 2, 0, Math.max(0, 1 - nextWidth)); + const y = clamp(centerY - nextHeight / 2, 0, Math.max(0, 1 - nextHeight)); + + return { + x, + y, + width: nextWidth, + height: nextHeight, + }; +} + function getRectUnion( leftRect: NormalizedLocalEditRegion["rect"], rightRect: NormalizedLocalEditRegion["rect"] @@ -938,6 +1028,40 @@ function getScaleAdjustPatchWorkspaceRect( return workspaceRects.reduce((combined, rect) => getRectUnion(combined, rect)); } +function shouldUseScaleAdjustEdgeLockedFullFrameReference(params: { + regions: NormalizedLocalEditRegion[]; + scalePercent?: ScaleAdjustPercent | null; + sourceHeight: number; + sourceWidth: number; +}) { + const { regions, scalePercent, sourceHeight, sourceWidth } = params; + if (!regions.length || !hasScaleAdjustShrinkTarget(regions, scalePercent)) { + return false; + } + if (!regions.some((region) => isScaleAdjustFurnitureProductLikePrompt(region.prompt))) { + return false; + } + + const workspaceRect = expandRectToProviderAspectLimit( + getScaleAdjustPatchWorkspaceRect(regions, scalePercent), + sourceWidth, + sourceHeight + ); + const pixelWidth = workspaceRect.width * sourceWidth; + const pixelHeight = workspaceRect.height * sourceHeight; + if (pixelWidth <= 0 || pixelHeight <= 0) return false; + + const ratio = pixelWidth / pixelHeight; + const touchesCanvasEdge = + workspaceRect.x <= 0.002 || + workspaceRect.y <= 0.002 || + workspaceRect.x + workspaceRect.width >= 0.998 || + workspaceRect.y + workspaceRect.height >= 0.998; + const nearProviderAspectLimit = ratio <= 0.42 || ratio >= 2.4; + + return touchesCanvasEdge && nearProviderAspectLimit; +} + function expandPixelRectForCrop(params: { height: number; pixelRect: NormalizedLocalEditRegion["pixelRect"]; @@ -1336,6 +1460,13 @@ function isScaleAdjustPersonLikePrompt(prompt: string) { ); } +function isScaleAdjustFurnitureProductLikePrompt(prompt: string) { + const text = normalizeScaleAdjustModelText(prompt); + return /furniture|product|commodity|goods|object|item|subject|sofa|couch|chair|table|desk|cabinet|shelf|bed|bench|lamp|basket|rack|stool|drawer|wardrobe|主物体|主体|物体|商品|产品|家具|沙发|椅|桌|柜|架|床|凳|灯|篮|抽屉|衣柜/.test( + text + ); +} + function hasScaleAdjustPersonPixelSignal(params: { box: { left: number; top: number; width: number; height: number }; raw: Buffer; @@ -2781,7 +2912,11 @@ async function createScaleAdjustPatchReference(params: { sourceWidth, userId, } = params; - const workspaceRect = getScaleAdjustPatchWorkspaceRect(regions, scalePercent); + const workspaceRect = expandRectToProviderAspectLimit( + getScaleAdjustPatchWorkspaceRect(regions, scalePercent), + sourceWidth, + sourceHeight + ); const workspacePixelRect = rectToPixels(workspaceRect, sourceWidth, sourceHeight); const shouldUseStructureLockedDraft = hasScaleAdjustStructureTarget( regions, @@ -3890,7 +4025,7 @@ export async function POST(request: NextRequest) { const useFalScaleAdjustCoordinatePrompt = useScaleAdjustCoordinatePrompt && resolvedScaleAdjustModelEntry?.provider?.type === "fal-image" && - !useScaleAdjustFullFrameFalEdit; + getScaleAdjustModelFamily(resolvedScaleAdjustModelEntry) === "nano-banana-2"; const useScaleAdjustVisualLayoutGuide = useScaleAdjustCoordinatePrompt && !useFalScaleAdjustCoordinatePrompt && @@ -4152,7 +4287,7 @@ export async function POST(request: NextRequest) { const imageRequest: CanvasGenerateImageRequest = { allowFailover: isScaleAdjustOperation ? false : undefined, aspectRatio: scaleAdjustPatchReference - ? inferClosestAspectRatio( + ? inferScaleAdjustPatchAspectRatio( scaleAdjustPatchReference.workspacePixelRect.width, scaleAdjustPatchReference.workspacePixelRect.height ) From 72b1f8f1cf075008e8086f140ae98fd05df94826 Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 9 Jul 2026 14:39:38 +0800 Subject: [PATCH 59/70] fix: keep image attachment add button visible --- components/canvas/CanvasChatPanel.tsx | 62 +++++++++++++++------------ 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index a95e017c3..d9030fc5a 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -15265,33 +15265,41 @@ export function CanvasChatPanel({
) : null} {!isWorkspacePresentation && !isDetailGenerationMode && !isVideoWorkspace && activeComposerMode === "image" ? ( - +
+ {attachments.map((attachment, index) => ( +
+ {attachment.name { + if (event.currentTarget.src !== attachment.url) { + event.currentTarget.src = attachment.url; + } + }} + /> +
+ ))} + +
) : null} {isClawdMode && clawdTableFiles.length > 0 ? (
From 65f5a13f13c526c9dc69aa982faf40c363b8c8bc Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 9 Jul 2026 14:47:18 +0800 Subject: [PATCH 60/70] Rewrite canvas scale adjust local edit --- app/api/canvas/local-edit/route.ts | 3926 ++-------------------------- 1 file changed, 149 insertions(+), 3777 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 24cc075ed..ea8771892 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -2,10 +2,7 @@ import { NextRequest, NextResponse } from "next/server"; import sharp from "sharp"; import { auth } from "@/app/(auth)/auth"; -import { - executeRmbgJob, - fetchImageBufferFromUrl, -} from "@/lib/ai/rmbg/service"; +import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service"; import { createCanvasImageTask, enqueueCanvasImageTask, @@ -13,14 +10,9 @@ import { } from "@/lib/canvas/image-task-queue"; import type { CanvasGenerateImageRequest } from "@/lib/canvas/generate-image"; import { - getEnabledAiModelsWithProvider, getPointsSystemConfig, getSiteSettings, } from "@/lib/db/queries"; -import { - modelLacksReferenceImageInputSupport, -} from "@/lib/ai/reference-image-capability"; -import { isFalGptImage2EditModelId } from "@/lib/ai/providers/fal-image"; import { getImageToolCreditCost } from "@/lib/points/config"; import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge"; import { uploadFile } from "@/lib/storage/service"; @@ -101,48 +93,12 @@ type NormalizedLocalEditRegion = { const MAX_LOCAL_EDIT_REGIONS = 8; const MAX_REFERENCE_IMAGES_PER_REGION = 3; -const MAX_SCALE_ADJUST_CROP_REFERENCES = 4; -const SCALE_ADJUST_CROP_PADDING_RATIO = 0.18; -const SCALE_ADJUST_CROP_MAX_EDGE = 1536; -const SCALE_ADJUST_PATCH_WORKSPACE_MIN_MARGIN = 0.045; -const SCALE_ADJUST_PATCH_WORKSPACE_MAX_MARGIN = 0.18; -const SCALE_ADJUST_LAYOUT_GUIDE_WASH_COLOR = "#f5f7fb"; -const SCALE_ADJUST_LAYOUT_GUIDE_WASH_OPACITY = 0.46; -const SCALE_ADJUST_STRUCTURE_LOCK_MIN_MASK_RATIO = 0.006; -const SCALE_ADJUST_STRUCTURE_LOCK_COMPONENT_MIN_RATIO = 0.00018; -const SCALE_ADJUST_RMBG_MIN_ALPHA_RATIO = 0.012; -const SCALE_ADJUST_RMBG_MAX_SOLID_ALPHA_RATIO = 0.92; -const SCALE_ADJUST_RMBG_ALPHA_CLEAN_THRESHOLD = 42; const MIN_REGION_EDGE_PX = 24; const SCALE_ADJUST_OPERATION = "scale-adjust"; type LocalEditOperation = "local-edit" | typeof SCALE_ADJUST_OPERATION; const MIN_SCALE_ADJUST_PERCENT = -80; const MAX_SCALE_ADJUST_PERCENT = 200; type ScaleAdjustPercent = number; -type ScaleAdjustCropReference = { - regionIndex: number; - url: string; -}; -type ScaleAdjustPatchReference = { - sourceCropUrl: string; - structureLockedDraft?: boolean; - targetDraftUrl: string; - workspaceRect: NormalizedLocalEditRegion["rect"]; - workspacePixelRect: { - left: number; - top: number; - width: number; - height: number; - }; -}; -type ScaleAdjustStructureLockedComposite = { - method?: "rmbg-cutout" | "pixel-mask"; - overlay: { input: Buffer; left: number; top: number }; - wash: { input: Buffer; left: number; top: number }; -}; -type AiModelWithProviderForSelection = Awaited< - ReturnType ->[number]; const DEFAULT_REGION_COLORS = [ "#ff3b30", "#1677ff", @@ -199,32 +155,6 @@ function inferClosestAspectRatio(width: number, height: number) { return closest; } -function inferScaleAdjustPatchAspectRatio(width: number, height: number) { - if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) { - return "1:1"; - } - const sourceRatio = width / height; - const providerSafeRatios = [ - "1:1", - "2:3", - "3:2", - "3:4", - "4:3", - "9:16", - "16:9", - ] as const; - let closest: (typeof providerSafeRatios)[number] = "1:1"; - let closestDiff = Number.POSITIVE_INFINITY; - for (const candidate of providerSafeRatios) { - const diff = Math.abs(parseAspectRatioValue(candidate) - sourceRatio); - if (diff < closestDiff) { - closest = candidate; - closestDiff = diff; - } - } - return closest; -} - function resolveLocalEditAspectRatio(input: unknown, sourceWidth: number, sourceHeight: number) { const requested = String(input || "").trim(); if ( @@ -259,336 +189,6 @@ function normalizeHexColor(input: unknown, fallback: string) { return /^#[0-9a-f]{6}$/i.test(expanded) ? expanded.toLowerCase() : fallback; } -function findModelSelectionEntry( - models: AiModelWithProviderForSelection[], - modelRef: string -) { - const normalizedRef = String(modelRef || "").trim(); - if (!normalizedRef) return null; - return ( - models.find( - (entry) => - entry.model.id === normalizedRef || entry.model.modelId === normalizedRef - ) || null - ); -} - -function isScaleAdjustDedicatedDetailModel( - entry: AiModelWithProviderForSelection | null | undefined -) { - if ( - !entry?.model?.supportsImageGen || - modelLacksReferenceImageInputSupport(entry.model, entry.provider) - ) { - return false; - } - const label = `${entry.model.displayName || ""} ${entry.model.modelId || ""}` - .toLowerCase() - .replace(/\s+/g, ""); - return label.includes("详情专线"); -} - -function normalizeScaleAdjustModelText(input: unknown) { - return String(input || "") - .trim() - .toLowerCase() - .normalize("NFKC") - .replace(/[「」『』【】[\]()（）{}]/g, " ") - .replace(/[_/|:：,，.。+]+/g, "-") - .replace(/\s+/g, " "); -} - -function resolveScaleAdjustModelFamilyFromText(input: unknown) { - const normalized = normalizeScaleAdjustModelText(input); - const compact = normalized.replace(/[^a-z0-9]+/g, ""); - if (!normalized && !compact) return ""; - if ( - normalized.includes("gemini-3.1-flash-image-preview") || - normalized.includes("nano-banana-2") || - compact.includes("nanobanana2") - ) { - return "nano-banana-2"; - } - if ( - normalized.includes("gemini-3-pro-image-preview") || - normalized.includes("nano-banana-pro") || - compact.includes("nanobananapro") - ) { - return "nano-banana-pro"; - } - if (normalized.includes("gpt-image-2") || compact.includes("gptimage2")) { - return "gpt-image-2"; - } - return ""; -} - -function getScaleAdjustModelFamily( - entry: AiModelWithProviderForSelection | null | undefined -) { - if (!entry) return ""; - return ( - resolveScaleAdjustModelFamilyFromText(entry.model.modelId) || - resolveScaleAdjustModelFamilyFromText(entry.model.displayName) || - resolveScaleAdjustModelFamilyFromText(entry.model.id) - ); -} - -function isScaleAdjustMaskCapableModel( - entry: AiModelWithProviderForSelection | null | undefined -) { - if (!entry || getScaleAdjustModelFamily(entry) !== "gpt-image-2") { - return false; - } - if (entry.provider.type === "openai-image") return true; - if (entry.provider.type === "fal-image") { - return isFalGptImage2EditModelId(entry.model.modelId || ""); - } - return false; -} - -function isScaleAdjustCoordinateGuideModel( - entry: AiModelWithProviderForSelection | null | undefined -) { - return getScaleAdjustModelFamily(entry) === "nano-banana-2"; -} - -function isScaleAdjustFalNanoBananaEditModel( - entry: AiModelWithProviderForSelection | null | undefined -) { - if ( - !entry || - entry.provider.type !== "fal-image" || - getScaleAdjustModelFamily(entry) !== "nano-banana-2" - ) { - return false; - } - const label = `${entry.model.displayName || ""} ${entry.model.modelId || ""}` - .toLowerCase() - .replace(/\s+/g, " "); - const capabilities = - entry.model.capabilities && typeof entry.model.capabilities === "object" - ? (entry.model.capabilities as Record) - : {}; - const defaultParams = - entry.model.defaultParams && typeof entry.model.defaultParams === "object" - ? (entry.model.defaultParams as Record) - : {}; - const editEndpoint = - typeof capabilities.editEndpoint === "string" - ? capabilities.editEndpoint - : typeof defaultParams.editEndpoint === "string" - ? defaultParams.editEndpoint - : ""; - return ( - label.includes("/edit") || - /\bedit\b/.test(label) || - capabilities.autoRouteEditEndpoint === true || - editEndpoint.toLowerCase().includes("/edit") - ); -} - -function isScaleAdjustHealthyModel( - entry: AiModelWithProviderForSelection | null | undefined -) { - return !entry?.model?.healthStatus || entry.model.healthStatus === "healthy"; -} - -function getScaleAdjustProviderPriority(providerType: string) { - switch (providerType) { - case "ruxa-image": - return 10; - case "snapgen-image": - return 20; - case "krea": - return 30; - case "gemini": - case "grsai-gpt-image": - return 40; - case "alibaba-image": - case "bfl-image": - case "openai-image": - case "stepfun-image": - case "volcano-vision": - return 60; - case "fal-image": - return 1000; - default: - return 80; - } -} - -function getScaleAdjustModelHealthPenalty( - healthStatus: string | null | undefined -) { - if (healthStatus === "degraded") return 25; - if (healthStatus === "auto_disabled") return 200; - return 0; -} - -function getScaleAdjustCandidatePriority(entry: AiModelWithProviderForSelection) { - return ( - getScaleAdjustProviderPriority(entry.provider.type) + - getScaleAdjustModelHealthPenalty(entry.model.healthStatus) - ); -} - -function isScaleAdjustUsableImageEditModel( - entry: AiModelWithProviderForSelection | null | undefined -) { - return Boolean( - entry?.model?.supportsImageGen && - !modelLacksReferenceImageInputSupport(entry.model, entry.provider) - ); -} - -function compareScaleAdjustProviderCandidates( - left: AiModelWithProviderForSelection, - right: AiModelWithProviderForSelection, - preferredModelRef: string -) { - const leftPriority = getScaleAdjustCandidatePriority(left); - const rightPriority = getScaleAdjustCandidatePriority(right); - if (leftPriority !== rightPriority) return leftPriority - rightPriority; - - const leftDedicated = isScaleAdjustDedicatedDetailModel(left); - const rightDedicated = isScaleAdjustDedicatedDetailModel(right); - if (leftDedicated !== rightDedicated) return leftDedicated ? -1 : 1; - - const leftExact = - left.model.id === preferredModelRef || left.model.modelId === preferredModelRef; - const rightExact = - right.model.id === preferredModelRef || right.model.modelId === preferredModelRef; - if (leftExact !== rightExact) return leftExact ? -1 : 1; - - return 0; -} - -function resolveScaleAdjustPreferredModelId(params: { - configuredModelId: string; - honorRequestedModelBeforeGpt?: boolean; - preferFalNanoBananaEdit?: boolean; - preferMaskCapableGpt?: boolean; - preferProviderTypes?: string[]; - requestedModelId: string; - models: AiModelWithProviderForSelection[]; -}) { - const requestedModel = findModelSelectionEntry( - params.models, - params.requestedModelId - ); - const configuredModel = findModelSelectionEntry( - params.models, - params.configuredModelId - ); - if ( - params.honorRequestedModelBeforeGpt && - requestedModel && - isScaleAdjustUsableImageEditModel(requestedModel) - ) { - return requestedModel.model.id || params.requestedModelId; - } - - const gptImage2Candidates = params.models - .filter((entry) => isScaleAdjustUsableImageEditModel(entry)) - .filter((entry) => getScaleAdjustModelFamily(entry) === "gpt-image-2") - .sort((left, right) => - compareScaleAdjustProviderCandidates( - left, - right, - params.requestedModelId || params.configuredModelId - ) - ); - if (params.preferMaskCapableGpt) { - const maskCapableGptImage2 = gptImage2Candidates - .filter(isScaleAdjustMaskCapableModel) - .filter(isScaleAdjustHealthyModel); - if (maskCapableGptImage2[0]) { - return maskCapableGptImage2[0].model.id; - } - } - for (const providerType of params.preferProviderTypes || []) { - const providerCandidate = gptImage2Candidates.find( - (entry) => - entry.provider.type === providerType && isScaleAdjustHealthyModel(entry) - ); - if (providerCandidate) { - return providerCandidate.model.id; - } - } - if (params.preferFalNanoBananaEdit) { - const falEditCandidate = params.models - .filter((entry) => isScaleAdjustUsableImageEditModel(entry)) - .filter(isScaleAdjustFalNanoBananaEditModel) - .filter(isScaleAdjustHealthyModel) - .sort((left, right) => - compareScaleAdjustProviderCandidates( - left, - right, - params.requestedModelId || params.configuredModelId - ) - )[0]; - if (falEditCandidate) { - return falEditCandidate.model.id; - } - } - if (gptImage2Candidates[0]) { - return gptImage2Candidates[0].model.id; - } - - if ( - isScaleAdjustUsableImageEditModel(requestedModel) && - requestedModel && - getScaleAdjustModelFamily(requestedModel) !== "nano-banana-2" && - requestedModel.provider.type !== "alibaba-image" - ) { - return requestedModel.model.id || params.requestedModelId; - } - - const autoDetailModel = params.models.find(isScaleAdjustDedicatedDetailModel); - const preferredModel = - (isScaleAdjustDedicatedDetailModel(requestedModel) ? requestedModel : null) || - (isScaleAdjustDedicatedDetailModel(configuredModel) ? configuredModel : null) || - autoDetailModel || - configuredModel || - requestedModel || - null; - const preferredModelRef = - preferredModel?.model.id || - preferredModel?.model.modelId || - params.configuredModelId || - params.requestedModelId; - const preferredFamily = - getScaleAdjustModelFamily(preferredModel) || - resolveScaleAdjustModelFamilyFromText(params.configuredModelId) || - resolveScaleAdjustModelFamilyFromText(params.requestedModelId); - - if (preferredFamily) { - const sameFamilyCandidates = params.models - .filter((entry) => isScaleAdjustUsableImageEditModel(entry)) - .filter((entry) => getScaleAdjustModelFamily(entry) === preferredFamily) - .sort((left, right) => - compareScaleAdjustProviderCandidates(left, right, preferredModelRef) - ); - if (sameFamilyCandidates[0]) { - return sameFamilyCandidates[0].model.id; - } - } - - if (isScaleAdjustDedicatedDetailModel(requestedModel)) { - return requestedModel?.model.id || params.requestedModelId; - } - - if (isScaleAdjustDedicatedDetailModel(configuredModel)) { - return configuredModel?.model.id || params.configuredModelId; - } - - if (autoDetailModel) { - return autoDetailModel.model.id; - } - - return params.configuredModelId || params.requestedModelId; -} - function normalizeHttpImageUrl(input: unknown) { const raw = String(input || "").trim(); if (!/^https?:\/\//i.test(raw)) return ""; @@ -741,2230 +341,6 @@ function getScaleAdjustTargetRect( return region.rect; } -function hasScaleAdjustEnlargeTarget( - regions: NormalizedLocalEditRegion[], - scalePercent?: ScaleAdjustPercent | null -) { - return regions.some((region) => { - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const widthFactor = - region.rect.width > 0 ? targetRect.width / region.rect.width : 1; - const heightFactor = - region.rect.height > 0 ? targetRect.height / region.rect.height : 1; - return widthFactor > 1.08 || heightFactor > 1.08; - }); -} - -function hasScaleAdjustShrinkTarget( - regions: NormalizedLocalEditRegion[], - scalePercent?: ScaleAdjustPercent | null -) { - return regions.some((region) => { - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const widthFactor = - region.rect.width > 0 ? targetRect.width / region.rect.width : 1; - const heightFactor = - region.rect.height > 0 ? targetRect.height / region.rect.height : 1; - return widthFactor < 0.92 || heightFactor < 0.92; - }); -} - -function hasScaleAdjustStructureTarget( - regions: NormalizedLocalEditRegion[], - scalePercent?: ScaleAdjustPercent | null -) { - return regions.some((region) => { - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const widthFactor = - region.rect.width > 0 ? targetRect.width / region.rect.width : 1; - const heightFactor = - region.rect.height > 0 ? targetRect.height / region.rect.height : 1; - return ( - widthFactor > 1.08 || - heightFactor > 1.08 || - widthFactor < 0.92 || - heightFactor < 0.92 - ); - }); -} - -function expandRectWithMargins( - rect: NormalizedLocalEditRegion["rect"], - marginX: number, - marginY: number -) { - const left = clamp(rect.x - marginX, 0, 1); - const top = clamp(rect.y - marginY, 0, 1); - const right = clamp(rect.x + rect.width + marginX, left + 0.001, 1); - const bottom = clamp(rect.y + rect.height + marginY, top + 0.001, 1); - - return { - x: left, - y: top, - width: Math.max(0.001, right - left), - height: Math.max(0.001, bottom - top), - }; -} - -function expandRectToProviderAspectLimit( - rect: NormalizedLocalEditRegion["rect"], - sourceWidth: number, - sourceHeight: number, - maxAspectRatio = 3 -) { - if ( - !Number.isFinite(sourceWidth) || - !Number.isFinite(sourceHeight) || - sourceWidth <= 0 || - sourceHeight <= 0 || - maxAspectRatio <= 1 - ) { - return rect; - } - - const pixelWidth = rect.width * sourceWidth; - const pixelHeight = rect.height * sourceHeight; - if (pixelWidth <= 0 || pixelHeight <= 0) return rect; - - const ratio = pixelWidth / pixelHeight; - let nextWidth = rect.width; - let nextHeight = rect.height; - - if (ratio > maxAspectRatio) { - nextHeight = Math.min(1, pixelWidth / maxAspectRatio / sourceHeight); - } else if (ratio < 1 / maxAspectRatio) { - nextWidth = Math.min(1, (pixelHeight / maxAspectRatio) / sourceWidth); - } else { - return rect; - } - - const centerX = rect.x + rect.width / 2; - const centerY = rect.y + rect.height / 2; - const x = clamp(centerX - nextWidth / 2, 0, Math.max(0, 1 - nextWidth)); - const y = clamp(centerY - nextHeight / 2, 0, Math.max(0, 1 - nextHeight)); - - return { - x, - y, - width: nextWidth, - height: nextHeight, - }; -} - -function getRectUnion( - leftRect: NormalizedLocalEditRegion["rect"], - rightRect: NormalizedLocalEditRegion["rect"] -) { - const left = Math.min(leftRect.x, rightRect.x); - const top = Math.min(leftRect.y, rightRect.y); - const right = Math.max( - leftRect.x + leftRect.width, - rightRect.x + rightRect.width - ); - const bottom = Math.max( - leftRect.y + leftRect.height, - rightRect.y + rightRect.height - ); - return { - x: clamp(left, 0, 1), - y: clamp(top, 0, 1), - width: Math.max(0.001, clamp(right, 0, 1) - clamp(left, 0, 1)), - height: Math.max(0.001, clamp(bottom, 0, 1) - clamp(top, 0, 1)), - }; -} - -function rectsDifferMeaningfully( - leftRect: NormalizedLocalEditRegion["rect"], - rightRect: NormalizedLocalEditRegion["rect"], - epsilon = 0.0005 -) { - return ( - Math.abs(leftRect.x - rightRect.x) > epsilon || - Math.abs(leftRect.y - rightRect.y) > epsilon || - Math.abs(leftRect.width - rightRect.width) > epsilon || - Math.abs(leftRect.height - rightRect.height) > epsilon - ); -} - -function getScaleAdjustWorkspaceRect( - sourceRect: NormalizedLocalEditRegion["rect"], - targetRect: NormalizedLocalEditRegion["rect"], - scalePercent?: ScaleAdjustPercent | null -) { - const sourceWidth = Math.max(0.0001, sourceRect.width); - const sourceHeight = Math.max(0.0001, sourceRect.height); - const widthDeltaRatio = Math.abs(targetRect.width / sourceWidth - 1); - const heightDeltaRatio = Math.abs(targetRect.height / sourceHeight - 1); - const areaDeltaRatio = Math.abs( - Math.sqrt( - Math.max(0.0001, targetRect.width * targetRect.height) / - Math.max(0.0001, sourceWidth * sourceHeight) - ) - 1 - ); - const sourceCenterX = sourceRect.x + sourceRect.width / 2; - const sourceCenterY = sourceRect.y + sourceRect.height / 2; - const targetCenterX = targetRect.x + targetRect.width / 2; - const targetCenterY = targetRect.y + targetRect.height / 2; - const movementRatioX = Math.abs(targetCenterX - sourceCenterX) / sourceWidth; - const movementRatioY = Math.abs(targetCenterY - sourceCenterY) / sourceHeight; - const explicitChangeRatio = Math.max( - widthDeltaRatio, - heightDeltaRatio, - areaDeltaRatio, - movementRatioX, - movementRatioY - ); - const fallbackScaleRatio = - typeof scalePercent === "number" && Number.isFinite(scalePercent) - ? Math.min(1, Math.abs(scalePercent) / 100) - : 0; - const changeRatio = Math.max(explicitChangeRatio, fallbackScaleRatio); - const unionRect = getRectUnion(sourceRect, targetRect); - if (changeRatio <= 0 && !rectsDifferMeaningfully(unionRect, targetRect)) { - return targetRect; - } - - const marginX = Math.min( - 0.16, - Math.max( - 0.018, - unionRect.width * 0.04, - sourceRect.width * changeRatio * 0.24 - ) - ); - const marginY = Math.min( - 0.16, - Math.max( - 0.018, - unionRect.height * 0.04, - sourceRect.height * changeRatio * 0.24 - ) - ); - - return expandRectWithMargins(unionRect, marginX, marginY); -} - -function buildScaleAdjustCompositeRegions( - regions: NormalizedLocalEditRegion[], - scalePercent?: ScaleAdjustPercent | null -) { - return regions.flatMap((region) => { - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const workspaceRect = getScaleAdjustWorkspaceRect( - region.rect, - targetRect, - scalePercent - ); - return rectsDifferMeaningfully(workspaceRect, targetRect) && - rectsDifferMeaningfully(workspaceRect, region.rect) - ? [region.rect, targetRect, workspaceRect] - : [region.rect, targetRect]; - }); -} - -function buildScaleAdjustPatchCompositeRegions( - regions: NormalizedLocalEditRegion[], - scalePercent?: ScaleAdjustPercent | null -) { - return regions.flatMap((region) => [ - region.rect, - getScaleAdjustTargetRect(region, scalePercent), - ]); -} - -function buildScaleAdjustProtectedChangeRegions( - regions: NormalizedLocalEditRegion[], - scalePercent?: ScaleAdjustPercent | null -) { - return regions.flatMap((region) => { - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const unionRect = getRectUnion(region.rect, targetRect); - const marginX = Math.min( - 0.045, - Math.max(0.012, unionRect.width * 0.08) - ); - const marginY = Math.min( - 0.045, - Math.max(0.012, unionRect.height * 0.08) - ); - - return [ - expandRectWithMargins(region.rect, marginX, marginY), - expandRectWithMargins(targetRect, marginX, marginY), - ]; - }); -} - -function getScaleAdjustPatchWorkspaceRect( - regions: NormalizedLocalEditRegion[], - scalePercent?: ScaleAdjustPercent | null -) { - const workspaceRects = regions.map((region) => { - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const baseWorkspace = getScaleAdjustWorkspaceRect( - region.rect, - targetRect, - scalePercent - ); - const marginX = Math.min( - SCALE_ADJUST_PATCH_WORKSPACE_MAX_MARGIN, - Math.max( - SCALE_ADJUST_PATCH_WORKSPACE_MIN_MARGIN, - baseWorkspace.width * 0.22, - region.rect.width * 0.18 - ) - ); - const marginY = Math.min( - SCALE_ADJUST_PATCH_WORKSPACE_MAX_MARGIN, - Math.max( - SCALE_ADJUST_PATCH_WORKSPACE_MIN_MARGIN, - baseWorkspace.height * 0.22, - region.rect.height * 0.18 - ) - ); - return expandRectWithMargins(baseWorkspace, marginX, marginY); - }); - - return workspaceRects.reduce((combined, rect) => getRectUnion(combined, rect)); -} - -function shouldUseScaleAdjustEdgeLockedFullFrameReference(params: { - regions: NormalizedLocalEditRegion[]; - scalePercent?: ScaleAdjustPercent | null; - sourceHeight: number; - sourceWidth: number; -}) { - const { regions, scalePercent, sourceHeight, sourceWidth } = params; - if (!regions.length || !hasScaleAdjustShrinkTarget(regions, scalePercent)) { - return false; - } - if (!regions.some((region) => isScaleAdjustFurnitureProductLikePrompt(region.prompt))) { - return false; - } - - const workspaceRect = expandRectToProviderAspectLimit( - getScaleAdjustPatchWorkspaceRect(regions, scalePercent), - sourceWidth, - sourceHeight - ); - const pixelWidth = workspaceRect.width * sourceWidth; - const pixelHeight = workspaceRect.height * sourceHeight; - if (pixelWidth <= 0 || pixelHeight <= 0) return false; - - const ratio = pixelWidth / pixelHeight; - const touchesCanvasEdge = - workspaceRect.x <= 0.002 || - workspaceRect.y <= 0.002 || - workspaceRect.x + workspaceRect.width >= 0.998 || - workspaceRect.y + workspaceRect.height >= 0.998; - const nearProviderAspectLimit = ratio <= 0.42 || ratio >= 2.4; - - return touchesCanvasEdge && nearProviderAspectLimit; -} - -function expandPixelRectForCrop(params: { - height: number; - pixelRect: NormalizedLocalEditRegion["pixelRect"]; - width: number; -}) { - const { height, pixelRect, width } = params; - const padX = Math.round(pixelRect.width * SCALE_ADJUST_CROP_PADDING_RATIO); - const padY = Math.round(pixelRect.height * SCALE_ADJUST_CROP_PADDING_RATIO); - const left = clamp(pixelRect.left - padX, 0, Math.max(0, width - 1)); - const top = clamp(pixelRect.top - padY, 0, Math.max(0, height - 1)); - const right = clamp( - pixelRect.left + pixelRect.width + padX, - left + 1, - width - ); - const bottom = clamp( - pixelRect.top + pixelRect.height + padY, - top + 1, - height - ); - - return { - left, - top, - width: Math.max(1, right - left), - height: Math.max(1, bottom - top), - }; -} - -async function createScaleAdjustCropReferences(params: { - regions: NormalizedLocalEditRegion[]; - sourceBuffer: Buffer; - sourceHeight: number; - sourceWidth: number; -}): Promise { - const references: ScaleAdjustCropReference[] = []; - const regionsForCrop = params.regions.slice(0, MAX_SCALE_ADJUST_CROP_REFERENCES); - - for (const region of regionsForCrop) { - const focusBox = getScaleAdjustFocusedSourceBox({ - region, - sourceBox: region.pixelRect, - }); - const cropBox = getScaleAdjustFocusDirection(region) - ? focusBox - : expandPixelRectForCrop({ - height: params.sourceHeight, - pixelRect: region.pixelRect, - width: params.sourceWidth, - }); - const cropBuffer = await sharp(params.sourceBuffer, { failOn: "none" }) - .extract(cropBox) - .resize({ - fit: "inside", - height: SCALE_ADJUST_CROP_MAX_EDGE, - withoutEnlargement: true, - width: SCALE_ADJUST_CROP_MAX_EDGE, - }) - .png({ compressionLevel: 6 }) - .toBuffer(); - const upload = await uploadFile( - `canvas-scale-adjust-crop-r${region.index}-${Date.now()}-${Math.random() - .toString(36) - .slice(2, 8)}.png`, - toArrayBuffer(cropBuffer) - ); - references.push({ - regionIndex: region.index, - url: upload.url, - }); - } - - return references; -} - -function getScaleAdjustVisualGuidePlacement(params: { - height: number; - region: NormalizedLocalEditRegion; - scalePercent?: ScaleAdjustPercent | null; - width: number; -}) { - const { height, region, scalePercent, width } = params; - const source = region.pixelRect; - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const target = region.targetPixelRect || rectToPixels(targetRect, width, height); - const sourceLeft = clamp(source.left, 0, Math.max(0, width - 1)); - const sourceTop = clamp(source.top, 0, Math.max(0, height - 1)); - const sourceRight = clamp( - source.left + source.width, - sourceLeft + 1, - width - ); - const sourceBottom = clamp( - source.top + source.height, - sourceTop + 1, - height - ); - const sourceFootprintBox = { - left: Math.round(sourceLeft), - top: Math.round(sourceTop), - width: Math.max(1, Math.round(sourceRight - sourceLeft)), - height: Math.max(1, Math.round(sourceBottom - sourceTop)), - }; - const sourceBox = getScaleAdjustFocusedSourceBox({ - region, - sourceBox: sourceFootprintBox, - }); - const sourceWidth = sourceBox.width; - const sourceHeight = sourceBox.height; - const widthScale = target.width / Math.max(1, sourceWidth); - const heightScale = target.height / Math.max(1, sourceHeight); - const uniformScale = Math.max( - 0.05, - Math.min( - Number.isFinite(widthScale) && widthScale > 0 ? widthScale : 1, - Number.isFinite(heightScale) && heightScale > 0 ? heightScale : 1 - ) - ); - const placedWidth = clamp( - Math.round(sourceWidth * uniformScale), - 1, - Math.max(1, width) - ); - const placedHeight = clamp( - Math.round(sourceHeight * uniformScale), - 1, - Math.max(1, height) - ); - const targetCenterX = target.left + target.width / 2; - const targetCenterY = target.top + target.height / 2; - const placedLeft = clamp( - Math.round(targetCenterX - placedWidth / 2), - 0, - Math.max(0, width - placedWidth) - ); - const placedTop = clamp( - Math.round(targetCenterY - placedHeight / 2), - 0, - Math.max(0, height - placedHeight) - ); - - return { - placedHeight: Math.max(1, Math.round(placedHeight)), - placedLeft: Math.max(0, Math.round(placedLeft)), - placedTop: Math.max(0, Math.round(placedTop)), - placedWidth: Math.max(1, Math.round(placedWidth)), - sourceBox: { - left: sourceBox.left, - top: sourceBox.top, - width: sourceWidth, - height: sourceHeight, - }, - sourceFootprintBox, - targetBox: target, - uniformScale, - }; -} - -function getScaleAdjustFocusDirection(region: NormalizedLocalEditRegion) { - const text = normalizeScaleAdjustModelText(region.prompt); - const compact = text.replace(/[^a-z0-9\u4e00-\u9fa5]+/g, ""); - if (text.includes("left") || compact.includes("左")) return "left"; - if (text.includes("right") || compact.includes("右")) return "right"; - if (text.includes("top") || compact.includes("上")) return "top"; - if (text.includes("bottom") || compact.includes("下")) return "bottom"; - return ""; -} - -function getScaleAdjustFocusedSourceBox(params: { - region: NormalizedLocalEditRegion; - sourceBox: { left: number; top: number; width: number; height: number }; -}) { - const { region, sourceBox } = params; - const direction = getScaleAdjustFocusDirection(region); - if (!direction) return sourceBox; - - const minWidth = Math.max(1, Math.round(sourceBox.width * 0.42)); - const minHeight = Math.max(1, Math.round(sourceBox.height * 0.42)); - let left = sourceBox.left; - let top = sourceBox.top; - let width = sourceBox.width; - let height = sourceBox.height; - - if (direction === "left") { - width = Math.max(minWidth, Math.round(sourceBox.width * 0.68)); - } else if (direction === "right") { - const nextWidth = Math.max(minWidth, Math.round(sourceBox.width * 0.68)); - left = sourceBox.left + sourceBox.width - nextWidth; - width = nextWidth; - } else if (direction === "top") { - height = Math.max(minHeight, Math.round(sourceBox.height * 0.72)); - } else if (direction === "bottom") { - const nextHeight = Math.max(minHeight, Math.round(sourceBox.height * 0.72)); - top = sourceBox.top + sourceBox.height - nextHeight; - height = nextHeight; - } - - return { - left, - top, - width, - height, - }; -} - -function buildScaleAdjustPatchMaskSvg(params: { height: number; width: number }) { - const { height, width } = params; - const blur = Math.max(2, Math.min(14, Math.round(Math.min(width, height) * 0.035))); - const inset = Math.min( - Math.floor(Math.min(width, height) / 8), - Math.max(1, Math.round(blur * 0.7)) - ); - const rectWidth = Math.max(1, width - inset * 2); - const rectHeight = Math.max(1, height - inset * 2); - - return Buffer.from( - ` - - - - - - - ` - ); -} - -function formatScaleAdjustWashColor( - background?: { b: number; g: number; r: number } | null -) { - if (!background) return SCALE_ADJUST_LAYOUT_GUIDE_WASH_COLOR; - const r = clamp(Math.round(background.r), 24, 242); - const g = clamp(Math.round(background.g), 24, 242); - const b = clamp(Math.round(background.b), 24, 242); - return `rgb(${r}, ${g}, ${b})`; -} - -function buildScaleAdjustSourceFootprintWashSvg(params: { - boxes: Array<{ - background?: { b: number; g: number; r: number } | null; - left: number; - top: number; - width: number; - height: number; - }>; - height: number; - width: number; -}) { - const { boxes, height, width } = params; - const defs = boxes - .map((box, index) => { - const blur = Math.max( - 14, - Math.min(72, Math.round(Math.min(box.width, box.height) * 0.14)) - ); - return ` - - `; - }) - .join(""); - const rects = boxes - .map((box, index) => { - const blur = Math.max( - 14, - Math.min(72, Math.round(Math.min(box.width, box.height) * 0.14)) - ); - const pad = Math.max(blur, Math.round(Math.min(box.width, box.height) * 0.055)); - const left = Math.max(0, box.left - blur); - const top = Math.max(0, box.top - blur); - const right = Math.min(width, box.left + box.width + blur); - const bottom = Math.min(height, box.top + box.height + blur); - const rectWidth = Math.max(1, right - left); - const rectHeight = Math.max(1, bottom - top); - const radius = Math.max( - 8, - Math.min(Math.round(blur * 1.8), Math.round(rectWidth / 3), Math.round(rectHeight / 3)) - ); - const innerLeft = Math.max(0, box.left + pad); - const innerTop = Math.max(0, box.top + pad); - const innerRight = Math.min(width, box.left + box.width - pad); - const innerBottom = Math.min(height, box.top + box.height - pad); - const fill = formatScaleAdjustWashColor(box.background); - const inner = - innerRight > innerLeft + 4 && innerBottom > innerTop + 4 - ? `` - : ""; - return `${inner}`; - }) - .join(""); - - return Buffer.from( - ` - - ${defs} - - ${rects} - ` - ); -} - -function buildScaleAdjustTargetFootprintShadowSvg(params: { - height: number; - placements: Array>; - width: number; -}) { - const { height, placements, width } = params; - const minSide = Math.max(1, Math.min(width, height)); - const blur = Math.max(4, Math.min(18, Math.round(minSide * 0.014))); - const rects = placements - .map((placement) => { - const pad = Math.max(2, Math.round(blur * 0.45)); - const left = clamp( - placement.placedLeft - pad, - 0, - Math.max(0, width - 1) - ); - const top = clamp( - placement.placedTop - pad, - 0, - Math.max(0, height - 1) - ); - const right = clamp( - placement.placedLeft + placement.placedWidth + pad, - left + 1, - width - ); - const bottom = clamp( - placement.placedTop + placement.placedHeight + pad, - top + 1, - height - ); - const radius = Math.max(4, Math.round(blur * 1.2)); - return ``; - }) - .join(""); - - return Buffer.from( - ` - - - - - - ${rects} - ` - ); -} - -function getScaleAdjustRawPixelIndex(width: number, x: number, y: number) { - return (y * width + x) * 3; -} - -function getScaleAdjustPixelLuma(r: number, g: number, b: number) { - return r * 0.299 + g * 0.587 + b * 0.114; -} - -function getScaleAdjustPixelStats(r: number, g: number, b: number) { - const max = Math.max(r, g, b); - const min = Math.min(r, g, b); - return { - luma: getScaleAdjustPixelLuma(r, g, b), - saturation: max > 0 ? (max - min) / max : 0, - }; -} - -function isScaleAdjustLikelySkinPixel(r: number, g: number, b: number) { - return ( - r > 132 && - g > 82 && - b > 58 && - r > g * 1.06 && - g > b * 1.03 && - r - b < 135 - ); -} - -function isScaleAdjustLikelyWhiteFabricPixel( - r: number, - g: number, - b: number, - luma: number, - saturation: number -) { - return ( - luma > 158 && - saturation < 0.3 && - Math.abs(r - g) < 34 && - Math.abs(g - b) < 44 - ); -} - -function isScaleAdjustPersonLikePrompt(prompt: string) { - const text = normalizeScaleAdjustModelText(prompt); - return /person|people|human|woman|man|girl|boy|model|face|body|hand|arm|leg|clothing|人物|人像|真人|模特|女人|男人|女孩|男孩|小孩|脸|身体|手|胳膊|腿|衣服/.test( - text - ); -} - -function isScaleAdjustFurnitureProductLikePrompt(prompt: string) { - const text = normalizeScaleAdjustModelText(prompt); - return /furniture|product|commodity|goods|object|item|subject|sofa|couch|chair|table|desk|cabinet|shelf|bed|bench|lamp|basket|rack|stool|drawer|wardrobe|主物体|主体|物体|商品|产品|家具|沙发|椅|桌|柜|架|床|凳|灯|篮|抽屉|衣柜/.test( - text - ); -} - -function hasScaleAdjustPersonPixelSignal(params: { - box: { left: number; top: number; width: number; height: number }; - raw: Buffer; - width: number; -}) { - const { box, raw, width } = params; - let sampled = 0; - let skin = 0; - let whiteFabric = 0; - const step = Math.max(1, Math.floor(Math.min(box.width, box.height) / 260)); - - for (let localY = 0; localY < box.height; localY += step) { - const y = box.top + localY; - for (let localX = 0; localX < box.width; localX += step) { - const x = box.left + localX; - const rawIndex = getScaleAdjustRawPixelIndex(width, x, y); - const r = raw[rawIndex] || 0; - const g = raw[rawIndex + 1] || 0; - const b = raw[rawIndex + 2] || 0; - const { luma, saturation } = getScaleAdjustPixelStats(r, g, b); - sampled += 1; - if (isScaleAdjustLikelySkinPixel(r, g, b)) { - skin += 1; - } else if ( - isScaleAdjustLikelyWhiteFabricPixel(r, g, b, luma, saturation) && - luma < 242 - ) { - whiteFabric += 1; - } - } - } - - if (sampled <= 0) return false; - const skinRatio = skin / sampled; - const fabricRatio = whiteFabric / sampled; - return skin >= 18 && skinRatio >= 0.0025 && fabricRatio >= 0.035; -} - -function estimateScaleAdjustBoxBackground(params: { - box: { left: number; top: number; width: number; height: number }; - raw: Buffer; - width: number; -}) { - const { box, raw, width } = params; - const samples: Array<{ b: number; g: number; luma: number; r: number }> = []; - const borderX = Math.max(2, Math.round(box.width * 0.08)); - const borderY = Math.max(2, Math.round(box.height * 0.08)); - - for (let localY = 0; localY < box.height; localY += 1) { - const y = box.top + localY; - const isBorderY = localY < borderY || localY >= box.height - borderY; - for (let localX = 0; localX < box.width; localX += 1) { - const isBorderX = localX < borderX || localX >= box.width - borderX; - if (!isBorderX && !isBorderY) continue; - const x = box.left + localX; - const rawIndex = getScaleAdjustRawPixelIndex(width, x, y); - const r = raw[rawIndex] || 0; - const g = raw[rawIndex + 1] || 0; - const b = raw[rawIndex + 2] || 0; - const { luma, saturation } = getScaleAdjustPixelStats(r, g, b); - if (isScaleAdjustLikelySkinPixel(r, g, b)) continue; - if (saturation > 0.42 && luma < 185) continue; - samples.push({ b, g, luma, r }); - } - } - - if (samples.length === 0) { - return { b: 245, g: 247, luma: 246, r: 245 }; - } - - samples.sort((left, right) => right.luma - left.luma); - const kept = samples.slice(0, Math.max(12, Math.ceil(samples.length * 0.38))); - const totals = kept.reduce( - (acc, sample) => { - acc.r += sample.r; - acc.g += sample.g; - acc.b += sample.b; - acc.luma += sample.luma; - return acc; - }, - { b: 0, g: 0, luma: 0, r: 0 } - ); - - return { - b: totals.b / kept.length, - g: totals.g / kept.length, - luma: totals.luma / kept.length, - r: totals.r / kept.length, - }; -} - -function dilateScaleAdjustStructureMask(params: { - height: number; - mask: Uint8Array; - radius: number; - width: number; -}) { - const { height, mask, radius, width } = params; - if (radius <= 0) return mask; - const output = new Uint8Array(mask.length); - for (let y = 0; y < height; y += 1) { - for (let x = 0; x < width; x += 1) { - if (mask[y * width + x] === 0) continue; - const left = Math.max(0, x - radius); - const right = Math.min(width - 1, x + radius); - const top = Math.max(0, y - radius); - const bottom = Math.min(height - 1, y + radius); - for (let yy = top; yy <= bottom; yy += 1) { - for (let xx = left; xx <= right; xx += 1) { - output[yy * width + xx] = 255; - } - } - } - } - return output; -} - -function buildScaleAdjustStructureInitialMask(params: { - box: { left: number; top: number; width: number; height: number }; - includePersonPixels?: boolean; - raw: Buffer; - width: number; -}) { - const { box, includePersonPixels = false, raw, width } = params; - const background = estimateScaleAdjustBoxBackground(params); - const mask = new Uint8Array(box.width * box.height); - - for (let localY = 0; localY < box.height; localY += 1) { - const y = box.top + localY; - for (let localX = 0; localX < box.width; localX += 1) { - const x = box.left + localX; - const rawIndex = getScaleAdjustRawPixelIndex(width, x, y); - const r = raw[rawIndex] || 0; - const g = raw[rawIndex + 1] || 0; - const b = raw[rawIndex + 2] || 0; - const { luma, saturation } = getScaleAdjustPixelStats(r, g, b); - const likelySkin = isScaleAdjustLikelySkinPixel(r, g, b); - const likelyWhiteFabric = isScaleAdjustLikelyWhiteFabricPixel( - r, - g, - b, - luma, - saturation - ); - const colorDistance = Math.sqrt( - (r - background.r) ** 2 + - (g - background.g) ** 2 + - (b - background.b) ** 2 - ); - - if (includePersonPixels && likelySkin) { - mask[localY * box.width + localX] = 255; - continue; - } - if ( - includePersonPixels && - likelyWhiteFabric && - luma < 246 && - (colorDistance > 12 || luma < background.luma - 4) - ) { - mask[localY * box.width + localX] = 255; - continue; - } - if (likelySkin) continue; - if (likelyWhiteFabric) { - continue; - } - - const darkStructure = luma < Math.min(136, background.luma - 38); - const saturatedStructure = - saturation > 0.16 && luma < 218 && colorDistance > 38; - const woodOrCaneStructure = - r > 46 && - g > 26 && - b < 132 && - r >= g * 0.9 && - g >= b * 1.02 && - saturation > 0.18 && - luma < 218; - - if (darkStructure || saturatedStructure || woodOrCaneStructure) { - mask[localY * box.width + localX] = 255; - } - } - } - - return dilateScaleAdjustStructureMask({ - height: box.height, - mask, - radius: Math.max(1, Math.round(Math.min(box.width, box.height) * 0.004)), - width: box.width, - }); -} - -function filterScaleAdjustStructureComponents(params: { - box: { width: number; height: number }; - mask: Uint8Array; -}) { - const { box, mask } = params; - const width = box.width; - const height = box.height; - const visited = new Uint8Array(mask.length); - const output = new Uint8Array(mask.length); - const queueX: number[] = []; - const queueY: number[] = []; - const minComponentArea = Math.max( - 10, - Math.round(width * height * SCALE_ADJUST_STRUCTURE_LOCK_COMPONENT_MIN_RATIO) - ); - - for (let y = 0; y < height; y += 1) { - for (let x = 0; x < width; x += 1) { - const startIndex = y * width + x; - if (mask[startIndex] === 0 || visited[startIndex]) continue; - - queueX.length = 0; - queueY.length = 0; - queueX.push(x); - queueY.push(y); - visited[startIndex] = 1; - let head = 0; - let area = 0; - let minX = x; - let minY = y; - let maxX = x; - let maxY = y; - - while (head < queueX.length) { - const cx = queueX[head] || 0; - const cy = queueY[head] || 0; - head += 1; - area += 1; - minX = Math.min(minX, cx); - minY = Math.min(minY, cy); - maxX = Math.max(maxX, cx); - maxY = Math.max(maxY, cy); - - for (let dy = -1; dy <= 1; dy += 1) { - for (let dx = -1; dx <= 1; dx += 1) { - if (dx === 0 && dy === 0) continue; - const nx = cx + dx; - const ny = cy + dy; - if (nx < 0 || ny < 0 || nx >= width || ny >= height) continue; - const nextIndex = ny * width + nx; - if (mask[nextIndex] === 0 || visited[nextIndex]) continue; - visited[nextIndex] = 1; - queueX.push(nx); - queueY.push(ny); - } - } - } - - const componentWidth = maxX - minX + 1; - const componentHeight = maxY - minY + 1; - const componentBoxArea = Math.max(1, componentWidth * componentHeight); - const componentDensity = area / componentBoxArea; - const centerXRatio = (minX + componentWidth / 2) / width; - const centerYRatio = (minY + componentHeight / 2) / height; - const touchesTop = minY <= height * 0.04; - const touchesRight = maxX >= width * 0.88; - const edgeSliver = touchesRight && componentWidth < width * 0.18; - const broadTopBackground = - touchesTop && - centerYRatio <= 0.45 && - componentWidth >= width * 0.42 && - componentHeight >= height * 0.18 && - componentDensity >= 0.18; - const elongated = - componentHeight > componentWidth * 2.4 || - componentWidth > componentHeight * 2.4; - const likelyStructure = - area >= minComponentArea || - (elongated && area >= Math.max(8, minComponentArea * 0.4)); - const centeredEnough = centerXRatio >= 0.04 && centerXRatio <= 0.88; - const fillSurfaceComponent = - componentWidth >= width * 0.14 && - componentHeight >= height * 0.045 && - componentWidth <= width * 0.82 && - componentHeight <= height * 0.42 && - componentDensity >= 0.035 && - componentDensity <= 0.72; - - if ( - likelyStructure && - centeredEnough && - !edgeSliver && - !broadTopBackground - ) { - if (fillSurfaceComponent) { - for (let fillY = minY; fillY <= maxY; fillY += 1) { - for (let fillX = minX; fillX <= maxX; fillX += 1) { - output[fillY * width + fillX] = 255; - } - } - } else { - for (let index = 0; index < queueX.length; index += 1) { - output[(queueY[index] || 0) * width + (queueX[index] || 0)] = 255; - } - } - } - } - } - - return dilateScaleAdjustStructureMask({ - height, - mask: output, - radius: Math.max(1, Math.round(Math.min(width, height) * 0.006)), - width, - }); -} - -function solidifyScaleAdjustStructureMask(params: { - box: { width: number; height: number }; - mask: Uint8Array; -}) { - const { box, mask } = params; - const width = box.width; - const height = box.height; - const bridgeGap = Math.max(2, Math.round(Math.min(width, height) * 0.006)); - const output = dilateScaleAdjustStructureMask({ - height, - mask, - radius: Math.max(1, Math.round(Math.min(width, height) * 0.004)), - width, - }); - - for (let y = 0; y < height; y += 1) { - let x = 0; - while (x < width) { - while (x < width && output[y * width + x] !== 0) x += 1; - const gapStart = x; - while (x < width && output[y * width + x] === 0) x += 1; - const gapEnd = x - 1; - const gapLength = gapEnd - gapStart + 1; - if ( - gapLength > 0 && - gapLength <= bridgeGap && - gapStart > 0 && - gapEnd < width - 1 && - output[y * width + gapStart - 1] !== 0 && - output[y * width + gapEnd + 1] !== 0 - ) { - for (let fillX = gapStart; fillX <= gapEnd; fillX += 1) { - output[y * width + fillX] = 255; - } - } - } - } - - for (let x = 0; x < width; x += 1) { - let y = 0; - while (y < height) { - while (y < height && output[y * width + x] !== 0) y += 1; - const gapStart = y; - while (y < height && output[y * width + x] === 0) y += 1; - const gapEnd = y - 1; - const gapLength = gapEnd - gapStart + 1; - if ( - gapLength > 0 && - gapLength <= bridgeGap && - gapStart > 0 && - gapEnd < height - 1 && - output[(gapStart - 1) * width + x] !== 0 && - output[(gapEnd + 1) * width + x] !== 0 - ) { - for (let fillY = gapStart; fillY <= gapEnd; fillY += 1) { - output[fillY * width + x] = 255; - } - } - } - } - - const panelRowThreshold = Math.max(8, Math.round(width * 0.055)); - const panelMinHeight = Math.max(12, Math.round(height * 0.035)); - const panelMaxGap = Math.max(1, Math.round(height * 0.006)); - let panelStart = -1; - let panelGap = 0; - const getRowMarkedCount = (rowY: number) => { - let count = 0; - for (let rowX = 0; rowX < width; rowX += 1) { - if (output[rowY * width + rowX] !== 0) count += 1; - } - return count; - }; - const fillPanelBand = (startY: number, endY: number) => { - const bandHeight = endY - startY + 1; - if (bandHeight < panelMinHeight) return; - let minX = width; - let maxX = -1; - let marked = 0; - for (let bandY = startY; bandY <= endY; bandY += 1) { - for (let bandX = 0; bandX < width; bandX += 1) { - if (output[bandY * width + bandX] === 0) continue; - minX = Math.min(minX, bandX); - maxX = Math.max(maxX, bandX); - marked += 1; - } - } - if (maxX < minX) return; - const span = maxX - minX + 1; - const density = marked / Math.max(1, span * bandHeight); - if ( - span < width * 0.16 || - span > width * 0.86 || - density < 0.025 || - density > 0.78 - ) { - return; - } - for (let fillY = startY; fillY <= endY; fillY += 1) { - for (let fillX = minX; fillX <= maxX; fillX += 1) { - output[fillY * width + fillX] = 255; - } - } - }; - - for (let y = 0; y < height; y += 1) { - const hasPanelRow = getRowMarkedCount(y) >= panelRowThreshold; - if (hasPanelRow) { - if (panelStart < 0) panelStart = y; - panelGap = 0; - continue; - } - if (panelStart < 0) continue; - panelGap += 1; - if (panelGap <= panelMaxGap) continue; - fillPanelBand(panelStart, y - panelGap); - panelStart = -1; - panelGap = 0; - } - if (panelStart >= 0) { - fillPanelBand(panelStart, height - 1 - panelGap); - } - - return dilateScaleAdjustStructureMask({ - height, - mask: output, - radius: Math.max(1, Math.round(Math.min(width, height) * 0.004)), - width, - }); -} - -function getScaleAdjustMaskBoundingBox(params: { - height: number; - mask: Uint8Array; - width: number; -}) { - const { height, mask, width } = params; - let left = width; - let top = height; - let right = -1; - let bottom = -1; - let count = 0; - - for (let y = 0; y < height; y += 1) { - for (let x = 0; x < width; x += 1) { - if (mask[y * width + x] === 0) continue; - left = Math.min(left, x); - top = Math.min(top, y); - right = Math.max(right, x); - bottom = Math.max(bottom, y); - count += 1; - } - } - - if (right < left || bottom < top) return null; - return { - count, - left, - top, - width: right - left + 1, - height: bottom - top + 1, - }; -} - -function extractScaleAdjustMaskBuffer(params: { - box: { left: number; top: number; width: number; height: number }; - mask: Uint8Array; - maskWidth: number; -}) { - const { box, mask, maskWidth } = params; - const output = Buffer.alloc(box.width * box.height); - for (let y = 0; y < box.height; y += 1) { - const sourceStart = (box.top + y) * maskWidth + box.left; - const targetStart = y * box.width; - for (let x = 0; x < box.width; x += 1) { - output[targetStart + x] = mask[sourceStart + x] || 0; - } - } - return output; -} - -async function buildScaleAdjustMaskWashPatch(params: { - background: { b: number; g: number; r: number }; - box: { width: number; height: number }; - maskBuffer: Buffer; -}) { - const { background, box, maskBuffer } = params; - const alpha = await sharp(maskBuffer, { - raw: { channels: 1, height: box.height, width: box.width }, - }) - .blur(Math.max(1.2, Math.min(box.width, box.height) * 0.018)) - .raw() - .toBuffer(); - const rgba = Buffer.alloc(box.width * box.height * 4); - for (let index = 0; index < box.width * box.height; index += 1) { - const offset = index * 4; - rgba[offset] = Math.round(background.r); - rgba[offset + 1] = Math.round(background.g); - rgba[offset + 2] = Math.round(background.b); - rgba[offset + 3] = Math.round((alpha[index] || 0) * 0.9); - } - return sharp(rgba, { - raw: { channels: 4, height: box.height, width: box.width }, - }) - .png({ compressionLevel: 6 }) - .toBuffer(); -} - -function getScaleAdjustAlphaBoundingBox(params: { - height: number; - rawRgba: Buffer; - threshold?: number; - width: number; -}) { - const { height, rawRgba, threshold = 18, width } = params; - let left = width; - let top = height; - let right = -1; - let bottom = -1; - let count = 0; - - for (let y = 0; y < height; y += 1) { - for (let x = 0; x < width; x += 1) { - const alpha = rawRgba[(y * width + x) * 4 + 3] || 0; - if (alpha < threshold) continue; - left = Math.min(left, x); - top = Math.min(top, y); - right = Math.max(right, x); - bottom = Math.max(bottom, y); - count += 1; - } - } - - if (right < left || bottom < top) return null; - return { - count, - left, - top, - width: right - left + 1, - height: bottom - top + 1, - }; -} - -function thresholdScaleAdjustAlphaMask(params: { - maskBuffer: Buffer | Uint8Array; - threshold?: number; -}) { - const { maskBuffer, threshold = 96 } = params; - const output = new Uint8Array(maskBuffer.length); - for (let index = 0; index < maskBuffer.length; index += 1) { - output[index] = (maskBuffer[index] || 0) >= threshold ? 255 : 0; - } - return output; -} - -async function applyScaleAdjustAlphaMaskToRgba(params: { - alphaMask: Uint8Array; - height: number; - rawRgba: Buffer; - width: number; -}) { - const { alphaMask, height, rawRgba, width } = params; - const blur = Math.max(0.45, Math.min(width, height) * 0.0035); - const softenedAlpha = await sharp(Buffer.from(alphaMask), { - raw: { channels: 1, height, width }, - }) - .blur(blur) - .raw() - .toBuffer(); - const output = Buffer.from(rawRgba); - const pixelCount = Math.min(alphaMask.length, Math.floor(output.length / 4)); - for (let index = 0; index < pixelCount; index += 1) { - output[index * 4 + 3] = softenedAlpha[index] || 0; - } - return output; -} - -function getScaleAdjustAlphaBandingStats(params: { - height: number; - maskBuffer: Buffer | Uint8Array; - threshold?: number; - width: number; -}) { - const { height, maskBuffer, threshold = 24, width } = params; - const mask = thresholdScaleAdjustAlphaMask({ maskBuffer, threshold }); - const bbox = getScaleAdjustMaskBoundingBox({ height, mask, width }); - if (!bbox) { - return { - activeRowRatio: 0, - bboxHeightRatio: 0, - bboxWidthRatio: 0, - detected: false, - transitionRatio: 0, - wideRowRatio: 0, - }; - } - - let activeRows = 0; - let transitions = 0; - let wideRows = 0; - let previousCoverage: number | null = null; - for (let y = bbox.top; y < bbox.top + bbox.height; y += 1) { - let rowCount = 0; - for (let x = bbox.left; x < bbox.left + bbox.width; x += 1) { - if (mask[y * width + x] !== 0) rowCount += 1; - } - const coverage = rowCount / Math.max(1, bbox.width); - if (coverage >= 0.08) activeRows += 1; - if (coverage >= 0.45) wideRows += 1; - if ( - previousCoverage !== null && - Math.abs(coverage - previousCoverage) >= 0.28 - ) { - transitions += 1; - } - previousCoverage = coverage; - } - - const bboxHeightRatio = bbox.height / Math.max(1, height); - const bboxWidthRatio = bbox.width / Math.max(1, width); - const activeRowRatio = activeRows / Math.max(1, bbox.height); - const transitionRatio = transitions / Math.max(1, bbox.height - 1); - const wideRowRatio = wideRows / Math.max(1, bbox.height); - const detected = - (bboxWidthRatio >= 0.38 && - bboxHeightRatio >= 0.22 && - wideRowRatio >= 0.18 && - transitionRatio >= 0.12) || - (bboxWidthRatio >= 0.55 && - activeRowRatio >= 0.45 && - transitionRatio >= 0.09); - - return { - activeRowRatio: Number(activeRowRatio.toFixed(3)), - bboxHeightRatio: Number(bboxHeightRatio.toFixed(3)), - bboxWidthRatio: Number(bboxWidthRatio.toFixed(3)), - detected, - transitionRatio: Number(transitionRatio.toFixed(3)), - wideRowRatio: Number(wideRowRatio.toFixed(3)), - }; -} - -async function cleanScaleAdjustAlphaChannel(params: { - height: number; - referenceRawRgb?: Buffer | null; - rawRgba: Buffer; - threshold?: number; - width: number; -}) { - const { - height, - referenceRawRgb = null, - rawRgba, - threshold = SCALE_ADJUST_RMBG_ALPHA_CLEAN_THRESHOLD, - width, - } = params; - const pixelCount = Math.floor(rawRgba.length / 4); - const cleanedRaw = Buffer.from(rawRgba); - const cleanedAlpha = Buffer.alloc(pixelCount); - const range = Math.max(1, 255 - threshold); - const hasReference = - referenceRawRgb && referenceRawRgb.length >= pixelCount * 3; - - for (let index = 0; index < pixelCount; index += 1) { - const colorIndex = index * 4; - const alphaIndex = index * 4 + 3; - const sourceAlpha = rawRgba[alphaIndex] || 0; - let matteArtifact = false; - if (hasReference && sourceAlpha >= threshold) { - const r = rawRgba[colorIndex] || 0; - const g = rawRgba[colorIndex + 1] || 0; - const b = rawRgba[colorIndex + 2] || 0; - const luma = getScaleAdjustPixelLuma(r, g, b); - const channelSpread = Math.max(r, g, b) - Math.min(r, g, b); - const refIndex = index * 3; - const refLuma = getScaleAdjustPixelLuma( - referenceRawRgb?.[refIndex] || 0, - referenceRawRgb?.[refIndex + 1] || 0, - referenceRawRgb?.[refIndex + 2] || 0 - ); - matteArtifact = luma <= 18 && channelSpread <= 18 && refLuma >= 42; - } - const nextAlpha = - sourceAlpha < threshold || matteArtifact - ? 0 - : Math.max(0, Math.min(255, Math.round(((sourceAlpha - threshold) / range) * 255))); - cleanedRaw[alphaIndex] = nextAlpha; - cleanedAlpha[index] = nextAlpha; - } - - return { cleanedAlpha, cleanedRaw }; -} - -async function resizeScaleAdjustAlphaMask(params: { - height: number; - maskBuffer: Buffer; - targetHeight: number; - targetWidth: number; - width: number; -}) { - const { height, maskBuffer, targetHeight, targetWidth, width } = params; - const { data, info } = await sharp(maskBuffer, { - raw: { channels: 1, height, width }, - }) - .resize(targetWidth, targetHeight, { - background: { alpha: 1, b: 0, g: 0, r: 0 }, - fit: "contain", - kernel: sharp.kernel.lanczos3, - }) - .greyscale() - .raw() - .toBuffer({ resolveWithObject: true }); - if (info.channels === 1) return Buffer.from(data); - - const output = Buffer.alloc(targetWidth * targetHeight); - for (let index = 0; index < targetWidth * targetHeight; index += 1) { - output[index] = data[index * info.channels] || 0; - } - return output; -} - -async function buildScaleAdjustRmbgStructureLockedComposite(params: { - placement: ReturnType; - raw: Buffer; - sourceBuffer: Buffer; - sourceImageUrl?: string; - sourceWidth: number; - userId?: string; -}): Promise { - const { - placement, - raw, - sourceBuffer, - sourceImageUrl, - sourceWidth, - userId, - } = params; - if (!userId) return null; - - try { - const sourceCropBuffer = await sharp(sourceBuffer, { failOn: "none" }) - .extract(placement.sourceBox) - .png({ compressionLevel: 6 }) - .toBuffer(); - const cutoutResult = await executeRmbgJob({ - userId, - imageBase64: sourceCropBuffer.toString("base64"), - imageBuffer: sourceCropBuffer, - prompt: "比例调整主体抠图：只保留框选的目标人物/家具/商品主体，输出透明背景 PNG", - skipGallerySave: true, - sourceImageUrl, - }); - const cutoutBuffer = await fetchImageBufferFromUrl(cutoutResult.original_url); - const normalizedCutout = await sharp(cutoutBuffer, { failOn: "none" }) - .rotate() - .ensureAlpha() - .png({ compressionLevel: 6 }) - .toBuffer(); - const cutoutMetadata = await sharp(normalizedCutout, { failOn: "none" }).metadata(); - const cutoutWidth = Math.round(Number(cutoutMetadata.width || 0)); - const cutoutHeight = Math.round(Number(cutoutMetadata.height || 0)); - if (cutoutWidth <= 0 || cutoutHeight <= 0) return null; - const sameSizeCutout = - Math.abs(cutoutWidth - placement.sourceBox.width) <= 2 && - Math.abs(cutoutHeight - placement.sourceBox.height) <= 2; - const sourceCropRaw = sameSizeCutout - ? await sharp(sourceCropBuffer, { failOn: "none" }) - .removeAlpha() - .toColorspace("srgb") - .raw() - .toBuffer() - : null; - - const cutoutRaw = await sharp(normalizedCutout, { failOn: "none" }) - .ensureAlpha() - .raw() - .toBuffer(); - const { cleanedAlpha, cleanedRaw } = await cleanScaleAdjustAlphaChannel({ - height: cutoutHeight, - referenceRawRgb: sourceCropRaw, - rawRgba: cutoutRaw, - width: cutoutWidth, - }); - const highConfidenceCutoutMask = thresholdScaleAdjustAlphaMask({ - maskBuffer: cleanedAlpha, - threshold: 96, - }); - const highConfidenceCutoutBbox = getScaleAdjustMaskBoundingBox({ - height: cutoutHeight, - mask: highConfidenceCutoutMask, - width: cutoutWidth, - }); - if (!highConfidenceCutoutBbox) return null; - const refinedCutoutRaw = await applyScaleAdjustAlphaMaskToRgba({ - alphaMask: highConfidenceCutoutMask, - height: cutoutHeight, - rawRgba: cleanedRaw, - width: cutoutWidth, - }); - const cleanedCutout = await sharp(refinedCutoutRaw, { - raw: { channels: 4, height: cutoutHeight, width: cutoutWidth }, - }) - .png({ compressionLevel: 6 }) - .toBuffer(); - const alphaBbox = getScaleAdjustAlphaBoundingBox({ - height: cutoutHeight, - rawRgba: refinedCutoutRaw, - width: cutoutWidth, - }); - if (!alphaBbox) return null; - - const alphaRatio = alphaBbox.count / Math.max(1, cutoutWidth * cutoutHeight); - const bboxCoversCanvas = - alphaBbox.width >= cutoutWidth * 0.97 && - alphaBbox.height >= cutoutHeight * 0.97; - if ( - alphaRatio < SCALE_ADJUST_RMBG_MIN_ALPHA_RATIO || - (alphaRatio > SCALE_ADJUST_RMBG_MAX_SOLID_ALPHA_RATIO && bboxCoversCanvas) - ) { - return null; - } - - const fullAlphaMask = Buffer.from(highConfidenceCutoutMask); - const sourceMaskBuffer = sameSizeCutout - ? fullAlphaMask - : await resizeScaleAdjustAlphaMask({ - height: cutoutHeight, - maskBuffer: fullAlphaMask, - targetHeight: placement.sourceBox.height, - targetWidth: placement.sourceBox.width, - width: cutoutWidth, - }); - const alphaBandingStats = getScaleAdjustAlphaBandingStats({ - height: placement.sourceBox.height, - maskBuffer: sourceMaskBuffer, - width: placement.sourceBox.width, - }); - if (alphaBandingStats.detected) { - console.warn( - "[Canvas Local Edit] Scale adjust RMBG alpha banding detected; fallback to pixel mask:", - alphaBandingStats - ); - return null; - } - const background = estimateScaleAdjustBoxBackground({ - box: placement.sourceBox, - raw, - width: sourceWidth, - }); - const highConfidenceSourceMask = thresholdScaleAdjustAlphaMask({ - maskBuffer: sourceMaskBuffer, - threshold: 96, - }); - const filteredSourceMask = filterScaleAdjustStructureComponents({ - box: placement.sourceBox, - mask: highConfidenceSourceMask, - }); - const filteredSourceBbox = getScaleAdjustMaskBoundingBox({ - height: placement.sourceBox.height, - mask: filteredSourceMask, - width: placement.sourceBox.width, - }); - const sourceWashBaseMask = filteredSourceBbox - ? solidifyScaleAdjustStructureMask({ - box: placement.sourceBox, - mask: filteredSourceMask, - }) - : highConfidenceSourceMask; - const sourceWashMask = dilateScaleAdjustStructureMask({ - height: placement.sourceBox.height, - mask: sourceWashBaseMask, - radius: Math.max( - 2, - Math.round( - Math.min(placement.sourceBox.width, placement.sourceBox.height) * 0.014 - ) - ), - width: placement.sourceBox.width, - }); - const sourceWashPatch = await buildScaleAdjustMaskWashPatch({ - background, - box: placement.sourceBox, - maskBuffer: Buffer.from(sourceWashMask), - }); - - const subjectCutout = await sharp(cleanedCutout, { failOn: "none" }) - .extract(alphaBbox) - .png({ compressionLevel: 6 }) - .toBuffer(); - const widthFactor = - placement.targetBox.width / Math.max(1, placement.sourceFootprintBox.width); - const heightFactor = - placement.targetBox.height / Math.max(1, placement.sourceFootprintBox.height); - const structureScale = Math.max( - 0.05, - Math.min( - 3.5, - Math.sqrt( - Math.max(0.0025, widthFactor) * Math.max(0.0025, heightFactor) - ) - ) - ); - let placedWidth = Math.max(1, Math.round(alphaBbox.width * structureScale)); - let placedHeight = Math.max(1, Math.round(alphaBbox.height * structureScale)); - if (!sameSizeCutout) { - const maxPlacedWidth = Math.max(1, Math.round(placement.targetBox.width * 1.06)); - const maxPlacedHeight = Math.max(1, Math.round(placement.targetBox.height * 1.06)); - const fitRatio = Math.min( - 1, - maxPlacedWidth / Math.max(1, placedWidth), - maxPlacedHeight / Math.max(1, placedHeight) - ); - placedWidth = Math.max(1, Math.round(placedWidth * fitRatio)); - placedHeight = Math.max(1, Math.round(placedHeight * fitRatio)); - } - - const sourceSubjectBox = sameSizeCutout - ? { - left: placement.sourceBox.left + alphaBbox.left, - top: placement.sourceBox.top + alphaBbox.top, - width: alphaBbox.width, - height: alphaBbox.height, - } - : null; - const sourceCenterX = sourceSubjectBox - ? sourceSubjectBox.left + sourceSubjectBox.width / 2 - : placement.sourceBox.left + placement.sourceBox.width / 2; - const sourceCenterY = sourceSubjectBox - ? sourceSubjectBox.top + sourceSubjectBox.height / 2 - : placement.sourceBox.top + placement.sourceBox.height / 2; - const targetCenterX = sourceSubjectBox - ? placement.targetBox.left + - ((sourceCenterX - placement.sourceFootprintBox.left) / - Math.max(1, placement.sourceFootprintBox.width)) * - placement.targetBox.width - : placement.targetBox.left + placement.targetBox.width / 2; - const targetCenterY = sourceSubjectBox - ? placement.targetBox.top + - ((sourceCenterY - placement.sourceFootprintBox.top) / - Math.max(1, placement.sourceFootprintBox.height)) * - placement.targetBox.height - : placement.targetBox.top + placement.targetBox.height / 2; - const structureOverlay = await sharp(subjectCutout, { failOn: "none" }) - .resize(placedWidth, placedHeight, { - fit: "fill", - kernel: sharp.kernel.lanczos3, - }) - .png({ compressionLevel: 6 }) - .toBuffer(); - - console.info("[Canvas Local Edit] Scale adjust RMBG structure draft applied:", { - alphaRatio: Number(alphaRatio.toFixed(3)), - cutoutHeight, - cutoutWidth, - placedHeight, - placedWidth, - sameSizeCutout, - alphaBandingStats, - sourceHeight: placement.sourceBox.height, - sourceWidth: placement.sourceBox.width, - }); - - return { - method: "rmbg-cutout", - overlay: { - input: structureOverlay, - left: Math.round(targetCenterX - placedWidth / 2), - top: Math.round(targetCenterY - placedHeight / 2), - }, - wash: { - input: sourceWashPatch, - left: placement.sourceBox.left, - top: placement.sourceBox.top, - }, - }; - } catch (error) { - console.warn("[Canvas Local Edit] Scale adjust RMBG structure draft failed; fallback to pixel mask:", { - error: error instanceof Error ? error.message : String(error), - }); - return null; - } -} - -async function buildScaleAdjustStructureLockedComposite(params: { - includePersonPixels?: boolean; - placement: ReturnType; - raw: Buffer; - sourceBuffer: Buffer; - sourceImageUrl?: string; - sourceWidth: number; - userId?: string; -}): Promise { - const { - includePersonPixels = false, - placement, - raw, - sourceBuffer, - sourceImageUrl, - sourceWidth, - userId, - } = params; - // The current RMBG provider can bake horizontal matte stripes into furniture cutouts. - const allowRmbgStructureLock = - process.env.CANVAS_SCALE_ADJUST_ENABLE_RMBG_STRUCTURE === "1" || - (includePersonPixels && - process.env.CANVAS_SCALE_ADJUST_DISABLE_PERSON_RMBG !== "1"); - const rmbgComposite = !allowRmbgStructureLock - ? null - : await buildScaleAdjustRmbgStructureLockedComposite({ - placement, - raw, - sourceBuffer, - sourceImageUrl, - sourceWidth, - userId, - }); - if (rmbgComposite) return rmbgComposite; - if (includePersonPixels) return null; - - const initialMask = buildScaleAdjustStructureInitialMask({ - box: placement.sourceBox, - includePersonPixels, - raw, - width: sourceWidth, - }); - const mask = filterScaleAdjustStructureComponents({ - box: placement.sourceBox, - mask: initialMask, - }); - const structureBbox = getScaleAdjustMaskBoundingBox({ - height: placement.sourceBox.height, - mask, - width: placement.sourceBox.width, - }); - if (!structureBbox) return null; - const maskRatio = structureBbox.count / Math.max(1, placement.sourceBox.width * placement.sourceBox.height); - if (maskRatio < SCALE_ADJUST_STRUCTURE_LOCK_MIN_MASK_RATIO) return null; - const subjectMask = solidifyScaleAdjustStructureMask({ - box: placement.sourceBox, - mask, - }); - const bbox = getScaleAdjustMaskBoundingBox({ - height: placement.sourceBox.height, - mask: subjectMask, - width: placement.sourceBox.width, - }); - if (!bbox) return null; - - const sourceSubjectBox = { - left: placement.sourceBox.left + bbox.left, - top: placement.sourceBox.top + bbox.top, - width: bbox.width, - height: bbox.height, - }; - const sourceSubjectMask = extractScaleAdjustMaskBuffer({ - box: bbox, - mask: subjectMask, - maskWidth: placement.sourceBox.width, - }); - const background = estimateScaleAdjustBoxBackground({ - box: placement.sourceBox, - raw, - width: sourceWidth, - }); - const sourceWashMask = dilateScaleAdjustStructureMask({ - height: placement.sourceBox.height, - mask: subjectMask, - radius: Math.max( - 2, - Math.round(Math.min(placement.sourceBox.width, placement.sourceBox.height) * 0.012) - ), - width: placement.sourceBox.width, - }); - const sourceWashPatch = await buildScaleAdjustMaskWashPatch({ - background, - box: placement.sourceBox, - maskBuffer: Buffer.from(sourceWashMask), - }); - const subjectCrop = await sharp(sourceBuffer, { failOn: "none" }) - .extract(sourceSubjectBox) - .removeAlpha() - .png({ compressionLevel: 6 }) - .toBuffer(); - const subjectAlpha = await sharp(sourceSubjectMask, { - raw: { channels: 1, height: bbox.height, width: bbox.width }, - }) - .blur(Math.max(0.7, Math.min(bbox.width, bbox.height) * 0.006)) - .png({ compressionLevel: 6 }) - .toBuffer(); - const widthFactor = - placement.targetBox.width / Math.max(1, placement.sourceFootprintBox.width); - const heightFactor = - placement.targetBox.height / Math.max(1, placement.sourceFootprintBox.height); - const structureScale = Math.max( - 0.05, - Math.min( - 3.5, - Math.sqrt( - Math.max(0.0025, widthFactor) * Math.max(0.0025, heightFactor) - ) - ) - ); - const placedWidth = Math.max(1, Math.round(bbox.width * structureScale)); - const placedHeight = Math.max(1, Math.round(bbox.height * structureScale)); - const sourceCenterX = sourceSubjectBox.left + sourceSubjectBox.width / 2; - const sourceCenterY = sourceSubjectBox.top + sourceSubjectBox.height / 2; - const targetCenterX = - placement.targetBox.left + - ((sourceCenterX - placement.sourceFootprintBox.left) / - Math.max(1, placement.sourceFootprintBox.width)) * - placement.targetBox.width; - const targetCenterY = - placement.targetBox.top + - ((sourceCenterY - placement.sourceFootprintBox.top) / - Math.max(1, placement.sourceFootprintBox.height)) * - placement.targetBox.height; - const structureOverlay = await sharp(subjectCrop, { failOn: "none" }) - .joinChannel(subjectAlpha) - .resize(placedWidth, placedHeight, { - fit: "fill", - kernel: sharp.kernel.lanczos3, - }) - .png({ compressionLevel: 6 }) - .toBuffer(); - - return { - method: "pixel-mask", - overlay: { - input: structureOverlay, - left: Math.round(targetCenterX - placedWidth / 2), - top: Math.round(targetCenterY - placedHeight / 2), - }, - wash: { - input: sourceWashPatch, - left: placement.sourceBox.left, - top: placement.sourceBox.top, - }, - }; -} - -function getScaleAdjustEditMaskBoxes(params: { - regions: NormalizedLocalEditRegion[]; - scalePercent?: ScaleAdjustPercent | null; - sourceHeight: number; - sourceWidth: number; -}) { - const { regions, scalePercent, sourceHeight, sourceWidth } = params; - return regions.map((region) => { - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const workspaceRect = getScaleAdjustWorkspaceRect( - region.rect, - targetRect, - scalePercent - ); - return rectToPixels(workspaceRect, sourceWidth, sourceHeight); - }); -} - -async function buildScaleAdjustEditMaskBuffer(params: { - regions: NormalizedLocalEditRegion[]; - scalePercent?: ScaleAdjustPercent | null; - sourceHeight: number; - sourceWidth: number; -}) { - const { regions, scalePercent, sourceHeight, sourceWidth } = params; - const boxes = getScaleAdjustEditMaskBoxes({ - regions, - scalePercent, - sourceHeight, - sourceWidth, - }); - const minSide = Math.max(1, Math.min(sourceWidth, sourceHeight)); - const blur = Math.max(5, Math.min(28, Math.round(minSide * 0.018))); - const pad = Math.max(2, Math.round(blur * 0.55)); - const rects = boxes - .map((box) => { - const left = clamp(box.left - pad, 0, Math.max(0, sourceWidth - 1)); - const top = clamp(box.top - pad, 0, Math.max(0, sourceHeight - 1)); - const right = clamp( - box.left + box.width + pad, - left + 1, - sourceWidth - ); - const bottom = clamp( - box.top + box.height + pad, - top + 1, - sourceHeight - ); - return ``; - }) - .join(""); - const editMaskSvg = Buffer.from( - ` - - - - - - - ${rects} - ` - ); - const editAlpha = await sharp(editMaskSvg, { failOn: "none" }) - .resize(sourceWidth, sourceHeight, { fit: "fill" }) - .greyscale() - .raw() - .toBuffer(); - const rgba = Buffer.alloc(sourceWidth * sourceHeight * 4); - for (let index = 0; index < sourceWidth * sourceHeight; index += 1) { - const offset = index * 4; - const editable = editAlpha[index] || 0; - rgba[offset] = 0; - rgba[offset + 1] = 0; - rgba[offset + 2] = 0; - rgba[offset + 3] = clamp(255 - editable, 0, 255); - } - return sharp(rgba, { - raw: { channels: 4, height: sourceHeight, width: sourceWidth }, - }) - .png({ compressionLevel: 9 }) - .toBuffer(); -} - -async function buildScaleAdjustVisualLayoutGuideBuffer(params: { - regions: NormalizedLocalEditRegion[]; - scalePercent?: ScaleAdjustPercent | null; - sourceBuffer: Buffer; - sourceHeight: number; - sourceImageUrl?: string; - sourceWidth: number; - structureLockAppliedRef?: { value: boolean }; - structureLockedDraft?: boolean; - userId?: string; -}) { - const { - regions, - scalePercent, - sourceBuffer, - sourceHeight, - sourceImageUrl, - sourceWidth, - structureLockAppliedRef, - structureLockedDraft = false, - userId, - } = params; - const sourceRaw = - structureLockedDraft || regions.length > 0 - ? await sharp(sourceBuffer, { failOn: "none" }) - .removeAlpha() - .toColorspace("srgb") - .raw() - .toBuffer() - : null; - const sourceFootprintBoxes: Array<{ - background?: { b: number; g: number; r: number } | null; - left: number; - top: number; - width: number; - height: number; - }> = []; - const placements: Array> = - []; - const composites: Array<{ input: Buffer; left: number; top: number }> = []; - const subjectWashComposites: Array<{ - input: Buffer; - left: number; - top: number; - }> = []; - - for (const region of regions) { - const placement = getScaleAdjustVisualGuidePlacement({ - height: sourceHeight, - region, - scalePercent, - width: sourceWidth, - }); - placements.push(placement); - const targetWidthFactor = - placement.targetBox.width / Math.max(1, placement.sourceFootprintBox.width); - const targetHeightFactor = - placement.targetBox.height / Math.max(1, placement.sourceFootprintBox.height); - const shouldUseStructureLock = - structureLockedDraft && - sourceRaw && - (targetWidthFactor > 1.08 || - targetHeightFactor > 1.08 || - targetWidthFactor < 0.92 || - targetHeightFactor < 0.92); - const structureLockedComposite = shouldUseStructureLock - ? await buildScaleAdjustStructureLockedComposite({ - includePersonPixels: - isScaleAdjustPersonLikePrompt(region.prompt) || - hasScaleAdjustPersonPixelSignal({ - box: placement.sourceBox, - raw: sourceRaw, - width: sourceWidth, - }), - placement, - raw: sourceRaw, - sourceBuffer, - sourceImageUrl, - sourceWidth, - userId, - }) - : null; - - if (structureLockedComposite) { - if (structureLockAppliedRef) { - structureLockAppliedRef.value = true; - } - subjectWashComposites.push(structureLockedComposite.wash); - composites.push(structureLockedComposite.overlay); - continue; - } - - const shouldUseShrinkCutoutDraft = - process.env.CANVAS_SCALE_ADJUST_ENABLE_RMBG_SHRINK_CUTOUT === "1" && - !structureLockedDraft && - sourceRaw && - userId && - (targetWidthFactor < 0.92 || targetHeightFactor < 0.92); - const shrinkCutoutComposite = shouldUseShrinkCutoutDraft - ? await buildScaleAdjustRmbgStructureLockedComposite({ - placement, - raw: sourceRaw, - sourceBuffer, - sourceImageUrl, - sourceWidth, - userId, - }) - : null; - - if (shrinkCutoutComposite) { - subjectWashComposites.push(shrinkCutoutComposite.wash); - composites.push(shrinkCutoutComposite.overlay); - continue; - } - - sourceFootprintBoxes.push({ - ...placement.sourceFootprintBox, - background: sourceRaw - ? estimateScaleAdjustBoxBackground({ - box: placement.sourceFootprintBox, - raw: sourceRaw, - width: sourceWidth, - }) - : null, - }); - - const resizedCropBuffer = await sharp(sourceBuffer, { failOn: "none" }) - .extract(placement.sourceBox) - .resize(placement.placedWidth, placement.placedHeight, { - fit: "fill", - }) - .png({ compressionLevel: 6 }) - .toBuffer(); - const cropBuffer = await sharp(resizedCropBuffer, { failOn: "none" }) - .ensureAlpha() - .composite([ - { - input: buildScaleAdjustPatchMaskSvg({ - height: placement.placedHeight, - width: placement.placedWidth, - }), - blend: "dest-in", - }, - ]) - .png({ compressionLevel: 6 }) - .toBuffer(); - composites.push({ - input: cropBuffer, - left: placement.placedLeft, - top: placement.placedTop, - }); - } - - const washComposite = sourceFootprintBoxes.length - ? [ - { - input: buildScaleAdjustSourceFootprintWashSvg({ - boxes: sourceFootprintBoxes, - height: sourceHeight, - width: sourceWidth, - }), - left: 0, - top: 0, - }, - ] - : []; - const targetShadowComposite = placements.length - ? structureLockedDraft - ? [] - : [ - { - input: buildScaleAdjustTargetFootprintShadowSvg({ - height: sourceHeight, - placements, - width: sourceWidth, - }), - left: 0, - top: 0, - }, - ] - : []; - - return sharp(sourceBuffer, { failOn: "none" }) - .composite([ - ...washComposite, - ...subjectWashComposites, - ...targetShadowComposite, - ...composites, - ]) - .png({ compressionLevel: 6 }) - .toBuffer(); -} - -async function createScaleAdjustPatchReference(params: { - regions: NormalizedLocalEditRegion[]; - scalePercent?: ScaleAdjustPercent | null; - sourceBuffer: Buffer; - sourceHeight: number; - sourceImageUrl?: string; - sourceWidth: number; - userId?: string; -}): Promise { - const { - regions, - scalePercent, - sourceBuffer, - sourceHeight, - sourceImageUrl, - sourceWidth, - userId, - } = params; - const workspaceRect = expandRectToProviderAspectLimit( - getScaleAdjustPatchWorkspaceRect(regions, scalePercent), - sourceWidth, - sourceHeight - ); - const workspacePixelRect = rectToPixels(workspaceRect, sourceWidth, sourceHeight); - const shouldUseStructureLockedDraft = hasScaleAdjustStructureTarget( - regions, - scalePercent - ); - const structureLockAppliedRef = { value: false }; - const targetDraftBuffer = await buildScaleAdjustVisualLayoutGuideBuffer({ - regions, - scalePercent, - sourceBuffer, - sourceHeight, - sourceImageUrl, - sourceWidth, - structureLockAppliedRef, - structureLockedDraft: shouldUseStructureLockedDraft, - userId, - }); - const [sourceCropBuffer, targetDraftCropBuffer] = await Promise.all([ - sharp(sourceBuffer, { failOn: "none" }) - .extract(workspacePixelRect) - .png({ compressionLevel: 6 }) - .toBuffer(), - sharp(targetDraftBuffer, { failOn: "none" }) - .extract(workspacePixelRect) - .png({ compressionLevel: 6 }) - .toBuffer(), - ]); - const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; - const [sourceCropUpload, targetDraftUpload] = await Promise.all([ - uploadFile( - `canvas-scale-adjust-workspace-source-${suffix}.png`, - toArrayBuffer(sourceCropBuffer) - ), - uploadFile( - `canvas-scale-adjust-workspace-target-draft-${suffix}.png`, - toArrayBuffer(targetDraftCropBuffer) - ), - ]); - - return { - sourceCropUrl: sourceCropUpload.url, - structureLockedDraft: structureLockAppliedRef.value, - targetDraftUrl: targetDraftUpload.url, - workspacePixelRect, - workspaceRect, - }; -} - function wrapText(text: string, maxChars: number, maxLines = 3) { const compact = text.replace(/\s+/g, " ").trim(); if (!compact) return []; @@ -3099,278 +475,6 @@ function buildAnnotationSvg(params: { `); } -function buildScaleAdjustAnnotationSvg(params: { - width: number; - height: number; - regions: NormalizedLocalEditRegion[]; - scalePercent?: ScaleAdjustPercent | null; -}) { - const { width, height, regions, scalePercent } = params; - const minSide = Math.max(1, Math.min(width, height)); - const strokeWidth = Math.max(2, Math.round(minSide * 0.0034)); - const fontSize = Math.max(16, Math.round(minSide * 0.018)); - const labelPadX = Math.round(fontSize * 0.45); - const labelPadY = Math.round(fontSize * 0.24); - const targetColor = "#111827"; - const sourceColor = "#9ca3af"; - const helperColor = "#4b5563"; - const label = (text: string, x: number, y: number, color: string) => { - const labelWidth = Math.round(text.length * fontSize * 0.62 + labelPadX * 2); - const labelHeight = Math.round(fontSize * 1.2 + labelPadY * 2); - const left = clamp(x, 6, Math.max(6, width - labelWidth - 6)); - const top = clamp(y, 6, Math.max(6, height - labelHeight - 6)); - return ` - - ${escapeXml(text)} - `; - }; - - const regionSvg = regions - .map((region) => { - const source = region.pixelRect; - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const target = - region.targetPixelRect || rectToPixels(targetRect, width, height); - const radius = Math.max(8, Math.round(strokeWidth * 2)); - const sourceCenterX = source.left + source.width / 2; - const sourceCenterY = source.top + source.height / 2; - const targetCenterX = target.left + target.width / 2; - const targetCenterY = target.top + target.height / 2; - const widthFactor = source.width > 0 ? target.width / source.width : 1; - const heightFactor = source.height > 0 ? target.height / source.height : 1; - const isShrink = widthFactor < 0.92 || heightFactor < 0.92; - const isEnlarge = widthFactor > 1.08 || heightFactor > 1.08; - const actionLabel = isShrink ? "SHRINK" : isEnlarge ? "ENLARGE" : "MOVE"; - const scaleLabel = `${actionLabel} ${widthFactor.toFixed(2)}W ${heightFactor.toFixed(2)}H`; - const instructionLabel = isShrink - ? "OLD OUTSIDE -> BG" - : isEnlarge - ? "TARGET = FINAL FOOTPRINT" - : "FINAL SUBJECT CENTER"; - const centerMoved = - Math.hypot(targetCenterX - sourceCenterX, targetCenterY - sourceCenterY) > - Math.max(8, minSide * 0.012); - - return ` - - - - - - - ${ - centerMoved - ? `` - : "" - } - ${label(`OLD ${region.index}`, source.left + strokeWidth, source.top + strokeWidth, sourceColor)} - ${label(`TARGET ${region.index}`, target.left + strokeWidth, target.top + strokeWidth, targetColor)} - ${label(scaleLabel, target.left + strokeWidth, target.top + target.height + strokeWidth * 2, helperColor)} - ${label(instructionLabel, source.left + strokeWidth, source.top + source.height + strokeWidth * 2, helperColor)} - `; - }) - .join(""); - - return Buffer.from(` - - - - - - - - - ${regionSvg} - - `); -} - -function buildScaleAdjustZoomedCoordinateSvg(params: { - width: number; - height: number; - regions: NormalizedLocalEditRegion[]; - scalePercent?: ScaleAdjustPercent | null; -}) { - const { width, height, regions, scalePercent } = params; - const minSide = Math.max(1, Math.min(width, height)); - const strokeWidth = Math.max(3, Math.round(minSide * 0.006)); - const fontSize = Math.max(22, Math.round(minSide * 0.026)); - const smallFontSize = Math.max(18, Math.round(fontSize * 0.72)); - const panelMarginX = Math.round(width * 0.1); - const panelMarginY = Math.round(height * 0.12); - const panelLeft = panelMarginX; - const panelTop = panelMarginY; - const panelWidth = Math.max(1, width - panelMarginX * 2); - const panelHeight = Math.max(1, height - panelMarginY * 2); - const sourceColor = "#9ca3af"; - const targetColor = "#111827"; - const helperColor = "#4b5563"; - const padX = Math.round(fontSize * 0.42); - const labelHeight = Math.round(fontSize * 1.35); - const label = (text: string, x: number, y: number, color: string) => { - const labelWidth = Math.min( - Math.round(text.length * fontSize * 0.58 + padX * 2), - Math.max(160, width - 16) - ); - const left = clamp(x, 8, Math.max(8, width - labelWidth - 8)); - const top = clamp(y, 8, Math.max(8, height - labelHeight - 8)); - return ` - - ${escapeXml(text)} - `; - }; - - const regionSvg = regions - .map((region) => { - const source = region.pixelRect; - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const target = - region.targetPixelRect || rectToPixels(targetRect, width, height); - const sourceRight = source.left + source.width; - const sourceBottom = source.top + source.height; - const targetRight = target.left + target.width; - const targetBottom = target.top + target.height; - const unionLeft = Math.min(source.left, target.left); - const unionTop = Math.min(source.top, target.top); - const unionRight = Math.max(sourceRight, targetRight); - const unionBottom = Math.max(sourceBottom, targetBottom); - const marginX = Math.max(12, Math.round((unionRight - unionLeft) * 0.35)); - const marginY = Math.max(12, Math.round((unionBottom - unionTop) * 0.35)); - const viewLeft = unionLeft - marginX; - const viewTop = unionTop - marginY; - const viewWidth = Math.max(1, unionRight - unionLeft + marginX * 2); - const viewHeight = Math.max(1, unionBottom - unionTop + marginY * 2); - const scale = Math.min( - panelWidth / viewWidth, - panelHeight / viewHeight - ); - const drawnWidth = viewWidth * scale; - const drawnHeight = viewHeight * scale; - const originX = panelLeft + (panelWidth - drawnWidth) / 2 - viewLeft * scale; - const originY = panelTop + (panelHeight - drawnHeight) / 2 - viewTop * scale; - const project = (box: { left: number; top: number; width: number; height: number }) => ({ - left: Math.round(originX + box.left * scale), - top: Math.round(originY + box.top * scale), - width: Math.max(1, Math.round(box.width * scale)), - height: Math.max(1, Math.round(box.height * scale)), - }); - const sourceBox = project(source); - const targetBox = project(target); - const sourceCenterX = sourceBox.left + sourceBox.width / 2; - const sourceCenterY = sourceBox.top + sourceBox.height / 2; - const targetCenterX = targetBox.left + targetBox.width / 2; - const targetCenterY = targetBox.top + targetBox.height / 2; - const widthFactor = source.width > 0 ? target.width / source.width : 1; - const heightFactor = source.height > 0 ? target.height / source.height : 1; - const actionLabel = - widthFactor < 0.92 || heightFactor < 0.92 - ? "SHRINK" - : widthFactor > 1.08 || heightFactor > 1.08 - ? "ENLARGE" - : "MOVE"; - const scaleLabel = `${actionLabel} ${widthFactor.toFixed(2)}W ${heightFactor.toFixed(2)}H`; - const radius = Math.max(10, Math.round(strokeWidth * 2.2)); - const centerMoved = - Math.hypot(targetCenterX - sourceCenterX, targetCenterY - sourceCenterY) > - Math.max(10, minSide * 0.018); - - return ` - - - - - ${ - centerMoved - ? `` - : "" - } - ${label(`OLD ${region.index}`, sourceBox.left, sourceBox.top - labelHeight - strokeWidth, sourceColor)} - ${label(`TARGET ${region.index}`, targetBox.left, targetBox.top + strokeWidth, targetColor)} - ${label(scaleLabel, targetBox.left, targetBox.top + targetBox.height + strokeWidth * 2, helperColor)} - ${label("OLD OUTSIDE -> BG", sourceBox.left, sourceBox.top + sourceBox.height + strokeWidth * 4 + labelHeight, helperColor)} - `; - }) - .join(""); - - return Buffer.from(` - - - - - - - - - LOCAL SCALE MAP - USE PROMPT PIXELS FOR ABSOLUTE POSITION - ${regionSvg} - - `); -} - -function buildScaleAdjustMarkedPhotoGuideSvg(params: { - width: number; - height: number; - regions: NormalizedLocalEditRegion[]; - scalePercent?: ScaleAdjustPercent | null; -}) { - const { width, height, regions, scalePercent } = params; - const minSide = Math.max(1, Math.min(width, height)); - const strokeWidth = Math.max(3, Math.round(minSide * 0.005)); - const currentColor = "#00b8d4"; - const finalColor = "#2563eb"; - - const regionSvg = regions - .map((region) => { - const source = region.pixelRect; - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const target = - region.targetPixelRect || rectToPixels(targetRect, width, height); - const radius = Math.max(8, Math.round(strokeWidth * 2)); - return ` - - - - - `; - }) - .join(""); - - return Buffer.from(` - - - ${regionSvg} - - `); -} - -async function buildScaleAdjustMarkedPhotoGuideBuffer(params: { - sourceBuffer: Buffer; - width: number; - height: number; - regions: NormalizedLocalEditRegion[]; - scalePercent?: ScaleAdjustPercent | null; -}) { - const { sourceBuffer, width, height, regions, scalePercent } = params; - return sharp(sourceBuffer, { failOn: "none" }) - .rotate() - .resize(width, height, { fit: "cover", position: "center" }) - .modulate({ brightness: 1.04, saturation: 0.45 }) - .composite([ - { - input: buildScaleAdjustMarkedPhotoGuideSvg({ - height, - regions, - scalePercent, - width, - }), - left: 0, - top: 0, - }, - ]) - .png({ compressionLevel: 6 }) - .toBuffer(); -} - function buildLocalEditPrompt(regions: NormalizedLocalEditRegion[]) { const regionLines = regions .map((region) => { @@ -3414,426 +518,58 @@ function buildLocalEditPrompt(regions: NormalizedLocalEditRegion[]) { ].join("\n"); } -function buildScaleAdjustPrompt( +function buildSimpleScaleAdjustPrompt( regions: NormalizedLocalEditRegion[], scalePercent: ScaleAdjustPercent | null | undefined, - cropReferences: ScaleAdjustCropReference[] = [], - sourceSize?: { width: number; height: number } -) { - const sourceSizeLine = sourceSize - ? `最终输出必须保持与输入图片1完全相同的画幅和像素尺寸：${sourceSize.width} x ${sourceSize.height} px。` - : "最终输出必须保持与输入图片1完全相同的画幅和像素尺寸。"; - const regionLines = regions - .map((region) => { - const cropIndex = cropReferences.findIndex( - (reference) => reference.regionIndex === region.index - ); - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const sourceCenterX = region.rect.x + region.rect.width / 2; - const sourceCenterY = region.rect.y + region.rect.height / 2; - const targetCenterX = targetRect.x + targetRect.width / 2; - const targetCenterY = targetRect.y + targetRect.height / 2; - const moved = - Math.hypot(targetCenterX - sourceCenterX, targetCenterY - sourceCenterY) > - 0.01; - const targetPixelRect = - region.targetPixelRect || - (sourceSize - ? rectToPixels(targetRect, sourceSize.width, sourceSize.height) - : null); - const widthFactor = - region.rect.width > 0 ? targetRect.width / region.rect.width : 1; - const heightFactor = - region.rect.height > 0 ? targetRect.height / region.rect.height : 1; - const sourcePixelLine = `源框像素坐标：x=${region.pixelRect.left}, y=${region.pixelRect.top}, w=${region.pixelRect.width}, h=${region.pixelRect.height}`; - const targetPixelLine = targetPixelRect - ? `目标框像素坐标：x=${targetPixelRect.left}, y=${targetPixelRect.top}, w=${targetPixelRect.width}, h=${targetPixelRect.height}` - : `目标框相对坐标：x=${Math.round(targetRect.x * 1000) / 10}%, y=${Math.round(targetRect.y * 1000) / 10}%, w=${Math.round(targetRect.width * 1000) / 10}%, h=${Math.round(targetRect.height * 1000) / 10}%`; - const scaleLine = `目标视觉占位约为原主体宽度的 ${widthFactor.toFixed(2)} 倍、高度的 ${heightFactor.toFixed(2)} 倍；执行时必须以等比主体缩放为准，不要拉伸主体去硬贴目标框。`; - const targetPositionNote = moved - ? "目标框同时指定用户希望的最终目标大小和目标位置，目标主体调整后的视觉中心应移动到目标框中心附近。" - : "目标框指定用户希望的最终目标大小，目标主体调整后应围绕当前主体位置自然放置。"; - const cropNote = - cropIndex >= 0 - ? `；输入图片${cropIndex + 3}是该区域的干净局部裁切参考图，只用于识别目标主体的身份、轮廓与原始比例` - : ""; - return `区域 ${region.index}${cropNote}：请在输入图片1中找到源框范围内最主要、可独立调整的商品/物体主体。${sourcePixelLine}；${targetPixelLine}；${scaleLine}${targetPositionNote}源框只是“寻找主体”的范围，不是最终边界，也不是要缩放或贴回的矩形补丁。目标框只是主体调整后的视觉占位参考，不要求主体填满矩形，更不能改变主体自身宽高比、透视或形体。请像真实图片编辑一样先保留锁定的原始画布，再只把这个主体整体等比变大/变小/移动到目标视觉占位；主体缩放后露出或遮挡的极小背景区域需要自然重建。严禁把输入图片1整张图、输入图片2目标布局参考图、任意裁切图、源框矩形区域或目标框矩形区域当成图片块整体缩放、平移、复制、裁切或贴回。如果源框中只覆盖到主体的一部分，请根据局部裁切参考和可见结构推断该主体的完整可见轮廓，只调整主体，不调整矩形框内背景。`; - }) - .join("\n"); - const cropLines = cropReferences - .map( - (reference, index) => - `输入图片${index + 3}：区域 ${reference.regionIndex} 的原图干净局部裁切参考图，只用于确认要缩放的主体身份、轮廓和宽高比，不能当成输出底图，也不能把裁切图整体贴回。` - ) - .join("\n"); - - return [ - "这是一项局部主体比例 inpaint/edit 任务，不是整图缩放、整图裁切或重新构图任务。", - sourceSizeLine, - "输入图片1是原始干净图，也是最终输出必须继承的唯一画面基底。除目标主体尺寸变化与必要背景补全外，输入图片1中的文字、Logo、装饰、排版、背景、光线、构图边界和所有未选区域必须保持不变。", - "输入图片2是一张与输入图片1尺寸和画幅完全相同的完整画面编辑意图图，不是输出底图。它保留原图完整画幅作为稳定参考，并把源框旧主体占位强淡化为“这里的原尺寸主体需要被目标尺寸替换并自然修复背景”的指令；同时把同一目标主体粗略缩放后放在用户希望的最终位置和大小。请只读取目标主体的最终视觉占位、中心位置和原画幅关系；输入图片2里被淡化的旧主体不能表示最终还要保留原尺寸主体。不得学习、复刻或输出淡化痕迹、阴影占位、矩形裁切边缘、拼接痕迹或参考图粗糙画质。", - cropLines - ? `输入图片3及之后是框选区域的干净局部裁切参考图，作用是帮助你确认“要缩放的是哪个主体”。这些裁切图不是输出底图，不得被整体放大、贴回或覆盖到原图上。\n${cropLines}` - : "", - "输入图片2中的完整画面编辑意图是强约束：淡化的 OLD 占位表示旧尺寸主体要被移除/修复，清晰的 TARGET 占位表示最终主体大小和中心位置；整张图不应缩放、裁切或平移。最终输出必须回到输入图片1的真实照片环境中完成自然编辑，不得直接粘贴 input2 的矩形图块；主体不需要填满目标矩形，必须保持主体自身原始宽高比和透视形体。", - "如果系统或模型只能读取一张参考图，必须优先以输入图片1为输出基底，不得清理、改写或删除输入图片1中框选区域外的任何文字和图形。", - "系统已根据用户框选区域计算出下方每个区域的原图坐标；框选区域只是定位目标主体的搜索范围，不是最终画面的裁切块、贴片、蒙版边界，也不是放大后的空间上限，更不是要整体缩放的矩形图层。", - "最终结果必须保持原始干净图的画幅比例、构图边界、镜头透视、光线、色温、材质质感和清晰度；严禁整张图整体缩放、裁切、平移、重新排版或重新生成。", - "Highest priority rule: do NOT zoom, crop, resize, reframe, pan, or regenerate the full canvas. Treat image 1 as a locked background plate; edit only the selected subject and the smallest necessary background around its old and new footprint.", - "请不要生成任何彩色框、矩形边框、箭头、连线、角点、编号圆点、提示词气泡、标注文字、UI 控件或半透明色块。", - "请同时使用下方每个区域的源框/目标框像素坐标和输入图片2的目标布局参考来理解目标占位。坐标和参考图只用于指导主体位置和大小，不是要求把某个矩形图块缩放后贴回。", - "", - "任务：先在输入图片1中识别源框范围内最主要、可独立调整的主体物体，再只对这个“主体物体”做局部比例调整。调整后的主体大小和位置必须接近目标框表示的视觉占位；不接受看起来和原图几乎相同的输出。", - regionLines, - "", - "关键要求：", - "1. 只调整目标主体物体本身的整体视觉比例，不得保留原尺寸；主体轮廓宽度和高度必须同步等比变化，宽高比必须与原主体一致，不能为了贴合目标框而拉宽、压窄、拉高或压扁。", - "2. 严禁把整个矩形框选区域、输入图片1整张图或任意裁切图当成图片块整体缩放、平移、复制、裁切或贴回；最终画面不得出现矩形拼接边缘、色差硬边、补丁感、裁切线或局部贴片痕迹。", - "2.1 严禁把输入图片1整张图整体放大、缩小、裁切或重新构图；画面中的标题、Logo、页面文字、边框、窗框、背景家具或非目标主体必须作为像素锚点保持原位置和原比例，不能跟着目标主体变化。", - "3. 如果输入图片2中的目标框明显小于源框，目标主体应自然变小，主体缩小后露出的背景、桌面、墙面、阴影、纹理和透视必须被重新生成并无缝补全，选区内应出现明显新增留白或背景。", - "4. 如果输入图片2中的目标框明显大于源框，目标主体应自然变大；允许主体自然超出用户手画的原始源框，使用周围真实背景完成边缘、阴影和遮挡衔接。宽度和高度都要同步变大，不能出现“高度变高但宽度变窄”的非等比变形；主体边缘与遮挡关系要合理，不能产生突兀裁切。", - "5. 如果目标框中心与源框中心不同，允许主体同步移动到目标框中心附近；如果主体靠近画布边界导致目标空间不足，请用能完整放入画面的最大统一倍率整体调整，绝不能为了塞进画面而只压缩宽度或只拉伸高度。", - "6. 框选区域内非目标主体的背景、墙面、桌面、植物、文字和其它物体应保持原场景逻辑，不要跟着主体一起缩放。", - "6.1 比例调整后的主体边缘必须和周围环境自然融合，选区边界不能被肉眼看出；不得出现一整块矩形区域清晰度、颜色、光照、纹理或透视突然变化。", - "6.2 框选区域通常会包含主体周围的背景留白，背景留白不是目标主体。请锁定这些非主体背景的窗框、文字、地毯纹理、墙面、桌面边缘和透视线，只重建主体缩放后被遮挡或露出的极小必要区域；不要把框选矩形当作整块背景重新生成。", - "6.3 如果框选范围内没有清晰、完整、可独立缩放的主体物体，请保持该区域背景结构稳定，不要为了制造变化而重绘窗框、文字、墙面、地面或其它非主体元素。", - "7. 未受比例调整影响的其它画面区域必须保持原图不变，尤其是框选区域外的文字、Logo、标题、尺寸标注、装饰条和页面排版不得被清理、改写、模糊或重绘。", - "8. 最终输出必须是一张完整、干净、无缝的照片，不得出现任何选区框、说明文字、对比线、比例数字或 UI 元素。", - "9. 输出前自检：如果目标主体大小与原图相比没有明显变化，或者主体宽高比发生非等比变形，必须重新生成；禁止返回看起来和原图相同或被拉伸的结果。", - ].join("\n"); -} - -function buildScaleAdjustCoordinatePrompt( - regions: NormalizedLocalEditRegion[], - scalePercent: ScaleAdjustPercent | null | undefined, - sourceSize: { width: number; height: number }, - cropReferences: ScaleAdjustCropReference[] = [], - options: { - falSafe?: boolean; - markedPhotoGuide?: boolean; - visualLayoutGuide?: boolean; - } = {} -) { - const regionLines = regions - .map((region) => { - const cropIndex = cropReferences.findIndex( - (reference) => reference.regionIndex === region.index - ); - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const targetPixelRect = - region.targetPixelRect || - rectToPixels(targetRect, sourceSize.width, sourceSize.height); - const widthFactor = - region.rect.width > 0 ? targetRect.width / region.rect.width : 1; - const heightFactor = - region.rect.height > 0 ? targetRect.height / region.rect.height : 1; - const sourceRight = region.pixelRect.left + region.pixelRect.width; - const sourceBottom = region.pixelRect.top + region.pixelRect.height; - const targetRight = targetPixelRect.left + targetPixelRect.width; - const targetBottom = targetPixelRect.top + targetPixelRect.height; - const regionUserInstruction = region.prompt.trim(); - const userInstructionLine = regionUserInstruction - ? ` User instruction for this region: ${regionUserInstruction}.` - : ""; - const isShrink = widthFactor < 0.92 || heightFactor < 0.92; - const isEnlarge = widthFactor > 1.08 || heightFactor > 1.08; - const scaleDirection = - isShrink - ? "make the selected subject clearly smaller" - : isEnlarge - ? "make the selected subject clearly larger" - : "move/fit the selected subject to the target box"; - const footprintRule = isShrink - ? "After shrinking, the original OLD footprint outside the TARGET box must be cleared and rebuilt as natural background/floor/rug/wall; the selected subject must not still touch the OLD box edges." - : isEnlarge - ? "After enlarging, the same selected subject should visibly occupy more of the TARGET footprint, with natural shadows and occlusion; do not just move it or stretch one axis." - : "Keep the subject centered on the TARGET box without changing the full canvas."; - - const cropNote = - cropIndex >= 0 - ? options.falSafe - ? ` Input image ${cropIndex + 3} is an extra identity and silhouette reference.` - : ` Input image ${cropIndex + 3} is a clean crop reference for this same target subject identity and silhouette; use it only to recognize the subject, never paste it, and do not preserve its old size.` - : ""; - if (options.falSafe) { - const actionPhrase = - isShrink - ? "resize the selected subject proportionally smaller and place it inside the final footprint" - : isEnlarge - ? "resize the same selected subject proportionally larger toward the final footprint" - : "place the same selected subject on the final footprint"; - const blendPhrase = - isShrink - ? "Blend the newly visible floor, rug, wall, window area, shadow, and background around it naturally." - : isEnlarge - ? "Blend edges, contact shadows, and occlusion with the surrounding room naturally." - : "Keep the surrounding room stable."; - return `Region ${region.index}: current target-subject search box in source photo left=${region.pixelRect.left}, top=${region.pixelRect.top}, right=${sourceRight}, bottom=${sourceBottom}, width=${region.pixelRect.width}, height=${region.pixelRect.height}; requested final footprint left=${targetPixelRect.left}, top=${targetPixelRect.top}, right=${targetRight}, bottom=${targetBottom}, width=${targetPixelRect.width}, height=${targetPixelRect.height}; scale factor width=${widthFactor.toFixed(2)}, height=${heightFactor.toFixed(2)}. Edit direction: ${actionPhrase}. ${blendPhrase}${cropNote}`; - } - return `Region ${region.index}: OLD search box in input 1 left=${region.pixelRect.left}, top=${region.pixelRect.top}, right=${sourceRight}, bottom=${sourceBottom}, width=${region.pixelRect.width}, height=${region.pixelRect.height}; TARGET final visual footprint left=${targetPixelRect.left}, top=${targetPixelRect.top}, right=${targetRight}, bottom=${targetBottom}, width=${targetPixelRect.width}, height=${targetPixelRect.height}; target scale is ${widthFactor.toFixed(2)}x width and ${heightFactor.toFixed(2)}x height. Main action: ${scaleDirection}.${userInstructionLine} The final selected-subject silhouette must clearly leave the OLD footprint edges and align much closer to TARGET than OLD; returning an almost unchanged target subject footprint is not acceptable. ${footprintRule}${cropNote}`; - }) - .join("\n"); - const cropLines = cropReferences - .map( - (reference, index) => - `Input image ${index + 3}: clean crop for Region ${reference.regionIndex}; identity/shape reference only, not a patch or layer.` - ) - .join("\n"); - - if (options.falSafe) { - return [ - "Interior selected-subject scale photo edit.", - `Return one realistic photo with the same camera frame and pixel size as input image 1: ${sourceSize.width} x ${sourceSize.height} px.`, - "Input image 1 is the source photo. Input image 2 is a marked copy of the source photo for location guidance only and should not appear in the final photo.", - "In the guide, the cyan dashed box marks the existing selected target subject area and the blue solid box marks the requested final footprint.", - "Resize only the selected target subject indicated by each current box. The target may be a person/model/child/body part, furniture, product, or another object. Keep the camera, room layout, walls, windows, floor, rug, sofa, lamp, text, background, and all non-target subjects stable.", - "Preserve the target identity, person anatomy/face/clothing or product material/structure, perspective, contact shadows, and natural aspect ratio/proportions.", - "For a smaller final footprint, show the selected subject smaller and blend newly visible floor, rug, wall, window area, shadow, and background around it naturally. For a larger final footprint, scale the same selected subject proportionally with natural edges and contact shadows.", - "Return a clean photo without guide marks, boxes, arrows, overlays, patch edges, color mismatch, or full-frame zoom.", - cropLines ? `${cropLines}` : "", - "", - "Region instructions:", - regionLines, - ].join("\n"); - } - - const coordinateGuideLine = options.visualLayoutGuide - ? "输入图片2是一张完整画面的粗略目标布局草稿，不是输出底图。它把 OLD 源框旧主体占位洗淡，表示旧尺寸主体必须从原尺寸位置移除并自然修复背景；同时把目标主体粗略缩放/移动到 TARGET 最终占位附近，这个缩小后的位置和尺寸是最终结果必须匹配的几何目标。请把 input 2 当成最终构图/比例的强约束：如果 input 2 中目标主体比 input 1 小，最终输出也必须明显变小，不能保留 input 1 的原尺寸主体。不要复制 input 2 的淡化痕迹、粗糙拼接、局部透明边、矩形补丁、亮度变化或参考图画质；最终必须以 input 1 的真实照片为唯一画面基底重新完成无缝图片编辑。" - : options.markedPhotoGuide - ? "输入图片2是一张原图浅化后的完整画面位置参考图，不是输出底图。青色虚线框表示当前主体搜索框（对应下方 OLD），蓝色实线框表示目标最终占位（对应下方 TARGET）。请用输入图片2理解主体在真实室内画面中的位置关系，但最终图不能出现任何框线、色块、标注或参考图的浅化效果。" - : "Input image 2 is a zoomed neutral coordinate instruction map, not a photo and not output content. It enlarges the OLD and TARGET boxes so their relative scale is easy to read; use the numeric pixel coordinates below for the exact absolute position in input image 1. The OLD box/label locates the current selected subject; the TARGET box/label gives the required final footprint, center, and scale factor. If the map says OLD OUTSIDE -> BG, rebuild the old full-size subject area outside TARGET as natural background in the final photo. Read the boxes, labels, arrows, and scale text only as instructions; never output any map graphics."; - - return [ - "Local selected-subject scale edit. Use input image 1 as the clean original photo and the locked camera frame.", - "One-shot cost constraint: produce the requested edit in this generation without relying on later retries or alternate candidates; an unchanged result is not acceptable.", - "中文硬约束：只调整 OLD 框中的主要目标主体，目标可以是人物、模特、小孩、手臂身体、商品或家具。TARGET 框是最终占位；如果是缩小，OLD 框里超出 TARGET 的旧主体占位必须改为自然背景；如果是放大，同一目标主体必须明显变大。禁止整图缩放、整图裁切、整图重绘。", - "一次成功硬约束：不要保守返回原尺寸。目标主体的可见轮廓必须明显从 OLD 原始占位变到 TARGET 占位；不接受看起来和原图几乎相同的结果。", - `Return one seamless realistic photo with the same frame and pixel size as input image 1: ${sourceSize.width} x ${sourceSize.height} px.`, - coordinateGuideLine, - cropLines ? `${cropLines}` : "", - "", - "Task:", - "Find the main selected target subject inside each OLD box in input image 1. The target can be a person/model/child/body part, furniture, product, or another movable object. Resize and, if needed, move that subject only so its final visible footprint matches the TARGET box shown in input image 2 and the numeric TARGET coordinates below.", - "Do a real subject-scale edit, not a full image zoom. The camera, room layout, windows, wall decor, text, floor, rug, and all non-target subjects must remain stable.", - "If an OLD box contains more than one object/person, choose the dominant target subject inside that box, usually the largest or most central selected item/person. Keep adjacent lamps, plants, side tables, pillows, wall decor, windows, text, background objects, and non-target people at their original size unless they are clearly part of the same target subject.", - "Keep the target identity, person face/anatomy/clothing or product material/structure, perspective, contact shadows, and natural aspect ratio/proportions from input image 1. Do not create a second copy or extra parts. Do not preserve the original target size just because the scene looks realistic; the user explicitly wants a visible scale change.", - "For beds, sofas, tables, chairs, cabinets, benches, people, models, children, hands, and other large visible subjects, still perform the scale edit on that selected subject. Do not treat the selected subject as locked background.", - "If the target is smaller, the selected subject must visibly become smaller and be mostly contained by the TARGET footprint. Rebuild the OLD area outside the TARGET footprint as newly exposed floor, rug, wall, bedding, shadows, or background. If the target is larger, enlarge the same selected subject naturally without stretching it.", - "缩小时请优先让目标主体明显缩小，而不是为了保守而保持原样。不接受原尺寸保留、只轻微变形、或只改变整张图构图。", - "Do not keep the selected subject at its original footprint. The edited subject should fit the TARGET rectangle much better than the OLD rectangle.", - "", - "Region requirements:", - regionLines, - "", - "Success check: compare the final selected-subject footprint with input image 1 and the neutral coordinate map in input image 2. The selected subject size must clearly change from the original and align much closer to TARGET than OLD, while the rest of the photo stays stable.", - "Do not paste rectangular image areas. Do not output coordinate-map lines, labels, boxes, arrows, scale text, muted wash, overlays, seams, hard patch edges, haze, color mismatch, full-canvas zoom, crop, pan, or redesign.", - "Return only the clean edited photo.", - ].join("\n"); -} - -function buildScaleAdjustCoordinateSystemPrompt() { - return [ - "You are an image editing model for local selected-subject scale adjustment.", - "Input image 1 is the only final photo base. Input image 2 is a zoomed neutral coordinate instruction map, not a photo layer.", - "Follow both English and Chinese instructions. The Chinese lines are hard constraints for avoiding an unchanged selected target subject and full-canvas zoom.", - "Generate a clean realistic final photo by editing the selected target subject itself and naturally rebuilding only the tiny surrounding background that becomes visible.", - "The target subject can be a person/model/child/body part, furniture, product, or another movable object. Prefer a clear visible size change over a conservative unchanged result; an unchanged target subject is not acceptable.", - "When shrinking, rebuild the old full-size target-subject footprint outside the target footprint as natural background. Large selected subjects are editable and must not remain locked at original size.", - "Do not paste, composite, clone, crop, resize, or copy rectangular image patches from any input.", - "Never output guide artifacts from the edit map: no faded wash, placeholder shadow, overlay patch, rough crop rectangle, lines, labels, blocks, seams, rectangular borders, arrows, masks, UI marks, or annotations.", - "Keep the camera frame, lighting, wall, floor, rug, windows, background furniture, text, logos, and all non-target regions stable.", - ].join("\n"); -} - -function buildScaleAdjustTextOnlyPrompt( - regions: NormalizedLocalEditRegion[], - scalePercent: ScaleAdjustPercent | null | undefined, - cropReferences: ScaleAdjustCropReference[], sourceSize: { width: number; height: number } ) { - const cropLines = cropReferences - .map( - (reference, index) => - `Input image ${index + 2}: clean crop reference for Region ${reference.regionIndex}; use it only to identify the selected target subject and its natural aspect ratio/proportions, never paste the crop into the output.` - ) - .join("\n"); const regionLines = regions .map((region) => { - const cropIndex = cropReferences.findIndex( - (reference) => reference.regionIndex === region.index - ); const targetRect = getScaleAdjustTargetRect(region, scalePercent); const targetPixelRect = region.targetPixelRect || rectToPixels(targetRect, sourceSize.width, sourceSize.height); - const widthFactor = - region.rect.width > 0 ? targetRect.width / region.rect.width : 1; - const heightFactor = - region.rect.height > 0 ? targetRect.height / region.rect.height : 1; const sourceRight = region.pixelRect.left + region.pixelRect.width; const sourceBottom = region.pixelRect.top + region.pixelRect.height; const targetRight = targetPixelRect.left + targetPixelRect.width; const targetBottom = targetPixelRect.top + targetPixelRect.height; - const cropNote = - cropIndex >= 0 - ? ` Input image ${cropIndex + 2} is the clean crop reference for this target-subject identity and silhouette.` - : ""; - return `Region ${region.index}: hidden coordinate set A current area x0=${region.pixelRect.left}, y0=${region.pixelRect.top}, x1=${sourceRight}, y1=${sourceBottom}, width=${region.pixelRect.width}, height=${region.pixelRect.height}; hidden coordinate set B final size/location x0=${targetPixelRect.left}, y0=${targetPixelRect.top}, x1=${targetRight}, y1=${targetBottom}, width=${targetPixelRect.width}, height=${targetPixelRect.height}; final visual scale width=${widthFactor.toFixed(2)}, height=${heightFactor.toFixed(2)}. User request: ${region.prompt || "make the same selected target subject match set B"}.${cropNote}`; - }) - .join("\n"); - - return [ - "Local selected-subject scale edit. Input image 1 is the locked source photo and the only final canvas base.", - `Return one clean realistic photo with exactly the same camera frame and pixel size as input image 1: ${sourceSize.width} x ${sourceSize.height} px.`, - cropLines ? cropLines : "", - "The final photo must contain only natural scene content. Never add artificial graphic overlays, visible coordinate aids, labels, lines, UI elements, or text.", - "Only edit the main selected target subject inside each current pixel area. The target can be a person/model/child/body part, furniture, product, or another object. Keep all other non-target subjects, furniture, sofa, lamps, wall, windows, rug, floor, decor, lighting, and camera frame stable.", - "Use the numeric pixel sets below as hidden instructions only; they are not visual content and must not be drawn.", - "When set B is smaller than set A, make the same selected target subject visibly smaller and naturally rebuild the newly visible floor, rug, wall, shadow, or background around it. Do not leave the old full-size subject silhouette.", - "Keep the target identity, person anatomy/face/clothing or product material/structure, perspective, contact shadows, and natural aspect ratio/proportions. Do not stretch, squash, duplicate, or paste a rectangular crop.", - "中文硬约束：输入图1是锁定底图，只能调整 A 像素范围里的主要目标主体，目标可以是人物、模特、小孩、手臂身体、商品或家具，让它变成 B 像素范围对应的大小和位置。A/B 坐标只是隐藏指令，最终照片里绝对不能画出任何线条、框、文字、标注或 UI 元素。", - "", - "Region instructions:", - regionLines, - "", - "Success check before returning: the selected target subject must be clearly closer to set B than to set A, while the rest of the photo remains visually unchanged and clean.", - ].filter(Boolean).join("\n"); -} - -function toPatchRelativePixelBox( - box: { left: number; top: number; width: number; height: number }, - workspace: ScaleAdjustPatchReference["workspacePixelRect"] -) { - const left = clamp(box.left - workspace.left, 0, Math.max(0, workspace.width - 1)); - const top = clamp(box.top - workspace.top, 0, Math.max(0, workspace.height - 1)); - const right = clamp( - box.left + box.width - workspace.left, - left + 1, - workspace.width - ); - const bottom = clamp( - box.top + box.height - workspace.top, - top + 1, - workspace.height - ); - - return { - left, - top, - width: Math.max(1, right - left), - height: Math.max(1, bottom - top), - }; -} - -function buildScaleAdjustPatchPrompt( - regions: NormalizedLocalEditRegion[], - scalePercent: ScaleAdjustPercent | null | undefined, - patchReference: ScaleAdjustPatchReference, - sourceSize: { width: number; height: number }, - cropReferences: ScaleAdjustCropReference[] = [], - options: { includeSourceReference?: boolean; targetDraftPrimary?: boolean } = {} -) { - const targetDraftPrimary = options.targetDraftPrimary === true; - const includeSourceReference = options.includeSourceReference !== false; - const structureLockedDraft = patchReference.structureLockedDraft === true; - const sourceInputIndex = targetDraftPrimary && includeSourceReference ? 2 : 1; - const draftInputIndex = targetDraftPrimary ? 1 : 2; - const cropReferenceStartIndex = - targetDraftPrimary && !includeSourceReference ? 2 : 3; - const firstInputRole = targetDraftPrimary - ? structureLockedDraft - ? includeSourceReference - ? "Input image 1 is a rough subject-structure-locked target crop. The selected target subject was moved and scaled using source-photo pixels, so its final size, position, silhouette, material/skin/clothing, perspective, and visible structure are the hard target. If the subject is a person, keep the same person identity, face, body proportions, pose, hair, hands, and clothing while cleaning rough edges. If the subject is furniture/product, keep rods, panels, weave, wheels, material, and product structure. Repair rough background holes, edge halos, blur, occlusion, contact shadows, and surface artifacts without redesigning the subject. Input image 2 is only the clean background and non-target reference, never the old size reference." - : "Input image 1 is a rough subject-structure-locked target crop and the only visual reference. The selected target subject was moved and scaled using source-photo pixels, so its final size, position, silhouette, material/skin/clothing, perspective, and visible structure are the hard target. If the subject is a person, keep the same person identity, face, body proportions, pose, hair, hands, and clothing while cleaning rough edges. If the subject is furniture/product, keep rods, panels, weave, wheels, material, and product structure. Repair rough background holes, edge halos, blur, occlusion, contact shadows, and surface artifacts without redesigning the subject." - : includeSourceReference - ? "Input image 1 is the rough edited target-layout crop. It is the hard layout target: the selected subject size and center in the final output must match this draft, but the rough wash, ghosted old footprint, pale wipe, placeholder shadow, blur, and patch artifacts are only edit instructions and must be cleaned. Never reproduce the pale wipe or any rectangular haze as photo content." - : "Input image 1 is the rough edited target-layout crop and the only visual reference. It is the hard layout target: the selected subject size and center in the final output must match this draft. Clean away the rough wash, ghosted old footprint, pale wipe, placeholder shadow, blur, and patch artifacts. Never reproduce the pale wipe or any rectangular haze as photo content." - : "Input image 1 is the exact crop from the original photo and is the locked crop base."; - const secondInputRole = targetDraftPrimary - ? includeSourceReference - ? "Input image 2 is the original clean crop. Use it for stable crop framing, background texture, lighting, perspective, target identity, and non-target person/object preservation. Do not restore the selected subject to its old size from this image." - : "" - : "Input image 2 is a rough target-layout draft of the same crop. It is the hard layout target for the selected subject size and position. Use it to understand the final footprint and newly visible background area, then remove any rough seams, washed areas, placeholder shadows, blur, or patch artifacts from that draft."; - const cropLines = cropReferences - .map( - (reference, index) => - `Input image ${index + cropReferenceStartIndex}: clean identity crop for Region ${reference.regionIndex}; use it only to recognize the target subject shape, identity, anatomy/materials, and natural aspect ratio, not its old size.` - ) - .join("\n"); - const regionLines = regions - .map((region) => { - const cropIndex = cropReferences.findIndex( - (reference) => reference.regionIndex === region.index - ); - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const targetPixelRect = - region.targetPixelRect || - rectToPixels(targetRect, sourceSize.width, sourceSize.height); - const sourceInPatch = toPatchRelativePixelBox( - region.pixelRect, - patchReference.workspacePixelRect - ); - const targetInPatch = toPatchRelativePixelBox( - targetPixelRect, - patchReference.workspacePixelRect - ); const widthFactor = region.rect.width > 0 ? targetRect.width / region.rect.width : 1; const heightFactor = region.rect.height > 0 ? targetRect.height / region.rect.height : 1; + const moved = + Math.hypot( + targetRect.x + targetRect.width / 2 - (region.rect.x + region.rect.width / 2), + targetRect.y + targetRect.height / 2 - (region.rect.y + region.rect.height / 2) + ) > 0.01; const action = widthFactor < 0.92 || heightFactor < 0.92 - ? "make the same selected subject visibly smaller" + ? "把同一个主体等比缩小" : widthFactor > 1.08 || heightFactor > 1.08 - ? "make the same selected subject visibly larger" - : "move the same selected subject to the final footprint"; - const cropNote = - cropIndex >= 0 - ? ` Input image ${cropIndex + cropReferenceStartIndex} is the clean identity reference for this exact target subject.` - : ""; - return `Region ${region.index}: current search area inside input image ${sourceInputIndex} crop is x=${sourceInPatch.left}, y=${sourceInPatch.top}, w=${sourceInPatch.width}, h=${sourceInPatch.height}; final visual footprint inside input image ${draftInputIndex} / output crop is x=${targetInPatch.left}, y=${targetInPatch.top}, w=${targetInPatch.width}, h=${targetInPatch.height}; requested scale width=${widthFactor.toFixed(2)}, height=${heightFactor.toFixed(2)}. User request: ${region.prompt || action}. Main action: ${action}. Treat the main selected subject inside the search area as the target; it may be a person, model, child, body part, furniture, product, or other object. The final subject must be much closer to the final footprint than to the current search area.${cropNote}`; + ? "把同一个主体等比放大" + : "把同一个主体调整到目标位置和大小"; + return [ + `区域 ${region.index}:`, + `源框 A: left=${region.pixelRect.left}, top=${region.pixelRect.top}, right=${sourceRight}, bottom=${sourceBottom}, width=${region.pixelRect.width}, height=${region.pixelRect.height}.`, + `目标框 B: left=${targetPixelRect.left}, top=${targetPixelRect.top}, right=${targetRight}, bottom=${targetBottom}, width=${targetPixelRect.width}, height=${targetPixelRect.height}.`, + `目标比例约为宽 ${widthFactor.toFixed(2)}x、高 ${heightFactor.toFixed(2)}x。`, + `请识别 A 框内的主要商品/物体/人物主体，只${action}，${moved ? "并移动到 B 框中心附近" : "保持在当前位置附近"}。`, + ].join(" "); }) .join("\n"); - const chineseHardConstraint = (() => { - if (!targetDraftPrimary) { - return "中文硬约束：只输出输入图1同一裁切范围的干净局部照片；输入图2是目标大小/位置草稿，必须按它的目标尺寸和目标位置输出，不是要保留旧尺寸。目标家具必须相对原图产生明显比例变化，非目标背景和裁切边缘要稳定自然。"; - } - if (structureLockedDraft) { - return includeSourceReference - ? "中文硬约束：输入图1已经用原图像素把目标主体放到目标尺寸和目标位置，目标主体可以是人物、模特、小孩、手臂身体、商品或家具。人物目标必须按输入图1保留同一身份、脸部、身体比例、姿态、发型、手部和服装；商品/家具目标必须保留产品结构、杆件、面板、编织纹理、轮子、材质和透视。输入图2只能作为干净背景、光影和非目标元素参考，绝对不能把目标主体恢复成输入图2里的旧尺寸。不能重新设计或替换目标主体，只清理草稿边缘、黑斑、破洞、脏点、灰块、皮肤/服装/产品表面伪影。" - : "中文硬约束：输入图1已经用原图像素把目标主体放到目标尺寸和目标位置，目标主体可以是人物、模特、小孩、手臂身体、商品或家具。人物目标必须按输入图1保留同一身份、脸部、身体比例、姿态、发型、手部和服装；商品/家具目标必须保留产品结构、杆件、面板、编织纹理、轮子、材质和透视。不能重新设计或替换目标主体，只清理草稿边缘、黑斑、破洞、脏点、灰块、皮肤/服装/产品表面伪影。"; - } - return includeSourceReference - ? "中文硬约束：输入图1是目标大小/位置草稿，必须按它的目标尺寸和目标位置输出；输入图2只是原始质感、非目标人物和背景参考，绝对不要把目标主体恢复成输入图2里的旧尺寸。草稿里的淡色擦除层/雾块只是提示旧位置需要补背景，不是画面内容，最终不能出现矩形雾块、白块、残影或拼接感。" - : "中文硬约束：输入图1是唯一视觉参考和目标大小/位置草稿，必须按它的目标尺寸和目标位置输出。不要找回旧尺寸目标主体。草稿里的淡色擦除层/雾块只是提示旧位置需要补背景，不是画面内容，最终不能出现矩形雾块、白块、残影或拼接感。"; - })(); return [ - "Local selected-subject scale edit inside a cropped workspace.", - `Return exactly one clean realistic crop with the same framing and aspect as input image 1. The crop represents full-photo pixels left=${patchReference.workspacePixelRect.left}, top=${patchReference.workspacePixelRect.top}, width=${patchReference.workspacePixelRect.width}, height=${patchReference.workspacePixelRect.height}.`, - firstInputRole, - secondInputRole, - cropLines, - "Only edit the selected target subject inside the search area. The target subject can be a person/model/child/body part, furniture, product, or another movable object. Keep the crop borders, wall, window frame, floor, rug, sofa, lamps, decor, lighting, camera perspective, and all non-target subjects stable.", - "If the selected target is a person, scale the whole visible person/body coherently to the final footprint while preserving identity, face, pose, anatomy, hand shape, hair, clothing, and natural human proportions. If a person appears outside the selected target, preserve that non-target person without resizing or redrawing them.", - "When the final footprint is smaller, the selected subject must clearly shrink; erase the old larger subject footprint outside the final footprint and replace it with natural background/floor/rug/wall/shadow that matches the original crop.", - "If input image 1 still shows any faint, transparent, blurred, or ghosted old-size copy of the selected subject outside the final footprint, treat it as an erase marker only. Do not keep it. The only selected target subject that may remain is the smaller clean subject inside the final footprint.", - structureLockedDraft - ? "When the final footprint is larger, the target draft already places the same selected subject at the required larger size. Do not redesign the subject geometry/anatomy; keep the locked person identity/anatomy/clothing or product structure/material/perspective while cleaning background gaps, boundary blending, rough surface artifacts, natural occlusion, and contact shadows." - : "When the final footprint is larger, enlarge the selected subject itself to match the target draft while keeping nearby non-target people and hands natural. If the target is a person, preserve anatomy and identity instead of inventing a new person.", - "Keep the same target identity, perspective, visible details, contact shadows, and natural aspect ratio/proportions. Do not stretch, squash, duplicate, or replace it with a different subject.", - "Hard failure condition: if the selected target subject remains near its original old size, the answer is wrong. The output must visibly change the selected subject footprint to match the target draft footprint.", - structureLockedDraft - ? "Remove every rough draft artifact from input image 1: gray fill patches, beige wash, flat painted areas, jagged mask holes, black speckled stains, dark blotches, rectangular crop edges, and ghost fragments. Human skin, face, hands, hair, and clothing must look continuous and natural when the target is a person. Woven panels, wheels, rods, hooks, and product surfaces must remain continuous and clean when the target is furniture/product." - : "", - "Do not output boxes, lines, labels, coordinate marks, UI marks, text, arrows, overlays, rough layout artifacts, rectangular patch edges, or a full-room image.", - chineseHardConstraint, + "对输入图片做一次简单的图生图局部比例编辑。", + `最终输出保持原图完整画幅和像素尺寸：${sourceSize.width} x ${sourceSize.height} px。`, + "只编辑用户框选的主体本身，保持主体外观、材质、文字、图案、透视和宽高比，不要拉伸或压扁。", + "不要缩放整张图，不要裁剪整张图，不要重绘未选区域。", + "如果主体变小，原位置露出的区域按原图背景自然补齐；白底或纯色背景必须保持干净一致。", + "如果主体变大，让同一个主体自然占据更大的目标范围，并保持边缘、阴影和背景衔接自然。", + "A/B 坐标只是隐藏编辑指令，最终图片里绝不能出现框线、箭头、文字、标注、UI 或半透明色块。", "", - "Region instructions:", + "编辑要求:", regionLines, - "", - "Success check before returning: the selected target subject footprint must be clearly closer to the final footprint than the current search area, and the crop must look like one natural photograph.", - ].filter(Boolean).join("\n"); + ].join("\n"); } export async function POST(request: NextRequest) { @@ -3924,22 +660,9 @@ export async function POST(request: NextRequest) { ); } - const shouldUseScaleAdjustDetailModel = - operation === SCALE_ADJUST_OPERATION && - (hasScaleAdjustTargetRects || scalePercent !== null); - const [pointsConfig, siteSettingsForModel, scaleAdjustModelCandidates] = - await Promise.all([ + const [pointsConfig, siteSettingsForModel] = await Promise.all([ getPointsSystemConfig(), getSiteSettings().catch(() => null), - shouldUseScaleAdjustDetailModel - ? getEnabledAiModelsWithProvider(false, true).catch((error) => { - console.warn( - "[Canvas Local Edit] Failed to load scale adjust model candidates:", - error - ); - return [] as AiModelWithProviderForSelection[]; - }) - : Promise.resolve([] as AiModelWithProviderForSelection[]), ]); const pointsCharge = await prepareFeaturePointsCharge({ userId: session.user.id, @@ -3954,21 +677,101 @@ export async function POST(request: NextRequest) { return NextResponse.json(pointsCharge.payload, { status: pointsCharge.status }); } - const isScaleAdjustOperation = + if ( operation === SCALE_ADJUST_OPERATION && - (hasScaleAdjustTargetRects || scalePercent !== null); - const hasScaleAdjustEnlarge = isScaleAdjustOperation - ? hasScaleAdjustEnlargeTarget(regions, scalePercent) - : false; - const hasScaleAdjustShrink = isScaleAdjustOperation - ? hasScaleAdjustShrinkTarget(regions, scalePercent) - : false; - const preferScaleAdjustMaskCapableModel = - hasScaleAdjustShrink && !hasScaleAdjustEnlarge; - const preferScaleAdjustFalNanoBananaEdit = - hasScaleAdjustShrink && !hasScaleAdjustEnlarge; - const preferScaleAdjustProviderTypes = - hasScaleAdjustShrink && !hasScaleAdjustEnlarge ? ["grsai-gpt-image"] : []; + (hasScaleAdjustTargetRects || scalePercent !== null) + ) { + const displayPrompt = buildLocalEditDisplayPrompt({ operation, scalePercent }); + const resolvedAspectRatio = resolveLocalEditAspectRatio( + body.aspectRatio, + sourceWidth, + sourceHeight + ); + const requestedModel = + typeof body.model === "string" && body.model.trim() + ? body.model.trim() + : undefined; + const configuredDetailModel = + typeof siteSettingsForModel?.canvasDetailDefaultModelId === "string" + ? siteSettingsForModel.canvasDetailDefaultModelId.trim() + : ""; + const resolvedModel = requestedModel || configuredDetailModel; + const requestedImageSize = + typeof body.imageSize === "string" && body.imageSize.trim() + ? body.imageSize.trim() + : "2K"; + const manualRegionReferenceImageUrls = regions.flatMap((region) => + region.referenceImages.map((reference) => reference.url) + ); + const prompt = buildSimpleScaleAdjustPrompt(regions, scalePercent, { + width: sourceWidth, + height: sourceHeight, + }); + const imageRequest: CanvasGenerateImageRequest = { + aspectRatio: resolvedAspectRatio, + billingFeature: "repaint", + debugLabel: "canvas-scale-adjust-simple", + debugMeta: { + displayPrompt, + operation, + requestedModel: requestedModel || null, + resolvedAspectRatio, + resolvedModel: resolvedModel || null, + scaleAdjustPromptStrategy: "simple-image-edit-v1", + scaleAdjustTargetRegionCount: regions.filter((region) => region.targetRect) + .length, + scalePercent, + sourceHeight, + sourceNodeId: body.sourceNodeId || null, + sourceWidth, + }, + displayPrompt, + imageCount: 1, + imageSize: requestedImageSize, + model: resolvedModel || undefined, + negativePrompt: + "不要画框线、箭头、坐标、标注、文字或 UI。不要缩放整张图、裁剪整张图、重绘未选区域。不要拉伸、压扁、复制或替换主体。", + prompt, + promptExtend: false, + referenceImages: [sourceImageUrl, ...manualRegionReferenceImageUrls], + ownerKey: session.user.id, + }; + + const task = await createCanvasImageTask({ + request: imageRequest as unknown as Record, + userId: session.user.id, + }); + await enqueueCanvasImageTask(task.id); + const currentTask = await getCanvasImageTaskForUser({ + taskId: task.id, + userId: session.user.id, + }); + + if (pointsCharge.chargeable) { + try { + await pointsCharge.consume({ + note: `比例调整消耗积分（${task.id}）`, + }); + } catch (pointsError) { + console.error("[Points] Failed to consume scale adjust points:", pointsError); + } + } + + return NextResponse.json({ + success: true, + annotatedReferenceUrl: sourceImageUrl, + aspectRatio: resolvedAspectRatio, + displayPrompt, + prompt: displayPrompt, + regions, + sourceHeight, + sourceWidth, + taskId: task.id, + status: currentTask?.status || task.status, + queueStatus: currentTask?.queueStatus, + }); + } + const displayPrompt = buildLocalEditDisplayPrompt({ operation, scalePercent }); const resolvedAspectRatio = resolveLocalEditAspectRatio( body.aspectRatio, @@ -3979,497 +782,66 @@ export async function POST(request: NextRequest) { typeof body.model === "string" && body.model.trim() ? body.model.trim() : undefined; - const configuredDetailModel = - typeof siteSettingsForModel?.canvasDetailDefaultModelId === "string" - ? siteSettingsForModel.canvasDetailDefaultModelId.trim() - : ""; - const detailFixedModel = isScaleAdjustOperation - ? resolveScaleAdjustPreferredModelId({ - configuredModelId: configuredDetailModel, - honorRequestedModelBeforeGpt: hasScaleAdjustEnlarge, - preferFalNanoBananaEdit: preferScaleAdjustFalNanoBananaEdit, - preferMaskCapableGpt: preferScaleAdjustMaskCapableModel, - preferProviderTypes: preferScaleAdjustProviderTypes, - requestedModelId: requestedModel || "", - models: scaleAdjustModelCandidates, - }) - : ""; - if ( - isScaleAdjustOperation && - detailFixedModel && - detailFixedModel !== configuredDetailModel - ) { - console.info("[Canvas Local Edit] Override scale adjust model to detail line:", { - configuredDetailModel: configuredDetailModel || null, - requestedModel: requestedModel || null, - resolvedModel: detailFixedModel, - }); - } - const resolvedModel = detailFixedModel || requestedModel; - const resolvedScaleAdjustModelEntry = isScaleAdjustOperation - ? findModelSelectionEntry(scaleAdjustModelCandidates, resolvedModel || "") - : null; - const useScaleAdjustCoordinateGuide = - isScaleAdjustOperation && - isScaleAdjustCoordinateGuideModel(resolvedScaleAdjustModelEntry); - const scaleAdjustModelFamily = getScaleAdjustModelFamily( - resolvedScaleAdjustModelEntry - ); - const useProviderTextOnlyCoordinatePrompt = - isScaleAdjustOperation && - resolvedScaleAdjustModelEntry?.provider?.type === "alibaba-image"; - const useScaleAdjustCoordinatePrompt = useScaleAdjustCoordinateGuide; - const useScaleAdjustFullFrameFalEdit = - isScaleAdjustOperation && - isScaleAdjustFalNanoBananaEditModel(resolvedScaleAdjustModelEntry); - const useFalScaleAdjustCoordinatePrompt = - useScaleAdjustCoordinatePrompt && - resolvedScaleAdjustModelEntry?.provider?.type === "fal-image" && - getScaleAdjustModelFamily(resolvedScaleAdjustModelEntry) === "nano-banana-2"; - const useScaleAdjustVisualLayoutGuide = - useScaleAdjustCoordinatePrompt && - !useFalScaleAdjustCoordinatePrompt && - !useProviderTextOnlyCoordinatePrompt; - const useScaleAdjustMarkedPhotoGuide = - useScaleAdjustCoordinatePrompt && !useScaleAdjustVisualLayoutGuide; - const useScaleAdjustStructureLockedPatchWorkflow = - isScaleAdjustOperation && - !useScaleAdjustFullFrameFalEdit && - hasScaleAdjustStructureTarget(regions, scalePercent) && - (!preferScaleAdjustMaskCapableModel || - !isScaleAdjustMaskCapableModel(resolvedScaleAdjustModelEntry)); - const useScaleAdjustEditMask = - isScaleAdjustOperation && - !useScaleAdjustStructureLockedPatchWorkflow && - isScaleAdjustMaskCapableModel(resolvedScaleAdjustModelEntry); - const useScaleAdjustPatchWorkflow = - isScaleAdjustOperation && - !useScaleAdjustFullFrameFalEdit && - (useScaleAdjustStructureLockedPatchWorkflow || !useScaleAdjustEditMask); - const sendScaleAdjustAnnotatedReferenceToModel = - isScaleAdjustOperation && - !useScaleAdjustPatchWorkflow && - !useProviderTextOnlyCoordinatePrompt; - const annotatedBuffer = isScaleAdjustOperation - ? useScaleAdjustCoordinatePrompt - ? useScaleAdjustVisualLayoutGuide - ? await buildScaleAdjustVisualLayoutGuideBuffer({ - regions, - scalePercent, - sourceBuffer, - sourceHeight, - sourceWidth, - }) - : useScaleAdjustMarkedPhotoGuide - ? await buildScaleAdjustMarkedPhotoGuideBuffer({ - height: sourceHeight, - regions, - scalePercent, - sourceBuffer, - width: sourceWidth, - }) - : await sharp( - buildScaleAdjustZoomedCoordinateSvg({ - height: sourceHeight, - regions, - scalePercent, - width: sourceWidth, - }), - { failOn: "none" } - ) - .png({ compressionLevel: 6 }) - .toBuffer() - : await buildScaleAdjustVisualLayoutGuideBuffer({ + const annotatedBuffer = await sharp(sourceBuffer, { failOn: "none" }) + .composite([ + { + input: buildAnnotationSvg({ + height: sourceHeight, + hidePromptText: false, regions, - scalePercent, - sourceBuffer, - sourceHeight, - sourceWidth, - }) - : await sharp(sourceBuffer, { failOn: "none" }) - .composite([ - { - input: buildAnnotationSvg({ - height: sourceHeight, - hidePromptText: false, - regions, - width: sourceWidth, - }), - left: 0, - top: 0, - }, - ]) - .png({ compressionLevel: 6 }) - .toBuffer(); - const annotatedFilename = `${isScaleAdjustOperation ? "canvas-scale-adjust-layout-guide" : "canvas-local-edit"}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.png`; + width: sourceWidth, + }), + left: 0, + top: 0, + }, + ]) + .png({ compressionLevel: 6 }) + .toBuffer(); + const annotatedFilename = `canvas-local-edit-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.png`; const annotatedUpload = await uploadFile( annotatedFilename, toArrayBuffer(annotatedBuffer) ); - const scaleAdjustEditMaskUpload = useScaleAdjustEditMask - ? await uploadFile( - `canvas-scale-adjust-edit-mask-${Date.now()}-${Math.random() - .toString(36) - .slice(2, 8)}.png`, - toArrayBuffer( - await buildScaleAdjustEditMaskBuffer({ - regions, - scalePercent, - sourceHeight, - sourceWidth, - }) - ) - ) - : null; - const scaleAdjustPatchReference = useScaleAdjustPatchWorkflow - ? await createScaleAdjustPatchReference({ - regions, - scalePercent, - sourceBuffer, - sourceHeight, - sourceImageUrl, - sourceWidth, - userId: session.user.id, - }) - : null; - const shouldCreateScaleAdjustCropReferences = - isScaleAdjustOperation && - (!scaleAdjustPatchReference && - (useScaleAdjustVisualLayoutGuide || - !useScaleAdjustCoordinatePrompt || - useProviderTextOnlyCoordinatePrompt || - (useScaleAdjustMarkedPhotoGuide && !useFalScaleAdjustCoordinatePrompt))); - const scaleAdjustCropReferences: ScaleAdjustCropReference[] = - shouldCreateScaleAdjustCropReferences - ? await createScaleAdjustCropReferences({ - regions, - sourceBuffer, - sourceHeight, - sourceWidth, - }) - : []; - const useScaleAdjustPatchTargetDraftPrimary = Boolean(scaleAdjustPatchReference); - const includeScaleAdjustPatchSourceReference = - Boolean(scaleAdjustPatchReference); - const prompt = isScaleAdjustOperation - ? scaleAdjustPatchReference - ? buildScaleAdjustPatchPrompt( - regions, - scalePercent, - scaleAdjustPatchReference, - { width: sourceWidth, height: sourceHeight }, - scaleAdjustCropReferences, - { - includeSourceReference: includeScaleAdjustPatchSourceReference, - targetDraftPrimary: useScaleAdjustPatchTargetDraftPrimary, - } - ) - : useProviderTextOnlyCoordinatePrompt - ? buildScaleAdjustTextOnlyPrompt( - regions, - scalePercent, - scaleAdjustCropReferences, - { width: sourceWidth, height: sourceHeight } - ) - : useScaleAdjustCoordinatePrompt - ? buildScaleAdjustCoordinatePrompt( - regions, - scalePercent, - { - width: sourceWidth, - height: sourceHeight, - }, - scaleAdjustCropReferences, - { - falSafe: useFalScaleAdjustCoordinatePrompt, - markedPhotoGuide: useScaleAdjustMarkedPhotoGuide, - visualLayoutGuide: useScaleAdjustVisualLayoutGuide, - } - ) - : buildScaleAdjustPrompt( - regions, - scalePercent, - scaleAdjustCropReferences, - { width: sourceWidth, height: sourceHeight } - ) - : buildLocalEditPrompt(regions); - const scaleAdjustCoordinateNegativePrompt = useFalScaleAdjustCoordinatePrompt - ? "guide marks, labels, boxes, arrows, overlays, patch edges, color mismatch, full-frame zoom, unchanged target subject, duplicate subject, duplicate furniture, duplicate person, stretched furniture, distorted furniture, deformed person, changed face, extra limbs" - : "visible annotation, guide overlay, translucent overlay, target placement overlay, faded wash, placeholder shadow, text label, arrows, boxes, rectangle border, UI marks, pasted patch, rough layout patch, hard crop edge, seam, hard rectangular edge, color mismatch, different-resolution area, whole image zoom, whole image crop, whole image pan, whole image redesign, unchanged target subject, old-size target subject, unchanged target person, old-size target person, unchanged target furniture, old-size target furniture, stretched subject, squashed subject, distorted subject, stretched furniture, squashed furniture, duplicate subject, duplicate person, duplicate furniture, second copy, changed face, deformed person, broken anatomy, extra limbs, extra fingers, extra legs, extra rails, side panels, shelves, cabinets, lattice panels"; - const scaleAdjustTextOnlyNegativePrompt = - "artificial graphic overlay, visible coordinate aid, visible line, visible label, UI element, annotation text, full image redesign, full-frame zoom, crop, pan, pasted crop, hard patch edge, color mismatch, changed sofa, changed lamp, changed wall, changed rug, duplicate subject, duplicate person, duplicate furniture, changed face, deformed person, extra limbs, stretched subject, stretched furniture, distorted furniture"; - const scaleAdjustPatchNegativePrompt = - "full room image, changed crop border, artificial graphic overlay, visible coordinate aid, visible line, visible label, UI element, annotation text, guide mark, pasted crop, hard patch edge, rectangular haze, white haze block, pale wipe block, color mismatch, gray draft patch, beige wash patch, flat painted area, jagged mask hole, black speckled stain, dark blotch, duplicate subject, duplicate person, duplicate furniture, unchanged target subject, unchanged target person, unchanged target furniture, old-size target subject, changed target identity, changed face, deformed person, broken anatomy, extra limbs, extra fingers, changed locked product, redrawn product interior, different furniture, broken woven panel, dirty woven panel, wheel inside woven panel, stretched subject, stretched furniture, distorted furniture"; + const prompt = buildLocalEditPrompt(regions); const manualRegionReferenceImageUrls = regions.flatMap((region) => region.referenceImages.map((reference) => reference.url) ); - const scaleAdjustCropReferenceImageUrls = scaleAdjustCropReferences.map( - (reference) => reference.url - ); - const referenceImages = isScaleAdjustOperation - ? scaleAdjustPatchReference - ? useScaleAdjustPatchTargetDraftPrimary - ? [ - scaleAdjustPatchReference.targetDraftUrl, - ...(includeScaleAdjustPatchSourceReference - ? [scaleAdjustPatchReference.sourceCropUrl] - : []), - ...scaleAdjustCropReferenceImageUrls, - ...manualRegionReferenceImageUrls, - ] - : [ - scaleAdjustPatchReference.sourceCropUrl, - scaleAdjustPatchReference.targetDraftUrl, - ...scaleAdjustCropReferenceImageUrls, - ...manualRegionReferenceImageUrls, - ] - : [ - sourceImageUrl, - ...(sendScaleAdjustAnnotatedReferenceToModel - ? [annotatedUpload.url] - : []), - ...scaleAdjustCropReferenceImageUrls, - ...manualRegionReferenceImageUrls, - ] - : [sourceImageUrl, ...manualRegionReferenceImageUrls]; - const scaleAdjustCompositeRegions = isScaleAdjustOperation - ? buildScaleAdjustCompositeRegions(regions, scalePercent) - : []; - const scaleAdjustPatchCompositeRegions = isScaleAdjustOperation - ? buildScaleAdjustPatchCompositeRegions(regions, scalePercent) - : []; - const scaleAdjustProtectedChangeRegions = - isScaleAdjustOperation && scaleAdjustPatchReference - ? buildScaleAdjustProtectedChangeRegions(regions, scalePercent) - : []; - const scaleAdjustProtectedRegions = - isScaleAdjustOperation && scaleAdjustPatchReference - ? scaleAdjustPatchReference.structureLockedDraft - ? [scaleAdjustPatchReference.workspaceRect] - : scaleAdjustProtectedChangeRegions - : scaleAdjustCompositeRegions; - const scaleAdjustVerificationConfig = isScaleAdjustOperation - ? { - allowOutsideChangedWhenChangedInside: false, - allowDeterministicFallback: false, - protectedRegions: scaleAdjustProtectedRegions, - rejectFailedCandidates: true, - sourceImageUrl, - regions: regions.map((region) => region.rect), - regionPrompts: regions.map((region) => region.prompt), - ...(typeof scalePercent === "number" - ? { scalePercent: scalePercent as ScaleAdjustPercent } - : {}), - ...(hasScaleAdjustTargetRects - ? { - targetRegions: regions - .map((region) => region.targetRect) - .filter( - ( - rect - ): rect is NonNullable => - Boolean(rect) - ), - } - : {}), - } - : undefined; + const referenceImages = [sourceImageUrl, ...manualRegionReferenceImageUrls]; const requestedImageSize = typeof body.imageSize === "string" && body.imageSize.trim() ? body.imageSize.trim() : "2K"; - const resolvedRequestImageSize = - scaleAdjustPatchReference?.structureLockedDraft === true - ? requestedImageSize - : scaleAdjustPatchReference - ? "1K" - : requestedImageSize; const imageRequest: CanvasGenerateImageRequest = { - allowFailover: isScaleAdjustOperation ? false : undefined, - aspectRatio: scaleAdjustPatchReference - ? inferScaleAdjustPatchAspectRatio( - scaleAdjustPatchReference.workspacePixelRect.width, - scaleAdjustPatchReference.workspacePixelRect.height - ) - : resolvedAspectRatio, + aspectRatio: resolvedAspectRatio, billingFeature: "repaint", - debugLabel: - operation === SCALE_ADJUST_OPERATION ? "canvas-scale-adjust" : "canvas-local-edit", + debugLabel: "canvas-local-edit", debugMeta: { annotatedReferenceUrl: annotatedUpload.url, - annotatedReferenceSentToModel: sendScaleAdjustAnnotatedReferenceToModel, - annotatedReferenceType: isScaleAdjustOperation - ? useProviderTextOnlyCoordinatePrompt - ? "scale-adjust-text-coordinate-only-debug-map" - : useScaleAdjustCoordinatePrompt - ? useScaleAdjustVisualLayoutGuide - ? "scale-adjust-visual-layout-guide" - : useScaleAdjustMarkedPhotoGuide - ? "scale-adjust-marked-photo-guide" - : "scale-adjust-neutral-coordinate-map" - : "scale-adjust-full-frame-semantic-edit-map" - : "source-overlay", - scaleAdjustEditMaskApplied: Boolean(scaleAdjustEditMaskUpload), - scaleAdjustEditMaskUrl: scaleAdjustEditMaskUpload?.url || null, - scaleAdjustEditMaskProviderType: - useScaleAdjustEditMask - ? resolvedScaleAdjustModelEntry?.provider?.type || null - : null, - cleanSourceReferenceIndex: - scaleAdjustPatchReference && - useScaleAdjustPatchTargetDraftPrimary && - !includeScaleAdjustPatchSourceReference - ? null - : scaleAdjustPatchReference && useScaleAdjustPatchTargetDraftPrimary - ? 2 - : 1, - configuredDetailModelId: configuredDetailModel || null, - detailFixedModelApplied: Boolean(detailFixedModel), - detailFixedModelId: detailFixedModel || null, + annotatedReferenceType: "source-overlay", displayPrompt, operation, requestedModel: requestedModel || null, resolvedAspectRatio, - scaleAdjustPatchWorkflowApplied: Boolean(scaleAdjustPatchReference), - scaleAdjustPatchStructureLockedDraftApplied: - scaleAdjustPatchReference?.structureLockedDraft === true, - scaleAdjustFullFrameFalEditApplied: useScaleAdjustFullFrameFalEdit, - scaleAdjustPatchReferenceOrder: - scaleAdjustPatchReference && useScaleAdjustPatchTargetDraftPrimary - ? includeScaleAdjustPatchSourceReference - ? "target-draft-source-identity" - : "target-draft-identity" - : scaleAdjustPatchReference - ? "source-target-draft-identity" - : null, - scaleAdjustPatchSourceCropUrl: - scaleAdjustPatchReference?.sourceCropUrl || null, - scaleAdjustPatchTargetDraftUrl: - scaleAdjustPatchReference?.targetDraftUrl || null, - scaleAdjustPatchWorkspacePixelRect: - scaleAdjustPatchReference?.workspacePixelRect || null, - scaleAdjustPatchWorkspaceRect: - scaleAdjustPatchReference?.workspaceRect || null, - resolvedModel: resolvedModel || null, - resolvedScaleAdjustProviderType: - resolvedScaleAdjustModelEntry?.provider?.type || null, - scaleAdjustModelFamily: scaleAdjustModelFamily || null, - scaleAdjustCoordinateGuideApplied: useScaleAdjustCoordinateGuide, - scaleAdjustCoordinatePromptApplied: - useScaleAdjustCoordinatePrompt || useProviderTextOnlyCoordinatePrompt, - scaleAdjustTextOnlyCoordinatePromptApplied: - useProviderTextOnlyCoordinatePrompt, - scaleAdjustNeutralCoordinateMapApplied: - isScaleAdjustOperation && - useScaleAdjustCoordinatePrompt && - !useScaleAdjustMarkedPhotoGuide && - !useScaleAdjustVisualLayoutGuide, - scaleAdjustMarkedPhotoGuideApplied: useScaleAdjustMarkedPhotoGuide, - scaleAdjustFalMarkedPhotoGuideApplied: useFalScaleAdjustCoordinatePrompt, - scaleAdjustVisualLayoutGuideApplied: - isScaleAdjustOperation && - !useProviderTextOnlyCoordinatePrompt && - (!useScaleAdjustCoordinatePrompt || useScaleAdjustVisualLayoutGuide), - falScaleAdjustCoordinateGuideApplied: - useScaleAdjustCoordinateGuide && - resolvedScaleAdjustModelEntry?.provider?.type === "fal-image", - falScaleAdjustSafePromptApplied: useFalScaleAdjustCoordinatePrompt, - scaleAdjustProviderFallbackPolicy: isScaleAdjustOperation - ? "disabled-one-shot-single-model" - : undefined, - scaleAdjustQualityFailoverPolicy: isScaleAdjustOperation - ? "disabled-reject-unchanged-ai-candidate" - : undefined, - scaleAdjustPreferSameNameProviderFailover: - false, - scaleAdjustStrictSameFamilyFailover: - false, manualRegionReferenceImageCount: manualRegionReferenceImageUrls.length, - regionReferenceImageCount: - referenceImages.length, + regionReferenceImageCount: referenceImages.length, regionCount: regions.length, - scalePercent, - scaleAdjustCropReferenceCount: scaleAdjustCropReferenceImageUrls.length, - scaleAdjustProtectionMode: hasScaleAdjustTargetRects - ? "target-rect" - : "scale-percent", - scaleAdjustTargetRegionCount: regions.filter((region) => region.targetRect) - .length, - scaleAdjustPromptStrategy: isScaleAdjustOperation - ? scaleAdjustPatchReference - ? scaleAdjustPatchReference.structureLockedDraft - ? "ai-patch-workspace-structure-locked-draft-v85-one-shot" - : "ai-patch-workspace-draft-v75-soft-wipe-erase-ghost-one-shot" - : useScaleAdjustEditMask - ? "mask-edit-gpt-image-2-scale-shrink-v74-one-shot" - : useFalScaleAdjustCoordinatePrompt - ? "fal-safe-marked-photo-box-guide-no-old-crop-v44-one-shot" - : useScaleAdjustVisualLayoutGuide - ? "same-name-focused-layout-crop-guide-v54-one-shot" - : useProviderTextOnlyCoordinatePrompt - ? "provider-text-coordinate-crop-v61-one-shot" - : useScaleAdjustMarkedPhotoGuide - ? "same-name-marked-photo-box-guide-no-old-crop-v45-one-shot" - : "locked-plate-semantic-target-map-no-old-crop-v15" - : undefined, - scaleAdjustPostCompositeApplied: Boolean(scaleAdjustPatchReference), - scaleAdjustUsesFullFrameOutput: - isScaleAdjustOperation && !scaleAdjustPatchReference, sourceHeight, sourceNodeId: body.sourceNodeId || null, sourceWidth, }, displayPrompt, - falSafetyTolerance: useScaleAdjustCoordinateGuide ? "6" : undefined, - falSystemPrompt: - useScaleAdjustCoordinatePrompt - ? buildScaleAdjustCoordinateSystemPrompt() - : undefined, - falThinkingLevel: useScaleAdjustCoordinateGuide ? "high" : undefined, imageCount: 1, - imageSize: resolvedRequestImageSize, - model: resolvedModel || undefined, + imageSize: requestedImageSize, + model: requestedModel || undefined, negativePrompt: - isScaleAdjustOperation - ? scaleAdjustPatchReference - ? scaleAdjustPatchNegativePrompt - : useProviderTextOnlyCoordinatePrompt - ? scaleAdjustTextOnlyNegativePrompt - : useScaleAdjustCoordinatePrompt - ? scaleAdjustCoordinateNegativePrompt - : "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、箭头、连线、角点、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、蓝色色块、色条、灰底坐标图、目标占位叠图、淡化痕迹、选区边框、白边、画幅压扁、画面拉伸、整张图缩放、全图放大、全图裁切、全图平移、文字位置变化、文字内容变化、文字模糊、Logo位置变化、主体拉伸、主体压扁、主体变窄、主体变高但变窄、横向挤压、纵向拉伸、透视变形、矩形贴片、硬边、拼接线、补丁感、局部裁切、色差边缘、背景断裂、背景错位、窗框错位、文字错位、地毯纹理错位、非主体背景重绘、主体未缩放、整块框选区域缩放" - : "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、选区边框、白边、画幅压扁、画面拉伸、主体截断、动物截断、缺头、缺尾巴、缺四肢、贴边裁切、矩形贴片、硬边、拼接线、补丁感、色差边缘、背景错位、地面纹理断裂、椅脚错位", + "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、选区边框、白边、画幅压扁、画面拉伸、主体截断、贴边裁切、矩形贴片、硬边、拼接线、补丁感、色差边缘、背景错位、地面纹理断裂、椅脚错位", prompt, promptExtend: false, - providerImageParams: scaleAdjustEditMaskUpload - ? { maskImageUrl: scaleAdjustEditMaskUpload.url } - : undefined, referenceImages, - scaleAdjustPatchComposite: scaleAdjustPatchReference - ? { - diffCompareBlurPx: 0.8, - diffHighThreshold: 42, - diffLowThreshold: 14, - featherMaxPx: 42, - featherMinPx: 10, - featherRatio: 0.018, - maskMode: scaleAdjustPatchReference.structureLockedDraft - ? "workspace" - : "diff", - regions: scaleAdjustPatchReference.structureLockedDraft - ? undefined - : scaleAdjustPatchCompositeRegions, - sourceImageUrl, - workspaceRegion: scaleAdjustPatchReference.workspaceRect, - } - : undefined, - scaleAdjustVerification: scaleAdjustVerificationConfig, - localEditComposite: isScaleAdjustOperation - ? undefined - : { - sourceImageUrl, - regions: regions.map((region) => region.rect), - }, + localEditComposite: { + sourceImageUrl, + regions: regions.map((region) => region.rect), + }, ownerKey: session.user.id, }; From adbbe4442a624641dbbd34c9d9f02dfe1e4fa1cb Mon Sep 17 00:00:00 2001 From: 134 Date: Thu, 9 Jul 2026 15:42:33 +0800 Subject: [PATCH 61/70] Use canvas default for scale adjust model fallback --- app/api/canvas/local-edit/route.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index ea8771892..0cf8f6f9d 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -695,7 +695,19 @@ export async function POST(request: NextRequest) { typeof siteSettingsForModel?.canvasDetailDefaultModelId === "string" ? siteSettingsForModel.canvasDetailDefaultModelId.trim() : ""; - const resolvedModel = requestedModel || configuredDetailModel; + const configuredCanvasModel = + typeof siteSettingsForModel?.canvasDefaultModelId === "string" + ? siteSettingsForModel.canvasDefaultModelId.trim() + : ""; + const configuredAutoCanvasModel = + typeof siteSettingsForModel?.canvasAutoModeDefaultModelId === "string" + ? siteSettingsForModel.canvasAutoModeDefaultModelId.trim() + : ""; + const resolvedModel = + requestedModel || + configuredDetailModel || + configuredCanvasModel || + configuredAutoCanvasModel; const requestedImageSize = typeof body.imageSize === "string" && body.imageSize.trim() ? body.imageSize.trim() From c52cf9aa908f8535d55d77575071daf99189a54c Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 9 Jul 2026 17:25:40 +0800 Subject: [PATCH 62/70] Apply canvas scale adjust rewrite --- app/api/canvas/local-edit/route.ts | 5615 +--------------------------- 1 file changed, 177 insertions(+), 5438 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index e238c1341..0cf8f6f9d 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -2,10 +2,7 @@ import { NextRequest, NextResponse } from "next/server"; import sharp from "sharp"; import { auth } from "@/app/(auth)/auth"; -import { - executeRmbgJob, - fetchImageBufferFromUrl, -} from "@/lib/ai/rmbg/service"; +import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service"; import { createCanvasImageTask, enqueueCanvasImageTask, @@ -13,14 +10,9 @@ import { } from "@/lib/canvas/image-task-queue"; import type { CanvasGenerateImageRequest } from "@/lib/canvas/generate-image"; import { - getEnabledAiModelsWithProvider, getPointsSystemConfig, getSiteSettings, } from "@/lib/db/queries"; -import { - modelLacksReferenceImageInputSupport, -} from "@/lib/ai/reference-image-capability"; -import { isFalGptImage2EditModelId } from "@/lib/ai/providers/fal-image"; import { getImageToolCreditCost } from "@/lib/points/config"; import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge"; import { uploadFile } from "@/lib/storage/service"; @@ -101,63 +93,12 @@ type NormalizedLocalEditRegion = { const MAX_LOCAL_EDIT_REGIONS = 8; const MAX_REFERENCE_IMAGES_PER_REGION = 3; -const MAX_SCALE_ADJUST_CROP_REFERENCES = 4; -const SCALE_ADJUST_CROP_PADDING_RATIO = 0.18; -const SCALE_ADJUST_CROP_MAX_EDGE = 1536; -const SCALE_ADJUST_PATCH_WORKSPACE_MIN_MARGIN = 0.045; -const SCALE_ADJUST_PATCH_WORKSPACE_MAX_MARGIN = 0.18; -const SCALE_ADJUST_LAYOUT_GUIDE_WASH_COLOR = "#f5f7fb"; -const SCALE_ADJUST_LAYOUT_GUIDE_WASH_OPACITY = 0.46; -const SCALE_ADJUST_STRUCTURE_LOCK_MIN_MASK_RATIO = 0.006; -const SCALE_ADJUST_STRUCTURE_LOCK_COMPONENT_MIN_RATIO = 0.00018; -const SCALE_ADJUST_RMBG_MIN_ALPHA_RATIO = 0.012; -const SCALE_ADJUST_RMBG_MAX_SOLID_ALPHA_RATIO = 0.92; -const SCALE_ADJUST_RMBG_ALPHA_CLEAN_THRESHOLD = 42; const MIN_REGION_EDGE_PX = 24; const SCALE_ADJUST_OPERATION = "scale-adjust"; type LocalEditOperation = "local-edit" | typeof SCALE_ADJUST_OPERATION; const MIN_SCALE_ADJUST_PERCENT = -80; const MAX_SCALE_ADJUST_PERCENT = 200; type ScaleAdjustPercent = number; -type ScaleAdjustCropReference = { - regionIndex: number; - url: string; -}; -type ScaleAdjustPatchReference = { - backgroundPlateCropUrl?: string | null; - backgroundPlateFullUrl?: string | null; - compositeClipMaskCropUrl?: string | null; - compositeClipMaskUrl?: string | null; - repairBlendMaskUrl?: string | null; - productLockMethod?: string | null; - productLockEditMaskUrl?: string | null; - productLockFalEditMaskUrl?: string | null; - productLockOverlayUrl?: string | null; - productLockProtectMaskUrl?: string | null; - sourceCropUrl: string; - structureLockedDraft?: boolean; - targetDraftFullUrl?: string | null; - targetDraftUrl: string; - workspaceRect: NormalizedLocalEditRegion["rect"]; - workspacePixelRect: { - left: number; - top: number; - width: number; - height: number; - }; -}; -type ScaleAdjustStructureLockedComposite = { - compositeClip?: { input: Buffer; left: number; top: number }; - method?: "pixel-mask" | "rect-crop-guide" | "rect-crop-guide-rmbg-lock" | "rmbg-cutout"; - overlay: { input: Buffer; left: number; top: number }; - productLock?: { input: Buffer; left: number; top: number }; - productLockProtectMask?: { input: Buffer; left: number; top: number }; - targetClip?: { input: Buffer; left: number; top: number }; - wash: { input: Buffer; left: number; top: number }; -}; -type AiModelWithProviderForSelection = Awaited< - ReturnType ->[number]; const DEFAULT_REGION_COLORS = [ "#ff3b30", "#1677ff", @@ -233,50 +174,6 @@ function toArrayBuffer(buffer: Buffer) { ) as ArrayBuffer; } -function getCanvasScaleAdjustPrepUploadTimeoutMs() { - const parsed = Number(process.env.CANVAS_SCALE_ADJUST_PREP_UPLOAD_TIMEOUT_MS); - if (!Number.isFinite(parsed)) return 25_000; - return Math.max(5_000, Math.min(60_000, Math.round(parsed))); -} - -async function uploadScaleAdjustPrepAsset( - filename: string, - buffer: Buffer | null | undefined -): Promise> | null> { - if (!buffer) return null; - const upload = uploadFile(filename, toArrayBuffer(buffer)); - upload.catch((error) => { - console.warn("[Canvas Local Edit] Scale adjust prep asset upload finished after timeout:", { - error: error instanceof Error ? error.message : String(error), - filename, - }); - }); - - let timeoutId: ReturnType | undefined; - const timeout = new Promise((resolve) => { - timeoutId = setTimeout(() => resolve(null), getCanvasScaleAdjustPrepUploadTimeoutMs()); - }); - - try { - const result = await Promise.race([upload, timeout]); - if (!result) { - console.warn("[Canvas Local Edit] Scale adjust prep asset upload timed out; continuing without it:", { - filename, - timeoutMs: getCanvasScaleAdjustPrepUploadTimeoutMs(), - }); - } - return result; - } catch (error) { - console.warn("[Canvas Local Edit] Scale adjust prep asset upload failed:", { - error: error instanceof Error ? error.message : String(error), - filename, - }); - return null; - } finally { - if (timeoutId) clearTimeout(timeoutId); - } -} - function escapeXml(value: string) { return value .replace(/&/g, "&") @@ -292,324 +189,6 @@ function normalizeHexColor(input: unknown, fallback: string) { return /^#[0-9a-f]{6}$/i.test(expanded) ? expanded.toLowerCase() : fallback; } -function findModelSelectionEntry( - models: AiModelWithProviderForSelection[], - modelRef: string -) { - const normalizedRef = String(modelRef || "").trim(); - if (!normalizedRef) return null; - return ( - models.find( - (entry) => - entry.model.id === normalizedRef || entry.model.modelId === normalizedRef - ) || null - ); -} - -function isScaleAdjustDedicatedDetailModel( - entry: AiModelWithProviderForSelection | null | undefined -) { - if ( - !entry?.model?.supportsImageGen || - modelLacksReferenceImageInputSupport(entry.model, entry.provider) - ) { - return false; - } - const label = `${entry.model.displayName || ""} ${entry.model.modelId || ""}` - .toLowerCase() - .replace(/\s+/g, ""); - return label.includes("详情专线"); -} - -function normalizeScaleAdjustModelText(input: unknown) { - return String(input || "") - .trim() - .toLowerCase() - .normalize("NFKC") - .replace(/[「」『』【】[\]()（）{}]/g, " ") - .replace(/[_/|:：,，.。+]+/g, "-") - .replace(/\s+/g, " "); -} - -function resolveScaleAdjustModelFamilyFromText(input: unknown) { - const normalized = normalizeScaleAdjustModelText(input); - const compact = normalized.replace(/[^a-z0-9]+/g, ""); - if (!normalized && !compact) return ""; - if ( - normalized.includes("gemini-3.1-flash-image-preview") || - normalized.includes("nano-banana-2") || - compact.includes("nanobanana2") - ) { - return "nano-banana-2"; - } - if ( - normalized.includes("gemini-3-pro-image-preview") || - normalized.includes("nano-banana-pro") || - compact.includes("nanobananapro") - ) { - return "nano-banana-pro"; - } - if (normalized.includes("gpt-image-2") || compact.includes("gptimage2")) { - return "gpt-image-2"; - } - return ""; -} - -function getScaleAdjustModelFamily( - entry: AiModelWithProviderForSelection | null | undefined -) { - if (!entry) return ""; - return ( - resolveScaleAdjustModelFamilyFromText(entry.model.modelId) || - resolveScaleAdjustModelFamilyFromText(entry.model.displayName) || - resolveScaleAdjustModelFamilyFromText(entry.model.id) - ); -} - -function isScaleAdjustMaskCapableModel( - entry: AiModelWithProviderForSelection | null | undefined -) { - if (!entry || getScaleAdjustModelFamily(entry) !== "gpt-image-2") { - return false; - } - if (entry.provider.type === "openai-image") return true; - if (entry.provider.type === "fal-image") { - return isFalGptImage2EditModelId(entry.model.modelId || ""); - } - return false; -} - -function isScaleAdjustCoordinateGuideModel( - entry: AiModelWithProviderForSelection | null | undefined -) { - return getScaleAdjustModelFamily(entry) === "nano-banana-2"; -} - -function isScaleAdjustFalNanoBananaEditModel( - entry: AiModelWithProviderForSelection | null | undefined -) { - if ( - !entry || - entry.provider.type !== "fal-image" || - getScaleAdjustModelFamily(entry) !== "nano-banana-2" - ) { - return false; - } - const label = `${entry.model.displayName || ""} ${entry.model.modelId || ""}` - .toLowerCase() - .replace(/\s+/g, " "); - return label.includes("/edit") || /\bedit\b/.test(label); -} - -function isScaleAdjustHealthyModel( - entry: AiModelWithProviderForSelection | null | undefined -) { - return !entry?.model?.healthStatus || entry.model.healthStatus === "healthy"; -} - -function getScaleAdjustProviderPriority(providerType: string) { - switch (providerType) { - case "ruxa-image": - return 10; - case "snapgen-image": - return 20; - case "krea": - return 30; - case "gemini": - case "grsai-gpt-image": - return 40; - case "alibaba-image": - case "bfl-image": - case "openai-image": - case "stepfun-image": - case "volcano-vision": - return 60; - case "fal-image": - return 1000; - default: - return 80; - } -} - -function getScaleAdjustModelHealthPenalty( - healthStatus: string | null | undefined -) { - if (healthStatus === "degraded") return 25; - if (healthStatus === "auto_disabled") return 200; - return 0; -} - -function getScaleAdjustCandidatePriority(entry: AiModelWithProviderForSelection) { - return ( - getScaleAdjustProviderPriority(entry.provider.type) + - getScaleAdjustModelHealthPenalty(entry.model.healthStatus) - ); -} - -function isScaleAdjustUsableImageEditModel( - entry: AiModelWithProviderForSelection | null | undefined -) { - return Boolean( - entry?.model?.supportsImageGen && - !modelLacksReferenceImageInputSupport(entry.model, entry.provider) - ); -} - -function compareScaleAdjustProviderCandidates( - left: AiModelWithProviderForSelection, - right: AiModelWithProviderForSelection, - preferredModelRef: string -) { - const leftPriority = getScaleAdjustCandidatePriority(left); - const rightPriority = getScaleAdjustCandidatePriority(right); - if (leftPriority !== rightPriority) return leftPriority - rightPriority; - - const leftDedicated = isScaleAdjustDedicatedDetailModel(left); - const rightDedicated = isScaleAdjustDedicatedDetailModel(right); - if (leftDedicated !== rightDedicated) return leftDedicated ? -1 : 1; - - const leftExact = - left.model.id === preferredModelRef || left.model.modelId === preferredModelRef; - const rightExact = - right.model.id === preferredModelRef || right.model.modelId === preferredModelRef; - if (leftExact !== rightExact) return leftExact ? -1 : 1; - - return 0; -} - -function resolveScaleAdjustPreferredModelId(params: { - configuredModelId: string; - honorRequestedModelBeforeGpt?: boolean; - preferFalNanoBananaEdit?: boolean; - preferMaskCapableGpt?: boolean; - preferProviderTypes?: string[]; - requestedModelId: string; - models: AiModelWithProviderForSelection[]; -}) { - const requestedModel = findModelSelectionEntry( - params.models, - params.requestedModelId - ); - const configuredModel = findModelSelectionEntry( - params.models, - params.configuredModelId - ); - if ( - requestedModel?.provider.type === "fal-image" && - requestedModel.model.healthStatus !== "auto_disabled" && - isScaleAdjustUsableImageEditModel(requestedModel) - ) { - return requestedModel.model.id || params.requestedModelId; - } - if ( - params.honorRequestedModelBeforeGpt && - requestedModel && - isScaleAdjustUsableImageEditModel(requestedModel) - ) { - return requestedModel.model.id || params.requestedModelId; - } - - const gptImage2Candidates = params.models - .filter((entry) => isScaleAdjustUsableImageEditModel(entry)) - .filter((entry) => getScaleAdjustModelFamily(entry) === "gpt-image-2") - .sort((left, right) => - compareScaleAdjustProviderCandidates( - left, - right, - params.requestedModelId || params.configuredModelId - ) - ); - for (const providerType of params.preferProviderTypes || []) { - const providerCandidate = gptImage2Candidates.find( - (entry) => - entry.provider.type === providerType && isScaleAdjustHealthyModel(entry) - ); - if (providerCandidate) { - return providerCandidate.model.id; - } - } - if (params.preferMaskCapableGpt) { - const maskCapableGptImage2 = gptImage2Candidates - .filter(isScaleAdjustMaskCapableModel) - .filter((entry) => entry.model.healthStatus !== "auto_disabled"); - if (maskCapableGptImage2[0]) { - return maskCapableGptImage2[0].model.id; - } - } - if (params.preferFalNanoBananaEdit) { - const falEditCandidate = params.models - .filter((entry) => isScaleAdjustUsableImageEditModel(entry)) - .filter(isScaleAdjustFalNanoBananaEditModel) - .filter(isScaleAdjustHealthyModel) - .sort((left, right) => - compareScaleAdjustProviderCandidates( - left, - right, - params.requestedModelId || params.configuredModelId - ) - )[0]; - if (falEditCandidate) { - return falEditCandidate.model.id; - } - } - if (gptImage2Candidates[0]) { - return gptImage2Candidates[0].model.id; - } - - if ( - isScaleAdjustUsableImageEditModel(requestedModel) && - requestedModel && - getScaleAdjustModelFamily(requestedModel) !== "nano-banana-2" && - requestedModel.provider.type !== "alibaba-image" - ) { - return requestedModel.model.id || params.requestedModelId; - } - - const autoDetailModel = params.models.find(isScaleAdjustDedicatedDetailModel); - const preferredModel = - (isScaleAdjustDedicatedDetailModel(requestedModel) ? requestedModel : null) || - (isScaleAdjustDedicatedDetailModel(configuredModel) ? configuredModel : null) || - autoDetailModel || - configuredModel || - requestedModel || - null; - const preferredModelRef = - preferredModel?.model.id || - preferredModel?.model.modelId || - params.configuredModelId || - params.requestedModelId; - const preferredFamily = - getScaleAdjustModelFamily(preferredModel) || - resolveScaleAdjustModelFamilyFromText(params.configuredModelId) || - resolveScaleAdjustModelFamilyFromText(params.requestedModelId); - - if (preferredFamily) { - const sameFamilyCandidates = params.models - .filter((entry) => isScaleAdjustUsableImageEditModel(entry)) - .filter((entry) => getScaleAdjustModelFamily(entry) === preferredFamily) - .sort((left, right) => - compareScaleAdjustProviderCandidates(left, right, preferredModelRef) - ); - if (sameFamilyCandidates[0]) { - return sameFamilyCandidates[0].model.id; - } - } - - if (isScaleAdjustDedicatedDetailModel(requestedModel)) { - return requestedModel?.model.id || params.requestedModelId; - } - - if (isScaleAdjustDedicatedDetailModel(configuredModel)) { - return configuredModel?.model.id || params.configuredModelId; - } - - if (autoDetailModel) { - return autoDetailModel.model.id; - } - - return params.configuredModelId || params.requestedModelId; -} - function normalizeHttpImageUrl(input: unknown) { const raw = String(input || "").trim(); if (!/^https?:\/\//i.test(raw)) return ""; @@ -762,3416 +341,6 @@ function getScaleAdjustTargetRect( return region.rect; } -function hasScaleAdjustEnlargeTarget( - regions: NormalizedLocalEditRegion[], - scalePercent?: ScaleAdjustPercent | null -) { - return regions.some((region) => { - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const widthFactor = - region.rect.width > 0 ? targetRect.width / region.rect.width : 1; - const heightFactor = - region.rect.height > 0 ? targetRect.height / region.rect.height : 1; - return widthFactor > 1.08 || heightFactor > 1.08; - }); -} - -function hasScaleAdjustShrinkTarget( - regions: NormalizedLocalEditRegion[], - scalePercent?: ScaleAdjustPercent | null -) { - return regions.some((region) => { - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const widthFactor = - region.rect.width > 0 ? targetRect.width / region.rect.width : 1; - const heightFactor = - region.rect.height > 0 ? targetRect.height / region.rect.height : 1; - return widthFactor < 0.92 || heightFactor < 0.92; - }); -} - -function hasScaleAdjustStructureTarget( - regions: NormalizedLocalEditRegion[], - scalePercent?: ScaleAdjustPercent | null -) { - return regions.some((region) => { - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const widthFactor = - region.rect.width > 0 ? targetRect.width / region.rect.width : 1; - const heightFactor = - region.rect.height > 0 ? targetRect.height / region.rect.height : 1; - return ( - widthFactor > 1.08 || - heightFactor > 1.08 || - widthFactor < 0.92 || - heightFactor < 0.92 - ); - }); -} - -function expandRectWithMargins( - rect: NormalizedLocalEditRegion["rect"], - marginX: number, - marginY: number -) { - const left = clamp(rect.x - marginX, 0, 1); - const top = clamp(rect.y - marginY, 0, 1); - const right = clamp(rect.x + rect.width + marginX, left + 0.001, 1); - const bottom = clamp(rect.y + rect.height + marginY, top + 0.001, 1); - - return { - x: left, - y: top, - width: Math.max(0.001, right - left), - height: Math.max(0.001, bottom - top), - }; -} - -function getRectUnion( - leftRect: NormalizedLocalEditRegion["rect"], - rightRect: NormalizedLocalEditRegion["rect"] -) { - const left = Math.min(leftRect.x, rightRect.x); - const top = Math.min(leftRect.y, rightRect.y); - const right = Math.max( - leftRect.x + leftRect.width, - rightRect.x + rightRect.width - ); - const bottom = Math.max( - leftRect.y + leftRect.height, - rightRect.y + rightRect.height - ); - return { - x: clamp(left, 0, 1), - y: clamp(top, 0, 1), - width: Math.max(0.001, clamp(right, 0, 1) - clamp(left, 0, 1)), - height: Math.max(0.001, clamp(bottom, 0, 1) - clamp(top, 0, 1)), - }; -} - -function rectsDifferMeaningfully( - leftRect: NormalizedLocalEditRegion["rect"], - rightRect: NormalizedLocalEditRegion["rect"], - epsilon = 0.0005 -) { - return ( - Math.abs(leftRect.x - rightRect.x) > epsilon || - Math.abs(leftRect.y - rightRect.y) > epsilon || - Math.abs(leftRect.width - rightRect.width) > epsilon || - Math.abs(leftRect.height - rightRect.height) > epsilon - ); -} - -function getScaleAdjustWorkspaceRect( - sourceRect: NormalizedLocalEditRegion["rect"], - targetRect: NormalizedLocalEditRegion["rect"], - scalePercent?: ScaleAdjustPercent | null -) { - const sourceWidth = Math.max(0.0001, sourceRect.width); - const sourceHeight = Math.max(0.0001, sourceRect.height); - const widthDeltaRatio = Math.abs(targetRect.width / sourceWidth - 1); - const heightDeltaRatio = Math.abs(targetRect.height / sourceHeight - 1); - const areaDeltaRatio = Math.abs( - Math.sqrt( - Math.max(0.0001, targetRect.width * targetRect.height) / - Math.max(0.0001, sourceWidth * sourceHeight) - ) - 1 - ); - const sourceCenterX = sourceRect.x + sourceRect.width / 2; - const sourceCenterY = sourceRect.y + sourceRect.height / 2; - const targetCenterX = targetRect.x + targetRect.width / 2; - const targetCenterY = targetRect.y + targetRect.height / 2; - const movementRatioX = Math.abs(targetCenterX - sourceCenterX) / sourceWidth; - const movementRatioY = Math.abs(targetCenterY - sourceCenterY) / sourceHeight; - const explicitChangeRatio = Math.max( - widthDeltaRatio, - heightDeltaRatio, - areaDeltaRatio, - movementRatioX, - movementRatioY - ); - const fallbackScaleRatio = - typeof scalePercent === "number" && Number.isFinite(scalePercent) - ? Math.min(1, Math.abs(scalePercent) / 100) - : 0; - const changeRatio = Math.max(explicitChangeRatio, fallbackScaleRatio); - const unionRect = getRectUnion(sourceRect, targetRect); - if (changeRatio <= 0 && !rectsDifferMeaningfully(unionRect, targetRect)) { - return targetRect; - } - - const marginX = Math.min( - 0.16, - Math.max( - 0.018, - unionRect.width * 0.04, - sourceRect.width * changeRatio * 0.24 - ) - ); - const marginY = Math.min( - 0.16, - Math.max( - 0.018, - unionRect.height * 0.04, - sourceRect.height * changeRatio * 0.24 - ) - ); - - return expandRectWithMargins(unionRect, marginX, marginY); -} - -function buildScaleAdjustCompositeRegions( - regions: NormalizedLocalEditRegion[], - scalePercent?: ScaleAdjustPercent | null -) { - return regions.flatMap((region) => { - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const workspaceRect = getScaleAdjustWorkspaceRect( - region.rect, - targetRect, - scalePercent - ); - return rectsDifferMeaningfully(workspaceRect, targetRect) && - rectsDifferMeaningfully(workspaceRect, region.rect) - ? [region.rect, targetRect, workspaceRect] - : [region.rect, targetRect]; - }); -} - -function buildScaleAdjustPatchCompositeRegions( - regions: NormalizedLocalEditRegion[], - scalePercent?: ScaleAdjustPercent | null -) { - return regions.flatMap((region) => [ - region.rect, - getScaleAdjustTargetRect(region, scalePercent), - ]); -} - -function buildScaleAdjustProtectedChangeRegions( - regions: NormalizedLocalEditRegion[], - scalePercent?: ScaleAdjustPercent | null -) { - return regions.flatMap((region) => { - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const unionRect = getRectUnion(region.rect, targetRect); - const marginX = Math.min( - 0.045, - Math.max(0.012, unionRect.width * 0.08) - ); - const marginY = Math.min( - 0.045, - Math.max(0.012, unionRect.height * 0.08) - ); - - return [ - expandRectWithMargins(region.rect, marginX, marginY), - expandRectWithMargins(targetRect, marginX, marginY), - ]; - }); -} - -function getScaleAdjustPatchWorkspaceRect( - regions: NormalizedLocalEditRegion[], - scalePercent?: ScaleAdjustPercent | null -) { - const workspaceRects = regions.map((region) => { - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const baseWorkspace = getScaleAdjustWorkspaceRect( - region.rect, - targetRect, - scalePercent - ); - const marginX = Math.min( - SCALE_ADJUST_PATCH_WORKSPACE_MAX_MARGIN, - Math.max( - SCALE_ADJUST_PATCH_WORKSPACE_MIN_MARGIN, - baseWorkspace.width * 0.22, - region.rect.width * 0.18 - ) - ); - const marginY = Math.min( - SCALE_ADJUST_PATCH_WORKSPACE_MAX_MARGIN, - Math.max( - SCALE_ADJUST_PATCH_WORKSPACE_MIN_MARGIN, - baseWorkspace.height * 0.22, - region.rect.height * 0.18 - ) - ); - return expandRectWithMargins(baseWorkspace, marginX, marginY); - }); - - return workspaceRects.reduce((combined, rect) => getRectUnion(combined, rect)); -} - -function expandPixelRectForCrop(params: { - height: number; - pixelRect: NormalizedLocalEditRegion["pixelRect"]; - width: number; -}) { - const { height, pixelRect, width } = params; - const padX = Math.round(pixelRect.width * SCALE_ADJUST_CROP_PADDING_RATIO); - const padY = Math.round(pixelRect.height * SCALE_ADJUST_CROP_PADDING_RATIO); - const left = clamp(pixelRect.left - padX, 0, Math.max(0, width - 1)); - const top = clamp(pixelRect.top - padY, 0, Math.max(0, height - 1)); - const right = clamp( - pixelRect.left + pixelRect.width + padX, - left + 1, - width - ); - const bottom = clamp( - pixelRect.top + pixelRect.height + padY, - top + 1, - height - ); - - return { - left, - top, - width: Math.max(1, right - left), - height: Math.max(1, bottom - top), - }; -} - -async function createScaleAdjustCropReferences(params: { - regions: NormalizedLocalEditRegion[]; - sourceBuffer: Buffer; - sourceHeight: number; - sourceWidth: number; -}): Promise { - const references: ScaleAdjustCropReference[] = []; - const regionsForCrop = params.regions.slice(0, MAX_SCALE_ADJUST_CROP_REFERENCES); - - for (const region of regionsForCrop) { - const focusBox = getScaleAdjustFocusedSourceBox({ - region, - sourceBox: region.pixelRect, - }); - const cropBox = getScaleAdjustFocusDirection(region) - ? focusBox - : expandPixelRectForCrop({ - height: params.sourceHeight, - pixelRect: region.pixelRect, - width: params.sourceWidth, - }); - const cropBuffer = await sharp(params.sourceBuffer, { failOn: "none" }) - .extract(cropBox) - .resize({ - fit: "inside", - height: SCALE_ADJUST_CROP_MAX_EDGE, - withoutEnlargement: true, - width: SCALE_ADJUST_CROP_MAX_EDGE, - }) - .png({ compressionLevel: 6 }) - .toBuffer(); - const upload = await uploadFile( - `canvas-scale-adjust-crop-r${region.index}-${Date.now()}-${Math.random() - .toString(36) - .slice(2, 8)}.png`, - toArrayBuffer(cropBuffer) - ); - references.push({ - regionIndex: region.index, - url: upload.url, - }); - } - - return references; -} - -function getScaleAdjustVisualGuidePlacement(params: { - height: number; - region: NormalizedLocalEditRegion; - scalePercent?: ScaleAdjustPercent | null; - width: number; -}) { - const { height, region, scalePercent, width } = params; - const source = region.pixelRect; - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const target = region.targetPixelRect || rectToPixels(targetRect, width, height); - const sourceLeft = clamp(source.left, 0, Math.max(0, width - 1)); - const sourceTop = clamp(source.top, 0, Math.max(0, height - 1)); - const sourceRight = clamp( - source.left + source.width, - sourceLeft + 1, - width - ); - const sourceBottom = clamp( - source.top + source.height, - sourceTop + 1, - height - ); - const sourceFootprintBox = { - left: Math.round(sourceLeft), - top: Math.round(sourceTop), - width: Math.max(1, Math.round(sourceRight - sourceLeft)), - height: Math.max(1, Math.round(sourceBottom - sourceTop)), - }; - const sourceBox = getScaleAdjustFocusedSourceBox({ - region, - sourceBox: sourceFootprintBox, - }); - const sourceWidth = sourceBox.width; - const sourceHeight = sourceBox.height; - const widthScale = target.width / Math.max(1, sourceWidth); - const heightScale = target.height / Math.max(1, sourceHeight); - const uniformScale = Math.max( - 0.05, - Math.min( - Number.isFinite(widthScale) && widthScale > 0 ? widthScale : 1, - Number.isFinite(heightScale) && heightScale > 0 ? heightScale : 1 - ) - ); - const placedWidth = clamp( - Math.round(sourceWidth * uniformScale), - 1, - Math.max(1, width) - ); - const placedHeight = clamp( - Math.round(sourceHeight * uniformScale), - 1, - Math.max(1, height) - ); - const targetCenterX = target.left + target.width / 2; - const targetCenterY = target.top + target.height / 2; - const placedLeft = clamp( - Math.round(targetCenterX - placedWidth / 2), - 0, - Math.max(0, width - placedWidth) - ); - const placedTop = clamp( - Math.round(targetCenterY - placedHeight / 2), - 0, - Math.max(0, height - placedHeight) - ); - - return { - placedHeight: Math.max(1, Math.round(placedHeight)), - placedLeft: Math.max(0, Math.round(placedLeft)), - placedTop: Math.max(0, Math.round(placedTop)), - placedWidth: Math.max(1, Math.round(placedWidth)), - sourceBox: { - left: sourceBox.left, - top: sourceBox.top, - width: sourceWidth, - height: sourceHeight, - }, - sourceFootprintBox, - targetBox: target, - uniformScale, - }; -} - -function getScaleAdjustFocusDirection(region: NormalizedLocalEditRegion) { - const text = normalizeScaleAdjustModelText(region.prompt); - const compact = text.replace(/[^a-z0-9\u4e00-\u9fa5]+/g, ""); - const protectedDirectionalPerson = - /(?:不要|别|勿|不改变|不能改变|不动|保持|保留|保护|非目标|do not|don't|dont|keep|preserve|without changing).{0,28}(?:left|right|左|右).{0,28}(?:person|people|human|woman|man|model|face|body|人物|人像|人|模特|女人|男人|脸|身体)/.test( - text - ) || - /(?:left|right|左|右).{0,16}(?:person|people|human|woman|man|model|face|body|人物|人像|人|模特|女人|男人|脸|身体).{0,28}(?:不要|别|勿|不改变|不能改变|不动|保持|保留|保护|非目标|do not|don't|dont|keep|preserve|without changing)/.test( - text - ) || - /(?:不要|别|勿|不改变|不能改变|不动|保持|保留|保护|非目标).{0,16}(?:左侧|右侧|左边|右边|左方|右方).{0,16}(?:人物|人像|人|模特|女人|男人|脸|身体)/.test( - compact - ); - - if (protectedDirectionalPerson) return ""; - if (text.includes("left") || compact.includes("左")) return "left"; - if (text.includes("right") || compact.includes("右")) return "right"; - if (text.includes("top") || compact.includes("上")) return "top"; - if (text.includes("bottom") || compact.includes("下")) return "bottom"; - return ""; -} - -function getScaleAdjustFocusedSourceBox(params: { - region: NormalizedLocalEditRegion; - sourceBox: { left: number; top: number; width: number; height: number }; -}) { - const { region, sourceBox } = params; - const direction = getScaleAdjustFocusDirection(region); - if (!direction) return sourceBox; - - const minWidth = Math.max(1, Math.round(sourceBox.width * 0.42)); - const minHeight = Math.max(1, Math.round(sourceBox.height * 0.42)); - let left = sourceBox.left; - let top = sourceBox.top; - let width = sourceBox.width; - let height = sourceBox.height; - - if (direction === "left") { - width = Math.max(minWidth, Math.round(sourceBox.width * 0.68)); - } else if (direction === "right") { - const nextWidth = Math.max(minWidth, Math.round(sourceBox.width * 0.68)); - left = sourceBox.left + sourceBox.width - nextWidth; - width = nextWidth; - } else if (direction === "top") { - height = Math.max(minHeight, Math.round(sourceBox.height * 0.72)); - } else if (direction === "bottom") { - const nextHeight = Math.max(minHeight, Math.round(sourceBox.height * 0.72)); - top = sourceBox.top + sourceBox.height - nextHeight; - height = nextHeight; - } - - return { - left, - top, - width, - height, - }; -} - -function buildScaleAdjustPatchMaskSvg(params: { height: number; width: number }) { - const { height, width } = params; - const blur = Math.max(2, Math.min(14, Math.round(Math.min(width, height) * 0.035))); - const inset = Math.min( - Math.floor(Math.min(width, height) / 8), - Math.max(1, Math.round(blur * 0.7)) - ); - const rectWidth = Math.max(1, width - inset * 2); - const rectHeight = Math.max(1, height - inset * 2); - - return Buffer.from( - ` - - - - - - - ` - ); -} - -function formatScaleAdjustWashColor( - background?: { b: number; g: number; r: number } | null -) { - if (!background) return SCALE_ADJUST_LAYOUT_GUIDE_WASH_COLOR; - const r = clamp(Math.round(background.r), 24, 242); - const g = clamp(Math.round(background.g), 24, 242); - const b = clamp(Math.round(background.b), 24, 242); - return `rgb(${r}, ${g}, ${b})`; -} - -function buildScaleAdjustSourceFootprintWashSvg(params: { - boxes: Array<{ - background?: { b: number; g: number; r: number } | null; - innerOpacity?: number; - left: number; - opacity?: number; - top: number; - width: number; - height: number; - }>; - height: number; - width: number; -}) { - const { boxes, height, width } = params; - const defs = boxes - .map((box, index) => { - const blur = Math.max( - 14, - Math.min(72, Math.round(Math.min(box.width, box.height) * 0.14)) - ); - return ` - - `; - }) - .join(""); - const rects = boxes - .map((box, index) => { - const blur = Math.max( - 14, - Math.min(72, Math.round(Math.min(box.width, box.height) * 0.14)) - ); - const pad = Math.max(blur, Math.round(Math.min(box.width, box.height) * 0.055)); - const left = Math.max(0, box.left - blur); - const top = Math.max(0, box.top - blur); - const right = Math.min(width, box.left + box.width + blur); - const bottom = Math.min(height, box.top + box.height + blur); - const rectWidth = Math.max(1, right - left); - const rectHeight = Math.max(1, bottom - top); - const radius = Math.max( - 8, - Math.min(Math.round(blur * 1.8), Math.round(rectWidth / 3), Math.round(rectHeight / 3)) - ); - const innerLeft = Math.max(0, box.left + pad); - const innerTop = Math.max(0, box.top + pad); - const innerRight = Math.min(width, box.left + box.width - pad); - const innerBottom = Math.min(height, box.top + box.height - pad); - const fill = formatScaleAdjustWashColor(box.background); - const opacity = - typeof box.opacity === "number" - ? clamp(box.opacity, 0, 1) - : SCALE_ADJUST_LAYOUT_GUIDE_WASH_OPACITY; - const innerOpacity = - typeof box.innerOpacity === "number" - ? clamp(box.innerOpacity, 0, 1) - : 0.18; - const inner = - innerRight > innerLeft + 4 && innerBottom > innerTop + 4 - ? `` - : ""; - return `${inner}`; - }) - .join(""); - - return Buffer.from( - ` - - ${defs} - - ${rects} - ` - ); -} - -function buildScaleAdjustTargetFootprintShadowSvg(params: { - height: number; - placements: Array>; - width: number; -}) { - const { height, placements, width } = params; - const minSide = Math.max(1, Math.min(width, height)); - const blur = Math.max(4, Math.min(18, Math.round(minSide * 0.014))); - const rects = placements - .map((placement) => { - const pad = Math.max(2, Math.round(blur * 0.45)); - const left = clamp( - placement.placedLeft - pad, - 0, - Math.max(0, width - 1) - ); - const top = clamp( - placement.placedTop - pad, - 0, - Math.max(0, height - 1) - ); - const right = clamp( - placement.placedLeft + placement.placedWidth + pad, - left + 1, - width - ); - const bottom = clamp( - placement.placedTop + placement.placedHeight + pad, - top + 1, - height - ); - const radius = Math.max(4, Math.round(blur * 1.2)); - return ``; - }) - .join(""); - - return Buffer.from( - ` - - - - - - ${rects} - ` - ); -} - -function getScaleAdjustRawPixelIndex(width: number, x: number, y: number) { - return (y * width + x) * 3; -} - -function getScaleAdjustPixelLuma(r: number, g: number, b: number) { - return r * 0.299 + g * 0.587 + b * 0.114; -} - -function getScaleAdjustPixelStats(r: number, g: number, b: number) { - const max = Math.max(r, g, b); - const min = Math.min(r, g, b); - return { - luma: getScaleAdjustPixelLuma(r, g, b), - saturation: max > 0 ? (max - min) / max : 0, - }; -} - -function isScaleAdjustLikelySkinPixel(r: number, g: number, b: number) { - return ( - r > 132 && - g > 82 && - b > 58 && - r > g * 1.06 && - g > b * 1.03 && - r - b < 135 - ); -} - -function isScaleAdjustLikelyWhiteFabricPixel( - r: number, - g: number, - b: number, - luma: number, - saturation: number -) { - return ( - luma > 158 && - saturation < 0.3 && - Math.abs(r - g) < 34 && - Math.abs(g - b) < 44 - ); -} - -function isScaleAdjustPersonLikePrompt(prompt: string) { - const text = normalizeScaleAdjustModelText(prompt); - return /person|people|human|woman|man|girl|boy|model|face|body|hand|arm|leg|clothing|人物|人像|真人|模特|女人|男人|女孩|男孩|小孩|脸|身体|手|胳膊|腿|衣服/.test( - text - ); -} - -function isScaleAdjustFurnitureProductLikePrompt(prompt: string) { - const text = normalizeScaleAdjustModelText(prompt); - return /furniture|product|object|chair|sofa|table|desk|cabinet|shelf|rack|lamp|cart|wheel|wood|woven|cane|rattan|panel|商品|产品|物体|家具|家居|椅|沙发|茶几|桌|柜|架|衣帽架|置物架|收纳|灯|车|轮|木|藤编|编织|面板|杆|挂包|帽子|伞/.test( - text - ); -} - -type ScaleAdjustProtectedPersonSide = "auto" | "left" | "right"; - -function getScaleAdjustProtectedPersonSide( - prompt: string -): ScaleAdjustProtectedPersonSide | null { - const text = normalizeScaleAdjustModelText(prompt); - if ( - !/person|people|human|woman|man|model|girl|boy|face|body|人物|人像|人|模特|女人|男人|女孩|男孩|脸|身体/.test( - text - ) - ) { - return null; - } - if (/right|右侧|右边|右方|右/.test(text)) return "right"; - if (/left|左侧|左边|左方|左/.test(text)) return "left"; - return "auto"; -} - -function hasScaleAdjustPersonPixelSignal(params: { - box: { left: number; top: number; width: number; height: number }; - raw: Buffer; - width: number; -}) { - const { box, raw, width } = params; - let sampled = 0; - let skin = 0; - let whiteFabric = 0; - const step = Math.max(1, Math.floor(Math.min(box.width, box.height) / 260)); - - for (let localY = 0; localY < box.height; localY += step) { - const y = box.top + localY; - for (let localX = 0; localX < box.width; localX += step) { - const x = box.left + localX; - const rawIndex = getScaleAdjustRawPixelIndex(width, x, y); - const r = raw[rawIndex] || 0; - const g = raw[rawIndex + 1] || 0; - const b = raw[rawIndex + 2] || 0; - const { luma, saturation } = getScaleAdjustPixelStats(r, g, b); - sampled += 1; - if (isScaleAdjustLikelySkinPixel(r, g, b)) { - skin += 1; - } else if ( - isScaleAdjustLikelyWhiteFabricPixel(r, g, b, luma, saturation) && - luma < 242 - ) { - whiteFabric += 1; - } - } - } - - if (sampled <= 0) return false; - const skinRatio = skin / sampled; - const fabricRatio = whiteFabric / sampled; - return skin >= 18 && skinRatio >= 0.0025 && fabricRatio >= 0.035; -} - -function estimateScaleAdjustBoxBackground(params: { - box: { left: number; top: number; width: number; height: number }; - raw: Buffer; - width: number; -}) { - const { box, raw, width } = params; - const samples: Array<{ b: number; g: number; luma: number; r: number }> = []; - const borderX = Math.max(2, Math.round(box.width * 0.08)); - const borderY = Math.max(2, Math.round(box.height * 0.08)); - - for (let localY = 0; localY < box.height; localY += 1) { - const y = box.top + localY; - const isBorderY = localY < borderY || localY >= box.height - borderY; - for (let localX = 0; localX < box.width; localX += 1) { - const isBorderX = localX < borderX || localX >= box.width - borderX; - if (!isBorderX && !isBorderY) continue; - const x = box.left + localX; - const rawIndex = getScaleAdjustRawPixelIndex(width, x, y); - const r = raw[rawIndex] || 0; - const g = raw[rawIndex + 1] || 0; - const b = raw[rawIndex + 2] || 0; - const { luma, saturation } = getScaleAdjustPixelStats(r, g, b); - if (isScaleAdjustLikelySkinPixel(r, g, b)) continue; - if (saturation > 0.42 && luma < 185) continue; - samples.push({ b, g, luma, r }); - } - } - - if (samples.length === 0) { - return { b: 245, g: 247, luma: 246, r: 245 }; - } - - samples.sort((left, right) => right.luma - left.luma); - const kept = samples.slice(0, Math.max(12, Math.ceil(samples.length * 0.38))); - const totals = kept.reduce( - (acc, sample) => { - acc.r += sample.r; - acc.g += sample.g; - acc.b += sample.b; - acc.luma += sample.luma; - return acc; - }, - { b: 0, g: 0, luma: 0, r: 0 } - ); - - return { - b: totals.b / kept.length, - g: totals.g / kept.length, - luma: totals.luma / kept.length, - r: totals.r / kept.length, - }; -} - -function dilateScaleAdjustStructureMask(params: { - height: number; - mask: Uint8Array; - radius: number; - width: number; -}) { - const { height, mask, radius, width } = params; - if (radius <= 0) return mask; - const output = new Uint8Array(mask.length); - for (let y = 0; y < height; y += 1) { - for (let x = 0; x < width; x += 1) { - if (mask[y * width + x] === 0) continue; - const left = Math.max(0, x - radius); - const right = Math.min(width - 1, x + radius); - const top = Math.max(0, y - radius); - const bottom = Math.min(height - 1, y + radius); - for (let yy = top; yy <= bottom; yy += 1) { - for (let xx = left; xx <= right; xx += 1) { - output[yy * width + xx] = 255; - } - } - } - } - return output; -} - -function buildScaleAdjustStructureInitialMask(params: { - box: { left: number; top: number; width: number; height: number }; - includePersonPixels?: boolean; - raw: Buffer; - width: number; -}) { - const { box, includePersonPixels = false, raw, width } = params; - const background = estimateScaleAdjustBoxBackground(params); - const mask = new Uint8Array(box.width * box.height); - - for (let localY = 0; localY < box.height; localY += 1) { - const y = box.top + localY; - for (let localX = 0; localX < box.width; localX += 1) { - const x = box.left + localX; - const rawIndex = getScaleAdjustRawPixelIndex(width, x, y); - const r = raw[rawIndex] || 0; - const g = raw[rawIndex + 1] || 0; - const b = raw[rawIndex + 2] || 0; - const { luma, saturation } = getScaleAdjustPixelStats(r, g, b); - const likelySkin = isScaleAdjustLikelySkinPixel(r, g, b); - const likelyWhiteFabric = isScaleAdjustLikelyWhiteFabricPixel( - r, - g, - b, - luma, - saturation - ); - const colorDistance = Math.sqrt( - (r - background.r) ** 2 + - (g - background.g) ** 2 + - (b - background.b) ** 2 - ); - - if (includePersonPixels && likelySkin) { - mask[localY * box.width + localX] = 255; - continue; - } - if ( - includePersonPixels && - likelyWhiteFabric && - luma < 246 && - (colorDistance > 12 || luma < background.luma - 4) - ) { - mask[localY * box.width + localX] = 255; - continue; - } - if (likelySkin) continue; - if (likelyWhiteFabric) { - continue; - } - - const darkStructure = luma < Math.min(136, background.luma - 38); - const saturatedStructure = - saturation > 0.16 && luma < 218 && colorDistance > 38; - const woodOrCaneStructure = - r > 46 && - g > 26 && - b < 132 && - r >= g * 0.9 && - g >= b * 1.02 && - saturation > 0.18 && - luma < 218; - - if (darkStructure || saturatedStructure || woodOrCaneStructure) { - mask[localY * box.width + localX] = 255; - } - } - } - - return dilateScaleAdjustStructureMask({ - height: box.height, - mask, - radius: Math.max(1, Math.round(Math.min(box.width, box.height) * 0.004)), - width: box.width, - }); -} - -function buildScaleAdjustProtectedPersonSeedMask(params: { - background?: { b: number; g: number; luma: number; r: number } | null; - box: { left: number; top: number; width: number; height: number }; - raw: Buffer; - side: ScaleAdjustProtectedPersonSide; - width: number; -}) { - const { background = null, box, raw, side, width } = params; - const candidate = new Uint8Array(box.width * box.height); - const anchor = new Uint8Array(box.width * box.height); - const visited = new Uint8Array(box.width * box.height); - const seed = new Uint8Array(box.width * box.height); - const sideAllowsX = (localX: number) => { - if (side === "right") return localX >= box.width * 0.52; - if (side === "left") return localX <= box.width * 0.48; - return true; - }; - - for (let localY = 0; localY < box.height; localY += 1) { - const y = box.top + localY; - for (let localX = 0; localX < box.width; localX += 1) { - if (!sideAllowsX(localX)) continue; - const x = box.left + localX; - const rawIndex = getScaleAdjustRawPixelIndex(width, x, y); - const r = raw[rawIndex] || 0; - const g = raw[rawIndex + 1] || 0; - const b = raw[rawIndex + 2] || 0; - const { luma, saturation } = getScaleAdjustPixelStats(r, g, b); - const likelySkin = isScaleAdjustLikelySkinPixel(r, g, b); - const colorDistance = background - ? Math.sqrt( - (r - background.r) ** 2 + - (g - background.g) ** 2 + - (b - background.b) ** 2 - ) - : 999; - const backgroundLikeLightPixel = - Boolean(background) && - luma >= (background?.luma || 0) - 3 && - colorDistance < 18; - const likelyWhiteFabric = - isScaleAdjustLikelyWhiteFabricPixel(r, g, b, luma, saturation) && - luma < 246 && - saturation < 0.22 && - !backgroundLikeLightPixel && - (colorDistance > 16 || !background || luma < background.luma - 5); - if (!likelySkin && !likelyWhiteFabric) continue; - const index = localY * box.width + localX; - candidate[index] = 255; - if (likelySkin) { - anchor[index] = 255; - } - } - } - - const queueX: number[] = []; - const queueY: number[] = []; - let seedCount = 0; - let anchorCount = 0; - for (let y = 0; y < box.height; y += 1) { - for (let x = 0; x < box.width; x += 1) { - const startIndex = y * box.width + x; - if (candidate[startIndex] === 0 || visited[startIndex]) continue; - - queueX.length = 0; - queueY.length = 0; - queueX.push(x); - queueY.push(y); - visited[startIndex] = 1; - let head = 0; - let area = 0; - let anchors = 0; - let minX = x; - let minY = y; - let maxX = x; - let maxY = y; - - while (head < queueX.length) { - const cx = queueX[head] || 0; - const cy = queueY[head] || 0; - head += 1; - area += 1; - if (anchor[cy * box.width + cx]) anchors += 1; - minX = Math.min(minX, cx); - minY = Math.min(minY, cy); - maxX = Math.max(maxX, cx); - maxY = Math.max(maxY, cy); - - for (let dy = -1; dy <= 1; dy += 1) { - for (let dx = -1; dx <= 1; dx += 1) { - if (dx === 0 && dy === 0) continue; - const nx = cx + dx; - const ny = cy + dy; - if (nx < 0 || ny < 0 || nx >= box.width || ny >= box.height) continue; - const nextIndex = ny * box.width + nx; - if (candidate[nextIndex] === 0 || visited[nextIndex]) continue; - visited[nextIndex] = 1; - queueX.push(nx); - queueY.push(ny); - } - } - } - - const componentWidth = maxX - minX + 1; - const componentHeight = maxY - minY + 1; - const broadBackgroundStripe = - anchors === 0 && - componentHeight >= box.height * 0.48 && - componentWidth >= box.width * 0.16; - const tinyUnanchoredPatch = anchors === 0 && area < box.width * box.height * 0.004; - if (anchors === 0 || broadBackgroundStripe || tinyUnanchoredPatch) { - continue; - } - - for (let index = 0; index < queueX.length; index += 1) { - const pixelX = queueX[index] || 0; - const pixelY = queueY[index] || 0; - const seedIndex = pixelY * box.width + pixelX; - if (seed[seedIndex]) continue; - seed[seedIndex] = 255; - seedCount += 1; - } - anchorCount += anchors; - } - } - - const seedRatio = seedCount / Math.max(1, box.width * box.height); - const anchorRatio = anchorCount / Math.max(1, box.width * box.height); - if (anchorCount < 12 || anchorRatio < 0.00012) return null; - if (seedCount < 24 || seedRatio < 0.0006) return null; - - return dilateScaleAdjustStructureMask({ - height: box.height, - mask: seed, - radius: Math.max(4, Math.round(Math.min(box.width, box.height) * 0.032)), - width: box.width, - }); -} - -function buildScaleAdjustProtectedPersonGeometryMask(params: { - box: { width: number; height: number }; - side: ScaleAdjustProtectedPersonSide | null; -}) { - const { box, side } = params; - if (!side) return null; - const mask = new Uint8Array(box.width * box.height); - for (let y = 0; y < box.height; y += 1) { - for (let x = 0; x < box.width; x += 1) { - const xr = x / Math.max(1, box.width); - const yr = y / Math.max(1, box.height); - const protect = - side === "right" - ? (xr >= 0.72 && yr >= 0.26) || - (xr >= 0.58 && yr >= 0.56) || - (xr >= 0.48 && yr >= 0.7) - : side === "left" - ? (xr <= 0.28 && yr >= 0.26) || - (xr <= 0.42 && yr >= 0.56) || - (xr <= 0.52 && yr >= 0.7) - : (yr >= 0.62 && (xr <= 0.2 || xr >= 0.8)) || - (yr >= 0.72 && (xr <= 0.34 || xr >= 0.66)); - if (protect) mask[y * box.width + x] = 255; - } - } - return mask; -} - -function filterScaleAdjustStructureComponents(params: { - box: { width: number; height: number }; - mask: Uint8Array; - protectedPersonMask?: Uint8Array | null; - protectedPersonSide?: ScaleAdjustProtectedPersonSide | null; -}) { - const { box, mask, protectedPersonMask = null, protectedPersonSide = null } = - params; - const width = box.width; - const height = box.height; - const visited = new Uint8Array(mask.length); - const output = new Uint8Array(mask.length); - const queueX: number[] = []; - const queueY: number[] = []; - const minComponentArea = Math.max( - 10, - Math.round(width * height * SCALE_ADJUST_STRUCTURE_LOCK_COMPONENT_MIN_RATIO) - ); - - for (let y = 0; y < height; y += 1) { - for (let x = 0; x < width; x += 1) { - const startIndex = y * width + x; - if (mask[startIndex] === 0 || visited[startIndex]) continue; - - queueX.length = 0; - queueY.length = 0; - queueX.push(x); - queueY.push(y); - visited[startIndex] = 1; - let head = 0; - let area = 0; - let minX = x; - let minY = y; - let maxX = x; - let maxY = y; - let protectedPersonOverlap = 0; - - while (head < queueX.length) { - const cx = queueX[head] || 0; - const cy = queueY[head] || 0; - head += 1; - area += 1; - minX = Math.min(minX, cx); - minY = Math.min(minY, cy); - maxX = Math.max(maxX, cx); - maxY = Math.max(maxY, cy); - if (protectedPersonMask?.[cy * width + cx]) { - protectedPersonOverlap += 1; - } - - for (let dy = -1; dy <= 1; dy += 1) { - for (let dx = -1; dx <= 1; dx += 1) { - if (dx === 0 && dy === 0) continue; - const nx = cx + dx; - const ny = cy + dy; - if (nx < 0 || ny < 0 || nx >= width || ny >= height) continue; - const nextIndex = ny * width + nx; - if (mask[nextIndex] === 0 || visited[nextIndex]) continue; - visited[nextIndex] = 1; - queueX.push(nx); - queueY.push(ny); - } - } - } - - const componentWidth = maxX - minX + 1; - const componentHeight = maxY - minY + 1; - const componentBoxArea = Math.max(1, componentWidth * componentHeight); - const componentDensity = area / componentBoxArea; - const centerXRatio = (minX + componentWidth / 2) / width; - const centerYRatio = (minY + componentHeight / 2) / height; - const touchesTop = minY <= height * 0.04; - const touchesRight = maxX >= width * 0.88; - const touchesLeft = minX <= width * 0.12; - const edgeSliver = touchesRight && componentWidth < width * 0.18; - const touchesProtectedPersonSide = - protectedPersonSide === "right" - ? maxX >= width * 0.68 - : protectedPersonSide === "left" - ? minX <= width * 0.32 - : protectedPersonSide === "auto" - ? touchesLeft || touchesRight - : false; - const shouldClipProtectedPersonSide = - Boolean(protectedPersonMask) && - protectedPersonOverlap > 0 && - touchesProtectedPersonSide; - const allowSurfaceFill = !protectedPersonSide; - const isProtectedPersonSafetySidePixel = ( - pixelX: number, - pixelY: number - ) => { - if (protectedPersonSide === "right") { - return ( - (pixelX >= width * 0.7 && pixelY >= height * 0.38) || - (pixelX >= width * 0.62 && pixelY >= height * 0.56) || - (pixelX >= width * 0.48 && pixelY >= height * 0.7) - ); - } - if (protectedPersonSide === "left") { - return ( - (pixelX <= width * 0.3 && pixelY >= height * 0.38) || - (pixelX <= width * 0.38 && pixelY >= height * 0.56) || - (pixelX <= width * 0.52 && pixelY >= height * 0.7) - ); - } - if (protectedPersonSide === "auto") { - return ( - (touchesRight && - pixelX >= width * 0.72 && - pixelY >= height * 0.42) || - (touchesLeft && - pixelX <= width * 0.28 && - pixelY >= height * 0.42) - ); - } - return false; - }; - const isProtectedPersonPixel = (pixelX: number, pixelY: number) => { - const pixelIndex = pixelY * width + pixelX; - if (protectedPersonMask?.[pixelIndex]) return true; - if (isProtectedPersonSafetySidePixel(pixelX, pixelY)) return true; - if (!shouldClipProtectedPersonSide) return false; - if (protectedPersonSide === "right") return pixelX >= width * 0.78; - if (protectedPersonSide === "left") return pixelX <= width * 0.22; - if (protectedPersonSide === "auto") { - return ( - (touchesRight && pixelX >= width * 0.82) || - (touchesLeft && pixelX <= width * 0.18) - ); - } - return false; - }; - const broadTopBackground = - touchesTop && - centerYRatio <= 0.45 && - componentWidth >= width * 0.42 && - componentHeight >= height * 0.18 && - componentDensity >= 0.18; - const elongated = - componentHeight > componentWidth * 2.4 || - componentWidth > componentHeight * 2.4; - const likelyStructure = - area >= minComponentArea || - (elongated && area >= Math.max(8, minComponentArea * 0.4)); - const centeredEnough = centerXRatio >= 0.04 && centerXRatio <= 0.88; - const fillSurfaceComponent = - componentWidth >= width * 0.14 && - componentHeight >= height * 0.045 && - componentWidth <= width * 0.82 && - componentHeight <= height * 0.42 && - componentDensity >= 0.035 && - componentDensity <= 0.72; - - if ( - likelyStructure && - centeredEnough && - !edgeSliver && - !broadTopBackground - ) { - if (fillSurfaceComponent && allowSurfaceFill) { - for (let fillY = minY; fillY <= maxY; fillY += 1) { - for (let fillX = minX; fillX <= maxX; fillX += 1) { - if (isProtectedPersonPixel(fillX, fillY)) continue; - output[fillY * width + fillX] = 255; - } - } - } else { - for (let index = 0; index < queueX.length; index += 1) { - const pixelX = queueX[index] || 0; - const pixelY = queueY[index] || 0; - if (isProtectedPersonPixel(pixelX, pixelY)) continue; - output[pixelY * width + pixelX] = 255; - } - } - } - } - } - - return dilateScaleAdjustStructureMask({ - height, - mask: output, - radius: Math.max(1, Math.round(Math.min(width, height) * 0.006)), - width, - }); -} - -function solidifyScaleAdjustStructureMask(params: { - box: { width: number; height: number }; - mask: Uint8Array; - maxPanelSpanRatio?: number; -}) { - const { box, mask, maxPanelSpanRatio = 0.86 } = params; - const width = box.width; - const height = box.height; - const bridgeGap = Math.max(2, Math.round(Math.min(width, height) * 0.006)); - const output = dilateScaleAdjustStructureMask({ - height, - mask, - radius: Math.max(1, Math.round(Math.min(width, height) * 0.004)), - width, - }); - - for (let y = 0; y < height; y += 1) { - let x = 0; - while (x < width) { - while (x < width && output[y * width + x] !== 0) x += 1; - const gapStart = x; - while (x < width && output[y * width + x] === 0) x += 1; - const gapEnd = x - 1; - const gapLength = gapEnd - gapStart + 1; - if ( - gapLength > 0 && - gapLength <= bridgeGap && - gapStart > 0 && - gapEnd < width - 1 && - output[y * width + gapStart - 1] !== 0 && - output[y * width + gapEnd + 1] !== 0 - ) { - for (let fillX = gapStart; fillX <= gapEnd; fillX += 1) { - output[y * width + fillX] = 255; - } - } - } - } - - for (let x = 0; x < width; x += 1) { - let y = 0; - while (y < height) { - while (y < height && output[y * width + x] !== 0) y += 1; - const gapStart = y; - while (y < height && output[y * width + x] === 0) y += 1; - const gapEnd = y - 1; - const gapLength = gapEnd - gapStart + 1; - if ( - gapLength > 0 && - gapLength <= bridgeGap && - gapStart > 0 && - gapEnd < height - 1 && - output[(gapStart - 1) * width + x] !== 0 && - output[(gapEnd + 1) * width + x] !== 0 - ) { - for (let fillY = gapStart; fillY <= gapEnd; fillY += 1) { - output[fillY * width + x] = 255; - } - } - } - } - - const panelRowThreshold = Math.max(8, Math.round(width * 0.055)); - const panelMinHeight = Math.max(12, Math.round(height * 0.035)); - const panelMaxGap = Math.max(1, Math.round(height * 0.006)); - let panelStart = -1; - let panelGap = 0; - const getRowMarkedCount = (rowY: number) => { - let count = 0; - for (let rowX = 0; rowX < width; rowX += 1) { - if (output[rowY * width + rowX] !== 0) count += 1; - } - return count; - }; - const fillPanelBand = (startY: number, endY: number) => { - const bandHeight = endY - startY + 1; - if (bandHeight < panelMinHeight) return; - let minX = width; - let maxX = -1; - let marked = 0; - for (let bandY = startY; bandY <= endY; bandY += 1) { - for (let bandX = 0; bandX < width; bandX += 1) { - if (output[bandY * width + bandX] === 0) continue; - minX = Math.min(minX, bandX); - maxX = Math.max(maxX, bandX); - marked += 1; - } - } - if (maxX < minX) return; - const span = maxX - minX + 1; - const density = marked / Math.max(1, span * bandHeight); - if ( - span < width * 0.16 || - span > width * maxPanelSpanRatio || - density < 0.025 || - density > 0.78 - ) { - return; - } - for (let fillY = startY; fillY <= endY; fillY += 1) { - for (let fillX = minX; fillX <= maxX; fillX += 1) { - output[fillY * width + fillX] = 255; - } - } - }; - - for (let y = 0; y < height; y += 1) { - const hasPanelRow = getRowMarkedCount(y) >= panelRowThreshold; - if (hasPanelRow) { - if (panelStart < 0) panelStart = y; - panelGap = 0; - continue; - } - if (panelStart < 0) continue; - panelGap += 1; - if (panelGap <= panelMaxGap) continue; - fillPanelBand(panelStart, y - panelGap); - panelStart = -1; - panelGap = 0; - } - if (panelStart >= 0) { - fillPanelBand(panelStart, height - 1 - panelGap); - } - - return dilateScaleAdjustStructureMask({ - height, - mask: output, - radius: Math.max(1, Math.round(Math.min(width, height) * 0.004)), - width, - }); -} - -function getScaleAdjustMaskBoundingBox(params: { - height: number; - mask: Uint8Array; - width: number; -}) { - const { height, mask, width } = params; - let left = width; - let top = height; - let right = -1; - let bottom = -1; - let count = 0; - - for (let y = 0; y < height; y += 1) { - for (let x = 0; x < width; x += 1) { - if (mask[y * width + x] === 0) continue; - left = Math.min(left, x); - top = Math.min(top, y); - right = Math.max(right, x); - bottom = Math.max(bottom, y); - count += 1; - } - } - - if (right < left || bottom < top) return null; - return { - count, - left, - top, - width: right - left + 1, - height: bottom - top + 1, - }; -} - -function extractScaleAdjustMaskBuffer(params: { - box: { left: number; top: number; width: number; height: number }; - mask: Uint8Array; - maskWidth: number; -}) { - const { box, mask, maskWidth } = params; - const output = Buffer.alloc(box.width * box.height); - for (let y = 0; y < box.height; y += 1) { - const sourceStart = (box.top + y) * maskWidth + box.left; - const targetStart = y * box.width; - for (let x = 0; x < box.width; x += 1) { - output[targetStart + x] = mask[sourceStart + x] || 0; - } - } - return output; -} - -async function buildScaleAdjustMaskWashPatch(params: { - background: { b: number; g: number; r: number }; - box: { width: number; height: number }; - maskBuffer: Buffer; -}) { - const { background, box, maskBuffer } = params; - const alpha = await sharp(maskBuffer, { - raw: { channels: 1, height: box.height, width: box.width }, - }) - .greyscale() - .blur(Math.max(1.2, Math.min(box.width, box.height) * 0.018)) - .greyscale() - .raw() - .toBuffer(); - const rgba = Buffer.alloc(box.width * box.height * 4); - for (let index = 0; index < box.width * box.height; index += 1) { - const offset = index * 4; - rgba[offset] = Math.round(background.r); - rgba[offset + 1] = Math.round(background.g); - rgba[offset + 2] = Math.round(background.b); - rgba[offset + 3] = Math.min(255, Math.round((alpha[index] || 0) * 0.985)); - } - return sharp(rgba, { - raw: { channels: 4, height: box.height, width: box.width }, - }) - .png({ compressionLevel: 6 }) - .toBuffer(); -} - -async function buildScaleAdjustCompositeClipMaskPatch(params: { - box: { width: number; height: number }; - maskBuffer: Buffer | Uint8Array; -}) { - const { box, maskBuffer } = params; - const alpha = await sharp(Buffer.from(maskBuffer), { - raw: { channels: 1, height: box.height, width: box.width }, - }) - .greyscale() - .blur(Math.max(0.8, Math.min(box.width, box.height) * 0.006)) - .greyscale() - .raw() - .toBuffer(); - const rgba = Buffer.alloc(box.width * box.height * 4); - for (let index = 0; index < box.width * box.height; index += 1) { - const offset = index * 4; - rgba[offset] = 255; - rgba[offset + 1] = 255; - rgba[offset + 2] = 255; - rgba[offset + 3] = alpha[index] || 0; - } - return sharp(rgba, { - raw: { channels: 4, height: box.height, width: box.width }, - }) - .png({ compressionLevel: 6 }) - .toBuffer(); -} - -async function buildScaleAdjustOverlayClipMaskPatch(imageBuffer: Buffer) { - const metadata = await sharp(imageBuffer, { failOn: "none" }).metadata(); - const width = Math.max(1, Math.round(Number(metadata.width || 0))); - const height = Math.max(1, Math.round(Number(metadata.height || 0))); - const alpha = await sharp(imageBuffer, { failOn: "none" }) - .ensureAlpha() - .extractChannel(3) - .greyscale() - .blur(Math.max(0.8, Math.min(width, height) * 0.006)) - .greyscale() - .raw() - .toBuffer(); - const rgba = Buffer.alloc(width * height * 4); - for (let index = 0; index < width * height; index += 1) { - const offset = index * 4; - const value = alpha[index] || 0; - rgba[offset] = 255; - rgba[offset + 1] = 255; - rgba[offset + 2] = 255; - rgba[offset + 3] = value < 6 ? 0 : Math.min(255, Math.round(value * 1.35)); - } - return sharp(rgba, { - raw: { channels: 4, height, width }, - }) - .png({ compressionLevel: 6 }) - .toBuffer(); -} - -async function buildScaleAdjustCompositeClipMaskBuffer(params: { - height: number; - sourceMaskComposites: Array<{ input: Buffer; left: number; top: number }>; - targetMaskComposites: Array<{ input: Buffer; left: number; top: number }>; - width: number; -}) { - const { height, sourceMaskComposites, targetMaskComposites, width } = params; - if (sourceMaskComposites.length === 0 && targetMaskComposites.length === 0) { - return null; - } - - return sharp({ - create: { - background: { alpha: 0, b: 255, g: 255, r: 255 }, - channels: 4, - height, - width, - }, - }) - .composite([...sourceMaskComposites, ...targetMaskComposites]) - .png({ compressionLevel: 6 }) - .toBuffer(); -} - -async function buildScaleAdjustProductLockRepairBlendMask(params: { - height: number; - productLockOverlayBuffer: Buffer | null; - productLockProtectMaskBuffer: Buffer | null; - sourceRepairMaskBuffer: Buffer | null; - width: number; -}) { - const { - height, - productLockOverlayBuffer, - productLockProtectMaskBuffer, - sourceRepairMaskBuffer, - width, - } = params; - if (!sourceRepairMaskBuffer) return null; - - const repairAlpha = await sharp(sourceRepairMaskBuffer, { failOn: "none" }) - .rotate() - .resize(width, height, { fit: "fill" }) - .ensureAlpha() - .extractChannel(3) - .greyscale() - .raw() - .toBuffer(); - const overlayAlpha = productLockOverlayBuffer - ? await sharp(productLockOverlayBuffer, { failOn: "none" }) - .rotate() - .resize(width, height, { fit: "fill" }) - .ensureAlpha() - .extractChannel(3) - .greyscale() - .threshold(8) - .blur(Math.max(6, Math.round(Math.min(width, height) * 0.012))) - .greyscale() - .raw() - .toBuffer() - : null; - const protectAlpha = productLockProtectMaskBuffer - ? await sharp(productLockProtectMaskBuffer, { failOn: "none" }) - .rotate() - .resize(width, height, { fit: "fill" }) - .ensureAlpha() - .extractChannel(3) - .greyscale() - .threshold(8) - .blur(Math.max(4, Math.round(Math.min(width, height) * 0.008))) - .greyscale() - .raw() - .toBuffer() - : null; - - const rgba = Buffer.alloc(width * height * 4); - let markedPixels = 0; - for (let index = 0; index < width * height; index += 1) { - const offset = index * 4; - const protectedValue = Math.max( - overlayAlpha?.[index] || 0, - protectAlpha?.[index] || 0 - ); - const alpha = clamp((repairAlpha[index] || 0) - protectedValue, 0, 255); - if (alpha > 8) markedPixels += 1; - rgba[offset] = 255; - rgba[offset + 1] = 255; - rgba[offset + 2] = 255; - rgba[offset + 3] = alpha; - } - if (markedPixels === 0) return sourceRepairMaskBuffer; - - return sharp(rgba, { - raw: { channels: 4, height, width }, - }) - .png({ compressionLevel: 6 }) - .toBuffer(); -} - -async function buildScaleAdjustTransparentEditMaskFromOpacityMask(params: { - height: number; - maskBuffer: Buffer; - width: number; -}) { - const { height, maskBuffer, width } = params; - const editAlpha = await sharp(maskBuffer, { failOn: "none" }) - .rotate() - .resize(width, height, { fit: "fill" }) - .ensureAlpha() - .extractChannel(3) - .greyscale() - .raw() - .toBuffer(); - const rgba = Buffer.alloc(width * height * 4); - for (let index = 0; index < width * height; index += 1) { - const offset = index * 4; - rgba[offset] = 0; - rgba[offset + 1] = 0; - rgba[offset + 2] = 0; - rgba[offset + 3] = clamp(255 - (editAlpha[index] || 0), 0, 255); - } - return sharp(rgba, { - raw: { channels: 4, height, width }, - }) - .png({ compressionLevel: 9 }) - .toBuffer(); -} - -async function buildScaleAdjustOpaqueEditMaskFromOpacityMask(params: { - height: number; - maskBuffer: Buffer; - width: number; -}) { - const { height, maskBuffer, width } = params; - const editAlpha = await sharp(maskBuffer, { failOn: "none" }) - .rotate() - .resize(width, height, { fit: "fill" }) - .ensureAlpha() - .extractChannel(3) - .greyscale() - .raw() - .toBuffer(); - const rgba = Buffer.alloc(width * height * 4); - for (let index = 0; index < width * height; index += 1) { - const offset = index * 4; - const value = editAlpha[index] || 0; - rgba[offset] = value; - rgba[offset + 1] = value; - rgba[offset + 2] = value; - rgba[offset + 3] = 255; - } - return sharp(rgba, { - raw: { channels: 4, height, width }, - }) - .png({ compressionLevel: 9 }) - .toBuffer(); -} - -function getScaleAdjustAlphaBoundingBox(params: { - height: number; - rawRgba: Buffer; - threshold?: number; - width: number; -}) { - const { height, rawRgba, threshold = 18, width } = params; - let left = width; - let top = height; - let right = -1; - let bottom = -1; - let count = 0; - - for (let y = 0; y < height; y += 1) { - for (let x = 0; x < width; x += 1) { - const alpha = rawRgba[(y * width + x) * 4 + 3] || 0; - if (alpha < threshold) continue; - left = Math.min(left, x); - top = Math.min(top, y); - right = Math.max(right, x); - bottom = Math.max(bottom, y); - count += 1; - } - } - - if (right < left || bottom < top) return null; - return { - count, - left, - top, - width: right - left + 1, - height: bottom - top + 1, - }; -} - -function thresholdScaleAdjustAlphaMask(params: { - maskBuffer: Buffer | Uint8Array; - threshold?: number; -}) { - const { maskBuffer, threshold = 96 } = params; - const output = new Uint8Array(maskBuffer.length); - for (let index = 0; index < maskBuffer.length; index += 1) { - output[index] = (maskBuffer[index] || 0) >= threshold ? 255 : 0; - } - return output; -} - -function removeScaleAdjustMaskFromAlpha(params: { - alphaMask: Buffer | Uint8Array; - removeMask: Uint8Array | null; -}) { - const { alphaMask, removeMask } = params; - if (!removeMask || removeMask.length !== alphaMask.length) return alphaMask; - const output = Buffer.from(alphaMask); - for (let index = 0; index < output.length; index += 1) { - if (removeMask[index]) output[index] = 0; - } - return output; -} - -async function applyScaleAdjustAlphaMaskToRgba(params: { - alphaMask: Uint8Array; - height: number; - rawRgba: Buffer; - width: number; -}) { - const { alphaMask, height, rawRgba, width } = params; - const blur = Math.max(0.45, Math.min(width, height) * 0.0035); - const softenedAlpha = await sharp(Buffer.from(alphaMask), { - raw: { channels: 1, height, width }, - }) - .greyscale() - .blur(blur) - .greyscale() - .raw() - .toBuffer(); - const output = Buffer.from(rawRgba); - const pixelCount = Math.min(alphaMask.length, Math.floor(output.length / 4)); - for (let index = 0; index < pixelCount; index += 1) { - output[index * 4 + 3] = softenedAlpha[index] || 0; - } - return output; -} - -function getScaleAdjustAlphaBandingStats(params: { - height: number; - maskBuffer: Buffer | Uint8Array; - threshold?: number; - width: number; -}) { - const { height, maskBuffer, threshold = 24, width } = params; - const mask = thresholdScaleAdjustAlphaMask({ maskBuffer, threshold }); - const bbox = getScaleAdjustMaskBoundingBox({ height, mask, width }); - if (!bbox) { - return { - activeRowRatio: 0, - bboxHeightRatio: 0, - bboxWidthRatio: 0, - detected: false, - transitionRatio: 0, - wideRowRatio: 0, - }; - } - - let activeRows = 0; - let transitions = 0; - let wideRows = 0; - let previousCoverage: number | null = null; - for (let y = bbox.top; y < bbox.top + bbox.height; y += 1) { - let rowCount = 0; - for (let x = bbox.left; x < bbox.left + bbox.width; x += 1) { - if (mask[y * width + x] !== 0) rowCount += 1; - } - const coverage = rowCount / Math.max(1, bbox.width); - if (coverage >= 0.08) activeRows += 1; - if (coverage >= 0.45) wideRows += 1; - if ( - previousCoverage !== null && - Math.abs(coverage - previousCoverage) >= 0.28 - ) { - transitions += 1; - } - previousCoverage = coverage; - } - - const bboxHeightRatio = bbox.height / Math.max(1, height); - const bboxWidthRatio = bbox.width / Math.max(1, width); - const activeRowRatio = activeRows / Math.max(1, bbox.height); - const transitionRatio = transitions / Math.max(1, bbox.height - 1); - const wideRowRatio = wideRows / Math.max(1, bbox.height); - const detected = - (bboxWidthRatio >= 0.38 && - bboxHeightRatio >= 0.22 && - wideRowRatio >= 0.18 && - transitionRatio >= 0.12) || - (bboxWidthRatio >= 0.55 && - activeRowRatio >= 0.45 && - transitionRatio >= 0.09); - - return { - activeRowRatio: Number(activeRowRatio.toFixed(3)), - bboxHeightRatio: Number(bboxHeightRatio.toFixed(3)), - bboxWidthRatio: Number(bboxWidthRatio.toFixed(3)), - detected, - transitionRatio: Number(transitionRatio.toFixed(3)), - wideRowRatio: Number(wideRowRatio.toFixed(3)), - }; -} - -async function cleanScaleAdjustAlphaChannel(params: { - height: number; - referenceRawRgb?: Buffer | null; - rawRgba: Buffer; - threshold?: number; - width: number; -}) { - const { - height, - referenceRawRgb = null, - rawRgba, - threshold = SCALE_ADJUST_RMBG_ALPHA_CLEAN_THRESHOLD, - width, - } = params; - const pixelCount = Math.floor(rawRgba.length / 4); - const cleanedRaw = Buffer.from(rawRgba); - const cleanedAlpha = Buffer.alloc(pixelCount); - const range = Math.max(1, 255 - threshold); - const hasReference = - referenceRawRgb && referenceRawRgb.length >= pixelCount * 3; - - for (let index = 0; index < pixelCount; index += 1) { - const colorIndex = index * 4; - const alphaIndex = index * 4 + 3; - const sourceAlpha = rawRgba[alphaIndex] || 0; - let matteArtifact = false; - if (hasReference && sourceAlpha >= threshold) { - const r = rawRgba[colorIndex] || 0; - const g = rawRgba[colorIndex + 1] || 0; - const b = rawRgba[colorIndex + 2] || 0; - const luma = getScaleAdjustPixelLuma(r, g, b); - const channelSpread = Math.max(r, g, b) - Math.min(r, g, b); - const refIndex = index * 3; - const refLuma = getScaleAdjustPixelLuma( - referenceRawRgb?.[refIndex] || 0, - referenceRawRgb?.[refIndex + 1] || 0, - referenceRawRgb?.[refIndex + 2] || 0 - ); - matteArtifact = luma <= 18 && channelSpread <= 18 && refLuma >= 42; - } - const nextAlpha = - sourceAlpha < threshold || matteArtifact - ? 0 - : Math.max(0, Math.min(255, Math.round(((sourceAlpha - threshold) / range) * 255))); - cleanedRaw[alphaIndex] = nextAlpha; - cleanedAlpha[index] = nextAlpha; - } - - return { cleanedAlpha, cleanedRaw }; -} - -async function resizeScaleAdjustAlphaMask(params: { - height: number; - maskBuffer: Buffer; - targetHeight: number; - targetWidth: number; - width: number; -}) { - const { height, maskBuffer, targetHeight, targetWidth, width } = params; - const { data, info } = await sharp(maskBuffer, { - raw: { channels: 1, height, width }, - }) - .resize(targetWidth, targetHeight, { - background: { alpha: 1, b: 0, g: 0, r: 0 }, - fit: "contain", - kernel: sharp.kernel.lanczos3, - }) - .greyscale() - .raw() - .toBuffer({ resolveWithObject: true }); - if (info.channels === 1) return Buffer.from(data); - - const output = Buffer.alloc(targetWidth * targetHeight); - for (let index = 0; index < targetWidth * targetHeight; index += 1) { - output[index] = data[index * info.channels] || 0; - } - return output; -} - -async function buildScaleAdjustRmbgStructureLockedComposite(params: { - placement: ReturnType; - protectedPersonSide?: ScaleAdjustProtectedPersonSide | null; - raw: Buffer; - sourceBuffer: Buffer; - sourceImageUrl?: string; - sourceWidth: number; - userId?: string; -}): Promise { - const { - placement, - protectedPersonSide = null, - raw, - sourceBuffer, - sourceImageUrl, - sourceWidth, - userId, - } = params; - if (!userId) return null; - - try { - const sourceCropBuffer = await sharp(sourceBuffer, { failOn: "none" }) - .extract(placement.sourceBox) - .png({ compressionLevel: 6 }) - .toBuffer(); - const cutoutResult = await executeRmbgJob({ - userId, - imageBase64: sourceCropBuffer.toString("base64"), - imageBuffer: sourceCropBuffer, - prompt: protectedPersonSide - ? "比例调整家具/商品抠图：只保留框选区域内需要缩放的目标家具/商品主体和它本身的配件结构，明确排除非目标人物、手、脚、腿、衣服、书本、身体遮挡物和背景，输出透明背景 PNG" - : "比例调整主体抠图：只保留框选的目标人物/家具/商品主体，输出透明背景 PNG", - skipGallerySave: true, - sourceImageUrl, - }); - const cutoutBuffer = await fetchImageBufferFromUrl(cutoutResult.original_url); - const normalizedCutout = await sharp(cutoutBuffer, { failOn: "none" }) - .rotate() - .ensureAlpha() - .png({ compressionLevel: 6 }) - .toBuffer(); - const cutoutMetadata = await sharp(normalizedCutout, { failOn: "none" }).metadata(); - const cutoutWidth = Math.round(Number(cutoutMetadata.width || 0)); - const cutoutHeight = Math.round(Number(cutoutMetadata.height || 0)); - if (cutoutWidth <= 0 || cutoutHeight <= 0) return null; - const sameSizeCutout = - Math.abs(cutoutWidth - placement.sourceBox.width) <= 2 && - Math.abs(cutoutHeight - placement.sourceBox.height) <= 2; - const sourceCropRaw = sameSizeCutout - ? await sharp(sourceCropBuffer, { failOn: "none" }) - .removeAlpha() - .toColorspace("srgb") - .raw() - .toBuffer() - : null; - - const cutoutRaw = await sharp(normalizedCutout, { failOn: "none" }) - .ensureAlpha() - .raw() - .toBuffer(); - const { cleanedAlpha, cleanedRaw } = await cleanScaleAdjustAlphaChannel({ - height: cutoutHeight, - referenceRawRgb: sourceCropRaw, - rawRgba: cutoutRaw, - width: cutoutWidth, - }); - const background = estimateScaleAdjustBoxBackground({ - box: placement.sourceBox, - raw, - width: sourceWidth, - }); - const protectedPersonMask = - protectedPersonSide && sameSizeCutout && sourceCropRaw - ? validateScaleAdjustProtectedPersonMask({ - box: { width: cutoutWidth, height: cutoutHeight }, - mask: buildScaleAdjustProtectedPersonSeedMask({ - background, - box: { left: 0, top: 0, width: cutoutWidth, height: cutoutHeight }, - raw: sourceCropRaw, - side: protectedPersonSide, - width: cutoutWidth, - }), - }) - : null; - const protectedPersonRemovalMask = - protectedPersonMask || - (protectedPersonSide && sameSizeCutout - ? buildScaleAdjustProtectedPersonGeometryMask({ - box: { width: cutoutWidth, height: cutoutHeight }, - side: protectedPersonSide, - }) - : null); - const sideProtectedCleanedAlpha = - protectedPersonSide && sameSizeCutout && sourceCropRaw - ? suppressScaleAdjustProtectedFurnitureDarkAlpha({ - alphaMask: Buffer.from(cleanedAlpha), - bbox: { left: 0, top: 0, width: cutoutWidth, height: cutoutHeight }, - clearProtectedOcclusionZones: true, - protectedPersonSide, - rawRgb: sourceCropRaw, - sourceBox: placement.sourceBox, - }) - : cleanedAlpha; - const protectedCleanedAlpha = removeScaleAdjustMaskFromAlpha({ - alphaMask: sideProtectedCleanedAlpha, - removeMask: protectedPersonRemovalMask, - }); - for (let index = 0; index < protectedCleanedAlpha.length; index += 1) { - cleanedRaw[index * 4 + 3] = protectedCleanedAlpha[index] || 0; - } - const highConfidenceCutoutMask = thresholdScaleAdjustAlphaMask({ - maskBuffer: protectedCleanedAlpha, - threshold: 96, - }); - const highConfidenceCutoutBbox = getScaleAdjustMaskBoundingBox({ - height: cutoutHeight, - mask: highConfidenceCutoutMask, - width: cutoutWidth, - }); - if (!highConfidenceCutoutBbox) return null; - - const fullAlphaMask = Buffer.from(highConfidenceCutoutMask); - const sourceMaskBuffer = sameSizeCutout - ? fullAlphaMask - : await resizeScaleAdjustAlphaMask({ - height: cutoutHeight, - maskBuffer: fullAlphaMask, - targetHeight: placement.sourceBox.height, - targetWidth: placement.sourceBox.width, - width: cutoutWidth, - }); - const alphaBandingStats = getScaleAdjustAlphaBandingStats({ - height: placement.sourceBox.height, - maskBuffer: sourceMaskBuffer, - width: placement.sourceBox.width, - }); - if (alphaBandingStats.detected) { - console.warn( - "[Canvas Local Edit] Scale adjust RMBG alpha banding detected; fallback to pixel mask:", - alphaBandingStats - ); - return null; - } - const highConfidenceSourceMask = thresholdScaleAdjustAlphaMask({ - maskBuffer: sourceMaskBuffer, - threshold: 96, - }); - const filteredSourceMask = filterScaleAdjustStructureComponents({ - box: placement.sourceBox, - mask: highConfidenceSourceMask, - protectedPersonMask, - protectedPersonSide, - }); - const filteredSourceBbox = getScaleAdjustMaskBoundingBox({ - height: placement.sourceBox.height, - mask: filteredSourceMask, - width: placement.sourceBox.width, - }); - const overlayAlphaMask = - sameSizeCutout && filteredSourceBbox - ? filteredSourceMask - : highConfidenceCutoutMask; - const overlayAlphaBbox = getScaleAdjustMaskBoundingBox({ - height: cutoutHeight, - mask: overlayAlphaMask, - width: cutoutWidth, - }); - if (!overlayAlphaBbox) return null; - const refinedCutoutRaw = await applyScaleAdjustAlphaMaskToRgba({ - alphaMask: overlayAlphaMask, - height: cutoutHeight, - rawRgba: cleanedRaw, - width: cutoutWidth, - }); - const cleanedCutout = await sharp(refinedCutoutRaw, { - raw: { channels: 4, height: cutoutHeight, width: cutoutWidth }, - }) - .png({ compressionLevel: 6 }) - .toBuffer(); - const alphaBbox = getScaleAdjustAlphaBoundingBox({ - height: cutoutHeight, - rawRgba: refinedCutoutRaw, - width: cutoutWidth, - }); - if (!alphaBbox) return null; - - const alphaRatio = - overlayAlphaBbox.count / Math.max(1, cutoutWidth * cutoutHeight); - const bboxCoversCanvas = - overlayAlphaBbox.width >= cutoutWidth * 0.97 && - overlayAlphaBbox.height >= cutoutHeight * 0.97; - if ( - alphaRatio < SCALE_ADJUST_RMBG_MIN_ALPHA_RATIO || - (alphaRatio > SCALE_ADJUST_RMBG_MAX_SOLID_ALPHA_RATIO && bboxCoversCanvas) - ) { - return null; - } - const sourceEditBaseMask = filteredSourceBbox - ? filteredSourceMask - : highConfidenceSourceMask; - const sourceEditMaskBbox = getScaleAdjustMaskBoundingBox({ - height: placement.sourceBox.height, - mask: sourceEditBaseMask, - width: placement.sourceBox.width, - }); - if (!sourceEditMaskBbox) return null; - const sourceWashBaseMask = sourceEditBaseMask; - const sourceWashMask = dilateScaleAdjustStructureMask({ - height: placement.sourceBox.height, - mask: sourceWashBaseMask, - radius: Math.max( - 2, - Math.round( - Math.min(placement.sourceBox.width, placement.sourceBox.height) * 0.014 - ) - ), - width: placement.sourceBox.width, - }); - const sourceCompositeClipMask = dilateScaleAdjustStructureMask({ - height: placement.sourceBox.height, - mask: sourceEditBaseMask, - radius: Math.max( - 4, - Math.round( - Math.min(placement.sourceBox.width, placement.sourceBox.height) * - (protectedPersonSide ? 0.018 : 0.014) - ) - ), - width: placement.sourceBox.width, - }); - const sourceWashMaskBbox = getScaleAdjustMaskBoundingBox({ - height: placement.sourceBox.height, - mask: sourceWashMask, - width: placement.sourceBox.width, - }); - const sourceCompositeClipBbox = getScaleAdjustMaskBoundingBox({ - height: placement.sourceBox.height, - mask: sourceCompositeClipMask, - width: placement.sourceBox.width, - }); - const sourceWashPatch = await buildScaleAdjustMaskWashPatch({ - background, - box: placement.sourceBox, - maskBuffer: Buffer.from(sourceWashMask), - }); - const compositeClipMaskPatch = await buildScaleAdjustCompositeClipMaskPatch({ - box: placement.sourceBox, - maskBuffer: sourceCompositeClipMask, - }); - - const subjectCutout = await sharp(cleanedCutout, { failOn: "none" }) - .extract(alphaBbox) - .png({ compressionLevel: 6 }) - .toBuffer(); - const widthFactor = - placement.targetBox.width / Math.max(1, placement.sourceFootprintBox.width); - const heightFactor = - placement.targetBox.height / Math.max(1, placement.sourceFootprintBox.height); - const structureScale = Math.max( - 0.05, - Math.min( - 3.5, - Math.sqrt( - Math.max(0.0025, widthFactor) * Math.max(0.0025, heightFactor) - ) - ) - ); - let placedWidth = Math.max(1, Math.round(alphaBbox.width * structureScale)); - let placedHeight = Math.max(1, Math.round(alphaBbox.height * structureScale)); - if (!sameSizeCutout) { - const maxPlacedWidth = Math.max(1, Math.round(placement.targetBox.width * 1.06)); - const maxPlacedHeight = Math.max(1, Math.round(placement.targetBox.height * 1.06)); - const fitRatio = Math.min( - 1, - maxPlacedWidth / Math.max(1, placedWidth), - maxPlacedHeight / Math.max(1, placedHeight) - ); - placedWidth = Math.max(1, Math.round(placedWidth * fitRatio)); - placedHeight = Math.max(1, Math.round(placedHeight * fitRatio)); - } - - const sourceSubjectBox = sameSizeCutout - ? { - left: placement.sourceBox.left + alphaBbox.left, - top: placement.sourceBox.top + alphaBbox.top, - width: alphaBbox.width, - height: alphaBbox.height, - } - : null; - const sourceCenterX = sourceSubjectBox - ? sourceSubjectBox.left + sourceSubjectBox.width / 2 - : placement.sourceBox.left + placement.sourceBox.width / 2; - const sourceCenterY = sourceSubjectBox - ? sourceSubjectBox.top + sourceSubjectBox.height / 2 - : placement.sourceBox.top + placement.sourceBox.height / 2; - const targetCenterX = sourceSubjectBox - ? placement.targetBox.left + - ((sourceCenterX - placement.sourceFootprintBox.left) / - Math.max(1, placement.sourceFootprintBox.width)) * - placement.targetBox.width - : placement.targetBox.left + placement.targetBox.width / 2; - const targetCenterY = sourceSubjectBox - ? placement.targetBox.top + - ((sourceCenterY - placement.sourceFootprintBox.top) / - Math.max(1, placement.sourceFootprintBox.height)) * - placement.targetBox.height - : placement.targetBox.top + placement.targetBox.height / 2; - const structureOverlay = await sharp(subjectCutout, { failOn: "none" }) - .resize(placedWidth, placedHeight, { - fit: "fill", - kernel: sharp.kernel.lanczos3, - }) - .png({ compressionLevel: 6 }) - .toBuffer(); - const targetClipMaskPatch = - await buildScaleAdjustOverlayClipMaskPatch(structureOverlay); - const productLockProtectMask = - protectedPersonRemovalMask || - (protectedPersonSide - ? buildScaleAdjustProtectedPersonGeometryMask({ - box: placement.sourceBox, - side: protectedPersonSide, - }) - : null); - const productLockProtectMaskPatch = productLockProtectMask - ? await buildScaleAdjustCompositeClipMaskPatch({ - box: placement.sourceBox, - maskBuffer: dilateScaleAdjustStructureMask({ - height: placement.sourceBox.height, - mask: productLockProtectMask, - radius: Math.max( - 2, - Math.round( - Math.min(placement.sourceBox.width, placement.sourceBox.height) * - 0.012 - ) - ), - width: placement.sourceBox.width, - }), - }) - : null; - - console.info("[Canvas Local Edit] Scale adjust RMBG structure draft applied:", { - alphaRatio: Number(alphaRatio.toFixed(3)), - cutoutHeight, - cutoutWidth, - filteredAlphaRatio: Number( - (overlayAlphaBbox.count / Math.max(1, cutoutWidth * cutoutHeight)).toFixed(3) - ), - placedHeight, - placedWidth, - protectedPersonSide, - protectedPersonGeometryMaskApplied: - Boolean(protectedPersonRemovalMask) && !protectedPersonMask, - protectedPersonMaskApplied: Boolean(protectedPersonMask), - sameSizeCutout, - alphaBandingStats, - sourceClipAlphaRatio: Number( - ( - (sourceCompositeClipBbox?.count || 0) / - Math.max(1, placement.sourceBox.width * placement.sourceBox.height) - ).toFixed(3) - ), - sourceEditAlphaRatio: Number( - ( - sourceEditMaskBbox.count / - Math.max(1, placement.sourceBox.width * placement.sourceBox.height) - ).toFixed(3) - ), - sourceHeight: placement.sourceBox.height, - sourceWashAlphaRatio: Number( - ( - (sourceWashMaskBbox?.count || 0) / - Math.max(1, placement.sourceBox.width * placement.sourceBox.height) - ).toFixed(3) - ), - sourceWidth: placement.sourceBox.width, - }); - - return { - compositeClip: { - input: compositeClipMaskPatch, - left: placement.sourceBox.left, - top: placement.sourceBox.top, - }, - method: "rmbg-cutout", - overlay: { - input: structureOverlay, - left: Math.round(targetCenterX - placedWidth / 2), - top: Math.round(targetCenterY - placedHeight / 2), - }, - productLock: { - input: structureOverlay, - left: Math.round(targetCenterX - placedWidth / 2), - top: Math.round(targetCenterY - placedHeight / 2), - }, - ...(productLockProtectMaskPatch - ? { - productLockProtectMask: { - input: productLockProtectMaskPatch, - left: placement.sourceBox.left, - top: placement.sourceBox.top, - }, - } - : {}), - targetClip: { - input: targetClipMaskPatch, - left: Math.round(targetCenterX - placedWidth / 2), - top: Math.round(targetCenterY - placedHeight / 2), - }, - wash: { - input: sourceWashPatch, - left: placement.sourceBox.left, - top: placement.sourceBox.top, - }, - }; - } catch (error) { - console.warn("[Canvas Local Edit] Scale adjust RMBG structure draft failed; fallback to pixel mask:", { - error: error instanceof Error ? error.message : String(error), - }); - return null; - } -} - -function validateScaleAdjustProtectedPersonMask(params: { - box: { width: number; height: number }; - mask: Uint8Array | null; -}) { - const { box, mask } = params; - if (!mask) return null; - const maskBox = getScaleAdjustMaskBoundingBox({ - height: box.height, - mask, - width: box.width, - }); - if (!maskBox) return null; - - const maskRatio = maskBox.count / Math.max(1, box.width * box.height); - const fullSideStripe = - maskBox.height >= box.height * 0.92 && maskBox.width >= box.width * 0.28; - if (maskRatio > 0.26 || fullSideStripe) { - console.warn( - "[Canvas Local Edit] Scale adjust protected person mask rejected as broad background:", - { - heightRatio: Number((maskBox.height / Math.max(1, box.height)).toFixed(3)), - maskRatio: Number(maskRatio.toFixed(3)), - widthRatio: Number((maskBox.width / Math.max(1, box.width)).toFixed(3)), - } - ); - return null; - } - - return mask; -} - -function suppressScaleAdjustProtectedFurnitureDarkAlpha(params: { - alphaMask: Buffer; - bbox: { left: number; top: number; width: number; height: number }; - clearProtectedOcclusionZones?: boolean; - protectedPersonSide: ScaleAdjustProtectedPersonSide | null; - rawRgb: Buffer; - sourceBox: { width: number; height: number }; -}) { - const { - alphaMask, - bbox, - clearProtectedOcclusionZones = false, - protectedPersonSide, - rawRgb, - sourceBox, - } = params; - if (!protectedPersonSide) return alphaMask; - - const output = Buffer.from(alphaMask); - const isProtectedSideArea = (sourceX: number, sourceY: number) => { - if (protectedPersonSide === "right") { - return ( - (sourceX >= sourceBox.width * 0.72 && sourceY >= sourceBox.height * 0.22) || - (sourceX >= sourceBox.width * 0.58 && sourceY >= sourceBox.height * 0.68) - ); - } - if (protectedPersonSide === "left") { - return ( - (sourceX <= sourceBox.width * 0.28 && sourceY >= sourceBox.height * 0.22) || - (sourceX <= sourceBox.width * 0.42 && sourceY >= sourceBox.height * 0.68) - ); - } - return sourceY >= sourceBox.height * 0.3; - }; - const isFarProtectedSideArea = (sourceX: number, sourceY: number) => { - if (protectedPersonSide === "right") { - return sourceX >= sourceBox.width * 0.84 && sourceY >= sourceBox.height * 0.3; - } - if (protectedPersonSide === "left") { - return sourceX <= sourceBox.width * 0.16 && sourceY >= sourceBox.height * 0.3; - } - return sourceY >= sourceBox.height * 0.58; - }; - const isProtectedOcclusionArea = (sourceX: number, sourceY: number) => { - if (!clearProtectedOcclusionZones) return false; - if (protectedPersonSide === "right") { - return ( - (sourceX >= sourceBox.width * 0.69 && sourceY >= sourceBox.height * 0.54) || - (sourceX >= sourceBox.width * 0.58 && sourceY >= sourceBox.height * 0.7) || - (sourceX >= sourceBox.width * 0.5 && sourceY >= sourceBox.height * 0.74) || - (sourceX >= sourceBox.width * 0.56 && - sourceY >= sourceBox.height * 0.58 && - sourceY <= sourceBox.height * 0.78) || - (sourceX >= sourceBox.width * 0.78 && sourceY >= sourceBox.height * 0.4) || - (sourceX >= sourceBox.width * 0.91 && sourceY >= sourceBox.height * 0.26) - ); - } - if (protectedPersonSide === "left") { - return ( - (sourceX <= sourceBox.width * 0.31 && sourceY >= sourceBox.height * 0.54) || - (sourceX <= sourceBox.width * 0.42 && sourceY >= sourceBox.height * 0.7) || - (sourceX <= sourceBox.width * 0.5 && sourceY >= sourceBox.height * 0.74) || - (sourceX <= sourceBox.width * 0.44 && - sourceY >= sourceBox.height * 0.58 && - sourceY <= sourceBox.height * 0.78) || - (sourceX <= sourceBox.width * 0.22 && sourceY >= sourceBox.height * 0.4) || - (sourceX <= sourceBox.width * 0.09 && sourceY >= sourceBox.height * 0.26) - ); - } - return sourceY >= sourceBox.height * 0.68; - }; - - let clearedPixels = 0; - let markedPixels = 0; - for (let localY = 0; localY < bbox.height; localY += 1) { - for (let localX = 0; localX < bbox.width; localX += 1) { - const alphaIndex = localY * bbox.width + localX; - const alpha = output[alphaIndex] || 0; - if (alpha === 0) continue; - markedPixels += 1; - - const colorIndex = alphaIndex * 3; - const r = rawRgb[colorIndex] || 0; - const g = rawRgb[colorIndex + 1] || 0; - const b = rawRgb[colorIndex + 2] || 0; - const { luma, saturation } = getScaleAdjustPixelStats(r, g, b); - const sourceX = bbox.left + localX; - const sourceY = bbox.top + localY; - const inProtectedSideArea = isProtectedSideArea(sourceX, sourceY); - const farProtectedSideArea = isFarProtectedSideArea(sourceX, sourceY); - const likelyHumanPixel = - isScaleAdjustLikelySkinPixel(r, g, b) || - (isScaleAdjustLikelyWhiteFabricPixel(r, g, b, luma, saturation) && - luma < 246 && - saturation < 0.24); - const neutralDark = - luma < 150 && - (saturation < 0.55 || (r < 110 && g < 110 && b < 110)); - const lowContrastDark = luma < 132 && saturation < 0.32; - const shouldClear = - isProtectedOcclusionArea(sourceX, sourceY) || - (inProtectedSideArea && likelyHumanPixel) || - (farProtectedSideArea && - (likelyHumanPixel || - (neutralDark && luma < 72) || - (lowContrastDark && saturation < 0.18))); - if (!shouldClear) continue; - - output[alphaIndex] = 0; - clearedPixels += 1; - } - } - - if (markedPixels > 0) { - console.info( - "[Canvas Local Edit] Scale adjust protected furniture dark alpha cleared:", - { - clearedRatio: Number((clearedPixels / markedPixels).toFixed(3)), - clearedPixels, - protectedPersonSide, - totalMarkedPixels: markedPixels, - } - ); - } - - return output; -} - -async function buildScaleAdjustStructureLockedComposite(params: { - includePersonPixels?: boolean; - placement: ReturnType; - preferPixelStructure?: boolean; - protectedPersonSide?: ScaleAdjustProtectedPersonSide | null; - raw: Buffer; - sourceBuffer: Buffer; - sourceImageUrl?: string; - sourceWidth: number; - userId?: string; -}): Promise { - const { - includePersonPixels = false, - placement, - preferPixelStructure = false, - protectedPersonSide = null, - raw, - sourceBuffer, - sourceImageUrl, - sourceWidth, - userId, - } = params; - const allowRmbgStructureLock = - process.env.CANVAS_SCALE_ADJUST_ENABLE_RMBG_PRODUCT_LOCK !== "0" && - ((preferPixelStructure && Boolean(protectedPersonSide)) || - process.env.CANVAS_SCALE_ADJUST_ENABLE_RMBG_STRUCTURE === "1" || - (includePersonPixels && - process.env.CANVAS_SCALE_ADJUST_DISABLE_PERSON_RMBG !== "1")); - const rmbgComposite = !allowRmbgStructureLock - ? null - : await buildScaleAdjustRmbgStructureLockedComposite({ - placement, - protectedPersonSide, - raw, - sourceBuffer, - sourceImageUrl, - sourceWidth, - userId, - }); - if (rmbgComposite) return rmbgComposite; - if (includePersonPixels) return null; - - const initialMask = buildScaleAdjustStructureInitialMask({ - box: placement.sourceBox, - includePersonPixels, - raw, - width: sourceWidth, - }); - const protectedPersonMask = - preferPixelStructure && protectedPersonSide - ? validateScaleAdjustProtectedPersonMask({ - box: placement.sourceBox, - mask: buildScaleAdjustProtectedPersonSeedMask({ - box: placement.sourceBox, - raw, - side: protectedPersonSide, - width: sourceWidth, - }), - }) - : null; - const mask = filterScaleAdjustStructureComponents({ - box: placement.sourceBox, - mask: initialMask, - protectedPersonMask, - protectedPersonSide, - }); - const structureBbox = getScaleAdjustMaskBoundingBox({ - height: placement.sourceBox.height, - mask, - width: placement.sourceBox.width, - }); - if (!structureBbox) return null; - const maskRatio = structureBbox.count / Math.max(1, placement.sourceBox.width * placement.sourceBox.height); - if (maskRatio < SCALE_ADJUST_STRUCTURE_LOCK_MIN_MASK_RATIO) return null; - const subjectMask = protectedPersonSide - ? dilateScaleAdjustStructureMask({ - height: placement.sourceBox.height, - mask, - radius: Math.max( - 1, - Math.round( - Math.min(placement.sourceBox.width, placement.sourceBox.height) * - 0.004 - ) - ), - width: placement.sourceBox.width, - }) - : solidifyScaleAdjustStructureMask({ - box: placement.sourceBox, - mask, - }); - const bbox = getScaleAdjustMaskBoundingBox({ - height: placement.sourceBox.height, - mask: subjectMask, - width: placement.sourceBox.width, - }); - if (!bbox) return null; - - const sourceSubjectBox = { - left: placement.sourceBox.left + bbox.left, - top: placement.sourceBox.top + bbox.top, - width: bbox.width, - height: bbox.height, - }; - const sourceSubjectMask = extractScaleAdjustMaskBuffer({ - box: bbox, - mask: subjectMask, - maskWidth: placement.sourceBox.width, - }); - const background = estimateScaleAdjustBoxBackground({ - box: placement.sourceBox, - raw, - width: sourceWidth, - }); - const sourceWashMask = dilateScaleAdjustStructureMask({ - height: placement.sourceBox.height, - mask: protectedPersonSide - ? dilateScaleAdjustStructureMask({ - height: placement.sourceBox.height, - mask, - radius: Math.max( - 3, - Math.round( - Math.min(placement.sourceBox.width, placement.sourceBox.height) * - 0.034 - ) - ), - width: placement.sourceBox.width, - }) - : subjectMask, - radius: Math.max( - 2, - Math.round( - Math.min(placement.sourceBox.width, placement.sourceBox.height) * - (protectedPersonSide ? 0.02 : 0.012) - ) - ), - width: placement.sourceBox.width, - }); - const sourceCompositeClipMask = dilateScaleAdjustStructureMask({ - height: placement.sourceBox.height, - mask, - radius: Math.max( - 1, - Math.round(Math.min(placement.sourceBox.width, placement.sourceBox.height) * 0.003) - ), - width: placement.sourceBox.width, - }); - const sourceWashPatch = await buildScaleAdjustMaskWashPatch({ - background, - box: placement.sourceBox, - maskBuffer: Buffer.from(sourceWashMask), - }); - const compositeClipMaskPatch = await buildScaleAdjustCompositeClipMaskPatch({ - box: placement.sourceBox, - maskBuffer: sourceCompositeClipMask, - }); - const subjectCrop = await sharp(sourceBuffer, { failOn: "none" }) - .extract(sourceSubjectBox) - .removeAlpha() - .png({ compressionLevel: 6 }) - .toBuffer(); - const sourceSubjectRaw = await sharp(sourceBuffer, { failOn: "none" }) - .extract(sourceSubjectBox) - .removeAlpha() - .toColorspace("srgb") - .raw() - .toBuffer(); - const alphaMaskForSubject = - preferPixelStructure && protectedPersonSide - ? suppressScaleAdjustProtectedFurnitureDarkAlpha({ - alphaMask: Buffer.from(sourceSubjectMask), - bbox, - protectedPersonSide, - rawRgb: sourceSubjectRaw, - sourceBox: placement.sourceBox, - }) - : sourceSubjectMask; - const subjectAlpha = await sharp(alphaMaskForSubject, { - raw: { channels: 1, height: bbox.height, width: bbox.width }, - }) - .greyscale() - .blur(Math.max(0.7, Math.min(bbox.width, bbox.height) * 0.006)) - .greyscale() - .png({ compressionLevel: 6 }) - .toBuffer(); - const widthFactor = - placement.targetBox.width / Math.max(1, placement.sourceFootprintBox.width); - const heightFactor = - placement.targetBox.height / Math.max(1, placement.sourceFootprintBox.height); - const structureScale = Math.max( - 0.05, - Math.min( - 3.5, - Math.sqrt( - Math.max(0.0025, widthFactor) * Math.max(0.0025, heightFactor) - ) - ) - ); - const placedWidth = Math.max(1, Math.round(bbox.width * structureScale)); - const placedHeight = Math.max(1, Math.round(bbox.height * structureScale)); - const sourceCenterX = sourceSubjectBox.left + sourceSubjectBox.width / 2; - const sourceCenterY = sourceSubjectBox.top + sourceSubjectBox.height / 2; - const targetCenterX = - placement.targetBox.left + - ((sourceCenterX - placement.sourceFootprintBox.left) / - Math.max(1, placement.sourceFootprintBox.width)) * - placement.targetBox.width; - const targetCenterY = - placement.targetBox.top + - ((sourceCenterY - placement.sourceFootprintBox.top) / - Math.max(1, placement.sourceFootprintBox.height)) * - placement.targetBox.height; - const useRectCropTargetGuide = preferPixelStructure && Boolean(protectedPersonSide); - const structureOverlay = useRectCropTargetGuide - ? await sharp(sourceBuffer, { failOn: "none" }) - .extract(placement.sourceBox) - .resize(placement.placedWidth, placement.placedHeight, { - fit: "fill", - kernel: sharp.kernel.lanczos3, - }) - .ensureAlpha() - .composite([ - { - input: buildScaleAdjustPatchMaskSvg({ - height: placement.placedHeight, - width: placement.placedWidth, - }), - blend: "dest-in", - }, - ]) - .png({ compressionLevel: 6 }) - .toBuffer() - : await sharp(subjectCrop, { failOn: "none" }) - .joinChannel(subjectAlpha) - .resize(placedWidth, placedHeight, { - fit: "fill", - kernel: sharp.kernel.lanczos3, - }) - .png({ compressionLevel: 6 }) - .toBuffer(); - const targetClipMaskPatch = - await buildScaleAdjustOverlayClipMaskPatch(structureOverlay); - const overlayLeft = useRectCropTargetGuide - ? placement.placedLeft - : Math.round(targetCenterX - placedWidth / 2); - const overlayTop = useRectCropTargetGuide - ? placement.placedTop - : Math.round(targetCenterY - placedHeight / 2); - - return { - compositeClip: { - input: compositeClipMaskPatch, - left: placement.sourceBox.left, - top: placement.sourceBox.top, - }, - method: useRectCropTargetGuide ? "rect-crop-guide" : "pixel-mask", - overlay: { - input: structureOverlay, - left: overlayLeft, - top: overlayTop, - }, - targetClip: { - input: targetClipMaskPatch, - left: overlayLeft, - top: overlayTop, - }, - wash: { - input: sourceWashPatch, - left: placement.sourceBox.left, - top: placement.sourceBox.top, - }, - }; -} - -function getScaleAdjustEditMaskBoxes(params: { - regions: NormalizedLocalEditRegion[]; - scalePercent?: ScaleAdjustPercent | null; - sourceHeight: number; - sourceWidth: number; -}) { - const { regions, scalePercent, sourceHeight, sourceWidth } = params; - return regions.map((region) => { - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const workspaceRect = getScaleAdjustWorkspaceRect( - region.rect, - targetRect, - scalePercent - ); - return rectToPixels(workspaceRect, sourceWidth, sourceHeight); - }); -} - -async function buildScaleAdjustEditMaskBuffer(params: { - regions: NormalizedLocalEditRegion[]; - scalePercent?: ScaleAdjustPercent | null; - sourceHeight: number; - sourceWidth: number; -}) { - const { regions, scalePercent, sourceHeight, sourceWidth } = params; - const boxes = getScaleAdjustEditMaskBoxes({ - regions, - scalePercent, - sourceHeight, - sourceWidth, - }); - const minSide = Math.max(1, Math.min(sourceWidth, sourceHeight)); - const blur = Math.max(5, Math.min(28, Math.round(minSide * 0.018))); - const pad = Math.max(2, Math.round(blur * 0.55)); - const rects = boxes - .map((box) => { - const left = clamp(box.left - pad, 0, Math.max(0, sourceWidth - 1)); - const top = clamp(box.top - pad, 0, Math.max(0, sourceHeight - 1)); - const right = clamp( - box.left + box.width + pad, - left + 1, - sourceWidth - ); - const bottom = clamp( - box.top + box.height + pad, - top + 1, - sourceHeight - ); - return ``; - }) - .join(""); - const editMaskSvg = Buffer.from( - ` - - - - - - - ${rects} - ` - ); - const editAlpha = await sharp(editMaskSvg, { failOn: "none" }) - .resize(sourceWidth, sourceHeight, { fit: "fill" }) - .greyscale() - .raw() - .toBuffer(); - const rgba = Buffer.alloc(sourceWidth * sourceHeight * 4); - for (let index = 0; index < sourceWidth * sourceHeight; index += 1) { - const offset = index * 4; - const editable = editAlpha[index] || 0; - rgba[offset] = 0; - rgba[offset + 1] = 0; - rgba[offset + 2] = 0; - rgba[offset + 3] = clamp(255 - editable, 0, 255); - } - return sharp(rgba, { - raw: { channels: 4, height: sourceHeight, width: sourceWidth }, - }) - .png({ compressionLevel: 9 }) - .toBuffer(); -} - -async function buildScaleAdjustVisualLayoutGuideBuffer(params: { - regions: NormalizedLocalEditRegion[]; - scalePercent?: ScaleAdjustPercent | null; - sourceBuffer: Buffer; - sourceHeight: number; - sourceImageUrl?: string; - sourceWidth: number; - structureLockAppliedRef?: { value: boolean }; - structureLockedDraft?: boolean; - userId?: string; -}) { - const { - regions, - scalePercent, - sourceBuffer, - sourceHeight, - sourceImageUrl, - sourceWidth, - structureLockAppliedRef, - structureLockedDraft = false, - userId, - } = params; - const sourceRaw = - structureLockedDraft || regions.length > 0 - ? await sharp(sourceBuffer, { failOn: "none" }) - .removeAlpha() - .toColorspace("srgb") - .raw() - .toBuffer() - : null; - const sourceFootprintBoxes: Array<{ - background?: { b: number; g: number; r: number } | null; - innerOpacity?: number; - left: number; - opacity?: number; - top: number; - width: number; - height: number; - }> = []; - const placements: Array> = - []; - const composites: Array<{ input: Buffer; left: number; top: number }> = []; - const subjectWashComposites: Array<{ - input: Buffer; - left: number; - top: number; - }> = []; - const compositeClipComposites: Array<{ - input: Buffer; - left: number; - top: number; - }> = []; - const targetClipComposites: Array<{ - input: Buffer; - left: number; - top: number; - }> = []; - const productLockComposites: Array<{ - input: Buffer; - left: number; - top: number; - }> = []; - const productLockProtectMaskComposites: Array<{ - input: Buffer; - left: number; - top: number; - }> = []; - - for (const region of regions) { - const placement = getScaleAdjustVisualGuidePlacement({ - height: sourceHeight, - region, - scalePercent, - width: sourceWidth, - }); - placements.push(placement); - const targetWidthFactor = - placement.targetBox.width / Math.max(1, placement.sourceFootprintBox.width); - const targetHeightFactor = - placement.targetBox.height / Math.max(1, placement.sourceFootprintBox.height); - const shouldUseStructureLock = - structureLockedDraft && - sourceRaw && - (targetWidthFactor > 1.08 || - targetHeightFactor > 1.08 || - targetWidthFactor < 0.92 || - targetHeightFactor < 0.92); - const preferPixelStructure = isScaleAdjustFurnitureProductLikePrompt( - region.prompt - ); - const protectedPersonSide = preferPixelStructure - ? getScaleAdjustProtectedPersonSide(region.prompt) - : null; - const hasPersonPixels = - !preferPixelStructure && - !!sourceRaw && - hasScaleAdjustPersonPixelSignal({ - box: placement.sourceBox, - raw: sourceRaw, - width: sourceWidth, - }); - const includePersonPixels = - !preferPixelStructure && - (isScaleAdjustPersonLikePrompt(region.prompt) || hasPersonPixels); - const structureLockedComposite = shouldUseStructureLock - ? await buildScaleAdjustStructureLockedComposite({ - includePersonPixels, - placement, - preferPixelStructure, - protectedPersonSide, - raw: sourceRaw, - sourceBuffer, - sourceImageUrl, - sourceWidth, - userId, - }) - : null; - - if (structureLockedComposite) { - if (structureLockAppliedRef) { - structureLockAppliedRef.value = true; - } - subjectWashComposites.push(structureLockedComposite.wash); - if (structureLockedComposite.compositeClip) { - compositeClipComposites.push(structureLockedComposite.compositeClip); - } - if (structureLockedComposite.targetClip && !structureLockedComposite.productLock) { - targetClipComposites.push(structureLockedComposite.targetClip); - } - if (structureLockedComposite.productLock) { - productLockComposites.push(structureLockedComposite.productLock); - } - if (structureLockedComposite.productLockProtectMask) { - productLockProtectMaskComposites.push( - structureLockedComposite.productLockProtectMask - ); - } - composites.push(structureLockedComposite.overlay); - continue; - } - - const shouldUseShrinkCutoutDraft = - process.env.CANVAS_SCALE_ADJUST_ENABLE_RMBG_SHRINK_CUTOUT === "1" && - !structureLockedDraft && - sourceRaw && - userId && - (targetWidthFactor < 0.92 || targetHeightFactor < 0.92); - const shrinkCutoutComposite = shouldUseShrinkCutoutDraft - ? await buildScaleAdjustRmbgStructureLockedComposite({ - placement, - protectedPersonSide, - raw: sourceRaw, - sourceBuffer, - sourceImageUrl, - sourceWidth, - userId, - }) - : null; - - if (shrinkCutoutComposite) { - subjectWashComposites.push(shrinkCutoutComposite.wash); - if (shrinkCutoutComposite.compositeClip) { - compositeClipComposites.push(shrinkCutoutComposite.compositeClip); - } - if (shrinkCutoutComposite.targetClip && !shrinkCutoutComposite.productLock) { - targetClipComposites.push(shrinkCutoutComposite.targetClip); - } - if (shrinkCutoutComposite.productLock) { - productLockComposites.push(shrinkCutoutComposite.productLock); - } - if (shrinkCutoutComposite.productLockProtectMask) { - productLockProtectMaskComposites.push( - shrinkCutoutComposite.productLockProtectMask - ); - } - composites.push(shrinkCutoutComposite.overlay); - continue; - } - - sourceFootprintBoxes.push({ - ...placement.sourceFootprintBox, - background: sourceRaw - ? estimateScaleAdjustBoxBackground({ - box: placement.sourceFootprintBox, - raw: sourceRaw, - width: sourceWidth, - }) - : null, - }); - - const resizedCropBuffer = await sharp(sourceBuffer, { failOn: "none" }) - .extract(placement.sourceBox) - .resize(placement.placedWidth, placement.placedHeight, { - fit: "fill", - }) - .png({ compressionLevel: 6 }) - .toBuffer(); - const cropBuffer = await sharp(resizedCropBuffer, { failOn: "none" }) - .ensureAlpha() - .composite([ - { - input: buildScaleAdjustPatchMaskSvg({ - height: placement.placedHeight, - width: placement.placedWidth, - }), - blend: "dest-in", - }, - ]) - .png({ compressionLevel: 6 }) - .toBuffer(); - composites.push({ - input: cropBuffer, - left: placement.placedLeft, - top: placement.placedTop, - }); - } - - const washComposite = sourceFootprintBoxes.length - ? [ - { - input: buildScaleAdjustSourceFootprintWashSvg({ - boxes: sourceFootprintBoxes, - height: sourceHeight, - width: sourceWidth, - }), - left: 0, - top: 0, - }, - ] - : []; - const targetShadowComposite = placements.length - ? structureLockedDraft - ? [] - : [ - { - input: buildScaleAdjustTargetFootprintShadowSvg({ - height: sourceHeight, - placements, - width: sourceWidth, - }), - left: 0, - top: 0, - }, - ] - : []; - - const buffer = await sharp(sourceBuffer, { failOn: "none" }) - .composite([ - ...washComposite, - ...subjectWashComposites, - ...targetShadowComposite, - ...composites, - ]) - .png({ compressionLevel: 6 }) - .toBuffer(); - const backgroundPlateBuffer = productLockComposites.length - ? await sharp(sourceBuffer, { failOn: "none" }) - .composite([ - ...washComposite, - ...subjectWashComposites, - ]) - .png({ compressionLevel: 6 }) - .toBuffer() - : null; - const productLockOverlayBuffer = productLockComposites.length - ? await sharp({ - create: { - background: { alpha: 0, b: 0, g: 0, r: 0 }, - channels: 4, - height: sourceHeight, - width: sourceWidth, - }, - }) - .composite(productLockComposites) - .png({ compressionLevel: 6 }) - .toBuffer() - : null; - const productLockProtectMaskBuffer = productLockProtectMaskComposites.length - ? await sharp({ - create: { - background: { alpha: 0, b: 255, g: 255, r: 255 }, - channels: 4, - height: sourceHeight, - width: sourceWidth, - }, - }) - .composite(productLockProtectMaskComposites) - .png({ compressionLevel: 6 }) - .toBuffer() - : null; - const compositeClipMaskBuffer = - compositeClipComposites.length || targetClipComposites.length - ? await buildScaleAdjustCompositeClipMaskBuffer({ - height: sourceHeight, - sourceMaskComposites: compositeClipComposites, - targetMaskComposites: targetClipComposites, - width: sourceWidth, - }) - : null; - const repairBlendMaskBuffer = productLockComposites.length - ? await buildScaleAdjustProductLockRepairBlendMask({ - height: sourceHeight, - productLockOverlayBuffer, - productLockProtectMaskBuffer, - sourceRepairMaskBuffer: compositeClipMaskBuffer, - width: sourceWidth, - }) - : compositeClipMaskBuffer; - const productLockEditMaskBuffer = - productLockComposites.length && compositeClipMaskBuffer - ? await buildScaleAdjustTransparentEditMaskFromOpacityMask({ - height: sourceHeight, - maskBuffer: compositeClipMaskBuffer, - width: sourceWidth, - }) - : null; - const productLockFalEditMaskBuffer = - productLockComposites.length && compositeClipMaskBuffer - ? await buildScaleAdjustOpaqueEditMaskFromOpacityMask({ - height: sourceHeight, - maskBuffer: compositeClipMaskBuffer, - width: sourceWidth, - }) - : null; - - return { - backgroundPlateBuffer, - buffer, - compositeClipMaskBuffer, - repairBlendMaskBuffer, - productLockEditMaskBuffer, - productLockFalEditMaskBuffer, - productLockOverlayBuffer, - productLockProtectMaskBuffer, - }; -} - -async function createScaleAdjustPatchReference(params: { - fullFrameReferenceOnly?: boolean; - regions: NormalizedLocalEditRegion[]; - scalePercent?: ScaleAdjustPercent | null; - sourceBuffer: Buffer; - sourceHeight: number; - sourceImageUrl?: string; - sourceWidth: number; - userId?: string; -}): Promise { - const { - regions, - scalePercent, - sourceBuffer, - sourceHeight, - sourceImageUrl, - sourceWidth, - userId, - } = params; - const workspaceRect = getScaleAdjustPatchWorkspaceRect(regions, scalePercent); - const workspacePixelRect = rectToPixels(workspaceRect, sourceWidth, sourceHeight); - const shouldUseStructureLockedDraft = hasScaleAdjustStructureTarget( - regions, - scalePercent - ); - const structureLockAppliedRef = { value: false }; - const targetDraftReference = await buildScaleAdjustVisualLayoutGuideBuffer({ - regions, - scalePercent, - sourceBuffer, - sourceHeight, - sourceImageUrl, - sourceWidth, - structureLockAppliedRef, - structureLockedDraft: shouldUseStructureLockedDraft, - userId, - }); - const targetDraftBuffer = targetDraftReference.buffer; - const suffix = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; - const targetDraftUploadBuffer = await sharp(targetDraftBuffer, { failOn: "none" }) - .flatten({ background: "#ffffff" }) - .jpeg({ quality: 90, mozjpeg: true }) - .toBuffer(); - const backgroundPlateUploadBuffer = targetDraftReference.backgroundPlateBuffer - ? await sharp(targetDraftReference.backgroundPlateBuffer, { failOn: "none" }) - .flatten({ background: "#ffffff" }) - .jpeg({ quality: 90, mozjpeg: true }) - .toBuffer() - : null; - const uploadOptionalScaleAdjustAsset = async ( - filename: string, - buffer: Buffer | null | undefined - ) => { - return uploadScaleAdjustPrepAsset(filename, buffer); - }; - if (params.fullFrameReferenceOnly && structureLockAppliedRef.value) { - const sourceFullUploadBuffer = await sharp(sourceBuffer, { failOn: "none" }) - .rotate() - .flatten({ background: "#ffffff" }) - .jpeg({ quality: 94, mozjpeg: true }) - .toBuffer(); - const sourceFullUpload = await uploadScaleAdjustPrepAsset( - `canvas-scale-adjust-full-source-${suffix}.jpg`, - sourceFullUploadBuffer - ); - const targetDraftFullUpload = await uploadScaleAdjustPrepAsset( - `canvas-scale-adjust-full-target-draft-${suffix}.jpg`, - targetDraftUploadBuffer - ); - const fullFrameFallbackUrl = targetDraftFullUpload?.url || sourceImageUrl; - if (!fullFrameFallbackUrl) { - throw new Error("比例调整预处理参考图上传失败，请稍后重试"); - } - const productLockOverlayUpload = await uploadOptionalScaleAdjustAsset( - `canvas-scale-adjust-product-lock-overlay-${suffix}.png`, - targetDraftReference.productLockOverlayBuffer - ); - const [productLockEditMaskUpload, productLockFalEditMaskUpload] = - await Promise.all([ - uploadOptionalScaleAdjustAsset( - `canvas-scale-adjust-product-lock-edit-mask-${suffix}.png`, - targetDraftReference.productLockEditMaskBuffer - ), - uploadOptionalScaleAdjustAsset( - `canvas-scale-adjust-product-lock-fal-edit-mask-${suffix}.png`, - targetDraftReference.productLockFalEditMaskBuffer - ), - ]); - const [ - backgroundPlateFullUpload, - productLockProtectMaskUpload, - compositeClipMaskUpload, - repairBlendMaskUpload, - ] = - await Promise.all([ - uploadOptionalScaleAdjustAsset( - `canvas-scale-adjust-background-plate-${suffix}.jpg`, - backgroundPlateUploadBuffer - ), - uploadOptionalScaleAdjustAsset( - `canvas-scale-adjust-product-lock-protect-mask-${suffix}.png`, - targetDraftReference.productLockProtectMaskBuffer - ), - uploadOptionalScaleAdjustAsset( - `canvas-scale-adjust-composite-clip-mask-${suffix}.png`, - targetDraftReference.compositeClipMaskBuffer - ), - uploadOptionalScaleAdjustAsset( - `canvas-scale-adjust-repair-blend-mask-${suffix}.png`, - targetDraftReference.repairBlendMaskBuffer - ), - ]); - - return { - backgroundPlateCropUrl: null, - backgroundPlateFullUrl: backgroundPlateFullUpload?.url || null, - compositeClipMaskUrl: compositeClipMaskUpload?.url || null, - repairBlendMaskUrl: repairBlendMaskUpload?.url || null, - productLockMethod: productLockOverlayUpload ? "rmbg-cutout" : null, - productLockEditMaskUrl: productLockEditMaskUpload?.url || null, - productLockFalEditMaskUrl: productLockFalEditMaskUpload?.url || null, - productLockOverlayUrl: productLockOverlayUpload?.url || null, - productLockProtectMaskUrl: productLockProtectMaskUpload?.url || null, - sourceCropUrl: sourceFullUpload?.url || sourceImageUrl || fullFrameFallbackUrl, - structureLockedDraft: structureLockAppliedRef.value, - targetDraftFullUrl: fullFrameFallbackUrl, - targetDraftUrl: fullFrameFallbackUrl, - workspacePixelRect, - workspaceRect, - }; - } - - const [sourceCropBuffer, targetDraftCropBuffer] = await Promise.all([ - sharp(sourceBuffer, { failOn: "none" }) - .extract(workspacePixelRect) - .png({ compressionLevel: 6 }) - .toBuffer(), - sharp(targetDraftBuffer, { failOn: "none" }) - .extract(workspacePixelRect) - .png({ compressionLevel: 6 }) - .toBuffer(), - ]); - const backgroundPlateCropBuffer = targetDraftReference.backgroundPlateBuffer - ? await sharp(targetDraftReference.backgroundPlateBuffer, { failOn: "none" }) - .extract(workspacePixelRect) - .png({ compressionLevel: 6 }) - .toBuffer() - : null; - const compositeClipMaskCropBuffer = targetDraftReference.compositeClipMaskBuffer - ? await sharp(targetDraftReference.compositeClipMaskBuffer, { failOn: "none" }) - .extract(workspacePixelRect) - .png({ compressionLevel: 6 }) - .toBuffer() - : null; - const [sourceCropUpload, targetDraftUpload, targetDraftFullUpload] = - await Promise.all([ - uploadScaleAdjustPrepAsset( - `canvas-scale-adjust-workspace-source-${suffix}.png`, - sourceCropBuffer - ), - uploadScaleAdjustPrepAsset( - `canvas-scale-adjust-workspace-target-draft-${suffix}.png`, - targetDraftCropBuffer - ), - uploadScaleAdjustPrepAsset( - `canvas-scale-adjust-full-target-draft-${suffix}.jpg`, - targetDraftUploadBuffer - ), - ]); - const [ - backgroundPlateFullUpload, - backgroundPlateCropUpload, - productLockOverlayUpload, - productLockProtectMaskUpload, - compositeClipMaskUpload, - compositeClipMaskCropUpload, - repairBlendMaskUpload, - productLockEditMaskUpload, - productLockFalEditMaskUpload, - ] = await Promise.all([ - uploadOptionalScaleAdjustAsset( - `canvas-scale-adjust-background-plate-${suffix}.jpg`, - backgroundPlateUploadBuffer - ), - uploadOptionalScaleAdjustAsset( - `canvas-scale-adjust-background-plate-crop-${suffix}.png`, - backgroundPlateCropBuffer - ), - uploadOptionalScaleAdjustAsset( - `canvas-scale-adjust-product-lock-overlay-${suffix}.png`, - targetDraftReference.productLockOverlayBuffer - ), - uploadOptionalScaleAdjustAsset( - `canvas-scale-adjust-product-lock-protect-mask-${suffix}.png`, - targetDraftReference.productLockProtectMaskBuffer - ), - uploadOptionalScaleAdjustAsset( - `canvas-scale-adjust-composite-clip-mask-${suffix}.png`, - targetDraftReference.compositeClipMaskBuffer - ), - uploadOptionalScaleAdjustAsset( - `canvas-scale-adjust-composite-clip-mask-crop-${suffix}.png`, - compositeClipMaskCropBuffer - ), - uploadOptionalScaleAdjustAsset( - `canvas-scale-adjust-repair-blend-mask-${suffix}.png`, - targetDraftReference.repairBlendMaskBuffer - ), - uploadOptionalScaleAdjustAsset( - `canvas-scale-adjust-product-lock-edit-mask-${suffix}.png`, - targetDraftReference.productLockEditMaskBuffer - ), - uploadOptionalScaleAdjustAsset( - `canvas-scale-adjust-product-lock-fal-edit-mask-${suffix}.png`, - targetDraftReference.productLockFalEditMaskBuffer - ), - ]); - const sourceCropUrl = sourceCropUpload?.url || sourceImageUrl || targetDraftFullUpload?.url; - const targetDraftUrl = - targetDraftUpload?.url || targetDraftFullUpload?.url || sourceImageUrl; - if (!sourceCropUrl || !targetDraftUrl) { - throw new Error("比例调整预处理参考图上传失败，请稍后重试"); - } - - return { - backgroundPlateCropUrl: backgroundPlateCropUpload?.url || null, - backgroundPlateFullUrl: backgroundPlateFullUpload?.url || null, - compositeClipMaskCropUrl: compositeClipMaskCropUpload?.url || null, - compositeClipMaskUrl: compositeClipMaskUpload?.url || null, - repairBlendMaskUrl: repairBlendMaskUpload?.url || null, - productLockMethod: productLockOverlayUpload ? "rmbg-cutout" : null, - productLockEditMaskUrl: productLockEditMaskUpload?.url || null, - productLockFalEditMaskUrl: productLockFalEditMaskUpload?.url || null, - productLockOverlayUrl: productLockOverlayUpload?.url || null, - productLockProtectMaskUrl: productLockProtectMaskUpload?.url || null, - sourceCropUrl, - structureLockedDraft: structureLockAppliedRef.value, - targetDraftFullUrl: targetDraftFullUpload?.url || sourceImageUrl || null, - targetDraftUrl, - workspacePixelRect, - workspaceRect, - }; -} - function wrapText(text: string, maxChars: number, maxLines = 3) { const compact = text.replace(/\s+/g, " ").trim(); if (!compact) return []; @@ -4306,278 +475,6 @@ function buildAnnotationSvg(params: { `); } -function buildScaleAdjustAnnotationSvg(params: { - width: number; - height: number; - regions: NormalizedLocalEditRegion[]; - scalePercent?: ScaleAdjustPercent | null; -}) { - const { width, height, regions, scalePercent } = params; - const minSide = Math.max(1, Math.min(width, height)); - const strokeWidth = Math.max(2, Math.round(minSide * 0.0034)); - const fontSize = Math.max(16, Math.round(minSide * 0.018)); - const labelPadX = Math.round(fontSize * 0.45); - const labelPadY = Math.round(fontSize * 0.24); - const targetColor = "#111827"; - const sourceColor = "#9ca3af"; - const helperColor = "#4b5563"; - const label = (text: string, x: number, y: number, color: string) => { - const labelWidth = Math.round(text.length * fontSize * 0.62 + labelPadX * 2); - const labelHeight = Math.round(fontSize * 1.2 + labelPadY * 2); - const left = clamp(x, 6, Math.max(6, width - labelWidth - 6)); - const top = clamp(y, 6, Math.max(6, height - labelHeight - 6)); - return ` - - ${escapeXml(text)} - `; - }; - - const regionSvg = regions - .map((region) => { - const source = region.pixelRect; - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const target = - region.targetPixelRect || rectToPixels(targetRect, width, height); - const radius = Math.max(8, Math.round(strokeWidth * 2)); - const sourceCenterX = source.left + source.width / 2; - const sourceCenterY = source.top + source.height / 2; - const targetCenterX = target.left + target.width / 2; - const targetCenterY = target.top + target.height / 2; - const widthFactor = source.width > 0 ? target.width / source.width : 1; - const heightFactor = source.height > 0 ? target.height / source.height : 1; - const isShrink = widthFactor < 0.92 || heightFactor < 0.92; - const isEnlarge = widthFactor > 1.08 || heightFactor > 1.08; - const actionLabel = isShrink ? "SHRINK" : isEnlarge ? "ENLARGE" : "MOVE"; - const scaleLabel = `${actionLabel} ${widthFactor.toFixed(2)}W ${heightFactor.toFixed(2)}H`; - const instructionLabel = isShrink - ? "OLD OUTSIDE -> BG" - : isEnlarge - ? "TARGET = FINAL FOOTPRINT" - : "FINAL SUBJECT CENTER"; - const centerMoved = - Math.hypot(targetCenterX - sourceCenterX, targetCenterY - sourceCenterY) > - Math.max(8, minSide * 0.012); - - return ` - - - - - - - ${ - centerMoved - ? `` - : "" - } - ${label(`OLD ${region.index}`, source.left + strokeWidth, source.top + strokeWidth, sourceColor)} - ${label(`TARGET ${region.index}`, target.left + strokeWidth, target.top + strokeWidth, targetColor)} - ${label(scaleLabel, target.left + strokeWidth, target.top + target.height + strokeWidth * 2, helperColor)} - ${label(instructionLabel, source.left + strokeWidth, source.top + source.height + strokeWidth * 2, helperColor)} - `; - }) - .join(""); - - return Buffer.from(` - - - - - - - - - ${regionSvg} - - `); -} - -function buildScaleAdjustZoomedCoordinateSvg(params: { - width: number; - height: number; - regions: NormalizedLocalEditRegion[]; - scalePercent?: ScaleAdjustPercent | null; -}) { - const { width, height, regions, scalePercent } = params; - const minSide = Math.max(1, Math.min(width, height)); - const strokeWidth = Math.max(3, Math.round(minSide * 0.006)); - const fontSize = Math.max(22, Math.round(minSide * 0.026)); - const smallFontSize = Math.max(18, Math.round(fontSize * 0.72)); - const panelMarginX = Math.round(width * 0.1); - const panelMarginY = Math.round(height * 0.12); - const panelLeft = panelMarginX; - const panelTop = panelMarginY; - const panelWidth = Math.max(1, width - panelMarginX * 2); - const panelHeight = Math.max(1, height - panelMarginY * 2); - const sourceColor = "#9ca3af"; - const targetColor = "#111827"; - const helperColor = "#4b5563"; - const padX = Math.round(fontSize * 0.42); - const labelHeight = Math.round(fontSize * 1.35); - const label = (text: string, x: number, y: number, color: string) => { - const labelWidth = Math.min( - Math.round(text.length * fontSize * 0.58 + padX * 2), - Math.max(160, width - 16) - ); - const left = clamp(x, 8, Math.max(8, width - labelWidth - 8)); - const top = clamp(y, 8, Math.max(8, height - labelHeight - 8)); - return ` - - ${escapeXml(text)} - `; - }; - - const regionSvg = regions - .map((region) => { - const source = region.pixelRect; - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const target = - region.targetPixelRect || rectToPixels(targetRect, width, height); - const sourceRight = source.left + source.width; - const sourceBottom = source.top + source.height; - const targetRight = target.left + target.width; - const targetBottom = target.top + target.height; - const unionLeft = Math.min(source.left, target.left); - const unionTop = Math.min(source.top, target.top); - const unionRight = Math.max(sourceRight, targetRight); - const unionBottom = Math.max(sourceBottom, targetBottom); - const marginX = Math.max(12, Math.round((unionRight - unionLeft) * 0.35)); - const marginY = Math.max(12, Math.round((unionBottom - unionTop) * 0.35)); - const viewLeft = unionLeft - marginX; - const viewTop = unionTop - marginY; - const viewWidth = Math.max(1, unionRight - unionLeft + marginX * 2); - const viewHeight = Math.max(1, unionBottom - unionTop + marginY * 2); - const scale = Math.min( - panelWidth / viewWidth, - panelHeight / viewHeight - ); - const drawnWidth = viewWidth * scale; - const drawnHeight = viewHeight * scale; - const originX = panelLeft + (panelWidth - drawnWidth) / 2 - viewLeft * scale; - const originY = panelTop + (panelHeight - drawnHeight) / 2 - viewTop * scale; - const project = (box: { left: number; top: number; width: number; height: number }) => ({ - left: Math.round(originX + box.left * scale), - top: Math.round(originY + box.top * scale), - width: Math.max(1, Math.round(box.width * scale)), - height: Math.max(1, Math.round(box.height * scale)), - }); - const sourceBox = project(source); - const targetBox = project(target); - const sourceCenterX = sourceBox.left + sourceBox.width / 2; - const sourceCenterY = sourceBox.top + sourceBox.height / 2; - const targetCenterX = targetBox.left + targetBox.width / 2; - const targetCenterY = targetBox.top + targetBox.height / 2; - const widthFactor = source.width > 0 ? target.width / source.width : 1; - const heightFactor = source.height > 0 ? target.height / source.height : 1; - const actionLabel = - widthFactor < 0.92 || heightFactor < 0.92 - ? "SHRINK" - : widthFactor > 1.08 || heightFactor > 1.08 - ? "ENLARGE" - : "MOVE"; - const scaleLabel = `${actionLabel} ${widthFactor.toFixed(2)}W ${heightFactor.toFixed(2)}H`; - const radius = Math.max(10, Math.round(strokeWidth * 2.2)); - const centerMoved = - Math.hypot(targetCenterX - sourceCenterX, targetCenterY - sourceCenterY) > - Math.max(10, minSide * 0.018); - - return ` - - - - - ${ - centerMoved - ? `` - : "" - } - ${label(`OLD ${region.index}`, sourceBox.left, sourceBox.top - labelHeight - strokeWidth, sourceColor)} - ${label(`TARGET ${region.index}`, targetBox.left, targetBox.top + strokeWidth, targetColor)} - ${label(scaleLabel, targetBox.left, targetBox.top + targetBox.height + strokeWidth * 2, helperColor)} - ${label("OLD OUTSIDE -> BG", sourceBox.left, sourceBox.top + sourceBox.height + strokeWidth * 4 + labelHeight, helperColor)} - `; - }) - .join(""); - - return Buffer.from(` - - - - - - - - - LOCAL SCALE MAP - USE PROMPT PIXELS FOR ABSOLUTE POSITION - ${regionSvg} - - `); -} - -function buildScaleAdjustMarkedPhotoGuideSvg(params: { - width: number; - height: number; - regions: NormalizedLocalEditRegion[]; - scalePercent?: ScaleAdjustPercent | null; -}) { - const { width, height, regions, scalePercent } = params; - const minSide = Math.max(1, Math.min(width, height)); - const strokeWidth = Math.max(3, Math.round(minSide * 0.005)); - const currentColor = "#00b8d4"; - const finalColor = "#2563eb"; - - const regionSvg = regions - .map((region) => { - const source = region.pixelRect; - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const target = - region.targetPixelRect || rectToPixels(targetRect, width, height); - const radius = Math.max(8, Math.round(strokeWidth * 2)); - return ` - - - - - `; - }) - .join(""); - - return Buffer.from(` - - - ${regionSvg} - - `); -} - -async function buildScaleAdjustMarkedPhotoGuideBuffer(params: { - sourceBuffer: Buffer; - width: number; - height: number; - regions: NormalizedLocalEditRegion[]; - scalePercent?: ScaleAdjustPercent | null; -}) { - const { sourceBuffer, width, height, regions, scalePercent } = params; - return sharp(sourceBuffer, { failOn: "none" }) - .rotate() - .resize(width, height, { fit: "cover", position: "center" }) - .modulate({ brightness: 1.04, saturation: 0.45 }) - .composite([ - { - input: buildScaleAdjustMarkedPhotoGuideSvg({ - height, - regions, - scalePercent, - width, - }), - left: 0, - top: 0, - }, - ]) - .png({ compressionLevel: 6 }) - .toBuffer(); -} - function buildLocalEditPrompt(regions: NormalizedLocalEditRegion[]) { const regionLines = regions .map((region) => { @@ -4621,623 +518,57 @@ function buildLocalEditPrompt(regions: NormalizedLocalEditRegion[]) { ].join("\n"); } -function buildScaleAdjustPrompt( +function buildSimpleScaleAdjustPrompt( regions: NormalizedLocalEditRegion[], scalePercent: ScaleAdjustPercent | null | undefined, - cropReferences: ScaleAdjustCropReference[] = [], - sourceSize?: { width: number; height: number } + sourceSize: { width: number; height: number } ) { - const sourceSizeLine = sourceSize - ? `最终输出必须保持与输入图片1完全相同的画幅和像素尺寸：${sourceSize.width} x ${sourceSize.height} px。` - : "最终输出必须保持与输入图片1完全相同的画幅和像素尺寸。"; const regionLines = regions .map((region) => { - const cropIndex = cropReferences.findIndex( - (reference) => reference.regionIndex === region.index - ); const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const sourceCenterX = region.rect.x + region.rect.width / 2; - const sourceCenterY = region.rect.y + region.rect.height / 2; - const targetCenterX = targetRect.x + targetRect.width / 2; - const targetCenterY = targetRect.y + targetRect.height / 2; + const targetPixelRect = + region.targetPixelRect || + rectToPixels(targetRect, sourceSize.width, sourceSize.height); + const sourceRight = region.pixelRect.left + region.pixelRect.width; + const sourceBottom = region.pixelRect.top + region.pixelRect.height; + const targetRight = targetPixelRect.left + targetPixelRect.width; + const targetBottom = targetPixelRect.top + targetPixelRect.height; + const widthFactor = + region.rect.width > 0 ? targetRect.width / region.rect.width : 1; + const heightFactor = + region.rect.height > 0 ? targetRect.height / region.rect.height : 1; const moved = - Math.hypot(targetCenterX - sourceCenterX, targetCenterY - sourceCenterY) > - 0.01; - const targetPixelRect = - region.targetPixelRect || - (sourceSize - ? rectToPixels(targetRect, sourceSize.width, sourceSize.height) - : null); - const widthFactor = - region.rect.width > 0 ? targetRect.width / region.rect.width : 1; - const heightFactor = - region.rect.height > 0 ? targetRect.height / region.rect.height : 1; - const sourcePixelLine = `源框像素坐标：x=${region.pixelRect.left}, y=${region.pixelRect.top}, w=${region.pixelRect.width}, h=${region.pixelRect.height}`; - const targetPixelLine = targetPixelRect - ? `目标框像素坐标：x=${targetPixelRect.left}, y=${targetPixelRect.top}, w=${targetPixelRect.width}, h=${targetPixelRect.height}` - : `目标框相对坐标：x=${Math.round(targetRect.x * 1000) / 10}%, y=${Math.round(targetRect.y * 1000) / 10}%, w=${Math.round(targetRect.width * 1000) / 10}%, h=${Math.round(targetRect.height * 1000) / 10}%`; - const scaleLine = `目标视觉占位约为原主体宽度的 ${widthFactor.toFixed(2)} 倍、高度的 ${heightFactor.toFixed(2)} 倍；执行时必须以等比主体缩放为准，不要拉伸主体去硬贴目标框。`; - const targetPositionNote = moved - ? "目标框同时指定用户希望的最终目标大小和目标位置，目标主体调整后的视觉中心应移动到目标框中心附近。" - : "目标框指定用户希望的最终目标大小，目标主体调整后应围绕当前主体位置自然放置。"; - const cropNote = - cropIndex >= 0 - ? `；输入图片${cropIndex + 3}是该区域的干净局部裁切参考图，只用于识别目标主体的身份、轮廓与原始比例` - : ""; - return `区域 ${region.index}${cropNote}：请在输入图片1中找到源框范围内最主要、可独立调整的商品/物体主体。${sourcePixelLine}；${targetPixelLine}；${scaleLine}${targetPositionNote}源框只是“寻找主体”的范围，不是最终边界，也不是要缩放或贴回的矩形补丁。目标框只是主体调整后的视觉占位参考，不要求主体填满矩形，更不能改变主体自身宽高比、透视或形体。请像真实图片编辑一样先保留锁定的原始画布，再只把这个主体整体等比变大/变小/移动到目标视觉占位；主体缩放后露出或遮挡的极小背景区域需要自然重建。严禁把输入图片1整张图、输入图片2目标布局参考图、任意裁切图、源框矩形区域或目标框矩形区域当成图片块整体缩放、平移、复制、裁切或贴回。如果源框中只覆盖到主体的一部分，请根据局部裁切参考和可见结构推断该主体的完整可见轮廓，只调整主体，不调整矩形框内背景。`; - }) - .join("\n"); - const cropLines = cropReferences - .map( - (reference, index) => - `输入图片${index + 3}：区域 ${reference.regionIndex} 的原图干净局部裁切参考图，只用于确认要缩放的主体身份、轮廓和宽高比，不能当成输出底图，也不能把裁切图整体贴回。` - ) - .join("\n"); - - return [ - "这是一项局部主体比例 inpaint/edit 任务，不是整图缩放、整图裁切或重新构图任务。", - sourceSizeLine, - "输入图片1是原始干净图，也是最终输出必须继承的唯一画面基底。除目标主体尺寸变化与必要背景补全外，输入图片1中的文字、Logo、装饰、排版、背景、光线、构图边界和所有未选区域必须保持不变。", - "输入图片2是一张与输入图片1尺寸和画幅完全相同的完整画面编辑意图图，不是输出底图。它保留原图完整画幅作为稳定参考，并把源框旧主体占位强淡化为“这里的原尺寸主体需要被目标尺寸替换并自然修复背景”的指令；同时把同一目标主体粗略缩放后放在用户希望的最终位置和大小。请只读取目标主体的最终视觉占位、中心位置和原画幅关系；输入图片2里被淡化的旧主体不能表示最终还要保留原尺寸主体。不得学习、复刻或输出淡化痕迹、阴影占位、矩形裁切边缘、拼接痕迹或参考图粗糙画质。", - cropLines - ? `输入图片3及之后是框选区域的干净局部裁切参考图，作用是帮助你确认“要缩放的是哪个主体”。这些裁切图不是输出底图，不得被整体放大、贴回或覆盖到原图上。\n${cropLines}` - : "", - "输入图片2中的完整画面编辑意图是强约束：淡化的 OLD 占位表示旧尺寸主体要被移除/修复，清晰的 TARGET 占位表示最终主体大小和中心位置；整张图不应缩放、裁切或平移。最终输出必须回到输入图片1的真实照片环境中完成自然编辑，不得直接粘贴 input2 的矩形图块；主体不需要填满目标矩形，必须保持主体自身原始宽高比和透视形体。", - "如果系统或模型只能读取一张参考图，必须优先以输入图片1为输出基底，不得清理、改写或删除输入图片1中框选区域外的任何文字和图形。", - "系统已根据用户框选区域计算出下方每个区域的原图坐标；框选区域只是定位目标主体的搜索范围，不是最终画面的裁切块、贴片、蒙版边界，也不是放大后的空间上限，更不是要整体缩放的矩形图层。", - "最终结果必须保持原始干净图的画幅比例、构图边界、镜头透视、光线、色温、材质质感和清晰度；严禁整张图整体缩放、裁切、平移、重新排版或重新生成。", - "Highest priority rule: do NOT zoom, crop, resize, reframe, pan, or regenerate the full canvas. Treat image 1 as a locked background plate; edit only the selected subject and the smallest necessary background around its old and new footprint.", - "请不要生成任何彩色框、矩形边框、箭头、连线、角点、编号圆点、提示词气泡、标注文字、UI 控件或半透明色块。", - "请同时使用下方每个区域的源框/目标框像素坐标和输入图片2的目标布局参考来理解目标占位。坐标和参考图只用于指导主体位置和大小，不是要求把某个矩形图块缩放后贴回。", - "", - "任务：先在输入图片1中识别源框范围内最主要、可独立调整的主体物体，再只对这个“主体物体”做局部比例调整。调整后的主体大小和位置必须接近目标框表示的视觉占位；不接受看起来和原图几乎相同的输出。", - regionLines, - "", - "关键要求：", - "1. 只调整目标主体物体本身的整体视觉比例，不得保留原尺寸；主体轮廓宽度和高度必须同步等比变化，宽高比必须与原主体一致，不能为了贴合目标框而拉宽、压窄、拉高或压扁。", - "2. 严禁把整个矩形框选区域、输入图片1整张图或任意裁切图当成图片块整体缩放、平移、复制、裁切或贴回；最终画面不得出现矩形拼接边缘、色差硬边、补丁感、裁切线或局部贴片痕迹。", - "2.1 严禁把输入图片1整张图整体放大、缩小、裁切或重新构图；画面中的标题、Logo、页面文字、边框、窗框、背景家具或非目标主体必须作为像素锚点保持原位置和原比例，不能跟着目标主体变化。", - "3. 如果输入图片2中的目标框明显小于源框，目标主体应自然变小，主体缩小后露出的背景、桌面、墙面、阴影、纹理和透视必须被重新生成并无缝补全，选区内应出现明显新增留白或背景。", - "4. 如果输入图片2中的目标框明显大于源框，目标主体应自然变大；允许主体自然超出用户手画的原始源框，使用周围真实背景完成边缘、阴影和遮挡衔接。宽度和高度都要同步变大，不能出现“高度变高但宽度变窄”的非等比变形；主体边缘与遮挡关系要合理，不能产生突兀裁切。", - "5. 如果目标框中心与源框中心不同，允许主体同步移动到目标框中心附近；如果主体靠近画布边界导致目标空间不足，请用能完整放入画面的最大统一倍率整体调整，绝不能为了塞进画面而只压缩宽度或只拉伸高度。", - "6. 框选区域内非目标主体的背景、墙面、桌面、植物、文字和其它物体应保持原场景逻辑，不要跟着主体一起缩放。", - "6.1 比例调整后的主体边缘必须和周围环境自然融合，选区边界不能被肉眼看出；不得出现一整块矩形区域清晰度、颜色、光照、纹理或透视突然变化。", - "6.2 框选区域通常会包含主体周围的背景留白，背景留白不是目标主体。请锁定这些非主体背景的窗框、文字、地毯纹理、墙面、桌面边缘和透视线，只重建主体缩放后被遮挡或露出的极小必要区域；不要把框选矩形当作整块背景重新生成。", - "6.3 如果框选范围内没有清晰、完整、可独立缩放的主体物体，请保持该区域背景结构稳定，不要为了制造变化而重绘窗框、文字、墙面、地面或其它非主体元素。", - "7. 未受比例调整影响的其它画面区域必须保持原图不变，尤其是框选区域外的文字、Logo、标题、尺寸标注、装饰条和页面排版不得被清理、改写、模糊或重绘。", - "8. 最终输出必须是一张完整、干净、无缝的照片，不得出现任何选区框、说明文字、对比线、比例数字或 UI 元素。", - "9. 输出前自检：如果目标主体大小与原图相比没有明显变化，或者主体宽高比发生非等比变形，必须重新生成；禁止返回看起来和原图相同或被拉伸的结果。", - ].join("\n"); -} - -function buildScaleAdjustCoordinatePrompt( - regions: NormalizedLocalEditRegion[], - scalePercent: ScaleAdjustPercent | null | undefined, - sourceSize: { width: number; height: number }, - cropReferences: ScaleAdjustCropReference[] = [], - options: { - falSafe?: boolean; - markedPhotoGuide?: boolean; - visualLayoutGuide?: boolean; - } = {} -) { - const regionLines = regions - .map((region) => { - const cropIndex = cropReferences.findIndex( - (reference) => reference.regionIndex === region.index - ); - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const targetPixelRect = - region.targetPixelRect || - rectToPixels(targetRect, sourceSize.width, sourceSize.height); - const widthFactor = - region.rect.width > 0 ? targetRect.width / region.rect.width : 1; - const heightFactor = - region.rect.height > 0 ? targetRect.height / region.rect.height : 1; - const sourceRight = region.pixelRect.left + region.pixelRect.width; - const sourceBottom = region.pixelRect.top + region.pixelRect.height; - const targetRight = targetPixelRect.left + targetPixelRect.width; - const targetBottom = targetPixelRect.top + targetPixelRect.height; - const regionUserInstruction = region.prompt.trim(); - const userInstructionLine = regionUserInstruction - ? ` User instruction for this region: ${regionUserInstruction}.` - : ""; - const isShrink = widthFactor < 0.92 || heightFactor < 0.92; - const isEnlarge = widthFactor > 1.08 || heightFactor > 1.08; - const scaleDirection = - isShrink - ? "make the selected subject clearly smaller" - : isEnlarge - ? "make the selected subject clearly larger" - : "move/fit the selected subject to the target box"; - const footprintRule = isShrink - ? "After shrinking, the original OLD footprint outside the TARGET box must be cleared and rebuilt as natural background/floor/rug/wall; the selected subject must not still touch the OLD box edges." - : isEnlarge - ? "After enlarging, the same selected subject should visibly occupy more of the TARGET footprint, with natural shadows and occlusion; do not just move it or stretch one axis." - : "Keep the subject centered on the TARGET box without changing the full canvas."; - - const cropNote = - cropIndex >= 0 - ? options.falSafe - ? ` Input image ${cropIndex + 3} is an extra identity and silhouette reference.` - : ` Input image ${cropIndex + 3} is a clean crop reference for this same target subject identity and silhouette; use it only to recognize the subject, never paste it, and do not preserve its old size.` - : ""; - if (options.falSafe) { - const actionPhrase = - isShrink - ? "resize the selected subject proportionally smaller and place it inside the final footprint" - : isEnlarge - ? "resize the same selected subject proportionally larger toward the final footprint" - : "place the same selected subject on the final footprint"; - const blendPhrase = - isShrink - ? "Blend the newly visible floor, rug, wall, window area, shadow, and background around it naturally." - : isEnlarge - ? "Blend edges, contact shadows, and occlusion with the surrounding room naturally." - : "Keep the surrounding room stable."; - return `Region ${region.index}: current target-subject search box in source photo left=${region.pixelRect.left}, top=${region.pixelRect.top}, right=${sourceRight}, bottom=${sourceBottom}, width=${region.pixelRect.width}, height=${region.pixelRect.height}; requested final footprint left=${targetPixelRect.left}, top=${targetPixelRect.top}, right=${targetRight}, bottom=${targetBottom}, width=${targetPixelRect.width}, height=${targetPixelRect.height}; scale factor width=${widthFactor.toFixed(2)}, height=${heightFactor.toFixed(2)}. Edit direction: ${actionPhrase}. ${blendPhrase}${cropNote}`; - } - return `Region ${region.index}: OLD search box in input 1 left=${region.pixelRect.left}, top=${region.pixelRect.top}, right=${sourceRight}, bottom=${sourceBottom}, width=${region.pixelRect.width}, height=${region.pixelRect.height}; TARGET final visual footprint left=${targetPixelRect.left}, top=${targetPixelRect.top}, right=${targetRight}, bottom=${targetBottom}, width=${targetPixelRect.width}, height=${targetPixelRect.height}; target scale is ${widthFactor.toFixed(2)}x width and ${heightFactor.toFixed(2)}x height. Main action: ${scaleDirection}.${userInstructionLine} The final selected-subject silhouette must clearly leave the OLD footprint edges and align much closer to TARGET than OLD; returning an almost unchanged target subject footprint is not acceptable. ${footprintRule}${cropNote}`; - }) - .join("\n"); - const cropLines = cropReferences - .map( - (reference, index) => - `Input image ${index + 3}: clean crop for Region ${reference.regionIndex}; identity/shape reference only, not a patch or layer.` - ) - .join("\n"); - - if (options.falSafe) { - return [ - "Interior selected-subject scale photo edit.", - `Return one realistic photo with the same camera frame and pixel size as input image 1: ${sourceSize.width} x ${sourceSize.height} px.`, - "Input image 1 is the source photo. Input image 2 is a marked copy of the source photo for location guidance only and should not appear in the final photo.", - "In the guide, the cyan dashed box marks the existing selected target subject area and the blue solid box marks the requested final footprint.", - "Resize only the selected target subject indicated by each current box. The target may be a person/model/child/body part, furniture, product, or another object. Keep the camera, room layout, walls, windows, floor, rug, sofa, lamp, text, background, and all non-target subjects stable.", - "Preserve the target identity, person anatomy/face/clothing or product material/structure, perspective, contact shadows, and natural aspect ratio/proportions.", - "For a smaller final footprint, show the selected subject smaller and blend newly visible floor, rug, wall, window area, shadow, and background around it naturally. For a larger final footprint, scale the same selected subject proportionally with natural edges and contact shadows.", - "Return a clean photo without guide marks, boxes, arrows, overlays, patch edges, color mismatch, or full-frame zoom.", - cropLines ? `${cropLines}` : "", - "", - "Region instructions:", - regionLines, - ].join("\n"); - } - - const coordinateGuideLine = options.visualLayoutGuide - ? "输入图片2是一张完整画面的粗略目标布局草稿，不是输出底图。它把 OLD 源框旧主体占位洗淡，表示旧尺寸主体必须从原尺寸位置移除并自然修复背景；同时把目标主体粗略缩放/移动到 TARGET 最终占位附近，这个缩小后的位置和尺寸是最终结果必须匹配的几何目标。请把 input 2 当成最终构图/比例的强约束：如果 input 2 中目标主体比 input 1 小，最终输出也必须明显变小，不能保留 input 1 的原尺寸主体。不要复制 input 2 的淡化痕迹、粗糙拼接、局部透明边、矩形补丁、亮度变化或参考图画质；最终必须以 input 1 的真实照片为唯一画面基底重新完成无缝图片编辑。" - : options.markedPhotoGuide - ? "输入图片2是一张原图浅化后的完整画面位置参考图，不是输出底图。青色虚线框表示当前主体搜索框（对应下方 OLD），蓝色实线框表示目标最终占位（对应下方 TARGET）。请用输入图片2理解主体在真实室内画面中的位置关系，但最终图不能出现任何框线、色块、标注或参考图的浅化效果。" - : "Input image 2 is a zoomed neutral coordinate instruction map, not a photo and not output content. It enlarges the OLD and TARGET boxes so their relative scale is easy to read; use the numeric pixel coordinates below for the exact absolute position in input image 1. The OLD box/label locates the current selected subject; the TARGET box/label gives the required final footprint, center, and scale factor. If the map says OLD OUTSIDE -> BG, rebuild the old full-size subject area outside TARGET as natural background in the final photo. Read the boxes, labels, arrows, and scale text only as instructions; never output any map graphics."; - - return [ - "Local selected-subject scale edit. Use input image 1 as the clean original photo and the locked camera frame.", - "One-shot cost constraint: produce the requested edit in this generation without relying on later retries or alternate candidates; an unchanged result is not acceptable.", - "中文硬约束：只调整 OLD 框中的主要目标主体，目标可以是人物、模特、小孩、手臂身体、商品或家具。TARGET 框是最终占位；如果是缩小，OLD 框里超出 TARGET 的旧主体占位必须改为自然背景；如果是放大，同一目标主体必须明显变大。禁止整图缩放、整图裁切、整图重绘。", - "一次成功硬约束：不要保守返回原尺寸。目标主体的可见轮廓必须明显从 OLD 原始占位变到 TARGET 占位；不接受看起来和原图几乎相同的结果。", - `Return one seamless realistic photo with the same frame and pixel size as input image 1: ${sourceSize.width} x ${sourceSize.height} px.`, - coordinateGuideLine, - cropLines ? `${cropLines}` : "", - "", - "Task:", - "Find the main selected target subject inside each OLD box in input image 1. The target can be a person/model/child/body part, furniture, product, or another movable object. Resize and, if needed, move that subject only so its final visible footprint matches the TARGET box shown in input image 2 and the numeric TARGET coordinates below.", - "Do a real subject-scale edit, not a full image zoom. The camera, room layout, windows, wall decor, text, floor, rug, and all non-target subjects must remain stable.", - "If an OLD box contains more than one object/person, choose the dominant target subject inside that box, usually the largest or most central selected item/person. Keep adjacent lamps, plants, side tables, pillows, wall decor, windows, text, background objects, and non-target people at their original size unless they are clearly part of the same target subject.", - "Keep the target identity, person face/anatomy/clothing or product material/structure, perspective, contact shadows, and natural aspect ratio/proportions from input image 1. Do not create a second copy or extra parts. Do not preserve the original target size just because the scene looks realistic; the user explicitly wants a visible scale change.", - "For beds, sofas, tables, chairs, cabinets, benches, people, models, children, hands, and other large visible subjects, still perform the scale edit on that selected subject. Do not treat the selected subject as locked background.", - "If the target is smaller, the selected subject must visibly become smaller and be mostly contained by the TARGET footprint. Rebuild the OLD area outside the TARGET footprint as newly exposed floor, rug, wall, bedding, shadows, or background. If the target is larger, enlarge the same selected subject naturally without stretching it.", - "缩小时请优先让目标主体明显缩小，而不是为了保守而保持原样。不接受原尺寸保留、只轻微变形、或只改变整张图构图。", - "Do not keep the selected subject at its original footprint. The edited subject should fit the TARGET rectangle much better than the OLD rectangle.", - "", - "Region requirements:", - regionLines, - "", - "Success check: compare the final selected-subject footprint with input image 1 and the neutral coordinate map in input image 2. The selected subject size must clearly change from the original and align much closer to TARGET than OLD, while the rest of the photo stays stable.", - "Do not paste rectangular image areas. Do not output coordinate-map lines, labels, boxes, arrows, scale text, muted wash, overlays, seams, hard patch edges, haze, color mismatch, full-canvas zoom, crop, pan, or redesign.", - "Return only the clean edited photo.", - ].join("\n"); -} - -function buildScaleAdjustCoordinateSystemPrompt() { - return [ - "You are an image editing model for local selected-subject scale adjustment.", - "Input image 1 is the only final photo base. Input image 2 is a zoomed neutral coordinate instruction map, not a photo layer.", - "Follow both English and Chinese instructions. The Chinese lines are hard constraints for avoiding an unchanged selected target subject and full-canvas zoom.", - "Generate a clean realistic final photo by editing the selected target subject itself and naturally rebuilding only the tiny surrounding background that becomes visible.", - "The target subject can be a person/model/child/body part, furniture, product, or another movable object. Prefer a clear visible size change over a conservative unchanged result; an unchanged target subject is not acceptable.", - "When shrinking, rebuild the old full-size target-subject footprint outside the target footprint as natural background. Large selected subjects are editable and must not remain locked at original size.", - "Do not paste, composite, clone, crop, resize, or copy rectangular image patches from any input.", - "Never output guide artifacts from the edit map: no faded wash, placeholder shadow, overlay patch, rough crop rectangle, lines, labels, blocks, seams, rectangular borders, arrows, masks, UI marks, or annotations.", - "Keep the camera frame, lighting, wall, floor, rug, windows, background furniture, text, logos, and all non-target regions stable.", - ].join("\n"); -} - -function buildScaleAdjustTextOnlyPrompt( - regions: NormalizedLocalEditRegion[], - scalePercent: ScaleAdjustPercent | null | undefined, - cropReferences: ScaleAdjustCropReference[], - sourceSize: { width: number; height: number } -) { - const cropLines = cropReferences - .map( - (reference, index) => - `Input image ${index + 2}: clean crop reference for Region ${reference.regionIndex}; use it only to identify the selected target subject and its natural aspect ratio/proportions, never paste the crop into the output.` - ) - .join("\n"); - const regionLines = regions - .map((region) => { - const cropIndex = cropReferences.findIndex( - (reference) => reference.regionIndex === region.index - ); - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const targetPixelRect = - region.targetPixelRect || - rectToPixels(targetRect, sourceSize.width, sourceSize.height); - const widthFactor = - region.rect.width > 0 ? targetRect.width / region.rect.width : 1; - const heightFactor = - region.rect.height > 0 ? targetRect.height / region.rect.height : 1; - const sourceRight = region.pixelRect.left + region.pixelRect.width; - const sourceBottom = region.pixelRect.top + region.pixelRect.height; - const targetRight = targetPixelRect.left + targetPixelRect.width; - const targetBottom = targetPixelRect.top + targetPixelRect.height; - const cropNote = - cropIndex >= 0 - ? ` Input image ${cropIndex + 2} is the clean crop reference for this target-subject identity and silhouette.` - : ""; - return `Region ${region.index}: hidden coordinate set A current area x0=${region.pixelRect.left}, y0=${region.pixelRect.top}, x1=${sourceRight}, y1=${sourceBottom}, width=${region.pixelRect.width}, height=${region.pixelRect.height}; hidden coordinate set B final size/location x0=${targetPixelRect.left}, y0=${targetPixelRect.top}, x1=${targetRight}, y1=${targetBottom}, width=${targetPixelRect.width}, height=${targetPixelRect.height}; final visual scale width=${widthFactor.toFixed(2)}, height=${heightFactor.toFixed(2)}. User request: ${region.prompt || "make the same selected target subject match set B"}.${cropNote}`; - }) - .join("\n"); - - return [ - "Local selected-subject scale edit. Input image 1 is the locked source photo and the only final canvas base.", - `Return one clean realistic photo with exactly the same camera frame and pixel size as input image 1: ${sourceSize.width} x ${sourceSize.height} px.`, - cropLines ? cropLines : "", - "The final photo must contain only natural scene content. Never add artificial graphic overlays, visible coordinate aids, labels, lines, UI elements, or text.", - "Only edit the main selected target subject inside each current pixel area. The target can be a person/model/child/body part, furniture, product, or another object. Keep all other non-target subjects, furniture, sofa, lamps, wall, windows, rug, floor, decor, lighting, and camera frame stable.", - "Use the numeric pixel sets below as hidden instructions only; they are not visual content and must not be drawn.", - "When set B is smaller than set A, make the same selected target subject visibly smaller and naturally rebuild the newly visible floor, rug, wall, shadow, or background around it. Do not leave the old full-size subject silhouette.", - "Keep the target identity, person anatomy/face/clothing or product material/structure, perspective, contact shadows, and natural aspect ratio/proportions. Do not stretch, squash, duplicate, or paste a rectangular crop.", - "中文硬约束：输入图1是锁定底图，只能调整 A 像素范围里的主要目标主体，目标可以是人物、模特、小孩、手臂身体、商品或家具，让它变成 B 像素范围对应的大小和位置。A/B 坐标只是隐藏指令，最终照片里绝对不能画出任何线条、框、文字、标注或 UI 元素。", - "", - "Region instructions:", - regionLines, - "", - "Success check before returning: the selected target subject must be clearly closer to set B than to set A, while the rest of the photo remains visually unchanged and clean.", - ].filter(Boolean).join("\n"); -} - -function toPatchRelativePixelBox( - box: { left: number; top: number; width: number; height: number }, - workspace: ScaleAdjustPatchReference["workspacePixelRect"] -) { - const left = clamp(box.left - workspace.left, 0, Math.max(0, workspace.width - 1)); - const top = clamp(box.top - workspace.top, 0, Math.max(0, workspace.height - 1)); - const right = clamp( - box.left + box.width - workspace.left, - left + 1, - workspace.width - ); - const bottom = clamp( - box.top + box.height - workspace.top, - top + 1, - workspace.height - ); - - return { - left, - top, - width: Math.max(1, right - left), - height: Math.max(1, bottom - top), - }; -} - -function buildScaleAdjustPatchPrompt( - regions: NormalizedLocalEditRegion[], - scalePercent: ScaleAdjustPercent | null | undefined, - patchReference: ScaleAdjustPatchReference, - sourceSize: { width: number; height: number }, - cropReferences: ScaleAdjustCropReference[] = [], - options: { - falSafe?: boolean; - includeSourceReference?: boolean; - targetDraftPrimary?: boolean; - } = {} -) { - const targetDraftPrimary = options.targetDraftPrimary === true; - const includeSourceReference = options.includeSourceReference !== false; - const falSafe = options.falSafe === true; - const structureLockedDraft = patchReference.structureLockedDraft === true; - const sourceInputIndex = targetDraftPrimary && includeSourceReference ? 2 : 1; - const draftInputIndex = targetDraftPrimary ? 1 : 2; - const cropReferenceStartIndex = - targetDraftPrimary && !includeSourceReference ? 2 : 3; - const firstInputRole = targetDraftPrimary - ? structureLockedDraft - ? includeSourceReference - ? "Input image 1 is a rough subject-structure-locked target crop. The selected target subject was moved and scaled using source-photo pixels, so its final size, position, silhouette, material/skin/clothing, perspective, and visible structure are the hard target. If the subject is a person, keep the same person identity, face, body proportions, pose, hair, hands, and clothing while cleaning rough edges. If the subject is furniture/product, keep rods, panels, weave, wheels, material, and product structure. Repair rough background holes, edge halos, blur, occlusion, contact shadows, and surface artifacts without redesigning the subject. Input image 2 is only the clean background and non-target reference, never the old size reference." - : "Input image 1 is a rough subject-structure-locked target crop and the only visual reference. The selected target subject was moved and scaled using source-photo pixels, so its final size, position, silhouette, material/skin/clothing, perspective, and visible structure are the hard target. If the subject is a person, keep the same person identity, face, body proportions, pose, hair, hands, and clothing while cleaning rough edges. If the subject is furniture/product, keep rods, panels, weave, wheels, material, and product structure. Repair rough background holes, edge halos, blur, occlusion, contact shadows, and surface artifacts without redesigning the subject." - : includeSourceReference - ? "Input image 1 is the rough edited target-layout crop. It is the hard layout target: the selected subject size and center in the final output must match this draft, but the rough wash, ghosted old footprint, pale wipe, placeholder shadow, blur, and patch artifacts are only edit instructions and must be cleaned. Never reproduce the pale wipe or any rectangular haze as photo content." - : "Input image 1 is the rough edited target-layout crop and the only visual reference. It is the hard layout target: the selected subject size and center in the final output must match this draft. Clean away the rough wash, ghosted old footprint, pale wipe, placeholder shadow, blur, and patch artifacts. Never reproduce the pale wipe or any rectangular haze as photo content." - : "Input image 1 is the exact crop from the original photo and is the locked crop base."; - const secondInputRole = targetDraftPrimary - ? includeSourceReference - ? "Input image 2 is the original clean crop. Use it for stable crop framing, background texture, lighting, perspective, target identity, and non-target person/object preservation. Do not restore the selected subject to its old size from this image." - : "" - : "Input image 2 is a rough target-layout draft of the same crop. It is the hard layout target for the selected subject size and position. Use it to understand the final footprint and newly visible background area, then remove any rough seams, washed areas, placeholder shadows, blur, or patch artifacts from that draft."; - const cropLines = cropReferences - .map( - (reference, index) => - `Input image ${index + cropReferenceStartIndex}: clean identity crop for Region ${reference.regionIndex}; use it only to recognize the target subject shape, identity, anatomy/materials, and natural aspect ratio, not its old size.` - ) - .join("\n"); - const regionLines = regions - .map((region) => { - const cropIndex = cropReferences.findIndex( - (reference) => reference.regionIndex === region.index - ); - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const targetPixelRect = - region.targetPixelRect || - rectToPixels(targetRect, sourceSize.width, sourceSize.height); - const sourceInPatch = toPatchRelativePixelBox( - region.pixelRect, - patchReference.workspacePixelRect - ); - const targetInPatch = toPatchRelativePixelBox( - targetPixelRect, - patchReference.workspacePixelRect - ); - const widthFactor = - region.rect.width > 0 ? targetRect.width / region.rect.width : 1; - const heightFactor = - region.rect.height > 0 ? targetRect.height / region.rect.height : 1; + Math.hypot( + targetRect.x + targetRect.width / 2 - (region.rect.x + region.rect.width / 2), + targetRect.y + targetRect.height / 2 - (region.rect.y + region.rect.height / 2) + ) > 0.01; const action = widthFactor < 0.92 || heightFactor < 0.92 - ? "make the same selected subject visibly smaller" + ? "把同一个主体等比缩小" : widthFactor > 1.08 || heightFactor > 1.08 - ? "make the same selected subject visibly larger" - : "move the same selected subject to the final footprint"; - const cropNote = - cropIndex >= 0 - ? ` Input image ${cropIndex + cropReferenceStartIndex} is the clean identity reference for this exact target subject.` - : ""; - return `Region ${region.index}: current search area inside input image ${sourceInputIndex} crop is x=${sourceInPatch.left}, y=${sourceInPatch.top}, w=${sourceInPatch.width}, h=${sourceInPatch.height}; final visual footprint inside input image ${draftInputIndex} / output crop is x=${targetInPatch.left}, y=${targetInPatch.top}, w=${targetInPatch.width}, h=${targetInPatch.height}; requested scale width=${widthFactor.toFixed(2)}, height=${heightFactor.toFixed(2)}. User request: ${region.prompt || action}. Main action: ${action}. Treat the main selected subject inside the search area as the target; it may be a person, model, child, body part, furniture, product, or other object. The final subject must be much closer to the final footprint than to the current search area.${cropNote}`; - }) - .join("\n"); - const falSafeRegionLines = regions - .map((region) => { - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const targetPixelRect = - region.targetPixelRect || - rectToPixels(targetRect, sourceSize.width, sourceSize.height); - const sourceInPatch = toPatchRelativePixelBox( - region.pixelRect, - patchReference.workspacePixelRect - ); - const targetInPatch = toPatchRelativePixelBox( - targetPixelRect, - patchReference.workspacePixelRect - ); - const widthFactor = - region.rect.width > 0 ? targetRect.width / region.rect.width : 1; - const heightFactor = - region.rect.height > 0 ? targetRect.height / region.rect.height : 1; - const action = - widthFactor < 0.92 || heightFactor < 0.92 - ? "make the same selected subject smaller" - : widthFactor > 1.08 || heightFactor > 1.08 - ? "make the same selected subject larger" - : "move the same selected subject to the final footprint"; - return `Region ${region.index}: original search area x=${sourceInPatch.left}, y=${sourceInPatch.top}, w=${sourceInPatch.width}, h=${sourceInPatch.height}; final target footprint x=${targetInPatch.left}, y=${targetInPatch.top}, w=${targetInPatch.width}, h=${targetInPatch.height}; scale width=${widthFactor.toFixed(2)}, height=${heightFactor.toFixed(2)}. ${region.prompt || action}`; - }) - .join("\n"); - const chineseHardConstraint = (() => { - if (!targetDraftPrimary) { - return "中文硬约束：只输出输入图1同一裁切范围的干净局部照片；输入图2是目标大小/位置草稿，必须按它的目标尺寸和目标位置输出，不是要保留旧尺寸。目标家具必须相对原图产生明显比例变化，非目标背景和裁切边缘要稳定自然。"; - } - if (structureLockedDraft) { - return includeSourceReference - ? "中文硬约束：输入图1已经用原图像素把目标主体放到目标尺寸和目标位置，目标主体可以是人物、模特、小孩、手臂身体、商品或家具。人物目标必须按输入图1保留同一身份、脸部、身体比例、姿态、发型、手部和服装；商品/家具目标必须保留产品结构、杆件、面板、编织纹理、轮子、材质和透视。输入图2只能作为干净背景、光影和非目标元素参考，绝对不能把目标主体恢复成输入图2里的旧尺寸。不能重新设计或替换目标主体，只清理草稿边缘、黑斑、破洞、脏点、灰块、皮肤/服装/产品表面伪影。" - : "中文硬约束：输入图1已经用原图像素把目标主体放到目标尺寸和目标位置，目标主体可以是人物、模特、小孩、手臂身体、商品或家具。人物目标必须按输入图1保留同一身份、脸部、身体比例、姿态、发型、手部和服装；商品/家具目标必须保留产品结构、杆件、面板、编织纹理、轮子、材质和透视。不能重新设计或替换目标主体，只清理草稿边缘、黑斑、破洞、脏点、灰块、皮肤/服装/产品表面伪影。"; - } - return includeSourceReference - ? "中文硬约束：输入图1是目标大小/位置草稿，必须按它的目标尺寸和目标位置输出；输入图2只是原始质感、非目标人物和背景参考，绝对不要把目标主体恢复成输入图2里的旧尺寸。草稿里的淡色擦除层/雾块只是提示旧位置需要补背景，不是画面内容，最终不能出现矩形雾块、白块、残影或拼接感。" - : "中文硬约束：输入图1是唯一视觉参考和目标大小/位置草稿，必须按它的目标尺寸和目标位置输出。不要找回旧尺寸目标主体。草稿里的淡色擦除层/雾块只是提示旧位置需要补背景，不是画面内容，最终不能出现矩形雾块、白块、残影或拼接感。"; - })(); - - if (falSafe) { - return [ - "Clean local image edit for a cropped workspace.", - `Return one clean realistic crop with the same framing and aspect as input image 1. This crop maps to full-photo pixels left=${patchReference.workspacePixelRect.left}, top=${patchReference.workspacePixelRect.top}, width=${patchReference.workspacePixelRect.width}, height=${patchReference.workspacePixelRect.height}.`, - "Input image 1 is the rough target draft. The selected subject is already placed at the requested final size and position; keep that final footprint.", - structureLockedDraft - ? "Structure lock: treat the selected subject pixels in input image 1 as the locked product/person layer. AI work is limited to background completion, boundary blending, occlusion, lighting, contact shadows, and removing rough draft artifacts. Do not repaint the product/person interior." - : "", - includeSourceReference - ? "Input image 2 is the original clean crop for background texture, lighting, perspective, and non-target objects only. Do not restore the selected subject to its old size." - : "", - "Use AI editing only to clean and blend the draft: remove halos, mask holes, rough edges, ghost fragments, haze blocks, dirty spots, and bad contact shadows.", - "Keep the same selected subject identity, product structure, material, perspective, and natural proportions. Do not replace, stretch, duplicate, or redesign it.", - "Rebuild newly visible floor, rug, wall, sofa edge, shadow, or background naturally where the old larger footprint was removed.", - "Do not output a full-room image. Do not zoom, crop, pan, or resize the whole canvas. Do not draw boxes, lines, labels, arrows, UI marks, overlays, text, seams, or rectangular patch edges.", - "中文硬约束：输入图1已经把目标主体放到最终大小和位置；只清理融合局部裁切，不要恢复旧尺寸，不要整图缩放，不要输出标注框、矩形雾块、贴片边、残影或全房间图。", - "", - "Region instructions:", - falSafeRegionLines, - "", - "Success check: the selected subject must stay much closer to the final target footprint than to the original search area, and the crop must look like one natural photograph.", - ].filter(Boolean).join("\n"); - } - - return [ - "Local selected-subject scale edit inside a cropped workspace.", - `Return exactly one clean realistic crop with the same framing and aspect as input image 1. The crop represents full-photo pixels left=${patchReference.workspacePixelRect.left}, top=${patchReference.workspacePixelRect.top}, width=${patchReference.workspacePixelRect.width}, height=${patchReference.workspacePixelRect.height}.`, - firstInputRole, - secondInputRole, - cropLines, - structureLockedDraft - ? "Structure lock: treat the selected subject pixels in input image 1 as the locked product/person layer. AI work is limited to background completion, boundary blending, occlusion, lighting, contact shadows, and removing rough draft artifacts. Do not repaint or redesign the target subject interior." - : "", - "Only edit the selected target subject inside the search area. The target subject can be a person/model/child/body part, furniture, product, or another movable object. Keep the crop borders, wall, window frame, floor, rug, sofa, lamps, decor, lighting, camera perspective, and all non-target subjects stable.", - "If the selected target is a person, scale the whole visible person/body coherently to the final footprint while preserving identity, face, pose, anatomy, hand shape, hair, clothing, and natural human proportions. If a person appears outside the selected target, preserve that non-target person without resizing or redrawing them.", - "When the final footprint is smaller, the selected subject must clearly shrink; erase the old larger subject footprint outside the final footprint and replace it with natural background/floor/rug/wall/shadow that matches the original crop.", - "If input image 1 still shows any faint, transparent, blurred, or ghosted old-size copy of the selected subject outside the final footprint, treat it as an erase marker only. Do not keep it. The only selected target subject that may remain is the smaller clean subject inside the final footprint.", - structureLockedDraft - ? "When the final footprint is larger, the target draft already places the same selected subject at the required larger size. Do not redesign the subject geometry/anatomy; keep the locked person identity/anatomy/clothing or product structure/material/perspective while cleaning background gaps, boundary blending, rough surface artifacts, natural occlusion, and contact shadows." - : "When the final footprint is larger, enlarge the selected subject itself to match the target draft while keeping nearby non-target people and hands natural. If the target is a person, preserve anatomy and identity instead of inventing a new person.", - "Keep the same target identity, perspective, visible details, contact shadows, and natural aspect ratio/proportions. Do not stretch, squash, duplicate, or replace it with a different subject.", - "Hard failure condition: if the selected target subject remains near its original old size, the answer is wrong. The output must visibly change the selected subject footprint to match the target draft footprint.", - structureLockedDraft - ? "Remove every rough draft artifact from input image 1: gray fill patches, beige wash, flat painted areas, jagged mask holes, black speckled stains, dark blotches, rectangular crop edges, and ghost fragments. Human skin, face, hands, hair, and clothing must look continuous and natural when the target is a person. Woven panels, wheels, rods, hooks, and product surfaces must remain continuous and clean when the target is furniture/product." - : "", - "Do not output boxes, lines, labels, coordinate marks, UI marks, text, arrows, overlays, rough layout artifacts, rectangular patch edges, or a full-room image.", - chineseHardConstraint, - "", - "Region instructions:", - regionLines, - "", - "Success check before returning: the selected target subject footprint must be clearly closer to the final footprint than the current search area, and the crop must look like one natural photograph.", - ].filter(Boolean).join("\n"); -} - -function buildScaleAdjustFullFramePatchPrompt( - regions: NormalizedLocalEditRegion[], - scalePercent: ScaleAdjustPercent | null | undefined, - sourceSize: { width: number; height: number } -) { - const regionLines = regions - .map((region) => { - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const targetPixelRect = - region.targetPixelRect || - rectToPixels(targetRect, sourceSize.width, sourceSize.height); - const widthFactor = - region.rect.width > 0 ? targetRect.width / region.rect.width : 1; - const heightFactor = - region.rect.height > 0 ? targetRect.height / region.rect.height : 1; - const action = - widthFactor < 0.92 || heightFactor < 0.92 - ? "make the same selected furniture/product visibly smaller" - : widthFactor > 1.08 || heightFactor > 1.08 - ? "make the same selected furniture/product visibly larger" - : "move the same selected furniture/product to the final footprint"; - return `Region ${region.index}: original selected search area in input image 1 is x=${region.pixelRect.left}, y=${region.pixelRect.top}, w=${region.pixelRect.width}, h=${region.pixelRect.height}; final target footprint shown by input image 2 is x=${targetPixelRect.left}, y=${targetPixelRect.top}, w=${targetPixelRect.width}, h=${targetPixelRect.height}; requested scale width=${widthFactor.toFixed(2)}, height=${heightFactor.toFixed(2)}. User request: ${region.prompt || action}. Main action: ${action}.`; + ? "把同一个主体等比放大" + : "把同一个主体调整到目标位置和大小"; + return [ + `区域 ${region.index}:`, + `源框 A: left=${region.pixelRect.left}, top=${region.pixelRect.top}, right=${sourceRight}, bottom=${sourceBottom}, width=${region.pixelRect.width}, height=${region.pixelRect.height}.`, + `目标框 B: left=${targetPixelRect.left}, top=${targetPixelRect.top}, right=${targetRight}, bottom=${targetBottom}, width=${targetPixelRect.width}, height=${targetPixelRect.height}.`, + `目标比例约为宽 ${widthFactor.toFixed(2)}x、高 ${heightFactor.toFixed(2)}x。`, + `请识别 A 框内的主要商品/物体/人物主体，只${action}，${moved ? "并移动到 B 框中心附近" : "保持在当前位置附近"}。`, + ].join(" "); }) .join("\n"); return [ - "Full-frame local image edit for furniture/product scale adjustment.", - "Return exactly one clean realistic full image with the same framing, camera, aspect ratio, and canvas extent as input image 1. Do not crop, zoom, pan, or resize the whole photo.", - "Input image 1 is the original clean full photo and the locked background plate for the final output. Preserve its framing, lighting, perspective, non-target people, non-target objects, and untouched areas.", - "Input image 2 is a full-photo target draft guide. It only shows where the selected furniture/product should end up and how large its final visual footprint should be. Treat the target draft size, center, silhouette, product structure, material, perspective, wheels, rods, panels, hooks, weave, and visible details as the hard target instruction.", - "The selected furniture/product in input image 1 is still at the old size and old position. Do not restore, keep, or prefer that old footprint. Use input image 1 for clean pixels and identity, then move/scale only the selected furniture/product to match input image 2.", - "AI work is limited to moving/scaling the locked selected furniture/product, natural background completion, boundary blending, occlusion, and contact shadows. Do not repaint, redesign, replace, stretch, duplicate, or distort the selected furniture/product interior.", - "The target draft can contain translucent pasted pixels, faint old-footprint washes, clipped fragments, suppressed dark details, partial mask silhouettes, or rectangular crop-guide edges. These are draft artifacts only. Use input image 1 to recover clean product details at the target size, but do not copy any dark blob, black hole, ghost fragment, transparent residue, pasted silhouette, crop rectangle, or old-size placement as final photo content.", - "If the target footprint intersects a non-target person, preserve that person as the foreground occluder exactly as in the original photo: do not move, cover, deform, redraw, or change their face, body, clothes, hands, legs, feet, or pose. Blend the furniture/product behind the person naturally where needed.", - "Erase the old larger furniture/product footprint outside the final target footprint and rebuild newly visible wall, curtain, floor, rug, shadow, and background naturally. Do not leave ghost copies, old-size remnants, rectangular haze, pale wash blocks, patch edges, black holes, dirty speckles, or visible seams.", - "Keep all non-target content stable: person identity and anatomy, sofa, window, curtain, cabinet, lamp, wall art, floor, rug, lighting, textural grain, and global color must remain consistent.", - "Do not output boxes, masks, labels, coordinate marks, UI marks, arrows, guide overlays, crop borders, or any annotation artifacts.", - "中文硬约束：输入图1是原始干净图，也是最终输出的锁定底板；输入图2只是完整画面的目标草稿，用来说明目标家具/商品的最终大小和最终位置。绝对不能把输入图1里的旧尺寸家具保留下来，也不能把输入图2的贴片痕迹当成最终内容。只移动/等比缩放目标家具并做背景补全、遮挡融合、阴影和边缘清理。草稿里的半透明贴片、暗色残块、黑洞、旧尺寸残影、矩形裁切边都只是错误遮罩提示，最终必须删除。非目标人物必须保持原样，不能被目标家具盖住、变形、换脸、改姿势或改腿脚。最终必须是一张完整自然照片，不能有矩形雾块、贴片边、黑洞、残影、标注框。", + "对输入图片做一次简单的图生图局部比例编辑。", + `最终输出保持原图完整画幅和像素尺寸：${sourceSize.width} x ${sourceSize.height} px。`, + "只编辑用户框选的主体本身，保持主体外观、材质、文字、图案、透视和宽高比，不要拉伸或压扁。", + "不要缩放整张图，不要裁剪整张图，不要重绘未选区域。", + "如果主体变小，原位置露出的区域按原图背景自然补齐；白底或纯色背景必须保持干净一致。", + "如果主体变大，让同一个主体自然占据更大的目标范围，并保持边缘、阴影和背景衔接自然。", + "A/B 坐标只是隐藏编辑指令，最终图片里绝不能出现框线、箭头、文字、标注、UI 或半透明色块。", "", - "Region instructions:", + "编辑要求:", regionLines, - "", - "Success check before returning: the selected furniture/product footprint is clearly closer to the final target footprint in input image 1 than to the original search area, all non-target people remain unchanged, and the full image looks like one natural photograph.", - ].join("\n"); -} - -function buildScaleAdjustProductLockBackgroundPrompt( - regions: NormalizedLocalEditRegion[], - scalePercent: ScaleAdjustPercent | null | undefined, - sourceSize: { width: number; height: number }, - options: { draftInput?: boolean; maskEdit?: boolean } = {} -) { - const regionLines = regions - .map((region) => { - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const targetPixelRect = - region.targetPixelRect || - rectToPixels(targetRect, sourceSize.width, sourceSize.height); - return `Region ${region.index}: remove the selected furniture/product from the original area x=${region.pixelRect.left}, y=${region.pixelRect.top}, w=${region.pixelRect.width}, h=${region.pixelRect.height}; keep the final target footprint x=${targetPixelRect.left}, y=${targetPixelRect.top}, w=${targetPixelRect.width}, h=${targetPixelRect.height} visually clean and empty because the locked product pixels will be composited there after generation. User request: ${region.prompt}.`; - }) - .join("\n"); - - return [ - "Full-frame background repair for product-locked scale adjustment.", - "Return exactly one clean realistic full image with the same framing, camera, aspect ratio, and canvas extent as input image 1. Do not crop, zoom, pan, or resize the whole photo.", - options.maskEdit - ? options.draftInput - ? "Input image 1 is a full-photo repair draft from the original photo. The old selected furniture/product footprint has been roughly wiped and the locked product may already appear near its final target footprint. The supplied edit mask marks the rough/old footprint area that must be repaired into natural background and seamless integration; do not edit outside the masked area." - : "Input image 1 is the original clean full photo. The supplied edit mask marks the old selected furniture/product footprint that must be repaired as natural background; do not edit outside the masked area." - : "Input image 1 is the original clean full photo. Input image 2 is a background plate reference that removes the selected furniture/product footprint and keeps the room/person/background layout stable.", - "Important: do not draw, recreate, move, scale, duplicate, or redesign the selected furniture/product in the final image. The system will composite the locked original product pixels into the final target footprint after your background image is returned.", - options.draftInput - ? "Your job is only to turn the rough wiped footprint and any visible draft haze, gray wash, ghost product silhouette, or edge patch into natural wall, curtain, floor, rug, shadow, sofa edge, and background. If the locked product is visible in the draft, keep the surrounding background and contact shadows natural, but do not invent a second product or preserve old-size remnants." - : "Your job is only to erase the old selected furniture/product footprint, complete newly exposed wall, curtain, floor, rug, shadow, sofa edge, and background naturally, and keep the target footprint clean enough for the locked product layer to sit there.", - "Preserve all non-target people exactly as foreground/background context: do not move, cover, deform, redraw, change face, change body, change hands, change legs, change clothes, or change pose.", - "Keep all non-target content stable: sofa, window, curtain, cabinet, lamp, wall art, floor, rug, lighting, textural grain, and global color must remain consistent.", - options.maskEdit - ? "The edit mask is only an invisible repair boundary. Do not output pale wash blocks, rough mask shapes, hard rectangular edges, black holes, ghost fragments, pasted silhouettes, or old-size residues." - : "Remove draft artifacts from input image 2: pale wash blocks, rough masks, missing patches, hard rectangular edges, black holes, ghost fragments, pasted silhouettes, and old-size residues. They are only edit guides, not final content.", - "Do not output boxes, masks, labels, coordinate marks, UI marks, arrows, guide overlays, crop borders, or any annotation artifacts.", - "中文硬约束：这是产品结构锁定流程。你只生成背景底图，不要生成、移动、缩放或重绘目标家具/商品；目标家具/商品会在你输出后由系统用原图锁定像素叠回目标位置。必须擦除旧位置的目标家具/商品并自然补全墙面、窗帘、地面、阴影和背景。非目标人物必须保持原样，不能变脸、变形、改变衣服、手、腿、脚或姿势。最终背景图不能有矩形雾块、贴片边、黑洞、残影、标注框，也不能出现第二个目标家具/商品。", - "", - "Region instructions:", - regionLines, - "", - "Success check before returning: the selected furniture/product is absent from the generated background plate, the old footprint is naturally repaired, the target footprint is clean, and the full image still looks like one natural photograph.", - ].join("\n"); -} - -function buildScaleAdjustProductLockBackgroundCropPrompt( - regions: NormalizedLocalEditRegion[], - scalePercent: ScaleAdjustPercent | null | undefined, - patchReference: ScaleAdjustPatchReference, - sourceSize: { width: number; height: number } -) { - const workspace = patchReference.workspacePixelRect; - const regionLines = regions - .map((region) => { - const targetRect = getScaleAdjustTargetRect(region, scalePercent); - const targetPixelRect = - region.targetPixelRect || - rectToPixels(targetRect, sourceSize.width, sourceSize.height); - const sourceInCrop = { - left: Math.max(0, region.pixelRect.left - workspace.left), - top: Math.max(0, region.pixelRect.top - workspace.top), - width: region.pixelRect.width, - height: region.pixelRect.height, - }; - const targetInCrop = { - left: Math.max(0, targetPixelRect.left - workspace.left), - top: Math.max(0, targetPixelRect.top - workspace.top), - width: targetPixelRect.width, - height: targetPixelRect.height, - }; - return `Region ${region.index}: in this crop, erase the old selected furniture/product footprint around x=${sourceInCrop.left}, y=${sourceInCrop.top}, w=${sourceInCrop.width}, h=${sourceInCrop.height}. Keep the future target footprint x=${targetInCrop.left}, y=${targetInCrop.top}, w=${targetInCrop.width}, h=${targetInCrop.height} empty and clean; the locked product pixels will be composited there after generation. User request: ${region.prompt}.`; - }) - .join("\n"); - - return [ - "Cropped background repair for product-locked scale adjustment.", - `Return exactly one clean realistic crop with the same framing, aspect ratio, and pixel extent as input image 1. This crop maps to full-photo pixels left=${workspace.left}, top=${workspace.top}, width=${workspace.width}, height=${workspace.height}.`, - patchReference.compositeClipMaskCropUrl - ? "Input image 1 is the original crop from the source photo. Input image 2 is a black/white alpha-mask guide only: white/bright pixels mark the old selected product footprint and target repair boundary to erase or keep empty; black pixels are context to preserve. Do not render the mask itself." - : "Input image 1 is the original crop from the source photo. Input image 2 is a rough empty-background repair guide crop where the selected furniture/product footprint has been wiped.", - "Important: do not draw, recreate, move, scale, duplicate, or redesign the selected furniture/product in your output. The system will composite the locked original product pixels into the target footprint after your crop is returned.", - "Your only job is to repair the old footprint into natural wall, curtain, floor, rug, shadow, sofa edge, and background, while keeping non-target people and non-target objects stable.", - "If input image 2 contains haze, gray wash, ghost silhouettes, crop edges, black holes, rough mask shapes, or pasted fragments, treat them as repair hints only. Do not preserve them as visible content.", - "Preserve any non-target person exactly: do not change face, body, hands, legs, clothes, pose, hair, or book. Do not cover the person with furniture.", - "Keep the target footprint empty and natural; do not place the selected furniture/product there.", - "Do not output boxes, masks, labels, coordinate marks, UI marks, arrows, guide overlays, text, seams, rectangular patch edges, haze blocks, ghost copies, or old-size remnants.", - "中文硬约束：这是产品结构锁定的局部背景修补。你只输出干净背景裁切，不要生成目标家具/商品；旧位置的目标家具/商品必须擦除并补成自然墙面、窗帘、地面、阴影和背景。目标家具/商品会在你输出后由系统用原图锁定像素叠回。最终不能有旧尺寸残影、矩形雾块、贴片边、黑洞、标注框，也不能改变人物。", - "", - "Region instructions:", - regionLines, - "", - "Success check before returning: the selected furniture/product is absent from the crop, the old footprint is naturally repaired, the future target footprint is clean, and the crop looks like one natural photograph.", ].join("\n"); } @@ -5329,22 +660,9 @@ export async function POST(request: NextRequest) { ); } - const shouldUseScaleAdjustDetailModel = - operation === SCALE_ADJUST_OPERATION && - (hasScaleAdjustTargetRects || scalePercent !== null); - const [pointsConfig, siteSettingsForModel, scaleAdjustModelCandidates] = - await Promise.all([ + const [pointsConfig, siteSettingsForModel] = await Promise.all([ getPointsSystemConfig(), getSiteSettings().catch(() => null), - shouldUseScaleAdjustDetailModel - ? getEnabledAiModelsWithProvider(false, true).catch((error) => { - console.warn( - "[Canvas Local Edit] Failed to load scale adjust model candidates:", - error - ); - return [] as AiModelWithProviderForSelection[]; - }) - : Promise.resolve([] as AiModelWithProviderForSelection[]), ]); const pointsCharge = await prepareFeaturePointsCharge({ userId: session.user.id, @@ -5359,23 +677,113 @@ export async function POST(request: NextRequest) { return NextResponse.json(pointsCharge.payload, { status: pointsCharge.status }); } - const isScaleAdjustOperation = + if ( operation === SCALE_ADJUST_OPERATION && - (hasScaleAdjustTargetRects || scalePercent !== null); - const hasScaleAdjustEnlarge = isScaleAdjustOperation - ? hasScaleAdjustEnlargeTarget(regions, scalePercent) - : false; - const hasScaleAdjustShrink = isScaleAdjustOperation - ? hasScaleAdjustShrinkTarget(regions, scalePercent) - : false; - const preferScaleAdjustMaskCapableModel = - hasScaleAdjustShrink && !hasScaleAdjustEnlarge; - const preferScaleAdjustFalNanoBananaEdit = - hasScaleAdjustShrink && !hasScaleAdjustEnlarge; - const preferScaleAdjustProviderTypes = - hasScaleAdjustShrink && !hasScaleAdjustEnlarge - ? ["snapgen-image", "grsai-gpt-image"] - : []; + (hasScaleAdjustTargetRects || scalePercent !== null) + ) { + const displayPrompt = buildLocalEditDisplayPrompt({ operation, scalePercent }); + const resolvedAspectRatio = resolveLocalEditAspectRatio( + body.aspectRatio, + sourceWidth, + sourceHeight + ); + const requestedModel = + typeof body.model === "string" && body.model.trim() + ? body.model.trim() + : undefined; + const configuredDetailModel = + typeof siteSettingsForModel?.canvasDetailDefaultModelId === "string" + ? siteSettingsForModel.canvasDetailDefaultModelId.trim() + : ""; + const configuredCanvasModel = + typeof siteSettingsForModel?.canvasDefaultModelId === "string" + ? siteSettingsForModel.canvasDefaultModelId.trim() + : ""; + const configuredAutoCanvasModel = + typeof siteSettingsForModel?.canvasAutoModeDefaultModelId === "string" + ? siteSettingsForModel.canvasAutoModeDefaultModelId.trim() + : ""; + const resolvedModel = + requestedModel || + configuredDetailModel || + configuredCanvasModel || + configuredAutoCanvasModel; + const requestedImageSize = + typeof body.imageSize === "string" && body.imageSize.trim() + ? body.imageSize.trim() + : "2K"; + const manualRegionReferenceImageUrls = regions.flatMap((region) => + region.referenceImages.map((reference) => reference.url) + ); + const prompt = buildSimpleScaleAdjustPrompt(regions, scalePercent, { + width: sourceWidth, + height: sourceHeight, + }); + const imageRequest: CanvasGenerateImageRequest = { + aspectRatio: resolvedAspectRatio, + billingFeature: "repaint", + debugLabel: "canvas-scale-adjust-simple", + debugMeta: { + displayPrompt, + operation, + requestedModel: requestedModel || null, + resolvedAspectRatio, + resolvedModel: resolvedModel || null, + scaleAdjustPromptStrategy: "simple-image-edit-v1", + scaleAdjustTargetRegionCount: regions.filter((region) => region.targetRect) + .length, + scalePercent, + sourceHeight, + sourceNodeId: body.sourceNodeId || null, + sourceWidth, + }, + displayPrompt, + imageCount: 1, + imageSize: requestedImageSize, + model: resolvedModel || undefined, + negativePrompt: + "不要画框线、箭头、坐标、标注、文字或 UI。不要缩放整张图、裁剪整张图、重绘未选区域。不要拉伸、压扁、复制或替换主体。", + prompt, + promptExtend: false, + referenceImages: [sourceImageUrl, ...manualRegionReferenceImageUrls], + ownerKey: session.user.id, + }; + + const task = await createCanvasImageTask({ + request: imageRequest as unknown as Record, + userId: session.user.id, + }); + await enqueueCanvasImageTask(task.id); + const currentTask = await getCanvasImageTaskForUser({ + taskId: task.id, + userId: session.user.id, + }); + + if (pointsCharge.chargeable) { + try { + await pointsCharge.consume({ + note: `比例调整消耗积分（${task.id}）`, + }); + } catch (pointsError) { + console.error("[Points] Failed to consume scale adjust points:", pointsError); + } + } + + return NextResponse.json({ + success: true, + annotatedReferenceUrl: sourceImageUrl, + aspectRatio: resolvedAspectRatio, + displayPrompt, + prompt: displayPrompt, + regions, + sourceHeight, + sourceWidth, + taskId: task.id, + status: currentTask?.status || task.status, + queueStatus: currentTask?.queueStatus, + }); + } + const displayPrompt = buildLocalEditDisplayPrompt({ operation, scalePercent }); const resolvedAspectRatio = resolveLocalEditAspectRatio( body.aspectRatio, @@ -5386,735 +794,66 @@ export async function POST(request: NextRequest) { typeof body.model === "string" && body.model.trim() ? body.model.trim() : undefined; - const configuredDetailModel = - typeof siteSettingsForModel?.canvasDetailDefaultModelId === "string" - ? siteSettingsForModel.canvasDetailDefaultModelId.trim() - : ""; - const detailFixedModel = isScaleAdjustOperation - ? resolveScaleAdjustPreferredModelId({ - configuredModelId: configuredDetailModel, - honorRequestedModelBeforeGpt: hasScaleAdjustEnlarge, - preferFalNanoBananaEdit: preferScaleAdjustFalNanoBananaEdit, - preferMaskCapableGpt: preferScaleAdjustMaskCapableModel, - preferProviderTypes: preferScaleAdjustProviderTypes, - requestedModelId: requestedModel || "", - models: scaleAdjustModelCandidates, - }) - : ""; - if ( - isScaleAdjustOperation && - detailFixedModel && - detailFixedModel !== configuredDetailModel - ) { - console.info("[Canvas Local Edit] Override scale adjust model to detail line:", { - configuredDetailModel: configuredDetailModel || null, - requestedModel: requestedModel || null, - resolvedModel: detailFixedModel, - }); - } - const resolvedModel = detailFixedModel || requestedModel; - const resolvedScaleAdjustModelEntry = isScaleAdjustOperation - ? findModelSelectionEntry(scaleAdjustModelCandidates, resolvedModel || "") - : null; - const useScaleAdjustCoordinateGuide = - isScaleAdjustOperation && - isScaleAdjustCoordinateGuideModel(resolvedScaleAdjustModelEntry); - const scaleAdjustModelFamily = getScaleAdjustModelFamily( - resolvedScaleAdjustModelEntry + const annotatedBuffer = await sharp(sourceBuffer, { failOn: "none" }) + .composite([ + { + input: buildAnnotationSvg({ + height: sourceHeight, + hidePromptText: false, + regions, + width: sourceWidth, + }), + left: 0, + top: 0, + }, + ]) + .png({ compressionLevel: 6 }) + .toBuffer(); + const annotatedFilename = `canvas-local-edit-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.png`; + const annotatedUpload = await uploadFile( + annotatedFilename, + toArrayBuffer(annotatedBuffer) ); - const useProviderTextOnlyCoordinatePrompt = - isScaleAdjustOperation && - resolvedScaleAdjustModelEntry?.provider?.type === "alibaba-image"; - const useScaleAdjustCoordinatePrompt = useScaleAdjustCoordinateGuide; - const useFalScaleAdjustCoordinatePrompt = - useScaleAdjustCoordinatePrompt && - resolvedScaleAdjustModelEntry?.provider?.type === "fal-image"; - const useScaleAdjustVisualLayoutGuide = - useScaleAdjustCoordinatePrompt && - !useFalScaleAdjustCoordinatePrompt && - !useProviderTextOnlyCoordinatePrompt; - const useScaleAdjustMarkedPhotoGuide = - useScaleAdjustCoordinatePrompt && !useScaleAdjustVisualLayoutGuide; - const isScaleAdjustFalNanoBananaEdit = - isScaleAdjustOperation && - isScaleAdjustFalNanoBananaEditModel(resolvedScaleAdjustModelEntry); - const useScaleAdjustFullFrameFalEdit = - isScaleAdjustFalNanoBananaEdit && !hasScaleAdjustShrink; - const useScaleAdjustStructureLockedPatchWorkflow = - isScaleAdjustOperation && - !useScaleAdjustFullFrameFalEdit && - hasScaleAdjustStructureTarget(regions, scalePercent) && - (hasScaleAdjustShrink || - !preferScaleAdjustMaskCapableModel || - !isScaleAdjustMaskCapableModel(resolvedScaleAdjustModelEntry)); - const useScaleAdjustEditMask = - isScaleAdjustOperation && - !useScaleAdjustStructureLockedPatchWorkflow && - isScaleAdjustMaskCapableModel(resolvedScaleAdjustModelEntry); - const useScaleAdjustPatchWorkflow = - isScaleAdjustOperation && - !useScaleAdjustFullFrameFalEdit && - (useScaleAdjustStructureLockedPatchWorkflow || !useScaleAdjustEditMask); - const sendScaleAdjustAnnotatedReferenceToModel = - isScaleAdjustOperation && - !useScaleAdjustPatchWorkflow && - !useProviderTextOnlyCoordinatePrompt; - const shouldUploadAnnotatedReference = - !isScaleAdjustOperation || sendScaleAdjustAnnotatedReferenceToModel; - const annotatedBuffer = shouldUploadAnnotatedReference - ? isScaleAdjustOperation - ? useScaleAdjustCoordinatePrompt - ? useScaleAdjustVisualLayoutGuide - ? ( - await buildScaleAdjustVisualLayoutGuideBuffer({ - regions, - scalePercent, - sourceBuffer, - sourceHeight, - sourceWidth, - }) - ).buffer - : useScaleAdjustMarkedPhotoGuide - ? await buildScaleAdjustMarkedPhotoGuideBuffer({ - height: sourceHeight, - regions, - scalePercent, - sourceBuffer, - width: sourceWidth, - }) - : await sharp( - buildScaleAdjustZoomedCoordinateSvg({ - height: sourceHeight, - regions, - scalePercent, - width: sourceWidth, - }), - { failOn: "none" } - ) - .png({ compressionLevel: 6 }) - .toBuffer() - : ( - await buildScaleAdjustVisualLayoutGuideBuffer({ - regions, - scalePercent, - sourceBuffer, - sourceHeight, - sourceWidth, - }) - ).buffer - : await sharp(sourceBuffer, { failOn: "none" }) - .composite([ - { - input: buildAnnotationSvg({ - height: sourceHeight, - hidePromptText: false, - regions, - width: sourceWidth, - }), - left: 0, - top: 0, - }, - ]) - .png({ compressionLevel: 6 }) - .toBuffer() - : null; - const annotatedFilename = `${isScaleAdjustOperation ? "canvas-scale-adjust-layout-guide" : "canvas-local-edit"}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.png`; - const annotatedUpload = annotatedBuffer - ? await uploadFile(annotatedFilename, toArrayBuffer(annotatedBuffer)) - : null; - const scaleAdjustEditMaskUpload = useScaleAdjustEditMask - ? await uploadFile( - `canvas-scale-adjust-edit-mask-${Date.now()}-${Math.random() - .toString(36) - .slice(2, 8)}.png`, - toArrayBuffer( - await buildScaleAdjustEditMaskBuffer({ - regions, - scalePercent, - sourceHeight, - sourceWidth, - }) - ) - ) - : null; - let scaleAdjustPatchReference: ScaleAdjustPatchReference | null = null; - let scaleAdjustPatchReferenceFailed = false; - if (useScaleAdjustPatchWorkflow) { - try { - scaleAdjustPatchReference = await createScaleAdjustPatchReference({ - regions, - scalePercent, - sourceBuffer, - sourceHeight, - sourceImageUrl, - sourceWidth, - fullFrameReferenceOnly: false, - userId: session.user.id, - }); - } catch (error) { - scaleAdjustPatchReferenceFailed = true; - console.warn( - "[Canvas Local Edit] Scale adjust patch reference failed; falling back to source-only coordinate prompt:", - { - error: error instanceof Error ? error.message : String(error), - } - ); - } - } - const shouldCreateScaleAdjustCropReferences = - isScaleAdjustOperation && - !scaleAdjustPatchReferenceFailed && - (!scaleAdjustPatchReference && - (useScaleAdjustVisualLayoutGuide || - !useScaleAdjustCoordinatePrompt || - useProviderTextOnlyCoordinatePrompt || - (useScaleAdjustMarkedPhotoGuide && !useFalScaleAdjustCoordinatePrompt))); - const scaleAdjustCropReferences: ScaleAdjustCropReference[] = - shouldCreateScaleAdjustCropReferences - ? await createScaleAdjustCropReferences({ - regions, - sourceBuffer, - sourceHeight, - sourceWidth, - }) - : []; - const useScaleAdjustFullFrameStructurePatch = - Boolean(scaleAdjustPatchReference?.structureLockedDraft) && - Boolean(scaleAdjustPatchReference?.targetDraftFullUrl) && - scaleAdjustPatchReference?.targetDraftUrl === - scaleAdjustPatchReference?.targetDraftFullUrl && - regions.some((region) => - isScaleAdjustFurnitureProductLikePrompt(region.prompt) - ); - const canUseScaleAdjustProductLockMaskEdit = - useScaleAdjustFullFrameStructurePatch && - isScaleAdjustMaskCapableModel(resolvedScaleAdjustModelEntry) && - Boolean(scaleAdjustPatchReference?.productLockEditMaskUrl); - const useScaleAdjustProductLockComposite = - Boolean(scaleAdjustPatchReference?.productLockOverlayUrl) && - (useScaleAdjustFullFrameStructurePatch - ? Boolean(scaleAdjustPatchReference?.backgroundPlateFullUrl) || - canUseScaleAdjustProductLockMaskEdit - : Boolean(scaleAdjustPatchReference)); - const useScaleAdjustProductLockMaskEdit = - useScaleAdjustProductLockComposite && - canUseScaleAdjustProductLockMaskEdit; - const useScaleAdjustProductLockCropBackground = - useScaleAdjustProductLockComposite && - !useScaleAdjustFullFrameStructurePatch && - Boolean( - scaleAdjustPatchReference?.compositeClipMaskCropUrl || - scaleAdjustPatchReference?.backgroundPlateCropUrl - ); - const useScaleAdjustPatchTargetDraftPrimary = Boolean(scaleAdjustPatchReference); - const includeScaleAdjustPatchSourceReference = - Boolean(scaleAdjustPatchReference); - const useFalScaleAdjustPatchPrompt = - Boolean(scaleAdjustPatchReference) && - !useScaleAdjustFullFrameStructurePatch && - resolvedScaleAdjustModelEntry?.provider?.type === "fal-image"; - const prompt = isScaleAdjustOperation - ? scaleAdjustPatchReference - ? useScaleAdjustFullFrameStructurePatch - ? useScaleAdjustProductLockComposite - ? buildScaleAdjustProductLockBackgroundPrompt( - regions, - scalePercent, - { width: sourceWidth, height: sourceHeight }, - { - draftInput: false, - maskEdit: useScaleAdjustProductLockMaskEdit, - } - ) - : buildScaleAdjustFullFramePatchPrompt( - regions, - scalePercent, - { width: sourceWidth, height: sourceHeight } - ) - : useScaleAdjustProductLockCropBackground - ? buildScaleAdjustProductLockBackgroundCropPrompt( - regions, - scalePercent, - scaleAdjustPatchReference, - { width: sourceWidth, height: sourceHeight } - ) - : buildScaleAdjustPatchPrompt( - regions, - scalePercent, - scaleAdjustPatchReference, - { width: sourceWidth, height: sourceHeight }, - scaleAdjustCropReferences, - { - falSafe: useFalScaleAdjustPatchPrompt, - includeSourceReference: includeScaleAdjustPatchSourceReference, - targetDraftPrimary: useScaleAdjustPatchTargetDraftPrimary, - } - ) - : useProviderTextOnlyCoordinatePrompt - ? buildScaleAdjustTextOnlyPrompt( - regions, - scalePercent, - scaleAdjustCropReferences, - { width: sourceWidth, height: sourceHeight } - ) - : useScaleAdjustCoordinatePrompt - ? buildScaleAdjustCoordinatePrompt( - regions, - scalePercent, - { - width: sourceWidth, - height: sourceHeight, - }, - scaleAdjustCropReferences, - { - falSafe: useFalScaleAdjustCoordinatePrompt, - markedPhotoGuide: useScaleAdjustMarkedPhotoGuide, - visualLayoutGuide: useScaleAdjustVisualLayoutGuide, - } - ) - : buildScaleAdjustPrompt( - regions, - scalePercent, - scaleAdjustCropReferences, - { width: sourceWidth, height: sourceHeight } - ) - : buildLocalEditPrompt(regions); - const scaleAdjustCoordinateNegativePrompt = useFalScaleAdjustCoordinatePrompt - ? "guide marks, labels, boxes, arrows, overlays, patch edges, color mismatch, full-frame zoom, unchanged target subject, duplicate subject, duplicate furniture, duplicate person, stretched furniture, distorted furniture, deformed person, changed face, extra limbs" - : "visible annotation, guide overlay, translucent overlay, target placement overlay, faded wash, placeholder shadow, text label, arrows, boxes, rectangle border, UI marks, pasted patch, rough layout patch, hard crop edge, seam, hard rectangular edge, color mismatch, different-resolution area, whole image zoom, whole image crop, whole image pan, whole image redesign, unchanged target subject, old-size target subject, unchanged target person, old-size target person, unchanged target furniture, old-size target furniture, stretched subject, squashed subject, distorted subject, stretched furniture, squashed furniture, duplicate subject, duplicate person, duplicate furniture, second copy, changed face, deformed person, broken anatomy, extra limbs, extra fingers, extra legs, extra rails, side panels, shelves, cabinets, lattice panels"; - const scaleAdjustTextOnlyNegativePrompt = - "artificial graphic overlay, visible coordinate aid, visible line, visible label, UI element, annotation text, full image redesign, full-frame zoom, crop, pan, pasted crop, hard patch edge, color mismatch, changed sofa, changed lamp, changed wall, changed rug, duplicate subject, duplicate person, duplicate furniture, changed face, deformed person, extra limbs, stretched subject, stretched furniture, distorted furniture"; - const scaleAdjustPatchNegativePrompt = - "full room image, changed crop border, artificial graphic overlay, visible coordinate aid, visible line, visible label, UI element, annotation text, guide mark, pasted crop, hard patch edge, rectangular haze, white haze block, pale wipe block, color mismatch, gray draft patch, beige wash patch, flat painted area, jagged mask hole, black speckled stain, dark blotch, duplicate subject, duplicate person, duplicate furniture, unchanged target subject, unchanged target person, unchanged target furniture, old-size target subject, changed target identity, changed face, deformed person, broken anatomy, extra limbs, extra fingers, changed locked product, redrawn product interior, different furniture, broken woven panel, dirty woven panel, wheel inside woven panel, stretched subject, stretched furniture, distorted furniture"; - const scaleAdjustFalPatchNegativePrompt = - "full room image, boxes, labels, UI marks, arrows, text, overlays, seams, rectangular patch edge, haze block, ghost copy, old-size subject, unchanged subject, duplicate subject, stretched subject, distorted product, changed product structure"; - const scaleAdjustFullFramePatchNegativePrompt = - "boxes, labels, UI marks, arrows, text, overlays, seams, rectangular patch edge, haze block, ghost copy, old-size subject, unchanged subject, duplicate subject, stretched subject, distorted product, changed product structure, changed person, changed face, deformed person, extra limbs, changed hands, changed feet, whole-image zoom, crop, pan"; - const scaleAdjustProductLockBackgroundNegativePrompt = - "selected furniture, selected product, moved furniture, scaled furniture, recreated product, duplicate furniture, duplicate product, old-size furniture, target object visible, product ghost, product remnant, boxes, labels, UI marks, arrows, text, overlays, seams, rectangular patch edge, haze block, black hole, changed person, changed face, deformed person, extra limbs, changed hands, changed feet, whole-image zoom, crop, pan"; + const prompt = buildLocalEditPrompt(regions); const manualRegionReferenceImageUrls = regions.flatMap((region) => region.referenceImages.map((reference) => reference.url) ); - const scaleAdjustCropReferenceImageUrls = scaleAdjustCropReferences.map( - (reference) => reference.url - ); - const referenceImages = isScaleAdjustOperation - ? scaleAdjustPatchReference - ? useScaleAdjustFullFrameStructurePatch - ? useScaleAdjustProductLockComposite - ? useScaleAdjustProductLockMaskEdit - ? [ - scaleAdjustPatchReference.sourceCropUrl, - ...scaleAdjustCropReferenceImageUrls, - ...manualRegionReferenceImageUrls, - ] - : [ - scaleAdjustPatchReference.sourceCropUrl, - scaleAdjustPatchReference.backgroundPlateFullUrl as string, - ...scaleAdjustCropReferenceImageUrls, - ...manualRegionReferenceImageUrls, - ] - : [ - sourceImageUrl, - scaleAdjustPatchReference.targetDraftFullUrl as string, - ...scaleAdjustCropReferenceImageUrls, - ...manualRegionReferenceImageUrls, - ] - : useScaleAdjustProductLockCropBackground - ? [ - scaleAdjustPatchReference.sourceCropUrl, - (scaleAdjustPatchReference.compositeClipMaskCropUrl || - scaleAdjustPatchReference.backgroundPlateCropUrl) as string, - ...scaleAdjustCropReferenceImageUrls, - ...manualRegionReferenceImageUrls, - ] - : useScaleAdjustPatchTargetDraftPrimary - ? [ - scaleAdjustPatchReference.targetDraftUrl, - ...(includeScaleAdjustPatchSourceReference - ? [scaleAdjustPatchReference.sourceCropUrl] - : []), - ...scaleAdjustCropReferenceImageUrls, - ...manualRegionReferenceImageUrls, - ] - : [ - scaleAdjustPatchReference.sourceCropUrl, - scaleAdjustPatchReference.targetDraftUrl, - ...scaleAdjustCropReferenceImageUrls, - ...manualRegionReferenceImageUrls, - ] - : [ - sourceImageUrl, - ...(sendScaleAdjustAnnotatedReferenceToModel && annotatedUpload?.url - ? [annotatedUpload.url] - : []), - ...scaleAdjustCropReferenceImageUrls, - ...manualRegionReferenceImageUrls, - ] - : [sourceImageUrl, ...manualRegionReferenceImageUrls]; - const scaleAdjustCompositeRegions = isScaleAdjustOperation - ? buildScaleAdjustCompositeRegions(regions, scalePercent) - : []; - const scaleAdjustPatchCompositeRegions = isScaleAdjustOperation - ? buildScaleAdjustPatchCompositeRegions(regions, scalePercent) - : []; - const scaleAdjustProtectedChangeRegions = - isScaleAdjustOperation && scaleAdjustPatchReference - ? buildScaleAdjustProtectedChangeRegions(regions, scalePercent) - : []; - const scaleAdjustProtectedRegions = - isScaleAdjustOperation && scaleAdjustPatchReference - ? scaleAdjustProtectedChangeRegions - : scaleAdjustCompositeRegions; - const scaleAdjustVerificationConfig = isScaleAdjustOperation - ? { - allowOutsideChangedWhenChangedInside: false, - allowDeterministicFallback: false, - protectedRegions: scaleAdjustProtectedRegions, - rejectFailedCandidates: true, - sourceImageUrl, - regions: regions.map((region) => region.rect), - regionPrompts: regions.map((region) => region.prompt), - ...(typeof scalePercent === "number" - ? { scalePercent: scalePercent as ScaleAdjustPercent } - : {}), - ...(hasScaleAdjustTargetRects - ? { - targetRegions: regions - .map((region) => region.targetRect) - .filter( - ( - rect - ): rect is NonNullable => - Boolean(rect) - ), - } - : {}), - ...(scaleAdjustPatchReference?.compositeClipMaskUrl - ? { - oldFootprintMaskImageUrl: - scaleAdjustPatchReference.compositeClipMaskUrl, - } - : {}), - } - : undefined; + const referenceImages = [sourceImageUrl, ...manualRegionReferenceImageUrls]; const requestedImageSize = typeof body.imageSize === "string" && body.imageSize.trim() ? body.imageSize.trim() : "2K"; - const resolvedRequestImageSize = - scaleAdjustPatchReference?.structureLockedDraft === true - ? requestedImageSize - : scaleAdjustPatchReference - ? "1K" - : requestedImageSize; const imageRequest: CanvasGenerateImageRequest = { - allowFailover: isScaleAdjustOperation ? true : undefined, - aspectRatio: scaleAdjustPatchReference && !useScaleAdjustFullFrameStructurePatch - ? inferClosestAspectRatio( - scaleAdjustPatchReference.workspacePixelRect.width, - scaleAdjustPatchReference.workspacePixelRect.height - ) - : resolvedAspectRatio, + aspectRatio: resolvedAspectRatio, billingFeature: "repaint", - debugLabel: - operation === SCALE_ADJUST_OPERATION ? "canvas-scale-adjust" : "canvas-local-edit", + debugLabel: "canvas-local-edit", debugMeta: { - annotatedReferenceUrl: annotatedUpload?.url || null, - annotatedReferenceSentToModel: sendScaleAdjustAnnotatedReferenceToModel, - annotatedReferenceType: isScaleAdjustOperation - ? useScaleAdjustFullFrameStructurePatch - ? useScaleAdjustProductLockComposite - ? useScaleAdjustProductLockMaskEdit - ? "scale-adjust-product-lock-mask-edit" - : "scale-adjust-product-lock-background-plate" - : "scale-adjust-full-frame-structure-target-draft" - : useProviderTextOnlyCoordinatePrompt - ? "scale-adjust-text-coordinate-only-debug-map" - : useScaleAdjustCoordinatePrompt - ? useScaleAdjustVisualLayoutGuide - ? "scale-adjust-visual-layout-guide" - : useScaleAdjustMarkedPhotoGuide - ? "scale-adjust-marked-photo-guide" - : "scale-adjust-neutral-coordinate-map" - : "scale-adjust-full-frame-semantic-edit-map" - : "source-overlay", - scaleAdjustEditMaskApplied: Boolean(scaleAdjustEditMaskUpload), - scaleAdjustEditMaskUrl: scaleAdjustEditMaskUpload?.url || null, - scaleAdjustEditMaskProviderType: - useScaleAdjustEditMask - ? resolvedScaleAdjustModelEntry?.provider?.type || null - : null, - cleanSourceReferenceIndex: - scaleAdjustPatchReference && useScaleAdjustFullFrameStructurePatch - ? 1 - : scaleAdjustPatchReference && - useScaleAdjustPatchTargetDraftPrimary && - !includeScaleAdjustPatchSourceReference - ? null - : scaleAdjustPatchReference && useScaleAdjustPatchTargetDraftPrimary - ? 2 - : 1, - configuredDetailModelId: configuredDetailModel || null, - detailFixedModelApplied: Boolean(detailFixedModel), - detailFixedModelId: detailFixedModel || null, + annotatedReferenceUrl: annotatedUpload.url, + annotatedReferenceType: "source-overlay", displayPrompt, operation, requestedModel: requestedModel || null, resolvedAspectRatio, - scaleAdjustPatchWorkflowApplied: Boolean(scaleAdjustPatchReference), - scaleAdjustPatchReferenceFailed, - scaleAdjustPatchForcedForShrink: - Boolean(scaleAdjustPatchReference) && hasScaleAdjustShrink, - scaleAdjustPatchStructureLockedDraftApplied: - scaleAdjustPatchReference?.structureLockedDraft === true, - scaleAdjustFullFrameStructurePatchApplied: - useScaleAdjustFullFrameStructurePatch, - scaleAdjustFullFrameFalEditApplied: useScaleAdjustFullFrameFalEdit, - scaleAdjustFullFrameFalEditDisabledForShrink: - isScaleAdjustFalNanoBananaEdit && hasScaleAdjustShrink, - scaleAdjustPatchReferenceOrder: - scaleAdjustPatchReference && useScaleAdjustFullFrameStructurePatch - ? useScaleAdjustProductLockMaskEdit - ? "full-source-product-lock-edit-mask" - : "full-source-full-target-draft" - : scaleAdjustPatchReference && useScaleAdjustPatchTargetDraftPrimary - ? includeScaleAdjustPatchSourceReference - ? "target-draft-source-identity" - : "target-draft-identity" - : scaleAdjustPatchReference - ? "source-target-draft-identity" - : null, - scaleAdjustPatchSourceCropUrl: - scaleAdjustPatchReference?.sourceCropUrl || null, - scaleAdjustPatchTargetDraftUrl: - scaleAdjustPatchReference?.targetDraftUrl || null, - scaleAdjustPatchTargetDraftFullUrl: - scaleAdjustPatchReference?.targetDraftFullUrl || null, - scaleAdjustBackgroundPlateFullUrl: - scaleAdjustPatchReference?.backgroundPlateFullUrl || null, - scaleAdjustProductLockApplied: - useScaleAdjustProductLockComposite, - scaleAdjustProductLockMethod: - scaleAdjustPatchReference?.productLockMethod || null, - scaleAdjustProductLockMaskEditApplied: - useScaleAdjustProductLockMaskEdit, - scaleAdjustRequiresProviderMaskInput: - useScaleAdjustProductLockMaskEdit, - scaleAdjustProductLockEditMaskUrl: - scaleAdjustPatchReference?.productLockEditMaskUrl || null, - scaleAdjustProductLockFalEditMaskUrl: - scaleAdjustPatchReference?.productLockFalEditMaskUrl || null, - scaleAdjustProductLockOverlayUrl: - scaleAdjustPatchReference?.productLockOverlayUrl || null, - scaleAdjustProductLockProtectMaskUrl: - scaleAdjustPatchReference?.productLockProtectMaskUrl || null, - scaleAdjustPatchCompositeClipMaskUrl: - scaleAdjustPatchReference?.compositeClipMaskUrl || null, - scaleAdjustPatchCompositeClipMaskCropUrl: - scaleAdjustPatchReference?.compositeClipMaskCropUrl || null, - scaleAdjustRepairBlendMaskUrl: - scaleAdjustPatchReference?.repairBlendMaskUrl || null, - scaleAdjustProviderMaskImageUrl: useScaleAdjustProductLockMaskEdit - ? scaleAdjustPatchReference?.productLockEditMaskUrl || null - : scaleAdjustEditMaskUpload?.url || null, - scaleAdjustPatchWorkspacePixelRect: - scaleAdjustPatchReference?.workspacePixelRect || null, - scaleAdjustPatchWorkspaceRect: - scaleAdjustPatchReference?.workspaceRect || null, - resolvedModel: resolvedModel || null, - resolvedScaleAdjustProviderType: - resolvedScaleAdjustModelEntry?.provider?.type || null, - scaleAdjustModelFamily: scaleAdjustModelFamily || null, - scaleAdjustCoordinateGuideApplied: useScaleAdjustCoordinateGuide, - scaleAdjustCoordinatePromptApplied: - useScaleAdjustCoordinatePrompt || useProviderTextOnlyCoordinatePrompt, - scaleAdjustTextOnlyCoordinatePromptApplied: - useProviderTextOnlyCoordinatePrompt, - scaleAdjustNeutralCoordinateMapApplied: - isScaleAdjustOperation && - useScaleAdjustCoordinatePrompt && - !useScaleAdjustMarkedPhotoGuide && - !useScaleAdjustVisualLayoutGuide, - scaleAdjustMarkedPhotoGuideApplied: useScaleAdjustMarkedPhotoGuide, - scaleAdjustFalMarkedPhotoGuideApplied: useFalScaleAdjustCoordinatePrompt, - scaleAdjustVisualLayoutGuideApplied: - isScaleAdjustOperation && - !useProviderTextOnlyCoordinatePrompt && - (!useScaleAdjustCoordinatePrompt || useScaleAdjustVisualLayoutGuide), - falScaleAdjustCoordinateGuideApplied: - useScaleAdjustCoordinateGuide && - resolvedScaleAdjustModelEntry?.provider?.type === "fal-image", - falScaleAdjustSafePromptApplied: useFalScaleAdjustCoordinatePrompt, - falScaleAdjustPatchSafePromptApplied: useFalScaleAdjustPatchPrompt, - scaleAdjustProviderFallbackPolicy: isScaleAdjustOperation - ? "single-output-provider-error-fallback-same-family-fal-last" - : undefined, - scaleAdjustQualityFailoverPolicy: isScaleAdjustOperation - ? "disabled-reject-unchanged-ai-candidate" - : undefined, - scaleAdjustPreferSameNameProviderFailover: - isScaleAdjustOperation, - scaleAdjustStrictSameFamilyFailover: - isScaleAdjustOperation, manualRegionReferenceImageCount: manualRegionReferenceImageUrls.length, - regionReferenceImageCount: - referenceImages.length, + regionReferenceImageCount: referenceImages.length, regionCount: regions.length, - scalePercent, - scaleAdjustCropReferenceCount: scaleAdjustCropReferenceImageUrls.length, - scaleAdjustProtectionMode: hasScaleAdjustTargetRects - ? "target-rect" - : "scale-percent", - scaleAdjustTargetRegionCount: regions.filter((region) => region.targetRect) - .length, - scaleAdjustPromptStrategy: isScaleAdjustOperation - ? scaleAdjustPatchReference - ? useScaleAdjustProductLockComposite - ? useScaleAdjustProductLockMaskEdit - ? "product-lock-mask-edit-rmbg-source-patch-one-shot" - : "product-lock-background-plate-rmbg-v2-source-patch-one-shot" - : useScaleAdjustFullFrameStructurePatch - ? "full-frame-source-locked-target-guide-v8-one-shot" - : scaleAdjustPatchReference.structureLockedDraft - ? "ai-patch-workspace-structure-locked-draft-v87-subject-alpha-clipped-one-shot" - : "ai-patch-workspace-draft-v75-soft-wipe-erase-ghost-one-shot" - : scaleAdjustPatchReferenceFailed - ? "source-only-coordinate-fallback-after-reference-upload-failure-v1-one-shot" - : useScaleAdjustEditMask - ? "mask-edit-gpt-image-2-scale-shrink-v74-one-shot" - : useFalScaleAdjustCoordinatePrompt - ? "fal-safe-marked-photo-box-guide-no-old-crop-v44-one-shot" - : useScaleAdjustVisualLayoutGuide - ? "same-name-focused-layout-crop-guide-v54-one-shot" - : useProviderTextOnlyCoordinatePrompt - ? "provider-text-coordinate-crop-v61-one-shot" - : useScaleAdjustMarkedPhotoGuide - ? "same-name-marked-photo-box-guide-no-old-crop-v45-one-shot" - : "locked-plate-semantic-target-map-no-old-crop-v15" - : undefined, - scaleAdjustPostCompositeApplied: - Boolean(scaleAdjustPatchReference) && - (!useScaleAdjustFullFrameStructurePatch || - useScaleAdjustProductLockComposite), - scaleAdjustUsesFullFrameOutput: - isScaleAdjustOperation && - (!scaleAdjustPatchReference || useScaleAdjustFullFrameStructurePatch), sourceHeight, sourceNodeId: body.sourceNodeId || null, sourceWidth, }, displayPrompt, - falSafetyTolerance: useScaleAdjustCoordinateGuide ? "6" : undefined, - falSystemPrompt: - useScaleAdjustCoordinatePrompt - ? buildScaleAdjustCoordinateSystemPrompt() - : undefined, - falThinkingLevel: useScaleAdjustCoordinateGuide ? "high" : undefined, imageCount: 1, - imageSize: resolvedRequestImageSize, - model: resolvedModel || undefined, + imageSize: requestedImageSize, + model: requestedModel || undefined, negativePrompt: - isScaleAdjustOperation - ? scaleAdjustPatchReference - ? useScaleAdjustFullFrameStructurePatch - ? useScaleAdjustProductLockComposite - ? scaleAdjustProductLockBackgroundNegativePrompt - : scaleAdjustFullFramePatchNegativePrompt - : useFalScaleAdjustPatchPrompt - ? scaleAdjustFalPatchNegativePrompt - : scaleAdjustPatchNegativePrompt - : useProviderTextOnlyCoordinatePrompt - ? scaleAdjustTextOnlyNegativePrompt - : useScaleAdjustCoordinatePrompt - ? scaleAdjustCoordinateNegativePrompt - : "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、箭头、连线、角点、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、蓝色色块、色条、灰底坐标图、目标占位叠图、淡化痕迹、选区边框、白边、画幅压扁、画面拉伸、整张图缩放、全图放大、全图裁切、全图平移、文字位置变化、文字内容变化、文字模糊、Logo位置变化、主体拉伸、主体压扁、主体变窄、主体变高但变窄、横向挤压、纵向拉伸、透视变形、矩形贴片、硬边、拼接线、补丁感、局部裁切、色差边缘、背景断裂、背景错位、窗框错位、文字错位、地毯纹理错位、非主体背景重绘、主体未缩放、整块框选区域缩放" - : "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、选区边框、白边、画幅压扁、画面拉伸、主体截断、动物截断、缺头、缺尾巴、缺四肢、贴边裁切、矩形贴片、硬边、拼接线、补丁感、色差边缘、背景错位、地面纹理断裂、椅脚错位", + "彩色框、蓝色边框、绿色边框、红色边框、橙色边框、矩形框、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、选区边框、白边、画幅压扁、画面拉伸、主体截断、贴边裁切、矩形贴片、硬边、拼接线、补丁感、色差边缘、背景错位、地面纹理断裂、椅脚错位", prompt, promptExtend: false, - providerImageParams: useScaleAdjustProductLockMaskEdit - ? { - falMaskImageUrl: - scaleAdjustPatchReference?.productLockFalEditMaskUrl || - (scaleAdjustPatchReference?.productLockEditMaskUrl as string), - maskImageUrl: - scaleAdjustPatchReference?.productLockEditMaskUrl as string, - } - : scaleAdjustEditMaskUpload - ? { maskImageUrl: scaleAdjustEditMaskUpload.url } - : undefined, referenceImages, - scaleAdjustPatchComposite: - useScaleAdjustProductLockComposite && useScaleAdjustFullFrameStructurePatch - ? { - diffAlphaCutoff: 18, - diffAlphaMultiplier: 2.6, - diffCompareBlurPx: 1.1, - diffHighThreshold: 64, - diffLowThreshold: 20, - featherMaxPx: 34, - featherMinPx: 8, - featherRatio: 0.015, - generatedFrame: "full", - maskMode: "diff", - regions: regions.map((region) => region.rect), - sourceImageUrl, - clipMaskImageUrl: - scaleAdjustPatchReference?.compositeClipMaskUrl || undefined, - workspaceRegion: - (scaleAdjustPatchReference as ScaleAdjustPatchReference) - .workspaceRect, - } - : scaleAdjustPatchReference && !useScaleAdjustFullFrameStructurePatch - ? { - diffAlphaCutoff: - scaleAdjustPatchReference.structureLockedDraft ? 24 : 8, - diffAlphaMultiplier: - scaleAdjustPatchReference.structureLockedDraft ? 2.25 : 3, - diffCompareBlurPx: - scaleAdjustPatchReference.structureLockedDraft ? 1.2 : 0.8, - diffHighThreshold: - scaleAdjustPatchReference.structureLockedDraft ? 72 : 42, - diffLowThreshold: - scaleAdjustPatchReference.structureLockedDraft ? 26 : 14, - featherMaxPx: - scaleAdjustPatchReference.structureLockedDraft ? 36 : 42, - featherMinPx: 10, - featherRatio: 0.018, - maskMode: "diff", - regions: scaleAdjustPatchCompositeRegions, - sourceImageUrl, - clipMaskImageUrl: - scaleAdjustPatchReference.compositeClipMaskUrl || undefined, - workspaceRegion: scaleAdjustPatchReference.workspaceRect, - } - : undefined, - scaleAdjustProductLockComposite: useScaleAdjustProductLockComposite - ? { - overlayImageUrl: - scaleAdjustPatchReference?.productLockOverlayUrl as string, - protectMaskImageUrl: - scaleAdjustPatchReference?.productLockProtectMaskUrl || undefined, - repairMaskImageUrl: - useScaleAdjustFullFrameStructurePatch - ? scaleAdjustPatchReference?.repairBlendMaskUrl || - scaleAdjustPatchReference?.compositeClipMaskUrl || - undefined - : undefined, - sourceImageUrl: useScaleAdjustFullFrameStructurePatch - ? sourceImageUrl - : undefined, - sourceHeight, - sourceWidth, - } - : undefined, - scaleAdjustVerification: scaleAdjustVerificationConfig, - localEditComposite: isScaleAdjustOperation - ? undefined - : { - sourceImageUrl, - regions: regions.map((region) => region.rect), - }, + localEditComposite: { + sourceImageUrl, + regions: regions.map((region) => region.rect), + }, ownerKey: session.user.id, }; @@ -6143,7 +882,7 @@ export async function POST(request: NextRequest) { return NextResponse.json({ success: true, - annotatedReferenceUrl: annotatedUpload?.url || null, + annotatedReferenceUrl: annotatedUpload.url, aspectRatio: resolvedAspectRatio, displayPrompt, prompt: displayPrompt, From 3252cbd2c829a745f942aa1d3553a2ed1e800885 Mon Sep 17 00:00:00 2001 From: 134 Date: Fri, 10 Jul 2026 12:57:01 +0800 Subject: [PATCH 63/70] feat: refine canvas workflows and unify application UI --- .../_components/access-security-dashboard.tsx | 698 + .../_components/admin-settings-navigation.tsx | 56 +- .../_components/local-ai-command-center.tsx | 42 +- .../_components/local-ai-section-shell.tsx | 24 +- .../local-ai-sections/planning-sections.tsx | 20 +- .../_components/setting-status-badge.tsx | 2 +- .../settings/_lib/settings-navigation.ts | 7 + .../settings/admin-settings-client.tsx | 131 +- .../agent-workflow-settings-client.tsx | 27 +- .../psd-templates-settings-client.tsx | 25 +- .../style-pro/style-pro-settings-client.tsx | 14 +- .../video-canvas-settings-client.tsx | 180 +- app/(auth)/auth.ts | 32 +- app/(chat)/api/files/direct-upload/route.ts | 18 +- app/(chat)/api/models/route.ts | 72 + app/api/admin/access-blocklist/route.ts | 207 + app/api/admin/access-logs/route.ts | 271 + app/api/admin/settings/route.ts | 24 +- app/api/canvas/[id]/route.ts | 63 +- app/api/canvas/workflow/analyze/route.ts | 456 +- app/api/favicon/route.ts | 9 + app/api/internal/access-log/route.ts | 147 + app/api/internal/ip-blocklist/check/route.ts | 140 + app/api/pick-object/route.ts | 119 +- app/api/site-settings/route.ts | 30 + app/api/user/membership-summary/route.ts | 30 +- app/api/video-canvas/config/route.ts | 4 + .../[id]/CanvasEditorLegacyPageClient.tsx | 31 +- app/canvas/[id]/CanvasPageClientRouter.tsx | 2 +- app/canvas/[id]/page.tsx | 11 +- app/globals.css | 136 + components/canvas/CanvasChatPanel.tsx | 132 +- .../canvas/LiblibVideoCanvasReplica.tsx | 14161 +++++++++++++++- components/elements/actions.tsx | 2 +- components/elements/conversation.tsx | 2 +- components/elements/message.tsx | 4 +- components/elements/prompt-input.tsx | 4 +- components/elements/response.tsx | 12 +- components/elements/suggestion.tsx | 2 +- components/home/home-animation-embed.tsx | 36 +- components/home/lovart-home.tsx | 100 +- .../home/lovart/lovart-featured-gallery.tsx | 10 +- .../home/lovart/lovart-hero-composer.tsx | 69 +- .../home/lovart/lovart-projects-section.tsx | 26 +- components/home/lovart/lovart-sidebar.tsx | 32 +- components/layout/brand-top-bar.tsx | 12 +- components/ui/button.tsx | 10 +- components/ui/card.tsx | 12 +- components/ui/dialog.tsx | 14 +- components/ui/input.tsx | 2 +- components/ui/label.tsx | 1 + components/ui/popover.tsx | 2 +- components/ui/select.tsx | 10 +- components/ui/sheet.tsx | 12 +- components/ui/tabs.tsx | 5 +- components/ui/textarea.tsx | 2 +- components/ui/tooltip.tsx | 2 +- docs/Claw模式技能使用宣导.md | 221 + docs/Claw模式技能使用宣导.txt | 197 + docs/Claw模式技能使用宣导2.pdf | Bin 0 -> 191290 bytes docs/video-canvas-competitor-audit.md | 364 + lib/ai/detail-agent-kimi-profile.ts | 11 +- lib/ai/detail-agent-model-config.ts | 55 +- lib/ai/providers/fal-image.ts | 9 + lib/ai/qwen3-vl.ts | 11 +- lib/ai/tools/generate-video.ts | 218 +- lib/client/upload-with-retry.ts | 2 +- .../migrations/0087_add_access_ip_events.sql | 35 + .../0088_add_access_ip_blocklist.sql | 20 + lib/db/migrations/meta/_journal.json | 14 + lib/db/queries.ts | 12 +- lib/db/schema.ts | 37 + lib/security/access-log.ts | 200 + lib/security/manual-ip-blocklist.ts | 88 + lib/storage/service.ts | 3 +- lib/video-canvas/config.ts | 207 + proxy.ts | 501 +- public/video-canvas/effects/bee-camera.webp | Bin 0 -> 3050362 bytes public/video-canvas/effects/cloud-dive.webp | Bin 0 -> 2110984 bytes public/video-canvas/effects/earth-dive.webp | Bin 0 -> 1934196 bytes public/video-canvas/effects/earth-zoom.webp | Bin 0 -> 1315378 bytes public/video-canvas/effects/global-zoom.webp | Bin 0 -> 1926700 bytes .../video-canvas/effects/horizon-flight.webp | Bin 0 -> 1784842 bytes public/video-canvas/effects/iris-push.webp | Bin 0 -> 1652144 bytes public/video-canvas/effects/multi-angle.webp | Bin 0 -> 1754590 bytes public/video-canvas/effects/prada-change.webp | Bin 0 -> 1407378 bytes .../effects/product-light-sweep.webp | Bin 0 -> 1020162 bytes .../video-canvas/effects/reverse-gravity.webp | Bin 0 -> 2052752 bytes .../video-canvas/effects/underwater-slow.webp | Bin 0 -> 1872114 bytes .../storage/configure-direct-upload-cors.ts | 147 +- .../storage/configure-multipart-lifecycle.ts | 181 +- services/canvas/index.ts | 12 + 92 files changed, 18716 insertions(+), 1533 deletions(-) create mode 100644 app/(admin)/settings/_components/access-security-dashboard.tsx create mode 100644 app/api/admin/access-blocklist/route.ts create mode 100644 app/api/admin/access-logs/route.ts create mode 100644 app/api/internal/access-log/route.ts create mode 100644 app/api/internal/ip-blocklist/check/route.ts create mode 100644 docs/Claw模式技能使用宣导.md create mode 100644 docs/Claw模式技能使用宣导.txt create mode 100644 docs/Claw模式技能使用宣导2.pdf create mode 100644 docs/video-canvas-competitor-audit.md create mode 100644 lib/db/migrations/0087_add_access_ip_events.sql create mode 100644 lib/db/migrations/0088_add_access_ip_blocklist.sql create mode 100644 lib/security/access-log.ts create mode 100644 lib/security/manual-ip-blocklist.ts create mode 100644 public/video-canvas/effects/bee-camera.webp create mode 100644 public/video-canvas/effects/cloud-dive.webp create mode 100644 public/video-canvas/effects/earth-dive.webp create mode 100644 public/video-canvas/effects/earth-zoom.webp create mode 100644 public/video-canvas/effects/global-zoom.webp create mode 100644 public/video-canvas/effects/horizon-flight.webp create mode 100644 public/video-canvas/effects/iris-push.webp create mode 100644 public/video-canvas/effects/multi-angle.webp create mode 100644 public/video-canvas/effects/prada-change.webp create mode 100644 public/video-canvas/effects/product-light-sweep.webp create mode 100644 public/video-canvas/effects/reverse-gravity.webp create mode 100644 public/video-canvas/effects/underwater-slow.webp diff --git a/app/(admin)/settings/_components/access-security-dashboard.tsx b/app/(admin)/settings/_components/access-security-dashboard.tsx new file mode 100644 index 000000000..2662cfeb7 --- /dev/null +++ b/app/(admin)/settings/_components/access-security-dashboard.tsx @@ -0,0 +1,698 @@ +"use client"; + +import { + Activity, + AlertTriangle, + Ban, + Clock3, + Globe2, + LogIn, + Plus, + RefreshCw, + Search, + ShieldAlert, + Unlock, + UserCheck, +} from "lucide-react"; +import { useMemo, useState } from "react"; +import { toast } from "sonner"; +import useSWR from "swr"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { Input } from "@/components/ui/input"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { cn, fetcher } from "@/lib/utils"; + +type Severity = "info" | "watch" | "high" | "critical"; + +type AccessLogEvent = { + id: string; + eventType: string; + ip: string; + userEmail: string | null; + userRole: string | null; + method: string | null; + path: string | null; + routeGroup: string | null; + statusCode: number | null; + outcome: string; + severity: Severity; + reasons: string[] | null; + userAgent: string | null; + createdAt: string; +}; + +type AccessLogTopIp = { + ip: string; + total: number; + uniqueUsers: number; + failureCount: number; + rateLimitedCount: number; + authFailureCount: number; + loginSuccessCount: number; + suspiciousCount: number; + firstSeen: string; + lastSeen: string; + severity: Severity; + failureRate: number; + reasons: string[]; +}; + +type AccessLogResponse = { + overview: { + totalEvents: number; + uniqueIps: number; + abnormalEvents: number; + abnormalIps: number; + rateLimitedEvents: number; + manualBlockedEvents: number; + loginSuccessEvents: number; + unauthorizedEvents: number; + }; + topIps: AccessLogTopIp[]; + hourlyStats: Array<{ + hour: string; + count: number; + abnormalCount: number; + }>; + pathStats: Array<{ + path: string | null; + routeGroup: string | null; + count: number; + abnormalCount: number; + }>; + recentEvents: AccessLogEvent[]; + rules: string[]; +}; + +type AccessBlocklistEntry = { + id: string; + ip: string; + reason: string | null; + isActive: boolean; + expiresAt: string | null; + createdAt: string; + updatedAt: string; + createdByEmail: string | null; +}; + +const severityLabels: Record = { + info: "正常", + watch: "关注", + high: "异常", + critical: "高危", +}; + +const severityClassNames: Record = { + info: "border-emerald-200 bg-emerald-50 text-emerald-700", + watch: "border-amber-200 bg-amber-50 text-amber-700", + high: "border-orange-200 bg-orange-50 text-orange-700", + critical: "border-red-200 bg-red-50 text-red-700", +}; + +function formatNumber(value: number | null | undefined) { + return new Intl.NumberFormat("zh-CN").format(Number(value || 0)); +} + +function formatPercent(value: number | null | undefined) { + return `${Math.round(Number(value || 0) * 100)}%`; +} + +function formatTime(value: string | null | undefined) { + if (!value) return "--"; + const date = new Date(value); + if (Number.isNaN(date.getTime())) return "--"; + return new Intl.DateTimeFormat("zh-CN", { + month: "2-digit", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + }).format(date); +} + +function getStatusClassName(statusCode: number | null) { + if (!statusCode) return "text-muted-foreground"; + if (statusCode >= 500) return "text-red-700"; + if (statusCode >= 400) return "text-orange-700"; + if (statusCode >= 300) return "text-sky-700"; + return "text-emerald-700"; +} + +function AccessSeverityBadge({ severity }: { severity: Severity }) { + return ( + + {severityLabels[severity]} + + ); +} + +export function AccessSecurityDashboard() { + const [hours, setHours] = useState("24"); + const [severity, setSeverity] = useState("all"); + const [query, setQuery] = useState(""); + const [blockIp, setBlockIp] = useState(""); + const [blockReason, setBlockReason] = useState(""); + const [isSavingBlock, setIsSavingBlock] = useState(false); + + const accessLogUrl = useMemo(() => { + const params = new URLSearchParams({ + hours, + severity, + limit: "160", + }); + if (query.trim()) params.set("q", query.trim()); + return `/api/admin/access-logs?${params.toString()}`; + }, [hours, query, severity]); + + const { data, isLoading, mutate } = useSWR( + accessLogUrl, + fetcher, + { refreshInterval: 30_000 } + ); + const { data: blocklistData, mutate: mutateBlocklist } = useSWR<{ + entries: AccessBlocklistEntry[]; + }>("/api/admin/access-blocklist", fetcher); + + const activeBlocklistEntries = useMemo( + () => (blocklistData?.entries || []).filter((entry) => entry.isActive), + [blocklistData?.entries] + ); + const blockedRules = useMemo( + () => new Set(activeBlocklistEntries.map((entry) => entry.ip.toLowerCase())), + [activeBlocklistEntries] + ); + + const saveBlockEntry = async (targetIp?: string, targetReason?: string) => { + const ip = String(targetIp || blockIp).trim(); + const reason = String(targetReason ?? blockReason).trim(); + if (!ip) { + toast.error("请输入要拉黑的 IP"); + return; + } + + setIsSavingBlock(true); + try { + const response = await fetch("/api/admin/access-blocklist", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ ip, reason }), + }); + const payload = await response.json().catch(() => ({})); + if (!response.ok) { + throw new Error(payload?.message || payload?.error || "拉黑失败"); + } + + toast.success(`${ip} 已加入手动黑名单`); + setBlockIp(""); + setBlockReason(""); + await Promise.all([mutateBlocklist(), mutate()]); + } catch (error) { + toast.error(error instanceof Error ? error.message : "拉黑失败"); + } finally { + setIsSavingBlock(false); + } + }; + + const updateBlockEntry = async (entry: AccessBlocklistEntry, isActive: boolean) => { + try { + const response = await fetch("/api/admin/access-blocklist", { + method: "PATCH", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + id: entry.id, + isActive, + reason: entry.reason || "", + expiresAt: entry.expiresAt || null, + }), + }); + const payload = await response.json().catch(() => ({})); + if (!response.ok) { + throw new Error(payload?.message || payload?.error || "更新失败"); + } + + toast.success(isActive ? `${entry.ip} 已重新拉黑` : `${entry.ip} 已解除拉黑`); + await Promise.all([mutateBlocklist(), mutate()]); + } catch (error) { + toast.error(error instanceof Error ? error.message : "更新失败"); + } + }; + + const overviewCards = [ + { + key: "events", + label: "访问记录", + value: formatNumber(data?.overview.totalEvents), + note: `${hours} 小时窗口`, + icon: Activity, + className: "border-slate-200 bg-slate-50 text-slate-900", + }, + { + key: "ips", + label: "独立 IP", + value: formatNumber(data?.overview.uniqueIps), + note: `${formatNumber(data?.overview.abnormalIps)} 个异常标注`, + icon: Globe2, + className: "border-sky-200 bg-sky-50 text-sky-900", + }, + { + key: "abnormal", + label: "异常事件", + value: formatNumber(data?.overview.abnormalEvents), + note: `${formatNumber(data?.overview.manualBlockedEvents)} 次手动阻断`, + icon: ShieldAlert, + className: "border-orange-200 bg-orange-50 text-orange-900", + }, + { + key: "login", + label: "成功登录", + value: formatNumber(data?.overview.loginSuccessEvents), + note: `${formatNumber(data?.overview.unauthorizedEvents)} 次未授权`, + icon: LogIn, + className: "border-emerald-200 bg-emerald-50 text-emerald-900", + }, + ]; + + const maxHourlyCount = Math.max( + 1, + ...(data?.hourlyStats || []).map((item) => item.count) + ); + const maxPathCount = Math.max( + 1, + ...(data?.pathStats || []).map((item) => item.count) + ); + + return ( +
+ + +
+
+ 访问安全分析 + + IP 访问记录、登录来源、异常标注与请求趋势 + +
+
+
+ + setQuery(event.target.value)} + placeholder="搜索 IP、路径、用户" + value={query} + /> +
+ + + +
+
+ + +
+ {overviewCards.map((item) => { + const Icon = item.icon; + return ( +
+
+
{item.label}
+ +
+
+ {item.value} +
+
{item.note}
+
+ ); + })} +
+ +
+
+
+ + 小时趋势 +
+
+ 橙色表示异常事件占比 +
+
+
+ {(data?.hourlyStats || []).map((item) => { + const height = Math.max(4, (item.count / maxHourlyCount) * 100); + const abnormalHeight = + item.count > 0 + ? Math.max(0, (item.abnormalCount / item.count) * height) + : 0; + return ( +
+
+
+ {item.hour} · {formatNumber(item.count)} 次 · 异常{" "} + {formatNumber(item.abnormalCount)} +
+
+ ); + })} + {(!data?.hourlyStats || data.hourlyStats.length === 0) && ( +
+ 暂无访问记录 +
+ )} +
+
+ + + + + + 手动 IP 黑名单 + + 管理员手动加入后，命中 IP 会被阻断并写入访问日志 + + + +
+ setBlockIp(event.target.value)} + placeholder="IP 或 IPv4 CIDR" + value={blockIp} + /> + setBlockReason(event.target.value)} + placeholder="拉黑原因" + value={blockReason} + /> + +
+ +
+ {activeBlocklistEntries.map((entry) => ( +
+
+
+
+ + + {entry.ip} + +
+
+ {entry.reason || "未填写原因"} +
+
+ {formatTime(entry.updatedAt)} +
+
+ +
+
+ ))} + {activeBlocklistEntries.length === 0 ? ( +
+ 暂无手动拉黑 IP +
+ ) : null} +
+ + + +
+ + + 异常 IP 标注 + + 按限流、失败率、认证异常、敏感路径和请求量综合排序 + + + +
+ {(data?.topIps || []).map((item) => ( +
+
+
+
+ + {item.ip} + + +
+
+ {formatNumber(item.total)} 次访问 + {formatNumber(item.uniqueUsers)} 个用户 + 失败率 {formatPercent(item.failureRate)} + 最近 {formatTime(item.lastSeen)} +
+
+
+ {(item.reasons.length > 0 ? item.reasons : ["正常访问"]).map( + (reason) => ( + + {reason} + + ) + )} + +
+
+
+ ))} + {(!data?.topIps || data.topIps.length === 0) && ( +
+ 暂无 IP 聚合数据 +
+ )} +
+ + + + + + 访问路径 + 按请求数量排序 + + +
+ {(data?.pathStats || []).map((item) => { + const abnormalRate = + item.count > 0 ? item.abnormalCount / item.count : 0; + return ( +
+
+ + {item.path || "/"} + + + {formatNumber(item.count)} + +
+
+
+
+
+
+ ); + })} + {(!data?.pathStats || data.pathStats.length === 0) && ( +
+ 暂无路径统计 +
+ )} +
+ + +
+ + + +
+
+ 访问日志列表 + 最近 {formatNumber(data?.recentEvents.length)} 条 +
+
+ {(data?.rules || []).map((rule) => ( + + {rule} + + ))} +
+
+ + +
+
+
+
时间
+
IP
+
状态
+
路径
+
用户
+
标注
+
+ {(data?.recentEvents || []).map((event) => ( +
+
+ {formatTime(event.createdAt)} +
+
{event.ip}
+
+ {event.statusCode || "--"} +
+
+
+ {event.method || "GET"} {event.path || "/"} +
+
+ {event.reasons?.join("、") || event.outcome} +
+
+
+
+ {event.userEmail || "匿名"} +
+
+ + {event.userRole || "unknown"} +
+
+
+ {event.severity === "high" || event.severity === "critical" ? ( + + ) : null} + +
+
+ ))} + {(!data?.recentEvents || data.recentEvents.length === 0) && ( +
+ 暂无访问日志 +
+ )} +
+
+ + +
+ ); +} diff --git a/app/(admin)/settings/_components/admin-settings-navigation.tsx b/app/(admin)/settings/_components/admin-settings-navigation.tsx index 346e073b7..f449fa28f 100644 --- a/app/(admin)/settings/_components/admin-settings-navigation.tsx +++ b/app/(admin)/settings/_components/admin-settings-navigation.tsx @@ -33,8 +33,8 @@ function SettingsQuickAction({ ? "border-foreground/10 bg-foreground text-background shadow-sm hover:!bg-foreground/90 hover:!text-background dark:border-white/20 dark:bg-white dark:text-black dark:hover:!bg-white/90 dark:hover:!text-black" : "border-transparent bg-transparent text-muted-foreground hover:!border-border/80 hover:!bg-zinc-100 hover:!text-foreground dark:hover:!bg-zinc-900/50"; const layoutClassName = stretched - ? "h-10 w-full min-w-0 justify-center gap-2 rounded-2xl border px-4" - : "h-9 shrink-0 gap-2 rounded-full border px-3.5"; + ? "h-8 w-full min-w-0 justify-center gap-1.5 rounded-md border px-2.5 text-xs" + : "h-8 shrink-0 gap-1.5 rounded-md border px-2.5 text-xs"; if (item.href) { return ( @@ -45,7 +45,7 @@ function SettingsQuickAction({ variant="ghost" > - + {item.label} @@ -59,7 +59,7 @@ function SettingsQuickAction({ size="sm" variant="ghost" > - + {item.label} ); @@ -84,11 +84,14 @@ export function AdminSettingsNavigation({ }; return ( -
-
+