From 4c86e01956e6501b4ffb09977351db13fbe8d5c1 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 20 May 2026 11:01:36 +0800 Subject: [PATCH] feat: add reference images to local edit regions --- app/api/canvas/local-edit/route.ts | 75 +++- components/canvas/nodes/ImageNode.tsx | 551 ++++++++++++++++++-------- services/canvas/index.ts | 6 + 3 files changed, 474 insertions(+), 158 deletions(-) diff --git a/app/api/canvas/local-edit/route.ts b/app/api/canvas/local-edit/route.ts index 733e91699..d2c70259e 100644 --- a/app/api/canvas/local-edit/route.ts +++ b/app/api/canvas/local-edit/route.ts @@ -30,6 +30,12 @@ type LocalEditRegionInput = { color?: string; prompt?: string; rect?: LocalEditRect; + referenceImages?: Array<{ + id?: string; + url?: string; + name?: string; + source?: string; + }>; }; type LocalEditRequest = { @@ -47,6 +53,12 @@ type NormalizedLocalEditRegion = { color: string; colorName: string; prompt: string; + referenceImages: Array<{ + id: string; + url: string; + name?: string; + source: "upload" | "canvas"; + }>; rect: { x: number; y: number; @@ -62,6 +74,7 @@ type NormalizedLocalEditRegion = { }; const MAX_LOCAL_EDIT_REGIONS = 8; +const MAX_REFERENCE_IMAGES_PER_REGION = 3; const MIN_REGION_EDGE_PX = 24; const DEFAULT_REGION_COLORS = [ "#ff3b30", @@ -100,6 +113,41 @@ function normalizeHexColor(input: unknown, fallback: string) { return /^#[0-9a-f]{6}$/i.test(expanded) ? expanded.toLowerCase() : fallback; } +function normalizeHttpImageUrl(input: unknown) { + const raw = String(input || "").trim(); + if (!/^https?:\/\//i.test(raw)) return ""; + return raw; +} + +function normalizeRegionReferenceImages( + region: LocalEditRegionInput, + itemIndex: number +): NormalizedLocalEditRegion["referenceImages"] { + if (!Array.isArray(region.referenceImages)) return []; + return region.referenceImages + .map((reference, referenceIndex) => { + const url = normalizeHttpImageUrl(reference?.url); + if (!url) return null; + const name = + typeof reference?.name === "string" && reference.name.trim() + ? reference.name.trim().slice(0, 80) + : ""; + return { + id: + typeof reference?.id === "string" && reference.id.trim() + ? reference.id.trim() + : `region-${itemIndex + 1}-reference-${referenceIndex + 1}`, + url, + ...(name ? { name } : {}), + source: reference?.source === "canvas" ? "canvas" as const : "upload" as const, + }; + }) + .filter((item): item is NormalizedLocalEditRegion["referenceImages"][number] => + Boolean(item) + ) + .slice(0, MAX_REFERENCE_IMAGES_PER_REGION); +} + function getColorName(hex: string) { const normalized = hex.toLowerCase(); const names: Record = { @@ -203,6 +251,7 @@ function normalizeRegions( color, colorName: getColorName(color), prompt: prompt.slice(0, 300), + referenceImages: normalizeRegionReferenceImages(region, itemIndex), rect, pixelRect, }; @@ -278,18 +327,32 @@ function buildLocalEditPrompt(regions: NormalizedLocalEditRegion[]) { const top = Math.round(region.rect.y * 1000) / 10; const width = Math.round(region.rect.width * 1000) / 10; const height = Math.round(region.rect.height * 1000) / 10; - return `区域 ${region.index}（${region.colorName}，编号 ${region.index}，原图位置 x=${left}%，y=${top}%，w=${width}%，h=${height}%）：${region.prompt}`; + const referenceNote = + region.referenceImages.length > 0 + ? `；该区域有 ${region.referenceImages.length} 张专属参考图，依次为后续输入图片中的“区域 ${region.index} 参考图”，只可用于该区域` + : ""; + return `区域 ${region.index}（${region.colorName}，编号 ${region.index}，原图位置 x=${left}%，y=${top}%，w=${width}%，h=${height}%${referenceNote}）：${region.prompt}`; }) .join("\n"); + const referenceLines = regions + .flatMap((region) => + region.referenceImages.map( + (reference, referenceIndex) => + `区域 ${region.index} 参考图 ${referenceIndex + 1}：${reference.name || "未命名参考图"}，只能影响区域 ${region.index}` + ) + ) + .join("\n"); return [ "输入图片1是原始干净图，也是最终输出必须继承的唯一画面基底。", "输入图片2是同一张图的彩色标注说明图，只用于理解编辑区域位置和区域提示词，不是画面参考。", + "输入图片3及之后如存在，是各区域专属参考图；专属参考图只服务于对应区域，不得改变其它区域或整张图的主体身份。", "请不要复刻、保留、描摹或生成输入图片2里的任何彩色框、编号圆点、提示词气泡、边框线、半透明色块或标注文字。", "如果输入图片2的标注元素与输入图片1冲突，必须完全忽略标注元素，只保留输入图片1的真实画面内容。", "", "请严格根据每个区域下方的提示词，只在对应彩色框附近完成局部编辑：", regionLines, + referenceLines ? `\n区域专属参考图说明：\n${referenceLines}` : "", "", "最终输出必须是干净照片：不得出现任何彩色框、蓝框、绿框、红框、矩形边框、编号、提示词气泡、标注文字或 UI 元素。", "未被彩色框标注的区域必须尽量保持原图不变。", @@ -368,6 +431,9 @@ export async function POST(request: NextRequest) { toArrayBuffer(annotatedBuffer) ); const prompt = buildLocalEditPrompt(regions); + const regionReferenceImageUrls = regions.flatMap((region) => + region.referenceImages.map((reference) => reference.url) + ); const imageRequest: CanvasGenerateImageRequest = { aspectRatio: @@ -378,6 +444,7 @@ export async function POST(request: NextRequest) { debugLabel: "canvas-local-edit", debugMeta: { annotatedReferenceUrl: annotatedUpload.url, + regionReferenceImageCount: regionReferenceImageUrls.length, regionCount: regions.length, sourceNodeId: body.sourceNodeId || null, }, @@ -394,7 +461,11 @@ export async function POST(request: NextRequest) { "彩色框、蓝色边框、绿色边框、红色边框、矩形框、编号圆点、提示词气泡、标注文字、UI 控件、半透明色块、选区边框", prompt, promptExtend: false, - referenceImages: [sourceImageUrl, annotatedUpload.url], + referenceImages: [ + sourceImageUrl, + annotatedUpload.url, + ...regionReferenceImageUrls, + ], ownerKey: session.user.id, }; diff --git a/components/canvas/nodes/ImageNode.tsx b/components/canvas/nodes/ImageNode.tsx index c81b26a93..030b69ffc 100644 --- a/components/canvas/nodes/ImageNode.tsx +++ b/components/canvas/nodes/ImageNode.tsx @@ -248,12 +248,21 @@ interface BoxCutoutPoint { y: number; } +interface LocalEditReferenceImage { + id: string; + url: string; + name?: string; + source?: "upload" | "canvas"; + createdAt: number; +} + interface LocalEditRegion { id: string; index: number; color: string; prompt: string; rect: NormalizedCropRect; + referenceImages?: LocalEditReferenceImage[]; createdAt: number; } @@ -261,6 +270,7 @@ const BOX_CUTOUT_DEFAULT_PADDING_RATIO = 0.08; const BOX_CUTOUT_MIN_EDGE_PX = 24; const LOCAL_EDIT_METADATA_KEY = "localEditRegions"; const LOCAL_EDIT_MAX_REGIONS = 8; +const LOCAL_EDIT_MAX_REFERENCE_IMAGES_PER_REGION = 3; const LOCAL_EDIT_MIN_EDGE_PX = BOX_CUTOUT_MIN_EDGE_PX; const LOCAL_EDIT_POLL_INTERVAL_MS = 2500; const LOCAL_EDIT_MAX_POLL_ATTEMPTS = 180; @@ -875,6 +885,8 @@ const normalizeTextEditItems = ( const createLocalEditRegionId = () => `local-edit-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; +const createLocalEditReferenceImageId = () => + `local-edit-ref-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; const normalizeLocalEditRect = (value: unknown): NormalizedCropRect | null => { if (!value || typeof value !== "object") return null; @@ -896,6 +908,40 @@ const normalizeLocalEditRect = (value: unknown): NormalizedCropRect | null => { return { x, y, width, height }; }; +const normalizeLocalEditReferenceImages = ( + value: unknown +): LocalEditReferenceImage[] => { + if (!Array.isArray(value)) return []; + const references: LocalEditReferenceImage[] = []; + for (const item of value) { + if (!item || typeof item !== "object") continue; + const record = item as Record; + const url = typeof record.url === "string" ? record.url.trim() : ""; + if (!isHttpImageUrl(url)) continue; + const name = + typeof record.name === "string" && record.name.trim() + ? record.name.trim().slice(0, 80) + : ""; + references.push({ + id: + typeof record.id === "string" && record.id.trim() + ? record.id.trim() + : createLocalEditReferenceImageId(), + url, + ...(name ? { name } : {}), + source: record.source === "canvas" ? "canvas" : "upload", + createdAt: + typeof record.createdAt === "number" && Number.isFinite(record.createdAt) + ? record.createdAt + : Date.now(), + }); + if (references.length >= LOCAL_EDIT_MAX_REFERENCE_IMAGES_PER_REGION) { + break; + } + } + return references; +}; + const reindexLocalEditRegions = ( regions: LocalEditRegion[] ): LocalEditRegion[] => @@ -907,38 +953,36 @@ const reindexLocalEditRegions = ( const normalizeLocalEditRegions = (value: unknown): LocalEditRegion[] => { if (!Array.isArray(value)) return []; - return reindexLocalEditRegions( - value - .map((item, index) => { - if (!item || typeof item !== "object") return null; - const record = item as Record; - const rect = normalizeLocalEditRect(record.rect); - if (!rect) return null; - return { - id: - typeof record.id === "string" && record.id.trim() - ? record.id.trim() - : createLocalEditRegionId(), - index: - typeof record.index === "number" && Number.isFinite(record.index) - ? record.index - : index + 1, - color: - typeof record.color === "string" && record.color.trim() - ? record.color.trim() - : LOCAL_EDIT_COLORS[index % LOCAL_EDIT_COLORS.length], - prompt: - typeof record.prompt === "string" ? record.prompt.trim() : "", - rect, - createdAt: - typeof record.createdAt === "number" && Number.isFinite(record.createdAt) - ? record.createdAt - : Date.now(), - } satisfies LocalEditRegion; - }) - .filter((item): item is LocalEditRegion => !!item) - .slice(0, LOCAL_EDIT_MAX_REGIONS) - ); + const regions: LocalEditRegion[] = []; + value.forEach((item, index) => { + if (!item || typeof item !== "object") return; + const record = item as Record; + const rect = normalizeLocalEditRect(record.rect); + if (!rect) return; + regions.push({ + id: + typeof record.id === "string" && record.id.trim() + ? record.id.trim() + : createLocalEditRegionId(), + index: + typeof record.index === "number" && Number.isFinite(record.index) + ? record.index + : index + 1, + color: + typeof record.color === "string" && record.color.trim() + ? record.color.trim() + : LOCAL_EDIT_COLORS[index % LOCAL_EDIT_COLORS.length], + prompt: + typeof record.prompt === "string" ? record.prompt.trim() : "", + rect, + referenceImages: normalizeLocalEditReferenceImages(record.referenceImages), + createdAt: + typeof record.createdAt === "number" && Number.isFinite(record.createdAt) + ? record.createdAt + : Date.now(), + }); + }); + return reindexLocalEditRegions(regions.slice(0, LOCAL_EDIT_MAX_REGIONS)); }; const getLocalEditRegionsFromMetadata = ( @@ -1838,6 +1882,10 @@ export function ImageNode({ const [editingLocalEditRegionId, setEditingLocalEditRegionId] = useState(null); const [isSubmittingLocalEdit, setIsSubmittingLocalEdit] = useState(false); + const [ + uploadingLocalEditReferenceRegionId, + setUploadingLocalEditReferenceRegionId, + ] = useState(null); const anglePreviewDragRef = useRef(null); const activeImageAdjustments = imageAdjustPanelOpen ? imageAdjustmentsDraft @@ -2384,9 +2432,13 @@ export function ImageNode({ const width = Number(style.width || 0); const height = Number(style.height || 0); const imageBottom = boxCutoutMetrics.offsetY + boxCutoutMetrics.renderedHeight; - const placeAbove = top + height + 58 > imageBottom; - const promptTop = placeAbove ? top - 46 : top + height + 8; - const promptWidth = Math.min(Math.max(width, 176), 300); + const referenceCount = region.referenceImages?.length || 0; + const promptStackHeight = referenceCount > 0 ? 82 : 42; + const placeAbove = top + height + promptStackHeight + 14 > imageBottom; + const promptTop = placeAbove + ? top - promptStackHeight - 8 + : top + height + 8; + const promptWidth = Math.min(Math.max(width, 236), 360); return { region, @@ -2467,6 +2519,7 @@ export function ImageNode({ setLocalEditDragEnd(null); setLocalEditRegionInteraction(null); setEditingLocalEditRegionId(null); + setUploadingLocalEditReferenceRegionId(null); }, []); const focusLocalEditPromptInput = useCallback((regionId: string) => { @@ -3402,6 +3455,7 @@ export function ImageNode({ LOCAL_EDIT_COLORS[localEditRegions.length % LOCAL_EDIT_COLORS.length], prompt: "", rect: nextRect, + referenceImages: [], createdAt: Date.now(), }; commitLocalEditRegions([...localEditRegions, nextRegion]); @@ -3618,6 +3672,95 @@ export function ImageNode({ [commitLocalEditRegions, editingLocalEditRegionId] ); + const handleLocalEditReferenceUpload = useCallback( + async (regionId: string, fileList: FileList | null) => { + const files = Array.from(fileList || []).filter((file) => + file.type.startsWith("image/") + ); + if (files.length === 0) return; + + const currentRegion = localEditRegionsRef.current.find( + (region) => region.id === regionId + ); + if (!currentRegion) return; + + const currentReferences = currentRegion.referenceImages || []; + const remainingSlots = + LOCAL_EDIT_MAX_REFERENCE_IMAGES_PER_REGION - currentReferences.length; + if (remainingSlots <= 0) { + toast.info( + `每个局部编辑区域最多添加 ${LOCAL_EDIT_MAX_REFERENCE_IMAGES_PER_REGION} 张参考图` + ); + return; + } + + const selectedFiles = files.slice(0, remainingSlots); + if (files.length > selectedFiles.length) { + toast.info( + `已自动保留前 ${LOCAL_EDIT_MAX_REFERENCE_IMAGES_PER_REGION} 张参考图` + ); + } + + setUploadingLocalEditReferenceRegionId(regionId); + try { + const uploadedReferences = await Promise.all( + selectedFiles.map(async (file) => { + const uploaded = await uploadChatAttachmentBlob(file, file.name); + return { + id: createLocalEditReferenceImageId(), + url: uploaded.url, + name: file.name, + source: "upload" as const, + createdAt: Date.now(), + } satisfies LocalEditReferenceImage; + }) + ); + + commitLocalEditRegions( + localEditRegionsRef.current.map((region) => + region.id === regionId + ? { + ...region, + referenceImages: [ + ...(region.referenceImages || []), + ...uploadedReferences, + ].slice(0, LOCAL_EDIT_MAX_REFERENCE_IMAGES_PER_REGION), + } + : region + ) + ); + toast.success( + uploadedReferences.length > 1 + ? `已添加 ${uploadedReferences.length} 张区域参考图` + : "已添加区域参考图" + ); + } catch (error) { + toast.error(error instanceof Error ? error.message : "参考图上传失败"); + } finally { + setUploadingLocalEditReferenceRegionId(null); + } + }, + [commitLocalEditRegions] + ); + + const removeLocalEditReferenceImage = useCallback( + (regionId: string, referenceId: string) => { + commitLocalEditRegions( + localEditRegionsRef.current.map((region) => + region.id === regionId + ? { + ...region, + referenceImages: (region.referenceImages || []).filter( + (reference) => reference.id !== referenceId + ), + } + : region + ) + ); + }, + [commitLocalEditRegions] + ); + const clearLocalEditRegions = useCallback(() => { localEditPromptDraftRefs.current = {}; commitLocalEditRegions([]); @@ -3696,6 +3839,12 @@ export function ImageNode({ color: region.color, prompt: region.prompt, rect: region.rect, + referenceImages: (region.referenceImages || []).map((reference) => ({ + id: reference.id, + url: reference.url, + name: reference.name, + source: reference.source, + })), })), }); @@ -8366,7 +8515,7 @@ export function ImageNode({ localEditPromptPortalOverlays.map(({ region, isEditing, style }) => createPortal(
event.stopPropagation()} style={style} > - {isEditing ? ( - { - const nextPrompt = event.currentTarget.value; - localEditPromptDraftRefs.current[region.id] = nextPrompt; - persistLocalEditRegionPrompt(region.id, nextPrompt); - if (isLocalEditControlTarget(event.relatedTarget)) { - return; +
+ {isEditing ? ( + { - if (isLocalEditControlTarget(document.activeElement)) { + onBlur={(event) => { + const nextPrompt = event.currentTarget.value; + localEditPromptDraftRefs.current[region.id] = nextPrompt; + persistLocalEditRegionPrompt(region.id, nextPrompt); + if (isLocalEditControlTarget(event.relatedTarget)) { return; } - setEditingLocalEditRegionId((current) => { - if (current === region.id) { - focusLocalEditPromptInput(region.id); + window.setTimeout(() => { + if (isLocalEditControlTarget(document.activeElement)) { + return; } - return current; - }); - }, 80); - }} - onClick={(event) => { - event.nativeEvent.stopImmediatePropagation?.(); - event.stopPropagation(); - event.currentTarget.focus({ preventScroll: true }); - }} - onFocus={(event) => { - event.stopPropagation(); - setEditingLocalEditRegionId((current) => - current === region.id ? current : region.id - ); - }} - onInput={(event) => { - localEditPromptDraftRefs.current[region.id] = - event.currentTarget.value; - }} + setEditingLocalEditRegionId((current) => { + if (current === region.id) { + focusLocalEditPromptInput(region.id); + } + return current; + }); + }, 80); + }} + onClick={(event) => { + event.nativeEvent.stopImmediatePropagation?.(); + event.stopPropagation(); + event.currentTarget.focus({ preventScroll: true }); + }} + onFocus={(event) => { + event.stopPropagation(); + setEditingLocalEditRegionId((current) => + current === region.id ? current : region.id + ); + }} + onInput={(event) => { + localEditPromptDraftRefs.current[region.id] = + event.currentTarget.value; + }} + onChange={(event) => { + localEditPromptDraftRefs.current[region.id] = + event.currentTarget.value; + }} + onKeyDown={(event) => { + event.nativeEvent.stopImmediatePropagation?.(); + event.stopPropagation(); + if (event.key === "Enter") { + event.preventDefault(); + persistLocalEditRegionPrompt( + region.id, + event.currentTarget.value + ); + setEditingLocalEditRegionId(null); + } + if (event.key === "Escape") { + event.preventDefault(); + persistLocalEditRegionPrompt( + region.id, + event.currentTarget.value + ); + setEditingLocalEditRegionId(null); + } + }} + onKeyUp={(event) => { + event.nativeEvent.stopImmediatePropagation?.(); + event.stopPropagation(); + }} + onMouseDown={(event) => { + event.nativeEvent.stopImmediatePropagation?.(); + event.stopPropagation(); + }} + onMouseDownCapture={(event) => { + event.nativeEvent.stopImmediatePropagation?.(); + event.stopPropagation(); + event.currentTarget.focus({ preventScroll: true }); + }} + onPointerDown={(event) => { + event.nativeEvent.stopImmediatePropagation?.(); + event.stopPropagation(); + }} + onPointerDownCapture={(event) => { + event.nativeEvent.stopImmediatePropagation?.(); + event.stopPropagation(); + event.currentTarget.focus({ preventScroll: true }); + }} + placeholder={`区域 ${region.index} 提示词`} + ref={(input) => { + localEditPromptInputRefs.current[region.id] = input; + }} + /> + ) : ( + + )} + { - localEditPromptDraftRefs.current[region.id] = - event.currentTarget.value; - }} - onKeyDown={(event) => { - event.nativeEvent.stopImmediatePropagation?.(); - event.stopPropagation(); - if (event.key === "Enter") { - event.preventDefault(); - persistLocalEditRegionPrompt( - region.id, - event.currentTarget.value - ); - setEditingLocalEditRegionId(null); - } - if (event.key === "Escape") { - event.preventDefault(); - persistLocalEditRegionPrompt( - region.id, - event.currentTarget.value - ); - setEditingLocalEditRegionId(null); - } - }} - onKeyUp={(event) => { - event.nativeEvent.stopImmediatePropagation?.(); - event.stopPropagation(); - }} - onMouseDown={(event) => { - event.nativeEvent.stopImmediatePropagation?.(); - event.stopPropagation(); - }} - onMouseDownCapture={(event) => { - event.nativeEvent.stopImmediatePropagation?.(); - event.stopPropagation(); - event.currentTarget.focus({ preventScroll: true }); - }} - onPointerDown={(event) => { - event.nativeEvent.stopImmediatePropagation?.(); - event.stopPropagation(); - }} - onPointerDownCapture={(event) => { - event.nativeEvent.stopImmediatePropagation?.(); - event.stopPropagation(); - event.currentTarget.focus({ preventScroll: true }); - }} - placeholder={`区域 ${region.index} 提示词`} - ref={(input) => { - localEditPromptInputRefs.current[region.id] = input; + void handleLocalEditReferenceUpload( + region.id, + event.currentTarget.files + ); + event.currentTarget.value = ""; }} + type="file" /> - ) : ( + - )} - +
+ {region.referenceImages?.length ? ( +
+ + 参考 + + {region.referenceImages.map((reference) => ( +
+ {reference.name + +
+ ))} +
+ ) : null}
, document.body ) diff --git a/services/canvas/index.ts b/services/canvas/index.ts index f030f6ce5..48754280a 100644 --- a/services/canvas/index.ts +++ b/services/canvas/index.ts @@ -307,6 +307,12 @@ export const imageGenerationService = { color: string; prompt: string; rect: NormalizedCropRect; + referenceImages?: Array<{ + id?: string; + url: string; + name?: string; + source?: "upload" | "canvas"; + }>; }>; model?: string; imageSize?: string;