"use client"; import { useCallback, useEffect, useMemo, useRef, useState, type ChangeEvent, type PointerEvent as ReactPointerEvent, } from "react"; import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle, } from "@/components/ui/dialog"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { Slider } from "@/components/ui/slider"; import { cn } from "@/lib/utils"; import { Eraser, Loader2, Paintbrush, Square, Circle as CircleIcon, Slash, Upload, RefreshCw, Sparkles, Trash2, Download, Undo2, Redo2, Send, } from "lucide-react"; import { toast } from "sonner"; type RealtimeConfig = { transport: { throttleInterval: number; maxBuffering: number; }; image: { maxInputResolution: number; jpegQuality: number; }; model: { id: string; defaults: { steps: number; cfgScale: number; strength: number; imageSize: "square" | "square_hd"; }; }; behavior?: { strictSketch?: boolean; }; }; type RealtimeStatus = "idle" | "connecting" | "live" | "error"; type RealtimeCanvasDialogProps = { open: boolean; onOpenChange: (open: boolean) => void; initialImageUrl?: string | null; onApplyResult: (payload: { imageUrl: string; prompt: string }) => void; }; const FALLBACK_PROMPT = "严格按照输入草图生成:保持主体轮廓、位置、比例和数量一致,只做材质与光影细化;禁止改变构图,禁止新增无关背景纹理。"; const CANVAS_BACKGROUND = "#ffffff"; const BRUSH_COLOR_PRESETS = [ "#18181b", "#ef4444", "#f59e0b", "#22c55e", "#3b82f6", "#a855f7", ] as const; const CANVAS_BACKGROUND_PRESETS = ["#ffffff", "#f4f4f5", "#fdf2f8", "#ecfeff", "#f0fdf4"] as const; const defaultRealtimeConfig: RealtimeConfig = { transport: { throttleInterval: 100, maxBuffering: 2, }, image: { maxInputResolution: 1024, jpegQuality: 0.55, }, model: { id: "runware:101@1", defaults: { steps: 8, cfgScale: 7, strength: 0.32, imageSize: "square", }, }, behavior: { strictSketch: true, }, }; function isShapeMode(mode: "brush" | "eraser" | "rect" | "circle" | "line") { return mode === "rect" || mode === "circle" || mode === "line"; } function createSessionSeed() { return Math.floor(Math.random() * 2147483647); } async function canvasToJpegDataUri( canvas: HTMLCanvasElement, maxInputResolution: number, jpegQuality: number, backgroundColor: string, options?: { autoCrop?: boolean; } ) { type CropBox = { x: number; y: number; width: number; height: number; foregroundRatio: number; }; const parseColor = (rawColor: string) => { const tempCanvas = document.createElement("canvas"); tempCanvas.width = 1; tempCanvas.height = 1; const tempCtx = tempCanvas.getContext("2d"); if (!tempCtx) { return { r: 255, g: 255, b: 255 }; } tempCtx.fillStyle = rawColor; const normalized = String(tempCtx.fillStyle || "").trim().toLowerCase(); if (normalized.startsWith("#")) { const hex = normalized.slice(1); if (hex.length === 3) { return { r: Number.parseInt(hex[0] + hex[0], 16), g: Number.parseInt(hex[1] + hex[1], 16), b: Number.parseInt(hex[2] + hex[2], 16), }; } if (hex.length >= 6) { return { r: Number.parseInt(hex.slice(0, 2), 16), g: Number.parseInt(hex.slice(2, 4), 16), b: Number.parseInt(hex.slice(4, 6), 16), }; } } const rgbMatch = normalized.match(/^rgba?\((\d+),\s*(\d+),\s*(\d+)/); if (rgbMatch) { return { r: Number(rgbMatch[1]), g: Number(rgbMatch[2]), b: Number(rgbMatch[3]), }; } return { r: 255, g: 255, b: 255 }; }; const findForegroundBox = (): CropBox | null => { const sourceWidth = canvas.width; const sourceHeight = canvas.height; const longestSide = Math.max(sourceWidth, sourceHeight); const analysisScale = longestSide > 320 ? 320 / longestSide : 1; const analysisWidth = Math.max(48, Math.round(sourceWidth * analysisScale)); const analysisHeight = Math.max(48, Math.round(sourceHeight * analysisScale)); const analysisCanvas = document.createElement("canvas"); analysisCanvas.width = analysisWidth; analysisCanvas.height = analysisHeight; const analysisCtx = analysisCanvas.getContext("2d", { willReadFrequently: true }); if (!analysisCtx) return null; analysisCtx.fillStyle = backgroundColor; analysisCtx.fillRect(0, 0, analysisWidth, analysisHeight); analysisCtx.drawImage(canvas, 0, 0, analysisWidth, analysisHeight); const imageData = analysisCtx.getImageData(0, 0, analysisWidth, analysisHeight); const data = imageData.data; const totalPixels = analysisWidth * analysisHeight; const mask = new Uint8Array(totalPixels); const bg = parseColor(backgroundColor); const tolerance = 18; let minX = analysisWidth; let minY = analysisHeight; let maxX = -1; let maxY = -1; let foregroundPixels = 0; for (let y = 0; y < analysisHeight; y += 1) { for (let x = 0; x < analysisWidth; x += 1) { const pixelIndex = y * analysisWidth + x; const offset = pixelIndex * 4; const alpha = data[offset + 3]; if (alpha < 16) continue; const dr = Math.abs(data[offset] - bg.r); const dg = Math.abs(data[offset + 1] - bg.g); const db = Math.abs(data[offset + 2] - bg.b); const isForeground = dr > tolerance || dg > tolerance || db > tolerance; if (!isForeground) continue; mask[pixelIndex] = 1; foregroundPixels += 1; if (x < minX) minX = x; if (x > maxX) maxX = x; if (y < minY) minY = y; if (y > maxY) maxY = y; } } if (foregroundPixels <= 0 || maxX < minX || maxY < minY) { return null; } const visited = new Uint8Array(totalPixels); const queue = new Uint32Array(totalPixels); const minComponentArea = Math.max(9, Math.floor(foregroundPixels * 0.0015)); let bestArea = 0; let bestMinX = minX; let bestMinY = minY; let bestMaxX = maxX; let bestMaxY = maxY; for (let idx = 0; idx < totalPixels; idx += 1) { if (!mask[idx] || visited[idx]) continue; let head = 0; let tail = 0; queue[tail] = idx; tail += 1; visited[idx] = 1; let area = 0; let componentMinX = analysisWidth; let componentMinY = analysisHeight; let componentMaxX = -1; let componentMaxY = -1; while (head < tail) { const current = queue[head]; head += 1; const x = current % analysisWidth; const y = Math.floor(current / analysisWidth); area += 1; if (x < componentMinX) componentMinX = x; if (x > componentMaxX) componentMaxX = x; if (y < componentMinY) componentMinY = y; if (y > componentMaxY) componentMaxY = y; const startY = Math.max(0, y - 1); const endY = Math.min(analysisHeight - 1, y + 1); const startX = Math.max(0, x - 1); const endX = Math.min(analysisWidth - 1, x + 1); for (let ny = startY; ny <= endY; ny += 1) { for (let nx = startX; nx <= endX; nx += 1) { const neighbor = ny * analysisWidth + nx; if (!mask[neighbor] || visited[neighbor]) continue; visited[neighbor] = 1; queue[tail] = neighbor; tail += 1; } } } if (area >= minComponentArea && area > bestArea) { bestArea = area; bestMinX = componentMinX; bestMinY = componentMinY; bestMaxX = componentMaxX; bestMaxY = componentMaxY; } } if (bestArea > 0) { minX = bestMinX; minY = bestMinY; maxX = bestMaxX; maxY = bestMaxY; } else { bestArea = foregroundPixels; } const rowCounts = new Uint32Array(analysisHeight); const colCounts = new Uint32Array(analysisWidth); for (let y = minY; y <= maxY; y += 1) { for (let x = minX; x <= maxX; x += 1) { const idx = y * analysisWidth + x; if (!mask[idx]) continue; rowCounts[y] += 1; colCounts[x] += 1; } } const trimPixels = Math.max(0, Math.floor(bestArea * 0.008)); if (trimPixels > 0) { let cumulative = 0; for (let x = minX; x <= maxX; x += 1) { cumulative += colCounts[x]; if (cumulative > trimPixels) { minX = x; break; } } cumulative = 0; for (let x = maxX; x >= minX; x -= 1) { cumulative += colCounts[x]; if (cumulative > trimPixels) { maxX = x; break; } } cumulative = 0; for (let y = minY; y <= maxY; y += 1) { cumulative += rowCounts[y]; if (cumulative > trimPixels) { minY = y; break; } } cumulative = 0; for (let y = maxY; y >= minY; y -= 1) { cumulative += rowCounts[y]; if (cumulative > trimPixels) { maxY = y; break; } } } const scaleX = sourceWidth / analysisWidth; const scaleY = sourceHeight / analysisHeight; const boxX = Math.max(0, Math.floor(minX * scaleX)); const boxY = Math.max(0, Math.floor(minY * scaleY)); const boxRight = Math.min(sourceWidth, Math.ceil((maxX + 1) * scaleX)); const boxBottom = Math.min(sourceHeight, Math.ceil((maxY + 1) * scaleY)); const boxWidth = Math.max(1, boxRight - boxX); const boxHeight = Math.max(1, boxBottom - boxY); const foregroundRatio = Math.min(1, bestArea / totalPixels); const paddingRatio = foregroundRatio < 0.08 ? 0.34 : foregroundRatio < 0.2 ? 0.28 : 0.22; const padding = Math.max(16, Math.floor(Math.max(boxWidth, boxHeight) * paddingRatio)); const x = Math.max(0, boxX - padding); const y = Math.max(0, boxY - padding); const right = Math.min(sourceWidth, boxRight + padding); const bottom = Math.min(sourceHeight, boxBottom + padding); return { x, y, width: Math.max(1, right - x), height: Math.max(1, bottom - y), foregroundRatio, }; }; const sourceWidth = canvas.width; const sourceHeight = canvas.height; const cropBox = options?.autoCrop ? findForegroundBox() : null; const drawSourceX = cropBox?.x || 0; const drawSourceY = cropBox?.y || 0; const drawSourceWidth = cropBox?.width || sourceWidth; const drawSourceHeight = cropBox?.height || sourceHeight; const autoCropEnabled = Boolean(options?.autoCrop && cropBox); const longestSide = Math.max(drawSourceWidth, drawSourceHeight); const scale = longestSide > maxInputResolution ? maxInputResolution / longestSide : 1; const exportWidth = autoCropEnabled ? Math.max(512, maxInputResolution) : Math.max(1, Math.round(drawSourceWidth * scale)); const exportHeight = autoCropEnabled ? Math.max(512, maxInputResolution) : Math.max(1, Math.round(drawSourceHeight * scale)); const exportCanvas = document.createElement("canvas"); exportCanvas.width = exportWidth; exportCanvas.height = exportHeight; const exportCtx = exportCanvas.getContext("2d"); if (!exportCtx) { throw new Error("无法初始化导出画布"); } exportCtx.fillStyle = backgroundColor; exportCtx.fillRect(0, 0, exportWidth, exportHeight); if (autoCropEnabled) { const fitRatio = (cropBox?.foregroundRatio || 0) < 0.12 ? 0.9 : 0.94; const fitScale = Math.min((exportWidth * fitRatio) / drawSourceWidth, (exportHeight * fitRatio) / drawSourceHeight); const targetWidth = Math.max(1, Math.round(drawSourceWidth * fitScale)); const targetHeight = Math.max(1, Math.round(drawSourceHeight * fitScale)); const targetX = Math.round((exportWidth - targetWidth) / 2); const targetY = Math.round((exportHeight - targetHeight) / 2); exportCtx.imageSmoothingEnabled = true; exportCtx.imageSmoothingQuality = "high"; exportCtx.drawImage( canvas, drawSourceX, drawSourceY, drawSourceWidth, drawSourceHeight, targetX, targetY, targetWidth, targetHeight ); if ((cropBox?.foregroundRatio || 0) < 0.28) { exportCtx.save(); exportCtx.globalAlpha = 0.22; exportCtx.drawImage( canvas, drawSourceX, drawSourceY, drawSourceWidth, drawSourceHeight, targetX, targetY, targetWidth, targetHeight ); exportCtx.restore(); } } else { exportCtx.drawImage( canvas, drawSourceX, drawSourceY, drawSourceWidth, drawSourceHeight, 0, 0, exportWidth, exportHeight ); } const blob = await new Promise((resolve) => { exportCanvas.toBlob(resolve, "image/jpeg", jpegQuality); }); if (!blob) { throw new Error("导出图片失败"); } return await new Promise((resolve, reject) => { const reader = new FileReader(); reader.onload = () => resolve(String(reader.result || "")); reader.onerror = () => reject(new Error("读取导出图片失败")); reader.readAsDataURL(blob); }); } export function RealtimeCanvasDialog({ open, onOpenChange, initialImageUrl, onApplyResult, }: RealtimeCanvasDialogProps) { const canvasRef = useRef(null); const previewCanvasRef = useRef(null); const canvasWrapRef = useRef(null); const connectionRef = useRef(null); const pointerDownRef = useRef(false); const lastPointRef = useRef<{ x: number; y: number } | null>(null); const shapeStartPointRef = useRef<{ x: number; y: number } | null>(null); const pendingFlushTimerRef = useRef(null); const flushInFlightRef = useRef(false); const dirtyRef = useRef(false); const seqRef = useRef(0); const requestSeqMapRef = useRef>(new Map()); const requestStartMapRef = useRef>(new Map()); const lastRenderedSeqRef = useRef(0); const metricsQueueRef = useRef([]); const sessionIdRef = useRef(`rt-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`); const seedRef = useRef(createSessionSeed()); const hasLoadedInitialImageRef = useRef(false); const canvasBackgroundRef = useRef(CANVAS_BACKGROUND); // History Management const historyRef = useRef([]); const historyIndexRef = useRef(-1); const cursorRef = useRef(null); const uploadInputRef = useRef(null); const [canUndo, setCanUndo] = useState(false); const [canRedo, setCanRedo] = useState(false); const [config, setConfig] = useState(defaultRealtimeConfig); const [status, setStatus] = useState("idle"); const [statusMessage, setStatusMessage] = useState("等待连接"); const [prompt, setPrompt] = useState(FALLBACK_PROMPT); const [previewUrl, setPreviewUrl] = useState(""); const [brushMode, setBrushMode] = useState<"brush" | "eraser" | "rect" | "circle" | "line">("brush"); const [brushColor, setBrushColor] = useState(BRUSH_COLOR_PRESETS[0]); const [canvasBackground, setCanvasBackground] = useState(canvasBackgroundRef.current); const [brushSize, setBrushSize] = useState(6); const [adaptiveThrottleMs, setAdaptiveThrottleMs] = useState( defaultRealtimeConfig.transport.throttleInterval ); const [refreshToken, setRefreshToken] = useState(0); const statusBadge = useMemo(() => { if (status === "live") return { label: "实时中", className: "bg-emerald-50 text-emerald-700" }; if (status === "connecting") { return { label: "连接中", className: "bg-blue-50 text-blue-700" }; } if (status === "error") return { label: "异常", className: "bg-rose-50 text-rose-700" }; return { label: "待启动", className: "bg-zinc-100 text-zinc-600" }; }, [status]); const pushMetric = useCallback((event: any) => { metricsQueueRef.current.push(event); if (metricsQueueRef.current.length > 120) { metricsQueueRef.current = metricsQueueRef.current.slice(-80); } }, []); const flushMetrics = useCallback(async () => { if (!open || metricsQueueRef.current.length === 0) return; const events = metricsQueueRef.current.splice(0, metricsQueueRef.current.length); try { await fetch("/api/realtime/metrics", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ sessionId: sessionIdRef.current, events, }), }); } catch { metricsQueueRef.current.unshift(...events.slice(-20)); } }, [open]); const saveHistory = useCallback(() => { const canvas = canvasRef.current; if (!canvas) return; const ctx = canvas.getContext("2d", { willReadFrequently: true }); if (!ctx) return; // If we are not at the end of history, truncate future history if (historyIndexRef.current < historyRef.current.length - 1) { historyRef.current = historyRef.current.slice(0, historyIndexRef.current + 1); } try { const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); historyRef.current.push(imageData); historyIndexRef.current = historyRef.current.length - 1; } catch (error) { console.warn("[RealtimeCanvas] saveHistory skipped:", error); return; } // Limit history size if (historyRef.current.length > 50) { historyRef.current.shift(); historyIndexRef.current--; } setCanUndo(historyIndexRef.current > 0); setCanRedo(false); }, []); const flushFrame = useCallback( async (force = false) => { if (!open) return; if (!connectionRef.current) return; if (flushInFlightRef.current) return; if (!dirtyRef.current && !force) return; const canvas = canvasRef.current; if (!canvas) return; flushInFlightRef.current = true; const seq = seqRef.current + 1; seqRef.current = seq; const requestId = `${sessionIdRef.current}-${seq}`; requestSeqMapRef.current.set(requestId, seq); requestStartMapRef.current.set(requestId, Date.now()); try { const imageDataUri = await canvasToJpegDataUri( canvas, config.image.maxInputResolution, config.image.jpegQuality, canvasBackgroundRef.current, { autoCrop: true } ); const activeConnection = connectionRef.current; if (!activeConnection) { setStatus("connecting"); setStatusMessage("连接重建中,正在重试..."); return; } setStatus("connecting"); setStatusMessage("正在请求实时生成..."); const response = await fetch("/api/realtime/generate", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ requestId, imageUrl: imageDataUri, prompt: prompt.trim() || FALLBACK_PROMPT, seed: seedRef.current, strictSketch: config.behavior?.strictSketch !== false, imageSize: config.model.defaults.imageSize, steps: config.model.defaults.steps, cfgScale: config.model.defaults.cfgScale, strength: config.model.defaults.strength, }), }); const payload = (await response.json().catch(() => ({}))) as { imageUrl?: string; imageDataURI?: string; cost?: number; message?: string; error?: string; }; if (!response.ok) { throw new Error(payload.message || payload.error || "实时生成失败"); } const nextPreviewUrl = String(payload.imageDataURI || payload.imageUrl || ""); if (!nextPreviewUrl) { throw new Error("实时生成未返回图像"); } const startedAt = requestStartMapRef.current.get(requestId) || Date.now(); const latencyMs = Date.now() - startedAt; requestStartMapRef.current.delete(requestId); requestSeqMapRef.current.delete(requestId); if (seq > 0 && seq < lastRenderedSeqRef.current) { return; } lastRenderedSeqRef.current = Math.max(lastRenderedSeqRef.current, seq); setPreviewUrl(nextPreviewUrl); setStatus("live"); setStatusMessage("实时渲染中"); dirtyRef.current = false; pushMetric({ name: "frame_send", ts: Date.now(), seq, }); pushMetric({ name: "frame_result", ts: Date.now(), seq, latencyMs, queueDepth: Number(payload.cost || 0), }); } catch (error) { requestStartMapRef.current.delete(requestId); requestSeqMapRef.current.delete(requestId); console.error("[RealtimeCanvas] flush frame failed:", error); setStatus("error"); setStatusMessage(error instanceof Error ? error.message : "发送失败"); } finally { flushInFlightRef.current = false; } }, [config, open, prompt, pushMetric] ); const scheduleFlush = useCallback( (force = false, markDirty = true) => { if (markDirty) { dirtyRef.current = true; } if (!dirtyRef.current) return; if (force) { if (pendingFlushTimerRef.current) { window.clearTimeout(pendingFlushTimerRef.current); pendingFlushTimerRef.current = null; } void flushFrame(true); return; } if (pendingFlushTimerRef.current) return; pendingFlushTimerRef.current = window.setTimeout(() => { pendingFlushTimerRef.current = null; void flushFrame(); }, adaptiveThrottleMs); }, [adaptiveThrottleMs, flushFrame] ); const scheduleFlushRef = useRef(scheduleFlush); const flushMetricsRef = useRef(flushMetrics); useEffect(() => { scheduleFlushRef.current = scheduleFlush; }, [scheduleFlush]); useEffect(() => { flushMetricsRef.current = flushMetrics; }, [flushMetrics]); const clearPreviewCanvas = useCallback(() => { const previewCanvas = previewCanvasRef.current; if (!previewCanvas) return; const previewCtx = previewCanvas.getContext("2d"); if (!previewCtx) return; previewCtx.clearRect(0, 0, previewCanvas.width, previewCanvas.height); }, []); const undo = useCallback(() => { if (historyIndexRef.current <= 0) return; historyIndexRef.current--; const imageData = historyRef.current[historyIndexRef.current]; const canvas = canvasRef.current; const ctx = canvas?.getContext("2d"); if (canvas && ctx && imageData) { ctx.putImageData(imageData, 0, 0); setCanUndo(historyIndexRef.current > 0); setCanRedo(historyIndexRef.current < historyRef.current.length - 1); scheduleFlush(true); } }, [scheduleFlush]); const redo = useCallback(() => { if (historyIndexRef.current >= historyRef.current.length - 1) return; historyIndexRef.current++; const imageData = historyRef.current[historyIndexRef.current]; const canvas = canvasRef.current; const ctx = canvas?.getContext("2d"); if (canvas && ctx && imageData) { ctx.putImageData(imageData, 0, 0); setCanUndo(historyIndexRef.current > 0); setCanRedo(historyIndexRef.current < historyRef.current.length - 1); scheduleFlush(true); } }, [scheduleFlush]); const resetCanvas = useCallback(() => { const canvas = canvasRef.current; if (!canvas) return; const ctx = canvas.getContext("2d"); if (!ctx) return; ctx.save(); ctx.globalCompositeOperation = "source-over"; ctx.fillStyle = canvasBackgroundRef.current; ctx.fillRect(0, 0, canvas.width, canvas.height); ctx.restore(); // Clear history and save initial state historyRef.current = []; historyIndexRef.current = -1; saveHistory(); }, [saveHistory]); const drawInitialImage = useCallback(async () => { if (!open || !initialImageUrl || hasLoadedInitialImageRef.current) return false; const canvas = canvasRef.current; if (!canvas) return false; const ctx = canvas.getContext("2d"); if (!ctx) return false; const image = new Image(); image.crossOrigin = "anonymous"; await new Promise((resolve, reject) => { image.onload = () => resolve(); image.onerror = () => reject(new Error("加载初始图片失败")); image.src = initialImageUrl; }); const scale = Math.min(canvas.width / image.width, canvas.height / image.height); const drawWidth = image.width * scale; const drawHeight = image.height * scale; const offsetX = (canvas.width - drawWidth) / 2; const offsetY = (canvas.height - drawHeight) / 2; ctx.drawImage(image, offsetX, offsetY, drawWidth, drawHeight); hasLoadedInitialImageRef.current = true; dirtyRef.current = true; saveHistory(); // Save initial image state return true; }, [initialImageUrl, open, saveHistory]); const initializeCanvas = useCallback(() => { const canvas = canvasRef.current; const previewCanvas = previewCanvasRef.current; const wrap = canvasWrapRef.current; if (!canvas || !previewCanvas || !wrap) return; const rect = wrap.getBoundingClientRect(); const dpr = Math.max(1, window.devicePixelRatio || 1); canvas.width = Math.max(640, Math.floor(rect.width * dpr)); canvas.height = Math.max(480, Math.floor(rect.height * dpr)); canvas.style.width = `${Math.floor(rect.width)}px`; canvas.style.height = `${Math.floor(rect.height)}px`; previewCanvas.width = canvas.width; previewCanvas.height = canvas.height; previewCanvas.style.width = canvas.style.width; previewCanvas.style.height = canvas.style.height; const ctx = canvas.getContext("2d"); const previewCtx = previewCanvas.getContext("2d"); if (!ctx) return; if (!previewCtx) return; ctx.setTransform(1, 0, 0, 1, 0, 0); ctx.scale(dpr, dpr); previewCtx.setTransform(1, 0, 0, 1, 0, 0); previewCtx.scale(dpr, dpr); previewCtx.clearRect(0, 0, previewCanvas.width, previewCanvas.height); resetCanvas(); hasLoadedInitialImageRef.current = false; }, [resetCanvas]); const getCanvasPoint = useCallback((event: ReactPointerEvent) => { const canvas = canvasRef.current; if (!canvas) return null; const rect = canvas.getBoundingClientRect(); return { x: event.clientX - rect.left, y: event.clientY - rect.top, }; }, []); const drawStroke = useCallback( (from: { x: number; y: number }, to: { x: number; y: number }) => { const canvas = canvasRef.current; if (!canvas) return; const ctx = canvas.getContext("2d"); if (!ctx) return; ctx.save(); if (brushMode === "eraser") { ctx.globalCompositeOperation = "destination-out"; ctx.strokeStyle = "rgba(0,0,0,1)"; ctx.lineWidth = brushSize * 1.45; } else { ctx.globalCompositeOperation = "source-over"; ctx.strokeStyle = brushColor; ctx.lineWidth = brushSize; } ctx.lineCap = "round"; ctx.lineJoin = "round"; ctx.beginPath(); ctx.moveTo(from.x, from.y); ctx.lineTo(to.x, to.y); ctx.stroke(); ctx.restore(); }, [brushMode, brushColor, brushSize] ); const drawShape = useCallback( ( from: { x: number; y: number }, to: { x: number; y: number }, targetCtx?: CanvasRenderingContext2D | null ) => { const defaultCtx = canvasRef.current?.getContext("2d"); const ctx = targetCtx || defaultCtx; if (!ctx) return; const minX = Math.min(from.x, to.x); const minY = Math.min(from.y, to.y); const width = Math.abs(to.x - from.x); const height = Math.abs(to.y - from.y); ctx.save(); ctx.globalCompositeOperation = "source-over"; ctx.strokeStyle = brushColor; ctx.lineWidth = Math.max(1, brushSize * 0.9); ctx.lineCap = "round"; ctx.lineJoin = "round"; if (brushMode === "rect") { ctx.strokeRect(minX, minY, width, height); } else if (brushMode === "circle") { const centerX = minX + width / 2; const centerY = minY + height / 2; const radiusX = width / 2; const radiusY = height / 2; ctx.beginPath(); ctx.ellipse(centerX, centerY, radiusX, radiusY, 0, 0, Math.PI * 2); ctx.stroke(); } else if (brushMode === "line") { ctx.beginPath(); ctx.moveTo(from.x, from.y); ctx.lineTo(to.x, to.y); ctx.stroke(); } ctx.restore(); }, [brushMode, brushColor, brushSize] ); const handlePointerDown = useCallback( (event: ReactPointerEvent) => { event.preventDefault(); const point = getCanvasPoint(event); if (!point) return; pointerDownRef.current = true; lastPointRef.current = point; if (isShapeMode(brushMode)) { shapeStartPointRef.current = point; clearPreviewCanvas(); } else { drawStroke(point, point); scheduleFlush(); } event.currentTarget.setPointerCapture(event.pointerId); }, [brushMode, clearPreviewCanvas, drawStroke, getCanvasPoint, scheduleFlush] ); const handlePointerLeave = useCallback(() => { const wasDrawing = pointerDownRef.current; if (cursorRef.current) { cursorRef.current.style.opacity = '0'; } pointerDownRef.current = false; lastPointRef.current = null; shapeStartPointRef.current = null; clearPreviewCanvas(); if (wasDrawing) { scheduleFlush(true, false); } }, [clearPreviewCanvas, scheduleFlush]); const handlePointerEnter = useCallback(() => { if (cursorRef.current) { cursorRef.current.style.opacity = brushMode === "brush" || brushMode === "eraser" ? "1" : "0"; } }, [brushMode]); const handlePointerMove = useCallback( (event: ReactPointerEvent) => { event.preventDefault(); // Update cursor const point = getCanvasPoint(event); if (point && cursorRef.current) { const showCursor = brushMode === "brush" || brushMode === "eraser"; if (showCursor) { cursorRef.current.style.opacity = "1"; cursorRef.current.style.transform = `translate(${point.x - brushSize / 2}px, ${point.y - brushSize / 2}px)`; cursorRef.current.style.width = `${brushSize}px`; cursorRef.current.style.height = `${brushSize}px`; } else { cursorRef.current.style.opacity = "0"; } } if (!pointerDownRef.current) return; if (!point || !lastPointRef.current) return; if (isShapeMode(brushMode) && shapeStartPointRef.current) { const previewCanvas = previewCanvasRef.current; const previewCtx = previewCanvas?.getContext("2d"); if (previewCanvas && previewCtx) { previewCtx.clearRect(0, 0, previewCanvas.width, previewCanvas.height); drawShape(shapeStartPointRef.current, point, previewCtx); } } else { drawStroke(lastPointRef.current, point); } lastPointRef.current = point; scheduleFlush(); }, [drawShape, drawStroke, getCanvasPoint, scheduleFlush, brushMode, brushSize] ); const handlePointerUp = useCallback( (event: ReactPointerEvent) => { event.preventDefault(); pointerDownRef.current = false; const endPoint = getCanvasPoint(event); const shapeStart = shapeStartPointRef.current; if (isShapeMode(brushMode) && endPoint && shapeStart) { drawShape(shapeStart, endPoint); clearPreviewCanvas(); } lastPointRef.current = null; shapeStartPointRef.current = null; try { event.currentTarget.releasePointerCapture(event.pointerId); } catch { // ignore } saveHistory(); // Save state after stroke scheduleFlush(true, false); }, [brushMode, clearPreviewCanvas, drawShape, getCanvasPoint, saveHistory, scheduleFlush] ); useEffect(() => { clearPreviewCanvas(); }, [brushMode, clearPreviewCanvas]); // Shortcuts useEffect(() => { if (!open) return; const handleKeyDown = (e: KeyboardEvent) => { // Undo/Redo if ((e.metaKey || e.ctrlKey) && e.key === 'z') { e.preventDefault(); if (e.shiftKey) { redo(); } else { undo(); } } // Brush Size if (e.key === '[') { setBrushSize(prev => Math.max(2, prev - 2)); } if (e.key === ']') { setBrushSize(prev => Math.min(50, prev + 2)); } }; window.addEventListener('keydown', handleKeyDown); return () => window.removeEventListener('keydown', handleKeyDown); }, [open, undo, redo]); useEffect(() => { if (!open) return; seedRef.current = createSessionSeed(); sessionIdRef.current = `rt-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; seqRef.current = 0; lastRenderedSeqRef.current = 0; requestSeqMapRef.current.clear(); requestStartMapRef.current.clear(); metricsQueueRef.current = []; }, [open]); useEffect(() => { if (!open) return; initializeCanvas(); void drawInitialImage().then((hasInitialImage) => { if (hasInitialImage) { scheduleFlushRef.current(true, false); } }); const onResize = () => initializeCanvas(); window.addEventListener("resize", onResize); return () => { window.removeEventListener("resize", onResize); }; }, [drawInitialImage, initializeCanvas, open]); useEffect(() => { if (!open) return; let cancelled = false; setStatus("connecting"); setStatusMessage("正在初始化实时连接..."); const start = async () => { try { let configResponse: Response | null = null; let configJson: Partial & { message?: string; retryAfter?: number } = {}; for (let attempt = 0; attempt < 3; attempt += 1) { configResponse = await fetch("/api/realtime/config", { cache: "no-store" }); configJson = (await configResponse.json().catch(() => ({}))) as Partial & { message?: string; retryAfter?: number; }; if (configResponse.ok) break; const retryAfterSeconds = Number(configJson.retryAfter || 0); const shouldRetry429 = configResponse.status === 429 && attempt < 2; if (!shouldRetry429) { throw new Error(configJson.message || "无法加载实时配置"); } const retryDelay = retryAfterSeconds > 0 ? retryAfterSeconds * 1000 : 700 * (attempt + 1); await new Promise((resolve) => window.setTimeout(resolve, retryDelay)); } if (!configResponse?.ok) { throw new Error(configJson.message || "无法加载实时配置"); } if (cancelled) return; const resolvedConfig: RealtimeConfig = { transport: { throttleInterval: configJson.transport?.throttleInterval || defaultRealtimeConfig.transport.throttleInterval, maxBuffering: configJson.transport?.maxBuffering || defaultRealtimeConfig.transport.maxBuffering, }, image: { maxInputResolution: configJson.image?.maxInputResolution || defaultRealtimeConfig.image.maxInputResolution, jpegQuality: configJson.image?.jpegQuality || defaultRealtimeConfig.image.jpegQuality, }, model: { id: configJson.model?.id || defaultRealtimeConfig.model.id, defaults: { steps: configJson.model?.defaults?.steps || defaultRealtimeConfig.model.defaults.steps, cfgScale: configJson.model?.defaults?.cfgScale || defaultRealtimeConfig.model.defaults.cfgScale, strength: configJson.model?.defaults?.strength || defaultRealtimeConfig.model.defaults.strength, imageSize: configJson.model?.defaults?.imageSize || defaultRealtimeConfig.model.defaults.imageSize, }, }, behavior: { strictSketch: configJson.behavior?.strictSketch !== false, }, }; setConfig(resolvedConfig); setAdaptiveThrottleMs(resolvedConfig.transport.throttleInterval); if (cancelled) return; connectionRef.current = { provider: "runware", modelId: resolvedConfig.model.id, reconnectToken: refreshToken, }; setStatus("live"); setStatusMessage("实时服务已就绪"); } catch (error) { if (cancelled) return; console.error("[RealtimeCanvas] init failed:", error); setStatus("error"); setStatusMessage(error instanceof Error ? error.message : "实时初始化失败"); } }; void start(); const metricTimer = window.setInterval(() => { void flushMetricsRef.current(); }, 5000); return () => { cancelled = true; if (pendingFlushTimerRef.current) { window.clearTimeout(pendingFlushTimerRef.current); pendingFlushTimerRef.current = null; } window.clearInterval(metricTimer); void flushMetricsRef.current(); connectionRef.current?.close?.(); connectionRef.current = null; setStatus("idle"); setStatusMessage("等待连接"); pointerDownRef.current = false; lastPointRef.current = null; }; }, [open, pushMetric, refreshToken]); const forceReconnect = useCallback(() => { connectionRef.current = null; setStatus("connecting"); setStatusMessage("正在重连..."); setRefreshToken((prev) => prev + 1); }, []); const clearSketch = useCallback(() => { resetCanvas(); clearPreviewCanvas(); dirtyRef.current = true; scheduleFlush(true, false); }, [clearPreviewCanvas, resetCanvas, scheduleFlush]); const handleSketchUpload = useCallback( async (event: ChangeEvent) => { const file = event.target.files?.[0]; if (!file) return; if (!file.type.startsWith("image/")) { toast.error("请上传图片文件"); event.currentTarget.value = ""; return; } try { const objectUrl = URL.createObjectURL(file); const image = new Image(); await new Promise((resolve, reject) => { image.onload = () => resolve(); image.onerror = () => reject(new Error("图片加载失败")); image.src = objectUrl; }); const canvas = canvasRef.current; const ctx = canvas?.getContext("2d"); if (!canvas || !ctx) { URL.revokeObjectURL(objectUrl); throw new Error("画布未初始化"); } const rect = canvas.getBoundingClientRect(); const canvasWidth = Math.max(1, rect.width); const canvasHeight = Math.max(1, rect.height); const scale = Math.min(canvasWidth / image.width, canvasHeight / image.height); const drawWidth = image.width * scale; const drawHeight = image.height * scale; const offsetX = (canvasWidth - drawWidth) / 2; const offsetY = (canvasHeight - drawHeight) / 2; ctx.drawImage(image, offsetX, offsetY, drawWidth, drawHeight); URL.revokeObjectURL(objectUrl); saveHistory(); scheduleFlush(true); toast.success("图片已导入绘画区"); } catch (error) { toast.error(error instanceof Error ? error.message : "导入图片失败"); } finally { event.currentTarget.value = ""; } }, [saveHistory, scheduleFlush] ); const handleCanvasBackgroundChange = useCallback( (nextColor: string) => { const canvas = canvasRef.current; if (!canvas) return; const ctx = canvas.getContext("2d"); if (!ctx) return; const snapshot = document.createElement("canvas"); snapshot.width = canvas.width; snapshot.height = canvas.height; const snapshotCtx = snapshot.getContext("2d"); if (!snapshotCtx) return; snapshotCtx.drawImage(canvas, 0, 0); ctx.save(); ctx.globalCompositeOperation = "source-over"; ctx.fillStyle = nextColor; ctx.fillRect(0, 0, canvas.width, canvas.height); ctx.drawImage(snapshot, 0, 0); ctx.restore(); canvasBackgroundRef.current = nextColor; setCanvasBackground(nextColor); saveHistory(); scheduleFlush(true); }, [saveHistory, scheduleFlush] ); const handleDownloadCanvas = useCallback(async () => { const canvas = canvasRef.current; if (!canvas) return; try { const dataUrl = await canvasToJpegDataUri(canvas, 2048, 0.92, canvasBackgroundRef.current); const anchor = document.createElement("a"); anchor.href = dataUrl; anchor.download = `realtime-canvas-${Date.now()}.jpg`; anchor.click(); toast.success("已保存当前画布"); } catch (error) { toast.error(error instanceof Error ? error.message : "保存失败"); } }, []); const handleApplyToCanvas = useCallback(() => { if (!previewUrl) { toast.error("请先生成预览结果"); return; } onApplyResult({ imageUrl: previewUrl, prompt: prompt.trim() || FALLBACK_PROMPT, }); onOpenChange(false); }, [onApplyResult, onOpenChange, previewUrl, prompt]); return ( 实时画布 左侧绘制草图,右侧实时预览。 {/* Top Header - Minimal */}
Realtime Canvas
{statusBadge.label}
{/* Main Canvas Area - Split View */}
{/* Left: Input Canvas */}
{/* Custom Brush Cursor */}
INPUT
{/* Right: Output Preview */}
{previewUrl ? ( 实时预览 ) : (
{status === "connecting" ? ( <> Connecting... ) : ( <> Start drawing to generate )}
)}
OUTPUT
{/* Floating Toolbar - Top (Lovart-like) */}
{BRUSH_COLOR_PRESETS.map((color) => (
setBrushSize(val)} />
{CANVAS_BACKGROUND_PRESETS.map((color) => (
{/* Floating Toolbar - Bottom (Prompt) */}
setPrompt(e.target.value)} onBlur={() => scheduleFlush(true, false)} placeholder="Describe what you want to create..." className="flex-1 h-10 border-0 bg-transparent focus-visible:ring-0 px-2 text-base shadow-none" />
{statusMessage} • Latency: {adaptiveThrottleMs}ms
); }