"use client"; import { useCallback, useEffect, useMemo, useRef, useState, type PointerEvent as ReactPointerEvent } from "react"; import * as fal from "@fal-ai/serverless-client"; import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle, } from "@/components/ui/dialog"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { Slider } from "@/components/ui/slider"; import { cn } from "@/lib/utils"; import { Eraser, Loader2, Paintbrush, RefreshCw, Send, Sparkles, Trash2, Wifi, WifiOff, Minimize2, Maximize2, Download, Image as ImageIcon, MoreHorizontal, Undo2, Redo2, X, } from "lucide-react"; import { toast } from "sonner"; type RealtimeConfig = { transport: { throttleInterval: number; maxBuffering: number; }; image: { maxInputResolution: number; jpegQuality: number; }; model: { id: string; defaults: { steps: number; scheduleMu: number; feedback: number; imageSize: "square" | "square_hd"; }; }; }; type RealtimeStatus = "idle" | "connecting" | "live" | "error"; type RealtimeCanvasDialogProps = { open: boolean; onOpenChange: (open: boolean) => void; initialImageUrl?: string | null; onApplyResult: (payload: { imageUrl: string; prompt: string }) => void; }; const FALLBACK_PROMPT = "将草图渲染为高质感产品设计图,保持结构和主轮廓一致。"; const defaultRealtimeConfig: RealtimeConfig = { transport: { throttleInterval: 100, maxBuffering: 2, }, image: { maxInputResolution: 1024, jpegQuality: 0.55, }, model: { id: "fal-ai/flux-2/klein/realtime", defaults: { steps: 3, scheduleMu: 2.3, feedback: 0.95, imageSize: "square", }, }, }; function buildImageDataUrl(image: any) { if (!image) return ""; if (typeof image.url === "string" && image.url.trim()) { return image.url; } if (typeof image.content === "string" && image.content.trim()) { if (image.content.startsWith("data:")) { return image.content; } const contentType = image.content_type || "image/jpeg"; return `data:${contentType};base64,${image.content}`; } return ""; } async function canvasToJpegDataUri( canvas: HTMLCanvasElement, maxInputResolution: number, jpegQuality: number ) { const sourceWidth = canvas.width; const sourceHeight = canvas.height; const longestSide = Math.max(sourceWidth, sourceHeight); const scale = longestSide > maxInputResolution ? maxInputResolution / longestSide : 1; const exportWidth = Math.max(1, Math.round(sourceWidth * scale)); const exportHeight = Math.max(1, Math.round(sourceHeight * scale)); const exportCanvas = document.createElement("canvas"); exportCanvas.width = exportWidth; exportCanvas.height = exportHeight; const exportCtx = exportCanvas.getContext("2d"); if (!exportCtx) { throw new Error("无法初始化导出画布"); } exportCtx.fillStyle = "#d8d8db"; exportCtx.fillRect(0, 0, exportWidth, exportHeight); exportCtx.drawImage(canvas, 0, 0, exportWidth, exportHeight); const blob = await new Promise((resolve) => { exportCanvas.toBlob(resolve, "image/jpeg", jpegQuality); }); if (!blob) { throw new Error("导出图片失败"); } return await new Promise((resolve, reject) => { const reader = new FileReader(); reader.onload = () => resolve(String(reader.result || "")); reader.onerror = () => reject(new Error("读取导出图片失败")); reader.readAsDataURL(blob); }); } export function RealtimeCanvasDialog({ open, onOpenChange, initialImageUrl, onApplyResult, }: RealtimeCanvasDialogProps) { const canvasRef = useRef(null); const canvasWrapRef = useRef(null); const connectionRef = useRef(null); const pointerDownRef = useRef(false); const lastPointRef = useRef<{ x: number; y: number } | null>(null); const pendingFlushTimerRef = useRef(null); const flushInFlightRef = useRef(false); const dirtyRef = useRef(false); const seqRef = useRef(0); const requestSeqMapRef = useRef>(new Map()); const requestStartMapRef = useRef>(new Map()); const lastRenderedSeqRef = useRef(0); const metricsQueueRef = useRef([]); const sessionIdRef = useRef(`rt-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`); const hasLoadedInitialImageRef = useRef(false); // History Management const historyRef = useRef([]); const historyIndexRef = useRef(-1); const cursorRef = useRef(null); const [canUndo, setCanUndo] = useState(false); const [canRedo, setCanRedo] = useState(false); const [config, setConfig] = useState(defaultRealtimeConfig); const [status, setStatus] = useState("idle"); const [statusMessage, setStatusMessage] = useState("等待连接"); const [prompt, setPrompt] = useState(FALLBACK_PROMPT); const [previewUrl, setPreviewUrl] = useState(""); const [isApplying, setIsApplying] = useState(false); const [brushMode, setBrushMode] = useState<"brush" | "eraser">("brush"); const [brushSize, setBrushSize] = useState(6); const [adaptiveThrottleMs, setAdaptiveThrottleMs] = useState( defaultRealtimeConfig.transport.throttleInterval ); const [refreshToken, setRefreshToken] = useState(0); const statusBadge = useMemo(() => { if (status === "live") return { label: "实时中", className: "bg-emerald-50 text-emerald-700" }; if (status === "connecting") { return { label: "连接中", className: "bg-blue-50 text-blue-700" }; } if (status === "error") return { label: "异常", className: "bg-rose-50 text-rose-700" }; return { label: "待启动", className: "bg-zinc-100 text-zinc-600" }; }, [status]); const pushMetric = useCallback((event: any) => { metricsQueueRef.current.push(event); if (metricsQueueRef.current.length > 120) { metricsQueueRef.current = metricsQueueRef.current.slice(-80); } }, []); const flushMetrics = useCallback(async () => { if (!open || metricsQueueRef.current.length === 0) return; const events = metricsQueueRef.current.splice(0, metricsQueueRef.current.length); try { await fetch("/api/realtime/metrics", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ sessionId: sessionIdRef.current, events, }), }); } catch { metricsQueueRef.current.unshift(...events.slice(-20)); } }, [open]); const saveHistory = useCallback(() => { const canvas = canvasRef.current; if (!canvas) return; const ctx = canvas.getContext("2d", { willReadFrequently: true }); if (!ctx) return; // If we are not at the end of history, truncate future history if (historyIndexRef.current < historyRef.current.length - 1) { historyRef.current = historyRef.current.slice(0, historyIndexRef.current + 1); } const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); historyRef.current.push(imageData); historyIndexRef.current = historyRef.current.length - 1; // Limit history size if (historyRef.current.length > 50) { historyRef.current.shift(); historyIndexRef.current--; } setCanUndo(historyIndexRef.current > 0); setCanRedo(false); }, []); const flushFrame = useCallback( async (force = false) => { if (!open) return; if (!connectionRef.current) return; if (flushInFlightRef.current) return; if (!dirtyRef.current && !force) return; const canvas = canvasRef.current; if (!canvas) return; flushInFlightRef.current = true; const seq = seqRef.current + 1; seqRef.current = seq; const requestId = `${sessionIdRef.current}-${seq}`; requestSeqMapRef.current.set(requestId, seq); requestStartMapRef.current.set(requestId, Date.now()); try { const imageDataUri = await canvasToJpegDataUri( canvas, config.image.maxInputResolution, config.image.jpegQuality ); dirtyRef.current = false; setStatus("connecting"); setStatusMessage("正在请求实时生成..."); connectionRef.current.send({ request_id: requestId, image_url: imageDataUri, prompt: prompt.trim() || FALLBACK_PROMPT, image_size: config.model.defaults.imageSize, num_inference_steps: config.model.defaults.steps, schedule_mu: config.model.defaults.scheduleMu, output_feedback_strength: config.model.defaults.feedback, }); pushMetric({ name: "frame_send", ts: Date.now(), seq, }); } catch (error) { console.error("[RealtimeCanvas] flush frame failed:", error); setStatus("error"); setStatusMessage(error instanceof Error ? error.message : "发送失败"); } finally { flushInFlightRef.current = false; } }, [config, open, prompt, pushMetric] ); const scheduleFlush = useCallback( (force = false) => { dirtyRef.current = true; if (force) { if (pendingFlushTimerRef.current) { window.clearTimeout(pendingFlushTimerRef.current); pendingFlushTimerRef.current = null; } void flushFrame(true); return; } if (pendingFlushTimerRef.current) return; pendingFlushTimerRef.current = window.setTimeout(() => { pendingFlushTimerRef.current = null; void flushFrame(); }, adaptiveThrottleMs); }, [adaptiveThrottleMs, flushFrame] ); const undo = useCallback(() => { if (historyIndexRef.current <= 0) return; historyIndexRef.current--; const imageData = historyRef.current[historyIndexRef.current]; const canvas = canvasRef.current; const ctx = canvas?.getContext("2d"); if (canvas && ctx && imageData) { ctx.putImageData(imageData, 0, 0); setCanUndo(historyIndexRef.current > 0); setCanRedo(historyIndexRef.current < historyRef.current.length - 1); scheduleFlush(true); } }, [scheduleFlush]); const redo = useCallback(() => { if (historyIndexRef.current >= historyRef.current.length - 1) return; historyIndexRef.current++; const imageData = historyRef.current[historyIndexRef.current]; const canvas = canvasRef.current; const ctx = canvas?.getContext("2d"); if (canvas && ctx && imageData) { ctx.putImageData(imageData, 0, 0); setCanUndo(historyIndexRef.current > 0); setCanRedo(historyIndexRef.current < historyRef.current.length - 1); scheduleFlush(true); } }, [scheduleFlush]); const resetCanvas = useCallback(() => { const canvas = canvasRef.current; if (!canvas) return; const ctx = canvas.getContext("2d"); if (!ctx) return; ctx.save(); ctx.globalCompositeOperation = "source-over"; ctx.fillStyle = "#d8d8db"; ctx.fillRect(0, 0, canvas.width, canvas.height); ctx.restore(); // Clear history and save initial state historyRef.current = []; historyIndexRef.current = -1; saveHistory(); }, [saveHistory]); const drawInitialImage = useCallback(async () => { if (!open || !initialImageUrl || hasLoadedInitialImageRef.current) return; const canvas = canvasRef.current; if (!canvas) return; const ctx = canvas.getContext("2d"); if (!ctx) return; const image = new Image(); image.crossOrigin = "anonymous"; await new Promise((resolve, reject) => { image.onload = () => resolve(); image.onerror = () => reject(new Error("加载初始图片失败")); image.src = initialImageUrl; }); const scale = Math.min(canvas.width / image.width, canvas.height / image.height); const drawWidth = image.width * scale; const drawHeight = image.height * scale; const offsetX = (canvas.width - drawWidth) / 2; const offsetY = (canvas.height - drawHeight) / 2; ctx.drawImage(image, offsetX, offsetY, drawWidth, drawHeight); hasLoadedInitialImageRef.current = true; dirtyRef.current = true; saveHistory(); // Save initial image state }, [initialImageUrl, open, saveHistory]); const initializeCanvas = useCallback(() => { const canvas = canvasRef.current; const wrap = canvasWrapRef.current; if (!canvas || !wrap) return; const rect = wrap.getBoundingClientRect(); const dpr = Math.max(1, window.devicePixelRatio || 1); canvas.width = Math.max(640, Math.floor(rect.width * dpr)); canvas.height = Math.max(480, Math.floor(rect.height * dpr)); canvas.style.width = `${Math.floor(rect.width)}px`; canvas.style.height = `${Math.floor(rect.height)}px`; const ctx = canvas.getContext("2d"); if (!ctx) return; ctx.setTransform(1, 0, 0, 1, 0, 0); ctx.scale(dpr, dpr); resetCanvas(); hasLoadedInitialImageRef.current = false; }, [resetCanvas]); const getCanvasPoint = useCallback((event: ReactPointerEvent) => { const canvas = canvasRef.current; if (!canvas) return null; const rect = canvas.getBoundingClientRect(); return { x: event.clientX - rect.left, y: event.clientY - rect.top, }; }, []); const drawStroke = useCallback( (from: { x: number; y: number }, to: { x: number; y: number }) => { const canvas = canvasRef.current; if (!canvas) return; const ctx = canvas.getContext("2d"); if (!ctx) return; ctx.save(); if (brushMode === "eraser") { ctx.globalCompositeOperation = "destination-out"; ctx.strokeStyle = "rgba(0,0,0,1)"; ctx.lineWidth = brushSize * 1.45; } else { ctx.globalCompositeOperation = "source-over"; ctx.strokeStyle = "#18181b"; ctx.lineWidth = brushSize; } ctx.lineCap = "round"; ctx.lineJoin = "round"; ctx.beginPath(); ctx.moveTo(from.x, from.y); ctx.lineTo(to.x, to.y); ctx.stroke(); ctx.restore(); }, [brushMode, brushSize] ); const handlePointerDown = useCallback( (event: ReactPointerEvent) => { const point = getCanvasPoint(event); if (!point) return; pointerDownRef.current = true; lastPointRef.current = point; event.currentTarget.setPointerCapture(event.pointerId); }, [getCanvasPoint] ); const handlePointerLeave = useCallback(() => { if (cursorRef.current) { cursorRef.current.style.opacity = '0'; } pointerDownRef.current = false; lastPointRef.current = null; scheduleFlush(true); }, [scheduleFlush]); const handlePointerEnter = useCallback(() => { if (cursorRef.current) { cursorRef.current.style.opacity = '1'; } }, []); const handlePointerMove = useCallback( (event: ReactPointerEvent) => { // Update cursor const point = getCanvasPoint(event); if (point && cursorRef.current) { cursorRef.current.style.transform = `translate(${point.x - brushSize / 2}px, ${point.y - brushSize / 2}px)`; cursorRef.current.style.width = `${brushSize}px`; cursorRef.current.style.height = `${brushSize}px`; } if (!pointerDownRef.current) return; if (!point || !lastPointRef.current) return; drawStroke(lastPointRef.current, point); lastPointRef.current = point; scheduleFlush(); }, [drawStroke, getCanvasPoint, scheduleFlush, brushSize] ); const handlePointerUp = useCallback( (event: ReactPointerEvent) => { pointerDownRef.current = false; lastPointRef.current = null; try { event.currentTarget.releasePointerCapture(event.pointerId); } catch { // ignore } saveHistory(); // Save state after stroke scheduleFlush(true); }, [scheduleFlush, saveHistory] ); // Shortcuts useEffect(() => { if (!open) return; const handleKeyDown = (e: KeyboardEvent) => { // Undo/Redo if ((e.metaKey || e.ctrlKey) && e.key === 'z') { e.preventDefault(); if (e.shiftKey) { redo(); } else { undo(); } } // Brush Size if (e.key === '[') { setBrushSize(prev => Math.max(2, prev - 2)); } if (e.key === ']') { setBrushSize(prev => Math.min(50, prev + 2)); } }; window.addEventListener('keydown', handleKeyDown); return () => window.removeEventListener('keydown', handleKeyDown); }, [open, undo, redo]); const uploadPreviewToCanvas = useCallback(async () => { if (!previewUrl) { toast.error("请先完成一帧实时预览"); return; } setIsApplying(true); try { const blob = await fetch(previewUrl).then((res) => res.blob()); const formData = new FormData(); const filename = `realtime-canvas-${Date.now()}.jpg`; formData.append("file", new File([blob], filename, { type: blob.type || "image/jpeg" })); const response = await fetch("/api/files/upload", { method: "POST", body: formData, }); const result = await response.json().catch(() => ({})); if (!response.ok || !result?.url) { throw new Error(result?.error || "上传到画布失败"); } onApplyResult({ imageUrl: String(result.url), prompt: prompt.trim() || FALLBACK_PROMPT, }); toast.success("已发送到画布"); onOpenChange(false); } catch (error) { toast.error(error instanceof Error ? error.message : "发送到画布失败"); } finally { setIsApplying(false); } }, [onApplyResult, onOpenChange, previewUrl, prompt]); useEffect(() => { if (!open) return; initializeCanvas(); void drawInitialImage().then(() => { scheduleFlush(true); }); const onResize = () => initializeCanvas(); window.addEventListener("resize", onResize); return () => { window.removeEventListener("resize", onResize); }; }, [drawInitialImage, initializeCanvas, open, scheduleFlush]); useEffect(() => { if (!open) return; let cancelled = false; setStatus("connecting"); setStatusMessage("正在初始化实时连接..."); const start = async () => { try { const configResponse = await fetch("/api/realtime/config", { cache: "no-store" }); const configJson = (await configResponse.json().catch(() => ({}))) as Partial; if (!configResponse.ok) { throw new Error((configJson as any)?.message || "无法加载实时配置"); } if (cancelled) return; const resolvedConfig: RealtimeConfig = { transport: { throttleInterval: configJson.transport?.throttleInterval || defaultRealtimeConfig.transport.throttleInterval, maxBuffering: configJson.transport?.maxBuffering || defaultRealtimeConfig.transport.maxBuffering, }, image: { maxInputResolution: configJson.image?.maxInputResolution || defaultRealtimeConfig.image.maxInputResolution, jpegQuality: configJson.image?.jpegQuality || defaultRealtimeConfig.image.jpegQuality, }, model: { id: configJson.model?.id || defaultRealtimeConfig.model.id, defaults: { steps: configJson.model?.defaults?.steps || defaultRealtimeConfig.model.defaults.steps, scheduleMu: configJson.model?.defaults?.scheduleMu || defaultRealtimeConfig.model.defaults.scheduleMu, feedback: configJson.model?.defaults?.feedback || defaultRealtimeConfig.model.defaults.feedback, imageSize: configJson.model?.defaults?.imageSize || defaultRealtimeConfig.model.defaults.imageSize, }, }, }; setConfig(resolvedConfig); setAdaptiveThrottleMs(resolvedConfig.transport.throttleInterval); fal.config({ proxyUrl: "/api/fal/proxy", }); const connection = fal.realtime.connect(resolvedConfig.model.id, { connectionKey: `canvas-realtime-${sessionIdRef.current}-${refreshToken}`, throttleInterval: resolvedConfig.transport.throttleInterval, maxBuffering: resolvedConfig.transport.maxBuffering, onResult: (result: any) => { const requestId = String(result?.request_id || ""); const seqFromRequest = requestSeqMapRef.current.get(requestId) || 0; const seq = Math.max(seqFromRequest, Number(result?.seq || 0)); if (seq > 0 && seq < lastRenderedSeqRef.current) { return; } const latestImage = Array.isArray(result?.images) ? result.images[result.images.length - 1] : null; const nextPreviewUrl = buildImageDataUrl(latestImage); if (!nextPreviewUrl) return; lastRenderedSeqRef.current = Math.max(lastRenderedSeqRef.current, seq); setPreviewUrl(nextPreviewUrl); setStatus("live"); setStatusMessage("实时渲染中"); if (requestId) { const startedAt = requestStartMapRef.current.get(requestId); if (startedAt) { pushMetric({ name: "frame_result", ts: Date.now(), seq, latencyMs: Date.now() - startedAt, }); } requestStartMapRef.current.delete(requestId); requestSeqMapRef.current.delete(requestId); } }, onError: (error: any) => { const errorMessage = error?.body?.message || error?.message || "实时连接异常,请稍后重试"; setStatus("error"); setStatusMessage(String(errorMessage)); if (Number(error?.status) === 429) { setAdaptiveThrottleMs((prev) => Math.min(240, prev + 40)); } pushMetric({ name: "frame_error", ts: Date.now(), message: String(errorMessage), }); }, }); if (cancelled) { connection.close(); return; } connectionRef.current = connection; setStatus("connecting"); setStatusMessage("连接成功,等待首帧..."); scheduleFlush(true); } catch (error) { console.error("[RealtimeCanvas] init failed:", error); setStatus("error"); setStatusMessage(error instanceof Error ? error.message : "实时初始化失败"); } }; void start(); const metricTimer = window.setInterval(() => { void flushMetrics(); }, 5000); return () => { cancelled = true; if (pendingFlushTimerRef.current) { window.clearTimeout(pendingFlushTimerRef.current); pendingFlushTimerRef.current = null; } window.clearInterval(metricTimer); void flushMetrics(); connectionRef.current?.close?.(); connectionRef.current = null; setStatus("idle"); setStatusMessage("等待连接"); pointerDownRef.current = false; lastPointRef.current = null; }; }, [flushMetrics, open, pushMetric, refreshToken, scheduleFlush]); const forceReconnect = useCallback(() => { connectionRef.current?.close?.(); connectionRef.current = null; setStatus("connecting"); setStatusMessage("正在重连..."); setRefreshToken((prev) => prev + 1); }, []); const clearSketch = useCallback(() => { resetCanvas(); dirtyRef.current = true; scheduleFlush(true); }, [resetCanvas, scheduleFlush]); return ( {/* Top Header - Minimal */}
Realtime Canvas
{statusBadge.label}
{/* Main Canvas Area - Split View */}
{/* Left: Input Canvas */}
{/* Custom Brush Cursor */}
INPUT
{/* Right: Output Preview */}
{previewUrl ? ( 实时预览 ) : (
{status === "connecting" ? ( <> Connecting... ) : ( <> Start drawing to generate )}
)}
OUTPUT
{/* Floating Toolbar - Left (Tools) */}
{brushSize}px setBrushSize(val)} className="h-24 w-4" />
{/* Floating Toolbar - Bottom (Prompt) */}
setPrompt(e.target.value)} onBlur={() => scheduleFlush(true)} placeholder="Describe what you want to create..." className="flex-1 h-10 border-0 bg-transparent focus-visible:ring-0 px-2 text-base shadow-none" />
{statusMessage} • Latency: {adaptiveThrottleMs}ms
{/* Floating Toolbar - Right (Actions) */}
); }