"use client"; import { useCallback, useEffect, useMemo, useRef, useState, type ChangeEvent, type PointerEvent as ReactPointerEvent, } from "react"; import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle, } from "@/components/ui/dialog"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { Slider } from "@/components/ui/slider"; import { cn } from "@/lib/utils"; import { Eraser, Loader2, Paintbrush, Square, Circle as CircleIcon, Slash, Upload, RefreshCw, Sparkles, Trash2, Download, Undo2, Redo2, Send, X, } from "lucide-react"; import { toast } from "sonner"; type RealtimeConfig = { transport: { throttleInterval: number; maxBuffering: number; }; image: { maxInputResolution: number; jpegQuality: number; }; model: { id: string; defaults: { steps: number; cfgScale: number; strength: number; imageSize: "square" | "square_hd"; }; }; behavior?: { strictSketch?: boolean; }; }; type RealtimeStatus = "idle" | "connecting" | "live" | "error"; type RealtimeCanvasDialogProps = { open: boolean; onOpenChange: (open: boolean) => void; initialImageUrl?: string | null; onApplyResult: (payload: { imageUrl: string; prompt: string }) => void; standalone?: boolean; }; const FALLBACK_PROMPT = "严格按照输入草图生成:保持主体轮廓、位置、比例和数量一致,只做材质与光影细化;禁止改变构图,禁止新增无关背景纹理。"; const CANVAS_BACKGROUND = "#ffffff"; const BRUSH_COLOR_PRESETS = [ "#18181b", "#ef4444", "#f59e0b", "#22c55e", "#3b82f6", "#a855f7", ] as const; const CANVAS_BACKGROUND_PRESETS = ["#ffffff", "#f4f4f5", "#fdf2f8", "#ecfeff", "#f0fdf4"] as const; const defaultRealtimeConfig: RealtimeConfig = { transport: { throttleInterval: 100, maxBuffering: 2, }, image: { maxInputResolution: 1024, jpegQuality: 0.55, }, model: { id: "runware:101@1", defaults: { steps: 8, cfgScale: 7, strength: 0.32, imageSize: "square", }, }, behavior: { strictSketch: true, }, }; function isShapeMode(mode: "brush" | "eraser" | "rect" | "circle" | "line") { return mode === "rect" || mode === "circle" || mode === "line"; } function createSessionSeed() { return Math.floor(Math.random() * 2147483647); } async function canvasToJpegDataUri( canvas: HTMLCanvasElement, maxInputResolution: number, jpegQuality: number, backgroundColor: string, options?: { autoCrop?: boolean; } ): Promise<{ dataUri: string; width: number; height: number }> { type CropBox = { x: number; y: number; width: number; height: number; foregroundRatio: number; }; const parseColor = (rawColor: string) => { const tempCanvas = document.createElement("canvas"); tempCanvas.width = 1; tempCanvas.height = 1; const tempCtx = tempCanvas.getContext("2d"); if (!tempCtx) { return { r: 255, g: 255, b: 255 }; } tempCtx.fillStyle = rawColor; const normalized = String(tempCtx.fillStyle || "").trim().toLowerCase(); if (normalized.startsWith("#")) { const hex = normalized.slice(1); if (hex.length === 3) { return { r: Number.parseInt(hex[0] + hex[0], 16), g: Number.parseInt(hex[1] + hex[1], 16), b: Number.parseInt(hex[2] + hex[2], 16), }; } if (hex.length >= 6) { return { r: Number.parseInt(hex.slice(0, 2), 16), g: Number.parseInt(hex.slice(2, 4), 16), b: Number.parseInt(hex.slice(4, 6), 16), }; } } const rgbMatch = normalized.match(/^rgba?\((\d+),\s*(\d+),\s*(\d+)/); if (rgbMatch) { return { r: Number(rgbMatch[1]), g: Number(rgbMatch[2]), b: Number(rgbMatch[3]), }; } return { r: 255, g: 255, b: 255 }; }; const findForegroundBox = (): CropBox | null => { const sourceWidth = canvas.width; const sourceHeight = canvas.height; const longestSide = Math.max(sourceWidth, sourceHeight); const analysisScale = longestSide > 320 ? 320 / longestSide : 1; const analysisWidth = Math.max(48, Math.round(sourceWidth * analysisScale)); const analysisHeight = Math.max(48, Math.round(sourceHeight * analysisScale)); const analysisCanvas = document.createElement("canvas"); analysisCanvas.width = analysisWidth; analysisCanvas.height = analysisHeight; const analysisCtx = analysisCanvas.getContext("2d", { willReadFrequently: true }); if (!analysisCtx) return null; analysisCtx.fillStyle = backgroundColor; analysisCtx.fillRect(0, 0, analysisWidth, analysisHeight); analysisCtx.drawImage(canvas, 0, 0, analysisWidth, analysisHeight); const imageData = analysisCtx.getImageData(0, 0, analysisWidth, analysisHeight); const data = imageData.data; const totalPixels = analysisWidth * analysisHeight; const mask = new Uint8Array(totalPixels); const bg = parseColor(backgroundColor); const tolerance = 18; let minX = analysisWidth; let minY = analysisHeight; let maxX = -1; let maxY = -1; let foregroundPixels = 0; for (let y = 0; y < analysisHeight; y += 1) { for (let x = 0; x < analysisWidth; x += 1) { const pixelIndex = y * analysisWidth + x; const offset = pixelIndex * 4; const alpha = data[offset + 3]; if (alpha < 16) continue; const dr = Math.abs(data[offset] - bg.r); const dg = Math.abs(data[offset + 1] - bg.g); const db = Math.abs(data[offset + 2] - bg.b); const isForeground = dr > tolerance || dg > tolerance || db > tolerance; if (!isForeground) continue; mask[pixelIndex] = 1; foregroundPixels += 1; if (x < minX) minX = x; if (x > maxX) maxX = x; if (y < minY) minY = y; if (y > maxY) maxY = y; } } if (foregroundPixels <= 0 || maxX < minX || maxY < minY) { return null; } const visited = new Uint8Array(totalPixels); const queue = new Uint32Array(totalPixels); const minComponentArea = Math.max(9, Math.floor(foregroundPixels * 0.0015)); let bestArea = 0; let bestMinX = minX; let bestMinY = minY; let bestMaxX = maxX; let bestMaxY = maxY; for (let idx = 0; idx < totalPixels; idx += 1) { if (!mask[idx] || visited[idx]) continue; let head = 0; let tail = 0; queue[tail] = idx; tail += 1; visited[idx] = 1; let area = 0; let componentMinX = analysisWidth; let componentMinY = analysisHeight; let componentMaxX = -1; let componentMaxY = -1; while (head < tail) { const current = queue[head]; head += 1; const x = current % analysisWidth; const y = Math.floor(current / analysisWidth); area += 1; if (x < componentMinX) componentMinX = x; if (x > componentMaxX) componentMaxX = x; if (y < componentMinY) componentMinY = y; if (y > componentMaxY) componentMaxY = y; const startY = Math.max(0, y - 1); const endY = Math.min(analysisHeight - 1, y + 1); const startX = Math.max(0, x - 1); const endX = Math.min(analysisWidth - 1, x + 1); for (let ny = startY; ny <= endY; ny += 1) { for (let nx = startX; nx <= endX; nx += 1) { const neighbor = ny * analysisWidth + nx; if (!mask[neighbor] || visited[neighbor]) continue; visited[neighbor] = 1; queue[tail] = neighbor; tail += 1; } } } if (area >= minComponentArea && area > bestArea) { bestArea = area; bestMinX = componentMinX; bestMinY = componentMinY; bestMaxX = componentMaxX; bestMaxY = componentMaxY; } } if (bestArea > 0) { minX = bestMinX; minY = bestMinY; maxX = bestMaxX; maxY = bestMaxY; } else { bestArea = foregroundPixels; } const rowCounts = new Uint32Array(analysisHeight); const colCounts = new Uint32Array(analysisWidth); for (let y = minY; y <= maxY; y += 1) { for (let x = minX; x <= maxX; x += 1) { const idx = y * analysisWidth + x; if (!mask[idx]) continue; rowCounts[y] += 1; colCounts[x] += 1; } } const trimPixels = Math.max(0, Math.floor(bestArea * 0.008)); if (trimPixels > 0) { let cumulative = 0; for (let x = minX; x <= maxX; x += 1) { cumulative += colCounts[x]; if (cumulative > trimPixels) { minX = x; break; } } cumulative = 0; for (let x = maxX; x >= minX; x -= 1) { cumulative += colCounts[x]; if (cumulative > trimPixels) { maxX = x; break; } } cumulative = 0; for (let y = minY; y <= maxY; y += 1) { cumulative += rowCounts[y]; if (cumulative > trimPixels) { minY = y; break; } } cumulative = 0; for (let y = maxY; y >= minY; y -= 1) { cumulative += rowCounts[y]; if (cumulative > trimPixels) { maxY = y; break; } } } const scaleX = sourceWidth / analysisWidth; const scaleY = sourceHeight / analysisHeight; const boxX = Math.max(0, Math.floor(minX * scaleX)); const boxY = Math.max(0, Math.floor(minY * scaleY)); const boxRight = Math.min(sourceWidth, Math.ceil((maxX + 1) * scaleX)); const boxBottom = Math.min(sourceHeight, Math.ceil((maxY + 1) * scaleY)); const boxWidth = Math.max(1, boxRight - boxX); const boxHeight = Math.max(1, boxBottom - boxY); const foregroundRatio = Math.min(1, bestArea / totalPixels); const paddingRatio = foregroundRatio < 0.08 ? 0.34 : foregroundRatio < 0.2 ? 0.28 : 0.22; const padding = Math.max(16, Math.floor(Math.max(boxWidth, boxHeight) * paddingRatio)); const x = Math.max(0, boxX - padding); const y = Math.max(0, boxY - padding); const right = Math.min(sourceWidth, boxRight + padding); const bottom = Math.min(sourceHeight, boxBottom + padding); return { x, y, width: Math.max(1, right - x), height: Math.max(1, bottom - y), foregroundRatio, }; }; const sourceWidth = canvas.width; const sourceHeight = canvas.height; const cropBox = options?.autoCrop ? findForegroundBox() : null; const drawSourceX = cropBox?.x || 0; const drawSourceY = cropBox?.y || 0; const drawSourceWidth = cropBox?.width || sourceWidth; const drawSourceHeight = cropBox?.height || sourceHeight; const autoCropEnabled = Boolean(options?.autoCrop && cropBox); const longestSide = Math.max(drawSourceWidth, drawSourceHeight); const scale = longestSide > maxInputResolution ? maxInputResolution / longestSide : 1; const exportWidth = autoCropEnabled ? Math.max(512, maxInputResolution) : Math.max(1, Math.round(drawSourceWidth * scale)); const exportHeight = autoCropEnabled ? Math.max(512, maxInputResolution) : Math.max(1, Math.round(drawSourceHeight * scale)); const exportCanvas = document.createElement("canvas"); exportCanvas.width = exportWidth; exportCanvas.height = exportHeight; const exportCtx = exportCanvas.getContext("2d"); if (!exportCtx) { throw new Error("无法初始化导出画布"); } exportCtx.fillStyle = backgroundColor; exportCtx.fillRect(0, 0, exportWidth, exportHeight); if (autoCropEnabled) { const fitRatio = (cropBox?.foregroundRatio || 0) < 0.12 ? 0.9 : 0.94; const fitScale = Math.min((exportWidth * fitRatio) / drawSourceWidth, (exportHeight * fitRatio) / drawSourceHeight); const targetWidth = Math.max(1, Math.round(drawSourceWidth * fitScale)); const targetHeight = Math.max(1, Math.round(drawSourceHeight * fitScale)); const targetX = Math.round((exportWidth - targetWidth) / 2); const targetY = Math.round((exportHeight - targetHeight) / 2); exportCtx.imageSmoothingEnabled = true; exportCtx.imageSmoothingQuality = "high"; exportCtx.drawImage( canvas, drawSourceX, drawSourceY, drawSourceWidth, drawSourceHeight, targetX, targetY, targetWidth, targetHeight ); if ((cropBox?.foregroundRatio || 0) < 0.28) { exportCtx.save(); exportCtx.globalAlpha = 0.22; exportCtx.drawImage( canvas, drawSourceX, drawSourceY, drawSourceWidth, drawSourceHeight, targetX, targetY, targetWidth, targetHeight ); exportCtx.restore(); } } else { exportCtx.drawImage( canvas, drawSourceX, drawSourceY, drawSourceWidth, drawSourceHeight, 0, 0, exportWidth, exportHeight ); } const blob = await new Promise((resolve) => { exportCanvas.toBlob(resolve, "image/jpeg", jpegQuality); }); if (!blob) { throw new Error("导出图片失败"); } const dataUri = await new Promise((resolve, reject) => { const reader = new FileReader(); reader.onload = () => resolve(String(reader.result || "")); reader.onerror = () => reject(new Error("读取导出图片失败")); reader.readAsDataURL(blob); }); return { dataUri, width: exportWidth, height: exportHeight, }; } type FlushMode = "preview" | "refine"; type NormalizedRegion = { x: number; y: number; width: number; height: number; }; function clamp01(value: number) { return Math.min(1, Math.max(0, value)); } function mergeRegion( current: NormalizedRegion | null, next: NormalizedRegion ): NormalizedRegion { if (!current) return next; const x1 = Math.min(current.x, next.x); const y1 = Math.min(current.y, next.y); const x2 = Math.max(current.x + current.width, next.x + next.width); const y2 = Math.max(current.y + current.height, next.y + next.height); return { x: clamp01(x1), y: clamp01(y1), width: clamp01(x2 - x1), height: clamp01(y2 - y1), }; } function toNormalizedRegion( canvas: HTMLCanvasElement, rect: { minX: number; minY: number; maxX: number; maxY: number; } ): NormalizedRegion | null { const bounds = canvas.getBoundingClientRect(); const width = Math.max(1, bounds.width); const height = Math.max(1, bounds.height); const x = clamp01(rect.minX / width); const y = clamp01(rect.minY / height); const right = clamp01(rect.maxX / width); const bottom = clamp01(rect.maxY / height); const regionWidth = right - x; const regionHeight = bottom - y; if (regionWidth <= 0 || regionHeight <= 0) return null; return { x, y, width: regionWidth, height: regionHeight }; } async function createMaskDataUri( width: number, height: number, region: NormalizedRegion ) { const canvas = document.createElement("canvas"); canvas.width = Math.max(1, width); canvas.height = Math.max(1, height); const ctx = canvas.getContext("2d"); if (!ctx) { throw new Error("无法初始化遮罩画布"); } const basePad = Math.max(12, Math.floor(Math.max(width, height) * 0.02)); let x = Math.floor(region.x * width) - basePad; let y = Math.floor(region.y * height) - basePad; let regionWidth = Math.ceil(region.width * width) + basePad * 2; let regionHeight = Math.ceil(region.height * height) + basePad * 2; const minPatch = Math.max(40, Math.floor(Math.max(width, height) * 0.1)); if (regionWidth < minPatch) { const delta = Math.ceil((minPatch - regionWidth) / 2); x -= delta; regionWidth += delta * 2; } if (regionHeight < minPatch) { const delta = Math.ceil((minPatch - regionHeight) / 2); y -= delta; regionHeight += delta * 2; } x = Math.max(0, x); y = Math.max(0, y); const right = Math.min(width, x + regionWidth); const bottom = Math.min(height, y + regionHeight); ctx.fillStyle = "#000000"; ctx.fillRect(0, 0, width, height); ctx.fillStyle = "#ffffff"; ctx.fillRect(x, y, Math.max(1, right - x), Math.max(1, bottom - y)); return canvas.toDataURL("image/png"); } export function RealtimeCanvasDialog({ open, onOpenChange, initialImageUrl, onApplyResult, standalone = false, }: RealtimeCanvasDialogProps) { const canvasRef = useRef(null); const previewCanvasRef = useRef(null); const canvasWrapRef = useRef(null); const connectionRef = useRef(null); const pointerDownRef = useRef(false); const lastPointRef = useRef<{ x: number; y: number } | null>(null); const shapeStartPointRef = useRef<{ x: number; y: number } | null>(null); const pendingFlushTimerRef = useRef(null); const flushInFlightRef = useRef(false); const dirtyRef = useRef(false); const dirtyRegionRef = useRef(null); const draftVersionRef = useRef(0); const pendingFlushModeRef = useRef(null); const pendingFlushForceRef = useRef(false); const latencyEmaRef = useRef(null); const baseThrottleMsRef = useRef(defaultRealtimeConfig.transport.throttleInterval); const pumpFlushRef = useRef<(() => void) | null>(null); const warningMessageRef = useRef(""); const seqRef = useRef(0); const requestStartMapRef = useRef>(new Map()); const lastRenderedSeqRef = useRef(0); const metricsQueueRef = useRef([]); const sessionIdRef = useRef(`rt-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`); const seedRef = useRef(createSessionSeed()); const hasLoadedInitialImageRef = useRef(false); const canvasBackgroundRef = useRef(CANVAS_BACKGROUND); // History Management const historyRef = useRef([]); const historyIndexRef = useRef(-1); const cursorRef = useRef(null); const uploadInputRef = useRef(null); const [canUndo, setCanUndo] = useState(false); const [canRedo, setCanRedo] = useState(false); const [config, setConfig] = useState(defaultRealtimeConfig); const [status, setStatus] = useState("idle"); const [statusMessage, setStatusMessage] = useState("等待连接"); const [prompt, setPrompt] = useState(FALLBACK_PROMPT); const [previewUrl, setPreviewUrl] = useState(""); const [brushMode, setBrushMode] = useState<"brush" | "eraser" | "rect" | "circle" | "line">("brush"); const [brushColor, setBrushColor] = useState(BRUSH_COLOR_PRESETS[0]); const [canvasBackground, setCanvasBackground] = useState(canvasBackgroundRef.current); const [brushSize, setBrushSize] = useState(6); const [adaptiveThrottleMs, setAdaptiveThrottleMs] = useState( defaultRealtimeConfig.transport.throttleInterval ); const [refreshToken, setRefreshToken] = useState(0); const statusBadge = useMemo(() => { if (status === "live") return { label: "实时中", className: "bg-emerald-50 text-emerald-700" }; if (status === "connecting") { return { label: "连接中", className: "bg-blue-50 text-blue-700" }; } if (status === "error") return { label: "异常", className: "bg-rose-50 text-rose-700" }; return { label: "待启动", className: "bg-zinc-100 text-zinc-600" }; }, [status]); const pushMetric = useCallback((event: any) => { metricsQueueRef.current.push(event); if (metricsQueueRef.current.length > 120) { metricsQueueRef.current = metricsQueueRef.current.slice(-80); } }, []); const flushMetrics = useCallback(async () => { if (!open || metricsQueueRef.current.length === 0) return; const events = metricsQueueRef.current.splice(0, metricsQueueRef.current.length); try { await fetch("/api/realtime/metrics", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ sessionId: sessionIdRef.current, events, }), }); } catch { metricsQueueRef.current.unshift(...events.slice(-20)); } }, [open]); const saveHistory = useCallback(() => { const canvas = canvasRef.current; if (!canvas) return; const ctx = canvas.getContext("2d", { willReadFrequently: true }); if (!ctx) return; // If we are not at the end of history, truncate future history if (historyIndexRef.current < historyRef.current.length - 1) { historyRef.current = historyRef.current.slice(0, historyIndexRef.current + 1); } try { const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); historyRef.current.push(imageData); historyIndexRef.current = historyRef.current.length - 1; } catch (error) { console.warn("[RealtimeCanvas] saveHistory skipped:", error); return; } // Limit history size if (historyRef.current.length > 50) { historyRef.current.shift(); historyIndexRef.current--; } setCanUndo(historyIndexRef.current > 0); setCanRedo(false); }, []); const markCanvasDirty = useCallback((region?: NormalizedRegion | null) => { dirtyRef.current = true; draftVersionRef.current += 1; if (!region) { dirtyRegionRef.current = { x: 0, y: 0, width: 1, height: 1 }; return; } dirtyRegionRef.current = mergeRegion(dirtyRegionRef.current, region); }, []); const markCanvasDirtyRect = useCallback( (rect: { minX: number; minY: number; maxX: number; maxY: number }) => { const canvas = canvasRef.current; if (!canvas) return; const region = toNormalizedRegion(canvas, rect); if (!region) return; markCanvasDirty(region); }, [markCanvasDirty] ); const markCanvasDirtyStroke = useCallback( (from: { x: number; y: number }, to: { x: number; y: number }, lineWidth: number) => { const pad = Math.max(6, Math.ceil(lineWidth * 1.2)); markCanvasDirtyRect({ minX: Math.min(from.x, to.x) - pad, minY: Math.min(from.y, to.y) - pad, maxX: Math.max(from.x, to.x) + pad, maxY: Math.max(from.y, to.y) + pad, }); }, [markCanvasDirtyRect] ); const flushFrame = useCallback( async (mode: FlushMode, force = false) => { if (!open) return; if (!connectionRef.current) return; if (flushInFlightRef.current) return; if (!dirtyRef.current && !force) return; const canvas = canvasRef.current; if (!canvas) return; flushInFlightRef.current = true; const seq = seqRef.current + 1; seqRef.current = seq; const requestId = `${sessionIdRef.current}-${seq}`; requestStartMapRef.current.set(requestId, Date.now()); const draftVersionAtSend = draftVersionRef.current; const dirtyRegionAtSend = dirtyRegionRef.current ? { ...dirtyRegionRef.current } : null; try { const exportedImage = await canvasToJpegDataUri( canvas, config.image.maxInputResolution, config.image.jpegQuality, canvasBackgroundRef.current, { autoCrop: false } ); const imageDataUri = exportedImage.dataUri; const shouldUseMask = Boolean(dirtyRegionAtSend) && mode !== "refine" && (dirtyRegionAtSend?.width || 0) * (dirtyRegionAtSend?.height || 0) < 0.92; const maskImageUrl = shouldUseMask && dirtyRegionAtSend ? await createMaskDataUri( exportedImage.width, exportedImage.height, dirtyRegionAtSend ) : undefined; const activeConnection = connectionRef.current; if (!activeConnection) { requestStartMapRef.current.delete(requestId); setStatus("connecting"); setStatusMessage("连接重建中,正在重试..."); return; } setStatus("connecting"); setStatusMessage("正在请求实时生成..."); const response = await fetch("/api/realtime/generate", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ requestId, imageUrl: imageDataUri, prompt: prompt.trim() || FALLBACK_PROMPT, seed: seedRef.current, strictSketch: config.behavior?.strictSketch !== false, imageSize: config.model.defaults.imageSize, steps: config.model.defaults.steps, cfgScale: config.model.defaults.cfgScale, strength: config.model.defaults.strength, qualityProfile: mode, maskImageUrl, roi: dirtyRegionAtSend || undefined, }), }); const payload = (await response.json().catch(() => ({}))) as { imageUrl?: string; imageDataURI?: string; cost?: number; transport?: string; warning?: string; message?: string; error?: string; }; if (!response.ok) { throw new Error(payload.message || payload.error || "实时生成失败"); } const nextPreviewUrl = String(payload.imageDataURI || payload.imageUrl || ""); if (!nextPreviewUrl) { throw new Error("实时生成未返回图像"); } const startedAt = requestStartMapRef.current.get(requestId) || Date.now(); const latencyMs = Date.now() - startedAt; requestStartMapRef.current.delete(requestId); if (seq > 0 && seq < lastRenderedSeqRef.current) { return; } lastRenderedSeqRef.current = Math.max(lastRenderedSeqRef.current, seq); setPreviewUrl(nextPreviewUrl); setStatus("live"); setStatusMessage( payload.transport === "ws" ? "实时渲染中(WS)" : "实时渲染中(HTTP)" ); const upstreamWarning = String(payload.warning || "").trim(); if (upstreamWarning && upstreamWarning !== warningMessageRef.current) { warningMessageRef.current = upstreamWarning; toast.warning(upstreamWarning); } if (draftVersionAtSend === draftVersionRef.current) { dirtyRef.current = false; dirtyRegionRef.current = null; } const previousLatency = latencyEmaRef.current; const nextLatency = previousLatency === null ? latencyMs : previousLatency * 0.72 + latencyMs * 0.28; latencyEmaRef.current = nextLatency; const computedThrottle = Math.round( Math.max(baseThrottleMsRef.current, Math.min(520, nextLatency * 0.45)) ); setAdaptiveThrottleMs((prev) => Math.abs(prev - computedThrottle) >= 8 ? computedThrottle : prev ); pushMetric({ name: "frame_send", ts: Date.now(), seq, }); pushMetric({ name: "frame_result", ts: Date.now(), seq, latencyMs, queueDepth: Number(payload.cost || 0), qualityProfile: mode, hasMask: Boolean(maskImageUrl), transport: payload.transport || "unknown", }); } catch (error) { requestStartMapRef.current.delete(requestId); console.error("[RealtimeCanvas] flush frame failed:", error); setStatus("error"); setStatusMessage(error instanceof Error ? error.message : "发送失败"); } finally { flushInFlightRef.current = false; if (pendingFlushModeRef.current) { if (pendingFlushTimerRef.current) { window.clearTimeout(pendingFlushTimerRef.current); pendingFlushTimerRef.current = null; } const shouldDelay = pendingFlushModeRef.current === "preview"; if (shouldDelay) { pendingFlushTimerRef.current = window.setTimeout(() => { pendingFlushTimerRef.current = null; pumpFlushRef.current?.(); }, adaptiveThrottleMs); } else { pumpFlushRef.current?.(); } } } }, [adaptiveThrottleMs, config, open, prompt, pushMetric] ); const pumpFlush = useCallback(() => { if (flushInFlightRef.current) return; const mode = pendingFlushModeRef.current; const force = pendingFlushForceRef.current; if (!mode) return; pendingFlushModeRef.current = null; pendingFlushForceRef.current = false; void flushFrame(mode, force); }, [flushFrame]); const scheduleFlush = useCallback( (force = false, markDirty = true, mode?: FlushMode) => { if (markDirty) { markCanvasDirty(null); } const nextMode: FlushMode = mode || (force ? "refine" : "preview"); if (pendingFlushModeRef.current === "refine" || nextMode === "refine") { pendingFlushModeRef.current = "refine"; } else { pendingFlushModeRef.current = "preview"; } pendingFlushForceRef.current = pendingFlushForceRef.current || force; if (!dirtyRef.current && !pendingFlushForceRef.current) { pendingFlushModeRef.current = null; pendingFlushForceRef.current = false; return; } if (force || pendingFlushModeRef.current === "refine") { if (pendingFlushTimerRef.current) { window.clearTimeout(pendingFlushTimerRef.current); pendingFlushTimerRef.current = null; } pumpFlush(); return; } if (pendingFlushTimerRef.current) return; pendingFlushTimerRef.current = window.setTimeout(() => { pendingFlushTimerRef.current = null; pumpFlush(); }, adaptiveThrottleMs); }, [adaptiveThrottleMs, markCanvasDirty, pumpFlush] ); const scheduleFlushRef = useRef(scheduleFlush); const flushMetricsRef = useRef(flushMetrics); useEffect(() => { scheduleFlushRef.current = scheduleFlush; }, [scheduleFlush]); useEffect(() => { pumpFlushRef.current = pumpFlush; }, [pumpFlush]); useEffect(() => { flushMetricsRef.current = flushMetrics; }, [flushMetrics]); const clearPreviewCanvas = useCallback(() => { const previewCanvas = previewCanvasRef.current; if (!previewCanvas) return; const previewCtx = previewCanvas.getContext("2d"); if (!previewCtx) return; previewCtx.clearRect(0, 0, previewCanvas.width, previewCanvas.height); }, []); const undo = useCallback(() => { if (historyIndexRef.current <= 0) return; historyIndexRef.current--; const imageData = historyRef.current[historyIndexRef.current]; const canvas = canvasRef.current; const ctx = canvas?.getContext("2d"); if (canvas && ctx && imageData) { ctx.putImageData(imageData, 0, 0); setCanUndo(historyIndexRef.current > 0); setCanRedo(historyIndexRef.current < historyRef.current.length - 1); scheduleFlush(true); } }, [scheduleFlush]); const redo = useCallback(() => { if (historyIndexRef.current >= historyRef.current.length - 1) return; historyIndexRef.current++; const imageData = historyRef.current[historyIndexRef.current]; const canvas = canvasRef.current; const ctx = canvas?.getContext("2d"); if (canvas && ctx && imageData) { ctx.putImageData(imageData, 0, 0); setCanUndo(historyIndexRef.current > 0); setCanRedo(historyIndexRef.current < historyRef.current.length - 1); scheduleFlush(true); } }, [scheduleFlush]); const resetCanvas = useCallback(() => { const canvas = canvasRef.current; if (!canvas) return; const ctx = canvas.getContext("2d"); if (!ctx) return; ctx.save(); ctx.globalCompositeOperation = "source-over"; ctx.fillStyle = canvasBackgroundRef.current; ctx.fillRect(0, 0, canvas.width, canvas.height); ctx.restore(); // Clear history and save initial state historyRef.current = []; historyIndexRef.current = -1; saveHistory(); }, [saveHistory]); const drawInitialImage = useCallback(async () => { if (!open || !initialImageUrl || hasLoadedInitialImageRef.current) return false; const canvas = canvasRef.current; if (!canvas) return false; const ctx = canvas.getContext("2d"); if (!ctx) return false; const image = new Image(); image.crossOrigin = "anonymous"; await new Promise((resolve, reject) => { image.onload = () => resolve(); image.onerror = () => reject(new Error("加载初始图片失败")); image.src = initialImageUrl; }); const scale = Math.min(canvas.width / image.width, canvas.height / image.height); const drawWidth = image.width * scale; const drawHeight = image.height * scale; const offsetX = (canvas.width - drawWidth) / 2; const offsetY = (canvas.height - drawHeight) / 2; ctx.drawImage(image, offsetX, offsetY, drawWidth, drawHeight); hasLoadedInitialImageRef.current = true; markCanvasDirty(null); saveHistory(); // Save initial image state return true; }, [initialImageUrl, markCanvasDirty, open, saveHistory]); const initializeCanvas = useCallback(() => { const canvas = canvasRef.current; const previewCanvas = previewCanvasRef.current; const wrap = canvasWrapRef.current; if (!canvas || !previewCanvas || !wrap) return; const rect = wrap.getBoundingClientRect(); const dpr = Math.max(1, window.devicePixelRatio || 1); canvas.width = Math.max(640, Math.floor(rect.width * dpr)); canvas.height = Math.max(480, Math.floor(rect.height * dpr)); canvas.style.width = `${Math.floor(rect.width)}px`; canvas.style.height = `${Math.floor(rect.height)}px`; previewCanvas.width = canvas.width; previewCanvas.height = canvas.height; previewCanvas.style.width = canvas.style.width; previewCanvas.style.height = canvas.style.height; const ctx = canvas.getContext("2d"); const previewCtx = previewCanvas.getContext("2d"); if (!ctx) return; if (!previewCtx) return; ctx.setTransform(1, 0, 0, 1, 0, 0); ctx.scale(dpr, dpr); previewCtx.setTransform(1, 0, 0, 1, 0, 0); previewCtx.scale(dpr, dpr); previewCtx.clearRect(0, 0, previewCanvas.width, previewCanvas.height); resetCanvas(); hasLoadedInitialImageRef.current = false; }, [resetCanvas]); const getCanvasPoint = useCallback((event: ReactPointerEvent) => { const canvas = canvasRef.current; if (!canvas) return null; const rect = canvas.getBoundingClientRect(); return { x: event.clientX - rect.left, y: event.clientY - rect.top, }; }, []); const drawStroke = useCallback( (from: { x: number; y: number }, to: { x: number; y: number }) => { const canvas = canvasRef.current; if (!canvas) return; const ctx = canvas.getContext("2d"); if (!ctx) return; ctx.save(); if (brushMode === "eraser") { ctx.globalCompositeOperation = "destination-out"; ctx.strokeStyle = "rgba(0,0,0,1)"; ctx.lineWidth = brushSize * 1.45; } else { ctx.globalCompositeOperation = "source-over"; ctx.strokeStyle = brushColor; ctx.lineWidth = brushSize; } ctx.lineCap = "round"; ctx.lineJoin = "round"; ctx.beginPath(); ctx.moveTo(from.x, from.y); ctx.lineTo(to.x, to.y); ctx.stroke(); ctx.restore(); }, [brushMode, brushColor, brushSize] ); const drawShape = useCallback( ( from: { x: number; y: number }, to: { x: number; y: number }, targetCtx?: CanvasRenderingContext2D | null ) => { const defaultCtx = canvasRef.current?.getContext("2d"); const ctx = targetCtx || defaultCtx; if (!ctx) return; const minX = Math.min(from.x, to.x); const minY = Math.min(from.y, to.y); const width = Math.abs(to.x - from.x); const height = Math.abs(to.y - from.y); ctx.save(); ctx.globalCompositeOperation = "source-over"; ctx.strokeStyle = brushColor; ctx.lineWidth = Math.max(1, brushSize * 0.9); ctx.lineCap = "round"; ctx.lineJoin = "round"; if (brushMode === "rect") { ctx.strokeRect(minX, minY, width, height); } else if (brushMode === "circle") { const centerX = minX + width / 2; const centerY = minY + height / 2; const radiusX = width / 2; const radiusY = height / 2; ctx.beginPath(); ctx.ellipse(centerX, centerY, radiusX, radiusY, 0, 0, Math.PI * 2); ctx.stroke(); } else if (brushMode === "line") { ctx.beginPath(); ctx.moveTo(from.x, from.y); ctx.lineTo(to.x, to.y); ctx.stroke(); } ctx.restore(); }, [brushMode, brushColor, brushSize] ); const handlePointerDown = useCallback( (event: ReactPointerEvent) => { event.preventDefault(); const point = getCanvasPoint(event); if (!point) return; pointerDownRef.current = true; lastPointRef.current = point; if (isShapeMode(brushMode)) { shapeStartPointRef.current = point; clearPreviewCanvas(); } else { drawStroke(point, point); const lineWidth = brushMode === "eraser" ? brushSize * 1.45 : brushSize; markCanvasDirtyStroke(point, point, lineWidth); scheduleFlush(false, false, "preview"); } event.currentTarget.setPointerCapture(event.pointerId); }, [ brushMode, brushSize, clearPreviewCanvas, drawStroke, getCanvasPoint, markCanvasDirtyStroke, scheduleFlush, ] ); const handlePointerLeave = useCallback(() => { const wasDrawing = pointerDownRef.current; if (cursorRef.current) { cursorRef.current.style.opacity = '0'; } pointerDownRef.current = false; lastPointRef.current = null; shapeStartPointRef.current = null; clearPreviewCanvas(); if (wasDrawing) { scheduleFlush(true, false, "refine"); } }, [clearPreviewCanvas, scheduleFlush]); const handlePointerEnter = useCallback(() => { if (cursorRef.current) { cursorRef.current.style.opacity = brushMode === "brush" || brushMode === "eraser" ? "1" : "0"; } }, [brushMode]); const handlePointerMove = useCallback( (event: ReactPointerEvent) => { event.preventDefault(); // Update cursor const point = getCanvasPoint(event); if (point && cursorRef.current) { const showCursor = brushMode === "brush" || brushMode === "eraser"; if (showCursor) { cursorRef.current.style.opacity = "1"; cursorRef.current.style.transform = `translate(${point.x - brushSize / 2}px, ${point.y - brushSize / 2}px)`; cursorRef.current.style.width = `${brushSize}px`; cursorRef.current.style.height = `${brushSize}px`; } else { cursorRef.current.style.opacity = "0"; } } if (!pointerDownRef.current) return; if (!point || !lastPointRef.current) return; if (isShapeMode(brushMode) && shapeStartPointRef.current) { const previewCanvas = previewCanvasRef.current; const previewCtx = previewCanvas?.getContext("2d"); if (previewCanvas && previewCtx) { previewCtx.clearRect(0, 0, previewCanvas.width, previewCanvas.height); drawShape(shapeStartPointRef.current, point, previewCtx); } } else { drawStroke(lastPointRef.current, point); const lineWidth = brushMode === "eraser" ? brushSize * 1.45 : brushSize; markCanvasDirtyStroke(lastPointRef.current, point, lineWidth); } lastPointRef.current = point; scheduleFlush(false, false, "preview"); }, [ drawShape, drawStroke, getCanvasPoint, scheduleFlush, brushMode, brushSize, markCanvasDirtyStroke, ] ); const handlePointerUp = useCallback( (event: ReactPointerEvent) => { event.preventDefault(); pointerDownRef.current = false; const endPoint = getCanvasPoint(event); const shapeStart = shapeStartPointRef.current; if (isShapeMode(brushMode) && endPoint && shapeStart) { drawShape(shapeStart, endPoint); const lineWidth = Math.max(1, brushSize * 0.9); const pad = Math.max(6, Math.ceil(lineWidth * 1.2)); markCanvasDirtyRect({ minX: Math.min(shapeStart.x, endPoint.x) - pad, minY: Math.min(shapeStart.y, endPoint.y) - pad, maxX: Math.max(shapeStart.x, endPoint.x) + pad, maxY: Math.max(shapeStart.y, endPoint.y) + pad, }); clearPreviewCanvas(); } lastPointRef.current = null; shapeStartPointRef.current = null; try { event.currentTarget.releasePointerCapture(event.pointerId); } catch { // ignore } saveHistory(); // Save state after stroke scheduleFlush(true, false, "refine"); }, [ brushMode, brushSize, clearPreviewCanvas, drawShape, getCanvasPoint, markCanvasDirtyRect, saveHistory, scheduleFlush, ] ); useEffect(() => { clearPreviewCanvas(); }, [brushMode, clearPreviewCanvas]); // Shortcuts useEffect(() => { if (!open) return; const handleKeyDown = (e: KeyboardEvent) => { // Undo/Redo if ((e.metaKey || e.ctrlKey) && e.key === 'z') { e.preventDefault(); if (e.shiftKey) { redo(); } else { undo(); } } // Brush Size if (e.key === '[') { setBrushSize(prev => Math.max(2, prev - 2)); } if (e.key === ']') { setBrushSize(prev => Math.min(50, prev + 2)); } }; window.addEventListener('keydown', handleKeyDown); return () => window.removeEventListener('keydown', handleKeyDown); }, [open, undo, redo]); useEffect(() => { if (!open) return; seedRef.current = createSessionSeed(); sessionIdRef.current = `rt-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; seqRef.current = 0; lastRenderedSeqRef.current = 0; requestStartMapRef.current.clear(); draftVersionRef.current = 0; dirtyRef.current = false; dirtyRegionRef.current = null; pendingFlushModeRef.current = null; pendingFlushForceRef.current = false; latencyEmaRef.current = null; warningMessageRef.current = ""; metricsQueueRef.current = []; }, [open]); useEffect(() => { if (!open) return; initializeCanvas(); void drawInitialImage().then((hasInitialImage) => { if (hasInitialImage) { scheduleFlushRef.current(true, false); } }); const onResize = () => initializeCanvas(); window.addEventListener("resize", onResize); return () => { window.removeEventListener("resize", onResize); }; }, [drawInitialImage, initializeCanvas, open]); useEffect(() => { if (!open) return; let cancelled = false; setStatus("connecting"); setStatusMessage("正在初始化实时连接..."); const start = async () => { try { let configResponse: Response | null = null; let configJson: Partial & { message?: string; retryAfter?: number } = {}; for (let attempt = 0; attempt < 3; attempt += 1) { configResponse = await fetch("/api/realtime/config", { cache: "no-store" }); configJson = (await configResponse.json().catch(() => ({}))) as Partial & { message?: string; retryAfter?: number; }; if (configResponse.ok) break; const retryAfterSeconds = Number(configJson.retryAfter || 0); const shouldRetry429 = configResponse.status === 429 && attempt < 2; if (!shouldRetry429) { throw new Error(configJson.message || "无法加载实时配置"); } const retryDelay = retryAfterSeconds > 0 ? retryAfterSeconds * 1000 : 700 * (attempt + 1); await new Promise((resolve) => window.setTimeout(resolve, retryDelay)); } if (!configResponse?.ok) { throw new Error(configJson.message || "无法加载实时配置"); } if (cancelled) return; const resolvedConfig: RealtimeConfig = { transport: { throttleInterval: configJson.transport?.throttleInterval || defaultRealtimeConfig.transport.throttleInterval, maxBuffering: configJson.transport?.maxBuffering || defaultRealtimeConfig.transport.maxBuffering, }, image: { maxInputResolution: configJson.image?.maxInputResolution || defaultRealtimeConfig.image.maxInputResolution, jpegQuality: configJson.image?.jpegQuality || defaultRealtimeConfig.image.jpegQuality, }, model: { id: configJson.model?.id || defaultRealtimeConfig.model.id, defaults: { steps: configJson.model?.defaults?.steps || defaultRealtimeConfig.model.defaults.steps, cfgScale: configJson.model?.defaults?.cfgScale || defaultRealtimeConfig.model.defaults.cfgScale, strength: configJson.model?.defaults?.strength || defaultRealtimeConfig.model.defaults.strength, imageSize: configJson.model?.defaults?.imageSize || defaultRealtimeConfig.model.defaults.imageSize, }, }, behavior: { strictSketch: configJson.behavior?.strictSketch !== false, }, }; setConfig(resolvedConfig); baseThrottleMsRef.current = resolvedConfig.transport.throttleInterval; setAdaptiveThrottleMs(resolvedConfig.transport.throttleInterval); if (cancelled) return; connectionRef.current = { provider: "runware", modelId: resolvedConfig.model.id, reconnectToken: refreshToken, }; setStatus("live"); setStatusMessage("实时服务已就绪"); } catch (error) { if (cancelled) return; console.error("[RealtimeCanvas] init failed:", error); setStatus("error"); setStatusMessage(error instanceof Error ? error.message : "实时初始化失败"); } }; void start(); const metricTimer = window.setInterval(() => { void flushMetricsRef.current(); }, 5000); return () => { cancelled = true; if (pendingFlushTimerRef.current) { window.clearTimeout(pendingFlushTimerRef.current); pendingFlushTimerRef.current = null; } window.clearInterval(metricTimer); void flushMetricsRef.current(); connectionRef.current?.close?.(); connectionRef.current = null; setStatus("idle"); setStatusMessage("等待连接"); pointerDownRef.current = false; lastPointRef.current = null; pendingFlushModeRef.current = null; pendingFlushForceRef.current = false; }; }, [open, pushMetric, refreshToken]); const forceReconnect = useCallback(() => { connectionRef.current = null; setStatus("connecting"); setStatusMessage("正在重连..."); setRefreshToken((prev) => prev + 1); }, []); const clearSketch = useCallback(() => { resetCanvas(); clearPreviewCanvas(); scheduleFlush(true, true, "refine"); }, [clearPreviewCanvas, resetCanvas, scheduleFlush]); const handleSketchUpload = useCallback( async (event: ChangeEvent) => { const file = event.target.files?.[0]; if (!file) return; if (!file.type.startsWith("image/")) { toast.error("请上传图片文件"); event.currentTarget.value = ""; return; } try { const objectUrl = URL.createObjectURL(file); const image = new Image(); await new Promise((resolve, reject) => { image.onload = () => resolve(); image.onerror = () => reject(new Error("图片加载失败")); image.src = objectUrl; }); const canvas = canvasRef.current; const ctx = canvas?.getContext("2d"); if (!canvas || !ctx) { URL.revokeObjectURL(objectUrl); throw new Error("画布未初始化"); } const rect = canvas.getBoundingClientRect(); const canvasWidth = Math.max(1, rect.width); const canvasHeight = Math.max(1, rect.height); const scale = Math.min(canvasWidth / image.width, canvasHeight / image.height); const drawWidth = image.width * scale; const drawHeight = image.height * scale; const offsetX = (canvasWidth - drawWidth) / 2; const offsetY = (canvasHeight - drawHeight) / 2; ctx.drawImage(image, offsetX, offsetY, drawWidth, drawHeight); URL.revokeObjectURL(objectUrl); saveHistory(); scheduleFlush(true); toast.success("图片已导入绘画区"); } catch (error) { toast.error(error instanceof Error ? error.message : "导入图片失败"); } finally { event.currentTarget.value = ""; } }, [saveHistory, scheduleFlush] ); const handleCanvasBackgroundChange = useCallback( (nextColor: string) => { const canvas = canvasRef.current; if (!canvas) return; const ctx = canvas.getContext("2d"); if (!ctx) return; const snapshot = document.createElement("canvas"); snapshot.width = canvas.width; snapshot.height = canvas.height; const snapshotCtx = snapshot.getContext("2d"); if (!snapshotCtx) return; snapshotCtx.drawImage(canvas, 0, 0); ctx.save(); ctx.globalCompositeOperation = "source-over"; ctx.fillStyle = nextColor; ctx.fillRect(0, 0, canvas.width, canvas.height); ctx.drawImage(snapshot, 0, 0); ctx.restore(); canvasBackgroundRef.current = nextColor; setCanvasBackground(nextColor); saveHistory(); scheduleFlush(true); }, [saveHistory, scheduleFlush] ); const handleDownloadCanvas = useCallback(async () => { const canvas = canvasRef.current; if (!canvas) return; try { const exportedImage = await canvasToJpegDataUri( canvas, 2048, 0.92, canvasBackgroundRef.current ); const anchor = document.createElement("a"); anchor.href = exportedImage.dataUri; anchor.download = `realtime-canvas-${Date.now()}.jpg`; anchor.click(); toast.success("已保存当前画布"); } catch (error) { toast.error(error instanceof Error ? error.message : "保存失败"); } }, []); const handleApplyToCanvas = useCallback(() => { if (!previewUrl) { toast.error("请先生成预览结果"); return; } onApplyResult({ imageUrl: previewUrl, prompt: prompt.trim() || FALLBACK_PROMPT, }); onOpenChange(false); }, [onApplyResult, onOpenChange, previewUrl, prompt]); const content = ( <> {standalone ? (

实时画布

左侧绘制草图,右侧实时预览。

) : ( 实时画布 左侧绘制草图,右侧实时预览。 )} {standalone ? ( ) : null} {/* Top Header - Minimal */}
Realtime Canvas
{statusBadge.label}
{/* Main Canvas Area - Split View */}
{/* Left: Input Canvas */}
{/* Custom Brush Cursor */}
INPUT
{/* Right: Output Preview */}
{previewUrl ? ( 实时预览 ) : (
{status === "connecting" ? ( <> Connecting... ) : ( <> Start drawing to generate )}
)}
OUTPUT
{/* Floating Toolbar - Top (Lovart-like) */}
{BRUSH_COLOR_PRESETS.map((color) => (
setBrushSize(val)} />
{CANVAS_BACKGROUND_PRESETS.map((color) => (
{/* Floating Toolbar - Bottom (Prompt) */}
setPrompt(e.target.value)} onBlur={() => scheduleFlush(true, false)} placeholder="Describe what you want to create..." className="flex-1 h-10 border-0 bg-transparent focus-visible:ring-0 px-2 text-base shadow-none" />
{statusMessage} • Latency: {adaptiveThrottleMs}ms
); if (standalone) { return (
{content}
); } return ( {content} ); }