ai-chatbot/components/canvas/RealtimeCanvasDialog.tsx
Your Name 9c4e51d3fd
Some checks failed
Lint / build (20) (push) Has been cancelled
Playwright Tests / test (push) Has been cancelled
chore(ui): sync latest manual canvas and settings updates
2026-02-22 16:38:54 +08:00

1275 lines
45 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"use client";
import {
useCallback,
useEffect,
useMemo,
useRef,
useState,
type ChangeEvent,
type PointerEvent as ReactPointerEvent,
} from "react";
import * as fal from "@fal-ai/serverless-client";
import {
Dialog,
DialogContent,
DialogDescription,
DialogHeader,
DialogTitle,
} from "@/components/ui/dialog";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { Slider } from "@/components/ui/slider";
import { cn } from "@/lib/utils";
import {
Eraser,
Loader2,
Paintbrush,
Square,
Circle as CircleIcon,
Slash,
Upload,
RefreshCw,
Sparkles,
Trash2,
Download,
Undo2,
Redo2,
} from "lucide-react";
import { toast } from "sonner";
type RealtimeConfig = {
transport: {
throttleInterval: number;
maxBuffering: number;
};
image: {
maxInputResolution: number;
jpegQuality: number;
};
model: {
id: string;
defaults: {
steps: number;
scheduleMu: number;
feedback: number;
imageSize: "square" | "square_hd";
};
};
};
type RealtimeStatus = "idle" | "connecting" | "live" | "error";
type RealtimeCanvasDialogProps = {
open: boolean;
onOpenChange: (open: boolean) => void;
initialImageUrl?: string | null;
onApplyResult: (payload: { imageUrl: string; prompt: string }) => void;
};
const FALLBACK_PROMPT = "将草图渲染为高质感产品设计图,保持结构和主轮廓一致。";
const CANVAS_BACKGROUND = "#ffffff";
const BRUSH_COLOR_PRESETS = [
"#18181b",
"#ef4444",
"#f59e0b",
"#22c55e",
"#3b82f6",
"#a855f7",
] as const;
const CANVAS_BACKGROUND_PRESETS = ["#ffffff", "#f4f4f5", "#fdf2f8", "#ecfeff", "#f0fdf4"] as const;
const defaultRealtimeConfig: RealtimeConfig = {
transport: {
throttleInterval: 100,
maxBuffering: 2,
},
image: {
maxInputResolution: 1024,
jpegQuality: 0.55,
},
model: {
id: "fal-ai/flux-2/klein/realtime",
defaults: {
steps: 3,
scheduleMu: 2.3,
feedback: 0.95,
imageSize: "square",
},
},
};
function buildImageDataUrl(image: any) {
if (!image) return "";
if (typeof image.url === "string" && image.url.trim()) {
return image.url;
}
if (typeof image.content === "string" && image.content.trim()) {
if (image.content.startsWith("data:")) {
return image.content;
}
const contentType = image.content_type || "image/jpeg";
return `data:${contentType};base64,${image.content}`;
}
return "";
}
function isShapeMode(mode: "brush" | "eraser" | "rect" | "circle" | "line") {
return mode === "rect" || mode === "circle" || mode === "line";
}
async function canvasToJpegDataUri(
canvas: HTMLCanvasElement,
maxInputResolution: number,
jpegQuality: number,
backgroundColor: string
) {
const sourceWidth = canvas.width;
const sourceHeight = canvas.height;
const longestSide = Math.max(sourceWidth, sourceHeight);
const scale = longestSide > maxInputResolution ? maxInputResolution / longestSide : 1;
const exportWidth = Math.max(1, Math.round(sourceWidth * scale));
const exportHeight = Math.max(1, Math.round(sourceHeight * scale));
const exportCanvas = document.createElement("canvas");
exportCanvas.width = exportWidth;
exportCanvas.height = exportHeight;
const exportCtx = exportCanvas.getContext("2d");
if (!exportCtx) {
throw new Error("无法初始化导出画布");
}
exportCtx.fillStyle = backgroundColor;
exportCtx.fillRect(0, 0, exportWidth, exportHeight);
exportCtx.drawImage(canvas, 0, 0, exportWidth, exportHeight);
const blob = await new Promise<Blob | null>((resolve) => {
exportCanvas.toBlob(resolve, "image/jpeg", jpegQuality);
});
if (!blob) {
throw new Error("导出图片失败");
}
return await new Promise<string>((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => resolve(String(reader.result || ""));
reader.onerror = () => reject(new Error("读取导出图片失败"));
reader.readAsDataURL(blob);
});
}
export function RealtimeCanvasDialog({
open,
onOpenChange,
initialImageUrl,
}: RealtimeCanvasDialogProps) {
const canvasRef = useRef<HTMLCanvasElement | null>(null);
const previewCanvasRef = useRef<HTMLCanvasElement | null>(null);
const canvasWrapRef = useRef<HTMLDivElement | null>(null);
const connectionRef = useRef<any>(null);
const pointerDownRef = useRef(false);
const lastPointRef = useRef<{ x: number; y: number } | null>(null);
const shapeStartPointRef = useRef<{ x: number; y: number } | null>(null);
const pendingFlushTimerRef = useRef<number | null>(null);
const flushInFlightRef = useRef(false);
const dirtyRef = useRef(false);
const seqRef = useRef(0);
const requestSeqMapRef = useRef<Map<string, number>>(new Map());
const requestStartMapRef = useRef<Map<string, number>>(new Map());
const lastRenderedSeqRef = useRef(0);
const metricsQueueRef = useRef<any[]>([]);
const sessionIdRef = useRef(`rt-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`);
const hasLoadedInitialImageRef = useRef(false);
const canvasBackgroundRef = useRef(CANVAS_BACKGROUND);
// History Management
const historyRef = useRef<ImageData[]>([]);
const historyIndexRef = useRef(-1);
const cursorRef = useRef<HTMLDivElement>(null);
const uploadInputRef = useRef<HTMLInputElement | null>(null);
const [canUndo, setCanUndo] = useState(false);
const [canRedo, setCanRedo] = useState(false);
const [config, setConfig] = useState<RealtimeConfig>(defaultRealtimeConfig);
const [status, setStatus] = useState<RealtimeStatus>("idle");
const [statusMessage, setStatusMessage] = useState("等待连接");
const [prompt, setPrompt] = useState(FALLBACK_PROMPT);
const [previewUrl, setPreviewUrl] = useState("");
const [brushMode, setBrushMode] = useState<"brush" | "eraser" | "rect" | "circle" | "line">("brush");
const [brushColor, setBrushColor] = useState<string>(BRUSH_COLOR_PRESETS[0]);
const [canvasBackground, setCanvasBackground] = useState<string>(canvasBackgroundRef.current);
const [brushSize, setBrushSize] = useState(6);
const [adaptiveThrottleMs, setAdaptiveThrottleMs] = useState<number>(
defaultRealtimeConfig.transport.throttleInterval
);
const [refreshToken, setRefreshToken] = useState(0);
const statusBadge = useMemo(() => {
if (status === "live") return { label: "实时中", className: "bg-emerald-50 text-emerald-700" };
if (status === "connecting") {
return { label: "连接中", className: "bg-blue-50 text-blue-700" };
}
if (status === "error") return { label: "异常", className: "bg-rose-50 text-rose-700" };
return { label: "待启动", className: "bg-zinc-100 text-zinc-600" };
}, [status]);
const pushMetric = useCallback((event: any) => {
metricsQueueRef.current.push(event);
if (metricsQueueRef.current.length > 120) {
metricsQueueRef.current = metricsQueueRef.current.slice(-80);
}
}, []);
const flushMetrics = useCallback(async () => {
if (!open || metricsQueueRef.current.length === 0) return;
const events = metricsQueueRef.current.splice(0, metricsQueueRef.current.length);
try {
await fetch("/api/realtime/metrics", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
sessionId: sessionIdRef.current,
events,
}),
});
} catch {
metricsQueueRef.current.unshift(...events.slice(-20));
}
}, [open]);
const saveHistory = useCallback(() => {
const canvas = canvasRef.current;
if (!canvas) return;
const ctx = canvas.getContext("2d", { willReadFrequently: true });
if (!ctx) return;
// If we are not at the end of history, truncate future history
if (historyIndexRef.current < historyRef.current.length - 1) {
historyRef.current = historyRef.current.slice(0, historyIndexRef.current + 1);
}
try {
const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
historyRef.current.push(imageData);
historyIndexRef.current = historyRef.current.length - 1;
} catch (error) {
console.warn("[RealtimeCanvas] saveHistory skipped:", error);
return;
}
// Limit history size
if (historyRef.current.length > 50) {
historyRef.current.shift();
historyIndexRef.current--;
}
setCanUndo(historyIndexRef.current > 0);
setCanRedo(false);
}, []);
const flushFrame = useCallback(
async (force = false) => {
if (!open) return;
if (!connectionRef.current) return;
if (flushInFlightRef.current) return;
if (!dirtyRef.current && !force) return;
const canvas = canvasRef.current;
if (!canvas) return;
flushInFlightRef.current = true;
const seq = seqRef.current + 1;
seqRef.current = seq;
const requestId = `${sessionIdRef.current}-${seq}`;
requestSeqMapRef.current.set(requestId, seq);
requestStartMapRef.current.set(requestId, Date.now());
try {
const imageDataUri = await canvasToJpegDataUri(
canvas,
config.image.maxInputResolution,
config.image.jpegQuality,
canvasBackgroundRef.current
);
dirtyRef.current = false;
setStatus("connecting");
setStatusMessage("正在请求实时生成...");
connectionRef.current.send({
request_id: requestId,
image_url: imageDataUri,
prompt: prompt.trim() || FALLBACK_PROMPT,
image_size: config.model.defaults.imageSize,
num_inference_steps: config.model.defaults.steps,
schedule_mu: config.model.defaults.scheduleMu,
output_feedback_strength: config.model.defaults.feedback,
});
pushMetric({
name: "frame_send",
ts: Date.now(),
seq,
});
} catch (error) {
console.error("[RealtimeCanvas] flush frame failed:", error);
setStatus("error");
setStatusMessage(error instanceof Error ? error.message : "发送失败");
} finally {
flushInFlightRef.current = false;
}
},
[config, open, prompt, pushMetric]
);
const scheduleFlush = useCallback(
(force = false, markDirty = true) => {
if (markDirty) {
dirtyRef.current = true;
}
if (!dirtyRef.current) return;
if (force) {
if (pendingFlushTimerRef.current) {
window.clearTimeout(pendingFlushTimerRef.current);
pendingFlushTimerRef.current = null;
}
void flushFrame(true);
return;
}
if (pendingFlushTimerRef.current) return;
pendingFlushTimerRef.current = window.setTimeout(() => {
pendingFlushTimerRef.current = null;
void flushFrame();
}, adaptiveThrottleMs);
},
[adaptiveThrottleMs, flushFrame]
);
const clearPreviewCanvas = useCallback(() => {
const previewCanvas = previewCanvasRef.current;
if (!previewCanvas) return;
const previewCtx = previewCanvas.getContext("2d");
if (!previewCtx) return;
previewCtx.clearRect(0, 0, previewCanvas.width, previewCanvas.height);
}, []);
const undo = useCallback(() => {
if (historyIndexRef.current <= 0) return;
historyIndexRef.current--;
const imageData = historyRef.current[historyIndexRef.current];
const canvas = canvasRef.current;
const ctx = canvas?.getContext("2d");
if (canvas && ctx && imageData) {
ctx.putImageData(imageData, 0, 0);
setCanUndo(historyIndexRef.current > 0);
setCanRedo(historyIndexRef.current < historyRef.current.length - 1);
scheduleFlush(true);
}
}, [scheduleFlush]);
const redo = useCallback(() => {
if (historyIndexRef.current >= historyRef.current.length - 1) return;
historyIndexRef.current++;
const imageData = historyRef.current[historyIndexRef.current];
const canvas = canvasRef.current;
const ctx = canvas?.getContext("2d");
if (canvas && ctx && imageData) {
ctx.putImageData(imageData, 0, 0);
setCanUndo(historyIndexRef.current > 0);
setCanRedo(historyIndexRef.current < historyRef.current.length - 1);
scheduleFlush(true);
}
}, [scheduleFlush]);
const resetCanvas = useCallback(() => {
const canvas = canvasRef.current;
if (!canvas) return;
const ctx = canvas.getContext("2d");
if (!ctx) return;
ctx.save();
ctx.globalCompositeOperation = "source-over";
ctx.fillStyle = canvasBackgroundRef.current;
ctx.fillRect(0, 0, canvas.width, canvas.height);
ctx.restore();
// Clear history and save initial state
historyRef.current = [];
historyIndexRef.current = -1;
saveHistory();
}, [saveHistory]);
const drawInitialImage = useCallback(async () => {
if (!open || !initialImageUrl || hasLoadedInitialImageRef.current) return;
const canvas = canvasRef.current;
if (!canvas) return;
const ctx = canvas.getContext("2d");
if (!ctx) return;
const image = new Image();
image.crossOrigin = "anonymous";
await new Promise<void>((resolve, reject) => {
image.onload = () => resolve();
image.onerror = () => reject(new Error("加载初始图片失败"));
image.src = initialImageUrl;
});
const scale = Math.min(canvas.width / image.width, canvas.height / image.height);
const drawWidth = image.width * scale;
const drawHeight = image.height * scale;
const offsetX = (canvas.width - drawWidth) / 2;
const offsetY = (canvas.height - drawHeight) / 2;
ctx.drawImage(image, offsetX, offsetY, drawWidth, drawHeight);
hasLoadedInitialImageRef.current = true;
dirtyRef.current = true;
saveHistory(); // Save initial image state
}, [initialImageUrl, open, saveHistory]);
const initializeCanvas = useCallback(() => {
const canvas = canvasRef.current;
const previewCanvas = previewCanvasRef.current;
const wrap = canvasWrapRef.current;
if (!canvas || !previewCanvas || !wrap) return;
const rect = wrap.getBoundingClientRect();
const dpr = Math.max(1, window.devicePixelRatio || 1);
canvas.width = Math.max(640, Math.floor(rect.width * dpr));
canvas.height = Math.max(480, Math.floor(rect.height * dpr));
canvas.style.width = `${Math.floor(rect.width)}px`;
canvas.style.height = `${Math.floor(rect.height)}px`;
previewCanvas.width = canvas.width;
previewCanvas.height = canvas.height;
previewCanvas.style.width = canvas.style.width;
previewCanvas.style.height = canvas.style.height;
const ctx = canvas.getContext("2d");
const previewCtx = previewCanvas.getContext("2d");
if (!ctx) return;
if (!previewCtx) return;
ctx.setTransform(1, 0, 0, 1, 0, 0);
ctx.scale(dpr, dpr);
previewCtx.setTransform(1, 0, 0, 1, 0, 0);
previewCtx.scale(dpr, dpr);
previewCtx.clearRect(0, 0, previewCanvas.width, previewCanvas.height);
resetCanvas();
hasLoadedInitialImageRef.current = false;
}, [resetCanvas]);
const getCanvasPoint = useCallback((event: ReactPointerEvent<HTMLCanvasElement>) => {
const canvas = canvasRef.current;
if (!canvas) return null;
const rect = canvas.getBoundingClientRect();
return {
x: event.clientX - rect.left,
y: event.clientY - rect.top,
};
}, []);
const drawStroke = useCallback(
(from: { x: number; y: number }, to: { x: number; y: number }) => {
const canvas = canvasRef.current;
if (!canvas) return;
const ctx = canvas.getContext("2d");
if (!ctx) return;
ctx.save();
if (brushMode === "eraser") {
ctx.globalCompositeOperation = "destination-out";
ctx.strokeStyle = "rgba(0,0,0,1)";
ctx.lineWidth = brushSize * 1.45;
} else {
ctx.globalCompositeOperation = "source-over";
ctx.strokeStyle = brushColor;
ctx.lineWidth = brushSize;
}
ctx.lineCap = "round";
ctx.lineJoin = "round";
ctx.beginPath();
ctx.moveTo(from.x, from.y);
ctx.lineTo(to.x, to.y);
ctx.stroke();
ctx.restore();
},
[brushMode, brushColor, brushSize]
);
const drawShape = useCallback(
(
from: { x: number; y: number },
to: { x: number; y: number },
targetCtx?: CanvasRenderingContext2D | null
) => {
const defaultCtx = canvasRef.current?.getContext("2d");
const ctx = targetCtx || defaultCtx;
if (!ctx) return;
const minX = Math.min(from.x, to.x);
const minY = Math.min(from.y, to.y);
const width = Math.abs(to.x - from.x);
const height = Math.abs(to.y - from.y);
ctx.save();
ctx.globalCompositeOperation = "source-over";
ctx.strokeStyle = brushColor;
ctx.lineWidth = Math.max(1, brushSize * 0.9);
ctx.lineCap = "round";
ctx.lineJoin = "round";
if (brushMode === "rect") {
ctx.strokeRect(minX, minY, width, height);
} else if (brushMode === "circle") {
const centerX = minX + width / 2;
const centerY = minY + height / 2;
const radiusX = width / 2;
const radiusY = height / 2;
ctx.beginPath();
ctx.ellipse(centerX, centerY, radiusX, radiusY, 0, 0, Math.PI * 2);
ctx.stroke();
} else if (brushMode === "line") {
ctx.beginPath();
ctx.moveTo(from.x, from.y);
ctx.lineTo(to.x, to.y);
ctx.stroke();
}
ctx.restore();
},
[brushMode, brushColor, brushSize]
);
const handlePointerDown = useCallback(
(event: ReactPointerEvent<HTMLCanvasElement>) => {
event.preventDefault();
const point = getCanvasPoint(event);
if (!point) return;
pointerDownRef.current = true;
lastPointRef.current = point;
if (isShapeMode(brushMode)) {
shapeStartPointRef.current = point;
clearPreviewCanvas();
} else {
drawStroke(point, point);
scheduleFlush();
}
event.currentTarget.setPointerCapture(event.pointerId);
},
[brushMode, clearPreviewCanvas, drawStroke, getCanvasPoint, scheduleFlush]
);
const handlePointerLeave = useCallback(() => {
const wasDrawing = pointerDownRef.current;
if (cursorRef.current) {
cursorRef.current.style.opacity = '0';
}
pointerDownRef.current = false;
lastPointRef.current = null;
shapeStartPointRef.current = null;
clearPreviewCanvas();
if (wasDrawing) {
scheduleFlush(true, false);
}
}, [clearPreviewCanvas, scheduleFlush]);
const handlePointerEnter = useCallback(() => {
if (cursorRef.current) {
cursorRef.current.style.opacity = brushMode === "brush" || brushMode === "eraser" ? "1" : "0";
}
}, [brushMode]);
const handlePointerMove = useCallback(
(event: ReactPointerEvent<HTMLCanvasElement>) => {
event.preventDefault();
// Update cursor
const point = getCanvasPoint(event);
if (point && cursorRef.current) {
const showCursor = brushMode === "brush" || brushMode === "eraser";
if (showCursor) {
cursorRef.current.style.opacity = "1";
cursorRef.current.style.transform = `translate(${point.x - brushSize / 2}px, ${point.y - brushSize / 2}px)`;
cursorRef.current.style.width = `${brushSize}px`;
cursorRef.current.style.height = `${brushSize}px`;
} else {
cursorRef.current.style.opacity = "0";
}
}
if (!pointerDownRef.current) return;
if (!point || !lastPointRef.current) return;
if (isShapeMode(brushMode) && shapeStartPointRef.current) {
const previewCanvas = previewCanvasRef.current;
const previewCtx = previewCanvas?.getContext("2d");
if (previewCanvas && previewCtx) {
previewCtx.clearRect(0, 0, previewCanvas.width, previewCanvas.height);
drawShape(shapeStartPointRef.current, point, previewCtx);
}
} else {
drawStroke(lastPointRef.current, point);
}
lastPointRef.current = point;
scheduleFlush();
},
[drawShape, drawStroke, getCanvasPoint, scheduleFlush, brushMode, brushSize]
);
const handlePointerUp = useCallback(
(event: ReactPointerEvent<HTMLCanvasElement>) => {
event.preventDefault();
pointerDownRef.current = false;
const endPoint = getCanvasPoint(event);
const shapeStart = shapeStartPointRef.current;
if (isShapeMode(brushMode) && endPoint && shapeStart) {
drawShape(shapeStart, endPoint);
clearPreviewCanvas();
}
lastPointRef.current = null;
shapeStartPointRef.current = null;
try {
event.currentTarget.releasePointerCapture(event.pointerId);
} catch {
// ignore
}
saveHistory(); // Save state after stroke
scheduleFlush(true, false);
},
[brushMode, clearPreviewCanvas, drawShape, getCanvasPoint, saveHistory, scheduleFlush]
);
useEffect(() => {
clearPreviewCanvas();
}, [brushMode, clearPreviewCanvas]);
// Shortcuts
useEffect(() => {
if (!open) return;
const handleKeyDown = (e: KeyboardEvent) => {
// Undo/Redo
if ((e.metaKey || e.ctrlKey) && e.key === 'z') {
e.preventDefault();
if (e.shiftKey) {
redo();
} else {
undo();
}
}
// Brush Size
if (e.key === '[') {
setBrushSize(prev => Math.max(2, prev - 2));
}
if (e.key === ']') {
setBrushSize(prev => Math.min(50, prev + 2));
}
};
window.addEventListener('keydown', handleKeyDown);
return () => window.removeEventListener('keydown', handleKeyDown);
}, [open, undo, redo]);
useEffect(() => {
if (!open) return;
initializeCanvas();
void drawInitialImage().then(() => {
scheduleFlush(true, false);
});
const onResize = () => initializeCanvas();
window.addEventListener("resize", onResize);
return () => {
window.removeEventListener("resize", onResize);
};
}, [drawInitialImage, initializeCanvas, open, scheduleFlush]);
useEffect(() => {
if (!open) return;
let cancelled = false;
setStatus("connecting");
setStatusMessage("正在初始化实时连接...");
const start = async () => {
try {
let configResponse: Response | null = null;
let configJson: Partial<RealtimeConfig> & { message?: string; retryAfter?: number } = {};
for (let attempt = 0; attempt < 3; attempt += 1) {
configResponse = await fetch("/api/realtime/config", { cache: "no-store" });
configJson = (await configResponse.json().catch(() => ({}))) as Partial<RealtimeConfig> & {
message?: string;
retryAfter?: number;
};
if (configResponse.ok) break;
const retryAfterSeconds = Number(configJson.retryAfter || 0);
const shouldRetry429 = configResponse.status === 429 && attempt < 2;
if (!shouldRetry429) {
throw new Error(configJson.message || "无法加载实时配置");
}
const retryDelay = retryAfterSeconds > 0 ? retryAfterSeconds * 1000 : 700 * (attempt + 1);
await new Promise((resolve) => window.setTimeout(resolve, retryDelay));
}
if (!configResponse?.ok) {
throw new Error(configJson.message || "无法加载实时配置");
}
if (cancelled) return;
const resolvedConfig: RealtimeConfig = {
transport: {
throttleInterval:
configJson.transport?.throttleInterval ||
defaultRealtimeConfig.transport.throttleInterval,
maxBuffering:
configJson.transport?.maxBuffering || defaultRealtimeConfig.transport.maxBuffering,
},
image: {
maxInputResolution:
configJson.image?.maxInputResolution || defaultRealtimeConfig.image.maxInputResolution,
jpegQuality: configJson.image?.jpegQuality || defaultRealtimeConfig.image.jpegQuality,
},
model: {
id: configJson.model?.id || defaultRealtimeConfig.model.id,
defaults: {
steps: configJson.model?.defaults?.steps || defaultRealtimeConfig.model.defaults.steps,
scheduleMu:
configJson.model?.defaults?.scheduleMu ||
defaultRealtimeConfig.model.defaults.scheduleMu,
feedback:
configJson.model?.defaults?.feedback ||
defaultRealtimeConfig.model.defaults.feedback,
imageSize:
configJson.model?.defaults?.imageSize ||
defaultRealtimeConfig.model.defaults.imageSize,
},
},
};
setConfig(resolvedConfig);
setAdaptiveThrottleMs(resolvedConfig.transport.throttleInterval);
fal.config({
proxyUrl: "/api/fal/proxy",
});
const connection = fal.realtime.connect(resolvedConfig.model.id, {
connectionKey: `canvas-realtime-${sessionIdRef.current}-${refreshToken}`,
throttleInterval: resolvedConfig.transport.throttleInterval,
maxBuffering: resolvedConfig.transport.maxBuffering,
onResult: (result: any) => {
const requestId = String(result?.request_id || "");
const seqFromRequest = requestSeqMapRef.current.get(requestId) || 0;
const seq = Math.max(seqFromRequest, Number(result?.seq || 0));
if (seq > 0 && seq < lastRenderedSeqRef.current) {
return;
}
const latestImage = Array.isArray(result?.images)
? result.images[result.images.length - 1]
: null;
const nextPreviewUrl = buildImageDataUrl(latestImage);
if (!nextPreviewUrl) return;
lastRenderedSeqRef.current = Math.max(lastRenderedSeqRef.current, seq);
setPreviewUrl(nextPreviewUrl);
setStatus("live");
setStatusMessage("实时渲染中");
if (requestId) {
const startedAt = requestStartMapRef.current.get(requestId);
if (startedAt) {
pushMetric({
name: "frame_result",
ts: Date.now(),
seq,
latencyMs: Date.now() - startedAt,
});
}
requestStartMapRef.current.delete(requestId);
requestSeqMapRef.current.delete(requestId);
}
},
onError: (error: any) => {
const errorMessage =
error?.body?.message || error?.message || "实时连接异常,请稍后重试";
setStatus("error");
setStatusMessage(String(errorMessage));
if (Number(error?.status) === 429) {
setAdaptiveThrottleMs((prev) => Math.min(240, prev + 40));
}
pushMetric({
name: "frame_error",
ts: Date.now(),
message: String(errorMessage),
});
},
});
if (cancelled) {
connection.close();
return;
}
connectionRef.current = connection;
setStatus("connecting");
setStatusMessage("连接成功,等待首帧...");
scheduleFlush(true, false);
} catch (error) {
console.error("[RealtimeCanvas] init failed:", error);
setStatus("error");
setStatusMessage(error instanceof Error ? error.message : "实时初始化失败");
}
};
void start();
const metricTimer = window.setInterval(() => {
void flushMetrics();
}, 5000);
return () => {
cancelled = true;
if (pendingFlushTimerRef.current) {
window.clearTimeout(pendingFlushTimerRef.current);
pendingFlushTimerRef.current = null;
}
window.clearInterval(metricTimer);
void flushMetrics();
connectionRef.current?.close?.();
connectionRef.current = null;
setStatus("idle");
setStatusMessage("等待连接");
pointerDownRef.current = false;
lastPointRef.current = null;
};
}, [flushMetrics, open, pushMetric, refreshToken, scheduleFlush]);
const forceReconnect = useCallback(() => {
connectionRef.current?.close?.();
connectionRef.current = null;
setStatus("connecting");
setStatusMessage("正在重连...");
setRefreshToken((prev) => prev + 1);
}, []);
const clearSketch = useCallback(() => {
resetCanvas();
clearPreviewCanvas();
dirtyRef.current = true;
scheduleFlush(true, false);
}, [clearPreviewCanvas, resetCanvas, scheduleFlush]);
const handleSketchUpload = useCallback(
async (event: ChangeEvent<HTMLInputElement>) => {
const file = event.target.files?.[0];
if (!file) return;
if (!file.type.startsWith("image/")) {
toast.error("请上传图片文件");
event.currentTarget.value = "";
return;
}
try {
const objectUrl = URL.createObjectURL(file);
const image = new Image();
await new Promise<void>((resolve, reject) => {
image.onload = () => resolve();
image.onerror = () => reject(new Error("图片加载失败"));
image.src = objectUrl;
});
const canvas = canvasRef.current;
const ctx = canvas?.getContext("2d");
if (!canvas || !ctx) {
URL.revokeObjectURL(objectUrl);
throw new Error("画布未初始化");
}
const rect = canvas.getBoundingClientRect();
const canvasWidth = Math.max(1, rect.width);
const canvasHeight = Math.max(1, rect.height);
const scale = Math.min(canvasWidth / image.width, canvasHeight / image.height);
const drawWidth = image.width * scale;
const drawHeight = image.height * scale;
const offsetX = (canvasWidth - drawWidth) / 2;
const offsetY = (canvasHeight - drawHeight) / 2;
ctx.drawImage(image, offsetX, offsetY, drawWidth, drawHeight);
URL.revokeObjectURL(objectUrl);
saveHistory();
scheduleFlush(true);
toast.success("图片已导入绘画区");
} catch (error) {
toast.error(error instanceof Error ? error.message : "导入图片失败");
} finally {
event.currentTarget.value = "";
}
},
[saveHistory, scheduleFlush]
);
const handleCanvasBackgroundChange = useCallback(
(nextColor: string) => {
const canvas = canvasRef.current;
if (!canvas) return;
const ctx = canvas.getContext("2d");
if (!ctx) return;
const snapshot = document.createElement("canvas");
snapshot.width = canvas.width;
snapshot.height = canvas.height;
const snapshotCtx = snapshot.getContext("2d");
if (!snapshotCtx) return;
snapshotCtx.drawImage(canvas, 0, 0);
ctx.save();
ctx.globalCompositeOperation = "source-over";
ctx.fillStyle = nextColor;
ctx.fillRect(0, 0, canvas.width, canvas.height);
ctx.drawImage(snapshot, 0, 0);
ctx.restore();
canvasBackgroundRef.current = nextColor;
setCanvasBackground(nextColor);
saveHistory();
scheduleFlush(true);
},
[saveHistory, scheduleFlush]
);
const handleDownloadCanvas = useCallback(async () => {
const canvas = canvasRef.current;
if (!canvas) return;
try {
const dataUrl = await canvasToJpegDataUri(canvas, 2048, 0.92, canvasBackgroundRef.current);
const anchor = document.createElement("a");
anchor.href = dataUrl;
anchor.download = `realtime-canvas-${Date.now()}.jpg`;
anchor.click();
toast.success("已保存当前画布");
} catch (error) {
toast.error(error instanceof Error ? error.message : "保存失败");
}
}, []);
return (
<Dialog open={open} onOpenChange={onOpenChange}>
<DialogContent className="h-[95vh] w-[98vw] max-w-[1800px] overflow-hidden rounded-[20px] border-0 bg-background p-0 shadow-2xl sm:rounded-[20px] md:rounded-[24px]">
<DialogHeader className="sr-only">
<DialogTitle>实时画布</DialogTitle>
<DialogDescription>左侧绘制草图,右侧实时预览。</DialogDescription>
</DialogHeader>
{/* Top Header - Minimal */}
<div className="absolute left-0 right-0 top-0 z-50 flex h-14 items-center justify-start px-6 bg-gradient-to-b from-black/20 to-transparent pointer-events-none">
<div className="pointer-events-auto flex items-center gap-3 rounded-full bg-background px-4 py-1.5 border border-border/50 shadow-sm">
<span className="font-semibold text-sm">Realtime Canvas</span>
<div className="h-4 w-[1px] bg-border" />
<span className={cn("flex items-center gap-1.5 text-xs font-medium", statusBadge.className, "bg-transparent p-0")}>
<div className={cn("h-2 w-2 rounded-full", status === "live" ? "bg-emerald-500 animate-pulse" : status === "error" ? "bg-rose-500" : "bg-zinc-400")} />
{statusBadge.label}
</span>
</div>
</div>
{/* Main Canvas Area - Split View */}
<div className="relative flex h-full w-full bg-[#f4f4f5] dark:bg-[#09090b]">
{/* Left: Input Canvas */}
<div className="relative flex-1 flex flex-col items-center justify-center overflow-hidden border-r border-border/10">
<div ref={canvasWrapRef} className="relative h-full w-full flex items-center justify-center bg-white dark:bg-[#18181b]">
<div className="absolute inset-0 pattern-grid-lg opacity-[0.03] pointer-events-none" />
<div className="relative">
<canvas
ref={canvasRef}
className={cn(
"touch-none shadow-sm block",
brushMode === "brush" || brushMode === "eraser" ? "cursor-none" : "cursor-crosshair"
)}
onPointerDown={handlePointerDown}
onPointerMove={handlePointerMove}
onPointerUp={handlePointerUp}
onPointerCancel={handlePointerUp}
onPointerEnter={handlePointerEnter}
onPointerLeave={handlePointerLeave}
/>
<canvas
ref={previewCanvasRef}
className="pointer-events-none absolute inset-0 touch-none"
/>
{/* Custom Brush Cursor */}
<div
ref={cursorRef}
className="pointer-events-none absolute top-0 left-0 rounded-full border border-zinc-500/50 bg-zinc-500/10 backdrop-invert transition-opacity duration-75 z-10"
style={{
width: brushSize,
height: brushSize,
opacity: 0,
willChange: 'transform, width, height'
}}
/>
</div>
</div>
<div className="absolute top-4 left-4 pointer-events-none z-10">
<span className="text-xs font-medium text-zinc-400 bg-black/5 px-2 py-1 rounded-md backdrop-blur-sm">INPUT</span>
</div>
</div>
{/* Right: Output Preview */}
<div className="relative flex-1 flex flex-col items-center justify-center overflow-hidden bg-zinc-50 dark:bg-[#0c0c0e]">
<div className="relative h-full w-full flex items-center justify-center">
{previewUrl ? (
<img src={previewUrl} alt="实时预览" className="max-h-full max-w-full object-contain shadow-sm" />
) : (
<div className="flex flex-col items-center justify-center text-zinc-400 gap-3">
{status === "connecting" ? (
<>
<Loader2 className="h-8 w-8 animate-spin opacity-50" />
<span className="text-sm">Connecting...</span>
</>
) : (
<>
<Sparkles className="h-8 w-8 opacity-20" />
<span className="text-sm">Start drawing to generate</span>
</>
)}
</div>
)}
</div>
<div className="absolute top-4 right-4 pointer-events-none">
<span className="text-xs font-medium text-zinc-400 bg-black/5 px-2 py-1 rounded-md backdrop-blur-sm">OUTPUT</span>
</div>
</div>
</div>
{/* Floating Toolbar - Top (Lovart-like) */}
<div className="absolute left-1/2 top-[72px] z-20 -translate-x-1/2 pointer-events-none">
<div className="pointer-events-auto flex items-center gap-1 rounded-[30px] border border-black/10 bg-white/90 p-2 shadow-xl backdrop-blur-xl">
<Button
size="icon"
variant="ghost"
className={cn(
"h-10 w-10 rounded-full text-zinc-600 hover:bg-zinc-100",
brushMode === "brush" && "bg-black text-white hover:bg-black"
)}
onClick={() => setBrushMode("brush")}
title="画笔"
>
<Paintbrush className="h-5 w-5" />
</Button>
<Button
size="icon"
variant="ghost"
className={cn(
"h-10 w-10 rounded-full text-zinc-600 hover:bg-zinc-100",
brushMode === "eraser" && "bg-black text-white hover:bg-black"
)}
onClick={() => setBrushMode("eraser")}
title="橡皮"
>
<Eraser className="h-5 w-5" />
</Button>
<Button
size="icon"
variant="ghost"
className={cn(
"h-10 w-10 rounded-full text-zinc-600 hover:bg-zinc-100",
brushMode === "rect" && "bg-black text-white hover:bg-black"
)}
onClick={() => setBrushMode("rect")}
title="矩形"
>
<Square className="h-5 w-5" />
</Button>
<Button
size="icon"
variant="ghost"
className={cn(
"h-10 w-10 rounded-full text-zinc-600 hover:bg-zinc-100",
brushMode === "circle" && "bg-black text-white hover:bg-black"
)}
onClick={() => setBrushMode("circle")}
title="圆形"
>
<CircleIcon className="h-5 w-5" />
</Button>
<Button
size="icon"
variant="ghost"
className={cn(
"h-10 w-10 rounded-full text-zinc-600 hover:bg-zinc-100",
brushMode === "line" && "bg-black text-white hover:bg-black"
)}
onClick={() => setBrushMode("line")}
title="直线"
>
<Slash className="h-5 w-5" />
</Button>
<div className="mx-1 h-7 w-px bg-black/10" />
<div className="flex items-center gap-1 px-1">
{BRUSH_COLOR_PRESETS.map((color) => (
<button
key={color}
type="button"
className={cn(
"h-5 w-5 rounded-full border border-black/15 transition-all",
brushColor === color && "scale-110 ring-2 ring-black/25"
)}
style={{ backgroundColor: color }}
onClick={() => setBrushColor(color)}
title={color}
/>
))}
<input
type="color"
value={brushColor}
onChange={(event) => setBrushColor(event.target.value)}
className="h-5 w-5 cursor-pointer rounded-full border border-black/15 bg-transparent p-0"
title="自定义颜色"
/>
</div>
<div className="mx-1 h-7 w-px bg-black/10" />
<div className="w-[88px] px-1">
<Slider
value={[brushSize]}
min={2}
max={50}
step={1}
onValueChange={([val]) => setBrushSize(val)}
/>
</div>
<div className="mx-1 h-7 w-px bg-black/10" />
<div className="flex items-center gap-1">
{CANVAS_BACKGROUND_PRESETS.map((color) => (
<button
key={color}
type="button"
className={cn(
"h-4 w-4 rounded border border-black/15 transition-all",
canvasBackground === color && "scale-110 ring-2 ring-black/20"
)}
style={{ backgroundColor: color }}
onClick={() => handleCanvasBackgroundChange(color)}
title="画布底色"
/>
))}
</div>
<div className="mx-1 h-7 w-px bg-black/10" />
<Button
size="icon"
variant="ghost"
className="h-10 w-10 rounded-full text-zinc-600 hover:bg-zinc-100"
onClick={() => uploadInputRef.current?.click()}
title="上传图片"
>
<Upload className="h-5 w-5" />
</Button>
<Button
size="icon"
variant="ghost"
className="h-10 w-10 rounded-full text-zinc-600 hover:bg-zinc-100"
onClick={clearSketch}
title="清空"
>
<Trash2 className="h-5 w-5" />
</Button>
<Button
size="icon"
variant="ghost"
className="h-10 w-10 rounded-full text-zinc-600 hover:bg-zinc-100"
onClick={undo}
disabled={!canUndo}
title="撤销"
>
<Undo2 className="h-5 w-5" />
</Button>
<Button
size="icon"
variant="ghost"
className="h-10 w-10 rounded-full text-zinc-600 hover:bg-zinc-100"
onClick={redo}
disabled={!canRedo}
title="重做"
>
<Redo2 className="h-5 w-5" />
</Button>
<Button
size="icon"
variant="ghost"
className="h-10 w-10 rounded-full text-zinc-600 hover:bg-zinc-100"
onClick={() => scheduleFlush(true)}
disabled={status === "connecting"}
title="生成"
>
{status === "connecting" ? (
<Loader2 className="h-5 w-5 animate-spin" />
) : (
<Sparkles className="h-5 w-5" />
)}
</Button>
<Button
size="icon"
variant="ghost"
className="h-10 w-10 rounded-full text-zinc-600 hover:bg-zinc-100"
onClick={handleDownloadCanvas}
title="保存"
>
<Download className="h-5 w-5" />
</Button>
<Button
size="icon"
variant="ghost"
className="h-10 w-10 rounded-full text-zinc-600 hover:bg-zinc-100"
onClick={forceReconnect}
title="重连"
>
<RefreshCw className="h-5 w-5" />
</Button>
</div>
<input
ref={uploadInputRef}
type="file"
accept="image/*"
className="hidden"
onChange={handleSketchUpload}
/>
</div>
{/* Floating Toolbar - Bottom (Prompt) */}
<div className="absolute bottom-8 left-1/2 -translate-x-1/2 w-full max-w-2xl px-4 z-20 pointer-events-none">
<div className="pointer-events-auto flex flex-col gap-2 p-1.5 bg-background/80 backdrop-blur-xl border border-border/50 shadow-2xl rounded-[24px]">
<div className="flex items-center gap-2 pl-2 pr-1.5 py-1">
<div className="flex h-8 w-8 items-center justify-center rounded-full bg-primary/10 text-primary">
<Sparkles className="h-4 w-4" />
</div>
<Input
value={prompt}
onChange={(e) => setPrompt(e.target.value)}
onBlur={() => scheduleFlush(true, false)}
placeholder="Describe what you want to create..."
className="flex-1 h-10 border-0 bg-transparent focus-visible:ring-0 px-2 text-base shadow-none"
/>
<Button
size="sm"
className="h-9 rounded-xl px-4 font-medium"
onClick={() => scheduleFlush(true)}
disabled={status === 'connecting'}
>
{status === 'connecting' ? <Loader2 className="h-4 w-4 animate-spin" /> : 'Generate'}
</Button>
</div>
</div>
<div className="text-center mt-2">
<span className="text-[10px] text-white/50 bg-black/40 px-3 py-1 rounded-full backdrop-blur-sm">
{statusMessage} • Latency: {adaptiveThrottleMs}ms
</span>
</div>
</div>
</DialogContent>
</Dialog>
);
}