956 lines
34 KiB
TypeScript
956 lines
34 KiB
TypeScript
"use client";
|
|
|
|
import { useCallback, useEffect, useMemo, useRef, useState, type PointerEvent as ReactPointerEvent } from "react";
|
|
import * as fal from "@fal-ai/serverless-client";
|
|
import {
|
|
Dialog,
|
|
DialogContent,
|
|
DialogDescription,
|
|
DialogHeader,
|
|
DialogTitle,
|
|
} from "@/components/ui/dialog";
|
|
import { Button } from "@/components/ui/button";
|
|
import { Input } from "@/components/ui/input";
|
|
import { Slider } from "@/components/ui/slider";
|
|
import { cn } from "@/lib/utils";
|
|
import {
|
|
Eraser,
|
|
Loader2,
|
|
Paintbrush,
|
|
RefreshCw,
|
|
Send,
|
|
Sparkles,
|
|
Trash2,
|
|
Wifi,
|
|
WifiOff,
|
|
Minimize2,
|
|
Maximize2,
|
|
Download,
|
|
Image as ImageIcon,
|
|
MoreHorizontal,
|
|
Undo2,
|
|
Redo2,
|
|
X,
|
|
} from "lucide-react";
|
|
import { toast } from "sonner";
|
|
|
|
type RealtimeConfig = {
|
|
transport: {
|
|
throttleInterval: number;
|
|
maxBuffering: number;
|
|
};
|
|
image: {
|
|
maxInputResolution: number;
|
|
jpegQuality: number;
|
|
};
|
|
model: {
|
|
id: string;
|
|
defaults: {
|
|
steps: number;
|
|
scheduleMu: number;
|
|
feedback: number;
|
|
imageSize: "square" | "square_hd";
|
|
};
|
|
};
|
|
};
|
|
|
|
type RealtimeStatus = "idle" | "connecting" | "live" | "error";
|
|
|
|
type RealtimeCanvasDialogProps = {
|
|
open: boolean;
|
|
onOpenChange: (open: boolean) => void;
|
|
initialImageUrl?: string | null;
|
|
onApplyResult: (payload: { imageUrl: string; prompt: string }) => void;
|
|
};
|
|
|
|
const FALLBACK_PROMPT = "将草图渲染为高质感产品设计图，保持结构和主轮廓一致。";
|
|
|
|
const defaultRealtimeConfig: RealtimeConfig = {
|
|
transport: {
|
|
throttleInterval: 100,
|
|
maxBuffering: 2,
|
|
},
|
|
image: {
|
|
maxInputResolution: 1024,
|
|
jpegQuality: 0.55,
|
|
},
|
|
model: {
|
|
id: "fal-ai/flux-2/klein/realtime",
|
|
defaults: {
|
|
steps: 3,
|
|
scheduleMu: 2.3,
|
|
feedback: 0.95,
|
|
imageSize: "square",
|
|
},
|
|
},
|
|
};
|
|
|
|
function buildImageDataUrl(image: any) {
|
|
if (!image) return "";
|
|
if (typeof image.url === "string" && image.url.trim()) {
|
|
return image.url;
|
|
}
|
|
if (typeof image.content === "string" && image.content.trim()) {
|
|
if (image.content.startsWith("data:")) {
|
|
return image.content;
|
|
}
|
|
const contentType = image.content_type || "image/jpeg";
|
|
return `data:${contentType};base64,${image.content}`;
|
|
}
|
|
return "";
|
|
}
|
|
|
|
async function canvasToJpegDataUri(
|
|
canvas: HTMLCanvasElement,
|
|
maxInputResolution: number,
|
|
jpegQuality: number
|
|
) {
|
|
const sourceWidth = canvas.width;
|
|
const sourceHeight = canvas.height;
|
|
const longestSide = Math.max(sourceWidth, sourceHeight);
|
|
const scale = longestSide > maxInputResolution ? maxInputResolution / longestSide : 1;
|
|
const exportWidth = Math.max(1, Math.round(sourceWidth * scale));
|
|
const exportHeight = Math.max(1, Math.round(sourceHeight * scale));
|
|
|
|
const exportCanvas = document.createElement("canvas");
|
|
exportCanvas.width = exportWidth;
|
|
exportCanvas.height = exportHeight;
|
|
const exportCtx = exportCanvas.getContext("2d");
|
|
if (!exportCtx) {
|
|
throw new Error("无法初始化导出画布");
|
|
}
|
|
|
|
exportCtx.fillStyle = "#d8d8db";
|
|
exportCtx.fillRect(0, 0, exportWidth, exportHeight);
|
|
exportCtx.drawImage(canvas, 0, 0, exportWidth, exportHeight);
|
|
|
|
const blob = await new Promise<Blob | null>((resolve) => {
|
|
exportCanvas.toBlob(resolve, "image/jpeg", jpegQuality);
|
|
});
|
|
if (!blob) {
|
|
throw new Error("导出图片失败");
|
|
}
|
|
|
|
return await new Promise<string>((resolve, reject) => {
|
|
const reader = new FileReader();
|
|
reader.onload = () => resolve(String(reader.result || ""));
|
|
reader.onerror = () => reject(new Error("读取导出图片失败"));
|
|
reader.readAsDataURL(blob);
|
|
});
|
|
}
|
|
|
|
export function RealtimeCanvasDialog({
|
|
open,
|
|
onOpenChange,
|
|
initialImageUrl,
|
|
onApplyResult,
|
|
}: RealtimeCanvasDialogProps) {
|
|
const canvasRef = useRef<HTMLCanvasElement | null>(null);
|
|
const canvasWrapRef = useRef<HTMLDivElement | null>(null);
|
|
const connectionRef = useRef<any>(null);
|
|
const pointerDownRef = useRef(false);
|
|
const lastPointRef = useRef<{ x: number; y: number } | null>(null);
|
|
const pendingFlushTimerRef = useRef<number | null>(null);
|
|
const flushInFlightRef = useRef(false);
|
|
const dirtyRef = useRef(false);
|
|
const seqRef = useRef(0);
|
|
const requestSeqMapRef = useRef<Map<string, number>>(new Map());
|
|
const requestStartMapRef = useRef<Map<string, number>>(new Map());
|
|
const lastRenderedSeqRef = useRef(0);
|
|
const metricsQueueRef = useRef<any[]>([]);
|
|
const sessionIdRef = useRef(`rt-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`);
|
|
const hasLoadedInitialImageRef = useRef(false);
|
|
|
|
// History Management
|
|
const historyRef = useRef<ImageData[]>([]);
|
|
const historyIndexRef = useRef(-1);
|
|
const cursorRef = useRef<HTMLDivElement>(null);
|
|
const [canUndo, setCanUndo] = useState(false);
|
|
const [canRedo, setCanRedo] = useState(false);
|
|
|
|
const [config, setConfig] = useState<RealtimeConfig>(defaultRealtimeConfig);
|
|
const [status, setStatus] = useState<RealtimeStatus>("idle");
|
|
const [statusMessage, setStatusMessage] = useState("等待连接");
|
|
const [prompt, setPrompt] = useState(FALLBACK_PROMPT);
|
|
const [previewUrl, setPreviewUrl] = useState("");
|
|
const [isApplying, setIsApplying] = useState(false);
|
|
const [brushMode, setBrushMode] = useState<"brush" | "eraser">("brush");
|
|
const [brushSize, setBrushSize] = useState(6);
|
|
const [adaptiveThrottleMs, setAdaptiveThrottleMs] = useState<number>(
|
|
defaultRealtimeConfig.transport.throttleInterval
|
|
);
|
|
const [refreshToken, setRefreshToken] = useState(0);
|
|
|
|
const statusBadge = useMemo(() => {
|
|
if (status === "live") return { label: "实时中", className: "bg-emerald-50 text-emerald-700" };
|
|
if (status === "connecting") {
|
|
return { label: "连接中", className: "bg-blue-50 text-blue-700" };
|
|
}
|
|
if (status === "error") return { label: "异常", className: "bg-rose-50 text-rose-700" };
|
|
return { label: "待启动", className: "bg-zinc-100 text-zinc-600" };
|
|
}, [status]);
|
|
|
|
const pushMetric = useCallback((event: any) => {
|
|
metricsQueueRef.current.push(event);
|
|
if (metricsQueueRef.current.length > 120) {
|
|
metricsQueueRef.current = metricsQueueRef.current.slice(-80);
|
|
}
|
|
}, []);
|
|
|
|
const flushMetrics = useCallback(async () => {
|
|
if (!open || metricsQueueRef.current.length === 0) return;
|
|
const events = metricsQueueRef.current.splice(0, metricsQueueRef.current.length);
|
|
try {
|
|
await fetch("/api/realtime/metrics", {
|
|
method: "POST",
|
|
headers: { "Content-Type": "application/json" },
|
|
body: JSON.stringify({
|
|
sessionId: sessionIdRef.current,
|
|
events,
|
|
}),
|
|
});
|
|
} catch {
|
|
metricsQueueRef.current.unshift(...events.slice(-20));
|
|
}
|
|
}, [open]);
|
|
|
|
const saveHistory = useCallback(() => {
|
|
const canvas = canvasRef.current;
|
|
if (!canvas) return;
|
|
const ctx = canvas.getContext("2d", { willReadFrequently: true });
|
|
if (!ctx) return;
|
|
|
|
// If we are not at the end of history, truncate future history
|
|
if (historyIndexRef.current < historyRef.current.length - 1) {
|
|
historyRef.current = historyRef.current.slice(0, historyIndexRef.current + 1);
|
|
}
|
|
|
|
const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
|
|
historyRef.current.push(imageData);
|
|
historyIndexRef.current = historyRef.current.length - 1;
|
|
|
|
// Limit history size
|
|
if (historyRef.current.length > 50) {
|
|
historyRef.current.shift();
|
|
historyIndexRef.current--;
|
|
}
|
|
|
|
setCanUndo(historyIndexRef.current > 0);
|
|
setCanRedo(false);
|
|
}, []);
|
|
|
|
const flushFrame = useCallback(
|
|
async (force = false) => {
|
|
if (!open) return;
|
|
if (!connectionRef.current) return;
|
|
if (flushInFlightRef.current) return;
|
|
if (!dirtyRef.current && !force) return;
|
|
|
|
const canvas = canvasRef.current;
|
|
if (!canvas) return;
|
|
|
|
flushInFlightRef.current = true;
|
|
const seq = seqRef.current + 1;
|
|
seqRef.current = seq;
|
|
const requestId = `${sessionIdRef.current}-${seq}`;
|
|
requestSeqMapRef.current.set(requestId, seq);
|
|
requestStartMapRef.current.set(requestId, Date.now());
|
|
|
|
try {
|
|
const imageDataUri = await canvasToJpegDataUri(
|
|
canvas,
|
|
config.image.maxInputResolution,
|
|
config.image.jpegQuality
|
|
);
|
|
|
|
dirtyRef.current = false;
|
|
setStatus("connecting");
|
|
setStatusMessage("正在请求实时生成...");
|
|
connectionRef.current.send({
|
|
request_id: requestId,
|
|
image_url: imageDataUri,
|
|
prompt: prompt.trim() || FALLBACK_PROMPT,
|
|
image_size: config.model.defaults.imageSize,
|
|
num_inference_steps: config.model.defaults.steps,
|
|
schedule_mu: config.model.defaults.scheduleMu,
|
|
output_feedback_strength: config.model.defaults.feedback,
|
|
});
|
|
|
|
pushMetric({
|
|
name: "frame_send",
|
|
ts: Date.now(),
|
|
seq,
|
|
});
|
|
} catch (error) {
|
|
console.error("[RealtimeCanvas] flush frame failed:", error);
|
|
setStatus("error");
|
|
setStatusMessage(error instanceof Error ? error.message : "发送失败");
|
|
} finally {
|
|
flushInFlightRef.current = false;
|
|
}
|
|
},
|
|
[config, open, prompt, pushMetric]
|
|
);
|
|
|
|
const scheduleFlush = useCallback(
|
|
(force = false) => {
|
|
dirtyRef.current = true;
|
|
if (force) {
|
|
if (pendingFlushTimerRef.current) {
|
|
window.clearTimeout(pendingFlushTimerRef.current);
|
|
pendingFlushTimerRef.current = null;
|
|
}
|
|
void flushFrame(true);
|
|
return;
|
|
}
|
|
if (pendingFlushTimerRef.current) return;
|
|
pendingFlushTimerRef.current = window.setTimeout(() => {
|
|
pendingFlushTimerRef.current = null;
|
|
void flushFrame();
|
|
}, adaptiveThrottleMs);
|
|
},
|
|
[adaptiveThrottleMs, flushFrame]
|
|
);
|
|
|
|
const undo = useCallback(() => {
|
|
if (historyIndexRef.current <= 0) return;
|
|
|
|
historyIndexRef.current--;
|
|
const imageData = historyRef.current[historyIndexRef.current];
|
|
const canvas = canvasRef.current;
|
|
const ctx = canvas?.getContext("2d");
|
|
|
|
if (canvas && ctx && imageData) {
|
|
ctx.putImageData(imageData, 0, 0);
|
|
setCanUndo(historyIndexRef.current > 0);
|
|
setCanRedo(historyIndexRef.current < historyRef.current.length - 1);
|
|
scheduleFlush(true);
|
|
}
|
|
}, [scheduleFlush]);
|
|
|
|
const redo = useCallback(() => {
|
|
if (historyIndexRef.current >= historyRef.current.length - 1) return;
|
|
|
|
historyIndexRef.current++;
|
|
const imageData = historyRef.current[historyIndexRef.current];
|
|
const canvas = canvasRef.current;
|
|
const ctx = canvas?.getContext("2d");
|
|
|
|
if (canvas && ctx && imageData) {
|
|
ctx.putImageData(imageData, 0, 0);
|
|
setCanUndo(historyIndexRef.current > 0);
|
|
setCanRedo(historyIndexRef.current < historyRef.current.length - 1);
|
|
scheduleFlush(true);
|
|
}
|
|
}, [scheduleFlush]);
|
|
|
|
const resetCanvas = useCallback(() => {
|
|
const canvas = canvasRef.current;
|
|
if (!canvas) return;
|
|
const ctx = canvas.getContext("2d");
|
|
if (!ctx) return;
|
|
ctx.save();
|
|
ctx.globalCompositeOperation = "source-over";
|
|
ctx.fillStyle = "#d8d8db";
|
|
ctx.fillRect(0, 0, canvas.width, canvas.height);
|
|
ctx.restore();
|
|
|
|
// Clear history and save initial state
|
|
historyRef.current = [];
|
|
historyIndexRef.current = -1;
|
|
saveHistory();
|
|
}, [saveHistory]);
|
|
|
|
const drawInitialImage = useCallback(async () => {
|
|
if (!open || !initialImageUrl || hasLoadedInitialImageRef.current) return;
|
|
const canvas = canvasRef.current;
|
|
if (!canvas) return;
|
|
const ctx = canvas.getContext("2d");
|
|
if (!ctx) return;
|
|
|
|
const image = new Image();
|
|
image.crossOrigin = "anonymous";
|
|
await new Promise<void>((resolve, reject) => {
|
|
image.onload = () => resolve();
|
|
image.onerror = () => reject(new Error("加载初始图片失败"));
|
|
image.src = initialImageUrl;
|
|
});
|
|
|
|
const scale = Math.min(canvas.width / image.width, canvas.height / image.height);
|
|
const drawWidth = image.width * scale;
|
|
const drawHeight = image.height * scale;
|
|
const offsetX = (canvas.width - drawWidth) / 2;
|
|
const offsetY = (canvas.height - drawHeight) / 2;
|
|
|
|
ctx.drawImage(image, offsetX, offsetY, drawWidth, drawHeight);
|
|
hasLoadedInitialImageRef.current = true;
|
|
dirtyRef.current = true;
|
|
saveHistory(); // Save initial image state
|
|
}, [initialImageUrl, open, saveHistory]);
|
|
|
|
const initializeCanvas = useCallback(() => {
|
|
const canvas = canvasRef.current;
|
|
const wrap = canvasWrapRef.current;
|
|
if (!canvas || !wrap) return;
|
|
const rect = wrap.getBoundingClientRect();
|
|
const dpr = Math.max(1, window.devicePixelRatio || 1);
|
|
canvas.width = Math.max(640, Math.floor(rect.width * dpr));
|
|
canvas.height = Math.max(480, Math.floor(rect.height * dpr));
|
|
canvas.style.width = `${Math.floor(rect.width)}px`;
|
|
canvas.style.height = `${Math.floor(rect.height)}px`;
|
|
const ctx = canvas.getContext("2d");
|
|
if (!ctx) return;
|
|
ctx.setTransform(1, 0, 0, 1, 0, 0);
|
|
ctx.scale(dpr, dpr);
|
|
resetCanvas();
|
|
hasLoadedInitialImageRef.current = false;
|
|
}, [resetCanvas]);
|
|
|
|
const getCanvasPoint = useCallback((event: ReactPointerEvent<HTMLCanvasElement>) => {
|
|
const canvas = canvasRef.current;
|
|
if (!canvas) return null;
|
|
const rect = canvas.getBoundingClientRect();
|
|
return {
|
|
x: event.clientX - rect.left,
|
|
y: event.clientY - rect.top,
|
|
};
|
|
}, []);
|
|
|
|
const drawStroke = useCallback(
|
|
(from: { x: number; y: number }, to: { x: number; y: number }) => {
|
|
const canvas = canvasRef.current;
|
|
if (!canvas) return;
|
|
const ctx = canvas.getContext("2d");
|
|
if (!ctx) return;
|
|
|
|
ctx.save();
|
|
if (brushMode === "eraser") {
|
|
ctx.globalCompositeOperation = "destination-out";
|
|
ctx.strokeStyle = "rgba(0,0,0,1)";
|
|
ctx.lineWidth = brushSize * 1.45;
|
|
} else {
|
|
ctx.globalCompositeOperation = "source-over";
|
|
ctx.strokeStyle = "#18181b";
|
|
ctx.lineWidth = brushSize;
|
|
}
|
|
ctx.lineCap = "round";
|
|
ctx.lineJoin = "round";
|
|
ctx.beginPath();
|
|
ctx.moveTo(from.x, from.y);
|
|
ctx.lineTo(to.x, to.y);
|
|
ctx.stroke();
|
|
ctx.restore();
|
|
},
|
|
[brushMode, brushSize]
|
|
);
|
|
|
|
const handlePointerDown = useCallback(
|
|
(event: ReactPointerEvent<HTMLCanvasElement>) => {
|
|
const point = getCanvasPoint(event);
|
|
if (!point) return;
|
|
pointerDownRef.current = true;
|
|
lastPointRef.current = point;
|
|
event.currentTarget.setPointerCapture(event.pointerId);
|
|
},
|
|
[getCanvasPoint]
|
|
);
|
|
|
|
const handlePointerLeave = useCallback(() => {
|
|
if (cursorRef.current) {
|
|
cursorRef.current.style.opacity = '0';
|
|
}
|
|
pointerDownRef.current = false;
|
|
lastPointRef.current = null;
|
|
scheduleFlush(true);
|
|
}, [scheduleFlush]);
|
|
|
|
const handlePointerEnter = useCallback(() => {
|
|
if (cursorRef.current) {
|
|
cursorRef.current.style.opacity = '1';
|
|
}
|
|
}, []);
|
|
|
|
const handlePointerMove = useCallback(
|
|
(event: ReactPointerEvent<HTMLCanvasElement>) => {
|
|
// Update cursor
|
|
const point = getCanvasPoint(event);
|
|
if (point && cursorRef.current) {
|
|
cursorRef.current.style.transform = `translate(${point.x - brushSize / 2}px, ${point.y - brushSize / 2}px)`;
|
|
cursorRef.current.style.width = `${brushSize}px`;
|
|
cursorRef.current.style.height = `${brushSize}px`;
|
|
}
|
|
|
|
if (!pointerDownRef.current) return;
|
|
if (!point || !lastPointRef.current) return;
|
|
drawStroke(lastPointRef.current, point);
|
|
lastPointRef.current = point;
|
|
scheduleFlush();
|
|
},
|
|
[drawStroke, getCanvasPoint, scheduleFlush, brushSize]
|
|
);
|
|
|
|
const handlePointerUp = useCallback(
|
|
(event: ReactPointerEvent<HTMLCanvasElement>) => {
|
|
pointerDownRef.current = false;
|
|
lastPointRef.current = null;
|
|
try {
|
|
event.currentTarget.releasePointerCapture(event.pointerId);
|
|
} catch {
|
|
// ignore
|
|
}
|
|
saveHistory(); // Save state after stroke
|
|
scheduleFlush(true);
|
|
},
|
|
[scheduleFlush, saveHistory]
|
|
);
|
|
|
|
// Shortcuts
|
|
useEffect(() => {
|
|
if (!open) return;
|
|
const handleKeyDown = (e: KeyboardEvent) => {
|
|
// Undo/Redo
|
|
if ((e.metaKey || e.ctrlKey) && e.key === 'z') {
|
|
e.preventDefault();
|
|
if (e.shiftKey) {
|
|
redo();
|
|
} else {
|
|
undo();
|
|
}
|
|
}
|
|
// Brush Size
|
|
if (e.key === '[') {
|
|
setBrushSize(prev => Math.max(2, prev - 2));
|
|
}
|
|
if (e.key === ']') {
|
|
setBrushSize(prev => Math.min(50, prev + 2));
|
|
}
|
|
};
|
|
window.addEventListener('keydown', handleKeyDown);
|
|
return () => window.removeEventListener('keydown', handleKeyDown);
|
|
}, [open, undo, redo]);
|
|
|
|
const uploadPreviewToCanvas = useCallback(async () => {
|
|
if (!previewUrl) {
|
|
toast.error("请先完成一帧实时预览");
|
|
return;
|
|
}
|
|
|
|
setIsApplying(true);
|
|
try {
|
|
const blob = await fetch(previewUrl).then((res) => res.blob());
|
|
const formData = new FormData();
|
|
const filename = `realtime-canvas-${Date.now()}.jpg`;
|
|
formData.append("file", new File([blob], filename, { type: blob.type || "image/jpeg" }));
|
|
|
|
const response = await fetch("/api/files/upload", {
|
|
method: "POST",
|
|
body: formData,
|
|
});
|
|
const result = await response.json().catch(() => ({}));
|
|
|
|
if (!response.ok || !result?.url) {
|
|
throw new Error(result?.error || "上传到画布失败");
|
|
}
|
|
|
|
onApplyResult({
|
|
imageUrl: String(result.url),
|
|
prompt: prompt.trim() || FALLBACK_PROMPT,
|
|
});
|
|
|
|
toast.success("已发送到画布");
|
|
onOpenChange(false);
|
|
} catch (error) {
|
|
toast.error(error instanceof Error ? error.message : "发送到画布失败");
|
|
} finally {
|
|
setIsApplying(false);
|
|
}
|
|
}, [onApplyResult, onOpenChange, previewUrl, prompt]);
|
|
|
|
useEffect(() => {
|
|
if (!open) return;
|
|
initializeCanvas();
|
|
void drawInitialImage().then(() => {
|
|
scheduleFlush(true);
|
|
});
|
|
const onResize = () => initializeCanvas();
|
|
window.addEventListener("resize", onResize);
|
|
return () => {
|
|
window.removeEventListener("resize", onResize);
|
|
};
|
|
}, [drawInitialImage, initializeCanvas, open, scheduleFlush]);
|
|
|
|
useEffect(() => {
|
|
if (!open) return;
|
|
|
|
let cancelled = false;
|
|
setStatus("connecting");
|
|
setStatusMessage("正在初始化实时连接...");
|
|
|
|
const start = async () => {
|
|
try {
|
|
const configResponse = await fetch("/api/realtime/config", { cache: "no-store" });
|
|
const configJson = (await configResponse.json().catch(() => ({}))) as Partial<RealtimeConfig>;
|
|
if (!configResponse.ok) {
|
|
throw new Error((configJson as any)?.message || "无法加载实时配置");
|
|
}
|
|
|
|
if (cancelled) return;
|
|
const resolvedConfig: RealtimeConfig = {
|
|
transport: {
|
|
throttleInterval:
|
|
configJson.transport?.throttleInterval ||
|
|
defaultRealtimeConfig.transport.throttleInterval,
|
|
maxBuffering:
|
|
configJson.transport?.maxBuffering || defaultRealtimeConfig.transport.maxBuffering,
|
|
},
|
|
image: {
|
|
maxInputResolution:
|
|
configJson.image?.maxInputResolution || defaultRealtimeConfig.image.maxInputResolution,
|
|
jpegQuality: configJson.image?.jpegQuality || defaultRealtimeConfig.image.jpegQuality,
|
|
},
|
|
model: {
|
|
id: configJson.model?.id || defaultRealtimeConfig.model.id,
|
|
defaults: {
|
|
steps: configJson.model?.defaults?.steps || defaultRealtimeConfig.model.defaults.steps,
|
|
scheduleMu:
|
|
configJson.model?.defaults?.scheduleMu ||
|
|
defaultRealtimeConfig.model.defaults.scheduleMu,
|
|
feedback:
|
|
configJson.model?.defaults?.feedback ||
|
|
defaultRealtimeConfig.model.defaults.feedback,
|
|
imageSize:
|
|
configJson.model?.defaults?.imageSize ||
|
|
defaultRealtimeConfig.model.defaults.imageSize,
|
|
},
|
|
},
|
|
};
|
|
|
|
setConfig(resolvedConfig);
|
|
setAdaptiveThrottleMs(resolvedConfig.transport.throttleInterval);
|
|
|
|
fal.config({
|
|
proxyUrl: "/api/fal/proxy",
|
|
});
|
|
|
|
const connection = fal.realtime.connect(resolvedConfig.model.id, {
|
|
connectionKey: `canvas-realtime-${sessionIdRef.current}-${refreshToken}`,
|
|
throttleInterval: resolvedConfig.transport.throttleInterval,
|
|
maxBuffering: resolvedConfig.transport.maxBuffering,
|
|
onResult: (result: any) => {
|
|
const requestId = String(result?.request_id || "");
|
|
const seqFromRequest = requestSeqMapRef.current.get(requestId) || 0;
|
|
const seq = Math.max(seqFromRequest, Number(result?.seq || 0));
|
|
if (seq > 0 && seq < lastRenderedSeqRef.current) {
|
|
return;
|
|
}
|
|
|
|
const latestImage = Array.isArray(result?.images)
|
|
? result.images[result.images.length - 1]
|
|
: null;
|
|
const nextPreviewUrl = buildImageDataUrl(latestImage);
|
|
if (!nextPreviewUrl) return;
|
|
|
|
lastRenderedSeqRef.current = Math.max(lastRenderedSeqRef.current, seq);
|
|
setPreviewUrl(nextPreviewUrl);
|
|
setStatus("live");
|
|
setStatusMessage("实时渲染中");
|
|
|
|
if (requestId) {
|
|
const startedAt = requestStartMapRef.current.get(requestId);
|
|
if (startedAt) {
|
|
pushMetric({
|
|
name: "frame_result",
|
|
ts: Date.now(),
|
|
seq,
|
|
latencyMs: Date.now() - startedAt,
|
|
});
|
|
}
|
|
requestStartMapRef.current.delete(requestId);
|
|
requestSeqMapRef.current.delete(requestId);
|
|
}
|
|
},
|
|
onError: (error: any) => {
|
|
const errorMessage =
|
|
error?.body?.message || error?.message || "实时连接异常，请稍后重试";
|
|
setStatus("error");
|
|
setStatusMessage(String(errorMessage));
|
|
if (Number(error?.status) === 429) {
|
|
setAdaptiveThrottleMs((prev) => Math.min(240, prev + 40));
|
|
}
|
|
pushMetric({
|
|
name: "frame_error",
|
|
ts: Date.now(),
|
|
message: String(errorMessage),
|
|
});
|
|
},
|
|
});
|
|
|
|
if (cancelled) {
|
|
connection.close();
|
|
return;
|
|
}
|
|
|
|
connectionRef.current = connection;
|
|
setStatus("connecting");
|
|
setStatusMessage("连接成功，等待首帧...");
|
|
scheduleFlush(true);
|
|
} catch (error) {
|
|
console.error("[RealtimeCanvas] init failed:", error);
|
|
setStatus("error");
|
|
setStatusMessage(error instanceof Error ? error.message : "实时初始化失败");
|
|
}
|
|
};
|
|
|
|
void start();
|
|
const metricTimer = window.setInterval(() => {
|
|
void flushMetrics();
|
|
}, 5000);
|
|
|
|
return () => {
|
|
cancelled = true;
|
|
if (pendingFlushTimerRef.current) {
|
|
window.clearTimeout(pendingFlushTimerRef.current);
|
|
pendingFlushTimerRef.current = null;
|
|
}
|
|
window.clearInterval(metricTimer);
|
|
void flushMetrics();
|
|
connectionRef.current?.close?.();
|
|
connectionRef.current = null;
|
|
setStatus("idle");
|
|
setStatusMessage("等待连接");
|
|
pointerDownRef.current = false;
|
|
lastPointRef.current = null;
|
|
};
|
|
}, [flushMetrics, open, pushMetric, refreshToken, scheduleFlush]);
|
|
|
|
const forceReconnect = useCallback(() => {
|
|
connectionRef.current?.close?.();
|
|
connectionRef.current = null;
|
|
setStatus("connecting");
|
|
setStatusMessage("正在重连...");
|
|
setRefreshToken((prev) => prev + 1);
|
|
}, []);
|
|
|
|
const clearSketch = useCallback(() => {
|
|
resetCanvas();
|
|
dirtyRef.current = true;
|
|
scheduleFlush(true);
|
|
}, [resetCanvas, scheduleFlush]);
|
|
|
|
return (
|
|
<Dialog open={open} onOpenChange={onOpenChange}>
|
|
<DialogContent className="h-[95vh] w-[98vw] max-w-[1800px] overflow-hidden rounded-[20px] border-0 bg-background p-0 shadow-2xl sm:rounded-[20px] md:rounded-[24px]">
|
|
{/* Top Header - Minimal */}
|
|
<div className="absolute left-0 right-0 top-0 z-50 flex h-14 items-center justify-between px-6 bg-gradient-to-b from-black/20 to-transparent pointer-events-none">
|
|
<div className="pointer-events-auto flex items-center gap-3 rounded-full bg-background/80 backdrop-blur-md px-4 py-1.5 border border-border/50 shadow-sm">
|
|
<span className="font-semibold text-sm">Realtime Canvas</span>
|
|
<div className="h-4 w-[1px] bg-border" />
|
|
<span className={cn("flex items-center gap-1.5 text-xs font-medium", statusBadge.className, "bg-transparent p-0")}>
|
|
<div className={cn("h-2 w-2 rounded-full", status === "live" ? "bg-emerald-500 animate-pulse" : status === "error" ? "bg-rose-500" : "bg-zinc-400")} />
|
|
{statusBadge.label}
|
|
</span>
|
|
</div>
|
|
|
|
<div className="pointer-events-auto flex items-center gap-2">
|
|
<Button
|
|
variant="ghost"
|
|
size="icon"
|
|
className="h-8 w-8 rounded-full bg-background/50 hover:bg-background/80 backdrop-blur-md text-foreground/80 hover:text-foreground transition-colors"
|
|
onClick={() => onOpenChange(false)}
|
|
>
|
|
<X className="h-4 w-4" />
|
|
</Button>
|
|
</div>
|
|
</div>
|
|
|
|
{/* Main Canvas Area - Split View */}
|
|
<div className="relative flex h-full w-full bg-[#f4f4f5] dark:bg-[#09090b]">
|
|
{/* Left: Input Canvas */}
|
|
<div className="relative flex-1 flex flex-col items-center justify-center overflow-hidden border-r border-border/10">
|
|
<div ref={canvasWrapRef} className="relative h-full w-full flex items-center justify-center bg-white dark:bg-[#18181b]">
|
|
<div className="absolute inset-0 pattern-grid-lg opacity-[0.03] pointer-events-none" />
|
|
|
|
<div className="relative">
|
|
<canvas
|
|
ref={canvasRef}
|
|
className="touch-none shadow-sm block cursor-none"
|
|
onPointerDown={handlePointerDown}
|
|
onPointerMove={handlePointerMove}
|
|
onPointerUp={handlePointerUp}
|
|
onPointerCancel={handlePointerUp}
|
|
onPointerEnter={handlePointerEnter}
|
|
onPointerLeave={handlePointerLeave}
|
|
/>
|
|
{/* Custom Brush Cursor */}
|
|
<div
|
|
ref={cursorRef}
|
|
className="pointer-events-none absolute top-0 left-0 rounded-full border border-zinc-500/50 bg-zinc-500/10 backdrop-invert transition-opacity duration-75 z-10"
|
|
style={{
|
|
width: brushSize,
|
|
height: brushSize,
|
|
opacity: 0,
|
|
willChange: 'transform, width, height'
|
|
}}
|
|
/>
|
|
</div>
|
|
</div>
|
|
<div className="absolute top-4 left-4 pointer-events-none z-10">
|
|
<span className="text-xs font-medium text-zinc-400 bg-black/5 px-2 py-1 rounded-md backdrop-blur-sm">INPUT</span>
|
|
</div>
|
|
</div>
|
|
|
|
{/* Right: Output Preview */}
|
|
<div className="relative flex-1 flex flex-col items-center justify-center overflow-hidden bg-zinc-50 dark:bg-[#0c0c0e]">
|
|
<div className="relative h-full w-full flex items-center justify-center">
|
|
{previewUrl ? (
|
|
<img src={previewUrl} alt="实时预览" className="max-h-full max-w-full object-contain shadow-sm" />
|
|
) : (
|
|
<div className="flex flex-col items-center justify-center text-zinc-400 gap-3">
|
|
{status === "connecting" ? (
|
|
<>
|
|
<Loader2 className="h-8 w-8 animate-spin opacity-50" />
|
|
<span className="text-sm">Connecting...</span>
|
|
</>
|
|
) : (
|
|
<>
|
|
<Sparkles className="h-8 w-8 opacity-20" />
|
|
<span className="text-sm">Start drawing to generate</span>
|
|
</>
|
|
)}
|
|
</div>
|
|
)}
|
|
</div>
|
|
<div className="absolute top-4 right-4 pointer-events-none">
|
|
<span className="text-xs font-medium text-zinc-400 bg-black/5 px-2 py-1 rounded-md backdrop-blur-sm">OUTPUT</span>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
{/* Floating Toolbar - Left (Tools) */}
|
|
<div className="absolute left-6 top-1/2 -translate-y-1/2 flex flex-col gap-4 z-20 pointer-events-none">
|
|
<div className="pointer-events-auto flex flex-col gap-2 p-2 bg-background/90 backdrop-blur-xl border border-border/50 shadow-xl rounded-2xl">
|
|
<Button
|
|
size="icon"
|
|
variant="ghost"
|
|
className="h-10 w-10 rounded-xl hover:bg-zinc-100 dark:hover:bg-zinc-800"
|
|
onClick={undo}
|
|
disabled={!canUndo}
|
|
title="Undo (Cmd+Z)"
|
|
>
|
|
<Undo2 className="h-5 w-5 opacity-70" />
|
|
</Button>
|
|
<Button
|
|
size="icon"
|
|
variant="ghost"
|
|
className="h-10 w-10 rounded-xl hover:bg-zinc-100 dark:hover:bg-zinc-800"
|
|
onClick={redo}
|
|
disabled={!canRedo}
|
|
title="Redo (Cmd+Shift+Z)"
|
|
>
|
|
<Redo2 className="h-5 w-5 opacity-70" />
|
|
</Button>
|
|
|
|
<div className="h-[1px] bg-border/50 mx-2 my-1" />
|
|
|
|
<Button
|
|
size="icon"
|
|
variant={brushMode === "brush" ? "secondary" : "ghost"}
|
|
className={cn("h-10 w-10 rounded-xl transition-all", brushMode === "brush" && "bg-primary text-primary-foreground shadow-sm")}
|
|
onClick={() => setBrushMode("brush")}
|
|
title="Brush"
|
|
>
|
|
<Paintbrush className="h-5 w-5" />
|
|
</Button>
|
|
<Button
|
|
size="icon"
|
|
variant={brushMode === "eraser" ? "secondary" : "ghost"}
|
|
className={cn("h-10 w-10 rounded-xl transition-all", brushMode === "eraser" && "bg-primary text-primary-foreground shadow-sm")}
|
|
onClick={() => setBrushMode("eraser")}
|
|
>
|
|
<Eraser className="h-5 w-5" />
|
|
</Button>
|
|
<div className="h-[1px] bg-border/50 mx-2 my-1" />
|
|
|
|
<div className="flex flex-col items-center gap-2 py-2">
|
|
<span className="text-[10px] font-medium text-muted-foreground">{brushSize}px</span>
|
|
<Slider
|
|
orientation="vertical"
|
|
value={[brushSize]}
|
|
min={2}
|
|
max={50}
|
|
step={1}
|
|
onValueChange={([val]) => setBrushSize(val)}
|
|
className="h-24 w-4"
|
|
/>
|
|
</div>
|
|
|
|
<div className="h-[1px] bg-border/50 mx-2 my-1" />
|
|
<Button
|
|
size="icon"
|
|
variant="ghost"
|
|
className="h-10 w-10 rounded-xl hover:bg-destructive/10 hover:text-destructive"
|
|
onClick={clearSketch}
|
|
>
|
|
<Trash2 className="h-5 w-5" />
|
|
</Button>
|
|
</div>
|
|
</div>
|
|
|
|
{/* Floating Toolbar - Bottom (Prompt) */}
|
|
<div className="absolute bottom-8 left-1/2 -translate-x-1/2 w-full max-w-2xl px-4 z-20 pointer-events-none">
|
|
<div className="pointer-events-auto flex flex-col gap-2 p-1.5 bg-background/80 backdrop-blur-xl border border-border/50 shadow-2xl rounded-[24px]">
|
|
<div className="flex items-center gap-2 pl-2 pr-1.5 py-1">
|
|
<div className="flex h-8 w-8 items-center justify-center rounded-full bg-primary/10 text-primary">
|
|
<Sparkles className="h-4 w-4" />
|
|
</div>
|
|
<Input
|
|
value={prompt}
|
|
onChange={(e) => setPrompt(e.target.value)}
|
|
onBlur={() => scheduleFlush(true)}
|
|
placeholder="Describe what you want to create..."
|
|
className="flex-1 h-10 border-0 bg-transparent focus-visible:ring-0 px-2 text-base shadow-none"
|
|
/>
|
|
<Button
|
|
size="sm"
|
|
className="h-9 rounded-xl px-4 font-medium"
|
|
onClick={() => scheduleFlush(true)}
|
|
disabled={status === 'connecting'}
|
|
>
|
|
{status === 'connecting' ? <Loader2 className="h-4 w-4 animate-spin" /> : 'Generate'}
|
|
</Button>
|
|
</div>
|
|
</div>
|
|
<div className="text-center mt-2">
|
|
<span className="text-[10px] text-white/50 bg-black/40 px-3 py-1 rounded-full backdrop-blur-sm">
|
|
{statusMessage} • Latency: {adaptiveThrottleMs}ms
|
|
</span>
|
|
</div>
|
|
</div>
|
|
|
|
{/* Floating Toolbar - Right (Actions) */}
|
|
<div className="absolute right-6 top-6 flex flex-col gap-3 z-20 pointer-events-none">
|
|
<div className="pointer-events-auto flex flex-col gap-2">
|
|
<Button
|
|
className="h-10 px-4 rounded-full shadow-lg bg-primary text-primary-foreground hover:bg-primary/90"
|
|
onClick={uploadPreviewToCanvas}
|
|
disabled={!previewUrl || isApplying}
|
|
>
|
|
{isApplying ? <Loader2 className="mr-2 h-4 w-4 animate-spin" /> : <Send className="mr-2 h-4 w-4" />}
|
|
Send to Canvas
|
|
</Button>
|
|
|
|
<Button
|
|
variant="secondary"
|
|
className="h-10 px-4 rounded-full shadow-md bg-background/80 backdrop-blur-md hover:bg-background"
|
|
onClick={forceReconnect}
|
|
>
|
|
<RefreshCw className="mr-2 h-4 w-4" />
|
|
Reconnect
|
|
</Button>
|
|
</div>
|
|
</div>
|
|
</DialogContent>
|
|
</Dialog>
|
|
);
|
|
}
|