1163 lines
42 KiB
TypeScript
1163 lines
42 KiB
TypeScript
"use client";
|
||
|
||
import {
|
||
useCallback,
|
||
useEffect,
|
||
useMemo,
|
||
useRef,
|
||
useState,
|
||
type ChangeEvent,
|
||
type PointerEvent as ReactPointerEvent,
|
||
} from "react";
|
||
import * as fal from "@fal-ai/serverless-client";
|
||
import {
|
||
Dialog,
|
||
DialogContent,
|
||
DialogDescription,
|
||
DialogHeader,
|
||
DialogTitle,
|
||
} from "@/components/ui/dialog";
|
||
import { Button } from "@/components/ui/button";
|
||
import { Input } from "@/components/ui/input";
|
||
import { Slider } from "@/components/ui/slider";
|
||
import { cn } from "@/lib/utils";
|
||
import {
|
||
Eraser,
|
||
Loader2,
|
||
Paintbrush,
|
||
Square,
|
||
Circle as CircleIcon,
|
||
Upload,
|
||
RefreshCw,
|
||
Send,
|
||
Sparkles,
|
||
Trash2,
|
||
Wifi,
|
||
WifiOff,
|
||
Minimize2,
|
||
Maximize2,
|
||
Download,
|
||
Image as ImageIcon,
|
||
MoreHorizontal,
|
||
Undo2,
|
||
Redo2,
|
||
X,
|
||
} from "lucide-react";
|
||
import { toast } from "sonner";
|
||
|
||
type RealtimeConfig = {
|
||
transport: {
|
||
throttleInterval: number;
|
||
maxBuffering: number;
|
||
};
|
||
image: {
|
||
maxInputResolution: number;
|
||
jpegQuality: number;
|
||
};
|
||
model: {
|
||
id: string;
|
||
defaults: {
|
||
steps: number;
|
||
scheduleMu: number;
|
||
feedback: number;
|
||
imageSize: "square" | "square_hd";
|
||
};
|
||
};
|
||
};
|
||
|
||
type RealtimeStatus = "idle" | "connecting" | "live" | "error";
|
||
|
||
type RealtimeCanvasDialogProps = {
|
||
open: boolean;
|
||
onOpenChange: (open: boolean) => void;
|
||
initialImageUrl?: string | null;
|
||
onApplyResult: (payload: { imageUrl: string; prompt: string }) => void;
|
||
};
|
||
|
||
const FALLBACK_PROMPT = "将草图渲染为高质感产品设计图，保持结构和主轮廓一致。";
|
||
const CANVAS_BACKGROUND = "#ffffff";
|
||
|
||
const defaultRealtimeConfig: RealtimeConfig = {
|
||
transport: {
|
||
throttleInterval: 100,
|
||
maxBuffering: 2,
|
||
},
|
||
image: {
|
||
maxInputResolution: 1024,
|
||
jpegQuality: 0.55,
|
||
},
|
||
model: {
|
||
id: "fal-ai/flux-2/klein/realtime",
|
||
defaults: {
|
||
steps: 3,
|
||
scheduleMu: 2.3,
|
||
feedback: 0.95,
|
||
imageSize: "square",
|
||
},
|
||
},
|
||
};
|
||
|
||
function buildImageDataUrl(image: any) {
|
||
if (!image) return "";
|
||
if (typeof image.url === "string" && image.url.trim()) {
|
||
return image.url;
|
||
}
|
||
if (typeof image.content === "string" && image.content.trim()) {
|
||
if (image.content.startsWith("data:")) {
|
||
return image.content;
|
||
}
|
||
const contentType = image.content_type || "image/jpeg";
|
||
return `data:${contentType};base64,${image.content}`;
|
||
}
|
||
return "";
|
||
}
|
||
|
||
function isShapeMode(mode: "brush" | "eraser" | "rect" | "circle") {
|
||
return mode === "rect" || mode === "circle";
|
||
}
|
||
|
||
async function canvasToJpegDataUri(
|
||
canvas: HTMLCanvasElement,
|
||
maxInputResolution: number,
|
||
jpegQuality: number
|
||
) {
|
||
const sourceWidth = canvas.width;
|
||
const sourceHeight = canvas.height;
|
||
const longestSide = Math.max(sourceWidth, sourceHeight);
|
||
const scale = longestSide > maxInputResolution ? maxInputResolution / longestSide : 1;
|
||
const exportWidth = Math.max(1, Math.round(sourceWidth * scale));
|
||
const exportHeight = Math.max(1, Math.round(sourceHeight * scale));
|
||
|
||
const exportCanvas = document.createElement("canvas");
|
||
exportCanvas.width = exportWidth;
|
||
exportCanvas.height = exportHeight;
|
||
const exportCtx = exportCanvas.getContext("2d");
|
||
if (!exportCtx) {
|
||
throw new Error("无法初始化导出画布");
|
||
}
|
||
|
||
exportCtx.fillStyle = CANVAS_BACKGROUND;
|
||
exportCtx.fillRect(0, 0, exportWidth, exportHeight);
|
||
exportCtx.drawImage(canvas, 0, 0, exportWidth, exportHeight);
|
||
|
||
const blob = await new Promise<Blob | null>((resolve) => {
|
||
exportCanvas.toBlob(resolve, "image/jpeg", jpegQuality);
|
||
});
|
||
if (!blob) {
|
||
throw new Error("导出图片失败");
|
||
}
|
||
|
||
return await new Promise<string>((resolve, reject) => {
|
||
const reader = new FileReader();
|
||
reader.onload = () => resolve(String(reader.result || ""));
|
||
reader.onerror = () => reject(new Error("读取导出图片失败"));
|
||
reader.readAsDataURL(blob);
|
||
});
|
||
}
|
||
|
||
export function RealtimeCanvasDialog({
|
||
open,
|
||
onOpenChange,
|
||
initialImageUrl,
|
||
onApplyResult,
|
||
}: RealtimeCanvasDialogProps) {
|
||
const canvasRef = useRef<HTMLCanvasElement | null>(null);
|
||
const canvasWrapRef = useRef<HTMLDivElement | null>(null);
|
||
const connectionRef = useRef<any>(null);
|
||
const pointerDownRef = useRef(false);
|
||
const lastPointRef = useRef<{ x: number; y: number } | null>(null);
|
||
const shapeStartPointRef = useRef<{ x: number; y: number } | null>(null);
|
||
const shapeSnapshotRef = useRef<ImageData | null>(null);
|
||
const pendingFlushTimerRef = useRef<number | null>(null);
|
||
const flushInFlightRef = useRef(false);
|
||
const dirtyRef = useRef(false);
|
||
const seqRef = useRef(0);
|
||
const requestSeqMapRef = useRef<Map<string, number>>(new Map());
|
||
const requestStartMapRef = useRef<Map<string, number>>(new Map());
|
||
const lastRenderedSeqRef = useRef(0);
|
||
const metricsQueueRef = useRef<any[]>([]);
|
||
const sessionIdRef = useRef(`rt-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`);
|
||
const hasLoadedInitialImageRef = useRef(false);
|
||
|
||
// History Management
|
||
const historyRef = useRef<ImageData[]>([]);
|
||
const historyIndexRef = useRef(-1);
|
||
const cursorRef = useRef<HTMLDivElement>(null);
|
||
const uploadInputRef = useRef<HTMLInputElement | null>(null);
|
||
const [canUndo, setCanUndo] = useState(false);
|
||
const [canRedo, setCanRedo] = useState(false);
|
||
|
||
const [config, setConfig] = useState<RealtimeConfig>(defaultRealtimeConfig);
|
||
const [status, setStatus] = useState<RealtimeStatus>("idle");
|
||
const [statusMessage, setStatusMessage] = useState("等待连接");
|
||
const [prompt, setPrompt] = useState(FALLBACK_PROMPT);
|
||
const [previewUrl, setPreviewUrl] = useState("");
|
||
const [isApplying, setIsApplying] = useState(false);
|
||
const [brushMode, setBrushMode] = useState<"brush" | "eraser" | "rect" | "circle">("brush");
|
||
const [brushSize, setBrushSize] = useState(6);
|
||
const [adaptiveThrottleMs, setAdaptiveThrottleMs] = useState<number>(
|
||
defaultRealtimeConfig.transport.throttleInterval
|
||
);
|
||
const [refreshToken, setRefreshToken] = useState(0);
|
||
|
||
const statusBadge = useMemo(() => {
|
||
if (status === "live") return { label: "实时中", className: "bg-emerald-50 text-emerald-700" };
|
||
if (status === "connecting") {
|
||
return { label: "连接中", className: "bg-blue-50 text-blue-700" };
|
||
}
|
||
if (status === "error") return { label: "异常", className: "bg-rose-50 text-rose-700" };
|
||
return { label: "待启动", className: "bg-zinc-100 text-zinc-600" };
|
||
}, [status]);
|
||
|
||
const pushMetric = useCallback((event: any) => {
|
||
metricsQueueRef.current.push(event);
|
||
if (metricsQueueRef.current.length > 120) {
|
||
metricsQueueRef.current = metricsQueueRef.current.slice(-80);
|
||
}
|
||
}, []);
|
||
|
||
const flushMetrics = useCallback(async () => {
|
||
if (!open || metricsQueueRef.current.length === 0) return;
|
||
const events = metricsQueueRef.current.splice(0, metricsQueueRef.current.length);
|
||
try {
|
||
await fetch("/api/realtime/metrics", {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({
|
||
sessionId: sessionIdRef.current,
|
||
events,
|
||
}),
|
||
});
|
||
} catch {
|
||
metricsQueueRef.current.unshift(...events.slice(-20));
|
||
}
|
||
}, [open]);
|
||
|
||
const saveHistory = useCallback(() => {
|
||
const canvas = canvasRef.current;
|
||
if (!canvas) return;
|
||
const ctx = canvas.getContext("2d", { willReadFrequently: true });
|
||
if (!ctx) return;
|
||
|
||
// If we are not at the end of history, truncate future history
|
||
if (historyIndexRef.current < historyRef.current.length - 1) {
|
||
historyRef.current = historyRef.current.slice(0, historyIndexRef.current + 1);
|
||
}
|
||
|
||
const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
|
||
historyRef.current.push(imageData);
|
||
historyIndexRef.current = historyRef.current.length - 1;
|
||
|
||
// Limit history size
|
||
if (historyRef.current.length > 50) {
|
||
historyRef.current.shift();
|
||
historyIndexRef.current--;
|
||
}
|
||
|
||
setCanUndo(historyIndexRef.current > 0);
|
||
setCanRedo(false);
|
||
}, []);
|
||
|
||
const flushFrame = useCallback(
|
||
async (force = false) => {
|
||
if (!open) return;
|
||
if (!connectionRef.current) return;
|
||
if (flushInFlightRef.current) return;
|
||
if (!dirtyRef.current && !force) return;
|
||
|
||
const canvas = canvasRef.current;
|
||
if (!canvas) return;
|
||
|
||
flushInFlightRef.current = true;
|
||
const seq = seqRef.current + 1;
|
||
seqRef.current = seq;
|
||
const requestId = `${sessionIdRef.current}-${seq}`;
|
||
requestSeqMapRef.current.set(requestId, seq);
|
||
requestStartMapRef.current.set(requestId, Date.now());
|
||
|
||
try {
|
||
const imageDataUri = await canvasToJpegDataUri(
|
||
canvas,
|
||
config.image.maxInputResolution,
|
||
config.image.jpegQuality
|
||
);
|
||
|
||
dirtyRef.current = false;
|
||
setStatus("connecting");
|
||
setStatusMessage("正在请求实时生成...");
|
||
connectionRef.current.send({
|
||
request_id: requestId,
|
||
image_url: imageDataUri,
|
||
prompt: prompt.trim() || FALLBACK_PROMPT,
|
||
image_size: config.model.defaults.imageSize,
|
||
num_inference_steps: config.model.defaults.steps,
|
||
schedule_mu: config.model.defaults.scheduleMu,
|
||
output_feedback_strength: config.model.defaults.feedback,
|
||
});
|
||
|
||
pushMetric({
|
||
name: "frame_send",
|
||
ts: Date.now(),
|
||
seq,
|
||
});
|
||
} catch (error) {
|
||
console.error("[RealtimeCanvas] flush frame failed:", error);
|
||
setStatus("error");
|
||
setStatusMessage(error instanceof Error ? error.message : "发送失败");
|
||
} finally {
|
||
flushInFlightRef.current = false;
|
||
}
|
||
},
|
||
[config, open, prompt, pushMetric]
|
||
);
|
||
|
||
const scheduleFlush = useCallback(
|
||
(force = false, markDirty = true) => {
|
||
if (markDirty) {
|
||
dirtyRef.current = true;
|
||
}
|
||
if (!dirtyRef.current) return;
|
||
if (force) {
|
||
if (pendingFlushTimerRef.current) {
|
||
window.clearTimeout(pendingFlushTimerRef.current);
|
||
pendingFlushTimerRef.current = null;
|
||
}
|
||
void flushFrame(true);
|
||
return;
|
||
}
|
||
if (pendingFlushTimerRef.current) return;
|
||
pendingFlushTimerRef.current = window.setTimeout(() => {
|
||
pendingFlushTimerRef.current = null;
|
||
void flushFrame();
|
||
}, adaptiveThrottleMs);
|
||
},
|
||
[adaptiveThrottleMs, flushFrame]
|
||
);
|
||
|
||
const undo = useCallback(() => {
|
||
if (historyIndexRef.current <= 0) return;
|
||
|
||
historyIndexRef.current--;
|
||
const imageData = historyRef.current[historyIndexRef.current];
|
||
const canvas = canvasRef.current;
|
||
const ctx = canvas?.getContext("2d");
|
||
|
||
if (canvas && ctx && imageData) {
|
||
ctx.putImageData(imageData, 0, 0);
|
||
setCanUndo(historyIndexRef.current > 0);
|
||
setCanRedo(historyIndexRef.current < historyRef.current.length - 1);
|
||
scheduleFlush(true);
|
||
}
|
||
}, [scheduleFlush]);
|
||
|
||
const redo = useCallback(() => {
|
||
if (historyIndexRef.current >= historyRef.current.length - 1) return;
|
||
|
||
historyIndexRef.current++;
|
||
const imageData = historyRef.current[historyIndexRef.current];
|
||
const canvas = canvasRef.current;
|
||
const ctx = canvas?.getContext("2d");
|
||
|
||
if (canvas && ctx && imageData) {
|
||
ctx.putImageData(imageData, 0, 0);
|
||
setCanUndo(historyIndexRef.current > 0);
|
||
setCanRedo(historyIndexRef.current < historyRef.current.length - 1);
|
||
scheduleFlush(true);
|
||
}
|
||
}, [scheduleFlush]);
|
||
|
||
const resetCanvas = useCallback(() => {
|
||
const canvas = canvasRef.current;
|
||
if (!canvas) return;
|
||
const ctx = canvas.getContext("2d");
|
||
if (!ctx) return;
|
||
ctx.save();
|
||
ctx.globalCompositeOperation = "source-over";
|
||
ctx.fillStyle = CANVAS_BACKGROUND;
|
||
ctx.fillRect(0, 0, canvas.width, canvas.height);
|
||
ctx.restore();
|
||
|
||
// Clear history and save initial state
|
||
historyRef.current = [];
|
||
historyIndexRef.current = -1;
|
||
saveHistory();
|
||
}, [saveHistory]);
|
||
|
||
const drawInitialImage = useCallback(async () => {
|
||
if (!open || !initialImageUrl || hasLoadedInitialImageRef.current) return;
|
||
const canvas = canvasRef.current;
|
||
if (!canvas) return;
|
||
const ctx = canvas.getContext("2d");
|
||
if (!ctx) return;
|
||
|
||
const image = new Image();
|
||
image.crossOrigin = "anonymous";
|
||
await new Promise<void>((resolve, reject) => {
|
||
image.onload = () => resolve();
|
||
image.onerror = () => reject(new Error("加载初始图片失败"));
|
||
image.src = initialImageUrl;
|
||
});
|
||
|
||
const scale = Math.min(canvas.width / image.width, canvas.height / image.height);
|
||
const drawWidth = image.width * scale;
|
||
const drawHeight = image.height * scale;
|
||
const offsetX = (canvas.width - drawWidth) / 2;
|
||
const offsetY = (canvas.height - drawHeight) / 2;
|
||
|
||
ctx.drawImage(image, offsetX, offsetY, drawWidth, drawHeight);
|
||
hasLoadedInitialImageRef.current = true;
|
||
dirtyRef.current = true;
|
||
saveHistory(); // Save initial image state
|
||
}, [initialImageUrl, open, saveHistory]);
|
||
|
||
const initializeCanvas = useCallback(() => {
|
||
const canvas = canvasRef.current;
|
||
const wrap = canvasWrapRef.current;
|
||
if (!canvas || !wrap) return;
|
||
const rect = wrap.getBoundingClientRect();
|
||
const dpr = Math.max(1, window.devicePixelRatio || 1);
|
||
canvas.width = Math.max(640, Math.floor(rect.width * dpr));
|
||
canvas.height = Math.max(480, Math.floor(rect.height * dpr));
|
||
canvas.style.width = `${Math.floor(rect.width)}px`;
|
||
canvas.style.height = `${Math.floor(rect.height)}px`;
|
||
const ctx = canvas.getContext("2d");
|
||
if (!ctx) return;
|
||
ctx.setTransform(1, 0, 0, 1, 0, 0);
|
||
ctx.scale(dpr, dpr);
|
||
resetCanvas();
|
||
hasLoadedInitialImageRef.current = false;
|
||
}, [resetCanvas]);
|
||
|
||
const getCanvasPoint = useCallback((event: ReactPointerEvent<HTMLCanvasElement>) => {
|
||
const canvas = canvasRef.current;
|
||
if (!canvas) return null;
|
||
const rect = canvas.getBoundingClientRect();
|
||
return {
|
||
x: event.clientX - rect.left,
|
||
y: event.clientY - rect.top,
|
||
};
|
||
}, []);
|
||
|
||
const drawStroke = useCallback(
|
||
(from: { x: number; y: number }, to: { x: number; y: number }) => {
|
||
const canvas = canvasRef.current;
|
||
if (!canvas) return;
|
||
const ctx = canvas.getContext("2d");
|
||
if (!ctx) return;
|
||
|
||
ctx.save();
|
||
if (brushMode === "eraser") {
|
||
ctx.globalCompositeOperation = "destination-out";
|
||
ctx.strokeStyle = "rgba(0,0,0,1)";
|
||
ctx.lineWidth = brushSize * 1.45;
|
||
} else {
|
||
ctx.globalCompositeOperation = "source-over";
|
||
ctx.strokeStyle = "#18181b";
|
||
ctx.lineWidth = brushSize;
|
||
}
|
||
ctx.lineCap = "round";
|
||
ctx.lineJoin = "round";
|
||
ctx.beginPath();
|
||
ctx.moveTo(from.x, from.y);
|
||
ctx.lineTo(to.x, to.y);
|
||
ctx.stroke();
|
||
ctx.restore();
|
||
},
|
||
[brushMode, brushSize]
|
||
);
|
||
|
||
const drawShape = useCallback(
|
||
(from: { x: number; y: number }, to: { x: number; y: number }) => {
|
||
const canvas = canvasRef.current;
|
||
if (!canvas) return;
|
||
const ctx = canvas.getContext("2d");
|
||
if (!ctx) return;
|
||
|
||
const minX = Math.min(from.x, to.x);
|
||
const minY = Math.min(from.y, to.y);
|
||
const width = Math.abs(to.x - from.x);
|
||
const height = Math.abs(to.y - from.y);
|
||
|
||
ctx.save();
|
||
ctx.globalCompositeOperation = "source-over";
|
||
ctx.strokeStyle = "#18181b";
|
||
ctx.lineWidth = Math.max(1, brushSize * 0.9);
|
||
ctx.lineCap = "round";
|
||
ctx.lineJoin = "round";
|
||
|
||
if (brushMode === "rect") {
|
||
ctx.strokeRect(minX, minY, width, height);
|
||
} else if (brushMode === "circle") {
|
||
const centerX = minX + width / 2;
|
||
const centerY = minY + height / 2;
|
||
const radiusX = width / 2;
|
||
const radiusY = height / 2;
|
||
ctx.beginPath();
|
||
ctx.ellipse(centerX, centerY, radiusX, radiusY, 0, 0, Math.PI * 2);
|
||
ctx.stroke();
|
||
}
|
||
ctx.restore();
|
||
},
|
||
[brushMode, brushSize]
|
||
);
|
||
|
||
const handlePointerDown = useCallback(
|
||
(event: ReactPointerEvent<HTMLCanvasElement>) => {
|
||
const point = getCanvasPoint(event);
|
||
if (!point) return;
|
||
pointerDownRef.current = true;
|
||
lastPointRef.current = point;
|
||
if (isShapeMode(brushMode)) {
|
||
const canvas = canvasRef.current;
|
||
const ctx = canvas?.getContext("2d", { willReadFrequently: true });
|
||
shapeStartPointRef.current = point;
|
||
shapeSnapshotRef.current =
|
||
canvas && ctx ? ctx.getImageData(0, 0, canvas.width, canvas.height) : null;
|
||
}
|
||
event.currentTarget.setPointerCapture(event.pointerId);
|
||
},
|
||
[brushMode, getCanvasPoint]
|
||
);
|
||
|
||
const handlePointerLeave = useCallback(() => {
|
||
const wasDrawing = pointerDownRef.current;
|
||
if (cursorRef.current) {
|
||
cursorRef.current.style.opacity = '0';
|
||
}
|
||
pointerDownRef.current = false;
|
||
lastPointRef.current = null;
|
||
shapeStartPointRef.current = null;
|
||
shapeSnapshotRef.current = null;
|
||
if (wasDrawing) {
|
||
scheduleFlush(true, false);
|
||
}
|
||
}, [scheduleFlush]);
|
||
|
||
const handlePointerEnter = useCallback(() => {
|
||
if (cursorRef.current) {
|
||
cursorRef.current.style.opacity = brushMode === "brush" || brushMode === "eraser" ? "1" : "0";
|
||
}
|
||
}, [brushMode]);
|
||
|
||
const handlePointerMove = useCallback(
|
||
(event: ReactPointerEvent<HTMLCanvasElement>) => {
|
||
// Update cursor
|
||
const point = getCanvasPoint(event);
|
||
if (point && cursorRef.current) {
|
||
const showCursor = brushMode === "brush" || brushMode === "eraser";
|
||
if (showCursor) {
|
||
cursorRef.current.style.opacity = "1";
|
||
cursorRef.current.style.transform = `translate(${point.x - brushSize / 2}px, ${point.y - brushSize / 2}px)`;
|
||
cursorRef.current.style.width = `${brushSize}px`;
|
||
cursorRef.current.style.height = `${brushSize}px`;
|
||
} else {
|
||
cursorRef.current.style.opacity = "0";
|
||
}
|
||
}
|
||
|
||
if (!pointerDownRef.current) return;
|
||
if (!point || !lastPointRef.current) return;
|
||
if (isShapeMode(brushMode) && shapeStartPointRef.current) {
|
||
const canvas = canvasRef.current;
|
||
const ctx = canvas?.getContext("2d");
|
||
if (ctx && shapeSnapshotRef.current) {
|
||
ctx.putImageData(shapeSnapshotRef.current, 0, 0);
|
||
drawShape(shapeStartPointRef.current, point);
|
||
}
|
||
} else {
|
||
drawStroke(lastPointRef.current, point);
|
||
}
|
||
lastPointRef.current = point;
|
||
scheduleFlush();
|
||
},
|
||
[drawShape, drawStroke, getCanvasPoint, scheduleFlush, brushMode, brushSize]
|
||
);
|
||
|
||
const handlePointerUp = useCallback(
|
||
(event: ReactPointerEvent<HTMLCanvasElement>) => {
|
||
pointerDownRef.current = false;
|
||
const endPoint = getCanvasPoint(event);
|
||
const shapeStart = shapeStartPointRef.current;
|
||
if (isShapeMode(brushMode) && endPoint && shapeStart) {
|
||
const canvas = canvasRef.current;
|
||
const ctx = canvas?.getContext("2d");
|
||
if (ctx && shapeSnapshotRef.current) {
|
||
ctx.putImageData(shapeSnapshotRef.current, 0, 0);
|
||
drawShape(shapeStart, endPoint);
|
||
}
|
||
}
|
||
lastPointRef.current = null;
|
||
shapeStartPointRef.current = null;
|
||
shapeSnapshotRef.current = null;
|
||
try {
|
||
event.currentTarget.releasePointerCapture(event.pointerId);
|
||
} catch {
|
||
// ignore
|
||
}
|
||
saveHistory(); // Save state after stroke
|
||
scheduleFlush(true, false);
|
||
},
|
||
[brushMode, drawShape, getCanvasPoint, saveHistory, scheduleFlush]
|
||
);
|
||
|
||
// Shortcuts
|
||
useEffect(() => {
|
||
if (!open) return;
|
||
const handleKeyDown = (e: KeyboardEvent) => {
|
||
// Undo/Redo
|
||
if ((e.metaKey || e.ctrlKey) && e.key === 'z') {
|
||
e.preventDefault();
|
||
if (e.shiftKey) {
|
||
redo();
|
||
} else {
|
||
undo();
|
||
}
|
||
}
|
||
// Brush Size
|
||
if (e.key === '[') {
|
||
setBrushSize(prev => Math.max(2, prev - 2));
|
||
}
|
||
if (e.key === ']') {
|
||
setBrushSize(prev => Math.min(50, prev + 2));
|
||
}
|
||
};
|
||
window.addEventListener('keydown', handleKeyDown);
|
||
return () => window.removeEventListener('keydown', handleKeyDown);
|
||
}, [open, undo, redo]);
|
||
|
||
const uploadPreviewToCanvas = useCallback(async () => {
|
||
if (!previewUrl) {
|
||
toast.error("请先完成一帧实时预览");
|
||
return;
|
||
}
|
||
|
||
setIsApplying(true);
|
||
try {
|
||
const blob = await fetch(previewUrl).then((res) => res.blob());
|
||
const formData = new FormData();
|
||
const filename = `realtime-canvas-${Date.now()}.jpg`;
|
||
formData.append("file", new File([blob], filename, { type: blob.type || "image/jpeg" }));
|
||
|
||
const response = await fetch("/api/files/upload", {
|
||
method: "POST",
|
||
body: formData,
|
||
});
|
||
const result = await response.json().catch(() => ({}));
|
||
|
||
if (!response.ok || !result?.url) {
|
||
throw new Error(result?.error || "上传到画布失败");
|
||
}
|
||
|
||
onApplyResult({
|
||
imageUrl: String(result.url),
|
||
prompt: prompt.trim() || FALLBACK_PROMPT,
|
||
});
|
||
|
||
toast.success("已发送到画布");
|
||
onOpenChange(false);
|
||
} catch (error) {
|
||
toast.error(error instanceof Error ? error.message : "发送到画布失败");
|
||
} finally {
|
||
setIsApplying(false);
|
||
}
|
||
}, [onApplyResult, onOpenChange, previewUrl, prompt]);
|
||
|
||
useEffect(() => {
|
||
if (!open) return;
|
||
initializeCanvas();
|
||
void drawInitialImage().then(() => {
|
||
scheduleFlush(true, false);
|
||
});
|
||
const onResize = () => initializeCanvas();
|
||
window.addEventListener("resize", onResize);
|
||
return () => {
|
||
window.removeEventListener("resize", onResize);
|
||
};
|
||
}, [drawInitialImage, initializeCanvas, open, scheduleFlush]);
|
||
|
||
useEffect(() => {
|
||
if (!open) return;
|
||
|
||
let cancelled = false;
|
||
setStatus("connecting");
|
||
setStatusMessage("正在初始化实时连接...");
|
||
|
||
const start = async () => {
|
||
try {
|
||
let configResponse: Response | null = null;
|
||
let configJson: Partial<RealtimeConfig> & { message?: string; retryAfter?: number } = {};
|
||
|
||
for (let attempt = 0; attempt < 3; attempt += 1) {
|
||
configResponse = await fetch("/api/realtime/config", { cache: "no-store" });
|
||
configJson = (await configResponse.json().catch(() => ({}))) as Partial<RealtimeConfig> & {
|
||
message?: string;
|
||
retryAfter?: number;
|
||
};
|
||
if (configResponse.ok) break;
|
||
|
||
const retryAfterSeconds = Number(configJson.retryAfter || 0);
|
||
const shouldRetry429 = configResponse.status === 429 && attempt < 2;
|
||
if (!shouldRetry429) {
|
||
throw new Error(configJson.message || "无法加载实时配置");
|
||
}
|
||
|
||
const retryDelay = retryAfterSeconds > 0 ? retryAfterSeconds * 1000 : 700 * (attempt + 1);
|
||
await new Promise((resolve) => window.setTimeout(resolve, retryDelay));
|
||
}
|
||
|
||
if (!configResponse?.ok) {
|
||
throw new Error(configJson.message || "无法加载实时配置");
|
||
}
|
||
|
||
if (cancelled) return;
|
||
const resolvedConfig: RealtimeConfig = {
|
||
transport: {
|
||
throttleInterval:
|
||
configJson.transport?.throttleInterval ||
|
||
defaultRealtimeConfig.transport.throttleInterval,
|
||
maxBuffering:
|
||
configJson.transport?.maxBuffering || defaultRealtimeConfig.transport.maxBuffering,
|
||
},
|
||
image: {
|
||
maxInputResolution:
|
||
configJson.image?.maxInputResolution || defaultRealtimeConfig.image.maxInputResolution,
|
||
jpegQuality: configJson.image?.jpegQuality || defaultRealtimeConfig.image.jpegQuality,
|
||
},
|
||
model: {
|
||
id: configJson.model?.id || defaultRealtimeConfig.model.id,
|
||
defaults: {
|
||
steps: configJson.model?.defaults?.steps || defaultRealtimeConfig.model.defaults.steps,
|
||
scheduleMu:
|
||
configJson.model?.defaults?.scheduleMu ||
|
||
defaultRealtimeConfig.model.defaults.scheduleMu,
|
||
feedback:
|
||
configJson.model?.defaults?.feedback ||
|
||
defaultRealtimeConfig.model.defaults.feedback,
|
||
imageSize:
|
||
configJson.model?.defaults?.imageSize ||
|
||
defaultRealtimeConfig.model.defaults.imageSize,
|
||
},
|
||
},
|
||
};
|
||
|
||
setConfig(resolvedConfig);
|
||
setAdaptiveThrottleMs(resolvedConfig.transport.throttleInterval);
|
||
|
||
fal.config({
|
||
proxyUrl: "/api/fal/proxy",
|
||
});
|
||
|
||
const connection = fal.realtime.connect(resolvedConfig.model.id, {
|
||
connectionKey: `canvas-realtime-${sessionIdRef.current}-${refreshToken}`,
|
||
throttleInterval: resolvedConfig.transport.throttleInterval,
|
||
maxBuffering: resolvedConfig.transport.maxBuffering,
|
||
onResult: (result: any) => {
|
||
const requestId = String(result?.request_id || "");
|
||
const seqFromRequest = requestSeqMapRef.current.get(requestId) || 0;
|
||
const seq = Math.max(seqFromRequest, Number(result?.seq || 0));
|
||
if (seq > 0 && seq < lastRenderedSeqRef.current) {
|
||
return;
|
||
}
|
||
|
||
const latestImage = Array.isArray(result?.images)
|
||
? result.images[result.images.length - 1]
|
||
: null;
|
||
const nextPreviewUrl = buildImageDataUrl(latestImage);
|
||
if (!nextPreviewUrl) return;
|
||
|
||
lastRenderedSeqRef.current = Math.max(lastRenderedSeqRef.current, seq);
|
||
setPreviewUrl(nextPreviewUrl);
|
||
setStatus("live");
|
||
setStatusMessage("实时渲染中");
|
||
|
||
if (requestId) {
|
||
const startedAt = requestStartMapRef.current.get(requestId);
|
||
if (startedAt) {
|
||
pushMetric({
|
||
name: "frame_result",
|
||
ts: Date.now(),
|
||
seq,
|
||
latencyMs: Date.now() - startedAt,
|
||
});
|
||
}
|
||
requestStartMapRef.current.delete(requestId);
|
||
requestSeqMapRef.current.delete(requestId);
|
||
}
|
||
},
|
||
onError: (error: any) => {
|
||
const errorMessage =
|
||
error?.body?.message || error?.message || "实时连接异常，请稍后重试";
|
||
setStatus("error");
|
||
setStatusMessage(String(errorMessage));
|
||
if (Number(error?.status) === 429) {
|
||
setAdaptiveThrottleMs((prev) => Math.min(240, prev + 40));
|
||
}
|
||
pushMetric({
|
||
name: "frame_error",
|
||
ts: Date.now(),
|
||
message: String(errorMessage),
|
||
});
|
||
},
|
||
});
|
||
|
||
if (cancelled) {
|
||
connection.close();
|
||
return;
|
||
}
|
||
|
||
connectionRef.current = connection;
|
||
setStatus("connecting");
|
||
setStatusMessage("连接成功，等待首帧...");
|
||
scheduleFlush(true, false);
|
||
} catch (error) {
|
||
console.error("[RealtimeCanvas] init failed:", error);
|
||
setStatus("error");
|
||
setStatusMessage(error instanceof Error ? error.message : "实时初始化失败");
|
||
}
|
||
};
|
||
|
||
void start();
|
||
const metricTimer = window.setInterval(() => {
|
||
void flushMetrics();
|
||
}, 5000);
|
||
|
||
return () => {
|
||
cancelled = true;
|
||
if (pendingFlushTimerRef.current) {
|
||
window.clearTimeout(pendingFlushTimerRef.current);
|
||
pendingFlushTimerRef.current = null;
|
||
}
|
||
window.clearInterval(metricTimer);
|
||
void flushMetrics();
|
||
connectionRef.current?.close?.();
|
||
connectionRef.current = null;
|
||
setStatus("idle");
|
||
setStatusMessage("等待连接");
|
||
pointerDownRef.current = false;
|
||
lastPointRef.current = null;
|
||
};
|
||
}, [flushMetrics, open, pushMetric, refreshToken, scheduleFlush]);
|
||
|
||
const forceReconnect = useCallback(() => {
|
||
connectionRef.current?.close?.();
|
||
connectionRef.current = null;
|
||
setStatus("connecting");
|
||
setStatusMessage("正在重连...");
|
||
setRefreshToken((prev) => prev + 1);
|
||
}, []);
|
||
|
||
const clearSketch = useCallback(() => {
|
||
resetCanvas();
|
||
dirtyRef.current = true;
|
||
scheduleFlush(true, false);
|
||
}, [resetCanvas, scheduleFlush]);
|
||
|
||
const handleSketchUpload = useCallback(
|
||
async (event: ChangeEvent<HTMLInputElement>) => {
|
||
const file = event.target.files?.[0];
|
||
if (!file) return;
|
||
if (!file.type.startsWith("image/")) {
|
||
toast.error("请上传图片文件");
|
||
event.currentTarget.value = "";
|
||
return;
|
||
}
|
||
|
||
try {
|
||
const objectUrl = URL.createObjectURL(file);
|
||
const image = new Image();
|
||
await new Promise<void>((resolve, reject) => {
|
||
image.onload = () => resolve();
|
||
image.onerror = () => reject(new Error("图片加载失败"));
|
||
image.src = objectUrl;
|
||
});
|
||
|
||
const canvas = canvasRef.current;
|
||
const ctx = canvas?.getContext("2d");
|
||
if (!canvas || !ctx) {
|
||
URL.revokeObjectURL(objectUrl);
|
||
throw new Error("画布未初始化");
|
||
}
|
||
|
||
const rect = canvas.getBoundingClientRect();
|
||
const canvasWidth = Math.max(1, rect.width);
|
||
const canvasHeight = Math.max(1, rect.height);
|
||
const scale = Math.min(canvasWidth / image.width, canvasHeight / image.height);
|
||
const drawWidth = image.width * scale;
|
||
const drawHeight = image.height * scale;
|
||
const offsetX = (canvasWidth - drawWidth) / 2;
|
||
const offsetY = (canvasHeight - drawHeight) / 2;
|
||
|
||
ctx.drawImage(image, offsetX, offsetY, drawWidth, drawHeight);
|
||
URL.revokeObjectURL(objectUrl);
|
||
saveHistory();
|
||
scheduleFlush(true);
|
||
toast.success("图片已导入绘画区");
|
||
} catch (error) {
|
||
toast.error(error instanceof Error ? error.message : "导入图片失败");
|
||
} finally {
|
||
event.currentTarget.value = "";
|
||
}
|
||
},
|
||
[saveHistory, scheduleFlush]
|
||
);
|
||
|
||
return (
|
||
<Dialog open={open} onOpenChange={onOpenChange}>
|
||
<DialogContent className="h-[95vh] w-[98vw] max-w-[1800px] overflow-hidden rounded-[20px] border-0 bg-background p-0 shadow-2xl sm:rounded-[20px] md:rounded-[24px]">
|
||
<DialogHeader className="sr-only">
|
||
<DialogTitle>实时画布</DialogTitle>
|
||
<DialogDescription>左侧绘制草图，右侧实时预览，并可将结果发送到无限画布。</DialogDescription>
|
||
</DialogHeader>
|
||
{/* Top Header - Minimal */}
|
||
<div className="absolute left-0 right-0 top-0 z-50 flex h-14 items-center justify-between px-6 bg-gradient-to-b from-black/20 to-transparent pointer-events-none">
|
||
<div className="pointer-events-auto flex items-center gap-3 rounded-full bg-background/80 backdrop-blur-md px-4 py-1.5 border border-border/50 shadow-sm">
|
||
<span className="font-semibold text-sm">Realtime Canvas</span>
|
||
<div className="h-4 w-[1px] bg-border" />
|
||
<span className={cn("flex items-center gap-1.5 text-xs font-medium", statusBadge.className, "bg-transparent p-0")}>
|
||
<div className={cn("h-2 w-2 rounded-full", status === "live" ? "bg-emerald-500 animate-pulse" : status === "error" ? "bg-rose-500" : "bg-zinc-400")} />
|
||
{statusBadge.label}
|
||
</span>
|
||
</div>
|
||
|
||
<div className="pointer-events-auto flex items-center gap-2">
|
||
<Button
|
||
variant="ghost"
|
||
size="icon"
|
||
className="h-8 w-8 rounded-full bg-background/50 hover:bg-background/80 backdrop-blur-md text-foreground/80 hover:text-foreground transition-colors"
|
||
onClick={() => onOpenChange(false)}
|
||
>
|
||
<X className="h-4 w-4" />
|
||
</Button>
|
||
</div>
|
||
</div>
|
||
|
||
{/* Main Canvas Area - Split View */}
|
||
<div className="relative flex h-full w-full bg-[#f4f4f5] dark:bg-[#09090b]">
|
||
{/* Left: Input Canvas */}
|
||
<div className="relative flex-1 flex flex-col items-center justify-center overflow-hidden border-r border-border/10">
|
||
<div ref={canvasWrapRef} className="relative h-full w-full flex items-center justify-center bg-white dark:bg-[#18181b]">
|
||
<div className="absolute inset-0 pattern-grid-lg opacity-[0.03] pointer-events-none" />
|
||
|
||
<div className="relative">
|
||
<canvas
|
||
ref={canvasRef}
|
||
className={cn(
|
||
"touch-none shadow-sm block",
|
||
brushMode === "brush" || brushMode === "eraser" ? "cursor-none" : "cursor-crosshair"
|
||
)}
|
||
onPointerDown={handlePointerDown}
|
||
onPointerMove={handlePointerMove}
|
||
onPointerUp={handlePointerUp}
|
||
onPointerCancel={handlePointerUp}
|
||
onPointerEnter={handlePointerEnter}
|
||
onPointerLeave={handlePointerLeave}
|
||
/>
|
||
{/* Custom Brush Cursor */}
|
||
<div
|
||
ref={cursorRef}
|
||
className="pointer-events-none absolute top-0 left-0 rounded-full border border-zinc-500/50 bg-zinc-500/10 backdrop-invert transition-opacity duration-75 z-10"
|
||
style={{
|
||
width: brushSize,
|
||
height: brushSize,
|
||
opacity: 0,
|
||
willChange: 'transform, width, height'
|
||
}}
|
||
/>
|
||
</div>
|
||
</div>
|
||
<div className="absolute top-4 left-4 pointer-events-none z-10">
|
||
<span className="text-xs font-medium text-zinc-400 bg-black/5 px-2 py-1 rounded-md backdrop-blur-sm">INPUT</span>
|
||
</div>
|
||
</div>
|
||
|
||
{/* Right: Output Preview */}
|
||
<div className="relative flex-1 flex flex-col items-center justify-center overflow-hidden bg-zinc-50 dark:bg-[#0c0c0e]">
|
||
<div className="relative h-full w-full flex items-center justify-center">
|
||
{previewUrl ? (
|
||
<img src={previewUrl} alt="实时预览" className="max-h-full max-w-full object-contain shadow-sm" />
|
||
) : (
|
||
<div className="flex flex-col items-center justify-center text-zinc-400 gap-3">
|
||
{status === "connecting" ? (
|
||
<>
|
||
<Loader2 className="h-8 w-8 animate-spin opacity-50" />
|
||
<span className="text-sm">Connecting...</span>
|
||
</>
|
||
) : (
|
||
<>
|
||
<Sparkles className="h-8 w-8 opacity-20" />
|
||
<span className="text-sm">Start drawing to generate</span>
|
||
</>
|
||
)}
|
||
</div>
|
||
)}
|
||
</div>
|
||
<div className="absolute top-4 right-4 pointer-events-none">
|
||
<span className="text-xs font-medium text-zinc-400 bg-black/5 px-2 py-1 rounded-md backdrop-blur-sm">OUTPUT</span>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
{/* Floating Toolbar - Left (Tools) */}
|
||
<div className="absolute left-6 top-1/2 -translate-y-1/2 flex flex-col gap-4 z-20 pointer-events-none">
|
||
<div className="pointer-events-auto flex flex-col gap-2 p-2 bg-background/90 backdrop-blur-xl border border-border/50 shadow-xl rounded-2xl">
|
||
<Button
|
||
size="icon"
|
||
variant="ghost"
|
||
className="h-10 w-10 rounded-xl hover:bg-zinc-100 dark:hover:bg-zinc-800"
|
||
onClick={undo}
|
||
disabled={!canUndo}
|
||
title="Undo (Cmd+Z)"
|
||
>
|
||
<Undo2 className="h-5 w-5 opacity-70" />
|
||
</Button>
|
||
<Button
|
||
size="icon"
|
||
variant="ghost"
|
||
className="h-10 w-10 rounded-xl hover:bg-zinc-100 dark:hover:bg-zinc-800"
|
||
onClick={redo}
|
||
disabled={!canRedo}
|
||
title="Redo (Cmd+Shift+Z)"
|
||
>
|
||
<Redo2 className="h-5 w-5 opacity-70" />
|
||
</Button>
|
||
|
||
<div className="h-[1px] bg-border/50 mx-2 my-1" />
|
||
|
||
<Button
|
||
size="icon"
|
||
variant={brushMode === "brush" ? "secondary" : "ghost"}
|
||
className={cn("h-10 w-10 rounded-xl transition-all", brushMode === "brush" && "bg-primary text-primary-foreground shadow-sm")}
|
||
onClick={() => setBrushMode("brush")}
|
||
title="Brush"
|
||
>
|
||
<Paintbrush className="h-5 w-5" />
|
||
</Button>
|
||
<Button
|
||
size="icon"
|
||
variant={brushMode === "eraser" ? "secondary" : "ghost"}
|
||
className={cn("h-10 w-10 rounded-xl transition-all", brushMode === "eraser" && "bg-primary text-primary-foreground shadow-sm")}
|
||
onClick={() => setBrushMode("eraser")}
|
||
title="Eraser"
|
||
>
|
||
<Eraser className="h-5 w-5" />
|
||
</Button>
|
||
<Button
|
||
size="icon"
|
||
variant={brushMode === "rect" ? "secondary" : "ghost"}
|
||
className={cn("h-10 w-10 rounded-xl transition-all", brushMode === "rect" && "bg-primary text-primary-foreground shadow-sm")}
|
||
onClick={() => setBrushMode("rect")}
|
||
title="Rectangle"
|
||
>
|
||
<Square className="h-5 w-5" />
|
||
</Button>
|
||
<Button
|
||
size="icon"
|
||
variant={brushMode === "circle" ? "secondary" : "ghost"}
|
||
className={cn("h-10 w-10 rounded-xl transition-all", brushMode === "circle" && "bg-primary text-primary-foreground shadow-sm")}
|
||
onClick={() => setBrushMode("circle")}
|
||
title="Circle"
|
||
>
|
||
<CircleIcon className="h-5 w-5" />
|
||
</Button>
|
||
<div className="h-[1px] bg-border/50 mx-2 my-1" />
|
||
|
||
<div className="flex flex-col items-center gap-2 py-2">
|
||
<span className="text-[10px] font-medium text-muted-foreground">{brushSize}px</span>
|
||
<Slider
|
||
orientation="vertical"
|
||
value={[brushSize]}
|
||
min={2}
|
||
max={50}
|
||
step={1}
|
||
onValueChange={([val]) => setBrushSize(val)}
|
||
className="h-24 w-4"
|
||
/>
|
||
</div>
|
||
|
||
<div className="h-[1px] bg-border/50 mx-2 my-1" />
|
||
<Button
|
||
size="icon"
|
||
variant="ghost"
|
||
className="h-10 w-10 rounded-xl hover:bg-destructive/10 hover:text-destructive"
|
||
onClick={clearSketch}
|
||
title="Clear"
|
||
>
|
||
<Trash2 className="h-5 w-5" />
|
||
</Button>
|
||
<Button
|
||
size="icon"
|
||
variant="ghost"
|
||
className="h-10 w-10 rounded-xl hover:bg-zinc-100 dark:hover:bg-zinc-800"
|
||
onClick={() => uploadInputRef.current?.click()}
|
||
title="Upload Image"
|
||
>
|
||
<Upload className="h-5 w-5" />
|
||
</Button>
|
||
<input
|
||
ref={uploadInputRef}
|
||
type="file"
|
||
accept="image/*"
|
||
className="hidden"
|
||
onChange={handleSketchUpload}
|
||
/>
|
||
</div>
|
||
</div>
|
||
|
||
{/* Floating Toolbar - Bottom (Prompt) */}
|
||
<div className="absolute bottom-8 left-1/2 -translate-x-1/2 w-full max-w-2xl px-4 z-20 pointer-events-none">
|
||
<div className="pointer-events-auto flex flex-col gap-2 p-1.5 bg-background/80 backdrop-blur-xl border border-border/50 shadow-2xl rounded-[24px]">
|
||
<div className="flex items-center gap-2 pl-2 pr-1.5 py-1">
|
||
<div className="flex h-8 w-8 items-center justify-center rounded-full bg-primary/10 text-primary">
|
||
<Sparkles className="h-4 w-4" />
|
||
</div>
|
||
<Input
|
||
value={prompt}
|
||
onChange={(e) => setPrompt(e.target.value)}
|
||
onBlur={() => scheduleFlush(true, false)}
|
||
placeholder="Describe what you want to create..."
|
||
className="flex-1 h-10 border-0 bg-transparent focus-visible:ring-0 px-2 text-base shadow-none"
|
||
/>
|
||
<Button
|
||
size="sm"
|
||
className="h-9 rounded-xl px-4 font-medium"
|
||
onClick={() => scheduleFlush(true)}
|
||
disabled={status === 'connecting'}
|
||
>
|
||
{status === 'connecting' ? <Loader2 className="h-4 w-4 animate-spin" /> : 'Generate'}
|
||
</Button>
|
||
</div>
|
||
</div>
|
||
<div className="text-center mt-2">
|
||
<span className="text-[10px] text-white/50 bg-black/40 px-3 py-1 rounded-full backdrop-blur-sm">
|
||
{statusMessage} • Latency: {adaptiveThrottleMs}ms
|
||
</span>
|
||
</div>
|
||
</div>
|
||
|
||
{/* Floating Toolbar - Right (Actions) */}
|
||
<div className="absolute right-6 top-6 flex flex-col gap-3 z-20 pointer-events-none">
|
||
<div className="pointer-events-auto flex flex-col gap-2">
|
||
<Button
|
||
className="h-10 px-4 rounded-full shadow-lg bg-primary text-primary-foreground hover:bg-primary/90"
|
||
onClick={uploadPreviewToCanvas}
|
||
disabled={!previewUrl || isApplying}
|
||
>
|
||
{isApplying ? <Loader2 className="mr-2 h-4 w-4 animate-spin" /> : <Send className="mr-2 h-4 w-4" />}
|
||
Send to Canvas
|
||
</Button>
|
||
|
||
<Button
|
||
variant="secondary"
|
||
className="h-10 px-4 rounded-full shadow-md bg-background/80 backdrop-blur-md hover:bg-background"
|
||
onClick={forceReconnect}
|
||
>
|
||
<RefreshCw className="mr-2 h-4 w-4" />
|
||
Reconnect
|
||
</Button>
|
||
</div>
|
||
</div>
|
||
</DialogContent>
|
||
</Dialog>
|
||
);
|
||
}
|