@@ -7092,6 +7202,192 @@ export function AdminSettingsClient() {
+ {/* 实时画布API配置 */}
+
+
+
+
+ 实时画布配置
+
+
+ 配置实时画布（左画右预览）功能使用的 fal realtime 参数与 API Key。该 Key 存储在数据库，前端不暴露。
+
+
+
+
+
+
+
+ 关闭后前端“实时画布”入口仍可显示，但会提示服务未启用。
+
+
+
+ setRealtimeCanvasConfig({ ...realtimeCanvasConfig, enabled: checked })
+ }
+ />
+
+
+
+ setRealtimeCanvasConfig({ ...realtimeCanvasConfig, falApiKey: e.target.value })
+ }
+ placeholder="fal_xxxxxxxxxxxxx"
+ disabled={!realtimeCanvasConfig.enabled}
+ />
+
+
+
+
+
+
+
+
+
+
+
+
+ setRealtimeCanvasConfig({
+ ...realtimeCanvasConfig,
+ scheduleMu: Number.parseFloat(e.target.value) || 2.3,
+ })
+ }
+ />
+
+
+
+
+
+
+
{/* 详情替换配置 */}
diff --git a/app/api/fal/proxy/route.ts b/app/api/fal/proxy/route.ts
new file mode 100644
index 000000000..d2151ad94
--- /dev/null
+++ b/app/api/fal/proxy/route.ts
@@ -0,0 +1,116 @@
+import { auth } from "@/app/(auth)/auth";
+import { getSystemSetting } from "@/lib/db/queries";
+
+const ALLOWED_FAL_HOSTS = new Set(["rest.alpha.fal.ai", "rest.fal.ai"]);
+
+type FalRealtimeConfig = {
+ enabled?: boolean;
+ falApiKey?: string;
+};
+
+async function getRealtimeCanvasConfig() {
+ const realtimeSetting = await getSystemSetting({ key: "realtime_canvas_config" }).catch(
+ () => null
+ );
+ return (realtimeSetting?.value || {}) as FalRealtimeConfig;
+}
+
+async function handleProxy(request: Request) {
+ const session = await auth();
+ if (!session?.user?.id) {
+ return Response.json({ code: "UNAUTHORIZED", message: "请先登录" }, { status: 401 });
+ }
+
+ const targetUrl = request.headers.get("x-fal-target-url");
+ if (!targetUrl) {
+ return Response.json(
+ { code: "INVALID_TARGET", message: "缺少 x-fal-target-url 请求头" },
+ { status: 400 }
+ );
+ }
+
+ let parsedUrl: URL;
+ try {
+ parsedUrl = new URL(targetUrl);
+ } catch {
+ return Response.json(
+ { code: "INVALID_TARGET", message: "x-fal-target-url 不是合法 URL" },
+ { status: 400 }
+ );
+ }
+
+ if (!ALLOWED_FAL_HOSTS.has(parsedUrl.host)) {
+ return Response.json(
+ { code: "TARGET_NOT_ALLOWED", message: "不允许代理该目标地址" },
+ { status: 403 }
+ );
+ }
+
+ const realtimeConfig = await getRealtimeCanvasConfig();
+ if (realtimeConfig.enabled === false) {
+ return Response.json(
+ {
+ code: "FEATURE_DISABLED",
+ message: "实时画布服务未启用（请在本地AI设置中开启）",
+ },
+ { status: 403 }
+ );
+ }
+
+ const falApiKey = realtimeConfig.falApiKey?.trim() || "";
+ if (!falApiKey) {
+ return Response.json(
+ {
+ code: "FAL_KEY_MISSING",
+ message: "未配置 FAL API Key（请在本地AI设置的实时画布配置中填写）",
+ },
+ { status: 500 }
+ );
+ }
+
+ let body: ArrayBuffer | undefined;
+ if (request.method !== "GET" && request.method !== "HEAD") {
+ body = await request.arrayBuffer();
+ }
+
+ const headers = new Headers();
+ headers.set("Authorization", `Key ${falApiKey}`);
+ headers.set("Accept", request.headers.get("accept") || "application/json");
+ const contentType = request.headers.get("content-type");
+ if (contentType) {
+ headers.set("Content-Type", contentType);
+ }
+
+ const upstreamResponse = await fetch(parsedUrl.toString(), {
+ method: request.method,
+ headers,
+ body,
+ cache: "no-store",
+ });
+
+ return new Response(upstreamResponse.body, {
+ status: upstreamResponse.status,
+ statusText: upstreamResponse.statusText,
+ headers: upstreamResponse.headers,
+ });
+}
+
+export async function GET(request: Request) {
+ return handleProxy(request);
+}
+
+export async function POST(request: Request) {
+ return handleProxy(request);
+}
+
+export async function PUT(request: Request) {
+ return handleProxy(request);
+}
+
+export async function PATCH(request: Request) {
+ return handleProxy(request);
+}
+
+export async function DELETE(request: Request) {
+ return handleProxy(request);
+}
diff --git a/app/api/realtime/config/route.ts b/app/api/realtime/config/route.ts
new file mode 100644
index 000000000..c61127051
--- /dev/null
+++ b/app/api/realtime/config/route.ts
@@ -0,0 +1,133 @@
+import { auth } from "@/app/(auth)/auth";
+import { getSystemSetting } from "@/lib/db/queries";
+
+type RealtimeCanvasConfig = {
+ enabled?: boolean;
+ falApiKey?: string;
+ throttleInterval?: number;
+ maxBuffering?: number;
+ maxInputResolution?: number;
+ jpegQuality?: number;
+ defaults?: {
+ steps?: number;
+ scheduleMu?: number;
+ feedback?: number;
+ imageSize?: "square" | "square_hd";
+ };
+};
+
+const defaultConfig = {
+ session: {
+ idleTimeoutMs: 120000,
+ },
+ transport: {
+ throttleInterval: 100,
+ maxBuffering: 2,
+ },
+ image: {
+ maxInputResolution: 1024,
+ jpegQuality: 0.55,
+ },
+ model: {
+ id: "fal-ai/flux-2/klein/realtime",
+ defaults: {
+ steps: 3,
+ scheduleMu: 2.3,
+ feedback: 0.95,
+ imageSize: "square" as const,
+ },
+ limits: {
+ steps: { min: 1, max: 6 },
+ feedback: { min: 0.7, max: 1.0 },
+ },
+ },
+ features: {
+ history: true,
+ abCompare: false,
+ },
+};
+
+function clamp(value: number, min: number, max: number) {
+ return Math.min(max, Math.max(min, value));
+}
+
+export async function GET() {
+ const session = await auth();
+ if (!session?.user?.id) {
+ return Response.json({ code: "UNAUTHORIZED", message: "请先登录" }, { status: 401 });
+ }
+
+ const setting = await getSystemSetting({ key: "realtime_canvas_config" }).catch(() => null);
+ const config = (setting?.value || {}) as RealtimeCanvasConfig;
+
+ if (config.enabled === false) {
+ return Response.json(
+ { code: "FEATURE_DISABLED", message: "实时画布功能未开启" },
+ { status: 403 }
+ );
+ }
+
+ if (!String(config.falApiKey || "").trim()) {
+ return Response.json(
+ {
+ code: "FAL_KEY_MISSING",
+ message: "未配置实时画布 API Key（请在本地AI设置中配置）",
+ },
+ { status: 500 }
+ );
+ }
+
+ const throttleInterval = clamp(
+ Number(config.throttleInterval || defaultConfig.transport.throttleInterval),
+ 60,
+ 260
+ );
+ const maxBuffering = clamp(
+ Number(config.maxBuffering || defaultConfig.transport.maxBuffering),
+ 1,
+ 6
+ );
+ const maxInputResolution = clamp(
+ Number(config.maxInputResolution || defaultConfig.image.maxInputResolution),
+ 512,
+ 1536
+ );
+ const jpegQuality = clamp(
+ Number(config.jpegQuality || defaultConfig.image.jpegQuality),
+ 0.35,
+ 0.82
+ );
+ const steps = clamp(Number(config.defaults?.steps || defaultConfig.model.defaults.steps), 1, 6);
+ const scheduleMu = clamp(
+ Number(config.defaults?.scheduleMu || defaultConfig.model.defaults.scheduleMu),
+ 1.5,
+ 3.5
+ );
+ const feedback = clamp(
+ Number(config.defaults?.feedback || defaultConfig.model.defaults.feedback),
+ 0.7,
+ 1.0
+ );
+ const imageSize = config.defaults?.imageSize === "square_hd" ? "square_hd" : "square";
+
+ return Response.json({
+ ...defaultConfig,
+ transport: {
+ throttleInterval,
+ maxBuffering,
+ },
+ image: {
+ maxInputResolution,
+ jpegQuality,
+ },
+ model: {
+ ...defaultConfig.model,
+ defaults: {
+ steps,
+ scheduleMu,
+ feedback,
+ imageSize,
+ },
+ },
+ });
+}
diff --git a/app/api/realtime/metrics/route.ts b/app/api/realtime/metrics/route.ts
new file mode 100644
index 000000000..3f779ebf4
--- /dev/null
+++ b/app/api/realtime/metrics/route.ts
@@ -0,0 +1,54 @@
+import { auth } from "@/app/(auth)/auth";
+
+type MetricEvent = {
+ name: string;
+ ts: number;
+ seq?: number;
+ latencyMs?: number;
+ queueDepth?: number;
+ networkType?: string;
+ deviceClass?: string;
+};
+
+type MetricsPayload = {
+ sessionId: string;
+ events: MetricEvent[];
+};
+
+export async function POST(request: Request) {
+ const session = await auth();
+ if (!session?.user?.id) {
+ return Response.json({ code: "UNAUTHORIZED", message: "请先登录" }, { status: 401 });
+ }
+
+ let payload: MetricsPayload;
+ try {
+ payload = (await request.json()) as MetricsPayload;
+ } catch {
+ return Response.json({ code: "BAD_REQUEST", message: "无效 JSON" }, { status: 400 });
+ }
+
+ if (!payload?.sessionId || !Array.isArray(payload.events)) {
+ return Response.json({ code: "BAD_REQUEST", message: "缺少 sessionId 或 events" }, { status: 400 });
+ }
+
+ if (payload.events.length > 200) {
+ return Response.json(
+ { code: "PAYLOAD_TOO_LARGE", message: "events 超过限制（200）" },
+ { status: 413 }
+ );
+ }
+
+ if (payload.events.length > 0) {
+ const latest = payload.events[payload.events.length - 1];
+ console.log("[RealtimeMetrics]", {
+ userId: session.user.id,
+ sessionId: payload.sessionId,
+ count: payload.events.length,
+ latest,
+ });
+ }
+
+ return Response.json({ ok: true });
+}
+
diff --git a/app/api/site-settings/route.ts b/app/api/site-settings/route.ts
index 7cb3f7c11..800259b87 100644
--- a/app/api/site-settings/route.ts
+++ b/app/api/site-settings/route.ts
@@ -283,6 +283,7 @@ export async function GET() {
tutorialEnabled: false,
rankingEnabled: true,
canvasEnabled: true,
+ realtimeCanvasEnabled: true,
canvasDefaultModelId: null,
canvasAutoModeDefaultModelId: null,
canvasDetailDefaultModelId: null,
@@ -364,6 +365,7 @@ const updateSchema = z.object({
tutorialEnabled: z.boolean().optional(),
rankingEnabled: z.boolean().optional(),
canvasEnabled: z.boolean().optional(),
+ realtimeCanvasEnabled: z.boolean().optional(),
aiDetailGenerationEnabled: z.boolean().optional(),
canvasDefaultModelId: z.string().nullable().optional(),
canvasAutoModeDefaultModelId: z.string().nullable().optional(),
@@ -754,6 +756,7 @@ export async function PUT(request: NextRequest) {
tutorialEnabled: siteSettingsPayload.tutorialEnabled ?? false,
rankingEnabled: siteSettingsPayload.rankingEnabled ?? true,
canvasEnabled: siteSettingsPayload.canvasEnabled ?? true,
+ realtimeCanvasEnabled: siteSettingsPayload.realtimeCanvasEnabled ?? true,
canvasDefaultModelId: siteSettingsPayload.canvasDefaultModelId,
canvasAutoModeDefaultModelId: siteSettingsPayload.canvasAutoModeDefaultModelId,
canvasDetailDefaultModelId: siteSettingsPayload.canvasDetailDefaultModelId,
diff --git a/app/canvas/[id]/page.tsx b/app/canvas/[id]/page.tsx
index 549cffead..aff4c905a 100644
--- a/app/canvas/[id]/page.tsx
+++ b/app/canvas/[id]/page.tsx
@@ -23,6 +23,7 @@ import {
Wand2,
Grid3X3,
Settings,
+ PenTool,
} from "lucide-react";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
@@ -58,6 +59,7 @@ import { CanvasWorkspace } from "@/components/canvas";
import { CanvasChatPanel } from "@/components/canvas/CanvasChatPanel";
import { CanvasFileListPanel } from "@/components/canvas/CanvasFileListPanel";
import { ObjectPickerOverlay } from "@/components/canvas/ObjectPickerOverlay";
+import { RealtimeCanvasDialog } from "@/components/canvas/RealtimeCanvasDialog";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { cn, fetcher } from "@/lib/utils";
import { MessageSquare, FolderOpen, Grid2X2 } from "lucide-react";
@@ -121,11 +123,13 @@ export default function CanvasEditorPage({ params }: PageProps) {
const [isPanelOpen, setIsPanelOpen] = useState(true);
// 面板模式: 'chat' 聊天面板, 'fileList' 文件列表
const [panelMode, setPanelMode] = useState<'chat' | 'fileList'>('chat');
+ const [isRealtimeCanvasOpen, setIsRealtimeCanvasOpen] = useState(false);
const { data: siteSettingsData } = useSWR<{
siteName?: string;
siteLogo?: string | null;
siteLogoDark?: string | null;
+ realtimeCanvasEnabled?: boolean;
}>("/api/site-settings", fetcher);
// 生成配置
@@ -353,6 +357,38 @@ export default function CanvasEditorPage({ params }: PageProps) {
});
};
+ const handleApplyRealtimeResult = useCallback(
+ ({ imageUrl, prompt: imagePrompt }: { imageUrl: string; prompt: string }) => {
+ const viewportX = viewport.x || 0;
+ const viewportY = viewport.y || 0;
+ const viewportZoom = viewport.zoom || 1;
+ const stageWidth = typeof window !== "undefined" ? window.innerWidth : 1280;
+ const stageHeight = typeof window !== "undefined" ? window.innerHeight : 720;
+ const targetWidth = 360;
+ const targetHeight = 360;
+
+ const x = (stageWidth * 0.45 - viewportX) / viewportZoom - targetWidth / 2;
+ const y = (stageHeight * 0.45 - viewportY) / viewportZoom - targetHeight / 2;
+
+ addNode({
+ type: "image",
+ position: { x, y },
+ size: { width: targetWidth, height: targetHeight },
+ data: {
+ src: imageUrl,
+ status: "completed",
+ title: "实时画布结果",
+ prompt: imagePrompt,
+ model: "fal-ai/flux-2/klein/realtime",
+ },
+ connections: [],
+ });
+
+ toast.success("实时画布结果已添加");
+ },
+ [addNode, viewport.x, viewport.y, viewport.zoom]
+ );
+
// 处理图片操作（通过事件分发到选中图片节点执行）
const handleAction = (action: string) => {
if (!selectedImageNode) {
@@ -717,6 +753,22 @@ export default function CanvasEditorPage({ params }: PageProps) {
添加图片
+ {(siteSettingsData?.realtimeCanvasEnabled ?? true) ? (
+
+
+
+
+ 实时画布
+
+ ) : null}
+
@@ -756,6 +808,13 @@ export default function CanvasEditorPage({ params }: PageProps) {
{/* 6. 物品拾取模式覆盖层 */}
+
+
);
diff --git a/components/canvas/RealtimeCanvasDialog.tsx b/components/canvas/RealtimeCanvasDialog.tsx
new file mode 100644
index 000000000..cf45c9433
--- /dev/null
+++ b/components/canvas/RealtimeCanvasDialog.tsx
@@ -0,0 +1,754 @@
+"use client";
+
+import { useCallback, useEffect, useMemo, useRef, useState, type PointerEvent as ReactPointerEvent } from "react";
+import * as fal from "@fal-ai/serverless-client";
+import {
+ Dialog,
+ DialogContent,
+ DialogDescription,
+ DialogHeader,
+ DialogTitle,
+} from "@/components/ui/dialog";
+import { Button } from "@/components/ui/button";
+import { Input } from "@/components/ui/input";
+import { Label } from "@/components/ui/label";
+import { cn } from "@/lib/utils";
+import {
+ Eraser,
+ Loader2,
+ Paintbrush,
+ RefreshCw,
+ Send,
+ Sparkles,
+ Trash2,
+ Wifi,
+ WifiOff,
+} from "lucide-react";
+import { toast } from "sonner";
+
+type RealtimeConfig = {
+ transport: {
+ throttleInterval: number;
+ maxBuffering: number;
+ };
+ image: {
+ maxInputResolution: number;
+ jpegQuality: number;
+ };
+ model: {
+ id: string;
+ defaults: {
+ steps: number;
+ scheduleMu: number;
+ feedback: number;
+ imageSize: "square" | "square_hd";
+ };
+ };
+};
+
+type RealtimeStatus = "idle" | "connecting" | "live" | "error";
+
+type RealtimeCanvasDialogProps = {
+ open: boolean;
+ onOpenChange: (open: boolean) => void;
+ initialImageUrl?: string | null;
+ onApplyResult: (payload: { imageUrl: string; prompt: string }) => void;
+};
+
+const FALLBACK_PROMPT = "将草图渲染为高质感产品设计图，保持结构和主轮廓一致。";
+
+const defaultRealtimeConfig: RealtimeConfig = {
+ transport: {
+ throttleInterval: 100,
+ maxBuffering: 2,
+ },
+ image: {
+ maxInputResolution: 1024,
+ jpegQuality: 0.55,
+ },
+ model: {
+ id: "fal-ai/flux-2/klein/realtime",
+ defaults: {
+ steps: 3,
+ scheduleMu: 2.3,
+ feedback: 0.95,
+ imageSize: "square",
+ },
+ },
+};
+
+function buildImageDataUrl(image: any) {
+ if (!image) return "";
+ if (typeof image.url === "string" && image.url.trim()) {
+ return image.url;
+ }
+ if (typeof image.content === "string" && image.content.trim()) {
+ if (image.content.startsWith("data:")) {
+ return image.content;
+ }
+ const contentType = image.content_type || "image/jpeg";
+ return `data:${contentType};base64,${image.content}`;
+ }
+ return "";
+}
+
+async function canvasToJpegDataUri(
+ canvas: HTMLCanvasElement,
+ maxInputResolution: number,
+ jpegQuality: number
+) {
+ const sourceWidth = canvas.width;
+ const sourceHeight = canvas.height;
+ const longestSide = Math.max(sourceWidth, sourceHeight);
+ const scale = longestSide > maxInputResolution ? maxInputResolution / longestSide : 1;
+ const exportWidth = Math.max(1, Math.round(sourceWidth * scale));
+ const exportHeight = Math.max(1, Math.round(sourceHeight * scale));
+
+ const exportCanvas = document.createElement("canvas");
+ exportCanvas.width = exportWidth;
+ exportCanvas.height = exportHeight;
+ const exportCtx = exportCanvas.getContext("2d");
+ if (!exportCtx) {
+ throw new Error("无法初始化导出画布");
+ }
+
+ exportCtx.fillStyle = "#d8d8db";
+ exportCtx.fillRect(0, 0, exportWidth, exportHeight);
+ exportCtx.drawImage(canvas, 0, 0, exportWidth, exportHeight);
+
+ const blob = await new Promise
((resolve) => {
+ exportCanvas.toBlob(resolve, "image/jpeg", jpegQuality);
+ });
+ if (!blob) {
+ throw new Error("导出图片失败");
+ }
+
+ return await new Promise((resolve, reject) => {
+ const reader = new FileReader();
+ reader.onload = () => resolve(String(reader.result || ""));
+ reader.onerror = () => reject(new Error("读取导出图片失败"));
+ reader.readAsDataURL(blob);
+ });
+}
+
+export function RealtimeCanvasDialog({
+ open,
+ onOpenChange,
+ initialImageUrl,
+ onApplyResult,
+}: RealtimeCanvasDialogProps) {
+ const canvasRef = useRef(null);
+ const canvasWrapRef = useRef(null);
+ const connectionRef = useRef(null);
+ const pointerDownRef = useRef(false);
+ const lastPointRef = useRef<{ x: number; y: number } | null>(null);
+ const pendingFlushTimerRef = useRef(null);
+ const flushInFlightRef = useRef(false);
+ const dirtyRef = useRef(false);
+ const seqRef = useRef(0);
+ const requestSeqMapRef = useRef>(new Map());
+ const requestStartMapRef = useRef>(new Map());
+ const lastRenderedSeqRef = useRef(0);
+ const metricsQueueRef = useRef([]);
+ const sessionIdRef = useRef(`rt-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`);
+ const hasLoadedInitialImageRef = useRef(false);
+ const [config, setConfig] = useState(defaultRealtimeConfig);
+ const [status, setStatus] = useState("idle");
+ const [statusMessage, setStatusMessage] = useState("等待连接");
+ const [prompt, setPrompt] = useState(FALLBACK_PROMPT);
+ const [previewUrl, setPreviewUrl] = useState("");
+ const [isApplying, setIsApplying] = useState(false);
+ const [brushMode, setBrushMode] = useState<"brush" | "eraser">("brush");
+ const [brushSize, setBrushSize] = useState(6);
+ const [adaptiveThrottleMs, setAdaptiveThrottleMs] = useState(
+ defaultRealtimeConfig.transport.throttleInterval
+ );
+ const [refreshToken, setRefreshToken] = useState(0);
+
+ const statusBadge = useMemo(() => {
+ if (status === "live") return { label: "实时中", className: "bg-emerald-50 text-emerald-700" };
+ if (status === "connecting") {
+ return { label: "连接中", className: "bg-blue-50 text-blue-700" };
+ }
+ if (status === "error") return { label: "异常", className: "bg-rose-50 text-rose-700" };
+ return { label: "待启动", className: "bg-zinc-100 text-zinc-600" };
+ }, [status]);
+
+ const pushMetric = useCallback((event: any) => {
+ metricsQueueRef.current.push(event);
+ if (metricsQueueRef.current.length > 120) {
+ metricsQueueRef.current = metricsQueueRef.current.slice(-80);
+ }
+ }, []);
+
+ const flushMetrics = useCallback(async () => {
+ if (!open || metricsQueueRef.current.length === 0) return;
+ const events = metricsQueueRef.current.splice(0, metricsQueueRef.current.length);
+ try {
+ await fetch("/api/realtime/metrics", {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({
+ sessionId: sessionIdRef.current,
+ events,
+ }),
+ });
+ } catch {
+ metricsQueueRef.current.unshift(...events.slice(-20));
+ }
+ }, [open]);
+
+ const resetCanvas = useCallback(() => {
+ const canvas = canvasRef.current;
+ if (!canvas) return;
+ const ctx = canvas.getContext("2d");
+ if (!ctx) return;
+ ctx.save();
+ ctx.globalCompositeOperation = "source-over";
+ ctx.fillStyle = "#d8d8db";
+ ctx.fillRect(0, 0, canvas.width, canvas.height);
+ ctx.restore();
+ }, []);
+
+ const drawInitialImage = useCallback(async () => {
+ if (!open || !initialImageUrl || hasLoadedInitialImageRef.current) return;
+ const canvas = canvasRef.current;
+ if (!canvas) return;
+ const ctx = canvas.getContext("2d");
+ if (!ctx) return;
+
+ const image = new Image();
+ image.crossOrigin = "anonymous";
+ await new Promise((resolve, reject) => {
+ image.onload = () => resolve();
+ image.onerror = () => reject(new Error("加载初始图片失败"));
+ image.src = initialImageUrl;
+ });
+
+ const scale = Math.min(canvas.width / image.width, canvas.height / image.height);
+ const drawWidth = image.width * scale;
+ const drawHeight = image.height * scale;
+ const offsetX = (canvas.width - drawWidth) / 2;
+ const offsetY = (canvas.height - drawHeight) / 2;
+
+ ctx.drawImage(image, offsetX, offsetY, drawWidth, drawHeight);
+ hasLoadedInitialImageRef.current = true;
+ dirtyRef.current = true;
+ }, [initialImageUrl, open]);
+
+ const flushFrame = useCallback(
+ async (force = false) => {
+ if (!open) return;
+ if (!connectionRef.current) return;
+ if (flushInFlightRef.current) return;
+ if (!dirtyRef.current && !force) return;
+
+ const canvas = canvasRef.current;
+ if (!canvas) return;
+
+ flushInFlightRef.current = true;
+ const seq = seqRef.current + 1;
+ seqRef.current = seq;
+ const requestId = `${sessionIdRef.current}-${seq}`;
+ requestSeqMapRef.current.set(requestId, seq);
+ requestStartMapRef.current.set(requestId, Date.now());
+
+ try {
+ const imageDataUri = await canvasToJpegDataUri(
+ canvas,
+ config.image.maxInputResolution,
+ config.image.jpegQuality
+ );
+
+ dirtyRef.current = false;
+ setStatus("connecting");
+ setStatusMessage("正在请求实时生成...");
+ connectionRef.current.send({
+ request_id: requestId,
+ image_url: imageDataUri,
+ prompt: prompt.trim() || FALLBACK_PROMPT,
+ image_size: config.model.defaults.imageSize,
+ num_inference_steps: config.model.defaults.steps,
+ schedule_mu: config.model.defaults.scheduleMu,
+ output_feedback_strength: config.model.defaults.feedback,
+ });
+
+ pushMetric({
+ name: "frame_send",
+ ts: Date.now(),
+ seq,
+ });
+ } catch (error) {
+ console.error("[RealtimeCanvas] flush frame failed:", error);
+ setStatus("error");
+ setStatusMessage(error instanceof Error ? error.message : "发送失败");
+ } finally {
+ flushInFlightRef.current = false;
+ }
+ },
+ [config, open, prompt, pushMetric]
+ );
+
+ const scheduleFlush = useCallback(
+ (force = false) => {
+ dirtyRef.current = true;
+ if (force) {
+ if (pendingFlushTimerRef.current) {
+ window.clearTimeout(pendingFlushTimerRef.current);
+ pendingFlushTimerRef.current = null;
+ }
+ void flushFrame(true);
+ return;
+ }
+ if (pendingFlushTimerRef.current) return;
+ pendingFlushTimerRef.current = window.setTimeout(() => {
+ pendingFlushTimerRef.current = null;
+ void flushFrame();
+ }, adaptiveThrottleMs);
+ },
+ [adaptiveThrottleMs, flushFrame]
+ );
+
+ const initializeCanvas = useCallback(() => {
+ const canvas = canvasRef.current;
+ const wrap = canvasWrapRef.current;
+ if (!canvas || !wrap) return;
+ const rect = wrap.getBoundingClientRect();
+ const dpr = Math.max(1, window.devicePixelRatio || 1);
+ canvas.width = Math.max(640, Math.floor(rect.width * dpr));
+ canvas.height = Math.max(480, Math.floor(rect.height * dpr));
+ canvas.style.width = `${Math.floor(rect.width)}px`;
+ canvas.style.height = `${Math.floor(rect.height)}px`;
+ const ctx = canvas.getContext("2d");
+ if (!ctx) return;
+ ctx.setTransform(1, 0, 0, 1, 0, 0);
+ ctx.scale(dpr, dpr);
+ resetCanvas();
+ hasLoadedInitialImageRef.current = false;
+ }, [resetCanvas]);
+
+ const getCanvasPoint = useCallback((event: ReactPointerEvent) => {
+ const canvas = canvasRef.current;
+ if (!canvas) return null;
+ const rect = canvas.getBoundingClientRect();
+ return {
+ x: event.clientX - rect.left,
+ y: event.clientY - rect.top,
+ };
+ }, []);
+
+ const drawStroke = useCallback(
+ (from: { x: number; y: number }, to: { x: number; y: number }) => {
+ const canvas = canvasRef.current;
+ if (!canvas) return;
+ const ctx = canvas.getContext("2d");
+ if (!ctx) return;
+
+ ctx.save();
+ if (brushMode === "eraser") {
+ ctx.globalCompositeOperation = "destination-out";
+ ctx.strokeStyle = "rgba(0,0,0,1)";
+ ctx.lineWidth = brushSize * 1.45;
+ } else {
+ ctx.globalCompositeOperation = "source-over";
+ ctx.strokeStyle = "#18181b";
+ ctx.lineWidth = brushSize;
+ }
+ ctx.lineCap = "round";
+ ctx.lineJoin = "round";
+ ctx.beginPath();
+ ctx.moveTo(from.x, from.y);
+ ctx.lineTo(to.x, to.y);
+ ctx.stroke();
+ ctx.restore();
+ },
+ [brushMode, brushSize]
+ );
+
+ const handlePointerDown = useCallback(
+ (event: ReactPointerEvent) => {
+ const point = getCanvasPoint(event);
+ if (!point) return;
+ pointerDownRef.current = true;
+ lastPointRef.current = point;
+ event.currentTarget.setPointerCapture(event.pointerId);
+ },
+ [getCanvasPoint]
+ );
+
+ const handlePointerMove = useCallback(
+ (event: ReactPointerEvent) => {
+ if (!pointerDownRef.current) return;
+ const point = getCanvasPoint(event);
+ if (!point || !lastPointRef.current) return;
+ drawStroke(lastPointRef.current, point);
+ lastPointRef.current = point;
+ scheduleFlush();
+ },
+ [drawStroke, getCanvasPoint, scheduleFlush]
+ );
+
+ const handlePointerUp = useCallback(
+ (event: ReactPointerEvent) => {
+ pointerDownRef.current = false;
+ lastPointRef.current = null;
+ try {
+ event.currentTarget.releasePointerCapture(event.pointerId);
+ } catch {
+ // ignore
+ }
+ scheduleFlush(true);
+ },
+ [scheduleFlush]
+ );
+
+ const uploadPreviewToCanvas = useCallback(async () => {
+ if (!previewUrl) {
+ toast.error("请先完成一帧实时预览");
+ return;
+ }
+
+ setIsApplying(true);
+ try {
+ const blob = await fetch(previewUrl).then((res) => res.blob());
+ const formData = new FormData();
+ const filename = `realtime-canvas-${Date.now()}.jpg`;
+ formData.append("file", new File([blob], filename, { type: blob.type || "image/jpeg" }));
+
+ const response = await fetch("/api/files/upload", {
+ method: "POST",
+ body: formData,
+ });
+ const result = await response.json().catch(() => ({}));
+
+ if (!response.ok || !result?.url) {
+ throw new Error(result?.error || "上传到画布失败");
+ }
+
+ onApplyResult({
+ imageUrl: String(result.url),
+ prompt: prompt.trim() || FALLBACK_PROMPT,
+ });
+
+ toast.success("已发送到画布");
+ onOpenChange(false);
+ } catch (error) {
+ toast.error(error instanceof Error ? error.message : "发送到画布失败");
+ } finally {
+ setIsApplying(false);
+ }
+ }, [onApplyResult, onOpenChange, previewUrl, prompt]);
+
+ useEffect(() => {
+ if (!open) return;
+ initializeCanvas();
+ void drawInitialImage().then(() => {
+ scheduleFlush(true);
+ });
+ const onResize = () => initializeCanvas();
+ window.addEventListener("resize", onResize);
+ return () => {
+ window.removeEventListener("resize", onResize);
+ };
+ }, [drawInitialImage, initializeCanvas, open, scheduleFlush]);
+
+ useEffect(() => {
+ if (!open) return;
+
+ let cancelled = false;
+ setStatus("connecting");
+ setStatusMessage("正在初始化实时连接...");
+
+ const start = async () => {
+ try {
+ const configResponse = await fetch("/api/realtime/config", { cache: "no-store" });
+ const configJson = (await configResponse.json().catch(() => ({}))) as Partial;
+ if (!configResponse.ok) {
+ throw new Error((configJson as any)?.message || "无法加载实时配置");
+ }
+
+ if (cancelled) return;
+ const resolvedConfig: RealtimeConfig = {
+ transport: {
+ throttleInterval:
+ configJson.transport?.throttleInterval ||
+ defaultRealtimeConfig.transport.throttleInterval,
+ maxBuffering:
+ configJson.transport?.maxBuffering || defaultRealtimeConfig.transport.maxBuffering,
+ },
+ image: {
+ maxInputResolution:
+ configJson.image?.maxInputResolution || defaultRealtimeConfig.image.maxInputResolution,
+ jpegQuality: configJson.image?.jpegQuality || defaultRealtimeConfig.image.jpegQuality,
+ },
+ model: {
+ id: configJson.model?.id || defaultRealtimeConfig.model.id,
+ defaults: {
+ steps: configJson.model?.defaults?.steps || defaultRealtimeConfig.model.defaults.steps,
+ scheduleMu:
+ configJson.model?.defaults?.scheduleMu ||
+ defaultRealtimeConfig.model.defaults.scheduleMu,
+ feedback:
+ configJson.model?.defaults?.feedback ||
+ defaultRealtimeConfig.model.defaults.feedback,
+ imageSize:
+ configJson.model?.defaults?.imageSize ||
+ defaultRealtimeConfig.model.defaults.imageSize,
+ },
+ },
+ };
+
+ setConfig(resolvedConfig);
+ setAdaptiveThrottleMs(resolvedConfig.transport.throttleInterval);
+
+ fal.config({
+ proxyUrl: "/api/fal/proxy",
+ });
+
+ const connection = fal.realtime.connect(resolvedConfig.model.id, {
+ connectionKey: `canvas-realtime-${sessionIdRef.current}-${refreshToken}`,
+ throttleInterval: resolvedConfig.transport.throttleInterval,
+ maxBuffering: resolvedConfig.transport.maxBuffering,
+ onResult: (result: any) => {
+ const requestId = String(result?.request_id || "");
+ const seqFromRequest = requestSeqMapRef.current.get(requestId) || 0;
+ const seq = Math.max(seqFromRequest, Number(result?.seq || 0));
+ if (seq > 0 && seq < lastRenderedSeqRef.current) {
+ return;
+ }
+
+ const latestImage = Array.isArray(result?.images)
+ ? result.images[result.images.length - 1]
+ : null;
+ const nextPreviewUrl = buildImageDataUrl(latestImage);
+ if (!nextPreviewUrl) return;
+
+ lastRenderedSeqRef.current = Math.max(lastRenderedSeqRef.current, seq);
+ setPreviewUrl(nextPreviewUrl);
+ setStatus("live");
+ setStatusMessage("实时渲染中");
+
+ if (requestId) {
+ const startedAt = requestStartMapRef.current.get(requestId);
+ if (startedAt) {
+ pushMetric({
+ name: "frame_result",
+ ts: Date.now(),
+ seq,
+ latencyMs: Date.now() - startedAt,
+ });
+ }
+ requestStartMapRef.current.delete(requestId);
+ requestSeqMapRef.current.delete(requestId);
+ }
+ },
+ onError: (error: any) => {
+ const errorMessage =
+ error?.body?.message || error?.message || "实时连接异常，请稍后重试";
+ setStatus("error");
+ setStatusMessage(String(errorMessage));
+ if (Number(error?.status) === 429) {
+ setAdaptiveThrottleMs((prev) => Math.min(240, prev + 40));
+ }
+ pushMetric({
+ name: "frame_error",
+ ts: Date.now(),
+ message: String(errorMessage),
+ });
+ },
+ });
+
+ if (cancelled) {
+ connection.close();
+ return;
+ }
+
+ connectionRef.current = connection;
+ setStatus("connecting");
+ setStatusMessage("连接成功，等待首帧...");
+ scheduleFlush(true);
+ } catch (error) {
+ console.error("[RealtimeCanvas] init failed:", error);
+ setStatus("error");
+ setStatusMessage(error instanceof Error ? error.message : "实时初始化失败");
+ }
+ };
+
+ void start();
+ const metricTimer = window.setInterval(() => {
+ void flushMetrics();
+ }, 5000);
+
+ return () => {
+ cancelled = true;
+ if (pendingFlushTimerRef.current) {
+ window.clearTimeout(pendingFlushTimerRef.current);
+ pendingFlushTimerRef.current = null;
+ }
+ window.clearInterval(metricTimer);
+ void flushMetrics();
+ connectionRef.current?.close?.();
+ connectionRef.current = null;
+ setStatus("idle");
+ setStatusMessage("等待连接");
+ pointerDownRef.current = false;
+ lastPointRef.current = null;
+ };
+ }, [flushMetrics, open, pushMetric, refreshToken, scheduleFlush]);
+
+ const forceReconnect = useCallback(() => {
+ connectionRef.current?.close?.();
+ connectionRef.current = null;
+ setStatus("connecting");
+ setStatusMessage("正在重连...");
+ setRefreshToken((prev) => prev + 1);
+ }, []);
+
+ const clearSketch = useCallback(() => {
+ resetCanvas();
+ dirtyRef.current = true;
+ scheduleFlush(true);
+ }, [resetCanvas, scheduleFlush]);
+
+ return (
+
+ );
+}
diff --git a/lib/db/migrations/0042_add_realtime_canvas_toggle.sql b/lib/db/migrations/0042_add_realtime_canvas_toggle.sql
new file mode 100644
index 000000000..851018213
--- /dev/null
+++ b/lib/db/migrations/0042_add_realtime_canvas_toggle.sql
@@ -0,0 +1,2 @@
+ALTER TABLE "SiteSettings"
+ADD COLUMN IF NOT EXISTS "realtimeCanvasEnabled" boolean NOT NULL DEFAULT true;
diff --git a/lib/db/migrations/meta/_journal.json b/lib/db/migrations/meta/_journal.json
index 3476c8667..696a93bb0 100644
--- a/lib/db/migrations/meta/_journal.json
+++ b/lib/db/migrations/meta/_journal.json
@@ -197,6 +197,13 @@
"when": 1772265600000,
"tag": "0041_add_user_high_vip",
"breakpoints": true
+ },
+ {
+ "idx": 28,
+ "version": "7",
+ "when": 1772280000000,
+ "tag": "0042_add_realtime_canvas_toggle",
+ "breakpoints": true
}
]
}
diff --git a/lib/db/schema.ts b/lib/db/schema.ts
index e08b58c6d..6891e817c 100644
--- a/lib/db/schema.ts
+++ b/lib/db/schema.ts
@@ -354,6 +354,7 @@ export const siteSettings = pgTable("SiteSettings", {
rankingEnabled: boolean("rankingEnabled").notNull().default(true), // 是否显示精选榜按钮
// 无限画布设置
canvasEnabled: boolean("canvasEnabled").notNull().default(true), // 是否启用无限画布功能
+ realtimeCanvasEnabled: boolean("realtimeCanvasEnabled").notNull().default(true), // 是否显示实时画布入口
canvasDefaultModelId: varchar("canvasDefaultModelId", { length: 128 }), // 画布默认选择的模型ID（null表示智能自动）
canvasAutoModeDefaultModelId: varchar("canvasAutoModeDefaultModelId", { length: 128 }), // 智能自动模式下实际使用的默认生图模型ID
canvasDetailDefaultModelId: varchar("canvasDetailDefaultModelId", { length: 128 }), // AI详情生成专用默认生图模型ID