From 18149083d49b888cacee80d5efe7912a6dd37249 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 22 Feb 2026 02:57:32 +0800 Subject: [PATCH] feat: add realtime canvas feature with admin db config --- .../settings/admin-settings-client.tsx | 296 +++++++ app/api/fal/proxy/route.ts | 116 +++ app/api/realtime/config/route.ts | 133 +++ app/api/realtime/metrics/route.ts | 54 ++ app/api/site-settings/route.ts | 3 + app/canvas/[id]/page.tsx | 59 ++ components/canvas/RealtimeCanvasDialog.tsx | 754 ++++++++++++++++++ .../0042_add_realtime_canvas_toggle.sql | 2 + lib/db/migrations/meta/_journal.json | 7 + lib/db/schema.ts | 1 + 10 files changed, 1425 insertions(+) create mode 100644 app/api/fal/proxy/route.ts create mode 100644 app/api/realtime/config/route.ts create mode 100644 app/api/realtime/metrics/route.ts create mode 100644 components/canvas/RealtimeCanvasDialog.tsx create mode 100644 lib/db/migrations/0042_add_realtime_canvas_toggle.sql diff --git a/app/(admin)/settings/admin-settings-client.tsx b/app/(admin)/settings/admin-settings-client.tsx index 55e2f39c1..0989ae7b6 100644 --- a/app/(admin)/settings/admin-settings-client.tsx +++ b/app/(admin)/settings/admin-settings-client.tsx @@ -55,6 +55,7 @@ import { Video, Loader2, Check, + Paintbrush, } from "lucide-react"; import { Dialog, @@ -197,6 +198,7 @@ type SiteSettings = { tutorialEnabled: boolean; rankingEnabled: boolean; canvasEnabled: boolean; + realtimeCanvasEnabled: boolean; aiDetailGenerationEnabled: boolean; canvasDefaultModelId: string | null; canvasAutoModeDefaultModelId: string | null; @@ -519,6 +521,7 @@ export function AdminSettingsClient() { tutorialEnabled: false, rankingEnabled: true, canvasEnabled: true, + realtimeCanvasEnabled: true, aiDetailGenerationEnabled: false, canvasDefaultModelId: null, canvasAutoModeDefaultModelId: null, @@ -587,6 +590,7 @@ export function AdminSettingsClient() { tutorialEnabled: siteSettingsData.tutorialEnabled ?? false, rankingEnabled: siteSettingsData.rankingEnabled ?? true, canvasEnabled: siteSettingsData.canvasEnabled ?? true, + realtimeCanvasEnabled: (siteSettingsData as any).realtimeCanvasEnabled ?? true, aiDetailGenerationEnabled: siteSettingsData.aiDetailGenerationEnabled ?? false, canvasDefaultModelId: siteSettingsData.canvasDefaultModelId || null, canvasAutoModeDefaultModelId: siteSettingsData.canvasAutoModeDefaultModelId || null, @@ -695,6 +699,20 @@ export function AdminSettingsClient() { timeout: 120, }); + // 实时画布API配置状态 + const [realtimeCanvasConfig, setRealtimeCanvasConfig] = useState({ + enabled: true, + falApiKey: "", + throttleInterval: 100, + maxBuffering: 2, + maxInputResolution: 1024, + jpegQuality: 0.55, + steps: 3, + scheduleMu: 2.3, + feedback: 0.95, + imageSize: "square" as "square" | "square_hd", + }); + // 详情替换配置状态 const [sceneFusionConfig, setSceneFusionConfig] = useState({ enabled: true, @@ -821,6 +839,42 @@ export function AdminSettingsClient() { } }, [angleEditConfigData]); + // 加载实时画布API配置 + const { data: realtimeCanvasConfigData } = useSWR( + "/api/admin/settings?key=realtime_canvas_config", + fetcher + ); + useEffect(() => { + if (realtimeCanvasConfigData?.setting?.value) { + const config = realtimeCanvasConfigData.setting.value as { + enabled?: boolean; + falApiKey?: string; + throttleInterval?: number; + maxBuffering?: number; + maxInputResolution?: number; + jpegQuality?: number; + defaults?: { + steps?: number; + scheduleMu?: number; + feedback?: number; + imageSize?: "square" | "square_hd"; + }; + }; + setRealtimeCanvasConfig({ + enabled: config.enabled ?? true, + falApiKey: config.falApiKey || "", + throttleInterval: Number(config.throttleInterval || 100), + maxBuffering: Number(config.maxBuffering || 2), + maxInputResolution: Number(config.maxInputResolution || 1024), + jpegQuality: Number(config.jpegQuality || 0.55), + steps: Number(config.defaults?.steps || 3), + scheduleMu: Number(config.defaults?.scheduleMu || 2.3), + feedback: Number(config.defaults?.feedback || 0.95), + imageSize: config.defaults?.imageSize === "square_hd" ? "square_hd" : "square", + }); + } + }, [realtimeCanvasConfigData]); + // 加载详情替换配置 const { data: sceneFusionConfigData } = useSWR("/api/admin/settings?key=scene_fusion_config", fetcher); useEffect(() => { @@ -1635,6 +1689,46 @@ export function AdminSettingsClient() { } }; + // --- 实时画布API配置逻辑 --- + const handleSaveRealtimeCanvasConfig = async () => { + try { + if (realtimeCanvasConfig.enabled && !realtimeCanvasConfig.falApiKey.trim()) { + toast.error("启用实时画布功能需要配置 fal.ai API Key"); + return; + } + + const response = await fetch("/api/admin/settings", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + key: "realtime_canvas_config", + value: { + enabled: realtimeCanvasConfig.enabled, + falApiKey: realtimeCanvasConfig.falApiKey.trim(), + throttleInterval: Math.min(260, Math.max(60, Number(realtimeCanvasConfig.throttleInterval) || 100)), + maxBuffering: Math.min(6, Math.max(1, Number(realtimeCanvasConfig.maxBuffering) || 2)), + maxInputResolution: Math.min( + 1536, + Math.max(512, Number(realtimeCanvasConfig.maxInputResolution) || 1024) + ), + jpegQuality: Math.min(0.82, Math.max(0.35, Number(realtimeCanvasConfig.jpegQuality) || 0.55)), + defaults: { + steps: Math.min(6, Math.max(1, Number(realtimeCanvasConfig.steps) || 3)), + scheduleMu: Math.min(3.5, Math.max(1.5, Number(realtimeCanvasConfig.scheduleMu) || 2.3)), + feedback: Math.min(1, Math.max(0.7, Number(realtimeCanvasConfig.feedback) || 0.95)), + imageSize: realtimeCanvasConfig.imageSize, + }, + }, + }), + }); + + if (!response.ok) throw new Error("保存失败"); + toast.success("实时画布配置保存成功"); + } catch (_error) { + toast.error("保存实时画布配置失败"); + } + }; + // --- 详情替换配置逻辑 --- const handleSaveSceneFusionConfig = async () => { try { @@ -4554,6 +4648,22 @@ export function AdminSettingsClient() { {siteSettings.canvasEnabled && ( <> +
+
+ +
+ 开启后，无限画布左侧工具栏显示“实时画布”按钮；关闭后前端隐藏。 +
+
+ + setSiteSettings({ ...siteSettings, realtimeCanvasEnabled: checked }) + } + /> +
+
@@ -7092,6 +7202,192 @@ export function AdminSettingsClient() { + {/* 实时画布API配置 */} + + + + + 实时画布配置 + + + 配置实时画布（左画右预览）功能使用的 fal realtime 参数与 API Key。该 Key 存储在数据库，前端不暴露。 + + + +
+
+ +
+ 关闭后前端“实时画布”入口仍可显示，但会提示服务未启用。 +
+
+ + setRealtimeCanvasConfig({ ...realtimeCanvasConfig, enabled: checked }) + } + /> +
+ +
+ + + setRealtimeCanvasConfig({ ...realtimeCanvasConfig, falApiKey: e.target.value }) + } + placeholder="fal_xxxxxxxxxxxxx" + disabled={!realtimeCanvasConfig.enabled} + /> +
+ +
+
+ + + setRealtimeCanvasConfig({ + ...realtimeCanvasConfig, + throttleInterval: Number.parseInt(e.target.value, 10) || 100, + }) + } + /> +
+
+ + + setRealtimeCanvasConfig({ + ...realtimeCanvasConfig, + maxBuffering: Number.parseInt(e.target.value, 10) || 2, + }) + } + /> +
+
+ +
+
+ + + setRealtimeCanvasConfig({ + ...realtimeCanvasConfig, + maxInputResolution: Number.parseInt(e.target.value, 10) || 1024, + }) + } + /> +
+
+ + + setRealtimeCanvasConfig({ + ...realtimeCanvasConfig, + jpegQuality: Number.parseFloat(e.target.value) || 0.55, + }) + } + /> +
+
+ +
+
+ + + setRealtimeCanvasConfig({ + ...realtimeCanvasConfig, + steps: Number.parseInt(e.target.value, 10) || 3, + }) + } + /> +
+
+ + + setRealtimeCanvasConfig({ + ...realtimeCanvasConfig, + feedback: Number.parseFloat(e.target.value) || 0.95, + }) + } + /> +
+
+ +
+
+ + + setRealtimeCanvasConfig({ + ...realtimeCanvasConfig, + scheduleMu: Number.parseFloat(e.target.value) || 2.3, + }) + } + /> +
+
+ + +
+
+ + + + + {/* 详情替换配置 */} diff --git a/app/api/fal/proxy/route.ts b/app/api/fal/proxy/route.ts new file mode 100644 index 000000000..d2151ad94 --- /dev/null +++ b/app/api/fal/proxy/route.ts @@ -0,0 +1,116 @@ +import { auth } from "@/app/(auth)/auth"; +import { getSystemSetting } from "@/lib/db/queries"; + +const ALLOWED_FAL_HOSTS = new Set(["rest.alpha.fal.ai", "rest.fal.ai"]); + +type FalRealtimeConfig = { + enabled?: boolean; + falApiKey?: string; +}; + +async function getRealtimeCanvasConfig() { + const realtimeSetting = await getSystemSetting({ key: "realtime_canvas_config" }).catch( + () => null + ); + return (realtimeSetting?.value || {}) as FalRealtimeConfig; +} + +async function handleProxy(request: Request) { + const session = await auth(); + if (!session?.user?.id) { + return Response.json({ code: "UNAUTHORIZED", message: "请先登录" }, { status: 401 }); + } + + const targetUrl = request.headers.get("x-fal-target-url"); + if (!targetUrl) { + return Response.json( + { code: "INVALID_TARGET", message: "缺少 x-fal-target-url 请求头" }, + { status: 400 } + ); + } + + let parsedUrl: URL; + try { + parsedUrl = new URL(targetUrl); + } catch { + return Response.json( + { code: "INVALID_TARGET", message: "x-fal-target-url 不是合法 URL" }, + { status: 400 } + ); + } + + if (!ALLOWED_FAL_HOSTS.has(parsedUrl.host)) { + return Response.json( + { code: "TARGET_NOT_ALLOWED", message: "不允许代理该目标地址" }, + { status: 403 } + ); + } + + const realtimeConfig = await getRealtimeCanvasConfig(); + if (realtimeConfig.enabled === false) { + return Response.json( + { + code: "FEATURE_DISABLED", + message: "实时画布服务未启用（请在本地AI设置中开启）", + }, + { status: 403 } + ); + } + + const falApiKey = realtimeConfig.falApiKey?.trim() || ""; + if (!falApiKey) { + return Response.json( + { + code: "FAL_KEY_MISSING", + message: "未配置 FAL API Key（请在本地AI设置的实时画布配置中填写）", + }, + { status: 500 } + ); + } + + let body: ArrayBuffer | undefined; + if (request.method !== "GET" && request.method !== "HEAD") { + body = await request.arrayBuffer(); + } + + const headers = new Headers(); + headers.set("Authorization", `Key ${falApiKey}`); + headers.set("Accept", request.headers.get("accept") || "application/json"); + const contentType = request.headers.get("content-type"); + if (contentType) { + headers.set("Content-Type", contentType); + } + + const upstreamResponse = await fetch(parsedUrl.toString(), { + method: request.method, + headers, + body, + cache: "no-store", + }); + + return new Response(upstreamResponse.body, { + status: upstreamResponse.status, + statusText: upstreamResponse.statusText, + headers: upstreamResponse.headers, + }); +} + +export async function GET(request: Request) { + return handleProxy(request); +} + +export async function POST(request: Request) { + return handleProxy(request); +} + +export async function PUT(request: Request) { + return handleProxy(request); +} + +export async function PATCH(request: Request) { + return handleProxy(request); +} + +export async function DELETE(request: Request) { + return handleProxy(request); +} diff --git a/app/api/realtime/config/route.ts b/app/api/realtime/config/route.ts new file mode 100644 index 000000000..c61127051 --- /dev/null +++ b/app/api/realtime/config/route.ts @@ -0,0 +1,133 @@ +import { auth } from "@/app/(auth)/auth"; +import { getSystemSetting } from "@/lib/db/queries"; + +type RealtimeCanvasConfig = { + enabled?: boolean; + falApiKey?: string; + throttleInterval?: number; + maxBuffering?: number; + maxInputResolution?: number; + jpegQuality?: number; + defaults?: { + steps?: number; + scheduleMu?: number; + feedback?: number; + imageSize?: "square" | "square_hd"; + }; +}; + +const defaultConfig = { + session: { + idleTimeoutMs: 120000, + }, + transport: { + throttleInterval: 100, + maxBuffering: 2, + }, + image: { + maxInputResolution: 1024, + jpegQuality: 0.55, + }, + model: { + id: "fal-ai/flux-2/klein/realtime", + defaults: { + steps: 3, + scheduleMu: 2.3, + feedback: 0.95, + imageSize: "square" as const, + }, + limits: { + steps: { min: 1, max: 6 }, + feedback: { min: 0.7, max: 1.0 }, + }, + }, + features: { + history: true, + abCompare: false, + }, +}; + +function clamp(value: number, min: number, max: number) { + return Math.min(max, Math.max(min, value)); +} + +export async function GET() { + const session = await auth(); + if (!session?.user?.id) { + return Response.json({ code: "UNAUTHORIZED", message: "请先登录" }, { status: 401 }); + } + + const setting = await getSystemSetting({ key: "realtime_canvas_config" }).catch(() => null); + const config = (setting?.value || {}) as RealtimeCanvasConfig; + + if (config.enabled === false) { + return Response.json( + { code: "FEATURE_DISABLED", message: "实时画布功能未开启" }, + { status: 403 } + ); + } + + if (!String(config.falApiKey || "").trim()) { + return Response.json( + { + code: "FAL_KEY_MISSING", + message: "未配置实时画布 API Key（请在本地AI设置中配置）", + }, + { status: 500 } + ); + } + + const throttleInterval = clamp( + Number(config.throttleInterval || defaultConfig.transport.throttleInterval), + 60, + 260 + ); + const maxBuffering = clamp( + Number(config.maxBuffering || defaultConfig.transport.maxBuffering), + 1, + 6 + ); + const maxInputResolution = clamp( + Number(config.maxInputResolution || defaultConfig.image.maxInputResolution), + 512, + 1536 + ); + const jpegQuality = clamp( + Number(config.jpegQuality || defaultConfig.image.jpegQuality), + 0.35, + 0.82 + ); + const steps = clamp(Number(config.defaults?.steps || defaultConfig.model.defaults.steps), 1, 6); + const scheduleMu = clamp( + Number(config.defaults?.scheduleMu || defaultConfig.model.defaults.scheduleMu), + 1.5, + 3.5 + ); + const feedback = clamp( + Number(config.defaults?.feedback || defaultConfig.model.defaults.feedback), + 0.7, + 1.0 + ); + const imageSize = config.defaults?.imageSize === "square_hd" ? "square_hd" : "square"; + + return Response.json({ + ...defaultConfig, + transport: { + throttleInterval, + maxBuffering, + }, + image: { + maxInputResolution, + jpegQuality, + }, + model: { + ...defaultConfig.model, + defaults: { + steps, + scheduleMu, + feedback, + imageSize, + }, + }, + }); +} diff --git a/app/api/realtime/metrics/route.ts b/app/api/realtime/metrics/route.ts new file mode 100644 index 000000000..3f779ebf4 --- /dev/null +++ b/app/api/realtime/metrics/route.ts @@ -0,0 +1,54 @@ +import { auth } from "@/app/(auth)/auth"; + +type MetricEvent = { + name: string; + ts: number; + seq?: number; + latencyMs?: number; + queueDepth?: number; + networkType?: string; + deviceClass?: string; +}; + +type MetricsPayload = { + sessionId: string; + events: MetricEvent[]; +}; + +export async function POST(request: Request) { + const session = await auth(); + if (!session?.user?.id) { + return Response.json({ code: "UNAUTHORIZED", message: "请先登录" }, { status: 401 }); + } + + let payload: MetricsPayload; + try { + payload = (await request.json()) as MetricsPayload; + } catch { + return Response.json({ code: "BAD_REQUEST", message: "无效 JSON" }, { status: 400 }); + } + + if (!payload?.sessionId || !Array.isArray(payload.events)) { + return Response.json({ code: "BAD_REQUEST", message: "缺少 sessionId 或 events" }, { status: 400 }); + } + + if (payload.events.length > 200) { + return Response.json( + { code: "PAYLOAD_TOO_LARGE", message: "events 超过限制（200）" }, + { status: 413 } + ); + } + + if (payload.events.length > 0) { + const latest = payload.events[payload.events.length - 1]; + console.log("[RealtimeMetrics]", { + userId: session.user.id, + sessionId: payload.sessionId, + count: payload.events.length, + latest, + }); + } + + return Response.json({ ok: true }); +} + diff --git a/app/api/site-settings/route.ts b/app/api/site-settings/route.ts index 7cb3f7c11..800259b87 100644 --- a/app/api/site-settings/route.ts +++ b/app/api/site-settings/route.ts @@ -283,6 +283,7 @@ export async function GET() { tutorialEnabled: false, rankingEnabled: true, canvasEnabled: true, + realtimeCanvasEnabled: true, canvasDefaultModelId: null, canvasAutoModeDefaultModelId: null, canvasDetailDefaultModelId: null, @@ -364,6 +365,7 @@ const updateSchema = z.object({ tutorialEnabled: z.boolean().optional(), rankingEnabled: z.boolean().optional(), canvasEnabled: z.boolean().optional(), + realtimeCanvasEnabled: z.boolean().optional(), aiDetailGenerationEnabled: z.boolean().optional(), canvasDefaultModelId: z.string().nullable().optional(), canvasAutoModeDefaultModelId: z.string().nullable().optional(), @@ -754,6 +756,7 @@ export async function PUT(request: NextRequest) { tutorialEnabled: siteSettingsPayload.tutorialEnabled ?? false, rankingEnabled: siteSettingsPayload.rankingEnabled ?? true, canvasEnabled: siteSettingsPayload.canvasEnabled ?? true, + realtimeCanvasEnabled: siteSettingsPayload.realtimeCanvasEnabled ?? true, canvasDefaultModelId: siteSettingsPayload.canvasDefaultModelId, canvasAutoModeDefaultModelId: siteSettingsPayload.canvasAutoModeDefaultModelId, canvasDetailDefaultModelId: siteSettingsPayload.canvasDetailDefaultModelId, diff --git a/app/canvas/[id]/page.tsx b/app/canvas/[id]/page.tsx index 549cffead..aff4c905a 100644 --- a/app/canvas/[id]/page.tsx +++ b/app/canvas/[id]/page.tsx @@ -23,6 +23,7 @@ import { Wand2, Grid3X3, Settings, + PenTool, } from "lucide-react"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; @@ -58,6 +59,7 @@ import { CanvasWorkspace } from "@/components/canvas"; import { CanvasChatPanel } from "@/components/canvas/CanvasChatPanel"; import { CanvasFileListPanel } from "@/components/canvas/CanvasFileListPanel"; import { ObjectPickerOverlay } from "@/components/canvas/ObjectPickerOverlay"; +import { RealtimeCanvasDialog } from "@/components/canvas/RealtimeCanvasDialog"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { cn, fetcher } from "@/lib/utils"; import { MessageSquare, FolderOpen, Grid2X2 } from "lucide-react"; @@ -121,11 +123,13 @@ export default function CanvasEditorPage({ params }: PageProps) { const [isPanelOpen, setIsPanelOpen] = useState(true); // 面板模式: 'chat' 聊天面板, 'fileList' 文件列表 const [panelMode, setPanelMode] = useState<'chat' | 'fileList'>('chat'); + const [isRealtimeCanvasOpen, setIsRealtimeCanvasOpen] = useState(false); const { data: siteSettingsData } = useSWR<{ siteName?: string; siteLogo?: string | null; siteLogoDark?: string | null; + realtimeCanvasEnabled?: boolean; }>("/api/site-settings", fetcher); // 生成配置 @@ -353,6 +357,38 @@ export default function CanvasEditorPage({ params }: PageProps) { }); }; + const handleApplyRealtimeResult = useCallback( + ({ imageUrl, prompt: imagePrompt }: { imageUrl: string; prompt: string }) => { + const viewportX = viewport.x || 0; + const viewportY = viewport.y || 0; + const viewportZoom = viewport.zoom || 1; + const stageWidth = typeof window !== "undefined" ? window.innerWidth : 1280; + const stageHeight = typeof window !== "undefined" ? window.innerHeight : 720; + const targetWidth = 360; + const targetHeight = 360; + + const x = (stageWidth * 0.45 - viewportX) / viewportZoom - targetWidth / 2; + const y = (stageHeight * 0.45 - viewportY) / viewportZoom - targetHeight / 2; + + addNode({ + type: "image", + position: { x, y }, + size: { width: targetWidth, height: targetHeight }, + data: { + src: imageUrl, + status: "completed", + title: "实时画布结果", + prompt: imagePrompt, + model: "fal-ai/flux-2/klein/realtime", + }, + connections: [], + }); + + toast.success("实时画布结果已添加"); + }, + [addNode, viewport.x, viewport.y, viewport.zoom] + ); + // 处理图片操作（通过事件分发到选中图片节点执行） const handleAction = (action: string) => { if (!selectedImageNode) { @@ -717,6 +753,22 @@ export default function CanvasEditorPage({ params }: PageProps) { 添加图片 + {(siteSettingsData?.realtimeCanvasEnabled ?? true) ? ( + + + + + 实时画布 + + ) : null} + @@ -756,6 +808,13 @@ export default function CanvasEditorPage({ params }: PageProps) { {/* 6. 物品拾取模式覆盖层 */} + +
); diff --git a/components/canvas/RealtimeCanvasDialog.tsx b/components/canvas/RealtimeCanvasDialog.tsx new file mode 100644 index 000000000..cf45c9433 --- /dev/null +++ b/components/canvas/RealtimeCanvasDialog.tsx @@ -0,0 +1,754 @@ +"use client"; + +import { useCallback, useEffect, useMemo, useRef, useState, type PointerEvent as ReactPointerEvent } from "react"; +import * as fal from "@fal-ai/serverless-client"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { cn } from "@/lib/utils"; +import { + Eraser, + Loader2, + Paintbrush, + RefreshCw, + Send, + Sparkles, + Trash2, + Wifi, + WifiOff, +} from "lucide-react"; +import { toast } from "sonner"; + +type RealtimeConfig = { + transport: { + throttleInterval: number; + maxBuffering: number; + }; + image: { + maxInputResolution: number; + jpegQuality: number; + }; + model: { + id: string; + defaults: { + steps: number; + scheduleMu: number; + feedback: number; + imageSize: "square" | "square_hd"; + }; + }; +}; + +type RealtimeStatus = "idle" | "connecting" | "live" | "error"; + +type RealtimeCanvasDialogProps = { + open: boolean; + onOpenChange: (open: boolean) => void; + initialImageUrl?: string | null; + onApplyResult: (payload: { imageUrl: string; prompt: string }) => void; +}; + +const FALLBACK_PROMPT = "将草图渲染为高质感产品设计图，保持结构和主轮廓一致。"; + +const defaultRealtimeConfig: RealtimeConfig = { + transport: { + throttleInterval: 100, + maxBuffering: 2, + }, + image: { + maxInputResolution: 1024, + jpegQuality: 0.55, + }, + model: { + id: "fal-ai/flux-2/klein/realtime", + defaults: { + steps: 3, + scheduleMu: 2.3, + feedback: 0.95, + imageSize: "square", + }, + }, +}; + +function buildImageDataUrl(image: any) { + if (!image) return ""; + if (typeof image.url === "string" && image.url.trim()) { + return image.url; + } + if (typeof image.content === "string" && image.content.trim()) { + if (image.content.startsWith("data:")) { + return image.content; + } + const contentType = image.content_type || "image/jpeg"; + return `data:${contentType};base64,${image.content}`; + } + return ""; +} + +async function canvasToJpegDataUri( + canvas: HTMLCanvasElement, + maxInputResolution: number, + jpegQuality: number +) { + const sourceWidth = canvas.width; + const sourceHeight = canvas.height; + const longestSide = Math.max(sourceWidth, sourceHeight); + const scale = longestSide > maxInputResolution ? maxInputResolution / longestSide : 1; + const exportWidth = Math.max(1, Math.round(sourceWidth * scale)); + const exportHeight = Math.max(1, Math.round(sourceHeight * scale)); + + const exportCanvas = document.createElement("canvas"); + exportCanvas.width = exportWidth; + exportCanvas.height = exportHeight; + const exportCtx = exportCanvas.getContext("2d"); + if (!exportCtx) { + throw new Error("无法初始化导出画布"); + } + + exportCtx.fillStyle = "#d8d8db"; + exportCtx.fillRect(0, 0, exportWidth, exportHeight); + exportCtx.drawImage(canvas, 0, 0, exportWidth, exportHeight); + + const blob = await new Promise((resolve) => { + exportCanvas.toBlob(resolve, "image/jpeg", jpegQuality); + }); + if (!blob) { + throw new Error("导出图片失败"); + } + + return await new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => resolve(String(reader.result || "")); + reader.onerror = () => reject(new Error("读取导出图片失败")); + reader.readAsDataURL(blob); + }); +} + +export function RealtimeCanvasDialog({ + open, + onOpenChange, + initialImageUrl, + onApplyResult, +}: RealtimeCanvasDialogProps) { + const canvasRef = useRef(null); + const canvasWrapRef = useRef(null); + const connectionRef = useRef(null); + const pointerDownRef = useRef(false); + const lastPointRef = useRef<{ x: number; y: number } | null>(null); + const pendingFlushTimerRef = useRef(null); + const flushInFlightRef = useRef(false); + const dirtyRef = useRef(false); + const seqRef = useRef(0); + const requestSeqMapRef = useRef>(new Map()); + const requestStartMapRef = useRef>(new Map()); + const lastRenderedSeqRef = useRef(0); + const metricsQueueRef = useRef([]); + const sessionIdRef = useRef(`rt-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`); + const hasLoadedInitialImageRef = useRef(false); + const [config, setConfig] = useState(defaultRealtimeConfig); + const [status, setStatus] = useState("idle"); + const [statusMessage, setStatusMessage] = useState("等待连接"); + const [prompt, setPrompt] = useState(FALLBACK_PROMPT); + const [previewUrl, setPreviewUrl] = useState(""); + const [isApplying, setIsApplying] = useState(false); + const [brushMode, setBrushMode] = useState<"brush" | "eraser">("brush"); + const [brushSize, setBrushSize] = useState(6); + const [adaptiveThrottleMs, setAdaptiveThrottleMs] = useState( + defaultRealtimeConfig.transport.throttleInterval + ); + const [refreshToken, setRefreshToken] = useState(0); + + const statusBadge = useMemo(() => { + if (status === "live") return { label: "实时中", className: "bg-emerald-50 text-emerald-700" }; + if (status === "connecting") { + return { label: "连接中", className: "bg-blue-50 text-blue-700" }; + } + if (status === "error") return { label: "异常", className: "bg-rose-50 text-rose-700" }; + return { label: "待启动", className: "bg-zinc-100 text-zinc-600" }; + }, [status]); + + const pushMetric = useCallback((event: any) => { + metricsQueueRef.current.push(event); + if (metricsQueueRef.current.length > 120) { + metricsQueueRef.current = metricsQueueRef.current.slice(-80); + } + }, []); + + const flushMetrics = useCallback(async () => { + if (!open || metricsQueueRef.current.length === 0) return; + const events = metricsQueueRef.current.splice(0, metricsQueueRef.current.length); + try { + await fetch("/api/realtime/metrics", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + sessionId: sessionIdRef.current, + events, + }), + }); + } catch { + metricsQueueRef.current.unshift(...events.slice(-20)); + } + }, [open]); + + const resetCanvas = useCallback(() => { + const canvas = canvasRef.current; + if (!canvas) return; + const ctx = canvas.getContext("2d"); + if (!ctx) return; + ctx.save(); + ctx.globalCompositeOperation = "source-over"; + ctx.fillStyle = "#d8d8db"; + ctx.fillRect(0, 0, canvas.width, canvas.height); + ctx.restore(); + }, []); + + const drawInitialImage = useCallback(async () => { + if (!open || !initialImageUrl || hasLoadedInitialImageRef.current) return; + const canvas = canvasRef.current; + if (!canvas) return; + const ctx = canvas.getContext("2d"); + if (!ctx) return; + + const image = new Image(); + image.crossOrigin = "anonymous"; + await new Promise((resolve, reject) => { + image.onload = () => resolve(); + image.onerror = () => reject(new Error("加载初始图片失败")); + image.src = initialImageUrl; + }); + + const scale = Math.min(canvas.width / image.width, canvas.height / image.height); + const drawWidth = image.width * scale; + const drawHeight = image.height * scale; + const offsetX = (canvas.width - drawWidth) / 2; + const offsetY = (canvas.height - drawHeight) / 2; + + ctx.drawImage(image, offsetX, offsetY, drawWidth, drawHeight); + hasLoadedInitialImageRef.current = true; + dirtyRef.current = true; + }, [initialImageUrl, open]); + + const flushFrame = useCallback( + async (force = false) => { + if (!open) return; + if (!connectionRef.current) return; + if (flushInFlightRef.current) return; + if (!dirtyRef.current && !force) return; + + const canvas = canvasRef.current; + if (!canvas) return; + + flushInFlightRef.current = true; + const seq = seqRef.current + 1; + seqRef.current = seq; + const requestId = `${sessionIdRef.current}-${seq}`; + requestSeqMapRef.current.set(requestId, seq); + requestStartMapRef.current.set(requestId, Date.now()); + + try { + const imageDataUri = await canvasToJpegDataUri( + canvas, + config.image.maxInputResolution, + config.image.jpegQuality + ); + + dirtyRef.current = false; + setStatus("connecting"); + setStatusMessage("正在请求实时生成..."); + connectionRef.current.send({ + request_id: requestId, + image_url: imageDataUri, + prompt: prompt.trim() || FALLBACK_PROMPT, + image_size: config.model.defaults.imageSize, + num_inference_steps: config.model.defaults.steps, + schedule_mu: config.model.defaults.scheduleMu, + output_feedback_strength: config.model.defaults.feedback, + }); + + pushMetric({ + name: "frame_send", + ts: Date.now(), + seq, + }); + } catch (error) { + console.error("[RealtimeCanvas] flush frame failed:", error); + setStatus("error"); + setStatusMessage(error instanceof Error ? error.message : "发送失败"); + } finally { + flushInFlightRef.current = false; + } + }, + [config, open, prompt, pushMetric] + ); + + const scheduleFlush = useCallback( + (force = false) => { + dirtyRef.current = true; + if (force) { + if (pendingFlushTimerRef.current) { + window.clearTimeout(pendingFlushTimerRef.current); + pendingFlushTimerRef.current = null; + } + void flushFrame(true); + return; + } + if (pendingFlushTimerRef.current) return; + pendingFlushTimerRef.current = window.setTimeout(() => { + pendingFlushTimerRef.current = null; + void flushFrame(); + }, adaptiveThrottleMs); + }, + [adaptiveThrottleMs, flushFrame] + ); + + const initializeCanvas = useCallback(() => { + const canvas = canvasRef.current; + const wrap = canvasWrapRef.current; + if (!canvas || !wrap) return; + const rect = wrap.getBoundingClientRect(); + const dpr = Math.max(1, window.devicePixelRatio || 1); + canvas.width = Math.max(640, Math.floor(rect.width * dpr)); + canvas.height = Math.max(480, Math.floor(rect.height * dpr)); + canvas.style.width = `${Math.floor(rect.width)}px`; + canvas.style.height = `${Math.floor(rect.height)}px`; + const ctx = canvas.getContext("2d"); + if (!ctx) return; + ctx.setTransform(1, 0, 0, 1, 0, 0); + ctx.scale(dpr, dpr); + resetCanvas(); + hasLoadedInitialImageRef.current = false; + }, [resetCanvas]); + + const getCanvasPoint = useCallback((event: ReactPointerEvent) => { + const canvas = canvasRef.current; + if (!canvas) return null; + const rect = canvas.getBoundingClientRect(); + return { + x: event.clientX - rect.left, + y: event.clientY - rect.top, + }; + }, []); + + const drawStroke = useCallback( + (from: { x: number; y: number }, to: { x: number; y: number }) => { + const canvas = canvasRef.current; + if (!canvas) return; + const ctx = canvas.getContext("2d"); + if (!ctx) return; + + ctx.save(); + if (brushMode === "eraser") { + ctx.globalCompositeOperation = "destination-out"; + ctx.strokeStyle = "rgba(0,0,0,1)"; + ctx.lineWidth = brushSize * 1.45; + } else { + ctx.globalCompositeOperation = "source-over"; + ctx.strokeStyle = "#18181b"; + ctx.lineWidth = brushSize; + } + ctx.lineCap = "round"; + ctx.lineJoin = "round"; + ctx.beginPath(); + ctx.moveTo(from.x, from.y); + ctx.lineTo(to.x, to.y); + ctx.stroke(); + ctx.restore(); + }, + [brushMode, brushSize] + ); + + const handlePointerDown = useCallback( + (event: ReactPointerEvent) => { + const point = getCanvasPoint(event); + if (!point) return; + pointerDownRef.current = true; + lastPointRef.current = point; + event.currentTarget.setPointerCapture(event.pointerId); + }, + [getCanvasPoint] + ); + + const handlePointerMove = useCallback( + (event: ReactPointerEvent) => { + if (!pointerDownRef.current) return; + const point = getCanvasPoint(event); + if (!point || !lastPointRef.current) return; + drawStroke(lastPointRef.current, point); + lastPointRef.current = point; + scheduleFlush(); + }, + [drawStroke, getCanvasPoint, scheduleFlush] + ); + + const handlePointerUp = useCallback( + (event: ReactPointerEvent) => { + pointerDownRef.current = false; + lastPointRef.current = null; + try { + event.currentTarget.releasePointerCapture(event.pointerId); + } catch { + // ignore + } + scheduleFlush(true); + }, + [scheduleFlush] + ); + + const uploadPreviewToCanvas = useCallback(async () => { + if (!previewUrl) { + toast.error("请先完成一帧实时预览"); + return; + } + + setIsApplying(true); + try { + const blob = await fetch(previewUrl).then((res) => res.blob()); + const formData = new FormData(); + const filename = `realtime-canvas-${Date.now()}.jpg`; + formData.append("file", new File([blob], filename, { type: blob.type || "image/jpeg" })); + + const response = await fetch("/api/files/upload", { + method: "POST", + body: formData, + }); + const result = await response.json().catch(() => ({})); + + if (!response.ok || !result?.url) { + throw new Error(result?.error || "上传到画布失败"); + } + + onApplyResult({ + imageUrl: String(result.url), + prompt: prompt.trim() || FALLBACK_PROMPT, + }); + + toast.success("已发送到画布"); + onOpenChange(false); + } catch (error) { + toast.error(error instanceof Error ? error.message : "发送到画布失败"); + } finally { + setIsApplying(false); + } + }, [onApplyResult, onOpenChange, previewUrl, prompt]); + + useEffect(() => { + if (!open) return; + initializeCanvas(); + void drawInitialImage().then(() => { + scheduleFlush(true); + }); + const onResize = () => initializeCanvas(); + window.addEventListener("resize", onResize); + return () => { + window.removeEventListener("resize", onResize); + }; + }, [drawInitialImage, initializeCanvas, open, scheduleFlush]); + + useEffect(() => { + if (!open) return; + + let cancelled = false; + setStatus("connecting"); + setStatusMessage("正在初始化实时连接..."); + + const start = async () => { + try { + const configResponse = await fetch("/api/realtime/config", { cache: "no-store" }); + const configJson = (await configResponse.json().catch(() => ({}))) as Partial; + if (!configResponse.ok) { + throw new Error((configJson as any)?.message || "无法加载实时配置"); + } + + if (cancelled) return; + const resolvedConfig: RealtimeConfig = { + transport: { + throttleInterval: + configJson.transport?.throttleInterval || + defaultRealtimeConfig.transport.throttleInterval, + maxBuffering: + configJson.transport?.maxBuffering || defaultRealtimeConfig.transport.maxBuffering, + }, + image: { + maxInputResolution: + configJson.image?.maxInputResolution || defaultRealtimeConfig.image.maxInputResolution, + jpegQuality: configJson.image?.jpegQuality || defaultRealtimeConfig.image.jpegQuality, + }, + model: { + id: configJson.model?.id || defaultRealtimeConfig.model.id, + defaults: { + steps: configJson.model?.defaults?.steps || defaultRealtimeConfig.model.defaults.steps, + scheduleMu: + configJson.model?.defaults?.scheduleMu || + defaultRealtimeConfig.model.defaults.scheduleMu, + feedback: + configJson.model?.defaults?.feedback || + defaultRealtimeConfig.model.defaults.feedback, + imageSize: + configJson.model?.defaults?.imageSize || + defaultRealtimeConfig.model.defaults.imageSize, + }, + }, + }; + + setConfig(resolvedConfig); + setAdaptiveThrottleMs(resolvedConfig.transport.throttleInterval); + + fal.config({ + proxyUrl: "/api/fal/proxy", + }); + + const connection = fal.realtime.connect(resolvedConfig.model.id, { + connectionKey: `canvas-realtime-${sessionIdRef.current}-${refreshToken}`, + throttleInterval: resolvedConfig.transport.throttleInterval, + maxBuffering: resolvedConfig.transport.maxBuffering, + onResult: (result: any) => { + const requestId = String(result?.request_id || ""); + const seqFromRequest = requestSeqMapRef.current.get(requestId) || 0; + const seq = Math.max(seqFromRequest, Number(result?.seq || 0)); + if (seq > 0 && seq < lastRenderedSeqRef.current) { + return; + } + + const latestImage = Array.isArray(result?.images) + ? result.images[result.images.length - 1] + : null; + const nextPreviewUrl = buildImageDataUrl(latestImage); + if (!nextPreviewUrl) return; + + lastRenderedSeqRef.current = Math.max(lastRenderedSeqRef.current, seq); + setPreviewUrl(nextPreviewUrl); + setStatus("live"); + setStatusMessage("实时渲染中"); + + if (requestId) { + const startedAt = requestStartMapRef.current.get(requestId); + if (startedAt) { + pushMetric({ + name: "frame_result", + ts: Date.now(), + seq, + latencyMs: Date.now() - startedAt, + }); + } + requestStartMapRef.current.delete(requestId); + requestSeqMapRef.current.delete(requestId); + } + }, + onError: (error: any) => { + const errorMessage = + error?.body?.message || error?.message || "实时连接异常，请稍后重试"; + setStatus("error"); + setStatusMessage(String(errorMessage)); + if (Number(error?.status) === 429) { + setAdaptiveThrottleMs((prev) => Math.min(240, prev + 40)); + } + pushMetric({ + name: "frame_error", + ts: Date.now(), + message: String(errorMessage), + }); + }, + }); + + if (cancelled) { + connection.close(); + return; + } + + connectionRef.current = connection; + setStatus("connecting"); + setStatusMessage("连接成功，等待首帧..."); + scheduleFlush(true); + } catch (error) { + console.error("[RealtimeCanvas] init failed:", error); + setStatus("error"); + setStatusMessage(error instanceof Error ? error.message : "实时初始化失败"); + } + }; + + void start(); + const metricTimer = window.setInterval(() => { + void flushMetrics(); + }, 5000); + + return () => { + cancelled = true; + if (pendingFlushTimerRef.current) { + window.clearTimeout(pendingFlushTimerRef.current); + pendingFlushTimerRef.current = null; + } + window.clearInterval(metricTimer); + void flushMetrics(); + connectionRef.current?.close?.(); + connectionRef.current = null; + setStatus("idle"); + setStatusMessage("等待连接"); + pointerDownRef.current = false; + lastPointRef.current = null; + }; + }, [flushMetrics, open, pushMetric, refreshToken, scheduleFlush]); + + const forceReconnect = useCallback(() => { + connectionRef.current?.close?.(); + connectionRef.current = null; + setStatus("connecting"); + setStatusMessage("正在重连..."); + setRefreshToken((prev) => prev + 1); + }, []); + + const clearSketch = useCallback(() => { + resetCanvas(); + dirtyRef.current = true; + scheduleFlush(true); + }, [resetCanvas, scheduleFlush]); + + return ( + + + +
+
+ 实时画布 + + 左侧画草图，右侧实时预览，底部输入提示词后持续渲染 + +
+
+ + {status === "error" ? : } + {statusBadge.label} + + +
+
+
{statusMessage}
+ + +
+
+
+
+ 草图画布 +
+ + + +
+
+
+ +
+
+ + setBrushSize(Number(event.target.value))} + className="h-2 w-32 accent-zinc-800" + /> + {brushSize}px +
+
+ +
+
+ 实时预览 + +
+
+ {previewUrl ? ( + 实时预览 + ) : ( +
+ {status === "connecting" ? ( + + + 正在等待首帧... + + ) : ( + 开始绘制后将显示实时预览 + )} +
+ )} +
+
+
+ +
+
+ setPrompt(event.target.value)} + onBlur={() => scheduleFlush(true)} + placeholder="输入提示词，例如：将草图渲染为高端工业设计风格..." + className="h-10 flex-1 rounded-xl border-zinc-300 bg-white" + /> + +
+
+ 提示：绘制过程会自动触发实时渲染。当前发送节流 {adaptiveThrottleMs}ms。 +
+
+
+ + + ); +} diff --git a/lib/db/migrations/0042_add_realtime_canvas_toggle.sql b/lib/db/migrations/0042_add_realtime_canvas_toggle.sql new file mode 100644 index 000000000..851018213 --- /dev/null +++ b/lib/db/migrations/0042_add_realtime_canvas_toggle.sql @@ -0,0 +1,2 @@ +ALTER TABLE "SiteSettings" +ADD COLUMN IF NOT EXISTS "realtimeCanvasEnabled" boolean NOT NULL DEFAULT true; diff --git a/lib/db/migrations/meta/_journal.json b/lib/db/migrations/meta/_journal.json index 3476c8667..696a93bb0 100644 --- a/lib/db/migrations/meta/_journal.json +++ b/lib/db/migrations/meta/_journal.json @@ -197,6 +197,13 @@ "when": 1772265600000, "tag": "0041_add_user_high_vip", "breakpoints": true + }, + { + "idx": 28, + "version": "7", + "when": 1772280000000, + "tag": "0042_add_realtime_canvas_toggle", + "breakpoints": true } ] } diff --git a/lib/db/schema.ts b/lib/db/schema.ts index e08b58c6d..6891e817c 100644 --- a/lib/db/schema.ts +++ b/lib/db/schema.ts @@ -354,6 +354,7 @@ export const siteSettings = pgTable("SiteSettings", { rankingEnabled: boolean("rankingEnabled").notNull().default(true), // 是否显示精选榜按钮 // 无限画布设置 canvasEnabled: boolean("canvasEnabled").notNull().default(true), // 是否启用无限画布功能 + realtimeCanvasEnabled: boolean("realtimeCanvasEnabled").notNull().default(true), // 是否显示实时画布入口 canvasDefaultModelId: varchar("canvasDefaultModelId", { length: 128 }), // 画布默认选择的模型ID（null表示智能自动） canvasAutoModeDefaultModelId: varchar("canvasAutoModeDefaultModelId", { length: 128 }), // 智能自动模式下实际使用的默认生图模型ID canvasDetailDefaultModelId: varchar("canvasDetailDefaultModelId", { length: 128 }), // AI详情生成专用默认生图模型ID