feat(realtime-canvas): switch realtime generation to runware

app/api/realtime/config/route.ts

@@ -3,6 +3,8 @@ import { getSystemSetting } from "@/lib/db/queries";
 
 type RealtimeCanvasConfig = {
   enabled?: boolean;
+  runwareApiKey?: string;
+  runwareModel?: string;
   falApiKey?: string;
   apiKey?: string;
   throttleInterval?: number;
@@ -11,16 +13,14 @@ type RealtimeCanvasConfig = {
   jpegQuality?: number;
   defaults?: {
     steps?: number;
+    cfgScale?: number;
+    strength?: number;
     scheduleMu?: number;
     feedback?: number;
     imageSize?: "square" | "square_hd";
   };
 };
 
-type AngleEditConfig = {
-  falApiKey?: string;
-};
-
 const defaultConfig = {
   session: {
     idleTimeoutMs: 120000,
@@ -34,16 +34,17 @@ const defaultConfig = {
     jpegQuality: 0.55,
   },
   model: {
-    id: "fal-ai/flux-2/klein/realtime",
+    id: "runware:400@4",
     defaults: {
-      steps: 3,
-      scheduleMu: 2.3,
-      feedback: 0.95,
+      steps: 4,
+      cfgScale: 3.5,
+      strength: 0.85,
       imageSize: "square" as const,
     },
     limits: {
-      steps: { min: 1, max: 6 },
-      feedback: { min: 0.7, max: 1.0 },
+      steps: { min: 1, max: 8 },
+      cfgScale: { min: 1, max: 12 },
+      strength: { min: 0.05, max: 1.0 },
     },
   },
   features: {
@@ -62,12 +63,8 @@ export async function GET() {
     return Response.json({ code: "UNAUTHORIZED", message: "请先登录" }, { status: 401 });
   }
 
-  const [setting, angleEditSetting] = await Promise.all([
-    getSystemSetting({ key: "realtime_canvas_config" }).catch(() => null),
-    getSystemSetting({ key: "angle_edit_config" }).catch(() => null),
-  ]);
+  const setting = await getSystemSetting({ key: "realtime_canvas_config" }).catch(() => null);
   const config = (setting?.value || {}) as RealtimeCanvasConfig;
-  const angleEditConfig = (angleEditSetting?.value || {}) as AngleEditConfig;
 
   if (config.enabled === false) {
     return Response.json(
@@ -76,20 +73,20 @@ export async function GET() {
     );
   }
 
-  const falApiKey = String(
-    config.falApiKey ||
+  const runwareApiKey = String(
+    config.runwareApiKey ||
+      config.falApiKey ||
       config.apiKey ||
-      angleEditConfig.falApiKey ||
-      process.env.FAL_KEY ||
-      process.env.FAL_API_KEY ||
+      process.env.RUNWARE_API_KEY ||
+      process.env.RUNWARE_KEY ||
       ""
   ).trim();
 
-  if (!falApiKey) {
+  if (!runwareApiKey) {
     return Response.json(
       {
-        code: "FAL_KEY_MISSING",
-        message: "未配置实时画布 API Key（请在本地AI设置中配置）",
+        code: "RUNWARE_KEY_MISSING",
+        message: "未配置 Runware API Key（请在服务器环境变量或设置中配置）",
       },
       { status: 500 }
     );
@@ -115,21 +112,39 @@ export async function GET() {
     0.35,
     0.82
   );
-  const steps = clamp(Number(config.defaults?.steps || defaultConfig.model.defaults.steps), 1, 6);
-  const scheduleMu = clamp(
-    Number(config.defaults?.scheduleMu || defaultConfig.model.defaults.scheduleMu),
-    1.5,
-    3.5
+  const steps = clamp(
+    Number(config.defaults?.steps || defaultConfig.model.defaults.steps),
+    defaultConfig.model.limits.steps.min,
+    defaultConfig.model.limits.steps.max
   );
-  const feedback = clamp(
-    Number(config.defaults?.feedback || defaultConfig.model.defaults.feedback),
-    0.7,
-    1.0
+  const cfgScale = clamp(
+    Number(
+      config.defaults?.cfgScale ??
+        config.defaults?.scheduleMu ??
+        defaultConfig.model.defaults.cfgScale
+    ),
+    defaultConfig.model.limits.cfgScale.min,
+    defaultConfig.model.limits.cfgScale.max
+  );
+  const strength = clamp(
+    Number(
+      config.defaults?.strength ??
+        config.defaults?.feedback ??
+        defaultConfig.model.defaults.strength
+    ),
+    defaultConfig.model.limits.strength.min,
+    defaultConfig.model.limits.strength.max
   );
   const imageSize = config.defaults?.imageSize === "square_hd" ? "square_hd" : "square";
+  const modelId = String(
+    config.runwareModel || process.env.RUNWARE_MODEL || defaultConfig.model.id
+  ).trim();
 
   return Response.json({
     ...defaultConfig,
+    provider: {
+      type: "runware",
+    },
     transport: {
       throttleInterval,
       maxBuffering,
@@ -140,10 +155,11 @@ export async function GET() {
     },
     model: {
       ...defaultConfig.model,
+      id: modelId || defaultConfig.model.id,
       defaults: {
         steps,
-        scheduleMu,
-        feedback,
+        cfgScale,
+        strength,
         imageSize,
       },
     },

app/api/realtime/generate/route.ts

@@ -0,0 +1,211 @@
+import { auth } from "@/app/(auth)/auth";
+import { getSystemSetting } from "@/lib/db/queries";
+
+type RealtimeCanvasConfig = {
+  enabled?: boolean;
+  runwareApiKey?: string;
+  runwareModel?: string;
+  falApiKey?: string;
+  apiKey?: string;
+  maxInputResolution?: number;
+  defaults?: {
+    steps?: number;
+    cfgScale?: number;
+    strength?: number;
+  };
+};
+
+type GeneratePayload = {
+  requestId?: string;
+  imageUrl?: string;
+  prompt?: string;
+  imageSize?: "square" | "square_hd";
+  steps?: number;
+  cfgScale?: number;
+  strength?: number;
+};
+
+function clamp(value: number, min: number, max: number) {
+  return Math.min(max, Math.max(min, value));
+}
+
+function resolveSquareSize(
+  imageSize: "square" | "square_hd" | undefined,
+  maxInputResolution: number
+) {
+  const preferred = imageSize === "square_hd" ? 1280 : 1024;
+  const capped = Math.min(preferred, maxInputResolution);
+  return clamp(Math.round(capped / 64) * 64, 512, 1536);
+}
+
+function parseRunwareError(payload: any) {
+  if (!payload) return "";
+  if (Array.isArray(payload)) {
+    const match = payload.find(
+      (item) =>
+        typeof item?.error === "string" ||
+        typeof item?.message === "string" ||
+        (Array.isArray(item?.errors) && item.errors.length > 0)
+    );
+    if (!match) return "";
+    if (typeof match.error === "string") return match.error;
+    if (typeof match.message === "string") return match.message;
+    if (Array.isArray(match.errors) && match.errors.length > 0) {
+      const firstError = match.errors[0];
+      if (typeof firstError === "string") return firstError;
+      if (typeof firstError?.message === "string") return firstError.message;
+    }
+    return "";
+  }
+  if (typeof payload?.error === "string") return payload.error;
+  if (typeof payload?.message === "string") return payload.message;
+  return "";
+}
+
+export async function POST(request: Request) {
+  const session = await auth();
+  if (!session?.user?.id) {
+    return Response.json({ code: "UNAUTHORIZED", message: "请先登录" }, { status: 401 });
+  }
+
+  let body: GeneratePayload;
+  try {
+    body = (await request.json()) as GeneratePayload;
+  } catch {
+    return Response.json({ code: "BAD_REQUEST", message: "无效 JSON" }, { status: 400 });
+  }
+
+  const imageUrl = String(body.imageUrl || "").trim();
+  if (!imageUrl) {
+    return Response.json({ code: "BAD_REQUEST", message: "缺少 imageUrl" }, { status: 400 });
+  }
+
+  const setting = await getSystemSetting({ key: "realtime_canvas_config" }).catch(() => null);
+  const config = (setting?.value || {}) as RealtimeCanvasConfig;
+
+  if (config.enabled === false) {
+    return Response.json(
+      { code: "FEATURE_DISABLED", message: "实时画布功能未开启" },
+      { status: 403 }
+    );
+  }
+
+  const runwareApiKey = String(
+    config.runwareApiKey ||
+      config.falApiKey ||
+      config.apiKey ||
+      process.env.RUNWARE_API_KEY ||
+      process.env.RUNWARE_KEY ||
+      ""
+  ).trim();
+  if (!runwareApiKey) {
+    return Response.json(
+      {
+        code: "RUNWARE_KEY_MISSING",
+        message: "未配置 Runware API Key",
+      },
+      { status: 500 }
+    );
+  }
+
+  const model = String(config.runwareModel || process.env.RUNWARE_MODEL || "runware:400@4").trim();
+  const maxInputResolution = clamp(Number(config.maxInputResolution || 1024), 512, 1536);
+  const size = resolveSquareSize(body.imageSize, maxInputResolution);
+  const steps = clamp(
+    Number(body.steps || config.defaults?.steps || 4),
+    1,
+    8
+  );
+  const cfgScale = clamp(
+    Number(body.cfgScale || config.defaults?.cfgScale || 3.5),
+    1,
+    12
+  );
+  const strength = clamp(
+    Number(body.strength || config.defaults?.strength || 0.85),
+    0.05,
+    1
+  );
+
+  const taskUUID = crypto.randomUUID();
+  const runwareRequest = [
+    {
+      taskType: "imageInference",
+      taskUUID,
+      model,
+      positivePrompt: String(body.prompt || "").trim() || "将草图渲染为高质感产品设计图，保持结构和主轮廓一致。",
+      seedImage: imageUrl,
+      width: size,
+      height: size,
+      strength,
+      CFGScale: cfgScale,
+      steps,
+      numberResults: 1,
+      outputType: "dataURI",
+      outputFormat: "JPG",
+    },
+  ];
+
+  const upstreamResponse = await fetch("https://api.runware.ai/v1", {
+    method: "POST",
+    headers: {
+      Authorization: `Bearer ${runwareApiKey}`,
+      "Content-Type": "application/json",
+      Accept: "application/json",
+    },
+    body: JSON.stringify(runwareRequest),
+    cache: "no-store",
+  });
+
+  const upstreamJson = await upstreamResponse.json().catch(() => null);
+  const upstreamError = parseRunwareError(upstreamJson);
+
+  if (!upstreamResponse.ok || upstreamError) {
+    return Response.json(
+      {
+        code: "RUNWARE_REQUEST_FAILED",
+        message: upstreamError || `Runware 请求失败（${upstreamResponse.status}）`,
+      },
+      { status: 502 }
+    );
+  }
+
+  const resultList = Array.isArray(upstreamJson)
+    ? upstreamJson
+    : Array.isArray(upstreamJson?.data)
+      ? upstreamJson.data
+      : [];
+  const imageResult =
+    resultList.find((item: any) => item?.taskUUID === taskUUID && (item?.imageDataURI || item?.imageURL)) ||
+    resultList.find((item: any) => item?.imageDataURI || item?.imageURL);
+
+  const imageDataURI = String(imageResult?.imageDataURI || "").trim();
+  const imageURL = String(imageResult?.imageURL || "").trim();
+  const outputImage = imageDataURI || imageURL;
+
+  if (!outputImage) {
+    return Response.json(
+      { code: "RUNWARE_EMPTY_OUTPUT", message: "Runware 未返回图像" },
+      { status: 502 }
+    );
+  }
+
+  const cost = Number(imageResult?.cost || 0);
+  console.log("[RealtimeRunware] frame success", {
+    userId: session.user.id,
+    requestId: body.requestId || null,
+    taskUUID,
+    model,
+    cost,
+  });
+
+  return Response.json({
+    ok: true,
+    provider: "runware",
+    model,
+    taskUUID,
+    imageDataURI: imageDataURI || undefined,
+    imageUrl: imageURL || undefined,
+    cost,
+  });
+}

components/canvas/RealtimeCanvasDialog.tsx

@@ -9,7 +9,6 @@ import {
   type ChangeEvent,
   type PointerEvent as ReactPointerEvent,
 } from "react";
-import * as fal from "@fal-ai/serverless-client";
 import {
   Dialog,
   DialogContent,
@@ -52,8 +51,8 @@ type RealtimeConfig = {
     id: string;
     defaults: {
       steps: number;
-      scheduleMu: number;
-      feedback: number;
+      cfgScale: number;
+      strength: number;
       imageSize: "square" | "square_hd";
     };
   };
@@ -90,37 +89,16 @@ const defaultRealtimeConfig: RealtimeConfig = {
     jpegQuality: 0.55,
   },
   model: {
-    id: "fal-ai/flux-2/klein/realtime",
+    id: "runware:400@4",
     defaults: {
-      steps: 3,
-      scheduleMu: 2.3,
-      feedback: 0.95,
+      steps: 4,
+      cfgScale: 3.5,
+      strength: 0.85,
       imageSize: "square",
     },
   },
 };
 
-function buildImageDataUrl(image: any) {
-  if (!image) return "";
-  if (typeof image === "string" && image.trim()) {
-    return image;
-  }
-  if (typeof image.url === "string" && image.url.trim()) {
-    return image.url;
-  }
-  if (typeof image.image_url === "string" && image.image_url.trim()) {
-    return image.image_url;
-  }
-  if (typeof image.content === "string" && image.content.trim()) {
-    if (image.content.startsWith("data:")) {
-      return image.content;
-    }
-    const contentType = image.content_type || "image/jpeg";
-    return `data:${contentType};base64,${image.content}`;
-  }
-  return "";
-}
-
 function isShapeMode(mode: "brush" | "eraser" | "rect" | "circle" | "line") {
   return mode === "rect" || mode === "circle" || mode === "line";
 }
@@ -300,25 +278,77 @@ export function RealtimeCanvasDialog({
           canvasBackgroundRef.current
         );
 
-        dirtyRef.current = false;
+        const activeConnection = connectionRef.current;
+        if (!activeConnection) {
+          setStatus("connecting");
+          setStatusMessage("连接重建中，正在重试...");
+          return;
+        }
+
         setStatus("connecting");
         setStatusMessage("正在请求实时生成...");
-        connectionRef.current.send({
-          request_id: requestId,
-          image_url: imageDataUri,
-          prompt: prompt.trim() || FALLBACK_PROMPT,
-          image_size: config.model.defaults.imageSize,
-          num_inference_steps: config.model.defaults.steps,
-          schedule_mu: config.model.defaults.scheduleMu,
-          output_feedback_strength: config.model.defaults.feedback,
+
+        const response = await fetch("/api/realtime/generate", {
+          method: "POST",
+          headers: { "Content-Type": "application/json" },
+          body: JSON.stringify({
+            requestId,
+            imageUrl: imageDataUri,
+            prompt: prompt.trim() || FALLBACK_PROMPT,
+            imageSize: config.model.defaults.imageSize,
+            steps: config.model.defaults.steps,
+            cfgScale: config.model.defaults.cfgScale,
+            strength: config.model.defaults.strength,
+          }),
         });
 
+        const payload = (await response.json().catch(() => ({}))) as {
+          imageUrl?: string;
+          imageDataURI?: string;
+          cost?: number;
+          message?: string;
+          error?: string;
+        };
+
+        if (!response.ok) {
+          throw new Error(payload.message || payload.error || "实时生成失败");
+        }
+
+        const nextPreviewUrl = String(payload.imageDataURI || payload.imageUrl || "");
+        if (!nextPreviewUrl) {
+          throw new Error("实时生成未返回图像");
+        }
+
+        const startedAt = requestStartMapRef.current.get(requestId) || Date.now();
+        const latencyMs = Date.now() - startedAt;
+        requestStartMapRef.current.delete(requestId);
+        requestSeqMapRef.current.delete(requestId);
+
+        if (seq > 0 && seq < lastRenderedSeqRef.current) {
+          return;
+        }
+
+        lastRenderedSeqRef.current = Math.max(lastRenderedSeqRef.current, seq);
+        setPreviewUrl(nextPreviewUrl);
+        setStatus("live");
+        setStatusMessage("实时渲染中");
+        dirtyRef.current = false;
+
         pushMetric({
           name: "frame_send",
           ts: Date.now(),
           seq,
         });
+        pushMetric({
+          name: "frame_result",
+          ts: Date.now(),
+          seq,
+          latencyMs,
+          queueDepth: Number(payload.cost || 0),
+        });
       } catch (error) {
+        requestStartMapRef.current.delete(requestId);
+        requestSeqMapRef.current.delete(requestId);
         console.error("[RealtimeCanvas] flush frame failed:", error);
         setStatus("error");
         setStatusMessage(error instanceof Error ? error.message : "发送失败");
@@ -352,6 +382,17 @@ export function RealtimeCanvasDialog({
     [adaptiveThrottleMs, flushFrame]
   );
 
+  const scheduleFlushRef = useRef(scheduleFlush);
+  const flushMetricsRef = useRef(flushMetrics);
+
+  useEffect(() => {
+    scheduleFlushRef.current = scheduleFlush;
+  }, [scheduleFlush]);
+
+  useEffect(() => {
+    flushMetricsRef.current = flushMetrics;
+  }, [flushMetrics]);
+
   const clearPreviewCanvas = useCallback(() => {
     const previewCanvas = previewCanvasRef.current;
     if (!previewCanvas) return;
@@ -677,7 +718,7 @@ export function RealtimeCanvasDialog({
     initializeCanvas();
     void drawInitialImage().then((hasInitialImage) => {
       if (hasInitialImage) {
-        scheduleFlush(true, false);
+        scheduleFlushRef.current(true, false);
       }
     });
     const onResize = () => initializeCanvas();
@@ -685,7 +726,7 @@ export function RealtimeCanvasDialog({
     return () => {
       window.removeEventListener("resize", onResize);
     };
-  }, [drawInitialImage, initializeCanvas, open, scheduleFlush]);
+  }, [drawInitialImage, initializeCanvas, open]);
 
   useEffect(() => {
     if (!open) return;
@@ -739,12 +780,12 @@ export function RealtimeCanvasDialog({
             id: configJson.model?.id || defaultRealtimeConfig.model.id,
             defaults: {
               steps: configJson.model?.defaults?.steps || defaultRealtimeConfig.model.defaults.steps,
-              scheduleMu:
-                configJson.model?.defaults?.scheduleMu ||
-                defaultRealtimeConfig.model.defaults.scheduleMu,
-              feedback:
-                configJson.model?.defaults?.feedback ||
-                defaultRealtimeConfig.model.defaults.feedback,
+              cfgScale:
+                configJson.model?.defaults?.cfgScale ||
+                defaultRealtimeConfig.model.defaults.cfgScale,
+              strength:
+                configJson.model?.defaults?.strength ||
+                defaultRealtimeConfig.model.defaults.strength,
               imageSize:
                 configJson.model?.defaults?.imageSize ||
                 defaultRealtimeConfig.model.defaults.imageSize,
@@ -755,78 +796,16 @@ export function RealtimeCanvasDialog({
         setConfig(resolvedConfig);
         setAdaptiveThrottleMs(resolvedConfig.transport.throttleInterval);
 
-        fal.config({
-          proxyUrl: "/api/fal/proxy",
-        });
-
-        const connection = fal.realtime.connect(resolvedConfig.model.id, {
-          connectionKey: `canvas-realtime-${sessionIdRef.current}-${refreshToken}`,
-          throttleInterval: resolvedConfig.transport.throttleInterval,
-          maxBuffering: resolvedConfig.transport.maxBuffering,
-          onResult: (result: any) => {
-            const requestId = String(result?.request_id || "");
-            const seqFromRequest = requestSeqMapRef.current.get(requestId) || 0;
-            const seq = Math.max(seqFromRequest, Number(result?.seq || 0));
-            if (seq > 0 && seq < lastRenderedSeqRef.current) {
-              return;
-            }
-
-            const imageGroups = [result?.images, result?.output?.images, result?.data?.images];
-            const firstImageGroup = imageGroups.find((group) => Array.isArray(group)) as any[] | undefined;
-            const latestImage =
-              (firstImageGroup && firstImageGroup[firstImageGroup.length - 1]) ||
-              result?.image ||
-              result?.output?.image ||
-              result?.data?.image ||
-              null;
-            const nextPreviewUrl = buildImageDataUrl(latestImage);
-            if (!nextPreviewUrl) return;
-
-            lastRenderedSeqRef.current = Math.max(lastRenderedSeqRef.current, seq);
-            setPreviewUrl(nextPreviewUrl);
-            setStatus("live");
-            setStatusMessage("实时渲染中");
-
-            if (requestId) {
-              const startedAt = requestStartMapRef.current.get(requestId);
-              if (startedAt) {
-                pushMetric({
-                  name: "frame_result",
-                  ts: Date.now(),
-                  seq,
-                  latencyMs: Date.now() - startedAt,
-                });
-              }
-              requestStartMapRef.current.delete(requestId);
-              requestSeqMapRef.current.delete(requestId);
-            }
-          },
-          onError: (error: any) => {
-            const errorMessage =
-              error?.body?.message || error?.message || "实时连接异常，请稍后重试";
-            setStatus("error");
-            setStatusMessage(String(errorMessage));
-            if (Number(error?.status) === 429) {
-              setAdaptiveThrottleMs((prev) => Math.min(240, prev + 40));
-            }
-            pushMetric({
-              name: "frame_error",
-              ts: Date.now(),
-              message: String(errorMessage),
-            });
-          },
-        });
-
-        if (cancelled) {
-          connection.close();
-          return;
-        }
-
-        connectionRef.current = connection;
-        setStatus("connecting");
-        setStatusMessage("连接成功，等待首帧...");
-        scheduleFlush(true, false);
+        if (cancelled) return;
+        connectionRef.current = {
+          provider: "runware",
+          modelId: resolvedConfig.model.id,
+          reconnectToken: refreshToken,
+        };
+        setStatus("live");
+        setStatusMessage("实时服务已就绪");
       } catch (error) {
+        if (cancelled) return;
         console.error("[RealtimeCanvas] init failed:", error);
         setStatus("error");
         setStatusMessage(error instanceof Error ? error.message : "实时初始化失败");
@@ -835,7 +814,7 @@ export function RealtimeCanvasDialog({
 
     void start();
     const metricTimer = window.setInterval(() => {
-      void flushMetrics();
+      void flushMetricsRef.current();
     }, 5000);
 
     return () => {
@@ -845,7 +824,7 @@ export function RealtimeCanvasDialog({
         pendingFlushTimerRef.current = null;
       }
       window.clearInterval(metricTimer);
-      void flushMetrics();
+      void flushMetricsRef.current();
       connectionRef.current?.close?.();
       connectionRef.current = null;
       setStatus("idle");
@@ -853,10 +832,9 @@ export function RealtimeCanvasDialog({
       pointerDownRef.current = false;
       lastPointRef.current = null;
     };
-  }, [flushMetrics, open, pushMetric, refreshToken, scheduleFlush]);
+  }, [open, pushMetric, refreshToken]);
 
   const forceReconnect = useCallback(() => {
-    connectionRef.current?.close?.();
     connectionRef.current = null;
     setStatus("connecting");
     setStatusMessage("正在重连...");