feat: improve realtime canvas sketch fidelity for runware

2026-02-25 16:27:35 +08:00 · 2026-02-25 16:27:35 +08:00 · 24275d9a3a
commit 24275d9a3a
parent 2c1b33bb6b
3 changed files with 226 additions and 40 deletions
--- a/app/api/realtime/config/route.ts
+++ b/app/api/realtime/config/route.ts
@ -7,6 +7,7 @@ type RealtimeCanvasConfig = {
  runwareModel?: string;
  falApiKey?: string;
  apiKey?: string;
+  strictSketch?: boolean;
  throttleInterval?: number;
  maxBuffering?: number;
  maxInputResolution?: number;
@ -37,13 +38,13 @@ const defaultConfig = {
    id: "runware:101@1",
    defaults: {
      steps: 8,
-      cfgScale: 6,
-      strength: 0.4,
+      cfgScale: 7,
+      strength: 0.32,
      imageSize: "square" as const,
    },
    limits: {
-      steps: { min: 1, max: 8 },
-      cfgScale: { min: 1, max: 12 },
+      steps: { min: 1, max: 12 },
+      cfgScale: { min: 1, max: 14 },
      strength: { min: 0.05, max: 1.0 },
    },
  },
@ -160,5 +161,8 @@ export async function GET() {
        imageSize,
      },
    },
+    behavior: {
+      strictSketch: config.strictSketch !== false,
+    },
  });
 }
--- a/app/api/realtime/generate/route.ts
+++ b/app/api/realtime/generate/route.ts
@ -7,6 +7,8 @@ type RealtimeCanvasConfig = {
  runwareModel?: string;
  falApiKey?: string;
  apiKey?: string;
+  negativePrompt?: string;
+  strictSketch?: boolean;
  maxInputResolution?: number;
  defaults?: {
    steps?: number;
@ -19,6 +21,8 @@ type GeneratePayload = {
  requestId?: string;
  imageUrl?: string;
  prompt?: string;
+  seed?: number;
+  strictSketch?: boolean;
  imageSize?: "square" | "square_hd";
  steps?: number;
  cfgScale?: number;
@ -62,6 +66,14 @@ function parseRunwareError(payload: any) {
  return "";
 }

+function resolveSeed(seed: unknown) {
+  const parsed = Number(seed);
+  if (!Number.isFinite(parsed)) return undefined;
+  const normalized = Math.floor(parsed);
+  if (normalized < 0 || normalized > 2147483647) return undefined;
+  return normalized;
+}
+
 export async function POST(request: Request) {
  const session = await auth();
  if (!session?.user?.id) {
@ -117,40 +129,50 @@ export async function POST(request: Request) {
  const steps = clamp(
    Number(body.steps || config.defaults?.steps || 8),
    1,
-    8
-  );
-  const cfgScale = clamp(
-    Number(body.cfgScale || config.defaults?.cfgScale || 6),
-    1,
    12
  );
+  const cfgScale = clamp(
+    Number(body.cfgScale || config.defaults?.cfgScale || 7),
+    1,
+    14
+  );
  const strength = clamp(
-    Number(body.strength || config.defaults?.strength || 0.4),
+    Number(body.strength || config.defaults?.strength || 0.32),
    0.05,
    1
  );
+  const strictSketch = Boolean(body.strictSketch ?? config.strictSketch ?? true);
+  const seed = resolveSeed(body.seed);

  const positivePrompt =
    String(body.prompt || "").trim() ||
    "严格按照输入草图生成：保持主体轮廓、位置、比例和数量一致，只做材质与光影细化；禁止改变构图，禁止新增无关背景纹理。";
+  const negativePrompt =
+    String(config.negativePrompt || "").trim() ||
+    "改变构图, 改变主体位置, 多余人物, 多余肢体, 重复物体, 镜像翻转, 错误透视, 模糊轮廓, 文字水印, 大面积背景噪点";

  const requestInference = async (modelId: string) => {
    const taskUUID = crypto.randomUUID();
+    const runtimeStrength = strictSketch ? clamp(strength, 0.12, 0.55) : strength;
+    const runtimeCfgScale = strictSketch ? clamp(cfgScale, 6, 14) : cfgScale;
    const runwareRequest = [
      {
        taskType: "imageInference",
        taskUUID,
        model: modelId,
        positivePrompt,
+        negativePrompt,
        seedImage: imageUrl,
        width: size,
        height: size,
-        strength,
-        CFGScale: cfgScale,
+        strength: runtimeStrength,
+        CFGScale: runtimeCfgScale,
        steps,
+        includeCost: true,
        numberResults: 1,
        outputType: "dataURI",
        outputFormat: "JPG",
+        ...(typeof seed === "number" ? { seed } : {}),
      },
    ];

@ -171,6 +193,8 @@ export async function POST(request: Request) {
    return {
      taskUUID,
      modelId,
+      runtimeStrength,
+      runtimeCfgScale,
      upstreamResponse,
      upstreamJson,
      upstreamError,
@ -229,6 +253,10 @@ export async function POST(request: Request) {
    requestId: body.requestId || null,
    taskUUID: requestResult.taskUUID,
    model: requestResult.modelId,
+    seed: typeof seed === "number" ? seed : null,
+    steps,
+    cfgScale: requestResult.runtimeCfgScale,
+    strength: requestResult.runtimeStrength,
    cost,
  });

--- a/components/canvas/RealtimeCanvasDialog.tsx
+++ b/components/canvas/RealtimeCanvasDialog.tsx
@ -56,6 +56,9 @@ type RealtimeConfig = {
      imageSize: "square" | "square_hd";
    };
  };
+  behavior?: {
+    strictSketch?: boolean;
+  };
 };

 type RealtimeStatus = "idle" | "connecting" | "live" | "error";
@ -93,17 +96,24 @@ const defaultRealtimeConfig: RealtimeConfig = {
    id: "runware:101@1",
    defaults: {
      steps: 8,
-      cfgScale: 6,
-      strength: 0.4,
+      cfgScale: 7,
+      strength: 0.32,
      imageSize: "square",
    },
  },
+  behavior: {
+    strictSketch: true,
+  },
 };

 function isShapeMode(mode: "brush" | "eraser" | "rect" | "circle" | "line") {
  return mode === "rect" || mode === "circle" || mode === "line";
 }

+function createSessionSeed() {
+  return Math.floor(Math.random() * 2147483647);
+}
+
 async function canvasToJpegDataUri(
  canvas: HTMLCanvasElement,
  maxInputResolution: number,
@ -113,6 +123,14 @@ async function canvasToJpegDataUri(
    autoCrop?: boolean;
  }
 ) {
+  type CropBox = {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+    foregroundRatio: number;
+  };
+
  const parseColor = (rawColor: string) => {
    const tempCanvas = document.createElement("canvas");
    tempCanvas.width = 1;
@ -154,26 +172,39 @@ async function canvasToJpegDataUri(
    return { r: 255, g: 255, b: 255 };
  };

-  const findForegroundBox = () => {
-    const sourceCtx = canvas.getContext("2d", { willReadFrequently: true });
-    if (!sourceCtx) return null;
-    const { width, height } = canvas;
-    const imageData = sourceCtx.getImageData(0, 0, width, height);
-    const data = imageData.data;
-    const rowCounts = new Uint32Array(height);
-    const colCounts = new Uint32Array(width);
-    const bg = parseColor(backgroundColor);
-    const tolerance = 16;
+  const findForegroundBox = (): CropBox | null => {
+    const sourceWidth = canvas.width;
+    const sourceHeight = canvas.height;
+    const longestSide = Math.max(sourceWidth, sourceHeight);
+    const analysisScale = longestSide > 320 ? 320 / longestSide : 1;
+    const analysisWidth = Math.max(48, Math.round(sourceWidth * analysisScale));
+    const analysisHeight = Math.max(48, Math.round(sourceHeight * analysisScale));
+    const analysisCanvas = document.createElement("canvas");
+    analysisCanvas.width = analysisWidth;
+    analysisCanvas.height = analysisHeight;
+    const analysisCtx = analysisCanvas.getContext("2d", { willReadFrequently: true });
+    if (!analysisCtx) return null;
+    analysisCtx.fillStyle = backgroundColor;
+    analysisCtx.fillRect(0, 0, analysisWidth, analysisHeight);
+    analysisCtx.drawImage(canvas, 0, 0, analysisWidth, analysisHeight);

-    let minX = width;
-    let minY = height;
+    const imageData = analysisCtx.getImageData(0, 0, analysisWidth, analysisHeight);
+    const data = imageData.data;
+    const totalPixels = analysisWidth * analysisHeight;
+    const mask = new Uint8Array(totalPixels);
+    const bg = parseColor(backgroundColor);
+    const tolerance = 18;
+
+    let minX = analysisWidth;
+    let minY = analysisHeight;
    let maxX = -1;
    let maxY = -1;
    let foregroundPixels = 0;

-    for (let y = 0; y < height; y += 1) {
-      for (let x = 0; x < width; x += 1) {
-        const offset = (y * width + x) * 4;
+    for (let y = 0; y < analysisHeight; y += 1) {
+      for (let x = 0; x < analysisWidth; x += 1) {
+        const pixelIndex = y * analysisWidth + x;
+        const offset = pixelIndex * 4;
        const alpha = data[offset + 3];
        if (alpha < 16) continue;
        const dr = Math.abs(data[offset] - bg.r);
@ -182,9 +213,8 @@ async function canvasToJpegDataUri(
        const isForeground = dr > tolerance || dg > tolerance || db > tolerance;
        if (!isForeground) continue;

+        mask[pixelIndex] = 1;
        foregroundPixels += 1;
-        rowCounts[y] += 1;
-        colCounts[x] += 1;
        if (x < minX) minX = x;
        if (x > maxX) maxX = x;
        if (y < minY) minY = y;
@ -196,7 +226,85 @@ async function canvasToJpegDataUri(
      return null;
    }

-    const trimPixels = Math.max(0, Math.floor(foregroundPixels * 0.01));
+    const visited = new Uint8Array(totalPixels);
+    const queue = new Uint32Array(totalPixels);
+    const minComponentArea = Math.max(9, Math.floor(foregroundPixels * 0.0015));
+    let bestArea = 0;
+    let bestMinX = minX;
+    let bestMinY = minY;
+    let bestMaxX = maxX;
+    let bestMaxY = maxY;
+
+    for (let idx = 0; idx < totalPixels; idx += 1) {
+      if (!mask[idx] || visited[idx]) continue;
+      let head = 0;
+      let tail = 0;
+      queue[tail] = idx;
+      tail += 1;
+      visited[idx] = 1;
+      let area = 0;
+      let componentMinX = analysisWidth;
+      let componentMinY = analysisHeight;
+      let componentMaxX = -1;
+      let componentMaxY = -1;
+
+      while (head < tail) {
+        const current = queue[head];
+        head += 1;
+        const x = current % analysisWidth;
+        const y = Math.floor(current / analysisWidth);
+        area += 1;
+        if (x < componentMinX) componentMinX = x;
+        if (x > componentMaxX) componentMaxX = x;
+        if (y < componentMinY) componentMinY = y;
+        if (y > componentMaxY) componentMaxY = y;
+
+        const startY = Math.max(0, y - 1);
+        const endY = Math.min(analysisHeight - 1, y + 1);
+        const startX = Math.max(0, x - 1);
+        const endX = Math.min(analysisWidth - 1, x + 1);
+
+        for (let ny = startY; ny <= endY; ny += 1) {
+          for (let nx = startX; nx <= endX; nx += 1) {
+            const neighbor = ny * analysisWidth + nx;
+            if (!mask[neighbor] || visited[neighbor]) continue;
+            visited[neighbor] = 1;
+            queue[tail] = neighbor;
+            tail += 1;
+          }
+        }
+      }
+
+      if (area >= minComponentArea && area > bestArea) {
+        bestArea = area;
+        bestMinX = componentMinX;
+        bestMinY = componentMinY;
+        bestMaxX = componentMaxX;
+        bestMaxY = componentMaxY;
+      }
+    }
+
+    if (bestArea > 0) {
+      minX = bestMinX;
+      minY = bestMinY;
+      maxX = bestMaxX;
+      maxY = bestMaxY;
+    } else {
+      bestArea = foregroundPixels;
+    }
+
+    const rowCounts = new Uint32Array(analysisHeight);
+    const colCounts = new Uint32Array(analysisWidth);
+    for (let y = minY; y <= maxY; y += 1) {
+      for (let x = minX; x <= maxX; x += 1) {
+        const idx = y * analysisWidth + x;
+        if (!mask[idx]) continue;
+        rowCounts[y] += 1;
+        colCounts[x] += 1;
+      }
+    }
+
+    const trimPixels = Math.max(0, Math.floor(bestArea * 0.008));
    if (trimPixels > 0) {
      let cumulative = 0;
      for (let x = minX; x <= maxX; x += 1) {
@ -235,19 +343,29 @@ async function canvasToJpegDataUri(
      }
    }

-    const boxWidth = Math.max(1, maxX - minX + 1);
-    const boxHeight = Math.max(1, maxY - minY + 1);
-    const padding = Math.max(20, Math.floor(Math.max(boxWidth, boxHeight) * 0.24));
-    const x = Math.max(0, minX - padding);
-    const y = Math.max(0, minY - padding);
-    const right = Math.min(width, maxX + 1 + padding);
-    const bottom = Math.min(height, maxY + 1 + padding);
+    const scaleX = sourceWidth / analysisWidth;
+    const scaleY = sourceHeight / analysisHeight;
+    const boxX = Math.max(0, Math.floor(minX * scaleX));
+    const boxY = Math.max(0, Math.floor(minY * scaleY));
+    const boxRight = Math.min(sourceWidth, Math.ceil((maxX + 1) * scaleX));
+    const boxBottom = Math.min(sourceHeight, Math.ceil((maxY + 1) * scaleY));
+
+    const boxWidth = Math.max(1, boxRight - boxX);
+    const boxHeight = Math.max(1, boxBottom - boxY);
+    const foregroundRatio = Math.min(1, bestArea / totalPixels);
+    const paddingRatio = foregroundRatio < 0.08 ? 0.34 : foregroundRatio < 0.2 ? 0.28 : 0.22;
+    const padding = Math.max(16, Math.floor(Math.max(boxWidth, boxHeight) * paddingRatio));
+    const x = Math.max(0, boxX - padding);
+    const y = Math.max(0, boxY - padding);
+    const right = Math.min(sourceWidth, boxRight + padding);
+    const bottom = Math.min(sourceHeight, boxBottom + padding);

    return {
      x,
      y,
      width: Math.max(1, right - x),
      height: Math.max(1, bottom - y),
+      foregroundRatio,
    };
  };

@ -281,11 +399,14 @@ async function canvasToJpegDataUri(
  exportCtx.fillRect(0, 0, exportWidth, exportHeight);

  if (autoCropEnabled) {
-    const fitScale = Math.min((exportWidth * 0.88) / drawSourceWidth, (exportHeight * 0.88) / drawSourceHeight);
+    const fitRatio = (cropBox?.foregroundRatio || 0) < 0.12 ? 0.9 : 0.94;
+    const fitScale = Math.min((exportWidth * fitRatio) / drawSourceWidth, (exportHeight * fitRatio) / drawSourceHeight);
    const targetWidth = Math.max(1, Math.round(drawSourceWidth * fitScale));
    const targetHeight = Math.max(1, Math.round(drawSourceHeight * fitScale));
    const targetX = Math.round((exportWidth - targetWidth) / 2);
    const targetY = Math.round((exportHeight - targetHeight) / 2);
+    exportCtx.imageSmoothingEnabled = true;
+    exportCtx.imageSmoothingQuality = "high";
    exportCtx.drawImage(
      canvas,
      drawSourceX,
@ -297,6 +418,22 @@ async function canvasToJpegDataUri(
      targetWidth,
      targetHeight
    );
+    if ((cropBox?.foregroundRatio || 0) < 0.28) {
+      exportCtx.save();
+      exportCtx.globalAlpha = 0.22;
+      exportCtx.drawImage(
+        canvas,
+        drawSourceX,
+        drawSourceY,
+        drawSourceWidth,
+        drawSourceHeight,
+        targetX,
+        targetY,
+        targetWidth,
+        targetHeight
+      );
+      exportCtx.restore();
+    }
  } else {
    exportCtx.drawImage(
      canvas,
@ -348,6 +485,7 @@ export function RealtimeCanvasDialog({
  const lastRenderedSeqRef = useRef(0);
  const metricsQueueRef = useRef<any[]>([]);
  const sessionIdRef = useRef(`rt-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`);
+  const seedRef = useRef<number>(createSessionSeed());
  const hasLoadedInitialImageRef = useRef(false);
  const canvasBackgroundRef = useRef(CANVAS_BACKGROUND);
  
@ -479,6 +617,8 @@ export function RealtimeCanvasDialog({
            requestId,
            imageUrl: imageDataUri,
            prompt: prompt.trim() || FALLBACK_PROMPT,
+            seed: seedRef.current,
+            strictSketch: config.behavior?.strictSketch !== false,
            imageSize: config.model.defaults.imageSize,
            steps: config.model.defaults.steps,
            cfgScale: config.model.defaults.cfgScale,
@ -897,6 +1037,17 @@ export function RealtimeCanvasDialog({
    return () => window.removeEventListener('keydown', handleKeyDown);
  }, [open, undo, redo]);

+  useEffect(() => {
+    if (!open) return;
+    seedRef.current = createSessionSeed();
+    sessionIdRef.current = `rt-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+    seqRef.current = 0;
+    lastRenderedSeqRef.current = 0;
+    requestSeqMapRef.current.clear();
+    requestStartMapRef.current.clear();
+    metricsQueueRef.current = [];
+  }, [open]);
+
  useEffect(() => {
    if (!open) return;
    initializeCanvas();
@ -975,6 +1126,9 @@ export function RealtimeCanvasDialog({
                defaultRealtimeConfig.model.defaults.imageSize,
            },
          },
+          behavior: {
+            strictSketch: configJson.behavior?.strictSketch !== false,
+          },
        };

        setConfig(resolvedConfig);