From 24275d9a3a652a8c7297a75a3cfa1e7d508ffcfd Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 25 Feb 2026 16:27:35 +0800 Subject: [PATCH] feat: improve realtime canvas sketch fidelity for runware --- app/api/realtime/config/route.ts | 12 +- app/api/realtime/generate/route.ts | 44 ++++- components/canvas/RealtimeCanvasDialog.tsx | 210 ++++++++++++++++++--- 3 files changed, 226 insertions(+), 40 deletions(-) diff --git a/app/api/realtime/config/route.ts b/app/api/realtime/config/route.ts index 148ffa6f6..150efce1a 100644 --- a/app/api/realtime/config/route.ts +++ b/app/api/realtime/config/route.ts @@ -7,6 +7,7 @@ type RealtimeCanvasConfig = { runwareModel?: string; falApiKey?: string; apiKey?: string; + strictSketch?: boolean; throttleInterval?: number; maxBuffering?: number; maxInputResolution?: number; @@ -37,13 +38,13 @@ const defaultConfig = { id: "runware:101@1", defaults: { steps: 8, - cfgScale: 6, - strength: 0.4, + cfgScale: 7, + strength: 0.32, imageSize: "square" as const, }, limits: { - steps: { min: 1, max: 8 }, - cfgScale: { min: 1, max: 12 }, + steps: { min: 1, max: 12 }, + cfgScale: { min: 1, max: 14 }, strength: { min: 0.05, max: 1.0 }, }, }, @@ -160,5 +161,8 @@ export async function GET() { imageSize, }, }, + behavior: { + strictSketch: config.strictSketch !== false, + }, }); } diff --git a/app/api/realtime/generate/route.ts b/app/api/realtime/generate/route.ts index 1bbe087a4..c899623ae 100644 --- a/app/api/realtime/generate/route.ts +++ b/app/api/realtime/generate/route.ts @@ -7,6 +7,8 @@ type RealtimeCanvasConfig = { runwareModel?: string; falApiKey?: string; apiKey?: string; + negativePrompt?: string; + strictSketch?: boolean; maxInputResolution?: number; defaults?: { steps?: number; @@ -19,6 +21,8 @@ type GeneratePayload = { requestId?: string; imageUrl?: string; prompt?: string; + seed?: number; + strictSketch?: boolean; imageSize?: "square" | "square_hd"; steps?: number; cfgScale?: number; @@ -62,6 +66,14 @@ function parseRunwareError(payload: any) { return ""; } +function resolveSeed(seed: unknown) { + const parsed = Number(seed); + if (!Number.isFinite(parsed)) return undefined; + const normalized = Math.floor(parsed); + if (normalized < 0 || normalized > 2147483647) return undefined; + return normalized; +} + export async function POST(request: Request) { const session = await auth(); if (!session?.user?.id) { @@ -117,40 +129,50 @@ export async function POST(request: Request) { const steps = clamp( Number(body.steps || config.defaults?.steps || 8), 1, - 8 - ); - const cfgScale = clamp( - Number(body.cfgScale || config.defaults?.cfgScale || 6), - 1, 12 ); + const cfgScale = clamp( + Number(body.cfgScale || config.defaults?.cfgScale || 7), + 1, + 14 + ); const strength = clamp( - Number(body.strength || config.defaults?.strength || 0.4), + Number(body.strength || config.defaults?.strength || 0.32), 0.05, 1 ); + const strictSketch = Boolean(body.strictSketch ?? config.strictSketch ?? true); + const seed = resolveSeed(body.seed); const positivePrompt = String(body.prompt || "").trim() || "严格按照输入草图生成：保持主体轮廓、位置、比例和数量一致，只做材质与光影细化；禁止改变构图，禁止新增无关背景纹理。"; + const negativePrompt = + String(config.negativePrompt || "").trim() || + "改变构图, 改变主体位置, 多余人物, 多余肢体, 重复物体, 镜像翻转, 错误透视, 模糊轮廓, 文字水印, 大面积背景噪点"; const requestInference = async (modelId: string) => { const taskUUID = crypto.randomUUID(); + const runtimeStrength = strictSketch ? clamp(strength, 0.12, 0.55) : strength; + const runtimeCfgScale = strictSketch ? clamp(cfgScale, 6, 14) : cfgScale; const runwareRequest = [ { taskType: "imageInference", taskUUID, model: modelId, positivePrompt, + negativePrompt, seedImage: imageUrl, width: size, height: size, - strength, - CFGScale: cfgScale, + strength: runtimeStrength, + CFGScale: runtimeCfgScale, steps, + includeCost: true, numberResults: 1, outputType: "dataURI", outputFormat: "JPG", + ...(typeof seed === "number" ? { seed } : {}), }, ]; @@ -171,6 +193,8 @@ export async function POST(request: Request) { return { taskUUID, modelId, + runtimeStrength, + runtimeCfgScale, upstreamResponse, upstreamJson, upstreamError, @@ -229,6 +253,10 @@ export async function POST(request: Request) { requestId: body.requestId || null, taskUUID: requestResult.taskUUID, model: requestResult.modelId, + seed: typeof seed === "number" ? seed : null, + steps, + cfgScale: requestResult.runtimeCfgScale, + strength: requestResult.runtimeStrength, cost, }); diff --git a/components/canvas/RealtimeCanvasDialog.tsx b/components/canvas/RealtimeCanvasDialog.tsx index 4c49a6cbd..303c5e80c 100644 --- a/components/canvas/RealtimeCanvasDialog.tsx +++ b/components/canvas/RealtimeCanvasDialog.tsx @@ -56,6 +56,9 @@ type RealtimeConfig = { imageSize: "square" | "square_hd"; }; }; + behavior?: { + strictSketch?: boolean; + }; }; type RealtimeStatus = "idle" | "connecting" | "live" | "error"; @@ -93,17 +96,24 @@ const defaultRealtimeConfig: RealtimeConfig = { id: "runware:101@1", defaults: { steps: 8, - cfgScale: 6, - strength: 0.4, + cfgScale: 7, + strength: 0.32, imageSize: "square", }, }, + behavior: { + strictSketch: true, + }, }; function isShapeMode(mode: "brush" | "eraser" | "rect" | "circle" | "line") { return mode === "rect" || mode === "circle" || mode === "line"; } +function createSessionSeed() { + return Math.floor(Math.random() * 2147483647); +} + async function canvasToJpegDataUri( canvas: HTMLCanvasElement, maxInputResolution: number, @@ -113,6 +123,14 @@ async function canvasToJpegDataUri( autoCrop?: boolean; } ) { + type CropBox = { + x: number; + y: number; + width: number; + height: number; + foregroundRatio: number; + }; + const parseColor = (rawColor: string) => { const tempCanvas = document.createElement("canvas"); tempCanvas.width = 1; @@ -154,26 +172,39 @@ async function canvasToJpegDataUri( return { r: 255, g: 255, b: 255 }; }; - const findForegroundBox = () => { - const sourceCtx = canvas.getContext("2d", { willReadFrequently: true }); - if (!sourceCtx) return null; - const { width, height } = canvas; - const imageData = sourceCtx.getImageData(0, 0, width, height); - const data = imageData.data; - const rowCounts = new Uint32Array(height); - const colCounts = new Uint32Array(width); - const bg = parseColor(backgroundColor); - const tolerance = 16; + const findForegroundBox = (): CropBox | null => { + const sourceWidth = canvas.width; + const sourceHeight = canvas.height; + const longestSide = Math.max(sourceWidth, sourceHeight); + const analysisScale = longestSide > 320 ? 320 / longestSide : 1; + const analysisWidth = Math.max(48, Math.round(sourceWidth * analysisScale)); + const analysisHeight = Math.max(48, Math.round(sourceHeight * analysisScale)); + const analysisCanvas = document.createElement("canvas"); + analysisCanvas.width = analysisWidth; + analysisCanvas.height = analysisHeight; + const analysisCtx = analysisCanvas.getContext("2d", { willReadFrequently: true }); + if (!analysisCtx) return null; + analysisCtx.fillStyle = backgroundColor; + analysisCtx.fillRect(0, 0, analysisWidth, analysisHeight); + analysisCtx.drawImage(canvas, 0, 0, analysisWidth, analysisHeight); - let minX = width; - let minY = height; + const imageData = analysisCtx.getImageData(0, 0, analysisWidth, analysisHeight); + const data = imageData.data; + const totalPixels = analysisWidth * analysisHeight; + const mask = new Uint8Array(totalPixels); + const bg = parseColor(backgroundColor); + const tolerance = 18; + + let minX = analysisWidth; + let minY = analysisHeight; let maxX = -1; let maxY = -1; let foregroundPixels = 0; - for (let y = 0; y < height; y += 1) { - for (let x = 0; x < width; x += 1) { - const offset = (y * width + x) * 4; + for (let y = 0; y < analysisHeight; y += 1) { + for (let x = 0; x < analysisWidth; x += 1) { + const pixelIndex = y * analysisWidth + x; + const offset = pixelIndex * 4; const alpha = data[offset + 3]; if (alpha < 16) continue; const dr = Math.abs(data[offset] - bg.r); @@ -182,9 +213,8 @@ async function canvasToJpegDataUri( const isForeground = dr > tolerance || dg > tolerance || db > tolerance; if (!isForeground) continue; + mask[pixelIndex] = 1; foregroundPixels += 1; - rowCounts[y] += 1; - colCounts[x] += 1; if (x < minX) minX = x; if (x > maxX) maxX = x; if (y < minY) minY = y; @@ -196,7 +226,85 @@ async function canvasToJpegDataUri( return null; } - const trimPixels = Math.max(0, Math.floor(foregroundPixels * 0.01)); + const visited = new Uint8Array(totalPixels); + const queue = new Uint32Array(totalPixels); + const minComponentArea = Math.max(9, Math.floor(foregroundPixels * 0.0015)); + let bestArea = 0; + let bestMinX = minX; + let bestMinY = minY; + let bestMaxX = maxX; + let bestMaxY = maxY; + + for (let idx = 0; idx < totalPixels; idx += 1) { + if (!mask[idx] || visited[idx]) continue; + let head = 0; + let tail = 0; + queue[tail] = idx; + tail += 1; + visited[idx] = 1; + let area = 0; + let componentMinX = analysisWidth; + let componentMinY = analysisHeight; + let componentMaxX = -1; + let componentMaxY = -1; + + while (head < tail) { + const current = queue[head]; + head += 1; + const x = current % analysisWidth; + const y = Math.floor(current / analysisWidth); + area += 1; + if (x < componentMinX) componentMinX = x; + if (x > componentMaxX) componentMaxX = x; + if (y < componentMinY) componentMinY = y; + if (y > componentMaxY) componentMaxY = y; + + const startY = Math.max(0, y - 1); + const endY = Math.min(analysisHeight - 1, y + 1); + const startX = Math.max(0, x - 1); + const endX = Math.min(analysisWidth - 1, x + 1); + + for (let ny = startY; ny <= endY; ny += 1) { + for (let nx = startX; nx <= endX; nx += 1) { + const neighbor = ny * analysisWidth + nx; + if (!mask[neighbor] || visited[neighbor]) continue; + visited[neighbor] = 1; + queue[tail] = neighbor; + tail += 1; + } + } + } + + if (area >= minComponentArea && area > bestArea) { + bestArea = area; + bestMinX = componentMinX; + bestMinY = componentMinY; + bestMaxX = componentMaxX; + bestMaxY = componentMaxY; + } + } + + if (bestArea > 0) { + minX = bestMinX; + minY = bestMinY; + maxX = bestMaxX; + maxY = bestMaxY; + } else { + bestArea = foregroundPixels; + } + + const rowCounts = new Uint32Array(analysisHeight); + const colCounts = new Uint32Array(analysisWidth); + for (let y = minY; y <= maxY; y += 1) { + for (let x = minX; x <= maxX; x += 1) { + const idx = y * analysisWidth + x; + if (!mask[idx]) continue; + rowCounts[y] += 1; + colCounts[x] += 1; + } + } + + const trimPixels = Math.max(0, Math.floor(bestArea * 0.008)); if (trimPixels > 0) { let cumulative = 0; for (let x = minX; x <= maxX; x += 1) { @@ -235,19 +343,29 @@ async function canvasToJpegDataUri( } } - const boxWidth = Math.max(1, maxX - minX + 1); - const boxHeight = Math.max(1, maxY - minY + 1); - const padding = Math.max(20, Math.floor(Math.max(boxWidth, boxHeight) * 0.24)); - const x = Math.max(0, minX - padding); - const y = Math.max(0, minY - padding); - const right = Math.min(width, maxX + 1 + padding); - const bottom = Math.min(height, maxY + 1 + padding); + const scaleX = sourceWidth / analysisWidth; + const scaleY = sourceHeight / analysisHeight; + const boxX = Math.max(0, Math.floor(minX * scaleX)); + const boxY = Math.max(0, Math.floor(minY * scaleY)); + const boxRight = Math.min(sourceWidth, Math.ceil((maxX + 1) * scaleX)); + const boxBottom = Math.min(sourceHeight, Math.ceil((maxY + 1) * scaleY)); + + const boxWidth = Math.max(1, boxRight - boxX); + const boxHeight = Math.max(1, boxBottom - boxY); + const foregroundRatio = Math.min(1, bestArea / totalPixels); + const paddingRatio = foregroundRatio < 0.08 ? 0.34 : foregroundRatio < 0.2 ? 0.28 : 0.22; + const padding = Math.max(16, Math.floor(Math.max(boxWidth, boxHeight) * paddingRatio)); + const x = Math.max(0, boxX - padding); + const y = Math.max(0, boxY - padding); + const right = Math.min(sourceWidth, boxRight + padding); + const bottom = Math.min(sourceHeight, boxBottom + padding); return { x, y, width: Math.max(1, right - x), height: Math.max(1, bottom - y), + foregroundRatio, }; }; @@ -281,11 +399,14 @@ async function canvasToJpegDataUri( exportCtx.fillRect(0, 0, exportWidth, exportHeight); if (autoCropEnabled) { - const fitScale = Math.min((exportWidth * 0.88) / drawSourceWidth, (exportHeight * 0.88) / drawSourceHeight); + const fitRatio = (cropBox?.foregroundRatio || 0) < 0.12 ? 0.9 : 0.94; + const fitScale = Math.min((exportWidth * fitRatio) / drawSourceWidth, (exportHeight * fitRatio) / drawSourceHeight); const targetWidth = Math.max(1, Math.round(drawSourceWidth * fitScale)); const targetHeight = Math.max(1, Math.round(drawSourceHeight * fitScale)); const targetX = Math.round((exportWidth - targetWidth) / 2); const targetY = Math.round((exportHeight - targetHeight) / 2); + exportCtx.imageSmoothingEnabled = true; + exportCtx.imageSmoothingQuality = "high"; exportCtx.drawImage( canvas, drawSourceX, @@ -297,6 +418,22 @@ async function canvasToJpegDataUri( targetWidth, targetHeight ); + if ((cropBox?.foregroundRatio || 0) < 0.28) { + exportCtx.save(); + exportCtx.globalAlpha = 0.22; + exportCtx.drawImage( + canvas, + drawSourceX, + drawSourceY, + drawSourceWidth, + drawSourceHeight, + targetX, + targetY, + targetWidth, + targetHeight + ); + exportCtx.restore(); + } } else { exportCtx.drawImage( canvas, @@ -348,6 +485,7 @@ export function RealtimeCanvasDialog({ const lastRenderedSeqRef = useRef(0); const metricsQueueRef = useRef([]); const sessionIdRef = useRef(`rt-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`); + const seedRef = useRef(createSessionSeed()); const hasLoadedInitialImageRef = useRef(false); const canvasBackgroundRef = useRef(CANVAS_BACKGROUND); @@ -479,6 +617,8 @@ export function RealtimeCanvasDialog({ requestId, imageUrl: imageDataUri, prompt: prompt.trim() || FALLBACK_PROMPT, + seed: seedRef.current, + strictSketch: config.behavior?.strictSketch !== false, imageSize: config.model.defaults.imageSize, steps: config.model.defaults.steps, cfgScale: config.model.defaults.cfgScale, @@ -897,6 +1037,17 @@ export function RealtimeCanvasDialog({ return () => window.removeEventListener('keydown', handleKeyDown); }, [open, undo, redo]); + useEffect(() => { + if (!open) return; + seedRef.current = createSessionSeed(); + sessionIdRef.current = `rt-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + seqRef.current = 0; + lastRenderedSeqRef.current = 0; + requestSeqMapRef.current.clear(); + requestStartMapRef.current.clear(); + metricsQueueRef.current = []; + }, [open]); + useEffect(() => { if (!open) return; initializeCanvas(); @@ -975,6 +1126,9 @@ export function RealtimeCanvasDialog({ defaultRealtimeConfig.model.defaults.imageSize, }, }, + behavior: { + strictSketch: configJson.behavior?.strictSketch !== false, + }, }; setConfig(resolvedConfig);