diff --git a/app/api/realtime/config/route.ts b/app/api/realtime/config/route.ts index bef77b7c5..5b5235c39 100644 --- a/app/api/realtime/config/route.ts +++ b/app/api/realtime/config/route.ts @@ -3,6 +3,8 @@ import { getSystemSetting } from "@/lib/db/queries"; type RealtimeCanvasConfig = { enabled?: boolean; + runwareApiKey?: string; + runwareModel?: string; falApiKey?: string; apiKey?: string; throttleInterval?: number; @@ -11,16 +13,14 @@ type RealtimeCanvasConfig = { jpegQuality?: number; defaults?: { steps?: number; + cfgScale?: number; + strength?: number; scheduleMu?: number; feedback?: number; imageSize?: "square" | "square_hd"; }; }; -type AngleEditConfig = { - falApiKey?: string; -}; - const defaultConfig = { session: { idleTimeoutMs: 120000, @@ -34,16 +34,17 @@ const defaultConfig = { jpegQuality: 0.55, }, model: { - id: "fal-ai/flux-2/klein/realtime", + id: "runware:400@4", defaults: { - steps: 3, - scheduleMu: 2.3, - feedback: 0.95, + steps: 4, + cfgScale: 3.5, + strength: 0.85, imageSize: "square" as const, }, limits: { - steps: { min: 1, max: 6 }, - feedback: { min: 0.7, max: 1.0 }, + steps: { min: 1, max: 8 }, + cfgScale: { min: 1, max: 12 }, + strength: { min: 0.05, max: 1.0 }, }, }, features: { @@ -62,12 +63,8 @@ export async function GET() { return Response.json({ code: "UNAUTHORIZED", message: "请先登录" }, { status: 401 }); } - const [setting, angleEditSetting] = await Promise.all([ - getSystemSetting({ key: "realtime_canvas_config" }).catch(() => null), - getSystemSetting({ key: "angle_edit_config" }).catch(() => null), - ]); + const setting = await getSystemSetting({ key: "realtime_canvas_config" }).catch(() => null); const config = (setting?.value || {}) as RealtimeCanvasConfig; - const angleEditConfig = (angleEditSetting?.value || {}) as AngleEditConfig; if (config.enabled === false) { return Response.json( @@ -76,20 +73,20 @@ export async function GET() { ); } - const falApiKey = String( - config.falApiKey || + const runwareApiKey = String( + config.runwareApiKey || + config.falApiKey || config.apiKey || - angleEditConfig.falApiKey || - process.env.FAL_KEY || - process.env.FAL_API_KEY || + process.env.RUNWARE_API_KEY || + process.env.RUNWARE_KEY || "" ).trim(); - if (!falApiKey) { + if (!runwareApiKey) { return Response.json( { - code: "FAL_KEY_MISSING", - message: "未配置实时画布 API Key(请在本地AI设置中配置)", + code: "RUNWARE_KEY_MISSING", + message: "未配置 Runware API Key(请在服务器环境变量或设置中配置)", }, { status: 500 } ); @@ -115,21 +112,39 @@ export async function GET() { 0.35, 0.82 ); - const steps = clamp(Number(config.defaults?.steps || defaultConfig.model.defaults.steps), 1, 6); - const scheduleMu = clamp( - Number(config.defaults?.scheduleMu || defaultConfig.model.defaults.scheduleMu), - 1.5, - 3.5 + const steps = clamp( + Number(config.defaults?.steps || defaultConfig.model.defaults.steps), + defaultConfig.model.limits.steps.min, + defaultConfig.model.limits.steps.max ); - const feedback = clamp( - Number(config.defaults?.feedback || defaultConfig.model.defaults.feedback), - 0.7, - 1.0 + const cfgScale = clamp( + Number( + config.defaults?.cfgScale ?? + config.defaults?.scheduleMu ?? + defaultConfig.model.defaults.cfgScale + ), + defaultConfig.model.limits.cfgScale.min, + defaultConfig.model.limits.cfgScale.max + ); + const strength = clamp( + Number( + config.defaults?.strength ?? + config.defaults?.feedback ?? + defaultConfig.model.defaults.strength + ), + defaultConfig.model.limits.strength.min, + defaultConfig.model.limits.strength.max ); const imageSize = config.defaults?.imageSize === "square_hd" ? "square_hd" : "square"; + const modelId = String( + config.runwareModel || process.env.RUNWARE_MODEL || defaultConfig.model.id + ).trim(); return Response.json({ ...defaultConfig, + provider: { + type: "runware", + }, transport: { throttleInterval, maxBuffering, @@ -140,10 +155,11 @@ export async function GET() { }, model: { ...defaultConfig.model, + id: modelId || defaultConfig.model.id, defaults: { steps, - scheduleMu, - feedback, + cfgScale, + strength, imageSize, }, }, diff --git a/app/api/realtime/generate/route.ts b/app/api/realtime/generate/route.ts new file mode 100644 index 000000000..a2126d733 --- /dev/null +++ b/app/api/realtime/generate/route.ts @@ -0,0 +1,211 @@ +import { auth } from "@/app/(auth)/auth"; +import { getSystemSetting } from "@/lib/db/queries"; + +type RealtimeCanvasConfig = { + enabled?: boolean; + runwareApiKey?: string; + runwareModel?: string; + falApiKey?: string; + apiKey?: string; + maxInputResolution?: number; + defaults?: { + steps?: number; + cfgScale?: number; + strength?: number; + }; +}; + +type GeneratePayload = { + requestId?: string; + imageUrl?: string; + prompt?: string; + imageSize?: "square" | "square_hd"; + steps?: number; + cfgScale?: number; + strength?: number; +}; + +function clamp(value: number, min: number, max: number) { + return Math.min(max, Math.max(min, value)); +} + +function resolveSquareSize( + imageSize: "square" | "square_hd" | undefined, + maxInputResolution: number +) { + const preferred = imageSize === "square_hd" ? 1280 : 1024; + const capped = Math.min(preferred, maxInputResolution); + return clamp(Math.round(capped / 64) * 64, 512, 1536); +} + +function parseRunwareError(payload: any) { + if (!payload) return ""; + if (Array.isArray(payload)) { + const match = payload.find( + (item) => + typeof item?.error === "string" || + typeof item?.message === "string" || + (Array.isArray(item?.errors) && item.errors.length > 0) + ); + if (!match) return ""; + if (typeof match.error === "string") return match.error; + if (typeof match.message === "string") return match.message; + if (Array.isArray(match.errors) && match.errors.length > 0) { + const firstError = match.errors[0]; + if (typeof firstError === "string") return firstError; + if (typeof firstError?.message === "string") return firstError.message; + } + return ""; + } + if (typeof payload?.error === "string") return payload.error; + if (typeof payload?.message === "string") return payload.message; + return ""; +} + +export async function POST(request: Request) { + const session = await auth(); + if (!session?.user?.id) { + return Response.json({ code: "UNAUTHORIZED", message: "请先登录" }, { status: 401 }); + } + + let body: GeneratePayload; + try { + body = (await request.json()) as GeneratePayload; + } catch { + return Response.json({ code: "BAD_REQUEST", message: "无效 JSON" }, { status: 400 }); + } + + const imageUrl = String(body.imageUrl || "").trim(); + if (!imageUrl) { + return Response.json({ code: "BAD_REQUEST", message: "缺少 imageUrl" }, { status: 400 }); + } + + const setting = await getSystemSetting({ key: "realtime_canvas_config" }).catch(() => null); + const config = (setting?.value || {}) as RealtimeCanvasConfig; + + if (config.enabled === false) { + return Response.json( + { code: "FEATURE_DISABLED", message: "实时画布功能未开启" }, + { status: 403 } + ); + } + + const runwareApiKey = String( + config.runwareApiKey || + config.falApiKey || + config.apiKey || + process.env.RUNWARE_API_KEY || + process.env.RUNWARE_KEY || + "" + ).trim(); + if (!runwareApiKey) { + return Response.json( + { + code: "RUNWARE_KEY_MISSING", + message: "未配置 Runware API Key", + }, + { status: 500 } + ); + } + + const model = String(config.runwareModel || process.env.RUNWARE_MODEL || "runware:400@4").trim(); + const maxInputResolution = clamp(Number(config.maxInputResolution || 1024), 512, 1536); + const size = resolveSquareSize(body.imageSize, maxInputResolution); + const steps = clamp( + Number(body.steps || config.defaults?.steps || 4), + 1, + 8 + ); + const cfgScale = clamp( + Number(body.cfgScale || config.defaults?.cfgScale || 3.5), + 1, + 12 + ); + const strength = clamp( + Number(body.strength || config.defaults?.strength || 0.85), + 0.05, + 1 + ); + + const taskUUID = crypto.randomUUID(); + const runwareRequest = [ + { + taskType: "imageInference", + taskUUID, + model, + positivePrompt: String(body.prompt || "").trim() || "将草图渲染为高质感产品设计图,保持结构和主轮廓一致。", + seedImage: imageUrl, + width: size, + height: size, + strength, + CFGScale: cfgScale, + steps, + numberResults: 1, + outputType: "dataURI", + outputFormat: "JPG", + }, + ]; + + const upstreamResponse = await fetch("https://api.runware.ai/v1", { + method: "POST", + headers: { + Authorization: `Bearer ${runwareApiKey}`, + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(runwareRequest), + cache: "no-store", + }); + + const upstreamJson = await upstreamResponse.json().catch(() => null); + const upstreamError = parseRunwareError(upstreamJson); + + if (!upstreamResponse.ok || upstreamError) { + return Response.json( + { + code: "RUNWARE_REQUEST_FAILED", + message: upstreamError || `Runware 请求失败(${upstreamResponse.status})`, + }, + { status: 502 } + ); + } + + const resultList = Array.isArray(upstreamJson) + ? upstreamJson + : Array.isArray(upstreamJson?.data) + ? upstreamJson.data + : []; + const imageResult = + resultList.find((item: any) => item?.taskUUID === taskUUID && (item?.imageDataURI || item?.imageURL)) || + resultList.find((item: any) => item?.imageDataURI || item?.imageURL); + + const imageDataURI = String(imageResult?.imageDataURI || "").trim(); + const imageURL = String(imageResult?.imageURL || "").trim(); + const outputImage = imageDataURI || imageURL; + + if (!outputImage) { + return Response.json( + { code: "RUNWARE_EMPTY_OUTPUT", message: "Runware 未返回图像" }, + { status: 502 } + ); + } + + const cost = Number(imageResult?.cost || 0); + console.log("[RealtimeRunware] frame success", { + userId: session.user.id, + requestId: body.requestId || null, + taskUUID, + model, + cost, + }); + + return Response.json({ + ok: true, + provider: "runware", + model, + taskUUID, + imageDataURI: imageDataURI || undefined, + imageUrl: imageURL || undefined, + cost, + }); +} diff --git a/components/canvas/RealtimeCanvasDialog.tsx b/components/canvas/RealtimeCanvasDialog.tsx index ccaf1e0ac..9d068c92d 100644 --- a/components/canvas/RealtimeCanvasDialog.tsx +++ b/components/canvas/RealtimeCanvasDialog.tsx @@ -9,7 +9,6 @@ import { type ChangeEvent, type PointerEvent as ReactPointerEvent, } from "react"; -import * as fal from "@fal-ai/serverless-client"; import { Dialog, DialogContent, @@ -52,8 +51,8 @@ type RealtimeConfig = { id: string; defaults: { steps: number; - scheduleMu: number; - feedback: number; + cfgScale: number; + strength: number; imageSize: "square" | "square_hd"; }; }; @@ -90,37 +89,16 @@ const defaultRealtimeConfig: RealtimeConfig = { jpegQuality: 0.55, }, model: { - id: "fal-ai/flux-2/klein/realtime", + id: "runware:400@4", defaults: { - steps: 3, - scheduleMu: 2.3, - feedback: 0.95, + steps: 4, + cfgScale: 3.5, + strength: 0.85, imageSize: "square", }, }, }; -function buildImageDataUrl(image: any) { - if (!image) return ""; - if (typeof image === "string" && image.trim()) { - return image; - } - if (typeof image.url === "string" && image.url.trim()) { - return image.url; - } - if (typeof image.image_url === "string" && image.image_url.trim()) { - return image.image_url; - } - if (typeof image.content === "string" && image.content.trim()) { - if (image.content.startsWith("data:")) { - return image.content; - } - const contentType = image.content_type || "image/jpeg"; - return `data:${contentType};base64,${image.content}`; - } - return ""; -} - function isShapeMode(mode: "brush" | "eraser" | "rect" | "circle" | "line") { return mode === "rect" || mode === "circle" || mode === "line"; } @@ -300,25 +278,77 @@ export function RealtimeCanvasDialog({ canvasBackgroundRef.current ); - dirtyRef.current = false; + const activeConnection = connectionRef.current; + if (!activeConnection) { + setStatus("connecting"); + setStatusMessage("连接重建中,正在重试..."); + return; + } + setStatus("connecting"); setStatusMessage("正在请求实时生成..."); - connectionRef.current.send({ - request_id: requestId, - image_url: imageDataUri, - prompt: prompt.trim() || FALLBACK_PROMPT, - image_size: config.model.defaults.imageSize, - num_inference_steps: config.model.defaults.steps, - schedule_mu: config.model.defaults.scheduleMu, - output_feedback_strength: config.model.defaults.feedback, + + const response = await fetch("/api/realtime/generate", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + requestId, + imageUrl: imageDataUri, + prompt: prompt.trim() || FALLBACK_PROMPT, + imageSize: config.model.defaults.imageSize, + steps: config.model.defaults.steps, + cfgScale: config.model.defaults.cfgScale, + strength: config.model.defaults.strength, + }), }); + const payload = (await response.json().catch(() => ({}))) as { + imageUrl?: string; + imageDataURI?: string; + cost?: number; + message?: string; + error?: string; + }; + + if (!response.ok) { + throw new Error(payload.message || payload.error || "实时生成失败"); + } + + const nextPreviewUrl = String(payload.imageDataURI || payload.imageUrl || ""); + if (!nextPreviewUrl) { + throw new Error("实时生成未返回图像"); + } + + const startedAt = requestStartMapRef.current.get(requestId) || Date.now(); + const latencyMs = Date.now() - startedAt; + requestStartMapRef.current.delete(requestId); + requestSeqMapRef.current.delete(requestId); + + if (seq > 0 && seq < lastRenderedSeqRef.current) { + return; + } + + lastRenderedSeqRef.current = Math.max(lastRenderedSeqRef.current, seq); + setPreviewUrl(nextPreviewUrl); + setStatus("live"); + setStatusMessage("实时渲染中"); + dirtyRef.current = false; + pushMetric({ name: "frame_send", ts: Date.now(), seq, }); + pushMetric({ + name: "frame_result", + ts: Date.now(), + seq, + latencyMs, + queueDepth: Number(payload.cost || 0), + }); } catch (error) { + requestStartMapRef.current.delete(requestId); + requestSeqMapRef.current.delete(requestId); console.error("[RealtimeCanvas] flush frame failed:", error); setStatus("error"); setStatusMessage(error instanceof Error ? error.message : "发送失败"); @@ -352,6 +382,17 @@ export function RealtimeCanvasDialog({ [adaptiveThrottleMs, flushFrame] ); + const scheduleFlushRef = useRef(scheduleFlush); + const flushMetricsRef = useRef(flushMetrics); + + useEffect(() => { + scheduleFlushRef.current = scheduleFlush; + }, [scheduleFlush]); + + useEffect(() => { + flushMetricsRef.current = flushMetrics; + }, [flushMetrics]); + const clearPreviewCanvas = useCallback(() => { const previewCanvas = previewCanvasRef.current; if (!previewCanvas) return; @@ -677,7 +718,7 @@ export function RealtimeCanvasDialog({ initializeCanvas(); void drawInitialImage().then((hasInitialImage) => { if (hasInitialImage) { - scheduleFlush(true, false); + scheduleFlushRef.current(true, false); } }); const onResize = () => initializeCanvas(); @@ -685,7 +726,7 @@ export function RealtimeCanvasDialog({ return () => { window.removeEventListener("resize", onResize); }; - }, [drawInitialImage, initializeCanvas, open, scheduleFlush]); + }, [drawInitialImage, initializeCanvas, open]); useEffect(() => { if (!open) return; @@ -739,12 +780,12 @@ export function RealtimeCanvasDialog({ id: configJson.model?.id || defaultRealtimeConfig.model.id, defaults: { steps: configJson.model?.defaults?.steps || defaultRealtimeConfig.model.defaults.steps, - scheduleMu: - configJson.model?.defaults?.scheduleMu || - defaultRealtimeConfig.model.defaults.scheduleMu, - feedback: - configJson.model?.defaults?.feedback || - defaultRealtimeConfig.model.defaults.feedback, + cfgScale: + configJson.model?.defaults?.cfgScale || + defaultRealtimeConfig.model.defaults.cfgScale, + strength: + configJson.model?.defaults?.strength || + defaultRealtimeConfig.model.defaults.strength, imageSize: configJson.model?.defaults?.imageSize || defaultRealtimeConfig.model.defaults.imageSize, @@ -755,78 +796,16 @@ export function RealtimeCanvasDialog({ setConfig(resolvedConfig); setAdaptiveThrottleMs(resolvedConfig.transport.throttleInterval); - fal.config({ - proxyUrl: "/api/fal/proxy", - }); - - const connection = fal.realtime.connect(resolvedConfig.model.id, { - connectionKey: `canvas-realtime-${sessionIdRef.current}-${refreshToken}`, - throttleInterval: resolvedConfig.transport.throttleInterval, - maxBuffering: resolvedConfig.transport.maxBuffering, - onResult: (result: any) => { - const requestId = String(result?.request_id || ""); - const seqFromRequest = requestSeqMapRef.current.get(requestId) || 0; - const seq = Math.max(seqFromRequest, Number(result?.seq || 0)); - if (seq > 0 && seq < lastRenderedSeqRef.current) { - return; - } - - const imageGroups = [result?.images, result?.output?.images, result?.data?.images]; - const firstImageGroup = imageGroups.find((group) => Array.isArray(group)) as any[] | undefined; - const latestImage = - (firstImageGroup && firstImageGroup[firstImageGroup.length - 1]) || - result?.image || - result?.output?.image || - result?.data?.image || - null; - const nextPreviewUrl = buildImageDataUrl(latestImage); - if (!nextPreviewUrl) return; - - lastRenderedSeqRef.current = Math.max(lastRenderedSeqRef.current, seq); - setPreviewUrl(nextPreviewUrl); - setStatus("live"); - setStatusMessage("实时渲染中"); - - if (requestId) { - const startedAt = requestStartMapRef.current.get(requestId); - if (startedAt) { - pushMetric({ - name: "frame_result", - ts: Date.now(), - seq, - latencyMs: Date.now() - startedAt, - }); - } - requestStartMapRef.current.delete(requestId); - requestSeqMapRef.current.delete(requestId); - } - }, - onError: (error: any) => { - const errorMessage = - error?.body?.message || error?.message || "实时连接异常,请稍后重试"; - setStatus("error"); - setStatusMessage(String(errorMessage)); - if (Number(error?.status) === 429) { - setAdaptiveThrottleMs((prev) => Math.min(240, prev + 40)); - } - pushMetric({ - name: "frame_error", - ts: Date.now(), - message: String(errorMessage), - }); - }, - }); - - if (cancelled) { - connection.close(); - return; - } - - connectionRef.current = connection; - setStatus("connecting"); - setStatusMessage("连接成功,等待首帧..."); - scheduleFlush(true, false); + if (cancelled) return; + connectionRef.current = { + provider: "runware", + modelId: resolvedConfig.model.id, + reconnectToken: refreshToken, + }; + setStatus("live"); + setStatusMessage("实时服务已就绪"); } catch (error) { + if (cancelled) return; console.error("[RealtimeCanvas] init failed:", error); setStatus("error"); setStatusMessage(error instanceof Error ? error.message : "实时初始化失败"); @@ -835,7 +814,7 @@ export function RealtimeCanvasDialog({ void start(); const metricTimer = window.setInterval(() => { - void flushMetrics(); + void flushMetricsRef.current(); }, 5000); return () => { @@ -845,7 +824,7 @@ export function RealtimeCanvasDialog({ pendingFlushTimerRef.current = null; } window.clearInterval(metricTimer); - void flushMetrics(); + void flushMetricsRef.current(); connectionRef.current?.close?.(); connectionRef.current = null; setStatus("idle"); @@ -853,10 +832,9 @@ export function RealtimeCanvasDialog({ pointerDownRef.current = false; lastPointRef.current = null; }; - }, [flushMetrics, open, pushMetric, refreshToken, scheduleFlush]); + }, [open, pushMetric, refreshToken]); const forceReconnect = useCallback(() => { - connectionRef.current?.close?.(); connectionRef.current = null; setStatus("connecting"); setStatusMessage("正在重连...");