877 lines
29 KiB
TypeScript
877 lines
29 KiB
TypeScript
import { auth } from "@/app/(auth)/auth";
|
||
import { getSystemSetting } from "@/lib/db/queries";
|
||
import { getRunwareWebSocketClient } from "@/lib/realtime/runware-ws";
|
||
|
||
type RealtimeCanvasConfig = {
|
||
enabled?: boolean;
|
||
runwareApiKey?: string;
|
||
runwareModel?: string;
|
||
runwareStylizeModel?: string;
|
||
runwareControlModel?: string;
|
||
runwareControlPreprocessor?: "canny" | "lineart" | "depth";
|
||
renderMode?: "strict-structure" | "balanced" | "stylize";
|
||
runwareUseWebSocket?: boolean;
|
||
falRealtimeEnabled?: boolean;
|
||
falRealtimeModel?: string;
|
||
falApiKey?: string;
|
||
apiKey?: string;
|
||
negativePrompt?: string;
|
||
strictSketch?: boolean;
|
||
maxInputResolution?: number;
|
||
defaults?: {
|
||
steps?: number;
|
||
cfgScale?: number;
|
||
strength?: number;
|
||
};
|
||
};
|
||
|
||
type AngleEditConfig = {
|
||
falApiKey?: string;
|
||
};
|
||
|
||
type GeneratePayload = {
|
||
requestId?: string;
|
||
imageUrl?: string;
|
||
maskImageUrl?: string;
|
||
qualityProfile?: "preview" | "refine";
|
||
roi?:
|
||
| {
|
||
x?: number;
|
||
y?: number;
|
||
width?: number;
|
||
height?: number;
|
||
}
|
||
| null;
|
||
prompt?: string;
|
||
seed?: number;
|
||
strictSketch?: boolean;
|
||
imageSize?: "square" | "square_hd";
|
||
steps?: number;
|
||
cfgScale?: number;
|
||
strength?: number;
|
||
};
|
||
|
||
const FALLBACK_REALTIME_PROMPT =
|
||
"严格按照输入草图生成：保持主体轮廓、位置、比例和数量一致，只做材质与光影细化；禁止改变构图，禁止新增无关背景纹理。";
|
||
const FALLBACK_FAL_REALTIME_MODEL = "fal-ai/flux-2/klein/realtime";
|
||
const FALLBACK_RUNWARE_STYLIZE_MODEL = "runware:106@1";
|
||
const FALLBACK_RUNWARE_CONTROL_MODEL = "runware:25@1";
|
||
|
||
function clamp(value: number, min: number, max: number) {
|
||
return Math.min(max, Math.max(min, value));
|
||
}
|
||
|
||
function resolveSquareSize(
|
||
imageSize: "square" | "square_hd" | undefined,
|
||
maxInputResolution: number
|
||
) {
|
||
const preferred = imageSize === "square_hd" ? 1280 : 1024;
|
||
const capped = Math.min(preferred, maxInputResolution);
|
||
return clamp(Math.round(capped / 64) * 64, 512, 1536);
|
||
}
|
||
|
||
function parseRunwareError(payload: any) {
|
||
if (!payload) return "";
|
||
if (Array.isArray(payload)) {
|
||
const match = payload.find(
|
||
(item) =>
|
||
typeof item?.error === "string" ||
|
||
typeof item?.message === "string" ||
|
||
(Array.isArray(item?.errors) && item.errors.length > 0)
|
||
);
|
||
if (!match) return "";
|
||
if (typeof match.error === "string") return match.error;
|
||
if (typeof match.message === "string") return match.message;
|
||
if (Array.isArray(match.errors) && match.errors.length > 0) {
|
||
const firstError = match.errors[0];
|
||
if (typeof firstError === "string") return firstError;
|
||
if (typeof firstError?.message === "string") return firstError.message;
|
||
}
|
||
return "";
|
||
}
|
||
if (typeof payload?.error === "string") return payload.error;
|
||
if (typeof payload?.message === "string") return payload.message;
|
||
if (Array.isArray(payload?.errors) && payload.errors.length > 0) {
|
||
const firstError = payload.errors[0];
|
||
if (typeof firstError?.message === "string") return firstError.message;
|
||
if (typeof firstError?.code === "string") return firstError.code;
|
||
}
|
||
return "";
|
||
}
|
||
|
||
function resolveSeed(seed: unknown) {
|
||
const parsed = Number(seed);
|
||
if (!Number.isFinite(parsed)) return undefined;
|
||
const normalized = Math.floor(parsed);
|
||
if (normalized < 0 || normalized > 2147483647) return undefined;
|
||
return normalized;
|
||
}
|
||
|
||
function shouldRetryWithoutMask(errorMessage: string) {
|
||
const normalized = errorMessage.toLowerCase();
|
||
return (
|
||
normalized.includes("unsupported use of 'maskimage'") ||
|
||
normalized.includes("unsupported use of \"maskimage\"") ||
|
||
normalized.includes("unsupported use of maskimage") ||
|
||
normalized.includes("maskimage is not supported")
|
||
);
|
||
}
|
||
|
||
function shouldRetryWithoutNegativePrompt(errorMessage: string) {
|
||
const normalized = errorMessage.toLowerCase();
|
||
return (
|
||
normalized.includes("unsupportedarchitecturenegativeprompt") ||
|
||
normalized.includes("negative prompt is not supported") ||
|
||
normalized.includes("unsupported use of 'negativeprompt'") ||
|
||
normalized.includes("unsupported use of \"negativeprompt\"")
|
||
);
|
||
}
|
||
|
||
function shouldRetryWithoutControlNet(errorMessage: string) {
|
||
const normalized = errorMessage.toLowerCase();
|
||
return (
|
||
normalized.includes("unsupportedcontrolnetmodel") ||
|
||
normalized.includes("unsupported use of 'controlnet'") ||
|
||
normalized.includes("unsupported use of \"controlnet\"") ||
|
||
normalized.includes("controlnet is not supported")
|
||
);
|
||
}
|
||
|
||
function resolveRenderModeSettings(
|
||
mode: "strict-structure" | "balanced" | "stylize"
|
||
) {
|
||
if (mode === "strict-structure") {
|
||
return {
|
||
controlWeight: 1,
|
||
seedStrengthMin: 0.14,
|
||
seedStrengthMax: 0.58,
|
||
};
|
||
}
|
||
if (mode === "stylize") {
|
||
return {
|
||
controlWeight: 0.72,
|
||
seedStrengthMin: 0.28,
|
||
seedStrengthMax: 0.8,
|
||
};
|
||
}
|
||
return {
|
||
controlWeight: 0.86,
|
||
seedStrengthMin: 0.2,
|
||
seedStrengthMax: 0.7,
|
||
};
|
||
}
|
||
|
||
function extractFalImageUrl(payload: any) {
|
||
const direct =
|
||
payload?.images?.[0]?.url ||
|
||
payload?.image?.url ||
|
||
payload?.output?.[0]?.url ||
|
||
payload?.data?.images?.[0]?.url;
|
||
if (typeof direct === "string" && direct.trim()) {
|
||
return direct.trim();
|
||
}
|
||
|
||
const queue: any[] = [payload];
|
||
const visited = new Set<any>();
|
||
while (queue.length > 0) {
|
||
const current = queue.shift();
|
||
if (!current || typeof current !== "object" || visited.has(current)) continue;
|
||
visited.add(current);
|
||
for (const value of Object.values(current)) {
|
||
if (typeof value === "string" && /^https?:\/\//i.test(value)) {
|
||
return value;
|
||
}
|
||
if (value && typeof value === "object") {
|
||
queue.push(value);
|
||
}
|
||
}
|
||
}
|
||
return "";
|
||
}
|
||
|
||
function extractRunwareOutputImage(payload: any, taskUUID: string) {
|
||
const resultList = Array.isArray(payload)
|
||
? payload
|
||
: Array.isArray(payload?.data)
|
||
? payload.data
|
||
: [];
|
||
const imageResult =
|
||
resultList.find(
|
||
(item: any) =>
|
||
item?.taskUUID === taskUUID && (item?.imageDataURI || item?.imageURL)
|
||
) ||
|
||
resultList.find((item: any) => item?.imageDataURI || item?.imageURL);
|
||
const imageDataURI = String(imageResult?.imageDataURI || "").trim();
|
||
const imageURL = String(imageResult?.imageURL || "").trim();
|
||
const outputImage = imageDataURI || imageURL;
|
||
return {
|
||
imageResult,
|
||
imageDataURI,
|
||
imageURL,
|
||
outputImage,
|
||
};
|
||
}
|
||
|
||
function extractControlGuideImage(payload: any, taskUUID: string) {
|
||
const resultList = Array.isArray(payload)
|
||
? payload
|
||
: Array.isArray(payload?.data)
|
||
? payload.data
|
||
: [];
|
||
const guideResult =
|
||
resultList.find(
|
||
(item: any) =>
|
||
item?.taskUUID === taskUUID && (item?.guideImageURL || item?.guideImageDataURI)
|
||
) ||
|
||
resultList.find((item: any) => item?.guideImageURL || item?.guideImageDataURI);
|
||
const guideImageDataURI = String(guideResult?.guideImageDataURI || "").trim();
|
||
const guideImageURL = String(guideResult?.guideImageURL || "").trim();
|
||
return guideImageDataURI || guideImageURL;
|
||
}
|
||
|
||
export async function POST(request: Request) {
|
||
const session = await auth();
|
||
if (!session?.user?.id) {
|
||
return Response.json({ code: "UNAUTHORIZED", message: "请先登录" }, { status: 401 });
|
||
}
|
||
|
||
let body: GeneratePayload;
|
||
try {
|
||
body = (await request.json()) as GeneratePayload;
|
||
} catch {
|
||
return Response.json({ code: "BAD_REQUEST", message: "无效 JSON" }, { status: 400 });
|
||
}
|
||
|
||
const imageUrl = String(body.imageUrl || "").trim();
|
||
if (!imageUrl) {
|
||
return Response.json({ code: "BAD_REQUEST", message: "缺少 imageUrl" }, { status: 400 });
|
||
}
|
||
const maskImageUrl = String(body.maskImageUrl || "").trim();
|
||
|
||
const [setting, angleEditSetting] = await Promise.all([
|
||
getSystemSetting({ key: "realtime_canvas_config" }).catch(() => null),
|
||
getSystemSetting({ key: "angle_edit_config" }).catch(() => null),
|
||
]);
|
||
const config = (setting?.value || {}) as RealtimeCanvasConfig;
|
||
const angleEditConfig = (angleEditSetting?.value || {}) as AngleEditConfig;
|
||
|
||
if (config.enabled === false) {
|
||
return Response.json(
|
||
{ code: "FEATURE_DISABLED", message: "实时画布功能未开启" },
|
||
{ status: 403 }
|
||
);
|
||
}
|
||
|
||
const runwareApiKey = String(
|
||
config.runwareApiKey ||
|
||
config.falApiKey ||
|
||
config.apiKey ||
|
||
process.env.RUNWARE_API_KEY ||
|
||
process.env.RUNWARE_KEY ||
|
||
""
|
||
).trim();
|
||
const falApiKey = String(
|
||
config.falApiKey ||
|
||
angleEditConfig.falApiKey ||
|
||
process.env.FAL_KEY ||
|
||
process.env.FAL_API_KEY ||
|
||
""
|
||
).trim();
|
||
if (!runwareApiKey) {
|
||
return Response.json(
|
||
{
|
||
code: "RUNWARE_KEY_MISSING",
|
||
message: "未配置 Runware API Key",
|
||
},
|
||
{ status: 500 }
|
||
);
|
||
}
|
||
|
||
const fallbackSeedImageModel = "runware:101@1";
|
||
const configuredModel = String(
|
||
config.runwareModel || process.env.RUNWARE_MODEL || fallbackSeedImageModel
|
||
).trim();
|
||
const model = configuredModel || fallbackSeedImageModel;
|
||
const configuredStylizeModel = String(
|
||
config.runwareStylizeModel ||
|
||
process.env.RUNWARE_REALTIME_STYLIZE_MODEL ||
|
||
FALLBACK_RUNWARE_STYLIZE_MODEL
|
||
).trim();
|
||
const stylizeModel = configuredStylizeModel || FALLBACK_RUNWARE_STYLIZE_MODEL;
|
||
const configuredControlModel = String(
|
||
config.runwareControlModel ||
|
||
process.env.RUNWARE_REALTIME_CONTROL_MODEL ||
|
||
FALLBACK_RUNWARE_CONTROL_MODEL
|
||
).trim();
|
||
const controlModel = configuredControlModel || FALLBACK_RUNWARE_CONTROL_MODEL;
|
||
const controlPreprocessor =
|
||
config.runwareControlPreprocessor === "lineart" ||
|
||
config.runwareControlPreprocessor === "depth"
|
||
? config.runwareControlPreprocessor
|
||
: "canny";
|
||
const renderMode =
|
||
config.renderMode === "strict-structure" ||
|
||
config.renderMode === "stylize" ||
|
||
config.renderMode === "balanced"
|
||
? config.renderMode
|
||
: "balanced";
|
||
const renderModeSettings = resolveRenderModeSettings(renderMode);
|
||
const falRealtimeModel = String(
|
||
config.falRealtimeModel || process.env.FAL_REALTIME_MODEL || FALLBACK_FAL_REALTIME_MODEL
|
||
).trim();
|
||
const preferWebSocket =
|
||
config.runwareUseWebSocket !== false &&
|
||
process.env.RUNWARE_REALTIME_USE_WS !== "0";
|
||
const maxInputResolution = clamp(Number(config.maxInputResolution || 1024), 512, 1536);
|
||
const size = resolveSquareSize(body.imageSize, maxInputResolution);
|
||
const qualityProfile = body.qualityProfile === "preview" ? "preview" : "refine";
|
||
const steps = clamp(
|
||
Number(body.steps || config.defaults?.steps || 8),
|
||
1,
|
||
12
|
||
);
|
||
const cfgScale = clamp(
|
||
Number(body.cfgScale || config.defaults?.cfgScale || 7),
|
||
1,
|
||
14
|
||
);
|
||
const strength = clamp(
|
||
Number(body.strength || config.defaults?.strength || 0.32),
|
||
0.05,
|
||
1
|
||
);
|
||
const strictSketch = Boolean(body.strictSketch ?? config.strictSketch ?? true);
|
||
const seed = resolveSeed(body.seed);
|
||
|
||
const rawPrompt = String(body.prompt || "").trim();
|
||
const hasUserPrompt =
|
||
rawPrompt.length > 0 && rawPrompt !== FALLBACK_REALTIME_PROMPT;
|
||
const positivePrompt = hasUserPrompt
|
||
? [
|
||
`User prompt: ${rawPrompt}`,
|
||
"Keep the object layout and perspective from the sketch.",
|
||
"Keep the exact silhouette and part topology from the sketch (same count of legs/supports and same frame structure).",
|
||
"The main object category must exactly match the user prompt.",
|
||
"Do not swap object class (for example, chair must remain chair, never animal).",
|
||
"Do not add bulky cushions, thick armrests, or boxy masses unless they clearly exist in the sketch.",
|
||
"Render as a full-color photorealistic product image with realistic material, lighting, and texture.",
|
||
"No sketch style, no line art outlines, no monochrome drawing.",
|
||
].join(" ")
|
||
: FALLBACK_REALTIME_PROMPT;
|
||
const negativePrompt =
|
||
String(config.negativePrompt || "").trim() ||
|
||
"改变构图, 改变主体位置, 多余人物, 多余肢体, 重复物体, 镜像翻转, 错误透视, 模糊轮廓, 文字水印, 大面积背景噪点, 线稿, 线描, 草图, 黑白素描, 轮廓线, line art, sketch, outline drawing, monochrome drawing, wrong object category, unrelated subject, animal when furniture requested, furniture when animal requested";
|
||
let fallbackWarning = "";
|
||
|
||
const useFalRealtime =
|
||
config.falRealtimeEnabled === true || process.env.FAL_REALTIME_ENABLED === "1";
|
||
|
||
if (hasUserPrompt && qualityProfile === "refine" && falApiKey && useFalRealtime) {
|
||
try {
|
||
const fal = await import("@fal-ai/serverless-client");
|
||
fal.config({
|
||
credentials: falApiKey,
|
||
});
|
||
|
||
const falInput: Record<string, unknown> = {
|
||
prompt: rawPrompt,
|
||
negative_prompt: negativePrompt,
|
||
image_url: imageUrl,
|
||
image_size: body.imageSize === "square_hd" ? "square_hd" : "square",
|
||
num_inference_steps: 6,
|
||
schedule_mu: 2.3,
|
||
output_feedback_strength: 0.95,
|
||
sync_mode: true,
|
||
};
|
||
|
||
const falResult: any = await fal.subscribe(falRealtimeModel, {
|
||
input: falInput,
|
||
logs: false,
|
||
timeout: 90_000,
|
||
});
|
||
|
||
const falImageUrl = extractFalImageUrl(falResult);
|
||
if (falImageUrl) {
|
||
console.log("[RealtimeFal] frame success", {
|
||
userId: session.user.id,
|
||
requestId: body.requestId || null,
|
||
model: falRealtimeModel,
|
||
promptLength: rawPrompt.length,
|
||
qualityProfile,
|
||
});
|
||
return Response.json({
|
||
ok: true,
|
||
provider: "fal",
|
||
model: falRealtimeModel,
|
||
transport: "fal-http",
|
||
imageUrl: falImageUrl,
|
||
cost: 0,
|
||
});
|
||
}
|
||
console.warn("[RealtimeFal] empty output, fallback to Runware", {
|
||
userId: session.user.id,
|
||
requestId: body.requestId || null,
|
||
model: falRealtimeModel,
|
||
});
|
||
} catch (error) {
|
||
fallbackWarning =
|
||
error instanceof Error && /unauthorized|forbidden/i.test(error.message)
|
||
? "FAL 实时模型鉴权失败，当前已回退为 Runware 草图模式；请在系统设置更新有效的 FAL API Key。"
|
||
: "FAL 实时模型调用失败，当前已回退为 Runware 草图模式。";
|
||
console.warn("[RealtimeFal] request failed, fallback to Runware", {
|
||
userId: session.user.id,
|
||
requestId: body.requestId || null,
|
||
model: falRealtimeModel,
|
||
error: error instanceof Error ? error.message : String(error),
|
||
});
|
||
}
|
||
}
|
||
|
||
type InferenceRequestResult = {
|
||
taskUUID: string;
|
||
modelId: string;
|
||
runtimeStrength: number;
|
||
runtimeCfgScale: number;
|
||
runtimeSteps: number;
|
||
effectiveStrictSketch: boolean;
|
||
runtimeSeed?: number;
|
||
usedMaskImage: boolean;
|
||
controlGuideImage: string;
|
||
controlNetModel: string;
|
||
path: "seed-image" | "controlnet";
|
||
transport: "ws" | "http";
|
||
upstreamStatus: number;
|
||
upstreamOk: boolean;
|
||
upstreamJson: any;
|
||
upstreamError: string;
|
||
wsFallbackError: string;
|
||
};
|
||
|
||
type RunwareTaskPayload = {
|
||
taskType: string;
|
||
taskUUID?: string;
|
||
[key: string]: unknown;
|
||
};
|
||
|
||
const sendRunwareTask = async (runwareTask: RunwareTaskPayload, timeoutMs = 90_000) => {
|
||
let transport: "ws" | "http" = "http";
|
||
let upstreamStatus = 0;
|
||
let upstreamOk = false;
|
||
let upstreamJson: any = null;
|
||
let upstreamError = "";
|
||
let wsFallbackError = "";
|
||
|
||
if (preferWebSocket) {
|
||
try {
|
||
const wsClient = getRunwareWebSocketClient(runwareApiKey);
|
||
upstreamJson = await wsClient.sendTask(runwareTask, timeoutMs);
|
||
upstreamError = parseRunwareError(upstreamJson);
|
||
upstreamStatus = upstreamError ? 502 : 200;
|
||
upstreamOk = !upstreamError;
|
||
transport = "ws";
|
||
} catch (error) {
|
||
wsFallbackError = error instanceof Error ? error.message : "Runware WebSocket 调用失败";
|
||
}
|
||
}
|
||
|
||
if (!upstreamJson) {
|
||
const upstreamResponse = await fetch("https://api.runware.ai/v1", {
|
||
method: "POST",
|
||
headers: {
|
||
Authorization: `Bearer ${runwareApiKey}`,
|
||
"Content-Type": "application/json",
|
||
Accept: "application/json",
|
||
},
|
||
body: JSON.stringify([runwareTask]),
|
||
cache: "no-store",
|
||
});
|
||
upstreamJson = await upstreamResponse.json().catch(() => null);
|
||
upstreamError = parseRunwareError(upstreamJson);
|
||
upstreamStatus = upstreamResponse.status;
|
||
upstreamOk = upstreamResponse.ok && !upstreamError;
|
||
transport = "http";
|
||
}
|
||
|
||
return {
|
||
transport,
|
||
upstreamStatus,
|
||
upstreamOk,
|
||
upstreamJson,
|
||
upstreamError,
|
||
wsFallbackError,
|
||
};
|
||
};
|
||
|
||
const requestInference = async (
|
||
modelId: string,
|
||
useMaskImage: boolean,
|
||
sourceImageUrl: string
|
||
): Promise<InferenceRequestResult> => {
|
||
const taskUUID = crypto.randomUUID();
|
||
const usePromptStylize = hasUserPrompt;
|
||
const previewMode = qualityProfile === "preview";
|
||
const isStylizePass =
|
||
usePromptStylize &&
|
||
qualityProfile === "refine" &&
|
||
Boolean(stylizeModel) &&
|
||
modelId === stylizeModel;
|
||
const relaxStructureForPrompt = usePromptStylize && !isStylizePass;
|
||
const effectiveStrictSketch = strictSketch && !relaxStructureForPrompt;
|
||
const profileStrength = usePromptStylize
|
||
? previewMode
|
||
? clamp(Math.max(strength, 0.58), 0.05, 1)
|
||
: isStylizePass
|
||
? clamp(Math.max(strength, 0.72), 0.05, 1)
|
||
: clamp(Math.max(strength, 0.9), 0.05, 1)
|
||
: previewMode
|
||
? clamp(strength * 0.85, 0.05, 1)
|
||
: strength;
|
||
const profileCfgScale = usePromptStylize
|
||
? previewMode
|
||
? clamp(Math.max(5.5, Math.min(cfgScale, 8.5)), 1, 14)
|
||
: isStylizePass
|
||
? clamp(Math.max(6.5, Math.min(cfgScale, 8.5)), 1, 14)
|
||
: clamp(Math.max(6, Math.min(cfgScale, 8.5)), 1, 14)
|
||
: previewMode
|
||
? clamp(cfgScale * 0.9, 1, 14)
|
||
: cfgScale;
|
||
const profileSteps = usePromptStylize
|
||
? previewMode
|
||
? clamp(Math.max(steps, 6), 1, 12)
|
||
: isStylizePass
|
||
? clamp(Math.max(steps, 8), 1, 10)
|
||
: clamp(Math.max(steps, 12), 1, 12)
|
||
: previewMode
|
||
? clamp(Math.min(steps, 6), 1, 12)
|
||
: clamp(Math.max(steps, 6), 1, 12);
|
||
const runtimeStrength = effectiveStrictSketch
|
||
? (() => {
|
||
const minStrength = usePromptStylize && !previewMode ? 0.2 : 0.12;
|
||
const maxStrength =
|
||
isStylizePass ? 0.62 : usePromptStylize && !previewMode ? 0.72 : 0.55;
|
||
return clamp(profileStrength, minStrength, maxStrength);
|
||
})()
|
||
: profileStrength;
|
||
const runtimeCfgScale = effectiveStrictSketch
|
||
? clamp(
|
||
profileCfgScale,
|
||
isStylizePass ? 6.5 : usePromptStylize && !previewMode ? 5 : 6,
|
||
14
|
||
)
|
||
: profileCfgScale;
|
||
const runtimeSeed = relaxStructureForPrompt ? undefined : seed;
|
||
const isRunware106 = modelId.toLowerCase().startsWith("runware:106@");
|
||
const inferenceSize = isRunware106 ? 1024 : size;
|
||
const runtimeStrengthLimited = clamp(
|
||
runtimeStrength,
|
||
renderModeSettings.seedStrengthMin,
|
||
renderModeSettings.seedStrengthMax
|
||
);
|
||
const runwareTask = {
|
||
taskType: "imageInference",
|
||
taskUUID,
|
||
model: modelId,
|
||
positivePrompt,
|
||
negativePrompt,
|
||
seedImage: sourceImageUrl,
|
||
width: inferenceSize,
|
||
height: inferenceSize,
|
||
strength: runtimeStrengthLimited,
|
||
CFGScale: runtimeCfgScale,
|
||
steps: profileSteps,
|
||
includeCost: true,
|
||
numberResults: 1,
|
||
outputType: "dataURI",
|
||
outputFormat: "JPG",
|
||
...(useMaskImage && maskImageUrl ? { maskImage: maskImageUrl } : {}),
|
||
...(typeof runtimeSeed === "number" ? { seed: runtimeSeed } : {}),
|
||
};
|
||
|
||
const upstream = await sendRunwareTask(runwareTask);
|
||
|
||
return {
|
||
taskUUID,
|
||
modelId,
|
||
runtimeStrength: runtimeStrengthLimited,
|
||
runtimeCfgScale,
|
||
runtimeSteps: profileSteps,
|
||
effectiveStrictSketch,
|
||
runtimeSeed,
|
||
usedMaskImage: useMaskImage && Boolean(maskImageUrl),
|
||
controlGuideImage: "",
|
||
controlNetModel: "",
|
||
path: "seed-image" as const,
|
||
...upstream,
|
||
};
|
||
};
|
||
|
||
const requestWithMaskFallback = async (
|
||
modelId: string,
|
||
useMaskImage: boolean,
|
||
sourceImageUrl: string
|
||
) => {
|
||
let requestResult = await requestInference(modelId, useMaskImage, sourceImageUrl);
|
||
let maskEnabled = useMaskImage;
|
||
|
||
if (
|
||
maskEnabled &&
|
||
shouldRetryWithoutMask(requestResult.upstreamError || "")
|
||
) {
|
||
requestResult = await requestInference(modelId, false, sourceImageUrl);
|
||
maskEnabled = false;
|
||
}
|
||
|
||
return { requestResult, maskEnabled };
|
||
};
|
||
|
||
const requestControlNetGuide = async (sourceImageUrl: string) => {
|
||
const taskUUID = crypto.randomUUID();
|
||
const controlTask = {
|
||
taskType: "imageControlNetPreProcess",
|
||
taskUUID,
|
||
inputImage: sourceImageUrl,
|
||
preProcessorType: controlPreprocessor,
|
||
};
|
||
const upstream = await sendRunwareTask(controlTask, 45_000);
|
||
return {
|
||
taskUUID,
|
||
guideImage: extractControlGuideImage(upstream.upstreamJson, taskUUID),
|
||
...upstream,
|
||
};
|
||
};
|
||
|
||
const requestControlNetInference = async (
|
||
modelId: string,
|
||
guideImage: string,
|
||
includeNegativePrompt: boolean
|
||
): Promise<InferenceRequestResult> => {
|
||
const taskUUID = crypto.randomUUID();
|
||
const previewMode = qualityProfile === "preview";
|
||
const profileSteps = previewMode
|
||
? clamp(Math.max(steps, 6), 1, 12)
|
||
: clamp(Math.max(steps, 10), 1, 12);
|
||
const profileCfgScale = previewMode
|
||
? clamp(Math.max(5, Math.min(cfgScale, 8)), 1, 14)
|
||
: clamp(Math.max(6, Math.min(cfgScale, 9)), 1, 14);
|
||
const controlWeight = previewMode
|
||
? clamp(renderModeSettings.controlWeight - 0.06, 0.55, 1)
|
||
: renderModeSettings.controlWeight;
|
||
|
||
const runwareTask: RunwareTaskPayload = {
|
||
taskType: "imageInference",
|
||
taskUUID,
|
||
model: modelId,
|
||
positivePrompt,
|
||
width: size,
|
||
height: size,
|
||
CFGScale: profileCfgScale,
|
||
steps: profileSteps,
|
||
includeCost: true,
|
||
numberResults: 1,
|
||
outputType: "dataURI",
|
||
outputFormat: "JPG",
|
||
...(typeof seed === "number" ? { seed } : {}),
|
||
...(includeNegativePrompt ? { negativePrompt } : {}),
|
||
controlNet: [
|
||
{
|
||
model: controlModel,
|
||
guideImage,
|
||
weight: controlWeight,
|
||
startStep: 0,
|
||
endStep: profileSteps,
|
||
controlMode: "balanced",
|
||
},
|
||
],
|
||
};
|
||
|
||
const upstream = await sendRunwareTask(runwareTask, 120_000);
|
||
return {
|
||
taskUUID,
|
||
modelId,
|
||
runtimeStrength: 0,
|
||
runtimeCfgScale: profileCfgScale,
|
||
runtimeSteps: profileSteps,
|
||
effectiveStrictSketch: strictSketch,
|
||
runtimeSeed: typeof seed === "number" ? seed : undefined,
|
||
usedMaskImage: false,
|
||
controlGuideImage: guideImage,
|
||
controlNetModel: controlModel,
|
||
path: "controlnet" as const,
|
||
...upstream,
|
||
};
|
||
};
|
||
|
||
const runTwoPassStylize = false;
|
||
let maskEnabled = Boolean(maskImageUrl);
|
||
let requestResult: InferenceRequestResult;
|
||
let pathLabel: "seed-image" | "controlnet" = "seed-image";
|
||
let controlGuideUsed = "";
|
||
let controlPreprocessUsed = false;
|
||
let controlGuideFallback = false;
|
||
|
||
const shouldUseControlNetPath =
|
||
hasUserPrompt && qualityProfile === "refine" && strictSketch;
|
||
|
||
if (shouldUseControlNetPath) {
|
||
const guideResult = await requestControlNetGuide(imageUrl);
|
||
const preprocessOk = guideResult.upstreamOk && !guideResult.upstreamError;
|
||
const guideImage = preprocessOk && guideResult.guideImage ? guideResult.guideImage : imageUrl;
|
||
controlGuideUsed = guideImage;
|
||
controlPreprocessUsed = preprocessOk;
|
||
controlGuideFallback = !preprocessOk || !guideResult.guideImage;
|
||
|
||
if (!preprocessOk && (guideResult.upstreamError || "").trim()) {
|
||
fallbackWarning = "ControlNet 预处理失败，已回退使用原始线稿引导。";
|
||
}
|
||
|
||
let controlResult = await requestControlNetInference(model, guideImage, true);
|
||
if (shouldRetryWithoutNegativePrompt(controlResult.upstreamError || "")) {
|
||
controlResult = await requestControlNetInference(model, guideImage, false);
|
||
}
|
||
|
||
if (controlResult.upstreamOk && !controlResult.upstreamError) {
|
||
requestResult = controlResult;
|
||
maskEnabled = false;
|
||
pathLabel = "controlnet";
|
||
} else if (shouldRetryWithoutControlNet(controlResult.upstreamError || "")) {
|
||
fallbackWarning = "当前模型不支持 ControlNet，已回退到 seedImage 模式。";
|
||
const fallbackPass = await requestWithMaskFallback(stylizeModel || model, maskEnabled, imageUrl);
|
||
requestResult = fallbackPass.requestResult;
|
||
maskEnabled = fallbackPass.maskEnabled;
|
||
pathLabel = "seed-image";
|
||
} else {
|
||
requestResult = controlResult;
|
||
maskEnabled = false;
|
||
pathLabel = "controlnet";
|
||
}
|
||
} else if (runTwoPassStylize) {
|
||
const basePass = await requestWithMaskFallback(model, maskEnabled, imageUrl);
|
||
requestResult = basePass.requestResult;
|
||
maskEnabled = basePass.maskEnabled;
|
||
|
||
if (requestResult.upstreamOk && !requestResult.upstreamError) {
|
||
const baseImage = extractRunwareOutputImage(
|
||
requestResult.upstreamJson,
|
||
requestResult.taskUUID
|
||
).outputImage;
|
||
if (baseImage) {
|
||
console.log("[RealtimeRunware] two-pass base success", {
|
||
userId: session.user.id,
|
||
requestId: body.requestId || null,
|
||
baseModel: model,
|
||
stylizeModel,
|
||
baseImageType: baseImage.startsWith("data:") ? "dataURI" : "url",
|
||
});
|
||
const stylizePass = await requestWithMaskFallback(stylizeModel, false, baseImage);
|
||
requestResult = stylizePass.requestResult;
|
||
maskEnabled = stylizePass.maskEnabled;
|
||
}
|
||
}
|
||
} else {
|
||
const initialModel =
|
||
hasUserPrompt && qualityProfile === "refine" ? stylizeModel || model : model;
|
||
const initialPass = await requestWithMaskFallback(initialModel, maskEnabled, imageUrl);
|
||
requestResult = initialPass.requestResult;
|
||
maskEnabled = initialPass.maskEnabled;
|
||
pathLabel = "seed-image";
|
||
}
|
||
|
||
if (pathLabel === "controlnet" && (!requestResult.upstreamOk || requestResult.upstreamError)) {
|
||
fallbackWarning = "ControlNet 推理失败，已自动回退为 seedImage 模式。";
|
||
const fallbackPass = await requestWithMaskFallback(stylizeModel || model, maskEnabled, imageUrl);
|
||
requestResult = fallbackPass.requestResult;
|
||
maskEnabled = fallbackPass.maskEnabled;
|
||
pathLabel = "seed-image";
|
||
}
|
||
|
||
const shouldRetryWithSeedImageModel =
|
||
(requestResult.upstreamError || "")
|
||
.toLowerCase()
|
||
.includes("unsupported use of 'seedimage' parameter") &&
|
||
model !== fallbackSeedImageModel;
|
||
|
||
if (shouldRetryWithSeedImageModel) {
|
||
fallbackWarning =
|
||
"当前模型不支持草图引导，已自动切换到兼容的实时草图模型。";
|
||
const fallbackResult = await requestWithMaskFallback(
|
||
fallbackSeedImageModel,
|
||
maskEnabled,
|
||
imageUrl
|
||
);
|
||
requestResult = fallbackResult.requestResult;
|
||
maskEnabled = fallbackResult.maskEnabled;
|
||
}
|
||
|
||
if (!requestResult.upstreamOk || requestResult.upstreamError) {
|
||
return Response.json(
|
||
{
|
||
code: "RUNWARE_REQUEST_FAILED",
|
||
message:
|
||
requestResult.upstreamError ||
|
||
requestResult.wsFallbackError ||
|
||
`Runware 请求失败（${requestResult.upstreamStatus}）`,
|
||
},
|
||
{ status: 502 }
|
||
);
|
||
}
|
||
|
||
const { imageResult, imageDataURI, imageURL, outputImage } = extractRunwareOutputImage(
|
||
requestResult.upstreamJson,
|
||
requestResult.taskUUID
|
||
);
|
||
|
||
if (!outputImage) {
|
||
return Response.json(
|
||
{ code: "RUNWARE_EMPTY_OUTPUT", message: "Runware 未返回图像" },
|
||
{ status: 502 }
|
||
);
|
||
}
|
||
|
||
const cost = Number(imageResult?.cost || 0);
|
||
console.log("[RealtimeRunware] frame success", {
|
||
userId: session.user.id,
|
||
requestId: body.requestId || null,
|
||
taskUUID: requestResult.taskUUID,
|
||
model: requestResult.modelId,
|
||
transport: requestResult.transport,
|
||
seed: typeof requestResult.runtimeSeed === "number" ? requestResult.runtimeSeed : null,
|
||
hasUserPrompt,
|
||
promptLength: rawPrompt.length,
|
||
promptPreview: hasUserPrompt ? rawPrompt.slice(0, 32) : null,
|
||
qualityProfile,
|
||
steps: requestResult.runtimeSteps,
|
||
cfgScale: requestResult.runtimeCfgScale,
|
||
strength: requestResult.runtimeStrength,
|
||
strictSketchApplied: requestResult.effectiveStrictSketch,
|
||
maskEnabled,
|
||
roi: body.roi || null,
|
||
cost,
|
||
renderMode,
|
||
path: pathLabel,
|
||
controlNetModel: pathLabel === "controlnet" ? requestResult.controlNetModel : null,
|
||
controlGuideType:
|
||
pathLabel === "controlnet" && controlGuideUsed
|
||
? controlGuideUsed.startsWith("data:")
|
||
? "dataURI"
|
||
: "url"
|
||
: null,
|
||
controlGuideFallback,
|
||
controlPreprocessUsed,
|
||
twoPassStylize: runTwoPassStylize,
|
||
fallbackWarning: fallbackWarning || null,
|
||
});
|
||
|
||
return Response.json({
|
||
ok: true,
|
||
provider: "runware",
|
||
model: requestResult.modelId,
|
||
transport: requestResult.transport,
|
||
taskUUID: requestResult.taskUUID,
|
||
path: pathLabel,
|
||
imageDataURI: imageDataURI || undefined,
|
||
imageUrl: imageURL || undefined,
|
||
cost,
|
||
warning: fallbackWarning || undefined,
|
||
});
|
||
}
|