import { auth } from "@/app/(auth)/auth"; import { getSystemSetting } from "@/lib/db/queries"; import { DEFAULT_BFL_MODEL, DEFAULT_REPLICATE_BASE_URL, DEFAULT_REPLICATE_MODEL, RealtimeCanvasConfig, RealtimeProviderConfig, resolveRealtimeProviders, } from "@/lib/realtime/config"; type GeneratePayload = { requestId?: string; imageUrl?: string; maskImageUrl?: string; qualityProfile?: "preview" | "refine"; roi?: | { x?: number; y?: number; width?: number; height?: number; } | null; prompt?: string; seed?: number; imageSize?: "square" | "square_hd"; steps?: number; cfgScale?: number; strength?: number; }; type BflSubmitResponse = { id?: string; polling_url?: string; response?: { id?: string; polling_url?: string; result?: unknown; }; result?: unknown; sample?: string; error?: unknown; message?: string; }; type ReplicatePredictionResponse = { id?: string; status?: string; output?: unknown; error?: unknown; detail?: unknown; urls?: { get?: string; }; }; const FALLBACK_REALTIME_PROMPT = "严格按照输入草图生成:保持主体轮廓、位置、比例和数量一致,只做材质与光影细化;禁止改变构图,禁止新增无关背景纹理。"; function clamp(value: number, min: number, max: number) { return Math.min(max, Math.max(min, value)); } function resolveSquareSize( imageSize: "square" | "square_hd" | undefined, maxInputResolution: number ) { const preferred = imageSize === "square_hd" ? 1280 : 1024; const capped = Math.min(preferred, maxInputResolution); return clamp(Math.round(capped / 64) * 64, 512, 1536); } function resolveSeed(seed: unknown) { const parsed = Number(seed); if (!Number.isFinite(parsed)) return undefined; const normalized = Math.floor(parsed); if (normalized < 0 || normalized > 2147483647) return undefined; return normalized; } function encodeModelPath(modelId: string) { return modelId .split("/") .filter(Boolean) .map((part) => encodeURIComponent(part)) .join("/"); } function sanitizeText(raw: string) { return raw.replace(/[\u2028\u2029]/g, " ").trim(); } function extractBflImageUrl(payload: any): string { const candidates = [ payload?.sample, payload?.result?.sample, payload?.response?.result?.sample, payload?.output?.[0], payload?.result?.output?.[0], payload?.images?.[0], payload?.result?.images?.[0], payload?.image_url, payload?.result?.image_url, payload?.url, payload?.result?.url, ]; for (const candidate of candidates) { const value = String(candidate || "").trim(); if (!value) continue; if (value.startsWith("data:")) return value; if (/^https?:\/\//i.test(value)) return value; } return ""; } function extractBflErrorMessage(payload: any, fallback: string) { const message = String( payload?.error?.message || payload?.error || payload?.detail || payload?.message || payload?.response?.error?.message || payload?.response?.error || "" ).trim(); return message || fallback; } function shouldRetryWithAlternateImageField(message: string) { const normalized = message.toLowerCase(); return ( normalized.includes("image_prompt") || normalized.includes("input_image") || normalized.includes("unsupported parameter") || normalized.includes("unknown field") || normalized.includes("invalid request") ); } function delay(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)); } function toImagePromptValue(imageUrl: string) { const trimmed = imageUrl.trim(); if (!trimmed) return ""; if (trimmed.startsWith("data:")) { const commaIndex = trimmed.indexOf(","); if (commaIndex > -1) { return trimmed.slice(commaIndex + 1).trim(); } } return trimmed; } function getBflImageFieldCandidates() { // BFL's current FLUX/Kontext editing APIs use `input_image` as the canonical field. // Keep `image_prompt` only as a legacy fallback in case an older compatible endpoint is used. return ["input_image", "image_prompt"] as const; } function getReplicateImageFieldCandidates(imagePrompt: string) { return [ { images: [imagePrompt] }, { image: imagePrompt }, { input_image: imagePrompt }, ] as const; } function isSelfHostedRealtimeProvider(provider: RealtimeProviderConfig) { return !provider.baseUrl.includes("api.bfl.ai"); } function splitReplicateModelIdentifier(modelId: string) { const [owner, ...rest] = modelId.split("/").filter(Boolean); const name = rest.join("/"); if (!owner || !name) { throw new Error(`无效的 Replicate 模型标识:${modelId}`); } return { owner, name }; } function extractReplicateImageUrl(payload: any): string { const output = payload?.output; const candidates = [ Array.isArray(output) ? output[0] : output, payload?.urls?.stream, payload?.result, ]; for (const candidate of candidates) { const value = String(candidate || "").trim(); if (!value) continue; if (value.startsWith("data:")) return value; if (/^https?:\/\//i.test(value)) return value; } return ""; } function extractReplicateErrorMessage(payload: any, fallback: string) { const message = String(payload?.detail || payload?.error || payload?.message || "").trim(); return message || fallback; } async function submitReplicatePrediction(params: { apiKey: string; model: string; input: Record; }): Promise { const { owner, name } = splitReplicateModelIdentifier(params.model); const endpoint = `${DEFAULT_REPLICATE_BASE_URL}/models/${encodeURIComponent(owner)}/${encodeURIComponent(name)}/predictions`; const response = await fetch(endpoint, { method: "POST", headers: { Authorization: `Bearer ${params.apiKey}`, "Content-Type": "application/json", Prefer: "wait=2", }, body: JSON.stringify({ input: params.input }), cache: "no-store", }); const payload = (await response.json().catch(() => null)) as ReplicatePredictionResponse | null; const imageUrl = extractReplicateImageUrl(payload); const taskId = String(payload?.id || "").trim(); const pollingUrl = String(payload?.urls?.get || "").trim(); if (response.ok && (imageUrl || taskId || pollingUrl)) { return { taskId, pollingUrl, imageUrl, errorMessage: "", status: response.status, }; } return { taskId, pollingUrl, imageUrl, errorMessage: extractReplicateErrorMessage(payload, `Replicate 请求失败(${response.status})`), status: response.status, }; } async function pollReplicateResult(params: { apiKey: string; pollingUrl?: string; taskId?: string; timeoutMs?: number; }) { const timeoutMs = Math.max(10_000, Number(params.timeoutMs || 60_000)); const deadline = Date.now() + timeoutMs; let lastMessage = ""; while (Date.now() < deadline) { const taskId = String(params.taskId || "").trim(); const url = String(params.pollingUrl || "").trim() || `${DEFAULT_REPLICATE_BASE_URL}/predictions/${encodeURIComponent(taskId)}`; const response = await fetch(url, { method: "GET", headers: { Authorization: `Bearer ${params.apiKey}`, Accept: "application/json", }, cache: "no-store", }); const payload = (await response.json().catch(() => null)) as ReplicatePredictionResponse | null; const imageUrl = extractReplicateImageUrl(payload); if (imageUrl) return { imageUrl, errorMessage: "", status: response.status }; const status = String(payload?.status || "").trim().toLowerCase(); if (status) { lastMessage = status; } if (status === "succeeded") { return { imageUrl: "", errorMessage: "Replicate 返回成功状态但未携带图像结果", status: response.status, }; } if (status === "failed" || status === "canceled") { return { imageUrl: "", errorMessage: extractReplicateErrorMessage(payload, "Replicate 任务执行失败"), status: response.status, }; } if (!response.ok) { return { imageUrl: "", errorMessage: extractReplicateErrorMessage(payload, `Replicate 轮询失败(${response.status})`), status: response.status, }; } await delay(250); } return { imageUrl: "", errorMessage: `Replicate 轮询超时${lastMessage ? `(${lastMessage})` : ""}`, status: 504, }; } type SubmitTaskResult = { taskId: string; pollingUrl: string; imageUrl: string; errorMessage: string; status: number; }; type ProviderAttemptResult = | { ok: true; imageUrl: string; taskId: string; imageField: "input_image" | "image_prompt"; fallbackWarning: string; provider: RealtimeProviderConfig; } | { ok: false; message: string; status: number; provider: RealtimeProviderConfig; }; async function submitBflTask(params: { apiKey: string; baseUrl: string; model: string; body: Record; }): Promise { const endpoint = `${params.baseUrl}/v1/${encodeModelPath(params.model)}`; const response = await fetch(endpoint, { method: "POST", headers: { "x-key": params.apiKey, "Content-Type": "application/json", Accept: "application/json", }, body: JSON.stringify(params.body), cache: "no-store", }); const payload = (await response.json().catch(() => null)) as BflSubmitResponse | null; const imageUrl = extractBflImageUrl(payload); const taskId = String(payload?.id || payload?.response?.id || "").trim(); const pollingUrl = String(payload?.polling_url || payload?.response?.polling_url || "").trim(); if (response.ok && (imageUrl || taskId || pollingUrl)) { return { taskId, pollingUrl, imageUrl, errorMessage: "", status: response.status, }; } return { taskId, pollingUrl, imageUrl, errorMessage: extractBflErrorMessage(payload, `BFL 请求失败(${response.status})`), status: response.status, }; } async function pollBflResult(params: { apiKey: string; baseUrl: string; pollingUrl?: string; taskId?: string; timeoutMs?: number; }) { const timeoutMs = Math.max(20_000, Number(params.timeoutMs || 100_000)); const deadline = Date.now() + timeoutMs; let lastMessage = ""; while (Date.now() < deadline) { const url = (() => { const direct = String(params.pollingUrl || "").trim(); if (direct) { if (/^https?:\/\//i.test(direct)) return direct; if (direct.startsWith("/")) return `${params.baseUrl}${direct}`; } const taskId = String(params.taskId || "").trim(); return `${params.baseUrl}/v1/get_result?id=${encodeURIComponent(taskId)}`; })(); const response = await fetch(url, { method: "GET", headers: { "x-key": params.apiKey, Accept: "application/json", }, cache: "no-store", }); const payload = await response.json().catch(() => null); const imageUrl = extractBflImageUrl(payload); if (imageUrl) return { imageUrl, errorMessage: "", status: response.status }; const status = String( payload?.status || payload?.task_status || payload?.response?.status || payload?.response?.task_status || "" ) .trim() .toLowerCase(); if (status) { lastMessage = status; } if (status === "ready" || status === "success" || status === "succeeded" || status === "completed") { return { imageUrl: "", errorMessage: "BFL 返回成功状态但未携带图像结果", status: response.status, }; } if (status === "error" || status === "failed" || status === "cancelled") { return { imageUrl: "", errorMessage: extractBflErrorMessage(payload, "BFL 任务执行失败"), status: response.status, }; } if (!response.ok) { return { imageUrl: "", errorMessage: extractBflErrorMessage(payload, `BFL 轮询失败(${response.status})`), status: response.status, }; } await delay(1_200); } return { imageUrl: "", errorMessage: `BFL 轮询超时${lastMessage ? `(${lastMessage})` : ""}`, status: 504, }; } async function generateWithProvider(params: { provider: RealtimeProviderConfig; imagePrompt: string; prompt: string; size: number; seed?: number; body: GeneratePayload; config: RealtimeCanvasConfig; }): Promise { const isReplicate = params.provider.type === "replicate"; const imageFieldCandidates = isReplicate ? getReplicateImageFieldCandidates(params.imagePrompt) : getBflImageFieldCandidates(); let selectedImageField: "input_image" | "image_prompt" = "input_image"; let submitResult: SubmitTaskResult | null = null; let fallbackWarning = ""; for (let index = 0; index < imageFieldCandidates.length; index += 1) { if (isReplicate) { const candidate = getReplicateImageFieldCandidates(params.imagePrompt)[index]; const requestBody: Record = { prompt: params.prompt, ...(typeof params.seed === "number" ? { seed: params.seed } : {}), ...candidate, }; submitResult = await submitReplicatePrediction({ apiKey: params.provider.apiKey, model: params.provider.model || DEFAULT_REPLICATE_MODEL, input: requestBody, }); } else { selectedImageField = imageFieldCandidates[index] as "input_image" | "image_prompt"; const requestBody: Record = { prompt: params.prompt, width: params.size, height: params.size, output_format: "jpeg", safety_tolerance: 2, [selectedImageField]: params.imagePrompt, ...(typeof params.seed === "number" ? { seed: params.seed } : {}), }; if (isSelfHostedRealtimeProvider(params.provider)) { requestBody.steps = clamp(Number(params.body.steps ?? params.config.defaults?.steps ?? 1), 1, 6); requestBody.cfg_scale = clamp( Number(params.body.cfgScale ?? params.config.defaults?.cfgScale ?? 1), 1, 14 ); requestBody.strength = clamp( Number(params.body.strength ?? params.config.defaults?.strength ?? 0.32), 0.05, 1 ); requestBody.preview_megapixels = clamp(Number(params.config.previewMegapixels ?? 0.4), 0.2, 0.8); requestBody.detail_megapixels = clamp(Number(params.config.detailMegapixels ?? 1.0), 0.6, 1.2); requestBody.output_jpeg_quality = clamp( Number(params.config.outputJpegQuality ?? 62), 35, 90 ); requestBody.quality_profile = params.body.qualityProfile || "refine"; } submitResult = await submitBflTask({ apiKey: params.provider.apiKey, baseUrl: params.provider.baseUrl, model: params.provider.model, body: requestBody, }); } if (!submitResult?.errorMessage) { break; } if ( index < imageFieldCandidates.length - 1 && shouldRetryWithAlternateImageField(submitResult.errorMessage) ) { fallbackWarning = "模型参数兼容性已自动切换,继续生成。"; continue; } return { ok: false, message: submitResult?.errorMessage || "实时生成提交失败", status: submitResult?.status || 502, provider: params.provider, }; } if (!submitResult) { return { ok: false, message: "实时生成提交失败", status: 502, provider: params.provider, }; } let finalImageUrl = submitResult.imageUrl; if (!finalImageUrl) { const pollResult = params.provider.type === "replicate" ? await pollReplicateResult({ apiKey: params.provider.apiKey, pollingUrl: submitResult.pollingUrl, taskId: submitResult.taskId, timeoutMs: 60_000, }) : await pollBflResult({ apiKey: params.provider.apiKey, baseUrl: params.provider.baseUrl, pollingUrl: submitResult.pollingUrl, taskId: submitResult.taskId, timeoutMs: 110_000, }); if (!pollResult.imageUrl) { return { ok: false, message: pollResult.errorMessage || "实时生成未返回图像", status: pollResult.status || 502, provider: params.provider, }; } finalImageUrl = pollResult.imageUrl; } return { ok: true, imageUrl: finalImageUrl, taskId: submitResult.taskId, imageField: selectedImageField, fallbackWarning, provider: params.provider, }; } export async function POST(request: Request) { const session = await auth(); if (!session?.user?.id) { return Response.json({ code: "UNAUTHORIZED", message: "请先登录" }, { status: 401 }); } let body: GeneratePayload; try { body = (await request.json()) as GeneratePayload; } catch { return Response.json({ code: "BAD_REQUEST", message: "无效 JSON" }, { status: 400 }); } const imageUrl = String(body.imageUrl || "").trim(); if (!imageUrl) { return Response.json({ code: "BAD_REQUEST", message: "缺少 imageUrl" }, { status: 400 }); } const setting = await getSystemSetting({ key: "realtime_canvas_config" }).catch(() => null); const config = (setting?.value || {}) as RealtimeCanvasConfig; if (config.enabled === false) { return Response.json( { code: "FEATURE_DISABLED", message: "实时画布功能未开启" }, { status: 403 } ); } const providers = resolveRealtimeProviders(config); if (providers.length === 0) { return Response.json( { code: "BFL_KEY_MISSING", message: "未配置可用的实时画布服务", }, { status: 500 } ); } const maxInputResolution = clamp(Number(config.maxInputResolution || 1024), 512, 1536); const size = resolveSquareSize(body.imageSize, maxInputResolution); const seed = resolveSeed(body.seed); const rawPrompt = sanitizeText(String(body.prompt || "").trim()) || FALLBACK_REALTIME_PROMPT; const negativePrompt = sanitizeText(String(config.negativePrompt || "").trim()) || "改变构图, 改变主体位置, 多余人物, 重复物体, 镜像翻转, 错误透视, 模糊轮廓, 文字水印, 线稿, 草图"; const positivePrompt = sanitizeText( [ rawPrompt, "Keep exact object silhouette, placement and count from the input sketch.", "Photorealistic material and lighting only, keep composition unchanged.", `Negative constraints: ${negativePrompt}`, ].join(" ") ); const imagePrompt = toImagePromptValue(imageUrl); if (!imagePrompt) { return Response.json( { code: "BAD_REQUEST", message: "输入图像格式无效", }, { status: 400 } ); } const providerErrors: string[] = []; for (const provider of providers) { const result = await generateWithProvider({ provider, imagePrompt, prompt: positivePrompt, size, seed, body, config, }); if (!result.ok) { providerErrors.push(`${provider.label}(${provider.baseUrl}): ${result.message}`); continue; } const warningParts = [result.fallbackWarning]; if (result.provider.source === "fallback") { warningParts.push("主实时服务异常,已自动切换到备用服务。"); } const warning = warningParts.filter(Boolean).join(" "); console.log("[RealtimeBFL] frame success", { userId: session.user.id, requestId: body.requestId || null, model: result.provider.model || DEFAULT_BFL_MODEL, providerSource: result.provider.source, providerBaseUrl: result.provider.baseUrl, taskId: result.taskId || null, hasPrompt: Boolean(rawPrompt && rawPrompt !== FALLBACK_REALTIME_PROMPT), imageField: result.imageField, qualityProfile: body.qualityProfile || "refine", size, roi: body.roi || null, hasMask: Boolean(body.maskImageUrl), warning: warning || null, }); return Response.json({ ok: true, provider: "bfl", providerSource: result.provider.source, model: result.provider.model, transport: "http-poll", taskUUID: result.taskId || undefined, imageDataURI: result.imageUrl.startsWith("data:") ? result.imageUrl : !/^https?:\/\//i.test(result.imageUrl) ? `data:image/jpeg;base64,${result.imageUrl}` : undefined, imageUrl: /^https?:\/\//i.test(result.imageUrl) ? result.imageUrl : undefined, warning: warning || undefined, }); } return Response.json( { code: "BFL_REQUEST_FAILED", message: providerErrors.length > 0 ? `主备用实时服务均不可用:${providerErrors.join(";")}` : "实时生成失败", }, { status: 502 } ); }