diff --git a/app/(admin)/settings/admin-settings-client.tsx b/app/(admin)/settings/admin-settings-client.tsx index 4fc1c830c..72422a7dc 100644 --- a/app/(admin)/settings/admin-settings-client.tsx +++ b/app/(admin)/settings/admin-settings-client.tsx @@ -900,13 +900,19 @@ export function AdminSettingsClient() { // 实时画布API配置状态 const [realtimeCanvasConfig, setRealtimeCanvasConfig] = useState({ enabled: true, + providerType: "bfl" as "bfl" | "replicate", bflApiKey: "", bflBaseUrl: DEFAULT_BFL_BASE_URL, bflModel: DEFAULT_BFL_REALTIME_MODEL, + replicateApiToken: "", + replicateModel: "black-forest-labs/flux-2-klein-4b", fallbackEnabled: false, + fallbackProviderType: "bfl" as "bfl" | "replicate", fallbackBflApiKey: "", fallbackBflBaseUrl: DEFAULT_BFL_BASE_URL, fallbackBflModel: DEFAULT_BFL_REALTIME_MODEL, + fallbackReplicateApiToken: "", + fallbackReplicateModel: "black-forest-labs/flux-2-klein-4b", renderMode: "balanced" as "strict-structure" | "balanced" | "stylize", throttleInterval: 100, maxBuffering: 2, @@ -1065,16 +1071,22 @@ export function AdminSettingsClient() { if (realtimeCanvasConfigData?.setting?.value) { const config = realtimeCanvasConfigData.setting.value as { enabled?: boolean; + providerType?: "bfl" | "replicate"; bflApiKey?: string; bflBaseUrl?: string; bflModel?: string; + replicateApiToken?: string; + replicateModel?: string; fallbackEnabled?: boolean; + fallbackProviderType?: "bfl" | "replicate"; fallbackBflApiKey?: string; fallbackBflBaseUrl?: string; fallbackBflModel?: string; fallbackRunwareApiKey?: string; fallbackFalApiKey?: string; fallbackRunwareModel?: string; + fallbackReplicateApiToken?: string; + fallbackReplicateModel?: string; runwareApiKey?: string; falApiKey?: string; runwareModel?: string; @@ -1095,10 +1107,14 @@ export function AdminSettingsClient() { }; setRealtimeCanvasConfig({ enabled: config.enabled ?? true, + providerType: config.providerType === "replicate" ? "replicate" : "bfl", bflApiKey: config.bflApiKey || config.runwareApiKey || config.falApiKey || "", bflBaseUrl: config.bflBaseUrl || DEFAULT_BFL_BASE_URL, bflModel: config.bflModel || config.runwareModel || DEFAULT_BFL_REALTIME_MODEL, + replicateApiToken: config.replicateApiToken || "", + replicateModel: config.replicateModel || "black-forest-labs/flux-2-klein-4b", fallbackEnabled: config.fallbackEnabled === true, + fallbackProviderType: config.fallbackProviderType === "replicate" ? "replicate" : "bfl", fallbackBflApiKey: config.fallbackBflApiKey || config.fallbackRunwareApiKey || @@ -1107,6 +1123,9 @@ export function AdminSettingsClient() { fallbackBflBaseUrl: config.fallbackBflBaseUrl || DEFAULT_BFL_BASE_URL, fallbackBflModel: config.fallbackBflModel || config.fallbackRunwareModel || DEFAULT_BFL_REALTIME_MODEL, + fallbackReplicateApiToken: config.fallbackReplicateApiToken || "", + fallbackReplicateModel: + config.fallbackReplicateModel || "black-forest-labs/flux-2-klein-4b", renderMode: config.renderMode === "strict-structure" || config.renderMode === "stylize" ? config.renderMode @@ -2208,23 +2227,48 @@ export function AdminSettingsClient() { // --- 实时画布API配置逻辑 --- const handleSaveRealtimeCanvasConfig = async () => { try { + const primaryProviderType = realtimeCanvasConfig.providerType; const bflApiKey = realtimeCanvasConfig.bflApiKey.trim(); const bflBaseUrl = realtimeCanvasConfig.bflBaseUrl.trim() || DEFAULT_BFL_BASE_URL; const bflModel = realtimeCanvasConfig.bflModel.trim() || DEFAULT_BFL_REALTIME_MODEL; + const replicateApiToken = realtimeCanvasConfig.replicateApiToken.trim(); + const replicateModel = + realtimeCanvasConfig.replicateModel.trim() || "black-forest-labs/flux-2-klein-4b"; const fallbackEnabled = realtimeCanvasConfig.fallbackEnabled; + const fallbackProviderType = realtimeCanvasConfig.fallbackProviderType; const fallbackBflApiKey = realtimeCanvasConfig.fallbackBflApiKey.trim(); const fallbackBflBaseUrl = realtimeCanvasConfig.fallbackBflBaseUrl.trim() || DEFAULT_BFL_BASE_URL; const fallbackBflModel = realtimeCanvasConfig.fallbackBflModel.trim() || DEFAULT_BFL_REALTIME_MODEL; + const fallbackReplicateApiToken = realtimeCanvasConfig.fallbackReplicateApiToken.trim(); + const fallbackReplicateModel = + realtimeCanvasConfig.fallbackReplicateModel.trim() || "black-forest-labs/flux-2-klein-4b"; - if (realtimeCanvasConfig.enabled && !bflApiKey) { - toast.error("启用实时画布功能需要配置 BFL API Key"); + if ( + realtimeCanvasConfig.enabled && + ((primaryProviderType === "bfl" && !bflApiKey) || + (primaryProviderType === "replicate" && !replicateApiToken)) + ) { + toast.error( + primaryProviderType === "replicate" + ? "启用实时画布功能需要配置 Replicate API Token" + : "启用实时画布功能需要配置 BFL API Key" + ); return; } - if (realtimeCanvasConfig.enabled && fallbackEnabled && !fallbackBflApiKey) { - toast.error("启用备用实时服务需要配置备用 BFL API Key"); + if ( + realtimeCanvasConfig.enabled && + fallbackEnabled && + ((fallbackProviderType === "bfl" && !fallbackBflApiKey) || + (fallbackProviderType === "replicate" && !fallbackReplicateApiToken)) + ) { + toast.error( + fallbackProviderType === "replicate" + ? "启用备用实时服务需要配置备用 Replicate API Token" + : "启用备用实时服务需要配置备用 BFL API Key" + ); return; } @@ -2233,9 +2277,10 @@ export function AdminSettingsClient() { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ - apiKey: bflApiKey, - modelId: bflModel, - baseUrl: bflBaseUrl, + providerType: primaryProviderType, + apiKey: primaryProviderType === "replicate" ? replicateApiToken : bflApiKey, + modelId: primaryProviderType === "replicate" ? replicateModel : bflModel, + baseUrl: primaryProviderType === "replicate" ? undefined : bflBaseUrl, }), }); const probeData = await probeResponse.json().catch(() => null); @@ -2249,9 +2294,14 @@ export function AdminSettingsClient() { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ - apiKey: fallbackBflApiKey, - modelId: fallbackBflModel, - baseUrl: fallbackBflBaseUrl, + providerType: fallbackProviderType, + apiKey: + fallbackProviderType === "replicate" + ? fallbackReplicateApiToken + : fallbackBflApiKey, + modelId: + fallbackProviderType === "replicate" ? fallbackReplicateModel : fallbackBflModel, + baseUrl: fallbackProviderType === "replicate" ? undefined : fallbackBflBaseUrl, }), }); const fallbackProbeData = await fallbackProbeResponse.json().catch(() => null); @@ -2269,13 +2319,19 @@ export function AdminSettingsClient() { key: "realtime_canvas_config", value: { enabled: realtimeCanvasConfig.enabled, + providerType: primaryProviderType, bflApiKey, bflBaseUrl, bflModel, + replicateApiToken, + replicateModel, fallbackEnabled, + fallbackProviderType, fallbackBflApiKey, fallbackBflBaseUrl, fallbackBflModel, + fallbackReplicateApiToken, + fallbackReplicateModel, runwareApiKey: bflApiKey, falApiKey: bflApiKey, runwareModel: bflModel, @@ -9129,48 +9185,109 @@ export function AdminSettingsClient() {
- - - setRealtimeCanvasConfig({ ...realtimeCanvasConfig, bflApiKey: e.target.value }) + +
-
- - - setRealtimeCanvasConfig({ ...realtimeCanvasConfig, bflBaseUrl: e.target.value }) - } - placeholder={DEFAULT_BFL_BASE_URL} - disabled={!realtimeCanvasConfig.enabled} - /> -
+ {realtimeCanvasConfig.providerType === "replicate" ? ( + <> +
+ + + setRealtimeCanvasConfig({ + ...realtimeCanvasConfig, + replicateApiToken: e.target.value, + }) + } + placeholder="r8_xxxxxxxxxxxxx" + disabled={!realtimeCanvasConfig.enabled} + /> +
-
- - - setRealtimeCanvasConfig({ ...realtimeCanvasConfig, bflModel: e.target.value }) - } - placeholder={DEFAULT_BFL_REALTIME_MODEL} - disabled={!realtimeCanvasConfig.enabled} - /> -
+
+ + + setRealtimeCanvasConfig({ + ...realtimeCanvasConfig, + replicateModel: e.target.value, + }) + } + placeholder="black-forest-labs/flux-2-klein-4b" + disabled={!realtimeCanvasConfig.enabled} + /> +
+ + ) : ( + <> +
+ + + setRealtimeCanvasConfig({ ...realtimeCanvasConfig, bflApiKey: e.target.value }) + } + placeholder="bfl_xxxxxxxxxxxxx" + disabled={!realtimeCanvasConfig.enabled} + /> +
+ +
+ + + setRealtimeCanvasConfig({ ...realtimeCanvasConfig, bflBaseUrl: e.target.value }) + } + placeholder={DEFAULT_BFL_BASE_URL} + disabled={!realtimeCanvasConfig.enabled} + /> +
+ +
+ + + setRealtimeCanvasConfig({ ...realtimeCanvasConfig, bflModel: e.target.value }) + } + placeholder={DEFAULT_BFL_REALTIME_MODEL} + disabled={!realtimeCanvasConfig.enabled} + /> +
+ + )}
@@ -9190,53 +9307,110 @@ export function AdminSettingsClient() {
- - - setRealtimeCanvasConfig({ - ...realtimeCanvasConfig, - fallbackBflApiKey: e.target.value, - }) + +
-
- - - setRealtimeCanvasConfig({ - ...realtimeCanvasConfig, - fallbackBflBaseUrl: e.target.value, - }) - } - placeholder={DEFAULT_BFL_BASE_URL} - disabled={!realtimeCanvasConfig.enabled || !realtimeCanvasConfig.fallbackEnabled} - /> -
+ {realtimeCanvasConfig.fallbackProviderType === "replicate" ? ( + <> +
+ + + setRealtimeCanvasConfig({ + ...realtimeCanvasConfig, + fallbackReplicateApiToken: e.target.value, + }) + } + placeholder="r8_xxxxxxxxxxxxx" + disabled={!realtimeCanvasConfig.enabled || !realtimeCanvasConfig.fallbackEnabled} + /> +
-
- - - setRealtimeCanvasConfig({ - ...realtimeCanvasConfig, - fallbackBflModel: e.target.value, - }) - } - placeholder={DEFAULT_BFL_REALTIME_MODEL} - disabled={!realtimeCanvasConfig.enabled || !realtimeCanvasConfig.fallbackEnabled} - /> -
+
+ + + setRealtimeCanvasConfig({ + ...realtimeCanvasConfig, + fallbackReplicateModel: e.target.value, + }) + } + placeholder="black-forest-labs/flux-2-klein-4b" + disabled={!realtimeCanvasConfig.enabled || !realtimeCanvasConfig.fallbackEnabled} + /> +
+ + ) : ( + <> +
+ + + setRealtimeCanvasConfig({ + ...realtimeCanvasConfig, + fallbackBflApiKey: e.target.value, + }) + } + placeholder="bfl_xxxxxxxxxxxxx" + disabled={!realtimeCanvasConfig.enabled || !realtimeCanvasConfig.fallbackEnabled} + /> +
+ +
+ + + setRealtimeCanvasConfig({ + ...realtimeCanvasConfig, + fallbackBflBaseUrl: e.target.value, + }) + } + placeholder={DEFAULT_BFL_BASE_URL} + disabled={!realtimeCanvasConfig.enabled || !realtimeCanvasConfig.fallbackEnabled} + /> +
+ +
+ + + setRealtimeCanvasConfig({ + ...realtimeCanvasConfig, + fallbackBflModel: e.target.value, + }) + } + placeholder={DEFAULT_BFL_REALTIME_MODEL} + disabled={!realtimeCanvasConfig.enabled || !realtimeCanvasConfig.fallbackEnabled} + /> +
+ + )}
diff --git a/app/api/admin/realtime/model-capability/route.ts b/app/api/admin/realtime/model-capability/route.ts index d9166755d..6d392a7e6 100644 --- a/app/api/admin/realtime/model-capability/route.ts +++ b/app/api/admin/realtime/model-capability/route.ts @@ -2,8 +2,11 @@ import { z } from "zod"; import { requireAdmin } from "@/lib/middleware/admin"; import { ChatSDKError } from "@/lib/errors"; import { probeBflModelSupport } from "@/lib/realtime/bfl-capability"; +import { probeReplicateModelSupport } from "@/lib/realtime/replicate-capability"; +import type { RealtimeProviderType } from "@/lib/realtime/config"; const schema = z.object({ + providerType: z.enum(["bfl", "replicate"]).optional(), apiKey: z.string().min(1), modelId: z.string().min(1), baseUrl: z.string().optional(), @@ -20,11 +23,19 @@ export async function POST(request: Request) { const body = await request.json(); const data = schema.parse(body); - const result = await probeBflModelSupport({ - apiKey: data.apiKey, - modelId: data.modelId, - baseUrl: data.baseUrl, - }); + const providerType: RealtimeProviderType = data.providerType === "replicate" ? "replicate" : "bfl"; + const result = + providerType === "replicate" + ? await probeReplicateModelSupport({ + apiKey: data.apiKey, + modelId: data.modelId, + baseUrl: data.baseUrl, + }) + : await probeBflModelSupport({ + apiKey: data.apiKey, + modelId: data.modelId, + baseUrl: data.baseUrl, + }); if (!result.supported) { return Response.json(result, { status: mapStatus(result.code) }); diff --git a/app/api/admin/settings/route.ts b/app/api/admin/settings/route.ts index faaae2080..18f87b7a2 100644 --- a/app/api/admin/settings/route.ts +++ b/app/api/admin/settings/route.ts @@ -7,6 +7,7 @@ import { } from "@/lib/db/queries"; import { ChatSDKError } from "@/lib/errors"; import { probeBflModelSupport } from "@/lib/realtime/bfl-capability"; +import { probeReplicateModelSupport } from "@/lib/realtime/replicate-capability"; import { normalizeRealtimeCanvasConfigForStorage, resolveRealtimeProviders, @@ -72,11 +73,18 @@ export async function POST(request: Request) { } for (const provider of providers) { - const probeResult = await probeBflModelSupport({ - apiKey: provider.apiKey, - modelId: provider.model, - baseUrl: provider.baseUrl, - }); + const probeResult = + provider.type === "replicate" + ? await probeReplicateModelSupport({ + apiKey: provider.apiKey, + modelId: provider.model, + baseUrl: provider.baseUrl, + }) + : await probeBflModelSupport({ + apiKey: provider.apiKey, + modelId: provider.model, + baseUrl: provider.baseUrl, + }); if (!probeResult.supported) { return Response.json( diff --git a/app/api/realtime/config/route.ts b/app/api/realtime/config/route.ts index 99060e0b9..a06fad509 100644 --- a/app/api/realtime/config/route.ts +++ b/app/api/realtime/config/route.ts @@ -117,7 +117,7 @@ export async function GET() { return Response.json({ ...defaultConfig, provider: { - type: "bfl", + type: activeProvider.type, baseUrl: activeProvider.baseUrl || DEFAULT_BFL_BASE_URL, }, failover: { @@ -125,7 +125,7 @@ export async function GET() { fallbackProvider: providers.length > 1 ? { - type: "bfl", + type: providers[1].type, baseUrl: providers[1].baseUrl, model: providers[1].model, } diff --git a/app/api/realtime/generate/route.ts b/app/api/realtime/generate/route.ts index cd77e243c..6005d7f1a 100644 --- a/app/api/realtime/generate/route.ts +++ b/app/api/realtime/generate/route.ts @@ -2,6 +2,8 @@ import { auth } from "@/app/(auth)/auth"; import { getSystemSetting } from "@/lib/db/queries"; import { DEFAULT_BFL_MODEL, + DEFAULT_REPLICATE_BASE_URL, + DEFAULT_REPLICATE_MODEL, RealtimeCanvasConfig, RealtimeProviderConfig, resolveRealtimeProviders, @@ -23,6 +25,9 @@ type GeneratePayload = { prompt?: string; seed?: number; imageSize?: "square" | "square_hd"; + steps?: number; + cfgScale?: number; + strength?: number; }; type BflSubmitResponse = { @@ -39,6 +44,17 @@ type BflSubmitResponse = { message?: string; }; +type ReplicatePredictionResponse = { + id?: string; + status?: string; + output?: unknown; + error?: unknown; + detail?: unknown; + urls?: { + get?: string; + }; +}; + const FALLBACK_REALTIME_PROMPT = "严格按照输入草图生成：保持主体轮廓、位置、比例和数量一致，只做材质与光影细化；禁止改变构图，禁止新增无关背景纹理。"; @@ -146,6 +162,159 @@ function getBflImageFieldCandidates() { return ["input_image", "image_prompt"] as const; } +function getReplicateImageFieldCandidates(imagePrompt: string) { + return [ + { images: [imagePrompt] }, + { image: imagePrompt }, + { input_image: imagePrompt }, + ] as const; +} + +function isSelfHostedRealtimeProvider(provider: RealtimeProviderConfig) { + return !provider.baseUrl.includes("api.bfl.ai"); +} + +function splitReplicateModelIdentifier(modelId: string) { + const [owner, ...rest] = modelId.split("/").filter(Boolean); + const name = rest.join("/"); + if (!owner || !name) { + throw new Error(`无效的 Replicate 模型标识：${modelId}`); + } + return { owner, name }; +} + +function extractReplicateImageUrl(payload: any): string { + const output = payload?.output; + const candidates = [ + Array.isArray(output) ? output[0] : output, + payload?.urls?.stream, + payload?.result, + ]; + + for (const candidate of candidates) { + const value = String(candidate || "").trim(); + if (!value) continue; + if (value.startsWith("data:")) return value; + if (/^https?:\/\//i.test(value)) return value; + } + + return ""; +} + +function extractReplicateErrorMessage(payload: any, fallback: string) { + const message = String(payload?.detail || payload?.error || payload?.message || "").trim(); + return message || fallback; +} + +async function submitReplicatePrediction(params: { + apiKey: string; + model: string; + input: Record; +}): Promise { + const { owner, name } = splitReplicateModelIdentifier(params.model); + const endpoint = `${DEFAULT_REPLICATE_BASE_URL}/models/${encodeURIComponent(owner)}/${encodeURIComponent(name)}/predictions`; + const response = await fetch(endpoint, { + method: "POST", + headers: { + Authorization: `Bearer ${params.apiKey}`, + "Content-Type": "application/json", + Prefer: "wait=2", + }, + body: JSON.stringify({ input: params.input }), + cache: "no-store", + }); + const payload = (await response.json().catch(() => null)) as ReplicatePredictionResponse | null; + const imageUrl = extractReplicateImageUrl(payload); + const taskId = String(payload?.id || "").trim(); + const pollingUrl = String(payload?.urls?.get || "").trim(); + + if (response.ok && (imageUrl || taskId || pollingUrl)) { + return { + taskId, + pollingUrl, + imageUrl, + errorMessage: "", + status: response.status, + }; + } + + return { + taskId, + pollingUrl, + imageUrl, + errorMessage: extractReplicateErrorMessage(payload, `Replicate 请求失败（${response.status}）`), + status: response.status, + }; +} + +async function pollReplicateResult(params: { + apiKey: string; + pollingUrl?: string; + taskId?: string; + timeoutMs?: number; +}) { + const timeoutMs = Math.max(10_000, Number(params.timeoutMs || 60_000)); + const deadline = Date.now() + timeoutMs; + let lastMessage = ""; + + while (Date.now() < deadline) { + const taskId = String(params.taskId || "").trim(); + const url = + String(params.pollingUrl || "").trim() || + `${DEFAULT_REPLICATE_BASE_URL}/predictions/${encodeURIComponent(taskId)}`; + + const response = await fetch(url, { + method: "GET", + headers: { + Authorization: `Bearer ${params.apiKey}`, + Accept: "application/json", + }, + cache: "no-store", + }); + + const payload = (await response.json().catch(() => null)) as ReplicatePredictionResponse | null; + const imageUrl = extractReplicateImageUrl(payload); + if (imageUrl) return { imageUrl, errorMessage: "", status: response.status }; + + const status = String(payload?.status || "").trim().toLowerCase(); + if (status) { + lastMessage = status; + } + + if (status === "succeeded") { + return { + imageUrl: "", + errorMessage: "Replicate 返回成功状态但未携带图像结果", + status: response.status, + }; + } + + if (status === "failed" || status === "canceled") { + return { + imageUrl: "", + errorMessage: extractReplicateErrorMessage(payload, "Replicate 任务执行失败"), + status: response.status, + }; + } + + if (!response.ok) { + return { + imageUrl: "", + errorMessage: extractReplicateErrorMessage(payload, `Replicate 轮询失败（${response.status}）`), + status: response.status, + }; + } + + await delay(250); + } + + return { + imageUrl: "", + errorMessage: `Replicate 轮询超时${lastMessage ? `（${lastMessage}）` : ""}`, + status: 504, + }; +} + type SubmitTaskResult = { taskId: string; pollingUrl: string; @@ -298,33 +467,75 @@ async function generateWithProvider(params: { prompt: string; size: number; seed?: number; + body: GeneratePayload; + config: RealtimeCanvasConfig; }): Promise { - const imageFieldCandidates = getBflImageFieldCandidates(); + const isReplicate = params.provider.type === "replicate"; + const imageFieldCandidates = isReplicate + ? getReplicateImageFieldCandidates(params.imagePrompt) + : getBflImageFieldCandidates(); - let selectedImageField: "input_image" | "image_prompt" = imageFieldCandidates[0]; + let selectedImageField: "input_image" | "image_prompt" = "input_image"; let submitResult: SubmitTaskResult | null = null; let fallbackWarning = ""; for (let index = 0; index < imageFieldCandidates.length; index += 1) { - selectedImageField = imageFieldCandidates[index]; - const requestBody: Record = { - prompt: params.prompt, - width: params.size, - height: params.size, - output_format: "jpeg", - safety_tolerance: 2, - [selectedImageField]: params.imagePrompt, - ...(typeof params.seed === "number" ? { seed: params.seed } : {}), - }; + if (isReplicate) { + const candidate = getReplicateImageFieldCandidates(params.imagePrompt)[index]; + const requestBody: Record = { + prompt: params.prompt, + ...(typeof params.seed === "number" ? { seed: params.seed } : {}), + ...candidate, + }; - submitResult = await submitBflTask({ - apiKey: params.provider.apiKey, - baseUrl: params.provider.baseUrl, - model: params.provider.model, - body: requestBody, - }); + submitResult = await submitReplicatePrediction({ + apiKey: params.provider.apiKey, + model: params.provider.model || DEFAULT_REPLICATE_MODEL, + input: requestBody, + }); + } else { + selectedImageField = imageFieldCandidates[index] as "input_image" | "image_prompt"; + const requestBody: Record = { + prompt: params.prompt, + width: params.size, + height: params.size, + output_format: "jpeg", + safety_tolerance: 2, + [selectedImageField]: params.imagePrompt, + ...(typeof params.seed === "number" ? { seed: params.seed } : {}), + }; - if (!submitResult.errorMessage) { + if (isSelfHostedRealtimeProvider(params.provider)) { + requestBody.steps = clamp(Number(params.body.steps ?? params.config.defaults?.steps ?? 1), 1, 6); + requestBody.cfg_scale = clamp( + Number(params.body.cfgScale ?? params.config.defaults?.cfgScale ?? 1), + 1, + 14 + ); + requestBody.strength = clamp( + Number(params.body.strength ?? params.config.defaults?.strength ?? 0.32), + 0.05, + 1 + ); + requestBody.preview_megapixels = clamp(Number(params.config.previewMegapixels ?? 0.4), 0.2, 0.8); + requestBody.detail_megapixels = clamp(Number(params.config.detailMegapixels ?? 1.0), 0.6, 1.2); + requestBody.output_jpeg_quality = clamp( + Number(params.config.outputJpegQuality ?? 62), + 35, + 90 + ); + requestBody.quality_profile = params.body.qualityProfile || "refine"; + } + + submitResult = await submitBflTask({ + apiKey: params.provider.apiKey, + baseUrl: params.provider.baseUrl, + model: params.provider.model, + body: requestBody, + }); + } + + if (!submitResult?.errorMessage) { break; } @@ -338,8 +549,8 @@ async function generateWithProvider(params: { return { ok: false, - message: submitResult.errorMessage || "实时生成提交失败", - status: submitResult.status || 502, + message: submitResult?.errorMessage || "实时生成提交失败", + status: submitResult?.status || 502, provider: params.provider, }; } @@ -355,13 +566,21 @@ async function generateWithProvider(params: { let finalImageUrl = submitResult.imageUrl; if (!finalImageUrl) { - const pollResult = await pollBflResult({ - apiKey: params.provider.apiKey, - baseUrl: params.provider.baseUrl, - pollingUrl: submitResult.pollingUrl, - taskId: submitResult.taskId, - timeoutMs: 110_000, - }); + const pollResult = + params.provider.type === "replicate" + ? await pollReplicateResult({ + apiKey: params.provider.apiKey, + pollingUrl: submitResult.pollingUrl, + taskId: submitResult.taskId, + timeoutMs: 60_000, + }) + : await pollBflResult({ + apiKey: params.provider.apiKey, + baseUrl: params.provider.baseUrl, + pollingUrl: submitResult.pollingUrl, + taskId: submitResult.taskId, + timeoutMs: 110_000, + }); if (!pollResult.imageUrl) { return { ok: false, @@ -459,6 +678,8 @@ export async function POST(request: Request) { prompt: positivePrompt, size, seed, + body, + config, }); if (!result.ok) { diff --git a/lib/realtime/config.ts b/lib/realtime/config.ts index 20d600592..140e8206d 100644 --- a/lib/realtime/config.ts +++ b/lib/realtime/config.ts @@ -1,8 +1,13 @@ export const DEFAULT_BFL_MODEL = "flux-2-klein-4b"; export const DEFAULT_BFL_BASE_URL = "https://api.bfl.ai"; +export const DEFAULT_REPLICATE_MODEL = "black-forest-labs/flux-2-klein-4b"; +export const DEFAULT_REPLICATE_BASE_URL = "https://api.replicate.com/v1"; + +export type RealtimeProviderType = "bfl" | "replicate"; export type RealtimeCanvasConfig = { enabled?: boolean; + providerType?: RealtimeProviderType; bflApiKey?: string; bflModel?: string; bflBaseUrl?: string; @@ -10,7 +15,10 @@ export type RealtimeCanvasConfig = { runwareModel?: string; falApiKey?: string; apiKey?: string; + replicateApiToken?: string; + replicateModel?: string; fallbackEnabled?: boolean; + fallbackProviderType?: RealtimeProviderType; fallbackBflApiKey?: string; fallbackBflModel?: string; fallbackBflBaseUrl?: string; @@ -18,6 +26,8 @@ export type RealtimeCanvasConfig = { fallbackRunwareModel?: string; fallbackFalApiKey?: string; fallbackApiKey?: string; + fallbackReplicateApiToken?: string; + fallbackReplicateModel?: string; negativePrompt?: string; renderMode?: "strict-structure" | "balanced" | "stylize"; strictSketch?: boolean; @@ -25,6 +35,9 @@ export type RealtimeCanvasConfig = { maxBuffering?: number; maxInputResolution?: number; jpegQuality?: number; + previewMegapixels?: number; + detailMegapixels?: number; + outputJpegQuality?: number; defaults?: { steps?: number; cfgScale?: number; @@ -36,6 +49,7 @@ export type RealtimeCanvasConfig = { }; export type RealtimeProviderConfig = { + type: RealtimeProviderType; apiKey: string; baseUrl: string; model: string; @@ -47,6 +61,10 @@ function trimString(value: unknown) { return String(value || "").trim(); } +function normalizeProviderType(raw: unknown): RealtimeProviderType { + return raw === "replicate" ? "replicate" : "bfl"; +} + export function normalizeBaseUrl(raw: string | undefined) { const next = trimString(raw || DEFAULT_BFL_BASE_URL); return (next || DEFAULT_BFL_BASE_URL).replace(/\/+$/, ""); @@ -64,7 +82,9 @@ function hasFallbackIntent(config: RealtimeCanvasConfig) { config.fallbackApiKey || config.fallbackBflBaseUrl || config.fallbackBflModel || - config.fallbackRunwareModel + config.fallbackRunwareModel || + config.fallbackReplicateApiToken || + config.fallbackReplicateModel ) ); } @@ -73,6 +93,22 @@ export function resolvePrimaryRealtimeProvider( config: RealtimeCanvasConfig, env: NodeJS.ProcessEnv = process.env ): RealtimeProviderConfig | null { + const type = normalizeProviderType(config.providerType); + + if (type === "replicate") { + const apiKey = trimString(config.replicateApiToken || env.REPLICATE_API_TOKEN); + if (!apiKey) return null; + + return { + type, + apiKey, + model: trimString(config.replicateModel) || DEFAULT_REPLICATE_MODEL, + baseUrl: DEFAULT_REPLICATE_BASE_URL, + source: "primary", + label: "主实时服务", + }; + } + const apiKey = trimString( config.bflApiKey || config.runwareApiKey || @@ -89,6 +125,7 @@ export function resolvePrimaryRealtimeProvider( const baseUrl = normalizeBaseUrl(config.bflBaseUrl || env.BFL_BASE_URL); return { + type, apiKey, model, baseUrl, @@ -102,6 +139,22 @@ export function resolveFallbackRealtimeProvider( ): RealtimeProviderConfig | null { if (!hasFallbackIntent(config)) return null; + const type = normalizeProviderType(config.fallbackProviderType); + + if (type === "replicate") { + const apiKey = trimString(config.fallbackReplicateApiToken); + if (!apiKey) return null; + + return { + type, + apiKey, + model: trimString(config.fallbackReplicateModel) || DEFAULT_REPLICATE_MODEL, + baseUrl: DEFAULT_REPLICATE_BASE_URL, + source: "fallback", + label: "备用实时服务", + }; + } + const apiKey = trimString( config.fallbackBflApiKey || config.fallbackRunwareApiKey || @@ -115,6 +168,7 @@ export function resolveFallbackRealtimeProvider( const baseUrl = normalizeBaseUrl(config.fallbackBflBaseUrl || DEFAULT_BFL_BASE_URL); return { + type, apiKey, model, baseUrl, @@ -166,21 +220,82 @@ export function normalizeRealtimeCanvasConfigForStorage( return { ...value, - bflApiKey: primary?.apiKey || "", - bflModel: primary?.model || DEFAULT_BFL_MODEL, - bflBaseUrl: primary?.baseUrl || normalizeBaseUrl(env.BFL_BASE_URL), - runwareApiKey: primary?.apiKey || "", - falApiKey: primary?.apiKey || "", - apiKey: primary?.apiKey || "", - runwareModel: primary?.model || DEFAULT_BFL_MODEL, + providerType: primary?.type || normalizeProviderType(value.providerType), + bflApiKey: + primary?.type === "bfl" + ? primary.apiKey + : trimString((value as RealtimeCanvasConfig).bflApiKey), + bflModel: + primary?.type === "bfl" + ? primary.model + : trimString((value as RealtimeCanvasConfig).bflModel) || DEFAULT_BFL_MODEL, + bflBaseUrl: + primary?.type === "bfl" + ? primary.baseUrl + : normalizeBaseUrl((value as RealtimeCanvasConfig).bflBaseUrl || env.BFL_BASE_URL), + runwareApiKey: + primary?.type === "bfl" + ? primary.apiKey + : trimString((value as RealtimeCanvasConfig).runwareApiKey), + falApiKey: + primary?.type === "bfl" + ? primary.apiKey + : trimString((value as RealtimeCanvasConfig).falApiKey), + apiKey: + primary?.type === "bfl" + ? primary.apiKey + : trimString((value as RealtimeCanvasConfig).apiKey), + runwareModel: + primary?.type === "bfl" + ? primary.model + : trimString((value as RealtimeCanvasConfig).runwareModel) || DEFAULT_BFL_MODEL, + replicateApiToken: + primary?.type === "replicate" + ? primary.apiKey + : trimString((value as RealtimeCanvasConfig).replicateApiToken), + replicateModel: + primary?.type === "replicate" + ? primary.model + : trimString((value as RealtimeCanvasConfig).replicateModel) || DEFAULT_REPLICATE_MODEL, fallbackEnabled: Boolean(fallback), - fallbackBflApiKey: fallback?.apiKey || "", - fallbackBflModel: fallback?.model || DEFAULT_BFL_MODEL, - fallbackBflBaseUrl: fallback?.baseUrl || DEFAULT_BFL_BASE_URL, - fallbackRunwareApiKey: fallback?.apiKey || "", - fallbackFalApiKey: fallback?.apiKey || "", - fallbackApiKey: fallback?.apiKey || "", - fallbackRunwareModel: fallback?.model || DEFAULT_BFL_MODEL, + fallbackProviderType: fallback?.type || normalizeProviderType(value.fallbackProviderType), + fallbackBflApiKey: + fallback?.type === "bfl" + ? fallback.apiKey + : trimString((value as RealtimeCanvasConfig).fallbackBflApiKey), + fallbackBflModel: + fallback?.type === "bfl" + ? fallback.model + : trimString((value as RealtimeCanvasConfig).fallbackBflModel) || DEFAULT_BFL_MODEL, + fallbackBflBaseUrl: + fallback?.type === "bfl" + ? fallback.baseUrl + : normalizeBaseUrl((value as RealtimeCanvasConfig).fallbackBflBaseUrl || DEFAULT_BFL_BASE_URL), + fallbackRunwareApiKey: + fallback?.type === "bfl" + ? fallback.apiKey + : trimString((value as RealtimeCanvasConfig).fallbackRunwareApiKey), + fallbackFalApiKey: + fallback?.type === "bfl" + ? fallback.apiKey + : trimString((value as RealtimeCanvasConfig).fallbackFalApiKey), + fallbackApiKey: + fallback?.type === "bfl" + ? fallback.apiKey + : trimString((value as RealtimeCanvasConfig).fallbackApiKey), + fallbackRunwareModel: + fallback?.type === "bfl" + ? fallback.model + : trimString((value as RealtimeCanvasConfig).fallbackRunwareModel) || DEFAULT_BFL_MODEL, + fallbackReplicateApiToken: + fallback?.type === "replicate" + ? fallback.apiKey + : trimString((value as RealtimeCanvasConfig).fallbackReplicateApiToken), + fallbackReplicateModel: + fallback?.type === "replicate" + ? fallback.model + : trimString((value as RealtimeCanvasConfig).fallbackReplicateModel) || + DEFAULT_REPLICATE_MODEL, renderMode, }; } diff --git a/lib/realtime/replicate-capability.ts b/lib/realtime/replicate-capability.ts new file mode 100644 index 000000000..146611547 --- /dev/null +++ b/lib/realtime/replicate-capability.ts @@ -0,0 +1,232 @@ +import sharp from "sharp"; + +const DEFAULT_REPLICATE_BASE_URL = "https://api.replicate.com/v1"; +const DEFAULT_REPLICATE_MODEL = "black-forest-labs/flux-2-klein-4b"; +const DEFAULT_TIMEOUT_MS = 20_000; + +export type ReplicateCapabilityCode = + | "SUPPORTED" + | "AUTH_FAILED" + | "MODEL_INVALID" + | "UPSTREAM_ERROR"; + +export type ReplicateModelProbeResult = { + supported: boolean; + code: ReplicateCapabilityCode; + message: string; + status?: number; +}; + +type ReplicateProbeParams = { + apiKey: string; + modelId?: string; + baseUrl?: string; + timeoutMs?: number; +}; + +let cachedProbeImageDataUri: string | null = null; + +function trimString(value: unknown) { + return String(value || "").trim(); +} + +function normalizeBaseUrl(raw: string | undefined) { + const next = trimString(raw || DEFAULT_REPLICATE_BASE_URL); + return (next || DEFAULT_REPLICATE_BASE_URL).replace(/\/+$/, ""); +} + +function toErrorMessage(payload: any, fallback: string) { + const direct = trimString( + payload?.detail || + payload?.error || + payload?.message || + payload?.title || + payload?.input?.message || + payload?.non_field_errors?.[0] + ); + return direct || fallback; +} + +function isModelInvalid(message: string) { + const normalized = message.toLowerCase(); + return ( + normalized.includes("model") && + (normalized.includes("not found") || + normalized.includes("invalid") || + normalized.includes("unknown") || + normalized.includes("does not exist")) + ); +} + +function splitModelIdentifier(modelId: string) { + const [owner, ...rest] = modelId.split("/").filter(Boolean); + const name = rest.join("/"); + if (!owner || !name) { + throw new Error(`无效的 Replicate 模型标识：${modelId}`); + } + return { owner, name }; +} + +function shouldRetryWithAltField(message: string) { + const normalized = message.toLowerCase(); + return ( + normalized.includes("images") || + normalized.includes("image") || + normalized.includes("input_image") || + normalized.includes("schema") || + normalized.includes("validation") + ); +} + +async function getProbeImageDataUri() { + if (cachedProbeImageDataUri) return cachedProbeImageDataUri; + + const buffer = await sharp({ + create: { + width: 64, + height: 64, + channels: 3, + background: { r: 255, g: 255, b: 255 }, + }, + }) + .png() + .toBuffer(); + + cachedProbeImageDataUri = `data:image/png;base64,${buffer.toString("base64")}`; + return cachedProbeImageDataUri; +} + +async function createPrediction(params: { + apiKey: string; + baseUrl: string; + modelId: string; + input: Record; + waitSeconds?: number; + signal?: AbortSignal; +}) { + const { owner, name } = splitModelIdentifier(params.modelId); + const endpoint = `${params.baseUrl}/models/${encodeURIComponent(owner)}/${encodeURIComponent(name)}/predictions`; + const response = await fetch(endpoint, { + method: "POST", + headers: { + Authorization: `Bearer ${params.apiKey}`, + "Content-Type": "application/json", + Prefer: `wait=${Math.max(1, Number(params.waitSeconds || 1))}`, + }, + body: JSON.stringify({ input: params.input }), + cache: "no-store", + signal: params.signal, + }); + + const payload = await response.json().catch(() => null); + return { response, payload }; +} + +export async function probeReplicateModelSupport( + params: ReplicateProbeParams +): Promise { + const apiKey = trimString(params.apiKey); + const modelId = trimString(params.modelId || DEFAULT_REPLICATE_MODEL) || DEFAULT_REPLICATE_MODEL; + const baseUrl = normalizeBaseUrl(params.baseUrl); + + if (!apiKey) { + return { + supported: false, + code: "AUTH_FAILED", + message: "缺少 Replicate API Token。", + }; + } + + const controller = new AbortController(); + const timer = setTimeout( + () => controller.abort(), + Math.max(3_000, Number(params.timeoutMs || DEFAULT_TIMEOUT_MS)) + ); + + try { + const image = await getProbeImageDataUri(); + const candidateInputs = [ + { prompt: "white square photo", images: [image] }, + { prompt: "white square photo", image }, + { prompt: "white square photo", input_image: image }, + ]; + + for (let index = 0; index < candidateInputs.length; index += 1) { + const { response, payload } = await createPrediction({ + apiKey, + baseUrl, + modelId, + input: candidateInputs[index], + waitSeconds: 1, + signal: controller.signal, + }); + + const message = toErrorMessage(payload, `Replicate 探针请求失败（${response.status}）`); + + if ( + response.ok && + (trimString(payload?.id) || + trimString(payload?.status) || + trimString(payload?.urls?.get)) + ) { + return { + supported: true, + code: "SUPPORTED", + message: "模型可用。", + status: response.status, + }; + } + + if (response.status === 401 || response.status === 403) { + return { + supported: false, + code: "AUTH_FAILED", + message: "Replicate API Token 无效或无权限，请检查后重试。", + status: response.status, + }; + } + + if (response.status === 404 || isModelInvalid(message)) { + return { + supported: false, + code: "MODEL_INVALID", + message: `模型 ${modelId} 不存在或不可用，请更换模型后重试。`, + status: response.status, + }; + } + + if (index < candidateInputs.length - 1 && shouldRetryWithAltField(message)) { + continue; + } + + return { + supported: false, + code: "UPSTREAM_ERROR", + message, + status: response.status, + }; + } + + return { + supported: false, + code: "UPSTREAM_ERROR", + message: "Replicate 探针请求失败。", + }; + } catch (error) { + if (error instanceof Error && error.name === "AbortError") { + return { + supported: false, + code: "UPSTREAM_ERROR", + message: "Replicate 探针请求超时，请稍后重试。", + }; + } + + return { + supported: false, + code: "UPSTREAM_ERROR", + message: error instanceof Error ? error.message : "Replicate 探针请求失败。", + }; + } finally { + clearTimeout(timer); + } +}