Add Replicate realtime canvas provider
Some checks failed
Lint / build (20) (push) Has been cancelled
Playwright Tests / test (push) Has been cancelled

This commit is contained in:
Your Name 2026-03-16 16:23:01 +08:00
parent 0a105b5c94
commit 77f1401333
7 changed files with 907 additions and 146 deletions

View File

@ -900,13 +900,19 @@ export function AdminSettingsClient() {
// 实时画布API配置状态
const [realtimeCanvasConfig, setRealtimeCanvasConfig] = useState({
enabled: true,
providerType: "bfl" as "bfl" | "replicate",
bflApiKey: "",
bflBaseUrl: DEFAULT_BFL_BASE_URL,
bflModel: DEFAULT_BFL_REALTIME_MODEL,
replicateApiToken: "",
replicateModel: "black-forest-labs/flux-2-klein-4b",
fallbackEnabled: false,
fallbackProviderType: "bfl" as "bfl" | "replicate",
fallbackBflApiKey: "",
fallbackBflBaseUrl: DEFAULT_BFL_BASE_URL,
fallbackBflModel: DEFAULT_BFL_REALTIME_MODEL,
fallbackReplicateApiToken: "",
fallbackReplicateModel: "black-forest-labs/flux-2-klein-4b",
renderMode: "balanced" as "strict-structure" | "balanced" | "stylize",
throttleInterval: 100,
maxBuffering: 2,
@ -1065,16 +1071,22 @@ export function AdminSettingsClient() {
if (realtimeCanvasConfigData?.setting?.value) {
const config = realtimeCanvasConfigData.setting.value as {
enabled?: boolean;
providerType?: "bfl" | "replicate";
bflApiKey?: string;
bflBaseUrl?: string;
bflModel?: string;
replicateApiToken?: string;
replicateModel?: string;
fallbackEnabled?: boolean;
fallbackProviderType?: "bfl" | "replicate";
fallbackBflApiKey?: string;
fallbackBflBaseUrl?: string;
fallbackBflModel?: string;
fallbackRunwareApiKey?: string;
fallbackFalApiKey?: string;
fallbackRunwareModel?: string;
fallbackReplicateApiToken?: string;
fallbackReplicateModel?: string;
runwareApiKey?: string;
falApiKey?: string;
runwareModel?: string;
@ -1095,10 +1107,14 @@ export function AdminSettingsClient() {
};
setRealtimeCanvasConfig({
enabled: config.enabled ?? true,
providerType: config.providerType === "replicate" ? "replicate" : "bfl",
bflApiKey: config.bflApiKey || config.runwareApiKey || config.falApiKey || "",
bflBaseUrl: config.bflBaseUrl || DEFAULT_BFL_BASE_URL,
bflModel: config.bflModel || config.runwareModel || DEFAULT_BFL_REALTIME_MODEL,
replicateApiToken: config.replicateApiToken || "",
replicateModel: config.replicateModel || "black-forest-labs/flux-2-klein-4b",
fallbackEnabled: config.fallbackEnabled === true,
fallbackProviderType: config.fallbackProviderType === "replicate" ? "replicate" : "bfl",
fallbackBflApiKey:
config.fallbackBflApiKey ||
config.fallbackRunwareApiKey ||
@ -1107,6 +1123,9 @@ export function AdminSettingsClient() {
fallbackBflBaseUrl: config.fallbackBflBaseUrl || DEFAULT_BFL_BASE_URL,
fallbackBflModel:
config.fallbackBflModel || config.fallbackRunwareModel || DEFAULT_BFL_REALTIME_MODEL,
fallbackReplicateApiToken: config.fallbackReplicateApiToken || "",
fallbackReplicateModel:
config.fallbackReplicateModel || "black-forest-labs/flux-2-klein-4b",
renderMode:
config.renderMode === "strict-structure" || config.renderMode === "stylize"
? config.renderMode
@ -2208,23 +2227,48 @@ export function AdminSettingsClient() {
// --- 实时画布API配置逻辑 ---
const handleSaveRealtimeCanvasConfig = async () => {
try {
const primaryProviderType = realtimeCanvasConfig.providerType;
const bflApiKey = realtimeCanvasConfig.bflApiKey.trim();
const bflBaseUrl = realtimeCanvasConfig.bflBaseUrl.trim() || DEFAULT_BFL_BASE_URL;
const bflModel = realtimeCanvasConfig.bflModel.trim() || DEFAULT_BFL_REALTIME_MODEL;
const replicateApiToken = realtimeCanvasConfig.replicateApiToken.trim();
const replicateModel =
realtimeCanvasConfig.replicateModel.trim() || "black-forest-labs/flux-2-klein-4b";
const fallbackEnabled = realtimeCanvasConfig.fallbackEnabled;
const fallbackProviderType = realtimeCanvasConfig.fallbackProviderType;
const fallbackBflApiKey = realtimeCanvasConfig.fallbackBflApiKey.trim();
const fallbackBflBaseUrl =
realtimeCanvasConfig.fallbackBflBaseUrl.trim() || DEFAULT_BFL_BASE_URL;
const fallbackBflModel =
realtimeCanvasConfig.fallbackBflModel.trim() || DEFAULT_BFL_REALTIME_MODEL;
const fallbackReplicateApiToken = realtimeCanvasConfig.fallbackReplicateApiToken.trim();
const fallbackReplicateModel =
realtimeCanvasConfig.fallbackReplicateModel.trim() || "black-forest-labs/flux-2-klein-4b";
if (realtimeCanvasConfig.enabled && !bflApiKey) {
toast.error("启用实时画布功能需要配置 BFL API Key");
if (
realtimeCanvasConfig.enabled &&
((primaryProviderType === "bfl" && !bflApiKey) ||
(primaryProviderType === "replicate" && !replicateApiToken))
) {
toast.error(
primaryProviderType === "replicate"
? "启用实时画布功能需要配置 Replicate API Token"
: "启用实时画布功能需要配置 BFL API Key"
);
return;
}
if (realtimeCanvasConfig.enabled && fallbackEnabled && !fallbackBflApiKey) {
toast.error("启用备用实时服务需要配置备用 BFL API Key");
if (
realtimeCanvasConfig.enabled &&
fallbackEnabled &&
((fallbackProviderType === "bfl" && !fallbackBflApiKey) ||
(fallbackProviderType === "replicate" && !fallbackReplicateApiToken))
) {
toast.error(
fallbackProviderType === "replicate"
? "启用备用实时服务需要配置备用 Replicate API Token"
: "启用备用实时服务需要配置备用 BFL API Key"
);
return;
}
@ -2233,9 +2277,10 @@ export function AdminSettingsClient() {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
apiKey: bflApiKey,
modelId: bflModel,
baseUrl: bflBaseUrl,
providerType: primaryProviderType,
apiKey: primaryProviderType === "replicate" ? replicateApiToken : bflApiKey,
modelId: primaryProviderType === "replicate" ? replicateModel : bflModel,
baseUrl: primaryProviderType === "replicate" ? undefined : bflBaseUrl,
}),
});
const probeData = await probeResponse.json().catch(() => null);
@ -2249,9 +2294,14 @@ export function AdminSettingsClient() {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
apiKey: fallbackBflApiKey,
modelId: fallbackBflModel,
baseUrl: fallbackBflBaseUrl,
providerType: fallbackProviderType,
apiKey:
fallbackProviderType === "replicate"
? fallbackReplicateApiToken
: fallbackBflApiKey,
modelId:
fallbackProviderType === "replicate" ? fallbackReplicateModel : fallbackBflModel,
baseUrl: fallbackProviderType === "replicate" ? undefined : fallbackBflBaseUrl,
}),
});
const fallbackProbeData = await fallbackProbeResponse.json().catch(() => null);
@ -2269,13 +2319,19 @@ export function AdminSettingsClient() {
key: "realtime_canvas_config",
value: {
enabled: realtimeCanvasConfig.enabled,
providerType: primaryProviderType,
bflApiKey,
bflBaseUrl,
bflModel,
replicateApiToken,
replicateModel,
fallbackEnabled,
fallbackProviderType,
fallbackBflApiKey,
fallbackBflBaseUrl,
fallbackBflModel,
fallbackReplicateApiToken,
fallbackReplicateModel,
runwareApiKey: bflApiKey,
falApiKey: bflApiKey,
runwareModel: bflModel,
@ -9129,48 +9185,109 @@ export function AdminSettingsClient() {
</div>
<div className="space-y-2">
<Label htmlFor="realtimeBflApiKey">
BFL API Key <span className="text-destructive">*</span>
</Label>
<Input
id="realtimeBflApiKey"
type="password"
value={realtimeCanvasConfig.bflApiKey}
onChange={(e) =>
setRealtimeCanvasConfig({ ...realtimeCanvasConfig, bflApiKey: e.target.value })
<Label>主服务供应商</Label>
<Select
value={realtimeCanvasConfig.providerType}
onValueChange={(value: "bfl" | "replicate") =>
setRealtimeCanvasConfig({ ...realtimeCanvasConfig, providerType: value })
}
placeholder="bfl_xxxxxxxxxxxxx"
disabled={!realtimeCanvasConfig.enabled}
/>
>
<SelectTrigger>
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value="bfl">BFL / 兼容接口</SelectItem>
<SelectItem value="replicate">Replicate</SelectItem>
</SelectContent>
</Select>
</div>
<div className="space-y-2">
<Label htmlFor="realtimeBflBaseUrl">BFL API Base URL</Label>
<Input
id="realtimeBflBaseUrl"
value={realtimeCanvasConfig.bflBaseUrl}
onChange={(e) =>
setRealtimeCanvasConfig({ ...realtimeCanvasConfig, bflBaseUrl: e.target.value })
}
placeholder={DEFAULT_BFL_BASE_URL}
disabled={!realtimeCanvasConfig.enabled}
/>
</div>
{realtimeCanvasConfig.providerType === "replicate" ? (
<>
<div className="space-y-2">
<Label htmlFor="realtimeReplicateApiToken">
Replicate API Token <span className="text-destructive">*</span>
</Label>
<Input
id="realtimeReplicateApiToken"
type="password"
value={realtimeCanvasConfig.replicateApiToken}
onChange={(e) =>
setRealtimeCanvasConfig({
...realtimeCanvasConfig,
replicateApiToken: e.target.value,
})
}
placeholder="r8_xxxxxxxxxxxxx"
disabled={!realtimeCanvasConfig.enabled}
/>
</div>
<div className="space-y-2">
<Label htmlFor="realtimeBflModel">
FLUX.2 模型 ID <span className="text-destructive">*</span>
</Label>
<Input
id="realtimeBflModel"
value={realtimeCanvasConfig.bflModel}
onChange={(e) =>
setRealtimeCanvasConfig({ ...realtimeCanvasConfig, bflModel: e.target.value })
}
placeholder={DEFAULT_BFL_REALTIME_MODEL}
disabled={!realtimeCanvasConfig.enabled}
/>
</div>
<div className="space-y-2">
<Label htmlFor="realtimeReplicateModel">
Replicate 模型 <span className="text-destructive">*</span>
</Label>
<Input
id="realtimeReplicateModel"
value={realtimeCanvasConfig.replicateModel}
onChange={(e) =>
setRealtimeCanvasConfig({
...realtimeCanvasConfig,
replicateModel: e.target.value,
})
}
placeholder="black-forest-labs/flux-2-klein-4b"
disabled={!realtimeCanvasConfig.enabled}
/>
</div>
</>
) : (
<>
<div className="space-y-2">
<Label htmlFor="realtimeBflApiKey">
BFL API Key <span className="text-destructive">*</span>
</Label>
<Input
id="realtimeBflApiKey"
type="password"
value={realtimeCanvasConfig.bflApiKey}
onChange={(e) =>
setRealtimeCanvasConfig({ ...realtimeCanvasConfig, bflApiKey: e.target.value })
}
placeholder="bfl_xxxxxxxxxxxxx"
disabled={!realtimeCanvasConfig.enabled}
/>
</div>
<div className="space-y-2">
<Label htmlFor="realtimeBflBaseUrl">BFL API Base URL</Label>
<Input
id="realtimeBflBaseUrl"
value={realtimeCanvasConfig.bflBaseUrl}
onChange={(e) =>
setRealtimeCanvasConfig({ ...realtimeCanvasConfig, bflBaseUrl: e.target.value })
}
placeholder={DEFAULT_BFL_BASE_URL}
disabled={!realtimeCanvasConfig.enabled}
/>
</div>
<div className="space-y-2">
<Label htmlFor="realtimeBflModel">
FLUX.2 模型 ID <span className="text-destructive">*</span>
</Label>
<Input
id="realtimeBflModel"
value={realtimeCanvasConfig.bflModel}
onChange={(e) =>
setRealtimeCanvasConfig({ ...realtimeCanvasConfig, bflModel: e.target.value })
}
placeholder={DEFAULT_BFL_REALTIME_MODEL}
disabled={!realtimeCanvasConfig.enabled}
/>
</div>
</>
)}
<div className="rounded-lg border border-dashed p-4 space-y-4">
<div className="flex items-center justify-between">
@ -9190,53 +9307,110 @@ export function AdminSettingsClient() {
</div>
<div className="space-y-2">
<Label htmlFor="realtimeFallbackBflApiKey">备用 BFL API Key</Label>
<Input
id="realtimeFallbackBflApiKey"
type="password"
value={realtimeCanvasConfig.fallbackBflApiKey}
onChange={(e) =>
setRealtimeCanvasConfig({
...realtimeCanvasConfig,
fallbackBflApiKey: e.target.value,
})
<Label>备用供应商</Label>
<Select
value={realtimeCanvasConfig.fallbackProviderType}
onValueChange={(value: "bfl" | "replicate") =>
setRealtimeCanvasConfig({ ...realtimeCanvasConfig, fallbackProviderType: value })
}
placeholder="bfl_xxxxxxxxxxxxx"
disabled={!realtimeCanvasConfig.enabled || !realtimeCanvasConfig.fallbackEnabled}
/>
>
<SelectTrigger>
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value="bfl">BFL / 兼容接口</SelectItem>
<SelectItem value="replicate">Replicate</SelectItem>
</SelectContent>
</Select>
</div>
<div className="space-y-2">
<Label htmlFor="realtimeFallbackBflBaseUrl">备用 BFL API Base URL</Label>
<Input
id="realtimeFallbackBflBaseUrl"
value={realtimeCanvasConfig.fallbackBflBaseUrl}
onChange={(e) =>
setRealtimeCanvasConfig({
...realtimeCanvasConfig,
fallbackBflBaseUrl: e.target.value,
})
}
placeholder={DEFAULT_BFL_BASE_URL}
disabled={!realtimeCanvasConfig.enabled || !realtimeCanvasConfig.fallbackEnabled}
/>
</div>
{realtimeCanvasConfig.fallbackProviderType === "replicate" ? (
<>
<div className="space-y-2">
<Label htmlFor="realtimeFallbackReplicateApiToken">备用 Replicate API Token</Label>
<Input
id="realtimeFallbackReplicateApiToken"
type="password"
value={realtimeCanvasConfig.fallbackReplicateApiToken}
onChange={(e) =>
setRealtimeCanvasConfig({
...realtimeCanvasConfig,
fallbackReplicateApiToken: e.target.value,
})
}
placeholder="r8_xxxxxxxxxxxxx"
disabled={!realtimeCanvasConfig.enabled || !realtimeCanvasConfig.fallbackEnabled}
/>
</div>
<div className="space-y-2">
<Label htmlFor="realtimeFallbackBflModel">备用 FLUX.2 模型 ID</Label>
<Input
id="realtimeFallbackBflModel"
value={realtimeCanvasConfig.fallbackBflModel}
onChange={(e) =>
setRealtimeCanvasConfig({
...realtimeCanvasConfig,
fallbackBflModel: e.target.value,
})
}
placeholder={DEFAULT_BFL_REALTIME_MODEL}
disabled={!realtimeCanvasConfig.enabled || !realtimeCanvasConfig.fallbackEnabled}
/>
</div>
<div className="space-y-2">
<Label htmlFor="realtimeFallbackReplicateModel">备用 Replicate 模型</Label>
<Input
id="realtimeFallbackReplicateModel"
value={realtimeCanvasConfig.fallbackReplicateModel}
onChange={(e) =>
setRealtimeCanvasConfig({
...realtimeCanvasConfig,
fallbackReplicateModel: e.target.value,
})
}
placeholder="black-forest-labs/flux-2-klein-4b"
disabled={!realtimeCanvasConfig.enabled || !realtimeCanvasConfig.fallbackEnabled}
/>
</div>
</>
) : (
<>
<div className="space-y-2">
<Label htmlFor="realtimeFallbackBflApiKey">备用 BFL API Key</Label>
<Input
id="realtimeFallbackBflApiKey"
type="password"
value={realtimeCanvasConfig.fallbackBflApiKey}
onChange={(e) =>
setRealtimeCanvasConfig({
...realtimeCanvasConfig,
fallbackBflApiKey: e.target.value,
})
}
placeholder="bfl_xxxxxxxxxxxxx"
disabled={!realtimeCanvasConfig.enabled || !realtimeCanvasConfig.fallbackEnabled}
/>
</div>
<div className="space-y-2">
<Label htmlFor="realtimeFallbackBflBaseUrl">备用 BFL API Base URL</Label>
<Input
id="realtimeFallbackBflBaseUrl"
value={realtimeCanvasConfig.fallbackBflBaseUrl}
onChange={(e) =>
setRealtimeCanvasConfig({
...realtimeCanvasConfig,
fallbackBflBaseUrl: e.target.value,
})
}
placeholder={DEFAULT_BFL_BASE_URL}
disabled={!realtimeCanvasConfig.enabled || !realtimeCanvasConfig.fallbackEnabled}
/>
</div>
<div className="space-y-2">
<Label htmlFor="realtimeFallbackBflModel">备用 FLUX.2 模型 ID</Label>
<Input
id="realtimeFallbackBflModel"
value={realtimeCanvasConfig.fallbackBflModel}
onChange={(e) =>
setRealtimeCanvasConfig({
...realtimeCanvasConfig,
fallbackBflModel: e.target.value,
})
}
placeholder={DEFAULT_BFL_REALTIME_MODEL}
disabled={!realtimeCanvasConfig.enabled || !realtimeCanvasConfig.fallbackEnabled}
/>
</div>
</>
)}
</div>
<div className="grid grid-cols-1 gap-4">

View File

@ -2,8 +2,11 @@ import { z } from "zod";
import { requireAdmin } from "@/lib/middleware/admin";
import { ChatSDKError } from "@/lib/errors";
import { probeBflModelSupport } from "@/lib/realtime/bfl-capability";
import { probeReplicateModelSupport } from "@/lib/realtime/replicate-capability";
import type { RealtimeProviderType } from "@/lib/realtime/config";
const schema = z.object({
providerType: z.enum(["bfl", "replicate"]).optional(),
apiKey: z.string().min(1),
modelId: z.string().min(1),
baseUrl: z.string().optional(),
@ -20,11 +23,19 @@ export async function POST(request: Request) {
const body = await request.json();
const data = schema.parse(body);
const result = await probeBflModelSupport({
apiKey: data.apiKey,
modelId: data.modelId,
baseUrl: data.baseUrl,
});
const providerType: RealtimeProviderType = data.providerType === "replicate" ? "replicate" : "bfl";
const result =
providerType === "replicate"
? await probeReplicateModelSupport({
apiKey: data.apiKey,
modelId: data.modelId,
baseUrl: data.baseUrl,
})
: await probeBflModelSupport({
apiKey: data.apiKey,
modelId: data.modelId,
baseUrl: data.baseUrl,
});
if (!result.supported) {
return Response.json(result, { status: mapStatus(result.code) });

View File

@ -7,6 +7,7 @@ import {
} from "@/lib/db/queries";
import { ChatSDKError } from "@/lib/errors";
import { probeBflModelSupport } from "@/lib/realtime/bfl-capability";
import { probeReplicateModelSupport } from "@/lib/realtime/replicate-capability";
import {
normalizeRealtimeCanvasConfigForStorage,
resolveRealtimeProviders,
@ -72,11 +73,18 @@ export async function POST(request: Request) {
}
for (const provider of providers) {
const probeResult = await probeBflModelSupport({
apiKey: provider.apiKey,
modelId: provider.model,
baseUrl: provider.baseUrl,
});
const probeResult =
provider.type === "replicate"
? await probeReplicateModelSupport({
apiKey: provider.apiKey,
modelId: provider.model,
baseUrl: provider.baseUrl,
})
: await probeBflModelSupport({
apiKey: provider.apiKey,
modelId: provider.model,
baseUrl: provider.baseUrl,
});
if (!probeResult.supported) {
return Response.json(

View File

@ -117,7 +117,7 @@ export async function GET() {
return Response.json({
...defaultConfig,
provider: {
type: "bfl",
type: activeProvider.type,
baseUrl: activeProvider.baseUrl || DEFAULT_BFL_BASE_URL,
},
failover: {
@ -125,7 +125,7 @@ export async function GET() {
fallbackProvider:
providers.length > 1
? {
type: "bfl",
type: providers[1].type,
baseUrl: providers[1].baseUrl,
model: providers[1].model,
}

View File

@ -2,6 +2,8 @@ import { auth } from "@/app/(auth)/auth";
import { getSystemSetting } from "@/lib/db/queries";
import {
DEFAULT_BFL_MODEL,
DEFAULT_REPLICATE_BASE_URL,
DEFAULT_REPLICATE_MODEL,
RealtimeCanvasConfig,
RealtimeProviderConfig,
resolveRealtimeProviders,
@ -23,6 +25,9 @@ type GeneratePayload = {
prompt?: string;
seed?: number;
imageSize?: "square" | "square_hd";
steps?: number;
cfgScale?: number;
strength?: number;
};
type BflSubmitResponse = {
@ -39,6 +44,17 @@ type BflSubmitResponse = {
message?: string;
};
type ReplicatePredictionResponse = {
id?: string;
status?: string;
output?: unknown;
error?: unknown;
detail?: unknown;
urls?: {
get?: string;
};
};
const FALLBACK_REALTIME_PROMPT =
"严格按照输入草图生成:保持主体轮廓、位置、比例和数量一致,只做材质与光影细化;禁止改变构图,禁止新增无关背景纹理。";
@ -146,6 +162,159 @@ function getBflImageFieldCandidates() {
return ["input_image", "image_prompt"] as const;
}
function getReplicateImageFieldCandidates(imagePrompt: string) {
return [
{ images: [imagePrompt] },
{ image: imagePrompt },
{ input_image: imagePrompt },
] as const;
}
function isSelfHostedRealtimeProvider(provider: RealtimeProviderConfig) {
return !provider.baseUrl.includes("api.bfl.ai");
}
function splitReplicateModelIdentifier(modelId: string) {
const [owner, ...rest] = modelId.split("/").filter(Boolean);
const name = rest.join("/");
if (!owner || !name) {
throw new Error(`无效的 Replicate 模型标识:${modelId}`);
}
return { owner, name };
}
function extractReplicateImageUrl(payload: any): string {
const output = payload?.output;
const candidates = [
Array.isArray(output) ? output[0] : output,
payload?.urls?.stream,
payload?.result,
];
for (const candidate of candidates) {
const value = String(candidate || "").trim();
if (!value) continue;
if (value.startsWith("data:")) return value;
if (/^https?:\/\//i.test(value)) return value;
}
return "";
}
function extractReplicateErrorMessage(payload: any, fallback: string) {
const message = String(payload?.detail || payload?.error || payload?.message || "").trim();
return message || fallback;
}
async function submitReplicatePrediction(params: {
apiKey: string;
model: string;
input: Record<string, unknown>;
}): Promise<SubmitTaskResult> {
const { owner, name } = splitReplicateModelIdentifier(params.model);
const endpoint = `${DEFAULT_REPLICATE_BASE_URL}/models/${encodeURIComponent(owner)}/${encodeURIComponent(name)}/predictions`;
const response = await fetch(endpoint, {
method: "POST",
headers: {
Authorization: `Bearer ${params.apiKey}`,
"Content-Type": "application/json",
Prefer: "wait=2",
},
body: JSON.stringify({ input: params.input }),
cache: "no-store",
});
const payload = (await response.json().catch(() => null)) as ReplicatePredictionResponse | null;
const imageUrl = extractReplicateImageUrl(payload);
const taskId = String(payload?.id || "").trim();
const pollingUrl = String(payload?.urls?.get || "").trim();
if (response.ok && (imageUrl || taskId || pollingUrl)) {
return {
taskId,
pollingUrl,
imageUrl,
errorMessage: "",
status: response.status,
};
}
return {
taskId,
pollingUrl,
imageUrl,
errorMessage: extractReplicateErrorMessage(payload, `Replicate 请求失败(${response.status})`),
status: response.status,
};
}
async function pollReplicateResult(params: {
apiKey: string;
pollingUrl?: string;
taskId?: string;
timeoutMs?: number;
}) {
const timeoutMs = Math.max(10_000, Number(params.timeoutMs || 60_000));
const deadline = Date.now() + timeoutMs;
let lastMessage = "";
while (Date.now() < deadline) {
const taskId = String(params.taskId || "").trim();
const url =
String(params.pollingUrl || "").trim() ||
`${DEFAULT_REPLICATE_BASE_URL}/predictions/${encodeURIComponent(taskId)}`;
const response = await fetch(url, {
method: "GET",
headers: {
Authorization: `Bearer ${params.apiKey}`,
Accept: "application/json",
},
cache: "no-store",
});
const payload = (await response.json().catch(() => null)) as ReplicatePredictionResponse | null;
const imageUrl = extractReplicateImageUrl(payload);
if (imageUrl) return { imageUrl, errorMessage: "", status: response.status };
const status = String(payload?.status || "").trim().toLowerCase();
if (status) {
lastMessage = status;
}
if (status === "succeeded") {
return {
imageUrl: "",
errorMessage: "Replicate 返回成功状态但未携带图像结果",
status: response.status,
};
}
if (status === "failed" || status === "canceled") {
return {
imageUrl: "",
errorMessage: extractReplicateErrorMessage(payload, "Replicate 任务执行失败"),
status: response.status,
};
}
if (!response.ok) {
return {
imageUrl: "",
errorMessage: extractReplicateErrorMessage(payload, `Replicate 轮询失败(${response.status})`),
status: response.status,
};
}
await delay(250);
}
return {
imageUrl: "",
errorMessage: `Replicate 轮询超时${lastMessage ? `(${lastMessage})` : ""}`,
status: 504,
};
}
type SubmitTaskResult = {
taskId: string;
pollingUrl: string;
@ -298,33 +467,75 @@ async function generateWithProvider(params: {
prompt: string;
size: number;
seed?: number;
body: GeneratePayload;
config: RealtimeCanvasConfig;
}): Promise<ProviderAttemptResult> {
const imageFieldCandidates = getBflImageFieldCandidates();
const isReplicate = params.provider.type === "replicate";
const imageFieldCandidates = isReplicate
? getReplicateImageFieldCandidates(params.imagePrompt)
: getBflImageFieldCandidates();
let selectedImageField: "input_image" | "image_prompt" = imageFieldCandidates[0];
let selectedImageField: "input_image" | "image_prompt" = "input_image";
let submitResult: SubmitTaskResult | null = null;
let fallbackWarning = "";
for (let index = 0; index < imageFieldCandidates.length; index += 1) {
selectedImageField = imageFieldCandidates[index];
const requestBody: Record<string, unknown> = {
prompt: params.prompt,
width: params.size,
height: params.size,
output_format: "jpeg",
safety_tolerance: 2,
[selectedImageField]: params.imagePrompt,
...(typeof params.seed === "number" ? { seed: params.seed } : {}),
};
if (isReplicate) {
const candidate = getReplicateImageFieldCandidates(params.imagePrompt)[index];
const requestBody: Record<string, unknown> = {
prompt: params.prompt,
...(typeof params.seed === "number" ? { seed: params.seed } : {}),
...candidate,
};
submitResult = await submitBflTask({
apiKey: params.provider.apiKey,
baseUrl: params.provider.baseUrl,
model: params.provider.model,
body: requestBody,
});
submitResult = await submitReplicatePrediction({
apiKey: params.provider.apiKey,
model: params.provider.model || DEFAULT_REPLICATE_MODEL,
input: requestBody,
});
} else {
selectedImageField = imageFieldCandidates[index] as "input_image" | "image_prompt";
const requestBody: Record<string, unknown> = {
prompt: params.prompt,
width: params.size,
height: params.size,
output_format: "jpeg",
safety_tolerance: 2,
[selectedImageField]: params.imagePrompt,
...(typeof params.seed === "number" ? { seed: params.seed } : {}),
};
if (!submitResult.errorMessage) {
if (isSelfHostedRealtimeProvider(params.provider)) {
requestBody.steps = clamp(Number(params.body.steps ?? params.config.defaults?.steps ?? 1), 1, 6);
requestBody.cfg_scale = clamp(
Number(params.body.cfgScale ?? params.config.defaults?.cfgScale ?? 1),
1,
14
);
requestBody.strength = clamp(
Number(params.body.strength ?? params.config.defaults?.strength ?? 0.32),
0.05,
1
);
requestBody.preview_megapixels = clamp(Number(params.config.previewMegapixels ?? 0.4), 0.2, 0.8);
requestBody.detail_megapixels = clamp(Number(params.config.detailMegapixels ?? 1.0), 0.6, 1.2);
requestBody.output_jpeg_quality = clamp(
Number(params.config.outputJpegQuality ?? 62),
35,
90
);
requestBody.quality_profile = params.body.qualityProfile || "refine";
}
submitResult = await submitBflTask({
apiKey: params.provider.apiKey,
baseUrl: params.provider.baseUrl,
model: params.provider.model,
body: requestBody,
});
}
if (!submitResult?.errorMessage) {
break;
}
@ -338,8 +549,8 @@ async function generateWithProvider(params: {
return {
ok: false,
message: submitResult.errorMessage || "实时生成提交失败",
status: submitResult.status || 502,
message: submitResult?.errorMessage || "实时生成提交失败",
status: submitResult?.status || 502,
provider: params.provider,
};
}
@ -355,13 +566,21 @@ async function generateWithProvider(params: {
let finalImageUrl = submitResult.imageUrl;
if (!finalImageUrl) {
const pollResult = await pollBflResult({
apiKey: params.provider.apiKey,
baseUrl: params.provider.baseUrl,
pollingUrl: submitResult.pollingUrl,
taskId: submitResult.taskId,
timeoutMs: 110_000,
});
const pollResult =
params.provider.type === "replicate"
? await pollReplicateResult({
apiKey: params.provider.apiKey,
pollingUrl: submitResult.pollingUrl,
taskId: submitResult.taskId,
timeoutMs: 60_000,
})
: await pollBflResult({
apiKey: params.provider.apiKey,
baseUrl: params.provider.baseUrl,
pollingUrl: submitResult.pollingUrl,
taskId: submitResult.taskId,
timeoutMs: 110_000,
});
if (!pollResult.imageUrl) {
return {
ok: false,
@ -459,6 +678,8 @@ export async function POST(request: Request) {
prompt: positivePrompt,
size,
seed,
body,
config,
});
if (!result.ok) {

View File

@ -1,8 +1,13 @@
export const DEFAULT_BFL_MODEL = "flux-2-klein-4b";
export const DEFAULT_BFL_BASE_URL = "https://api.bfl.ai";
export const DEFAULT_REPLICATE_MODEL = "black-forest-labs/flux-2-klein-4b";
export const DEFAULT_REPLICATE_BASE_URL = "https://api.replicate.com/v1";
export type RealtimeProviderType = "bfl" | "replicate";
export type RealtimeCanvasConfig = {
enabled?: boolean;
providerType?: RealtimeProviderType;
bflApiKey?: string;
bflModel?: string;
bflBaseUrl?: string;
@ -10,7 +15,10 @@ export type RealtimeCanvasConfig = {
runwareModel?: string;
falApiKey?: string;
apiKey?: string;
replicateApiToken?: string;
replicateModel?: string;
fallbackEnabled?: boolean;
fallbackProviderType?: RealtimeProviderType;
fallbackBflApiKey?: string;
fallbackBflModel?: string;
fallbackBflBaseUrl?: string;
@ -18,6 +26,8 @@ export type RealtimeCanvasConfig = {
fallbackRunwareModel?: string;
fallbackFalApiKey?: string;
fallbackApiKey?: string;
fallbackReplicateApiToken?: string;
fallbackReplicateModel?: string;
negativePrompt?: string;
renderMode?: "strict-structure" | "balanced" | "stylize";
strictSketch?: boolean;
@ -25,6 +35,9 @@ export type RealtimeCanvasConfig = {
maxBuffering?: number;
maxInputResolution?: number;
jpegQuality?: number;
previewMegapixels?: number;
detailMegapixels?: number;
outputJpegQuality?: number;
defaults?: {
steps?: number;
cfgScale?: number;
@ -36,6 +49,7 @@ export type RealtimeCanvasConfig = {
};
export type RealtimeProviderConfig = {
type: RealtimeProviderType;
apiKey: string;
baseUrl: string;
model: string;
@ -47,6 +61,10 @@ function trimString(value: unknown) {
return String(value || "").trim();
}
function normalizeProviderType(raw: unknown): RealtimeProviderType {
return raw === "replicate" ? "replicate" : "bfl";
}
export function normalizeBaseUrl(raw: string | undefined) {
const next = trimString(raw || DEFAULT_BFL_BASE_URL);
return (next || DEFAULT_BFL_BASE_URL).replace(/\/+$/, "");
@ -64,7 +82,9 @@ function hasFallbackIntent(config: RealtimeCanvasConfig) {
config.fallbackApiKey ||
config.fallbackBflBaseUrl ||
config.fallbackBflModel ||
config.fallbackRunwareModel
config.fallbackRunwareModel ||
config.fallbackReplicateApiToken ||
config.fallbackReplicateModel
)
);
}
@ -73,6 +93,22 @@ export function resolvePrimaryRealtimeProvider(
config: RealtimeCanvasConfig,
env: NodeJS.ProcessEnv = process.env
): RealtimeProviderConfig | null {
const type = normalizeProviderType(config.providerType);
if (type === "replicate") {
const apiKey = trimString(config.replicateApiToken || env.REPLICATE_API_TOKEN);
if (!apiKey) return null;
return {
type,
apiKey,
model: trimString(config.replicateModel) || DEFAULT_REPLICATE_MODEL,
baseUrl: DEFAULT_REPLICATE_BASE_URL,
source: "primary",
label: "主实时服务",
};
}
const apiKey = trimString(
config.bflApiKey ||
config.runwareApiKey ||
@ -89,6 +125,7 @@ export function resolvePrimaryRealtimeProvider(
const baseUrl = normalizeBaseUrl(config.bflBaseUrl || env.BFL_BASE_URL);
return {
type,
apiKey,
model,
baseUrl,
@ -102,6 +139,22 @@ export function resolveFallbackRealtimeProvider(
): RealtimeProviderConfig | null {
if (!hasFallbackIntent(config)) return null;
const type = normalizeProviderType(config.fallbackProviderType);
if (type === "replicate") {
const apiKey = trimString(config.fallbackReplicateApiToken);
if (!apiKey) return null;
return {
type,
apiKey,
model: trimString(config.fallbackReplicateModel) || DEFAULT_REPLICATE_MODEL,
baseUrl: DEFAULT_REPLICATE_BASE_URL,
source: "fallback",
label: "备用实时服务",
};
}
const apiKey = trimString(
config.fallbackBflApiKey ||
config.fallbackRunwareApiKey ||
@ -115,6 +168,7 @@ export function resolveFallbackRealtimeProvider(
const baseUrl = normalizeBaseUrl(config.fallbackBflBaseUrl || DEFAULT_BFL_BASE_URL);
return {
type,
apiKey,
model,
baseUrl,
@ -166,21 +220,82 @@ export function normalizeRealtimeCanvasConfigForStorage(
return {
...value,
bflApiKey: primary?.apiKey || "",
bflModel: primary?.model || DEFAULT_BFL_MODEL,
bflBaseUrl: primary?.baseUrl || normalizeBaseUrl(env.BFL_BASE_URL),
runwareApiKey: primary?.apiKey || "",
falApiKey: primary?.apiKey || "",
apiKey: primary?.apiKey || "",
runwareModel: primary?.model || DEFAULT_BFL_MODEL,
providerType: primary?.type || normalizeProviderType(value.providerType),
bflApiKey:
primary?.type === "bfl"
? primary.apiKey
: trimString((value as RealtimeCanvasConfig).bflApiKey),
bflModel:
primary?.type === "bfl"
? primary.model
: trimString((value as RealtimeCanvasConfig).bflModel) || DEFAULT_BFL_MODEL,
bflBaseUrl:
primary?.type === "bfl"
? primary.baseUrl
: normalizeBaseUrl((value as RealtimeCanvasConfig).bflBaseUrl || env.BFL_BASE_URL),
runwareApiKey:
primary?.type === "bfl"
? primary.apiKey
: trimString((value as RealtimeCanvasConfig).runwareApiKey),
falApiKey:
primary?.type === "bfl"
? primary.apiKey
: trimString((value as RealtimeCanvasConfig).falApiKey),
apiKey:
primary?.type === "bfl"
? primary.apiKey
: trimString((value as RealtimeCanvasConfig).apiKey),
runwareModel:
primary?.type === "bfl"
? primary.model
: trimString((value as RealtimeCanvasConfig).runwareModel) || DEFAULT_BFL_MODEL,
replicateApiToken:
primary?.type === "replicate"
? primary.apiKey
: trimString((value as RealtimeCanvasConfig).replicateApiToken),
replicateModel:
primary?.type === "replicate"
? primary.model
: trimString((value as RealtimeCanvasConfig).replicateModel) || DEFAULT_REPLICATE_MODEL,
fallbackEnabled: Boolean(fallback),
fallbackBflApiKey: fallback?.apiKey || "",
fallbackBflModel: fallback?.model || DEFAULT_BFL_MODEL,
fallbackBflBaseUrl: fallback?.baseUrl || DEFAULT_BFL_BASE_URL,
fallbackRunwareApiKey: fallback?.apiKey || "",
fallbackFalApiKey: fallback?.apiKey || "",
fallbackApiKey: fallback?.apiKey || "",
fallbackRunwareModel: fallback?.model || DEFAULT_BFL_MODEL,
fallbackProviderType: fallback?.type || normalizeProviderType(value.fallbackProviderType),
fallbackBflApiKey:
fallback?.type === "bfl"
? fallback.apiKey
: trimString((value as RealtimeCanvasConfig).fallbackBflApiKey),
fallbackBflModel:
fallback?.type === "bfl"
? fallback.model
: trimString((value as RealtimeCanvasConfig).fallbackBflModel) || DEFAULT_BFL_MODEL,
fallbackBflBaseUrl:
fallback?.type === "bfl"
? fallback.baseUrl
: normalizeBaseUrl((value as RealtimeCanvasConfig).fallbackBflBaseUrl || DEFAULT_BFL_BASE_URL),
fallbackRunwareApiKey:
fallback?.type === "bfl"
? fallback.apiKey
: trimString((value as RealtimeCanvasConfig).fallbackRunwareApiKey),
fallbackFalApiKey:
fallback?.type === "bfl"
? fallback.apiKey
: trimString((value as RealtimeCanvasConfig).fallbackFalApiKey),
fallbackApiKey:
fallback?.type === "bfl"
? fallback.apiKey
: trimString((value as RealtimeCanvasConfig).fallbackApiKey),
fallbackRunwareModel:
fallback?.type === "bfl"
? fallback.model
: trimString((value as RealtimeCanvasConfig).fallbackRunwareModel) || DEFAULT_BFL_MODEL,
fallbackReplicateApiToken:
fallback?.type === "replicate"
? fallback.apiKey
: trimString((value as RealtimeCanvasConfig).fallbackReplicateApiToken),
fallbackReplicateModel:
fallback?.type === "replicate"
? fallback.model
: trimString((value as RealtimeCanvasConfig).fallbackReplicateModel) ||
DEFAULT_REPLICATE_MODEL,
renderMode,
};
}

View File

@ -0,0 +1,232 @@
import sharp from "sharp";
const DEFAULT_REPLICATE_BASE_URL = "https://api.replicate.com/v1";
const DEFAULT_REPLICATE_MODEL = "black-forest-labs/flux-2-klein-4b";
const DEFAULT_TIMEOUT_MS = 20_000;
export type ReplicateCapabilityCode =
| "SUPPORTED"
| "AUTH_FAILED"
| "MODEL_INVALID"
| "UPSTREAM_ERROR";
export type ReplicateModelProbeResult = {
supported: boolean;
code: ReplicateCapabilityCode;
message: string;
status?: number;
};
type ReplicateProbeParams = {
apiKey: string;
modelId?: string;
baseUrl?: string;
timeoutMs?: number;
};
let cachedProbeImageDataUri: string | null = null;
function trimString(value: unknown) {
return String(value || "").trim();
}
function normalizeBaseUrl(raw: string | undefined) {
const next = trimString(raw || DEFAULT_REPLICATE_BASE_URL);
return (next || DEFAULT_REPLICATE_BASE_URL).replace(/\/+$/, "");
}
function toErrorMessage(payload: any, fallback: string) {
const direct = trimString(
payload?.detail ||
payload?.error ||
payload?.message ||
payload?.title ||
payload?.input?.message ||
payload?.non_field_errors?.[0]
);
return direct || fallback;
}
function isModelInvalid(message: string) {
const normalized = message.toLowerCase();
return (
normalized.includes("model") &&
(normalized.includes("not found") ||
normalized.includes("invalid") ||
normalized.includes("unknown") ||
normalized.includes("does not exist"))
);
}
function splitModelIdentifier(modelId: string) {
const [owner, ...rest] = modelId.split("/").filter(Boolean);
const name = rest.join("/");
if (!owner || !name) {
throw new Error(`无效的 Replicate 模型标识:${modelId}`);
}
return { owner, name };
}
function shouldRetryWithAltField(message: string) {
const normalized = message.toLowerCase();
return (
normalized.includes("images") ||
normalized.includes("image") ||
normalized.includes("input_image") ||
normalized.includes("schema") ||
normalized.includes("validation")
);
}
async function getProbeImageDataUri() {
if (cachedProbeImageDataUri) return cachedProbeImageDataUri;
const buffer = await sharp({
create: {
width: 64,
height: 64,
channels: 3,
background: { r: 255, g: 255, b: 255 },
},
})
.png()
.toBuffer();
cachedProbeImageDataUri = `data:image/png;base64,${buffer.toString("base64")}`;
return cachedProbeImageDataUri;
}
async function createPrediction(params: {
apiKey: string;
baseUrl: string;
modelId: string;
input: Record<string, unknown>;
waitSeconds?: number;
signal?: AbortSignal;
}) {
const { owner, name } = splitModelIdentifier(params.modelId);
const endpoint = `${params.baseUrl}/models/${encodeURIComponent(owner)}/${encodeURIComponent(name)}/predictions`;
const response = await fetch(endpoint, {
method: "POST",
headers: {
Authorization: `Bearer ${params.apiKey}`,
"Content-Type": "application/json",
Prefer: `wait=${Math.max(1, Number(params.waitSeconds || 1))}`,
},
body: JSON.stringify({ input: params.input }),
cache: "no-store",
signal: params.signal,
});
const payload = await response.json().catch(() => null);
return { response, payload };
}
export async function probeReplicateModelSupport(
params: ReplicateProbeParams
): Promise<ReplicateModelProbeResult> {
const apiKey = trimString(params.apiKey);
const modelId = trimString(params.modelId || DEFAULT_REPLICATE_MODEL) || DEFAULT_REPLICATE_MODEL;
const baseUrl = normalizeBaseUrl(params.baseUrl);
if (!apiKey) {
return {
supported: false,
code: "AUTH_FAILED",
message: "缺少 Replicate API Token。",
};
}
const controller = new AbortController();
const timer = setTimeout(
() => controller.abort(),
Math.max(3_000, Number(params.timeoutMs || DEFAULT_TIMEOUT_MS))
);
try {
const image = await getProbeImageDataUri();
const candidateInputs = [
{ prompt: "white square photo", images: [image] },
{ prompt: "white square photo", image },
{ prompt: "white square photo", input_image: image },
];
for (let index = 0; index < candidateInputs.length; index += 1) {
const { response, payload } = await createPrediction({
apiKey,
baseUrl,
modelId,
input: candidateInputs[index],
waitSeconds: 1,
signal: controller.signal,
});
const message = toErrorMessage(payload, `Replicate 探针请求失败(${response.status})`);
if (
response.ok &&
(trimString(payload?.id) ||
trimString(payload?.status) ||
trimString(payload?.urls?.get))
) {
return {
supported: true,
code: "SUPPORTED",
message: "模型可用。",
status: response.status,
};
}
if (response.status === 401 || response.status === 403) {
return {
supported: false,
code: "AUTH_FAILED",
message: "Replicate API Token 无效或无权限,请检查后重试。",
status: response.status,
};
}
if (response.status === 404 || isModelInvalid(message)) {
return {
supported: false,
code: "MODEL_INVALID",
message: `模型 ${modelId} 不存在或不可用,请更换模型后重试。`,
status: response.status,
};
}
if (index < candidateInputs.length - 1 && shouldRetryWithAltField(message)) {
continue;
}
return {
supported: false,
code: "UPSTREAM_ERROR",
message,
status: response.status,
};
}
return {
supported: false,
code: "UPSTREAM_ERROR",
message: "Replicate 探针请求失败。",
};
} catch (error) {
if (error instanceof Error && error.name === "AbortError") {
return {
supported: false,
code: "UPSTREAM_ERROR",
message: "Replicate 探针请求超时,请稍后重试。",
};
}
return {
supported: false,
code: "UPSTREAM_ERROR",
message: error instanceof Error ? error.message : "Replicate 探针请求失败。",
};
} finally {
clearTimeout(timer);
}
}