lunanting/ai-chatbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add fal provider for realtime canvas
Some checks failed
Lint / build (20) (push) Has been cancelled
Details
Playwright Tests / test (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Your Name 2026-03-31 13:36:28 +08:00
parent ade5344163
commit cd91094b0f
7 changed files with 596 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

248
app/(admin)/settings/admin-settings-client.tsx
View File

@ -1045,21 +1045,27 @@ export function AdminSettingsClient() {
});
// 实时画布API配置状态
const [realtimeCanvasConfig, setRealtimeCanvasConfig] = useState({
enabled: true,
providerType: "bfl" as "bfl" | "replicate",
bflApiKey: "",
bflBaseUrl: DEFAULT_BFL_BASE_URL,
bflModel: DEFAULT_BFL_REALTIME_MODEL,
replicateApiToken: "",
replicateModel: "black-forest-labs/flux-2-klein-4b",
fallbackEnabled: false,
fallbackProviderType: "bfl" as "bfl" | "replicate",
fallbackBflApiKey: "",
fallbackBflBaseUrl: DEFAULT_BFL_BASE_URL,
fallbackBflModel: DEFAULT_BFL_REALTIME_MODEL,
fallbackReplicateApiToken: "",
fallbackReplicateModel: "black-forest-labs/flux-2-klein-4b",
const [realtimeCanvasConfig, setRealtimeCanvasConfig] = useState({
enabled: true,
providerType: "bfl" as "bfl" | "replicate" | "fal",
bflApiKey: "",
bflBaseUrl: DEFAULT_BFL_BASE_URL,
bflModel: DEFAULT_BFL_REALTIME_MODEL,
falApiKey: "",
falModel: "",
falBaseUrl: "https://fal.run",
replicateApiToken: "",
replicateModel: "black-forest-labs/flux-2-klein-4b",
fallbackEnabled: false,
fallbackProviderType: "bfl" as "bfl" | "replicate" | "fal",
fallbackBflApiKey: "",
fallbackBflBaseUrl: DEFAULT_BFL_BASE_URL,
fallbackBflModel: DEFAULT_BFL_REALTIME_MODEL,
fallbackFalApiKey: "",
fallbackFalModel: "",
fallbackFalBaseUrl: "https://fal.run",
fallbackReplicateApiToken: "",
fallbackReplicateModel: "black-forest-labs/flux-2-klein-4b",
renderMode: "balanced" as "strict-structure" | "balanced" | "stylize",
throttleInterval: 100,
maxBuffering: 2,
@ -1319,24 +1325,28 @@ export function AdminSettingsClient() {
if (realtimeCanvasConfigData?.setting?.value) {
const config = realtimeCanvasConfigData.setting.value as {
enabled?: boolean;
providerType?: "bfl" | "replicate";
providerType?: "bfl" | "replicate" | "fal";
bflApiKey?: string;
bflBaseUrl?: string;
bflModel?: string;
falApiKey?: string;
falBaseUrl?: string;
falModel?: string;
replicateApiToken?: string;
replicateModel?: string;
fallbackEnabled?: boolean;
fallbackProviderType?: "bfl" | "replicate";
fallbackProviderType?: "bfl" | "replicate" | "fal";
fallbackBflApiKey?: string;
fallbackBflBaseUrl?: string;
fallbackBflModel?: string;
fallbackRunwareApiKey?: string;
fallbackFalApiKey?: string;
fallbackFalBaseUrl?: string;
fallbackFalModel?: string;
fallbackRunwareModel?: string;
fallbackReplicateApiToken?: string;
fallbackReplicateModel?: string;
runwareApiKey?: string;
falApiKey?: string;
runwareModel?: string;
renderMode?: "strict-structure" | "balanced" | "stylize";
throttleInterval?: number;
@ -1355,22 +1365,37 @@ export function AdminSettingsClient() {
};
setRealtimeCanvasConfig({
enabled: config.enabled ?? true,
providerType: config.providerType === "replicate" ? "replicate" : "bfl",
bflApiKey: config.bflApiKey || config.runwareApiKey || config.falApiKey || "",
providerType:
config.providerType === "replicate"
? "replicate"
: config.providerType === "fal"
? "fal"
: "bfl",
bflApiKey: config.bflApiKey || config.runwareApiKey || "",
bflBaseUrl: config.bflBaseUrl || DEFAULT_BFL_BASE_URL,
bflModel: config.bflModel || config.runwareModel || DEFAULT_BFL_REALTIME_MODEL,
falApiKey: config.falApiKey || "",
falBaseUrl: config.falBaseUrl || "https://fal.run",
falModel: config.falModel || "",
replicateApiToken: config.replicateApiToken || "",
replicateModel: config.replicateModel || "black-forest-labs/flux-2-klein-4b",
fallbackEnabled: config.fallbackEnabled === true,
fallbackProviderType: config.fallbackProviderType === "replicate" ? "replicate" : "bfl",
fallbackProviderType:
config.fallbackProviderType === "replicate"
? "replicate"
: config.fallbackProviderType === "fal"
? "fal"
: "bfl",
fallbackBflApiKey:
config.fallbackBflApiKey ||
config.fallbackRunwareApiKey ||
config.fallbackFalApiKey ||
"",
fallbackBflBaseUrl: config.fallbackBflBaseUrl || DEFAULT_BFL_BASE_URL,
fallbackBflModel:
config.fallbackBflModel || config.fallbackRunwareModel || DEFAULT_BFL_REALTIME_MODEL,
fallbackFalApiKey: config.fallbackFalApiKey || "",
fallbackFalBaseUrl: config.fallbackFalBaseUrl || "https://fal.run",
fallbackFalModel: config.fallbackFalModel || "",
fallbackReplicateApiToken: config.fallbackReplicateApiToken || "",
fallbackReplicateModel:
config.fallbackReplicateModel || "black-forest-labs/flux-2-klein-4b",
@ -2748,6 +2773,9 @@ export function AdminSettingsClient() {
const bflApiKey = realtimeCanvasConfig.bflApiKey.trim();
const bflBaseUrl = realtimeCanvasConfig.bflBaseUrl.trim() || DEFAULT_BFL_BASE_URL;
const bflModel = realtimeCanvasConfig.bflModel.trim() || DEFAULT_BFL_REALTIME_MODEL;
const falApiKey = realtimeCanvasConfig.falApiKey.trim();
const falBaseUrl = realtimeCanvasConfig.falBaseUrl.trim() || "https://fal.run";
const falModel = realtimeCanvasConfig.falModel.trim();
const replicateApiToken = realtimeCanvasConfig.replicateApiToken.trim();
const replicateModel =
realtimeCanvasConfig.replicateModel.trim() || "black-forest-labs/flux-2-klein-4b";
@ -2758,6 +2786,10 @@ export function AdminSettingsClient() {
realtimeCanvasConfig.fallbackBflBaseUrl.trim() || DEFAULT_BFL_BASE_URL;
const fallbackBflModel =
realtimeCanvasConfig.fallbackBflModel.trim() || DEFAULT_BFL_REALTIME_MODEL;
const fallbackFalApiKey = realtimeCanvasConfig.fallbackFalApiKey.trim();
const fallbackFalBaseUrl =
realtimeCanvasConfig.fallbackFalBaseUrl.trim() || "https://fal.run";
const fallbackFalModel = realtimeCanvasConfig.fallbackFalModel.trim();
const fallbackReplicateApiToken = realtimeCanvasConfig.fallbackReplicateApiToken.trim();
const fallbackReplicateModel =
realtimeCanvasConfig.fallbackReplicateModel.trim() || "black-forest-labs/flux-2-klein-4b";
@ -2765,12 +2797,15 @@ export function AdminSettingsClient() {
if (
realtimeCanvasConfig.enabled &&
((primaryProviderType === "bfl" && !bflApiKey) ||
(primaryProviderType === "replicate" && !replicateApiToken))
(primaryProviderType === "replicate" && !replicateApiToken) ||
(primaryProviderType === "fal" && (!falApiKey || !falModel)))
) {
toast.error(
primaryProviderType === "replicate"
? "启用实时画布功能需要配置 Replicate API Token"
: "启用实时画布功能需要配置 BFL API Key"
: primaryProviderType === "fal"
? "启用实时画布功能需要配置 FAL API Key 和模型 ID"
: "启用实时画布功能需要配置 BFL API Key"
);
return;
}
@ -2779,12 +2814,15 @@ export function AdminSettingsClient() {
realtimeCanvasConfig.enabled &&
fallbackEnabled &&
((fallbackProviderType === "bfl" && !fallbackBflApiKey) ||
(fallbackProviderType === "replicate" && !fallbackReplicateApiToken))
(fallbackProviderType === "replicate" && !fallbackReplicateApiToken) ||
(fallbackProviderType === "fal" && (!fallbackFalApiKey || !fallbackFalModel)))
) {
toast.error(
fallbackProviderType === "replicate"
? "启用备用实时服务需要配置备用 Replicate API Token"
: "启用备用实时服务需要配置备用 BFL API Key"
: fallbackProviderType === "fal"
? "启用备用实时服务需要配置备用 FAL API Key 和模型 ID"
: "启用备用实时服务需要配置备用 BFL API Key"
);
return;
}
@ -2795,9 +2833,24 @@ export function AdminSettingsClient() {
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
providerType: primaryProviderType,
apiKey: primaryProviderType === "replicate" ? replicateApiToken : bflApiKey,
modelId: primaryProviderType === "replicate" ? replicateModel : bflModel,
baseUrl: primaryProviderType === "replicate" ? undefined : bflBaseUrl,
apiKey:
primaryProviderType === "replicate"
? replicateApiToken
: primaryProviderType === "fal"
? falApiKey
: bflApiKey,
modelId:
primaryProviderType === "replicate"
? replicateModel
: primaryProviderType === "fal"
? falModel
: bflModel,
baseUrl:
primaryProviderType === "replicate"
? undefined
: primaryProviderType === "fal"
? falBaseUrl
: bflBaseUrl,
}),
});
const probeData = await probeResponse.json().catch(() => null);
@ -2815,10 +2868,21 @@ export function AdminSettingsClient() {
apiKey:
fallbackProviderType === "replicate"
? fallbackReplicateApiToken
: fallbackProviderType === "fal"
? fallbackFalApiKey
: fallbackBflApiKey,
modelId:
fallbackProviderType === "replicate" ? fallbackReplicateModel : fallbackBflModel,
baseUrl: fallbackProviderType === "replicate" ? undefined : fallbackBflBaseUrl,
fallbackProviderType === "replicate"
? fallbackReplicateModel
: fallbackProviderType === "fal"
? fallbackFalModel
: fallbackBflModel,
baseUrl:
fallbackProviderType === "replicate"
? undefined
: fallbackProviderType === "fal"
? fallbackFalBaseUrl
: fallbackBflBaseUrl,
}),
});
const fallbackProbeData = await fallbackProbeResponse.json().catch(() => null);
@ -2840,6 +2904,9 @@ export function AdminSettingsClient() {
bflApiKey,
bflBaseUrl,
bflModel,
falApiKey,
falBaseUrl,
falModel,
replicateApiToken,
replicateModel,
fallbackEnabled,
@ -2847,10 +2914,13 @@ export function AdminSettingsClient() {
fallbackBflApiKey,
fallbackBflBaseUrl,
fallbackBflModel,
fallbackFalApiKey,
fallbackFalBaseUrl,
fallbackFalModel,
fallbackReplicateApiToken,
fallbackReplicateModel,
runwareApiKey: bflApiKey,
falApiKey: bflApiKey,
apiKey: primaryProviderType === "fal" ? falApiKey : bflApiKey,
runwareModel: bflModel,
renderMode: realtimeCanvasConfig.renderMode,
throttleInterval: Math.min(260, Math.max(60, Number(realtimeCanvasConfig.throttleInterval) || 100)),
@ -10463,7 +10533,7 @@ export function AdminSettingsClient() {
<Label>主服务供应商</Label>
<Select
value={realtimeCanvasConfig.providerType}
onValueChange={(value: "bfl" | "replicate") =>
onValueChange={(value: "bfl" | "replicate" | "fal") =>
setRealtimeCanvasConfig({ ...realtimeCanvasConfig, providerType: value })
}
>
@ -10473,6 +10543,7 @@ export function AdminSettingsClient() {
<SelectContent>
<SelectItem value="bfl">BFL / 兼容接口</SelectItem>
<SelectItem value="replicate">Replicate</SelectItem>
<SelectItem value="fal">FAL</SelectItem>
</SelectContent>
</Select>
</div>
@ -10516,6 +10587,61 @@ export function AdminSettingsClient() {
/>
</div>
</>
) : realtimeCanvasConfig.providerType === "fal" ? (
<>
<div className="space-y-2">
<Label htmlFor="realtimeFalApiKey">
FAL API Key <span className="text-destructive">*</span>
</Label>
<Input
id="realtimeFalApiKey"
type="password"
value={realtimeCanvasConfig.falApiKey}
onChange={(e) =>
setRealtimeCanvasConfig({
...realtimeCanvasConfig,
falApiKey: e.target.value,
})
}
placeholder="FAL_KEY_ID:FAL_KEY_SECRET 或 token"
disabled={!realtimeCanvasConfig.enabled}
/>
</div>
<div className="space-y-2">
<Label htmlFor="realtimeFalModel">
FAL 模型 ID <span className="text-destructive">*</span>
</Label>
<Input
id="realtimeFalModel"
value={realtimeCanvasConfig.falModel}
onChange={(e) =>
setRealtimeCanvasConfig({
...realtimeCanvasConfig,
falModel: e.target.value,
})
}
placeholder="fal-ai/your-image-edit-model"
disabled={!realtimeCanvasConfig.enabled}
/>
</div>
<div className="space-y-2">
<Label htmlFor="realtimeFalBaseUrl">FAL Base URL</Label>
<Input
id="realtimeFalBaseUrl"
value={realtimeCanvasConfig.falBaseUrl}
onChange={(e) =>
setRealtimeCanvasConfig({
...realtimeCanvasConfig,
falBaseUrl: e.target.value,
})
}
placeholder="https://fal.run"
disabled={!realtimeCanvasConfig.enabled}
/>
</div>
</>
) : (
<>
<div className="space-y-2">
@ -10585,7 +10711,7 @@ export function AdminSettingsClient() {
<Label>备用供应商</Label>
<Select
value={realtimeCanvasConfig.fallbackProviderType}
onValueChange={(value: "bfl" | "replicate") =>
onValueChange={(value: "bfl" | "replicate" | "fal") =>
setRealtimeCanvasConfig({ ...realtimeCanvasConfig, fallbackProviderType: value })
}
>
@ -10595,6 +10721,7 @@ export function AdminSettingsClient() {
<SelectContent>
<SelectItem value="bfl">BFL / 兼容接口</SelectItem>
<SelectItem value="replicate">Replicate</SelectItem>
<SelectItem value="fal">FAL</SelectItem>
</SelectContent>
</Select>
</div>
@ -10634,6 +10761,57 @@ export function AdminSettingsClient() {
/>
</div>
</>
) : realtimeCanvasConfig.fallbackProviderType === "fal" ? (
<>
<div className="space-y-2">
<Label htmlFor="realtimeFallbackFalApiKey">备用 FAL API Key</Label>
<Input
id="realtimeFallbackFalApiKey"
type="password"
value={realtimeCanvasConfig.fallbackFalApiKey}
onChange={(e) =>
setRealtimeCanvasConfig({
...realtimeCanvasConfig,
fallbackFalApiKey: e.target.value,
})
}
placeholder="FAL_KEY_ID:FAL_KEY_SECRET 或 token"
disabled={!realtimeCanvasConfig.enabled || !realtimeCanvasConfig.fallbackEnabled}
/>
</div>
<div className="space-y-2">
<Label htmlFor="realtimeFallbackFalModel">备用 FAL 模型 ID</Label>
<Input
id="realtimeFallbackFalModel"
value={realtimeCanvasConfig.fallbackFalModel}
onChange={(e) =>
setRealtimeCanvasConfig({
...realtimeCanvasConfig,
fallbackFalModel: e.target.value,
})
}
placeholder="fal-ai/your-image-edit-model"
disabled={!realtimeCanvasConfig.enabled || !realtimeCanvasConfig.fallbackEnabled}
/>
</div>
<div className="space-y-2">
<Label htmlFor="realtimeFallbackFalBaseUrl">备用 FAL Base URL</Label>
<Input
id="realtimeFallbackFalBaseUrl"
value={realtimeCanvasConfig.fallbackFalBaseUrl}
onChange={(e) =>
setRealtimeCanvasConfig({
...realtimeCanvasConfig,
fallbackFalBaseUrl: e.target.value,
})
}
placeholder="https://fal.run"
disabled={!realtimeCanvasConfig.enabled || !realtimeCanvasConfig.fallbackEnabled}
/>
</div>
</>
) : (
<>
<div className="space-y-2">
@ -10711,7 +10889,7 @@ export function AdminSettingsClient() {
</div>
</div>
<p className="text-xs text-muted-foreground">
推荐：实时草图模型使用 `flux-2-klein-4b`，高质量可切换 `flux-2-pro-preview`。
推荐：BFL 实时草图模型使用 `flux-2-klein-4b`；FAL 需填写支持图像编辑输入的模型 ID。
</p>
<div className="grid grid-cols-2 gap-4">

15
app/api/admin/realtime/model-capability/route.ts
View File

@ -2,11 +2,12 @@ import { z } from "zod";
import { requireAdmin } from "@/lib/middleware/admin";
import { ChatSDKError } from "@/lib/errors";
import { probeBflModelSupport } from "@/lib/realtime/bfl-capability";
import { probeFalModelSupport } from "@/lib/realtime/fal-capability";
import { probeReplicateModelSupport } from "@/lib/realtime/replicate-capability";
import type { RealtimeProviderType } from "@/lib/realtime/config";
const schema = z.object({
providerType: z.enum(["bfl", "replicate"]).optional(),
providerType: z.enum(["bfl", "replicate", "fal"]).optional(),
apiKey: z.string().min(1),
modelId: z.string().min(1),
baseUrl: z.string().optional(),
@ -23,7 +24,12 @@ export async function POST(request: Request) {
const body = await request.json();
const data = schema.parse(body);
const providerType: RealtimeProviderType = data.providerType === "replicate" ? "replicate" : "bfl";
const providerType: RealtimeProviderType =
data.providerType === "replicate"
? "replicate"
: data.providerType === "fal"
? "fal"
: "bfl";
const result =
providerType === "replicate"
? await probeReplicateModelSupport({
@ -31,6 +37,11 @@ export async function POST(request: Request) {
modelId: data.modelId,
baseUrl: data.baseUrl,
})
: providerType === "fal"
? await probeFalModelSupport({
apiKey: data.apiKey,
modelId: data.modelId,
})
: await probeBflModelSupport({
apiKey: data.apiKey,
modelId: data.modelId,

6
app/api/admin/settings/route.ts
View File

@ -7,6 +7,7 @@ import {
} from "@/lib/db/queries";
import { ChatSDKError } from "@/lib/errors";
import { probeBflModelSupport } from "@/lib/realtime/bfl-capability";
import { probeFalModelSupport } from "@/lib/realtime/fal-capability";
import { probeReplicateModelSupport } from "@/lib/realtime/replicate-capability";
import {
normalizeRealtimeCanvasConfigForStorage,
@ -157,6 +158,11 @@ export async function POST(request: Request) {
modelId: provider.model,
baseUrl: provider.baseUrl,
})
: provider.type === "fal"
? await probeFalModelSupport({
apiKey: provider.apiKey,
modelId: provider.model,
})
: await probeBflModelSupport({
apiKey: provider.apiKey,
modelId: provider.model,

96
app/api/realtime/generate/route.ts
View File

@ -8,6 +8,7 @@ import {
RealtimeProviderConfig,
resolveRealtimeProviders,
} from "@/lib/realtime/config";
import { collectFalImageUrls, normalizeFalCredentials } from "@/lib/realtime/fal-shared";
import { ensurePublicImageUrl } from "@/lib/realtime/image-input";
type GeneratePayload = {
@ -56,6 +57,12 @@ type ReplicatePredictionResponse = {
};
};
type FalSubmitResult = {
imageUrl: string;
errorMessage: string;
status: number;
};
const FALLBACK_REALTIME_PROMPT =
"严格按照输入草图生成：保持主体轮廓、位置、比例和数量一致，只做材质与光影细化；禁止改变构图，禁止新增无关背景纹理。";
@ -207,6 +214,19 @@ function extractReplicateErrorMessage(payload: any, fallback: string) {
return message || fallback;
}
function extractFalErrorMessage(error: unknown, fallback: string) {
if (typeof error === "string") return error.trim() || fallback;
if (error instanceof Error && error.message.trim()) return error.message.trim();
if (error && typeof error === "object") {
const payload = error as Record<string, unknown>;
const message = String(
payload.message || payload.error || payload.detail || payload.reason || ""
).trim();
if (message) return message;
}
return fallback;
}
async function submitReplicatePrediction(params: {
apiKey: string;
model: string;
@ -316,6 +336,51 @@ async function pollReplicateResult(params: {
};
}
async function submitFalPrediction(params: {
apiKey: string;
model: string;
imageUrl: string;
prompt: string;
timeoutMs?: number;
}): Promise<FalSubmitResult> {
try {
const fal = await import("@fal-ai/serverless-client");
fal.config({
credentials: normalizeFalCredentials(params.apiKey),
});
const result = await fal.subscribe(params.model, {
input: {
prompt: params.prompt,
image_urls: [params.imageUrl],
},
logs: false,
timeout: Math.max(10_000, Number(params.timeoutMs || 90_000)),
});
const outputUrls = collectFalImageUrls(result);
if (outputUrls.length > 0) {
return {
imageUrl: outputUrls[0],
errorMessage: "",
status: 200,
};
}
return {
imageUrl: "",
errorMessage: "FAL 返回成功状态但未携带图像结果",
status: 502,
};
} catch (error) {
return {
imageUrl: "",
errorMessage: extractFalErrorMessage(error, "FAL 请求失败"),
status: 502,
};
}
}
type SubmitTaskResult = {
taskId: string;
pollingUrl: string;
@ -471,9 +536,12 @@ async function generateWithProvider(params: {
body: GeneratePayload;
config: RealtimeCanvasConfig;
}): Promise<ProviderAttemptResult> {
const isFal = params.provider.type === "fal";
const isReplicate = params.provider.type === "replicate";
const imageFieldCandidates = isReplicate
? getReplicateImageFieldCandidates(params.imagePrompt)
: isFal
? ["image_urls"] as const
: getBflImageFieldCandidates();
let selectedImageField: "input_image" | "image_prompt" = "input_image";
@ -481,7 +549,19 @@ async function generateWithProvider(params: {
let fallbackWarning = "";
for (let index = 0; index < imageFieldCandidates.length; index += 1) {
if (isReplicate) {
if (isFal) {
submitResult = {
taskId: "",
pollingUrl: "",
...(await submitFalPrediction({
apiKey: params.provider.apiKey,
model: params.provider.model,
imageUrl: params.imagePrompt,
prompt: params.prompt,
timeoutMs: 90_000,
})),
};
} else if (isReplicate) {
const candidate = getReplicateImageFieldCandidates(params.imagePrompt)[index];
const requestBody: Record<string, unknown> = {
prompt: params.prompt,
@ -541,6 +621,7 @@ async function generateWithProvider(params: {
}
if (
!isFal &&
index < imageFieldCandidates.length - 1 &&
shouldRetryWithAlternateImageField(submitResult.errorMessage)
) {
@ -566,7 +647,7 @@ async function generateWithProvider(params: {
}
let finalImageUrl = submitResult.imageUrl;
if (!finalImageUrl) {
if (!finalImageUrl && !isFal) {
const pollResult =
params.provider.type === "replicate"
? await pollReplicateResult({
@ -677,13 +758,16 @@ export async function POST(request: Request) {
for (const provider of providers) {
let providerImageInput = imagePrompt;
if (provider.type === "replicate") {
if (provider.type === "replicate" || provider.type === "fal") {
try {
replicateImageInputPromise =
replicateImageInputPromise ||
ensurePublicImageUrl({
imageUrl,
prefix: "realtime-replicate-input",
prefix:
provider.type === "fal"
? "realtime-fal-input"
: "realtime-replicate-input",
requestOrigin,
});
providerImageInput = await replicateImageInputPromise;
@ -734,7 +818,7 @@ export async function POST(request: Request) {
return Response.json({
ok: true,
provider: "bfl",
provider: result.provider.type,
providerSource: result.provider.source,
model: result.provider.model,
transport: "http-poll",
@ -753,7 +837,7 @@ export async function POST(request: Request) {
{
code: "BFL_REQUEST_FAILED",
message:
providerErrors.length > 0
providerErrors.length > 0
? `主备用实时服务均不可用：${providerErrors.join("；")}`
: "实时生成失败",
},

66
lib/realtime/config.ts
View File

@ -2,8 +2,10 @@ export const DEFAULT_BFL_MODEL = "flux-2-klein-4b";
export const DEFAULT_BFL_BASE_URL = "https://api.bfl.ai";
export const DEFAULT_REPLICATE_MODEL = "black-forest-labs/flux-2-klein-4b";
export const DEFAULT_REPLICATE_BASE_URL = "https://api.replicate.com/v1";
export const DEFAULT_FAL_BASE_URL = "https://fal.run";
export const DEFAULT_FAL_REALTIME_MODEL = "";
export type RealtimeProviderType = "bfl" | "replicate";
export type RealtimeProviderType = "bfl" | "replicate" | "fal";
export type RealtimeCanvasConfig = {
enabled?: boolean;
@ -14,6 +16,8 @@ export type RealtimeCanvasConfig = {
runwareApiKey?: string;
runwareModel?: string;
falApiKey?: string;
falModel?: string;
falBaseUrl?: string;
apiKey?: string;
replicateApiToken?: string;
replicateModel?: string;
@ -25,6 +29,8 @@ export type RealtimeCanvasConfig = {
fallbackRunwareApiKey?: string;
fallbackRunwareModel?: string;
fallbackFalApiKey?: string;
fallbackFalModel?: string;
fallbackFalBaseUrl?: string;
fallbackApiKey?: string;
fallbackReplicateApiToken?: string;
fallbackReplicateModel?: string;
@ -62,6 +68,7 @@ function trimString(value: unknown) {
}
function normalizeProviderType(raw: unknown): RealtimeProviderType {
if (raw === "fal") return "fal";
return raw === "replicate" ? "replicate" : "bfl";
}
@ -82,7 +89,8 @@ function hasFallbackIntent(config: RealtimeCanvasConfig) {
config.fallbackApiKey ||
config.fallbackBflBaseUrl ||
config.fallbackBflModel ||
config.fallbackRunwareModel ||
config.fallbackRunwareModel ||
config.fallbackFalModel ||
config.fallbackReplicateApiToken ||
config.fallbackReplicateModel
)
@ -109,6 +117,21 @@ export function resolvePrimaryRealtimeProvider(
};
}
if (type === "fal") {
const apiKey = trimString(config.falApiKey || config.apiKey || env.FAL_KEY || env.FAL_API_KEY);
const model = trimString(config.falModel || env.FAL_REALTIME_MODEL || env.FAL_MODEL);
if (!apiKey || !model) return null;
return {
type,
apiKey,
model,
baseUrl: trimString(config.falBaseUrl || env.FAL_BASE_URL || DEFAULT_FAL_BASE_URL),
source: "primary",
label: "主实时服务",
};
}
const apiKey = trimString(
config.bflApiKey ||
config.runwareApiKey ||
@ -155,6 +178,21 @@ export function resolveFallbackRealtimeProvider(
};
}
if (type === "fal") {
const apiKey = trimString(config.fallbackFalApiKey || config.fallbackApiKey);
const model = trimString(config.fallbackFalModel);
if (!apiKey || !model) return null;
return {
type,
apiKey,
model,
baseUrl: trimString(config.fallbackFalBaseUrl || DEFAULT_FAL_BASE_URL),
source: "fallback",
label: "备用实时服务",
};
}
const apiKey = trimString(
config.fallbackBflApiKey ||
config.fallbackRunwareApiKey ||
@ -238,11 +276,19 @@ export function normalizeRealtimeCanvasConfigForStorage(
? primary.apiKey
: trimString((value as RealtimeCanvasConfig).runwareApiKey),
falApiKey:
primary?.type === "bfl"
primary?.type === "bfl" || primary?.type === "fal"
? primary.apiKey
: trimString((value as RealtimeCanvasConfig).falApiKey),
falModel:
primary?.type === "fal"
? primary.model
: trimString((value as RealtimeCanvasConfig).falModel) || DEFAULT_FAL_REALTIME_MODEL,
falBaseUrl:
primary?.type === "fal"
? primary.baseUrl
: trimString((value as RealtimeCanvasConfig).falBaseUrl || env.FAL_BASE_URL || DEFAULT_FAL_BASE_URL),
apiKey:
primary?.type === "bfl"
primary?.type === "bfl" || primary?.type === "fal"
? primary.apiKey
: trimString((value as RealtimeCanvasConfig).apiKey),
runwareModel:
@ -276,11 +322,19 @@ export function normalizeRealtimeCanvasConfigForStorage(
? fallback.apiKey
: trimString((value as RealtimeCanvasConfig).fallbackRunwareApiKey),
fallbackFalApiKey:
fallback?.type === "bfl"
fallback?.type === "bfl" || fallback?.type === "fal"
? fallback.apiKey
: trimString((value as RealtimeCanvasConfig).fallbackFalApiKey),
fallbackFalModel:
fallback?.type === "fal"
? fallback.model
: trimString((value as RealtimeCanvasConfig).fallbackFalModel) || DEFAULT_FAL_REALTIME_MODEL,
fallbackFalBaseUrl:
fallback?.type === "fal"
? fallback.baseUrl
: trimString((value as RealtimeCanvasConfig).fallbackFalBaseUrl || DEFAULT_FAL_BASE_URL),
fallbackApiKey:
fallback?.type === "bfl"
fallback?.type === "bfl" || fallback?.type === "fal"
? fallback.apiKey
: trimString((value as RealtimeCanvasConfig).fallbackApiKey),
fallbackRunwareModel:

158
lib/realtime/fal-capability.ts Normal file
View File

@ -0,0 +1,158 @@
import sharp from "sharp";
import { ensurePublicImageUrl } from "@/lib/realtime/image-input";
import { collectFalImageUrls, normalizeFalCredentials } from "@/lib/realtime/fal-shared";
const DEFAULT_TIMEOUT_MS = 25_000;
export type FalCapabilityCode =
| "SUPPORTED"
| "AUTH_FAILED"
| "MODEL_INVALID"
| "UPSTREAM_ERROR";
export type FalModelProbeResult = {
supported: boolean;
code: FalCapabilityCode;
message: string;
status?: number;
};
type FalProbeParams = {
apiKey: string;
modelId?: string;
timeoutMs?: number;
};
let cachedProbeImageUrl: string | null = null;
async function getProbeImageUrl() {
if (cachedProbeImageUrl) return cachedProbeImageUrl;
const buffer = await sharp({
create: {
width: 64,
height: 64,
channels: 3,
background: { r: 255, g: 255, b: 255 },
},
})
.png()
.toBuffer();
cachedProbeImageUrl = await ensurePublicImageUrl({
imageUrl: `data:image/png;base64,${buffer.toString("base64")}`,
prefix: "realtime-fal-probe",
});
return cachedProbeImageUrl;
}
function toErrorMessage(error: unknown, fallback: string) {
if (!error) return fallback;
if (typeof error === "string") return error.trim() || fallback;
if (typeof error === "object") {
const anyError = error as Record<string, unknown>;
const message = String(
anyError.message || anyError.error || anyError.detail || anyError.reason || ""
).trim();
if (message) return message;
}
if (error instanceof Error && error.message.trim()) {
return error.message.trim();
}
return fallback;
}
function isModelInvalid(message: string) {
const normalized = message.toLowerCase();
return (
normalized.includes("model") &&
(normalized.includes("not found") ||
normalized.includes("invalid") ||
normalized.includes("unknown") ||
normalized.includes("does not exist"))
);
}
export async function probeFalModelSupport(
params: FalProbeParams
): Promise<FalModelProbeResult> {
const apiKey = normalizeFalCredentials(params.apiKey);
const modelId = String(params.modelId || "").trim();
if (!apiKey) {
return {
supported: false,
code: "AUTH_FAILED",
message: "缺少 FAL API Key。",
};
}
if (!modelId) {
return {
supported: false,
code: "MODEL_INVALID",
message: "缺少 FAL 模型 ID。",
};
}
try {
const imageUrl = await getProbeImageUrl();
const fal = await import("@fal-ai/serverless-client");
fal.config({
credentials: apiKey,
});
const result = await fal.subscribe(modelId, {
input: {
prompt: "white square photo",
image_urls: [imageUrl],
},
logs: false,
timeout: Math.max(5_000, Number(params.timeoutMs || DEFAULT_TIMEOUT_MS)),
});
const urls = collectFalImageUrls(result);
if (urls.length > 0) {
return {
supported: true,
code: "SUPPORTED",
message: "模型可用。",
status: 200,
};
}
return {
supported: false,
code: "UPSTREAM_ERROR",
message: "FAL 探针已返回，但未发现可用图片输出。",
status: 502,
};
} catch (error) {
const message = toErrorMessage(error, "FAL 探针请求失败。");
if (/401|403|unauthorized|forbidden|invalid api key|authentication/i.test(message)) {
return {
supported: false,
code: "AUTH_FAILED",
message: "FAL API Key 无效或无权限，请检查后重试。",
status: 401,
};
}
if (isModelInvalid(message)) {
return {
supported: false,
code: "MODEL_INVALID",
message: `模型 ${modelId} 不存在或不可用，请更换模型后重试。`,
status: 404,
};
}
return {
supported: false,
code: "UPSTREAM_ERROR",
message,
status: 502,
};
}
}

56
lib/realtime/fal-shared.ts Normal file
View File

@ -0,0 +1,56 @@
import "server-only";
export function normalizeFalCredentials(apiKey: string) {
const normalized = String(apiKey || "")
.replace(/\u00a0/g, " ")
.replace(/\u3000/g, " ")
.replace(/：/g, ":")
.trim();
const stripped = normalized.replace(/^bearer\s+/i, "").replace(/^key\s+/i, "");
const kvMatch = stripped.match(/^(?:fal[_-]?key|api[_-]?key|token)\s*[:：]\s*(.+)$/i);
if (kvMatch?.[1]) {
return kvMatch[1].trim();
}
return stripped;
}
export function collectFalImageUrls(payload: any): string[] {
const urls = new Set<string>();
const isLikelyImageUrl = (value: string) =>
/^https?:\/\//i.test(value) &&
(/\.(png|jpg|jpeg|webp|gif)(\?|$)/i.test(value) ||
value.includes("image") ||
value.includes("img") ||
value.includes("fal.media"));
const walk = (value: any, parentKey = "") => {
if (typeof value === "string") {
if (isLikelyImageUrl(value)) {
urls.add(value);
}
return;
}
if (Array.isArray(value)) {
for (const item of value) {
walk(item, parentKey);
}
return;
}
if (value && typeof value === "object") {
for (const [key, nestedValue] of Object.entries(value)) {
if (key === "url" && typeof nestedValue === "string" && /^https?:\/\//i.test(nestedValue)) {
if (/image|img|file|output|result|artifact/i.test(parentKey) || isLikelyImageUrl(nestedValue)) {
urls.add(nestedValue);
}
}
walk(nestedValue, key);
}
}
};
walk(payload);
return Array.from(urls);
}