Your Name 77f1401333
Some checks failed
Lint / build (20) (push) Has been cancelled
Playwright Tests / test (push) Has been cancelled
Add Replicate realtime canvas provider
2026-03-16 16:23:01 +08:00

740 lines
21 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import { auth } from "@/app/(auth)/auth";
import { getSystemSetting } from "@/lib/db/queries";
import {
DEFAULT_BFL_MODEL,
DEFAULT_REPLICATE_BASE_URL,
DEFAULT_REPLICATE_MODEL,
RealtimeCanvasConfig,
RealtimeProviderConfig,
resolveRealtimeProviders,
} from "@/lib/realtime/config";
type GeneratePayload = {
requestId?: string;
imageUrl?: string;
maskImageUrl?: string;
qualityProfile?: "preview" | "refine";
roi?:
| {
x?: number;
y?: number;
width?: number;
height?: number;
}
| null;
prompt?: string;
seed?: number;
imageSize?: "square" | "square_hd";
steps?: number;
cfgScale?: number;
strength?: number;
};
type BflSubmitResponse = {
id?: string;
polling_url?: string;
response?: {
id?: string;
polling_url?: string;
result?: unknown;
};
result?: unknown;
sample?: string;
error?: unknown;
message?: string;
};
type ReplicatePredictionResponse = {
id?: string;
status?: string;
output?: unknown;
error?: unknown;
detail?: unknown;
urls?: {
get?: string;
};
};
const FALLBACK_REALTIME_PROMPT =
"严格按照输入草图生成:保持主体轮廓、位置、比例和数量一致,只做材质与光影细化;禁止改变构图,禁止新增无关背景纹理。";
function clamp(value: number, min: number, max: number) {
return Math.min(max, Math.max(min, value));
}
function resolveSquareSize(
imageSize: "square" | "square_hd" | undefined,
maxInputResolution: number
) {
const preferred = imageSize === "square_hd" ? 1280 : 1024;
const capped = Math.min(preferred, maxInputResolution);
return clamp(Math.round(capped / 64) * 64, 512, 1536);
}
function resolveSeed(seed: unknown) {
const parsed = Number(seed);
if (!Number.isFinite(parsed)) return undefined;
const normalized = Math.floor(parsed);
if (normalized < 0 || normalized > 2147483647) return undefined;
return normalized;
}
function encodeModelPath(modelId: string) {
return modelId
.split("/")
.filter(Boolean)
.map((part) => encodeURIComponent(part))
.join("/");
}
function sanitizeText(raw: string) {
return raw.replace(/[\u2028\u2029]/g, " ").trim();
}
function extractBflImageUrl(payload: any): string {
const candidates = [
payload?.sample,
payload?.result?.sample,
payload?.response?.result?.sample,
payload?.output?.[0],
payload?.result?.output?.[0],
payload?.images?.[0],
payload?.result?.images?.[0],
payload?.image_url,
payload?.result?.image_url,
payload?.url,
payload?.result?.url,
];
for (const candidate of candidates) {
const value = String(candidate || "").trim();
if (!value) continue;
if (value.startsWith("data:")) return value;
if (/^https?:\/\//i.test(value)) return value;
}
return "";
}
function extractBflErrorMessage(payload: any, fallback: string) {
const message = String(
payload?.error?.message ||
payload?.error ||
payload?.detail ||
payload?.message ||
payload?.response?.error?.message ||
payload?.response?.error ||
""
).trim();
return message || fallback;
}
function shouldRetryWithAlternateImageField(message: string) {
const normalized = message.toLowerCase();
return (
normalized.includes("image_prompt") ||
normalized.includes("input_image") ||
normalized.includes("unsupported parameter") ||
normalized.includes("unknown field") ||
normalized.includes("invalid request")
);
}
function delay(ms: number) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
function toImagePromptValue(imageUrl: string) {
const trimmed = imageUrl.trim();
if (!trimmed) return "";
if (trimmed.startsWith("data:")) {
const commaIndex = trimmed.indexOf(",");
if (commaIndex > -1) {
return trimmed.slice(commaIndex + 1).trim();
}
}
return trimmed;
}
function getBflImageFieldCandidates() {
// BFL's current FLUX/Kontext editing APIs use `input_image` as the canonical field.
// Keep `image_prompt` only as a legacy fallback in case an older compatible endpoint is used.
return ["input_image", "image_prompt"] as const;
}
function getReplicateImageFieldCandidates(imagePrompt: string) {
return [
{ images: [imagePrompt] },
{ image: imagePrompt },
{ input_image: imagePrompt },
] as const;
}
function isSelfHostedRealtimeProvider(provider: RealtimeProviderConfig) {
return !provider.baseUrl.includes("api.bfl.ai");
}
function splitReplicateModelIdentifier(modelId: string) {
const [owner, ...rest] = modelId.split("/").filter(Boolean);
const name = rest.join("/");
if (!owner || !name) {
throw new Error(`无效的 Replicate 模型标识:${modelId}`);
}
return { owner, name };
}
function extractReplicateImageUrl(payload: any): string {
const output = payload?.output;
const candidates = [
Array.isArray(output) ? output[0] : output,
payload?.urls?.stream,
payload?.result,
];
for (const candidate of candidates) {
const value = String(candidate || "").trim();
if (!value) continue;
if (value.startsWith("data:")) return value;
if (/^https?:\/\//i.test(value)) return value;
}
return "";
}
function extractReplicateErrorMessage(payload: any, fallback: string) {
const message = String(payload?.detail || payload?.error || payload?.message || "").trim();
return message || fallback;
}
async function submitReplicatePrediction(params: {
apiKey: string;
model: string;
input: Record<string, unknown>;
}): Promise<SubmitTaskResult> {
const { owner, name } = splitReplicateModelIdentifier(params.model);
const endpoint = `${DEFAULT_REPLICATE_BASE_URL}/models/${encodeURIComponent(owner)}/${encodeURIComponent(name)}/predictions`;
const response = await fetch(endpoint, {
method: "POST",
headers: {
Authorization: `Bearer ${params.apiKey}`,
"Content-Type": "application/json",
Prefer: "wait=2",
},
body: JSON.stringify({ input: params.input }),
cache: "no-store",
});
const payload = (await response.json().catch(() => null)) as ReplicatePredictionResponse | null;
const imageUrl = extractReplicateImageUrl(payload);
const taskId = String(payload?.id || "").trim();
const pollingUrl = String(payload?.urls?.get || "").trim();
if (response.ok && (imageUrl || taskId || pollingUrl)) {
return {
taskId,
pollingUrl,
imageUrl,
errorMessage: "",
status: response.status,
};
}
return {
taskId,
pollingUrl,
imageUrl,
errorMessage: extractReplicateErrorMessage(payload, `Replicate 请求失败(${response.status})`),
status: response.status,
};
}
async function pollReplicateResult(params: {
apiKey: string;
pollingUrl?: string;
taskId?: string;
timeoutMs?: number;
}) {
const timeoutMs = Math.max(10_000, Number(params.timeoutMs || 60_000));
const deadline = Date.now() + timeoutMs;
let lastMessage = "";
while (Date.now() < deadline) {
const taskId = String(params.taskId || "").trim();
const url =
String(params.pollingUrl || "").trim() ||
`${DEFAULT_REPLICATE_BASE_URL}/predictions/${encodeURIComponent(taskId)}`;
const response = await fetch(url, {
method: "GET",
headers: {
Authorization: `Bearer ${params.apiKey}`,
Accept: "application/json",
},
cache: "no-store",
});
const payload = (await response.json().catch(() => null)) as ReplicatePredictionResponse | null;
const imageUrl = extractReplicateImageUrl(payload);
if (imageUrl) return { imageUrl, errorMessage: "", status: response.status };
const status = String(payload?.status || "").trim().toLowerCase();
if (status) {
lastMessage = status;
}
if (status === "succeeded") {
return {
imageUrl: "",
errorMessage: "Replicate 返回成功状态但未携带图像结果",
status: response.status,
};
}
if (status === "failed" || status === "canceled") {
return {
imageUrl: "",
errorMessage: extractReplicateErrorMessage(payload, "Replicate 任务执行失败"),
status: response.status,
};
}
if (!response.ok) {
return {
imageUrl: "",
errorMessage: extractReplicateErrorMessage(payload, `Replicate 轮询失败(${response.status})`),
status: response.status,
};
}
await delay(250);
}
return {
imageUrl: "",
errorMessage: `Replicate 轮询超时${lastMessage ? `(${lastMessage})` : ""}`,
status: 504,
};
}
type SubmitTaskResult = {
taskId: string;
pollingUrl: string;
imageUrl: string;
errorMessage: string;
status: number;
};
type ProviderAttemptResult =
| {
ok: true;
imageUrl: string;
taskId: string;
imageField: "input_image" | "image_prompt";
fallbackWarning: string;
provider: RealtimeProviderConfig;
}
| {
ok: false;
message: string;
status: number;
provider: RealtimeProviderConfig;
};
async function submitBflTask(params: {
apiKey: string;
baseUrl: string;
model: string;
body: Record<string, unknown>;
}): Promise<SubmitTaskResult> {
const endpoint = `${params.baseUrl}/v1/${encodeModelPath(params.model)}`;
const response = await fetch(endpoint, {
method: "POST",
headers: {
"x-key": params.apiKey,
"Content-Type": "application/json",
Accept: "application/json",
},
body: JSON.stringify(params.body),
cache: "no-store",
});
const payload = (await response.json().catch(() => null)) as BflSubmitResponse | null;
const imageUrl = extractBflImageUrl(payload);
const taskId = String(payload?.id || payload?.response?.id || "").trim();
const pollingUrl = String(payload?.polling_url || payload?.response?.polling_url || "").trim();
if (response.ok && (imageUrl || taskId || pollingUrl)) {
return {
taskId,
pollingUrl,
imageUrl,
errorMessage: "",
status: response.status,
};
}
return {
taskId,
pollingUrl,
imageUrl,
errorMessage: extractBflErrorMessage(payload, `BFL 请求失败(${response.status})`),
status: response.status,
};
}
async function pollBflResult(params: {
apiKey: string;
baseUrl: string;
pollingUrl?: string;
taskId?: string;
timeoutMs?: number;
}) {
const timeoutMs = Math.max(20_000, Number(params.timeoutMs || 100_000));
const deadline = Date.now() + timeoutMs;
let lastMessage = "";
while (Date.now() < deadline) {
const url = (() => {
const direct = String(params.pollingUrl || "").trim();
if (direct) {
if (/^https?:\/\//i.test(direct)) return direct;
if (direct.startsWith("/")) return `${params.baseUrl}${direct}`;
}
const taskId = String(params.taskId || "").trim();
return `${params.baseUrl}/v1/get_result?id=${encodeURIComponent(taskId)}`;
})();
const response = await fetch(url, {
method: "GET",
headers: {
"x-key": params.apiKey,
Accept: "application/json",
},
cache: "no-store",
});
const payload = await response.json().catch(() => null);
const imageUrl = extractBflImageUrl(payload);
if (imageUrl) return { imageUrl, errorMessage: "", status: response.status };
const status = String(
payload?.status ||
payload?.task_status ||
payload?.response?.status ||
payload?.response?.task_status ||
""
)
.trim()
.toLowerCase();
if (status) {
lastMessage = status;
}
if (status === "ready" || status === "success" || status === "succeeded" || status === "completed") {
return {
imageUrl: "",
errorMessage: "BFL 返回成功状态但未携带图像结果",
status: response.status,
};
}
if (status === "error" || status === "failed" || status === "cancelled") {
return {
imageUrl: "",
errorMessage: extractBflErrorMessage(payload, "BFL 任务执行失败"),
status: response.status,
};
}
if (!response.ok) {
return {
imageUrl: "",
errorMessage: extractBflErrorMessage(payload, `BFL 轮询失败(${response.status})`),
status: response.status,
};
}
await delay(1_200);
}
return {
imageUrl: "",
errorMessage: `BFL 轮询超时${lastMessage ? `(${lastMessage})` : ""}`,
status: 504,
};
}
async function generateWithProvider(params: {
provider: RealtimeProviderConfig;
imagePrompt: string;
prompt: string;
size: number;
seed?: number;
body: GeneratePayload;
config: RealtimeCanvasConfig;
}): Promise<ProviderAttemptResult> {
const isReplicate = params.provider.type === "replicate";
const imageFieldCandidates = isReplicate
? getReplicateImageFieldCandidates(params.imagePrompt)
: getBflImageFieldCandidates();
let selectedImageField: "input_image" | "image_prompt" = "input_image";
let submitResult: SubmitTaskResult | null = null;
let fallbackWarning = "";
for (let index = 0; index < imageFieldCandidates.length; index += 1) {
if (isReplicate) {
const candidate = getReplicateImageFieldCandidates(params.imagePrompt)[index];
const requestBody: Record<string, unknown> = {
prompt: params.prompt,
...(typeof params.seed === "number" ? { seed: params.seed } : {}),
...candidate,
};
submitResult = await submitReplicatePrediction({
apiKey: params.provider.apiKey,
model: params.provider.model || DEFAULT_REPLICATE_MODEL,
input: requestBody,
});
} else {
selectedImageField = imageFieldCandidates[index] as "input_image" | "image_prompt";
const requestBody: Record<string, unknown> = {
prompt: params.prompt,
width: params.size,
height: params.size,
output_format: "jpeg",
safety_tolerance: 2,
[selectedImageField]: params.imagePrompt,
...(typeof params.seed === "number" ? { seed: params.seed } : {}),
};
if (isSelfHostedRealtimeProvider(params.provider)) {
requestBody.steps = clamp(Number(params.body.steps ?? params.config.defaults?.steps ?? 1), 1, 6);
requestBody.cfg_scale = clamp(
Number(params.body.cfgScale ?? params.config.defaults?.cfgScale ?? 1),
1,
14
);
requestBody.strength = clamp(
Number(params.body.strength ?? params.config.defaults?.strength ?? 0.32),
0.05,
1
);
requestBody.preview_megapixels = clamp(Number(params.config.previewMegapixels ?? 0.4), 0.2, 0.8);
requestBody.detail_megapixels = clamp(Number(params.config.detailMegapixels ?? 1.0), 0.6, 1.2);
requestBody.output_jpeg_quality = clamp(
Number(params.config.outputJpegQuality ?? 62),
35,
90
);
requestBody.quality_profile = params.body.qualityProfile || "refine";
}
submitResult = await submitBflTask({
apiKey: params.provider.apiKey,
baseUrl: params.provider.baseUrl,
model: params.provider.model,
body: requestBody,
});
}
if (!submitResult?.errorMessage) {
break;
}
if (
index < imageFieldCandidates.length - 1 &&
shouldRetryWithAlternateImageField(submitResult.errorMessage)
) {
fallbackWarning = "模型参数兼容性已自动切换,继续生成。";
continue;
}
return {
ok: false,
message: submitResult?.errorMessage || "实时生成提交失败",
status: submitResult?.status || 502,
provider: params.provider,
};
}
if (!submitResult) {
return {
ok: false,
message: "实时生成提交失败",
status: 502,
provider: params.provider,
};
}
let finalImageUrl = submitResult.imageUrl;
if (!finalImageUrl) {
const pollResult =
params.provider.type === "replicate"
? await pollReplicateResult({
apiKey: params.provider.apiKey,
pollingUrl: submitResult.pollingUrl,
taskId: submitResult.taskId,
timeoutMs: 60_000,
})
: await pollBflResult({
apiKey: params.provider.apiKey,
baseUrl: params.provider.baseUrl,
pollingUrl: submitResult.pollingUrl,
taskId: submitResult.taskId,
timeoutMs: 110_000,
});
if (!pollResult.imageUrl) {
return {
ok: false,
message: pollResult.errorMessage || "实时生成未返回图像",
status: pollResult.status || 502,
provider: params.provider,
};
}
finalImageUrl = pollResult.imageUrl;
}
return {
ok: true,
imageUrl: finalImageUrl,
taskId: submitResult.taskId,
imageField: selectedImageField,
fallbackWarning,
provider: params.provider,
};
}
export async function POST(request: Request) {
const session = await auth();
if (!session?.user?.id) {
return Response.json({ code: "UNAUTHORIZED", message: "请先登录" }, { status: 401 });
}
let body: GeneratePayload;
try {
body = (await request.json()) as GeneratePayload;
} catch {
return Response.json({ code: "BAD_REQUEST", message: "无效 JSON" }, { status: 400 });
}
const imageUrl = String(body.imageUrl || "").trim();
if (!imageUrl) {
return Response.json({ code: "BAD_REQUEST", message: "缺少 imageUrl" }, { status: 400 });
}
const setting = await getSystemSetting({ key: "realtime_canvas_config" }).catch(() => null);
const config = (setting?.value || {}) as RealtimeCanvasConfig;
if (config.enabled === false) {
return Response.json(
{ code: "FEATURE_DISABLED", message: "实时画布功能未开启" },
{ status: 403 }
);
}
const providers = resolveRealtimeProviders(config);
if (providers.length === 0) {
return Response.json(
{
code: "BFL_KEY_MISSING",
message: "未配置可用的实时画布服务",
},
{ status: 500 }
);
}
const maxInputResolution = clamp(Number(config.maxInputResolution || 1024), 512, 1536);
const size = resolveSquareSize(body.imageSize, maxInputResolution);
const seed = resolveSeed(body.seed);
const rawPrompt = sanitizeText(String(body.prompt || "").trim()) || FALLBACK_REALTIME_PROMPT;
const negativePrompt =
sanitizeText(String(config.negativePrompt || "").trim()) ||
"改变构图, 改变主体位置, 多余人物, 重复物体, 镜像翻转, 错误透视, 模糊轮廓, 文字水印, 线稿, 草图";
const positivePrompt = sanitizeText(
[
rawPrompt,
"Keep exact object silhouette, placement and count from the input sketch.",
"Photorealistic material and lighting only, keep composition unchanged.",
`Negative constraints: ${negativePrompt}`,
].join(" ")
);
const imagePrompt = toImagePromptValue(imageUrl);
if (!imagePrompt) {
return Response.json(
{
code: "BAD_REQUEST",
message: "输入图像格式无效",
},
{ status: 400 }
);
}
const providerErrors: string[] = [];
for (const provider of providers) {
const result = await generateWithProvider({
provider,
imagePrompt,
prompt: positivePrompt,
size,
seed,
body,
config,
});
if (!result.ok) {
providerErrors.push(`${provider.label}(${provider.baseUrl}): ${result.message}`);
continue;
}
const warningParts = [result.fallbackWarning];
if (result.provider.source === "fallback") {
warningParts.push("主实时服务异常,已自动切换到备用服务。");
}
const warning = warningParts.filter(Boolean).join(" ");
console.log("[RealtimeBFL] frame success", {
userId: session.user.id,
requestId: body.requestId || null,
model: result.provider.model || DEFAULT_BFL_MODEL,
providerSource: result.provider.source,
providerBaseUrl: result.provider.baseUrl,
taskId: result.taskId || null,
hasPrompt: Boolean(rawPrompt && rawPrompt !== FALLBACK_REALTIME_PROMPT),
imageField: result.imageField,
qualityProfile: body.qualityProfile || "refine",
size,
roi: body.roi || null,
hasMask: Boolean(body.maskImageUrl),
warning: warning || null,
});
return Response.json({
ok: true,
provider: "bfl",
providerSource: result.provider.source,
model: result.provider.model,
transport: "http-poll",
taskUUID: result.taskId || undefined,
imageDataURI: result.imageUrl.startsWith("data:")
? result.imageUrl
: !/^https?:\/\//i.test(result.imageUrl)
? `data:image/jpeg;base64,${result.imageUrl}`
: undefined,
imageUrl: /^https?:\/\//i.test(result.imageUrl) ? result.imageUrl : undefined,
warning: warning || undefined,
});
}
return Response.json(
{
code: "BFL_REQUEST_FAILED",
message:
providerErrors.length > 0
? `主备用实时服务均不可用:${providerErrors.join(";")}`
: "实时生成失败",
},
{ status: 502 }
);
}