diff --git a/app/api/realtime/generate/route.ts b/app/api/realtime/generate/route.ts index 6005d7f1a..56e6b4121 100644 --- a/app/api/realtime/generate/route.ts +++ b/app/api/realtime/generate/route.ts @@ -8,6 +8,7 @@ import { RealtimeProviderConfig, resolveRealtimeProviders, } from "@/lib/realtime/config"; +import { ensurePublicImageUrl } from "@/lib/realtime/image-input"; type GeneratePayload = { requestId?: string; @@ -670,11 +671,33 @@ export async function POST(request: Request) { } const providerErrors: string[] = []; + const requestOrigin = new URL(request.url).origin; + let replicateImageInputPromise: Promise | null = null; for (const provider of providers) { + let providerImageInput = imagePrompt; + + if (provider.type === "replicate") { + try { + replicateImageInputPromise = + replicateImageInputPromise || + ensurePublicImageUrl({ + imageUrl, + prefix: "realtime-replicate-input", + requestOrigin, + }); + providerImageInput = await replicateImageInputPromise; + } catch (error) { + const message = + error instanceof Error ? error.message : "实时画布输入图上传失败"; + providerErrors.push(`${provider.label}(${provider.baseUrl}): ${message}`); + continue; + } + } + const result = await generateWithProvider({ provider, - imagePrompt, + imagePrompt: providerImageInput, prompt: positivePrompt, size, seed, diff --git a/lib/realtime/image-input.ts b/lib/realtime/image-input.ts new file mode 100644 index 000000000..eecadefb3 --- /dev/null +++ b/lib/realtime/image-input.ts @@ -0,0 +1,92 @@ +import "server-only"; + +import { uploadFile } from "@/lib/storage/service"; + +const DATA_IMAGE_URL_PATTERN = /^data:(image\/[a-z0-9.+-]+);base64,([a-z0-9+/=\s]+)$/i; + +const MIME_TYPE_EXTENSION_MAP: Record = { + "image/jpeg": "jpg", + "image/jpg": "jpg", + "image/png": "png", + "image/webp": "webp", + "image/gif": "gif", + "image/bmp": "bmp", + "image/tiff": "tiff", + "image/svg+xml": "svg", +}; + +function normalizeImageMimeType(mimeType: string) { + return String(mimeType || "").trim().toLowerCase(); +} + +function resolveImageExtension(mimeType: string) { + const normalized = normalizeImageMimeType(mimeType); + return MIME_TYPE_EXTENSION_MAP[normalized] || normalized.split("/")[1] || "png"; +} + +function toArrayBuffer(buffer: Buffer) { + return buffer.buffer.slice( + buffer.byteOffset, + buffer.byteOffset + buffer.byteLength + ) as ArrayBuffer; +} + +export function parseDataImageUrl(raw: string) { + const trimmed = String(raw || "").trim(); + const match = trimmed.match(DATA_IMAGE_URL_PATTERN); + if (!match) return null; + + const mimeType = normalizeImageMimeType(match[1]); + const base64Payload = match[2].replace(/\s+/g, ""); + + return { + mimeType, + base64Payload, + extension: resolveImageExtension(mimeType), + }; +} + +export async function uploadDataImageUrl(params: { + imageUrl: string; + prefix: string; +}) { + const parsed = parseDataImageUrl(params.imageUrl); + if (!parsed) { + throw new Error("图片必须是有效的 data:image;base64 格式"); + } + + const buffer = Buffer.from(parsed.base64Payload, "base64"); + const filename = `${params.prefix}-${Date.now()}-${Math.random() + .toString(36) + .slice(2, 8)}.${parsed.extension}`; + const uploaded = await uploadFile(filename, toArrayBuffer(buffer)); + return uploaded.url; +} + +export async function ensurePublicImageUrl(params: { + imageUrl: string; + prefix: string; + requestOrigin?: string; +}) { + const trimmed = String(params.imageUrl || "").trim(); + if (!trimmed) { + throw new Error("缺少输入图片"); + } + + if (/^https?:\/\//i.test(trimmed)) { + return trimmed; + } + + if (params.requestOrigin && trimmed.startsWith("/")) { + return new URL(trimmed, params.requestOrigin).toString(); + } + + if (trimmed.startsWith("data:")) { + return uploadDataImageUrl({ + imageUrl: trimmed, + prefix: params.prefix, + }); + } + + throw new Error("图片必须是公网 URL、站内相对路径或 data:image;base64"); +} diff --git a/lib/realtime/replicate-capability.ts b/lib/realtime/replicate-capability.ts index 146611547..f5c90aebe 100644 --- a/lib/realtime/replicate-capability.ts +++ b/lib/realtime/replicate-capability.ts @@ -1,4 +1,5 @@ import sharp from "sharp"; +import { ensurePublicImageUrl } from "@/lib/realtime/image-input"; const DEFAULT_REPLICATE_BASE_URL = "https://api.replicate.com/v1"; const DEFAULT_REPLICATE_MODEL = "black-forest-labs/flux-2-klein-4b"; @@ -24,7 +25,7 @@ type ReplicateProbeParams = { timeoutMs?: number; }; -let cachedProbeImageDataUri: string | null = null; +let cachedProbeImageUrl: string | null = null; function trimString(value: unknown) { return String(value || "").trim(); @@ -78,8 +79,8 @@ function shouldRetryWithAltField(message: string) { ); } -async function getProbeImageDataUri() { - if (cachedProbeImageDataUri) return cachedProbeImageDataUri; +async function getProbeImageUrl() { + if (cachedProbeImageUrl) return cachedProbeImageUrl; const buffer = await sharp({ create: { @@ -92,8 +93,11 @@ async function getProbeImageDataUri() { .png() .toBuffer(); - cachedProbeImageDataUri = `data:image/png;base64,${buffer.toString("base64")}`; - return cachedProbeImageDataUri; + cachedProbeImageUrl = await ensurePublicImageUrl({ + imageUrl: `data:image/png;base64,${buffer.toString("base64")}`, + prefix: "realtime-replicate-probe", + }); + return cachedProbeImageUrl; } async function createPrediction(params: { @@ -144,7 +148,7 @@ export async function probeReplicateModelSupport( ); try { - const image = await getProbeImageDataUri(); + const image = await getProbeImageUrl(); const candidateInputs = [ { prompt: "white square photo", images: [image] }, { prompt: "white square photo", image },