import "server-only"; import sharp from "sharp"; import { constrainImageDimensions } from "@/lib/ai/image-size-constraints"; import { createDeferredUploadedImage } from "@/lib/storage/image-archive"; import { uploadFile } from "@/lib/storage/service"; interface StepfunImageConfig { apiKey: string; baseURL: string; } export interface StepfunImageParams { aspectRatio?: string; imageSize?: string; imageCount?: number; size?: string; seed?: number; negativePrompt?: string; } interface UploadedImage { thumbnail: string; original: string; } const DEFAULT_BASE_URL = "https://api.stepfun.com"; const DEFAULT_TIMEOUT_MS = 600000; const DEFAULT_CFG_SCALE = 1.0; const DEFAULT_STEPS = 8; const DEFAULT_TEXT_MODE = true; const MAX_PROMPT_LENGTH = 512; function normalizeBaseURL(baseURL: string) { const normalized = String(baseURL || "").trim().replace(/\/+$/, ""); const base = normalized || DEFAULT_BASE_URL; if (/\/v1$/i.test(base)) return base; return `${base}/v1`; } function joinEndpoint(baseURL: string, path: string) { const normalizedPath = path.startsWith("/") ? path : `/${path}`; return `${normalizeBaseURL(baseURL)}${normalizedPath}`; } function normalizePrompt(prompt: string) { const normalized = String(prompt || "").trim(); if (normalized.length <= MAX_PROMPT_LENGTH) return normalized; console.warn("[StepFun Image] Prompt exceeds 512 characters, truncating for step-image-edit-2"); return normalized.slice(0, MAX_PROMPT_LENGTH); } function parseAspectRatioValue(ratio?: string | null) { const normalized = String(ratio || "").trim(); const [width, height] = normalized.split(":").map((item) => Number(item)); if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) { return 1; } return width / height; } function resolveStepfunSize(aspectRatio?: string, explicitSize?: string) { const normalizedExplicit = String(explicitSize || "").trim(); if (/^\d{3,4}x\d{3,4}$/i.test(normalizedExplicit)) { const [rawWidth, rawHeight] = normalizedExplicit.split("x").map((item) => Number(item)); const { width, height } = constrainImageDimensions( { width: rawWidth, height: rawHeight }, { maxEdge: 1360, maxPixels: 1360 * 1360, minEdge: 256, multiple: 8, } ); return `${width}x${height}`; } const ratio = parseAspectRatioValue(aspectRatio); const candidates = [ { size: "1024x1024", ratio: 1 }, { size: "768x1360", ratio: 1360 / 768 }, { size: "896x1184", ratio: 1184 / 896 }, { size: "1360x768", ratio: 768 / 1360 }, { size: "1184x896", ratio: 896 / 1184 }, ]; let best = candidates[0]; let minDiff = Number.POSITIVE_INFINITY; for (const candidate of candidates) { const diff = Math.abs(candidate.ratio - ratio); if (diff < minDiff) { best = candidate; minDiff = diff; } } return best.size; } function detectImageExtension(imageUrl: string, contentType?: string | null) { const type = String(contentType || "").toLowerCase(); if (type.includes("jpeg") || type.includes("jpg")) return "jpg"; if (type.includes("webp")) return "webp"; if (type.includes("png")) return "png"; const cleanUrl = imageUrl.split("?")[0]?.toLowerCase() || ""; if (cleanUrl.endsWith(".jpg") || cleanUrl.endsWith(".jpeg")) return "jpg"; if (cleanUrl.endsWith(".webp")) return "webp"; if (cleanUrl.endsWith(".png")) return "png"; return "png"; } async function createThumbnail(buffer: Buffer) { try { return await sharp(buffer) .resize(400, 400, { fit: "inside", withoutEnlargement: true }) .jpeg({ quality: 80 }) .toBuffer(); } catch (error) { console.error("[StepFun Image] Failed to create thumbnail:", error); return buffer; } } function generateFilename(prefix: string, extension: string) { const timestamp = Date.now(); const random = Math.random().toString(36).slice(2, 8); return `${prefix}-${timestamp}-${random}.${extension}`; } async function uploadGeneratedBuffer( buffer: Buffer, extension: string, mimeType: string ): Promise { const thumbnailBuffer = await createThumbnail(buffer); const originalFilename = generateFilename("stepfun-image-orig", extension); const originalArrayBuffer = buffer.buffer.slice( buffer.byteOffset, buffer.byteOffset + buffer.byteLength ) as ArrayBuffer; const originalUploaded = await uploadFile(originalFilename, originalArrayBuffer); const thumbFilename = generateFilename("stepfun-image-thumb", "jpg"); const thumbArrayBuffer = thumbnailBuffer.buffer.slice( thumbnailBuffer.byteOffset, thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength ) as ArrayBuffer; const thumbUploaded = await uploadFile(thumbFilename, thumbArrayBuffer); console.log("[StepFun Image] Uploaded generated image:", { original: originalUploaded.url, thumbnail: thumbUploaded.url, mimeType, }); return { thumbnail: thumbUploaded.url, original: originalUploaded.url, }; } async function downloadAndUploadImage(imageUrl: string): Promise { const deferred = await createDeferredUploadedImage(imageUrl, { providerType: "stepfun-image", }); if (deferred) return deferred; const response = await fetch(imageUrl, { cache: "no-store" }); if (!response.ok) { throw new Error(`Failed to download StepFun generated image: ${response.status}`); } const buffer = Buffer.from(await response.arrayBuffer()); const extension = detectImageExtension(imageUrl, response.headers.get("content-type")); const mimeType = response.headers.get("content-type") || `image/${extension === "jpg" ? "jpeg" : extension}`; return uploadGeneratedBuffer(buffer, extension, mimeType); } async function uploadBase64ToS3(base64Data: string, mimeType: string) { const buffer = Buffer.from(base64Data, "base64"); const extension = detectImageExtension("", mimeType); return uploadGeneratedBuffer( buffer, extension, mimeType || `image/${extension === "jpg" ? "jpeg" : extension}` ); } function extractPromptAndImages(messages: Array<{ role: string; content: any }>) { const lastUserMessage = messages.filter((message) => message.role === "user").pop(); if (!lastUserMessage) { throw new Error("No user message found"); } let prompt = ""; const imageUrls: string[] = []; const pushImageUrl = (value?: string) => { if (!value) return; const normalized = String(value).trim(); if (!normalized || imageUrls.includes(normalized)) return; imageUrls.push(normalized); }; if (typeof lastUserMessage.content === "string") { prompt = lastUserMessage.content.trim(); } else if (Array.isArray(lastUserMessage.content)) { for (const part of lastUserMessage.content) { if (part?.type === "text" && typeof part.text === "string") { prompt = `${prompt} ${part.text}`.trim(); } else if (part?.type === "image_url" && part.image_url?.url) { pushImageUrl(part.image_url.url); } else if (part?.type === "file") { pushImageUrl(part.url || part.data); } else if (part?.type === "image") { pushImageUrl(part.image || part.url); } } } if (!prompt) { throw new Error("No prompt found in user message"); } return { prompt: normalizePrompt(prompt), imageUrls }; } async function loadInputImage(source: string) { const normalized = String(source || "").trim(); if (!normalized) { throw new Error("输入图片地址为空"); } if (normalized.startsWith("data:")) { const match = normalized.match(/^data:([A-Za-z0-9.+/-]+);base64,(.+)$/); if (!match) { throw new Error("不支持的 Base64 图片格式"); } const mimeType = match[1] || "image/png"; const buffer = Buffer.from(match[2], "base64"); const extension = detectImageExtension("", mimeType); return { buffer, mimeType, filename: generateFilename("stepfun-image-edit-input", extension), }; } const response = await fetch(normalized, { cache: "no-store" }); if (!response.ok) { throw new Error(`Failed to fetch StepFun input image: ${response.status}`); } const buffer = Buffer.from(await response.arrayBuffer()); const mimeType = response.headers.get("content-type") || "image/png"; const extension = detectImageExtension(normalized, mimeType); return { buffer, mimeType, filename: generateFilename("stepfun-image-edit-input", extension), }; } function normalizeSeed(seed?: number) { if (typeof seed !== "number" || !Number.isFinite(seed)) return undefined; return Math.max(0, Math.floor(seed)); } function extractErrorMessage(payload: any) { if (!payload) return "StepFun Images 请求失败"; if (typeof payload === "string") return payload; if (payload.error?.message) return String(payload.error.message); if (payload.message) return String(payload.message); try { return JSON.stringify(payload); } catch { return "StepFun Images 请求失败"; } } async function parseJsonResponse(response: Response) { const rawText = await response.text(); try { return JSON.parse(rawText); } catch { return rawText; } } async function requestStepfunImage(params: { baseURL: string; apiKey: string; model: string; prompt: string; size?: string; imageUrls: string[]; seed?: number; }) { const { baseURL, apiKey, model, prompt, size, imageUrls, seed } = params; const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), DEFAULT_TIMEOUT_MS); try { if (imageUrls.length > 0) { const endpoint = joinEndpoint(baseURL, "/images/edits"); const formData = new FormData(); formData.append("model", model); formData.append("prompt", prompt); formData.append("response_format", "b64_json"); formData.append("cfg_scale", String(DEFAULT_CFG_SCALE)); formData.append("steps", String(DEFAULT_STEPS)); formData.append("text_mode", String(DEFAULT_TEXT_MODE)); const normalizedSeed = normalizeSeed(seed); if (normalizedSeed !== undefined) formData.append("seed", String(normalizedSeed)); for (const source of imageUrls.slice(0, 1)) { const image = await loadInputImage(source); formData.append("image", new Blob([image.buffer], { type: image.mimeType }), image.filename); } const response = await fetch(endpoint, { method: "POST", headers: { Authorization: `Bearer ${apiKey}`, }, body: formData, signal: controller.signal, }); const payload = await parseJsonResponse(response); if (!response.ok) { throw new Error(`${extractErrorMessage(payload)} (HTTP ${response.status})`); } return payload; } const endpoint = joinEndpoint(baseURL, "/images/generations"); const body: Record = { model, prompt, response_format: "b64_json", cfg_scale: DEFAULT_CFG_SCALE, steps: DEFAULT_STEPS, text_mode: DEFAULT_TEXT_MODE, ...(size ? { size } : {}), }; const normalizedSeed = normalizeSeed(seed); if (normalizedSeed !== undefined) body.seed = normalizedSeed; const response = await fetch(endpoint, { method: "POST", headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json", }, body: JSON.stringify(body), signal: controller.signal, }); const payload = await parseJsonResponse(response); if (!response.ok) { throw new Error(`${extractErrorMessage(payload)} (HTTP ${response.status})`); } return payload; } finally { clearTimeout(timeoutId); } } async function extractUploadedImages(payload: any): Promise { const data = Array.isArray(payload?.data) ? payload.data : []; const uploadedImages: UploadedImage[] = []; for (const item of data) { if (item?.url && typeof item.url === "string") { uploadedImages.push(await downloadAndUploadImage(item.url)); continue; } if (item?.b64_json && typeof item.b64_json === "string") { uploadedImages.push(await uploadBase64ToS3(item.b64_json, "image/png")); } } return uploadedImages; } function formatMarkdown(images: UploadedImage[], imageSize: string) { return images .map((image) => `![thumb|||${image.original}|||${imageSize}](${image.thumbnail})`) .join("\n\n"); } export function createStepfunImage({ apiKey, baseURL }: StepfunImageConfig) { return { provider: "stepfun-image", chat: (modelId: string) => ({ provider: "stepfun-image", doGenerate: async (options: { messages: Array<{ role: string; content: any }>; imageParams?: StepfunImageParams; overrideModelId?: string; }) => { const { prompt, imageUrls } = extractPromptAndImages(options.messages); const activeModelId = options.overrideModelId || modelId; const size = imageUrls.length > 0 ? undefined : resolveStepfunSize(options.imageParams?.aspectRatio, options.imageParams?.size); console.log("[StepFun Image] Submitting request", { model: activeModelId, size, imageCount: imageUrls.length, baseURL: normalizeBaseURL(baseURL), }); const payload = await requestStepfunImage({ baseURL, apiKey, model: activeModelId, prompt, size, imageUrls, seed: options.imageParams?.seed, }); const uploadedImages = await extractUploadedImages(payload); if (uploadedImages.length === 0) { throw new Error("StepFun Images 未返回可用图片"); } const text = formatMarkdown(uploadedImages, options.imageParams?.imageSize || "1K"); return { text, finishReason: "stop" as const, usage: { promptTokens: 0, completionTokens: text.length, }, }; }, doStream: async function* (options: { messages: Array<{ role: string; content: any }>; imageParams?: StepfunImageParams; overrideModelId?: string; }) { const result = await this.doGenerate(options); yield { type: "text-delta", textDelta: result.text, }; }, }), }; }