From 979c9603599163c1b600a2d7823e8ad4a5ad2bb8 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 29 Apr 2026 15:51:23 +0800 Subject: [PATCH] feat: add StepFun image provider --- .../settings/admin-settings-client.tsx | 12 + app/(chat)/api/chat/route.ts | 42 +- app/api/admin/ai-providers/route.ts | 4 + app/api/canvas/text-edit/route.ts | 4 + components/chat.tsx | 7 +- components/image-generation-loading.tsx | 1 + components/image-params-bar.tsx | 7 +- components/messages.tsx | 3 + lib/ai/dynamic-provider.ts | 14 + lib/ai/model-classification.ts | 1 + lib/ai/model-router.ts | 13 +- lib/ai/provider-defaults.ts | 1 + lib/ai/providers/stepfun-image.ts | 436 ++++++++++++++++++ lib/ai/runtime/generated-image-persistence.ts | 2 + lib/canvas/generate-image.ts | 20 +- lib/db/migrate.ts | 3 +- .../0077_add_stepfun_image_provider_type.sql | 36 ++ lib/db/migrations/meta/_journal.json | 7 + lib/db/queries.ts | 4 +- lib/db/schema.ts | 2 +- lib/image-url.ts | 4 + 21 files changed, 596 insertions(+), 27 deletions(-) create mode 100644 lib/ai/providers/stepfun-image.ts create mode 100644 lib/db/migrations/0077_add_stepfun_image_provider_type.sql diff --git a/app/(admin)/settings/admin-settings-client.tsx b/app/(admin)/settings/admin-settings-client.tsx index d8a3c621e..fb40e28f1 100644 --- a/app/(admin)/settings/admin-settings-client.tsx +++ b/app/(admin)/settings/admin-settings-client.tsx @@ -491,6 +491,7 @@ type AiProvider = { | "ruxa-image" | "fal-image" | "openai-image" + | "stepfun-image" | "midjourney" | "fal-video" | "zhipu" @@ -1274,6 +1275,7 @@ const PROVIDER_TYPES = [ { value: "ruxa-image", label: "RUXA生图" }, { value: "fal-image", label: "FAL 图片生成/编辑" }, { value: "openai-image", label: "OpenAI 图片兼容" }, + { value: "stepfun-image", label: "阶跃星辰 Step Image Edit" }, { value: "bltcy-video", label: "柏拉图统一视频" }, { value: "grsai-gpt-image", label: "Grsai GPT Image" }, { value: "grsai-video", label: "Grsai 视频模型" }, @@ -1306,6 +1308,7 @@ const PROVIDER_DEFAULT_BASE_URL: Record = { "ruxa-image": "https://api.ruxa.ai", "fal-image": "https://fal.run", "openai-image": "https://api.openai.com/v1", + "stepfun-image": "https://api.stepfun.com", "bltcy-video": "https://api.bltcy.ai", "grsai-gpt-image": "https://grsai.dakka.com.cn", "grsai-video": "https://grsai.dakka.com.cn", @@ -3232,6 +3235,7 @@ export function AdminSettingsClient() { | "ruxa-image" | "fal-image" | "openai-image" + | "stepfun-image" | "midjourney" | "fal-video" | "zhipu" @@ -8464,6 +8468,7 @@ export function AdminSettingsClient() { "ruxa-image": "bg-emerald-500", "fal-image": "bg-sky-500", "openai-image": "bg-green-600", + "stepfun-image": "bg-indigo-600", "grsai-gpt-image": "bg-emerald-600", "grsai-video": "bg-orange-600", midjourney: "bg-fuchsia-600", @@ -13667,6 +13672,13 @@ export function AdminSettingsClient() { Token 认证。保存后会自动添加所有同步图片编辑预设模型。

)} + {providerForm.type === "stepfun-image" && ( +

+ 阶跃星辰 Step Image Edit 使用 Bearer Token 认证，默认地址 + https://api.stepfun.com。模型 ID 可填写 step-image-edit-2， + 支持文生图和图片编辑。 +

+ )} {providerForm.type === "bfl-image" && isBltcyBflBaseUrl(providerForm.baseUrl) && (

diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index 1357f67ed..a6c2cb540 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -474,6 +474,7 @@ function normalizeImageLoadingProviderType(providerType?: string | null): string normalized === "ruxa-image" || normalized === "fal-image" || normalized === "openai-image" || + normalized === "stepfun-image" || normalized === "midjourney" || normalized === "runway" || normalized === "krea" || @@ -1302,6 +1303,7 @@ const IMAGE_FAILOVER_PROVIDER_ALLOWLIST = new Set([ "ruxa-image", "fal-image", "openai-image", + "stepfun-image", "runway", "krea", "volcano-vision", @@ -1372,6 +1374,7 @@ function getImageFailoverConvertProvider(providerType?: string | null) { case "ruxa-image": case "fal-image": case "openai-image": + case "stepfun-image": case "midjourney": case "runway": case "krea": @@ -2274,6 +2277,7 @@ async function convertMessagesWithImages( normalizedProviderType === "ruxa-image" || normalizedProviderType === "fal-image" || normalizedProviderType === "openai-image" || + normalizedProviderType === "stepfun-image" || normalizedProviderType === "midjourney" || normalizedProviderType === "runway" || normalizedProviderType === "krea" @@ -10240,6 +10244,7 @@ export async function POST(request: Request) { "ruxa-image", "fal-image", "openai-image", + "stepfun-image", "midjourney", "runway", "krea", @@ -10465,6 +10470,8 @@ export async function POST(request: Request) { ? "fal-image" : (languageModel as any).provider === "openai-image" ? "openai-image" + : (languageModel as any).provider === "stepfun-image" + ? "stepfun-image" : (languageModel as any).provider === "midjourney" ? "midjourney" : (languageModel as any).provider === "runway" @@ -10565,10 +10572,11 @@ export async function POST(request: Request) { "ruxa-image": 3, "fal-image": 4, "openai-image": 5, - runway: 6, - krea: 7, - gemini: 7, - "huanke-gemini": 8, + "stepfun-image": 6, + runway: 7, + krea: 8, + gemini: 8, + "huanke-gemini": 9, }; const crossProviderCandidates = getImageCrossProviderCandidates(allModels, { currentModelId: finalModelId, @@ -10608,6 +10616,8 @@ export async function POST(request: Request) { ? "fal-image" : candidate.provider.type === "openai-image" ? "openai-image" + : candidate.provider.type === "stepfun-image" + ? "stepfun-image" : candidate.provider.type === "midjourney" ? "midjourney" : candidate.provider.type === "runway" @@ -11001,11 +11011,12 @@ export async function POST(request: Request) { "ruxa-image": 2, "fal-image": 3, "openai-image": 4, - runway: 5, - krea: 6, - gemini: 6, - "huanke-gemini": 7, - "volcano-vision": 8, + "stepfun-image": 5, + runway: 6, + krea: 7, + gemini: 7, + "huanke-gemini": 8, + "volcano-vision": 9, }; const crossProviderCandidates = getImageCrossProviderCandidates(allModels, { currentModelId: finalModelId, @@ -11045,6 +11056,8 @@ export async function POST(request: Request) { ? "fal-image" : candidate.provider.type === "openai-image" ? "openai-image" + : candidate.provider.type === "stepfun-image" + ? "stepfun-image" : candidate.provider.type === "midjourney" ? "midjourney" : candidate.provider.type === "runway" @@ -11903,10 +11916,11 @@ export async function POST(request: Request) { "ruxa-image": 3, "fal-image": 4, "openai-image": 5, - runway: 6, - krea: 7, - gemini: 7, - "huanke-gemini": 8, + "stepfun-image": 6, + runway: 7, + krea: 8, + gemini: 8, + "huanke-gemini": 9, }; const crossProviderCandidates = getImageCrossProviderCandidates(allModels, { currentModelId: finalModelId, @@ -11951,6 +11965,8 @@ export async function POST(request: Request) { ? "fal-image" : candidate.provider.type === "openai-image" ? "openai-image" + : candidate.provider.type === "stepfun-image" + ? "stepfun-image" : candidate.provider.type === "midjourney" ? "midjourney" : candidate.provider.type === "runway" diff --git a/app/api/admin/ai-providers/route.ts b/app/api/admin/ai-providers/route.ts index ef825e5af..59d94ddbb 100644 --- a/app/api/admin/ai-providers/route.ts +++ b/app/api/admin/ai-providers/route.ts @@ -30,6 +30,7 @@ const providerTypes = [ "ruxa-image", "fal-image", "openai-image", + "stepfun-image", "midjourney", "fal-video", "zhipu", @@ -69,6 +70,7 @@ const DOUBAO_VIDEO_BASE_URL = "https://ark.cn-beijing.volces.com/api/v3"; const RUXA_VIDEO_BASE_URL = "https://api.ruxa.ai"; const RUXA_IMAGE_BASE_URL = "https://api.ruxa.ai"; const OPENAI_IMAGE_BASE_URL = "https://api.openai.com/v1"; +const STEPFUN_IMAGE_BASE_URL = "https://api.stepfun.com"; const BFL_IMAGE_BASE_URL = "https://api.bfl.ai"; const MIDJOURNEY_BASE_URL = "https://your-midjourney-api.example.com"; const TRIPO_3D_BASE_URL = "https://api.tripo3d.ai/v2/openapi"; @@ -230,6 +232,8 @@ export async function POST(request: Request) { ? data.baseUrl || RUXA_IMAGE_BASE_URL : data.type === "openai-image" ? data.baseUrl || OPENAI_IMAGE_BASE_URL + : data.type === "stepfun-image" + ? data.baseUrl || STEPFUN_IMAGE_BASE_URL : data.type === "midjourney" ? data.baseUrl || MIDJOURNEY_BASE_URL : data.type === "tripo-3d" diff --git a/app/api/canvas/text-edit/route.ts b/app/api/canvas/text-edit/route.ts index 489538eaa..f72b99d34 100644 --- a/app/api/canvas/text-edit/route.ts +++ b/app/api/canvas/text-edit/route.ts @@ -9,6 +9,7 @@ import { createBflImage } from "@/lib/ai/providers/bfl-image"; import { createRuxaImage } from "@/lib/ai/providers/ruxa-image"; import { createFalImage } from "@/lib/ai/providers/fal-image"; import { createOpenAIImage } from "@/lib/ai/providers/openai-image"; +import { createStepfunImage } from "@/lib/ai/providers/stepfun-image"; import { createMidjourney } from "@/lib/ai/providers/midjourney"; import { createRunway } from "@/lib/ai/providers/runway"; import { createKrea } from "@/lib/ai/providers/krea"; @@ -39,6 +40,7 @@ const SUPPORTED_TEXT_EDIT_PROVIDER_TYPES = new Set([ "ruxa-image", "fal-image", "openai-image", + "stepfun-image", "midjourney", "runway", "krea", @@ -98,6 +100,8 @@ function createTextEditProvider( return createFalImage({ baseURL, apiKey }); case "openai-image": return createOpenAIImage({ baseURL, apiKey }); + case "stepfun-image": + return createStepfunImage({ baseURL, apiKey }); case "midjourney": return createMidjourney({ baseURL, apiKey }); case "runway": diff --git a/components/chat.tsx b/components/chat.tsx index 0776a0cbb..559e64546 100644 --- a/components/chat.tsx +++ b/components/chat.tsx @@ -56,7 +56,12 @@ function normalizeImageParamsForProvider( const normalized = { ...params }; const currentCount = normalized.imageCount || 1; - if (providerType === "ruxa-image" || providerType === "openai-image" || providerType === "midjourney") { + if ( + providerType === "ruxa-image" || + providerType === "openai-image" || + providerType === "stepfun-image" || + providerType === "midjourney" + ) { normalized.imageCount = 1; return normalized; } diff --git a/components/image-generation-loading.tsx b/components/image-generation-loading.tsx index b434e8087..c873c6e3d 100644 --- a/components/image-generation-loading.tsx +++ b/components/image-generation-loading.tsx @@ -21,6 +21,7 @@ const PROVIDER_MODEL_LABEL: Record = { "ruxa-image": "RUXA 图片", "fal-image": "FAL 图片", "openai-image": "OpenAI 图片", + "stepfun-image": "StepFun 图片", midjourney: "Midjourney", "volcano-vision": "即梦 4.5", gemini: "Gemini 图片", diff --git a/components/image-params-bar.tsx b/components/image-params-bar.tsx index a4be2224c..479c0d0e5 100644 --- a/components/image-params-bar.tsx +++ b/components/image-params-bar.tsx @@ -113,7 +113,12 @@ export function ImageParamsBar({ const getImageCountOptions = () => { // RUXA 当前按单图任务处理（优先级最高） - if (providerType === "ruxa-image" || providerType === "openai-image" || providerType === "midjourney") { + if ( + providerType === "ruxa-image" || + providerType === "openai-image" || + providerType === "stepfun-image" || + providerType === "midjourney" + ) { return [1]; } // 优先使用模型配置中的 maxImages diff --git a/components/messages.tsx b/components/messages.tsx index dc2b02669..f9be29342 100644 --- a/components/messages.tsx +++ b/components/messages.tsx @@ -697,6 +697,7 @@ function PureMessages({ imageCount={imageCount} maxImages={ currentModelProviderType === "openai-image" || + currentModelProviderType === "stepfun-image" || currentModelProviderType === "midjourney" ? 1 : 6 @@ -708,6 +709,8 @@ function PureMessages({ ? "fal-image" : currentModelProviderType === "openai-image" ? "openai-image" + : currentModelProviderType === "stepfun-image" + ? "stepfun-image" : currentModelProviderType === "midjourney" ? "midjourney" : "alibaba-image" diff --git a/lib/ai/dynamic-provider.ts b/lib/ai/dynamic-provider.ts index 8cc183333..828c8e3c1 100644 --- a/lib/ai/dynamic-provider.ts +++ b/lib/ai/dynamic-provider.ts @@ -22,6 +22,7 @@ import { createBflImage } from "./providers/bfl-image"; import { createRuxaImage } from "./providers/ruxa-image"; import { createFalImage } from "./providers/fal-image"; import { createOpenAIImage } from "./providers/openai-image"; +import { createStepfunImage } from "./providers/stepfun-image"; import { createMidjourney } from "./providers/midjourney"; import { createFalVideo } from "./providers/fal-video"; import { createRuxaVideo } from "./providers/ruxa-video"; @@ -260,6 +261,15 @@ function getOrCreateProvider(provider: Pick = { // 图片编辑：优先 alibaba-image（灵魂4.5，支持提示词扩展） image_edit: { - providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "runway", "krea", "gemini", "huanke-gemini"], + providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "stepfun-image", "runway", "krea", "gemini", "huanke-gemini"], requireVision: true, requireImageGen: true, description: "图片编辑", @@ -170,7 +171,7 @@ const INTENT_MODEL_MAP: Record< // 图片生成：优先灵魂4.5 image_gen: { - providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "runway", "krea", "gemini", "huanke-gemini", "volcano-vision"], + providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "stepfun-image", "runway", "krea", "gemini", "huanke-gemini", "volcano-vision"], requireImageGen: true, description: "图片生成", }, @@ -198,25 +199,25 @@ const SUB_INTENT_OVERRIDES: Record< > = { // 抠图：alibaba-image 效果最好 remove_background: { - providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "runway", "krea", "gemini"], + providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "stepfun-image", "runway", "krea", "gemini"], reason: "抠图首选灵魂4.5", }, // 换背景：alibaba-image 支持场景生成 replace_background: { - providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "runway", "krea", "gemini"], + providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "stepfun-image", "runway", "krea", "gemini"], reason: "换背景首选灵魂4.5", }, // 图片增强 enhance: { - providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "runway", "krea", "gemini", "volcano-vision"], + providerTypes: ["alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "stepfun-image", "runway", "krea", "gemini", "volcano-vision"], reason: "增强首选灵魂4.5", }, // 风格迁移：gemini 可能更有创意 style_transfer: { - providerTypes: ["gemini", "alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "runway", "krea"], + providerTypes: ["gemini", "alibaba-image", "bfl-image", "ruxa-image", "fal-image", "openai-image", "stepfun-image", "runway", "krea"], reason: "风格迁移优选Gemini", }, }; diff --git a/lib/ai/provider-defaults.ts b/lib/ai/provider-defaults.ts index f09aab83c..82c5b9fd3 100644 --- a/lib/ai/provider-defaults.ts +++ b/lib/ai/provider-defaults.ts @@ -16,6 +16,7 @@ const PROVIDER_DEFAULT_BASE_URL: Partial> = { "ruxa-image": "https://api.ruxa.ai", "fal-image": "https://fal.run", "openai-image": "https://api.openai.com/v1", + "stepfun-image": "https://api.stepfun.com", "bfl-image": "https://api.bfl.ai", midjourney: "https://your-midjourney-api.example.com", "fal-video": "https://fal.run", diff --git a/lib/ai/providers/stepfun-image.ts b/lib/ai/providers/stepfun-image.ts new file mode 100644 index 000000000..0c94cdc51 --- /dev/null +++ b/lib/ai/providers/stepfun-image.ts @@ -0,0 +1,436 @@ +import "server-only"; + +import sharp from "sharp"; +import { uploadFile } from "@/lib/storage/service"; + +interface StepfunImageConfig { + apiKey: string; + baseURL: string; +} + +export interface StepfunImageParams { + aspectRatio?: string; + imageSize?: string; + imageCount?: number; + size?: string; + seed?: number; + negativePrompt?: string; +} + +interface UploadedImage { + thumbnail: string; + original: string; +} + +const DEFAULT_BASE_URL = "https://api.stepfun.com"; +const DEFAULT_TIMEOUT_MS = 300000; +const DEFAULT_CFG_SCALE = 1.0; +const DEFAULT_STEPS = 8; +const DEFAULT_TEXT_MODE = true; +const MAX_PROMPT_LENGTH = 512; + +function normalizeBaseURL(baseURL: string) { + const normalized = String(baseURL || "").trim().replace(/\/+$/, ""); + const base = normalized || DEFAULT_BASE_URL; + if (/\/v1$/i.test(base)) return base; + return `${base}/v1`; +} + +function joinEndpoint(baseURL: string, path: string) { + const normalizedPath = path.startsWith("/") ? path : `/${path}`; + return `${normalizeBaseURL(baseURL)}${normalizedPath}`; +} + +function normalizePrompt(prompt: string) { + const normalized = String(prompt || "").trim(); + if (normalized.length <= MAX_PROMPT_LENGTH) return normalized; + console.warn("[StepFun Image] Prompt exceeds 512 characters, truncating for step-image-edit-2"); + return normalized.slice(0, MAX_PROMPT_LENGTH); +} + +function parseAspectRatioValue(ratio?: string | null) { + const normalized = String(ratio || "").trim(); + const [width, height] = normalized.split(":").map((item) => Number(item)); + if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) { + return 1; + } + return width / height; +} + +function resolveStepfunSize(aspectRatio?: string, explicitSize?: string) { + const normalizedExplicit = String(explicitSize || "").trim(); + if (/^\d{3,4}x\d{3,4}$/i.test(normalizedExplicit)) { + return normalizedExplicit; + } + + const ratio = parseAspectRatioValue(aspectRatio); + const candidates = [ + { size: "1024x1024", ratio: 1 }, + { size: "768x1360", ratio: 1360 / 768 }, + { size: "896x1184", ratio: 1184 / 896 }, + { size: "1360x768", ratio: 768 / 1360 }, + { size: "1184x896", ratio: 896 / 1184 }, + ]; + + let best = candidates[0]; + let minDiff = Number.POSITIVE_INFINITY; + for (const candidate of candidates) { + const diff = Math.abs(candidate.ratio - ratio); + if (diff < minDiff) { + best = candidate; + minDiff = diff; + } + } + return best.size; +} + +function detectImageExtension(imageUrl: string, contentType?: string | null) { + const type = String(contentType || "").toLowerCase(); + if (type.includes("jpeg") || type.includes("jpg")) return "jpg"; + if (type.includes("webp")) return "webp"; + if (type.includes("png")) return "png"; + + const cleanUrl = imageUrl.split("?")[0]?.toLowerCase() || ""; + if (cleanUrl.endsWith(".jpg") || cleanUrl.endsWith(".jpeg")) return "jpg"; + if (cleanUrl.endsWith(".webp")) return "webp"; + if (cleanUrl.endsWith(".png")) return "png"; + return "png"; +} + +async function createThumbnail(buffer: Buffer) { + try { + return await sharp(buffer) + .resize(400, 400, { fit: "inside", withoutEnlargement: true }) + .jpeg({ quality: 80 }) + .toBuffer(); + } catch (error) { + console.error("[StepFun Image] Failed to create thumbnail:", error); + return buffer; + } +} + +function generateFilename(prefix: string, extension: string) { + const timestamp = Date.now(); + const random = Math.random().toString(36).slice(2, 8); + return `${prefix}-${timestamp}-${random}.${extension}`; +} + +async function uploadGeneratedBuffer( + buffer: Buffer, + extension: string, + mimeType: string +): Promise { + const thumbnailBuffer = await createThumbnail(buffer); + + const originalFilename = generateFilename("stepfun-image-orig", extension); + const originalArrayBuffer = buffer.buffer.slice( + buffer.byteOffset, + buffer.byteOffset + buffer.byteLength + ) as ArrayBuffer; + const originalUploaded = await uploadFile(originalFilename, originalArrayBuffer); + + const thumbFilename = generateFilename("stepfun-image-thumb", "jpg"); + const thumbArrayBuffer = thumbnailBuffer.buffer.slice( + thumbnailBuffer.byteOffset, + thumbnailBuffer.byteOffset + thumbnailBuffer.byteLength + ) as ArrayBuffer; + const thumbUploaded = await uploadFile(thumbFilename, thumbArrayBuffer); + + console.log("[StepFun Image] Uploaded generated image:", { + original: originalUploaded.url, + thumbnail: thumbUploaded.url, + mimeType, + }); + + return { + thumbnail: thumbUploaded.url, + original: originalUploaded.url, + }; +} + +async function downloadAndUploadImage(imageUrl: string): Promise { + const response = await fetch(imageUrl, { cache: "no-store" }); + if (!response.ok) { + throw new Error(`Failed to download StepFun generated image: ${response.status}`); + } + const buffer = Buffer.from(await response.arrayBuffer()); + const extension = detectImageExtension(imageUrl, response.headers.get("content-type")); + const mimeType = response.headers.get("content-type") || `image/${extension === "jpg" ? "jpeg" : extension}`; + return uploadGeneratedBuffer(buffer, extension, mimeType); +} + +async function uploadBase64ToS3(base64Data: string, mimeType: string) { + const buffer = Buffer.from(base64Data, "base64"); + const extension = detectImageExtension("", mimeType); + return uploadGeneratedBuffer( + buffer, + extension, + mimeType || `image/${extension === "jpg" ? "jpeg" : extension}` + ); +} + +function extractPromptAndImages(messages: Array<{ role: string; content: any }>) { + const lastUserMessage = messages.filter((message) => message.role === "user").pop(); + if (!lastUserMessage) { + throw new Error("No user message found"); + } + + let prompt = ""; + const imageUrls: string[] = []; + + const pushImageUrl = (value?: string) => { + if (!value) return; + const normalized = String(value).trim(); + if (!normalized || imageUrls.includes(normalized)) return; + imageUrls.push(normalized); + }; + + if (typeof lastUserMessage.content === "string") { + prompt = lastUserMessage.content.trim(); + } else if (Array.isArray(lastUserMessage.content)) { + for (const part of lastUserMessage.content) { + if (part?.type === "text" && typeof part.text === "string") { + prompt = `${prompt} ${part.text}`.trim(); + } else if (part?.type === "image_url" && part.image_url?.url) { + pushImageUrl(part.image_url.url); + } else if (part?.type === "file") { + pushImageUrl(part.url || part.data); + } else if (part?.type === "image") { + pushImageUrl(part.image || part.url); + } + } + } + + if (!prompt) { + throw new Error("No prompt found in user message"); + } + + return { prompt: normalizePrompt(prompt), imageUrls }; +} + +async function loadInputImage(source: string) { + const normalized = String(source || "").trim(); + if (!normalized) { + throw new Error("输入图片地址为空"); + } + + if (normalized.startsWith("data:")) { + const match = normalized.match(/^data:([A-Za-z0-9.+/-]+);base64,(.+)$/); + if (!match) { + throw new Error("不支持的 Base64 图片格式"); + } + const mimeType = match[1] || "image/png"; + const buffer = Buffer.from(match[2], "base64"); + const extension = detectImageExtension("", mimeType); + return { + buffer, + mimeType, + filename: generateFilename("stepfun-image-edit-input", extension), + }; + } + + const response = await fetch(normalized, { cache: "no-store" }); + if (!response.ok) { + throw new Error(`Failed to fetch StepFun input image: ${response.status}`); + } + const buffer = Buffer.from(await response.arrayBuffer()); + const mimeType = response.headers.get("content-type") || "image/png"; + const extension = detectImageExtension(normalized, mimeType); + return { + buffer, + mimeType, + filename: generateFilename("stepfun-image-edit-input", extension), + }; +} + +function normalizeSeed(seed?: number) { + if (typeof seed !== "number" || !Number.isFinite(seed)) return undefined; + return Math.max(0, Math.floor(seed)); +} + +function extractErrorMessage(payload: any) { + if (!payload) return "StepFun Images 请求失败"; + if (typeof payload === "string") return payload; + if (payload.error?.message) return String(payload.error.message); + if (payload.message) return String(payload.message); + try { + return JSON.stringify(payload); + } catch { + return "StepFun Images 请求失败"; + } +} + +async function parseJsonResponse(response: Response) { + const rawText = await response.text(); + try { + return JSON.parse(rawText); + } catch { + return rawText; + } +} + +async function requestStepfunImage(params: { + baseURL: string; + apiKey: string; + model: string; + prompt: string; + size?: string; + imageUrls: string[]; + seed?: number; +}) { + const { baseURL, apiKey, model, prompt, size, imageUrls, seed } = params; + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), DEFAULT_TIMEOUT_MS); + + try { + if (imageUrls.length > 0) { + const endpoint = joinEndpoint(baseURL, "/images/edits"); + const formData = new FormData(); + formData.append("model", model); + formData.append("prompt", prompt); + formData.append("response_format", "b64_json"); + formData.append("cfg_scale", String(DEFAULT_CFG_SCALE)); + formData.append("steps", String(DEFAULT_STEPS)); + formData.append("text_mode", String(DEFAULT_TEXT_MODE)); + const normalizedSeed = normalizeSeed(seed); + if (normalizedSeed !== undefined) formData.append("seed", String(normalizedSeed)); + + for (const source of imageUrls.slice(0, 1)) { + const image = await loadInputImage(source); + formData.append("image", new Blob([image.buffer], { type: image.mimeType }), image.filename); + } + + const response = await fetch(endpoint, { + method: "POST", + headers: { + Authorization: `Bearer ${apiKey}`, + }, + body: formData, + signal: controller.signal, + }); + const payload = await parseJsonResponse(response); + if (!response.ok) { + throw new Error(`${extractErrorMessage(payload)} (HTTP ${response.status})`); + } + return payload; + } + + const endpoint = joinEndpoint(baseURL, "/images/generations"); + const body: Record = { + model, + prompt, + response_format: "b64_json", + cfg_scale: DEFAULT_CFG_SCALE, + steps: DEFAULT_STEPS, + text_mode: DEFAULT_TEXT_MODE, + ...(size ? { size } : {}), + }; + const normalizedSeed = normalizeSeed(seed); + if (normalizedSeed !== undefined) body.seed = normalizedSeed; + + const response = await fetch(endpoint, { + method: "POST", + headers: { + Authorization: `Bearer ${apiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + signal: controller.signal, + }); + const payload = await parseJsonResponse(response); + if (!response.ok) { + throw new Error(`${extractErrorMessage(payload)} (HTTP ${response.status})`); + } + return payload; + } finally { + clearTimeout(timeoutId); + } +} + +async function extractUploadedImages(payload: any): Promise { + const data = Array.isArray(payload?.data) ? payload.data : []; + const uploadedImages: UploadedImage[] = []; + + for (const item of data) { + if (item?.url && typeof item.url === "string") { + uploadedImages.push(await downloadAndUploadImage(item.url)); + continue; + } + + if (item?.b64_json && typeof item.b64_json === "string") { + uploadedImages.push(await uploadBase64ToS3(item.b64_json, "image/png")); + } + } + + return uploadedImages; +} + +function formatMarkdown(images: UploadedImage[], imageSize: string) { + return images + .map((image) => `![thumb|||${image.original}|||${imageSize}](${image.thumbnail})`) + .join("\n\n"); +} + +export function createStepfunImage({ apiKey, baseURL }: StepfunImageConfig) { + return { + provider: "stepfun-image", + chat: (modelId: string) => ({ + provider: "stepfun-image", + doGenerate: async (options: { + messages: Array<{ role: string; content: any }>; + imageParams?: StepfunImageParams; + overrideModelId?: string; + }) => { + const { prompt, imageUrls } = extractPromptAndImages(options.messages); + const activeModelId = options.overrideModelId || modelId; + const size = + imageUrls.length > 0 + ? undefined + : resolveStepfunSize(options.imageParams?.aspectRatio, options.imageParams?.size); + + console.log("[StepFun Image] Submitting request", { + model: activeModelId, + size, + imageCount: imageUrls.length, + baseURL: normalizeBaseURL(baseURL), + }); + + const payload = await requestStepfunImage({ + baseURL, + apiKey, + model: activeModelId, + prompt, + size, + imageUrls, + seed: options.imageParams?.seed, + }); + + const uploadedImages = await extractUploadedImages(payload); + if (uploadedImages.length === 0) { + throw new Error("StepFun Images 未返回可用图片"); + } + + const text = formatMarkdown(uploadedImages, options.imageParams?.imageSize || "1K"); + return { + text, + finishReason: "stop" as const, + usage: { + promptTokens: 0, + completionTokens: text.length, + }, + }; + }, + doStream: async function* (options: { + messages: Array<{ role: string; content: any }>; + imageParams?: StepfunImageParams; + overrideModelId?: string; + }) { + const result = await this.doGenerate(options); + yield { + type: "text-delta", + textDelta: result.text, + }; + }, + }), + }; +} diff --git a/lib/ai/runtime/generated-image-persistence.ts b/lib/ai/runtime/generated-image-persistence.ts index b943a84d7..bc1350a93 100644 --- a/lib/ai/runtime/generated-image-persistence.ts +++ b/lib/ai/runtime/generated-image-persistence.ts @@ -53,6 +53,8 @@ export function parseImageAltMetadata(altText: string, thumbnailUrl: string): { originalUrl = thumbnailUrl.replace("alibaba-image-thumb-", "alibaba-image-orig-"); } else if (thumbnailUrl.includes("ruxa-image-thumb-")) { originalUrl = thumbnailUrl.replace("ruxa-image-thumb-", "ruxa-image-orig-"); + } else if (thumbnailUrl.includes("stepfun-image-thumb-")) { + originalUrl = thumbnailUrl.replace("stepfun-image-thumb-", "stepfun-image-orig-"); } else if (thumbnailUrl.includes("fal-image-thumb-")) { originalUrl = thumbnailUrl.replace("fal-image-thumb-", "fal-image-orig-"); } diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index d79d1c7c2..339a70267 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -7,6 +7,7 @@ import { createBflImage } from "@/lib/ai/providers/bfl-image"; import { createRuxaImage } from "@/lib/ai/providers/ruxa-image"; import { createFalImage } from "@/lib/ai/providers/fal-image"; import { createOpenAIImage } from "@/lib/ai/providers/openai-image"; +import { createStepfunImage } from "@/lib/ai/providers/stepfun-image"; import { createMidjourney } from "@/lib/ai/providers/midjourney"; import { createRunway } from "@/lib/ai/providers/runway"; import { createKrea } from "@/lib/ai/providers/krea"; @@ -47,6 +48,7 @@ const SUPPORTED_CANVAS_PROVIDER_TYPES = new Set([ "ruxa-image", "fal-image", "openai-image", + "stepfun-image", "midjourney", "runway", "krea", @@ -103,6 +105,7 @@ function getProviderBatchLimit(providerType: string) { case "ruxa-image": case "bfl-image": case "openai-image": + case "stepfun-image": case "midjourney": case "runway": return 1; @@ -191,7 +194,7 @@ export async function generateCanvasImage( } if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(fallbackProvider.type)) { throw new Error( - "当前图片模型类型暂不支持画布生成，请在后台选择 Gemini/Grsai GPT Image/阿里/BFL/RUXA/FAL/OpenAI/Runway/Krea 图片模型。" + "当前图片模型类型暂不支持画布生成，请在后台选择 Gemini/Grsai GPT Image/阿里/BFL/RUXA/FAL/OpenAI/StepFun/Runway/Krea 图片模型。" ); } if (fallbackModel.model.id !== selectedModel.model.id) { @@ -374,6 +377,19 @@ export async function generateCanvasImage( }); return result.text || ""; }; + } else if (provider.type === "stepfun-image") { + const stepfunProvider = createStepfunImage({ + apiKey: provider.apiKey, + baseURL: provider.baseUrl, + }); + const modelInstance = stepfunProvider.chat(selectedModel.model.modelId); + executeGenerateBatch = async (_batchImageCount) => { + const result = await modelInstance.doGenerate({ + messages, + imageParams, + }); + return result.text || ""; + }; } else if (provider.type === "midjourney") { const midjourneyProvider = createMidjourney({ apiKey: provider.apiKey, @@ -421,7 +437,7 @@ export async function generateCanvasImage( }; } else { throw new Error( - `不支持的图片生成模型类型: ${provider.type || "unknown"}，请改用 Gemini/Grsai GPT Image/阿里/BFL/RUXA/FAL/OpenAI/Midjourney/Runway/Krea 图片模型` + `不支持的图片生成模型类型: ${provider.type || "unknown"}，请改用 Gemini/Grsai GPT Image/阿里/BFL/RUXA/FAL/OpenAI/StepFun/Midjourney/Runway/Krea 图片模型` ); } diff --git a/lib/db/migrate.ts b/lib/db/migrate.ts index 3aa4d011b..deb3307a2 100644 --- a/lib/db/migrate.ts +++ b/lib/db/migrate.ts @@ -32,6 +32,7 @@ const providerBaseUrlDefaults = [ { type: "ruxa-image", baseUrl: "https://api.ruxa.ai" }, { type: "fal-image", baseUrl: "https://fal.run" }, { type: "openai-image", baseUrl: "https://api.openai.com/v1" }, + { type: "stepfun-image", baseUrl: "https://api.stepfun.com" }, { type: "midjourney", baseUrl: "https://your-midjourney-api.example.com" }, { type: "fal-video", baseUrl: "https://fal.run" }, { type: "tripo-3d", baseUrl: "https://api.tripo3d.ai/v2/openapi" }, @@ -98,7 +99,7 @@ const runMigrate = async () => { ADD CONSTRAINT "AiProvider_type_check" CHECK (type IN ( 'alibaba', 'openai', 'custom', 'local', 'gemini', - 'grsai-gpt-image', 'grsai-video', 'huanke-gemini', 'qianwen-video', 'doubao-video', 'ruxa-video', 'volcano-vision', 'volcano-video', 'alibaba-image', 'bfl-image', 'ruxa-image', 'fal-image', 'openai-image', 'midjourney', 'fal-video', 'zhipu', 'tripo-3d', 'worldlabs-3d', 'kittycad-cad', 'runway', 'krea' + 'grsai-gpt-image', 'grsai-video', 'huanke-gemini', 'qianwen-video', 'doubao-video', 'ruxa-video', 'volcano-vision', 'volcano-video', 'alibaba-image', 'bfl-image', 'ruxa-image', 'fal-image', 'openai-image', 'stepfun-image', 'midjourney', 'fal-video', 'zhipu', 'tripo-3d', 'worldlabs-3d', 'kittycad-cad', 'runway', 'krea' )); `); await db.execute(sql` diff --git a/lib/db/migrations/0077_add_stepfun_image_provider_type.sql b/lib/db/migrations/0077_add_stepfun_image_provider_type.sql new file mode 100644 index 000000000..6096d1d0d --- /dev/null +++ b/lib/db/migrations/0077_add_stepfun_image_provider_type.sql @@ -0,0 +1,36 @@ +ALTER TABLE "AiProvider" + DROP CONSTRAINT IF EXISTS "AiProvider_type_check"; + +ALTER TABLE "AiProvider" + ADD CONSTRAINT "AiProvider_type_check" + CHECK ( + "type" IN ( + 'alibaba', + 'openai', + 'custom', + 'local', + 'gemini', + 'grsai-gpt-image', + 'grsai-video', + 'huanke-gemini', + 'qianwen-video', + 'doubao-video', + 'ruxa-video', + 'volcano-vision', + 'volcano-video', + 'alibaba-image', + 'bfl-image', + 'ruxa-image', + 'fal-image', + 'openai-image', + 'stepfun-image', + 'midjourney', + 'fal-video', + 'zhipu', + 'tripo-3d', + 'worldlabs-3d', + 'kittycad-cad', + 'runway', + 'krea' + ) + ); diff --git a/lib/db/migrations/meta/_journal.json b/lib/db/migrations/meta/_journal.json index cc74ac8df..1d58b4f63 100644 --- a/lib/db/migrations/meta/_journal.json +++ b/lib/db/migrations/meta/_journal.json @@ -442,6 +442,13 @@ "when": 1777305600000, "tag": "0076_add_ai_model_text_edit_default", "breakpoints": true + }, + { + "idx": 63, + "version": "7", + "when": 1777392000000, + "tag": "0077_add_stepfun_image_provider_type", + "breakpoints": true } ] } diff --git a/lib/db/queries.ts b/lib/db/queries.ts index bdb4ab1ca..c937de70f 100644 --- a/lib/db/queries.ts +++ b/lib/db/queries.ts @@ -3751,7 +3751,7 @@ export async function createAiProvider({ updatedBy, }: { name: string; - type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "grsai-gpt-image" | "grsai-video" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "bfl-image" | "ruxa-image" | "fal-image" | "openai-image" | "midjourney" | "fal-video" | "zhipu" | "tripo-3d" | "worldlabs-3d" | "kittycad-cad" | "runway" | "krea"; + type: "alibaba" | "openai" | "custom" | "local" | "gemini" | "grsai-gpt-image" | "grsai-video" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "bfl-image" | "ruxa-image" | "fal-image" | "openai-image" | "stepfun-image" | "midjourney" | "fal-video" | "zhipu" | "tripo-3d" | "worldlabs-3d" | "kittycad-cad" | "runway" | "krea"; baseUrl: string; apiKey: string; isEnabled?: boolean; @@ -3789,7 +3789,7 @@ export async function updateAiProvider({ }: { id: string; name?: string; - type?: "alibaba" | "openai" | "custom" | "local" | "gemini" | "grsai-gpt-image" | "grsai-video" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "bfl-image" | "ruxa-image" | "fal-image" | "openai-image" | "midjourney" | "fal-video" | "zhipu" | "tripo-3d" | "worldlabs-3d" | "kittycad-cad" | "runway" | "krea"; + type?: "alibaba" | "openai" | "custom" | "local" | "gemini" | "grsai-gpt-image" | "grsai-video" | "huanke-gemini" | "qianwen-video" | "doubao-video" | "ruxa-video" | "volcano-vision" | "volcano-video" | "alibaba-image" | "bfl-image" | "ruxa-image" | "fal-image" | "openai-image" | "stepfun-image" | "midjourney" | "fal-video" | "zhipu" | "tripo-3d" | "worldlabs-3d" | "kittycad-cad" | "runway" | "krea"; baseUrl?: string; apiKey?: string; isEnabled?: boolean; diff --git a/lib/db/schema.ts b/lib/db/schema.ts index 06ff7fc51..496de7f51 100644 --- a/lib/db/schema.ts +++ b/lib/db/schema.ts @@ -276,7 +276,7 @@ export type ToolConfig = InferSelectModel; export const aiProvider = pgTable("AiProvider", { id: uuid("id").primaryKey().notNull().defaultRandom(), name: varchar("name", { length: 128 }).notNull(), // 提供商名称，如 "阿里云通义千问" - type: varchar("type", { enum: ["alibaba", "openai", "custom", "local", "gemini", "grsai-gpt-image", "grsai-video", "huanke-gemini", "qianwen-video", "doubao-video", "ruxa-video", "volcano-vision", "volcano-video", "alibaba-image", "ruxa-image", "fal-image", "openai-image", "midjourney", "fal-video", "zhipu", "tripo-3d", "worldlabs-3d", "kittycad-cad", "runway", "krea", "bfl-image"] }) + type: varchar("type", { enum: ["alibaba", "openai", "custom", "local", "gemini", "grsai-gpt-image", "grsai-video", "huanke-gemini", "qianwen-video", "doubao-video", "ruxa-video", "volcano-vision", "volcano-video", "alibaba-image", "ruxa-image", "fal-image", "openai-image", "stepfun-image", "midjourney", "fal-video", "zhipu", "tripo-3d", "worldlabs-3d", "kittycad-cad", "runway", "krea", "bfl-image"] }) .notNull() .default("custom"), // 提供商类型 baseUrl: text("baseUrl").notNull(), // API 基础 URL diff --git a/lib/image-url.ts b/lib/image-url.ts index c18e208a9..8de2cd926 100644 --- a/lib/image-url.ts +++ b/lib/image-url.ts @@ -16,6 +16,7 @@ export function isLikelyThumbnailUrl(url: string): boolean { lowerUrl.includes("volcano-thumb-") || lowerUrl.includes("alibaba-image-thumb-") || lowerUrl.includes("ruxa-image-thumb-") || + lowerUrl.includes("stepfun-image-thumb-") || lowerUrl.includes("fal-image-thumb-") ) { return true; @@ -42,6 +43,9 @@ export function inferOriginalImageUrl(url: string): string { if (url.includes("ruxa-image-thumb-")) { return url.replace("ruxa-image-thumb-", "ruxa-image-orig-"); } + if (url.includes("stepfun-image-thumb-")) { + return url.replace("stepfun-image-thumb-", "stepfun-image-orig-"); + } if (url.includes("fal-image-thumb-")) { return url.replace("fal-image-thumb-", "fal-image-orig-"); }