diff --git a/app/(admin)/settings/_components/local-ai-sections/image-pipeline-sections.tsx b/app/(admin)/settings/_components/local-ai-sections/image-pipeline-sections.tsx index 61cc133a1..226fd6d9a 100644 --- a/app/(admin)/settings/_components/local-ai-sections/image-pipeline-sections.tsx +++ b/app/(admin)/settings/_components/local-ai-sections/image-pipeline-sections.tsx @@ -2377,7 +2377,9 @@ export function LocalAiImagePipelineSections({ items={[ { label: - extendConfig.provider === "vip98k" + extendConfig.provider === "fal" + ? "Bria 扩图服务" + : extendConfig.provider === "vip98k" ? "VIP98K 服务商" : "阿里云服务商", tone: "enabled", @@ -2391,7 +2393,11 @@ export function LocalAiImagePipelineSections({ } title="图像拓展API配置" > - {extendConfig.provider === "vip98k" ? ( + {extendConfig.provider === "fal" ? ( +
+ 使用 Bria 生成新增区域并保留原图主体和文字。密钥从已启用的 fal 官方服务商读取。 +
+ ) : extendConfig.provider === "vip98k" ? (
使用 VIP98K 的 image-expansion 模型。API Key 从“AI 模型设置”中的 VIP98K 服务商读取，仅在这里维护扩展比例。 diff --git a/app/(admin)/settings/_lib/types.ts b/app/(admin)/settings/_lib/types.ts index 521541cbd..93dfff7b5 100644 --- a/app/(admin)/settings/_lib/types.ts +++ b/app/(admin)/settings/_lib/types.ts @@ -274,7 +274,7 @@ export type EnhanceConfig = { }; export type ExtendConfig = { - provider: "vip98k" | "dashscope"; + provider: "vip98k" | "dashscope" | "fal"; xScale: string; yScale: string; }; diff --git a/app/(admin)/settings/admin-settings-client.tsx b/app/(admin)/settings/admin-settings-client.tsx index 9217e86b2..9b27f130f 100644 --- a/app/(admin)/settings/admin-settings-client.tsx +++ b/app/(admin)/settings/admin-settings-client.tsx @@ -2576,7 +2576,7 @@ export function AdminSettingsClient() { // 图像拓展API配置状态 const [extendConfig, setExtendConfig] = useState({ - provider: "dashscope" as "vip98k" | "dashscope", + provider: "dashscope" as "vip98k" | "dashscope" | "fal", xScale: "1.5", // X轴放大倍数 yScale: "1.5", // Y轴放大倍数 }); @@ -2891,12 +2891,12 @@ export function AdminSettingsClient() { useEffect(() => { if (extendConfigData?.setting?.value) { const config = extendConfigData.setting.value as { - provider?: "vip98k" | "dashscope"; + provider?: "vip98k" | "dashscope" | "fal"; xScale?: number; yScale?: number; }; setExtendConfig({ - provider: config.provider === "vip98k" ? "vip98k" : "dashscope", + provider: config.provider === "vip98k" || config.provider === "fal" ? config.provider : "dashscope", xScale: config.xScale ? String(config.xScale) : "1.5", yScale: config.yScale ? String(config.yScale) : "1.5", }); diff --git a/lib/ai/extend/config.ts b/lib/ai/extend/config.ts index 2baf28205..fed15e03f 100644 --- a/lib/ai/extend/config.ts +++ b/lib/ai/extend/config.ts @@ -3,6 +3,19 @@ export type ExtendProviderCredential = { updatedAt?: Date | string | null; }; +export type ExtendProvider = "vip98k" | "dashscope" | "fal"; + +export function resolveFalExtendApiKey(providers: Array) { + for (const { apiKey, baseUrl } of providers) { + if (!apiKey || !baseUrl) continue; + const url = URL.canParse(baseUrl) ? new URL(baseUrl) : null; + if (url?.protocol === "https:" && ["queue.fal.run", "fal.run", "fal.ai", "api.fal.ai"].includes(url.hostname)) { + return apiKey.trim().replace(/^(?:Bearer|Key)\s+/i, ""); + } + } + return null; +} + function getUpdatedAtTimestamp(value: Date | string | null | undefined): number { if (!value) return 0; const timestamp = value instanceof Date ? value.getTime() : Date.parse(value); diff --git a/lib/ai/extend/fibo.ts b/lib/ai/extend/fibo.ts new file mode 100644 index 000000000..10ca2f660 --- /dev/null +++ b/lib/ai/extend/fibo.ts @@ -0,0 +1,74 @@ +import { createFalClient } from "@fal-ai/client"; +import sharp from "sharp"; + +export const FIBO_EXPAND_MODEL = "bria/fibo-edit-1.5/expand"; +export const FIBO_EXPAND_PROMPT = "Continue the same scene edge to edge, matching the existing lighting, colors, texture, perspective and cut lines. Preserve the original subject and all lettering; continue any caption background that reaches the side edges."; + +export function getFiboGeometry(width: number, height: number, xScale: number, yScale: number) { + const targetWidth = Math.round(width * xScale); + const targetHeight = Math.round(height * yScale); + const ratio = Math.min(1, Math.sqrt(24_000_000 / (targetWidth * targetHeight))); + const modelWidth = Math.max(1, Math.floor(targetWidth * ratio)); + const modelHeight = Math.max(1, Math.floor(targetHeight * ratio)); + const sourceWidth = Math.max(1, Math.round(width * modelWidth / targetWidth)); + const sourceHeight = Math.max(1, Math.round(height * modelHeight / targetHeight)); + const left = Math.floor((targetWidth - width) / 2); + const top = Math.floor((targetHeight - height) / 2); + return { + targetWidth, targetHeight, left, top, modelWidth, modelHeight, sourceWidth, sourceHeight, + modelLeft: Math.round(left * modelWidth / targetWidth), + modelTop: Math.round(top * modelHeight / targetHeight), + }; +} + +export async function composeFiboExpansion(source: Buffer, expanded: Buffer, geometry: ReturnType) { + const { targetWidth, targetHeight, left, top, modelWidth, modelHeight } = geometry; + const modelImage = sharp(expanded); + const info = await modelImage.metadata(); + if (info.width !== modelWidth || info.height !== modelHeight) { + throw new Error("扩图服务返回的图片尺寸不正确"); + } + + // Copy the source rectangle, including transparent pixels, without blending or resizing it. + const { data: original, info: sourceInfo } = await sharp(source).toColourspace("srgb").ensureAlpha().raw().toBuffer({ resolveWithObject: true }); + const result = await modelImage.resize(targetWidth, targetHeight, { fit: "fill" }).toColourspace("srgb").ensureAlpha().raw().toBuffer(); + const stride = sourceInfo.width * 4; + for (let row = 0; row < sourceInfo.height; row++) { + original.copy(result, ((top + row) * targetWidth + left) * 4, row * stride, (row + 1) * stride); + } + return sharp(result, { raw: { width: targetWidth, height: targetHeight, channels: 4 } }).png().toBuffer(); +} + +export async function expandWithFibo({ imageUrl, apiKey, xScale, yScale }: { + imageUrl: string; + apiKey: string; + xScale: number; + yScale: number; +}) { + const response = await fetch(imageUrl, { signal: AbortSignal.timeout(45_000) }); + if (!response.ok) throw new Error("无法读取待扩展的原图"); + const { data: source, info } = await sharp(Buffer.from(await response.arrayBuffer())) + .rotate().png().toBuffer({ resolveWithObject: true }); + const geometry = getFiboGeometry(info.width, info.height, xScale, yScale); + if (geometry.targetWidth * geometry.targetHeight > 64_000_000) { + throw new Error("扩图后的尺寸过大，请降低扩展倍数"); + } + const fal = createFalClient({ credentials: apiKey }); + const image = await fal.storage.upload(new Blob([new Uint8Array(source)], { type: "image/png" })); + const { data } = await fal.subscribe(FIBO_EXPAND_MODEL, { + input: { + image_url: image, + canvas_size: [geometry.modelWidth, geometry.modelHeight], + original_image_size: [geometry.sourceWidth, geometry.sourceHeight], + original_image_location: [geometry.modelLeft, geometry.modelTop], + retain_original_quality: true, + prompt: FIBO_EXPAND_PROMPT, + }, + timeout: 90_000, + }); + const outputUrl = (data as { image?: { url?: string } }).image?.url; + if (!outputUrl) throw new Error("扩图服务未返回图片"); + const output = await fetch(outputUrl, { signal: AbortSignal.timeout(25_000) }); + if (!output.ok) throw new Error("无法读取扩图结果"); + return composeFiboExpansion(source, Buffer.from(await output.arrayBuffer()), geometry); +} diff --git a/lib/ai/extend/service.ts b/lib/ai/extend/service.ts index f0ed39603..2a82456d0 100644 --- a/lib/ai/extend/service.ts +++ b/lib/ai/extend/service.ts @@ -5,14 +5,15 @@ import { getSystemSetting, upsertMediaAsset, } from "@/lib/db/queries"; -import { resolveAlibabaExtendApiKey } from "@/lib/ai/extend/config"; +import { type ExtendProvider, resolveAlibabaExtendApiKey, resolveFalExtendApiKey } from "@/lib/ai/extend/config"; +import { expandWithFibo, FIBO_EXPAND_MODEL } from "@/lib/ai/extend/fibo"; import { executeConfiguredVip98kTask } from "@/lib/ai/vip98k-runtime"; import { db } from "@/lib/db"; import { generatedImage } from "@/lib/db/schema"; import { uploadFile } from "@/lib/storage/service"; type ExtendConfig = { - provider?: "vip98k" | "dashscope"; + provider?: ExtendProvider; apiKey?: string; xScale?: number; yScale?: number; @@ -83,12 +84,12 @@ async function createThumbnail(buffer: Buffer): Promise { } } -async function uploadBufferPairToS3(buffer: Buffer): Promise<{ thumbnail: string; original: string }> { +async function uploadBufferPairToS3(buffer: Buffer, originalExtension = "jpg"): Promise<{ thumbnail: string; original: string }> { try { const timestamp = Date.now(); const randomStr = Math.random().toString(36).slice(2, 8); const thumbnailFilename = `extend-thumb-${timestamp}-${randomStr}.jpg`; - const originalFilename = `extend-orig-${timestamp}-${randomStr}.jpg`; + const originalFilename = `extend-orig-${timestamp}-${randomStr}.${originalExtension}`; const thumbnailBuffer = await createThumbnail(buffer); const thumbnailArrayBuffer = thumbnailBuffer.buffer.slice( thumbnailBuffer.byteOffset, @@ -473,6 +474,15 @@ async function getValidatedExtendConfig(): Promise { if (config.provider === "vip98k") { return config; } + if (config.provider === "fal") { + const [imageProviders, videoProviders] = await Promise.all([ + getAiProvidersByType({ type: "fal-image", enabledOnly: true }), + getAiProvidersByType({ type: "fal-video", enabledOnly: true }), + ]); + const apiKey = resolveFalExtendApiKey([...imageProviders, ...videoProviders]); + if (!apiKey) throw new ExtendServiceError("扩图服务未配置可用的 fal 密钥", 503); + return { ...config, apiKey }; + } const alibabaProviders = await getAiProvidersByType({ type: "alibaba-image", @@ -522,35 +532,29 @@ export async function executeExtendJob( throw new ExtendServiceError("无法获取输入图片URL", 400); } - const preparedInputImageUrl = - config.provider === "vip98k" - ? inputImageUrl - : await normalizeDashscopeExpansionInput(inputImageUrl); const startTime = Date.now(); - const extendedImageUrl = await extendImage( - preparedInputImageUrl, - config, - xScale, - yScale, - requestId, - ); - const imageResponse = await fetch(extendedImageUrl); - if (!imageResponse.ok) { - throw new ExtendServiceError( - "下载拓展图片失败", - 500, - `${imageResponse.status} ${imageResponse.statusText}`, - ); + let buffer: Buffer; + let modelId: string; + if (config.provider === "fal") { + buffer = await expandWithFibo({ imageUrl: inputImageUrl, apiKey: config.apiKey!, xScale, yScale }); + modelId = FIBO_EXPAND_MODEL; + } else { + const prepared = config.provider === "vip98k" ? inputImageUrl : await normalizeDashscopeExpansionInput(inputImageUrl); + const url = await extendImage(prepared, config, xScale, yScale, requestId); + const imageResponse = await fetch(url); + if (!imageResponse.ok) { + throw new ExtendServiceError("下载拓展图片失败", 500, `${imageResponse.status} ${imageResponse.statusText}`); + } + buffer = Buffer.from(await imageResponse.arrayBuffer()); + modelId = config.provider === "vip98k" ? "image-expansion" : "extend-1.0"; } - - const buffer = Buffer.from(await imageResponse.arrayBuffer()); const imageBase64Result = includeBase64 ? buffer.toString("base64") : undefined; - const uploadResult = await uploadBufferPairToS3(buffer); + const uploadResult = await uploadBufferPairToS3(buffer, config.provider === "fal" ? "png" : "jpg"); console.log("[Extend Service] Result uploaded:", { requestId, includeBase64, inputImageUrl: inputImageUrl.slice(0, 120), - extendedImageUrl: extendedImageUrl.slice(0, 120), + modelId, outputBytes: buffer.length, }); let mediaAssetId: string | null = null; @@ -567,7 +571,7 @@ export async function executeExtendJob( thumbnailUrl: uploadResult.thumbnail, originalUrl: uploadResult.original, prompt, - modelId: config.provider === "vip98k" ? "image-expansion" : "extend-1.0", + modelId, imageSize: "2K", }) .returning(); @@ -591,7 +595,7 @@ export async function executeExtendJob( thumbnailUrl: uploadResult.thumbnail, displayName: prompt.slice(0, 80), prompt, - modelId: config.provider === "vip98k" ? "image-expansion" : "extend-1.0", + modelId, sourceMessageId: messageId, sourceTable: "GeneratedImage", sourceRowId: generatedImageId, diff --git a/tests/routes/extend-provider-config.test.ts b/tests/routes/extend-provider-config.test.ts index 24ff7c2b3..ab62ea972 100644 --- a/tests/routes/extend-provider-config.test.ts +++ b/tests/routes/extend-provider-config.test.ts @@ -1,5 +1,7 @@ import { expect, test } from "@playwright/test"; -import { resolveAlibabaExtendApiKey } from "@/lib/ai/extend/config"; +import sharp from "sharp"; +import { resolveAlibabaExtendApiKey, resolveFalExtendApiKey } from "@/lib/ai/extend/config"; +import { composeFiboExpansion, getFiboGeometry } from "@/lib/ai/extend/fibo"; test.describe("image extension provider configuration", () => { test("uses the most recently updated Alibaba image provider key", () => { @@ -23,4 +25,32 @@ test.describe("image extension provider configuration", () => { ).toBe("legacy-key"); expect(resolveAlibabaExtendApiKey([], "")).toBeNull(); }); + + test("uses an official fal provider key, not a key configured for a relay", () => { + expect(resolveFalExtendApiKey([ + { baseUrl: "https://relay.example.test", apiKey: "relay-key" }, + { baseUrl: "https://queue.fal.run", apiKey: "Key official-key" }, + ])).toBe("official-key"); + }); +}); + +test("outpainting restores every original pixel in place and keeps the generated border", async () => { + const geometry = getFiboGeometry(13, 9, 1.5, 1.5); + const { targetWidth, targetHeight, left, top } = geometry; + const pixel = Buffer.from([42, 93, 190, 255, 30, 80, 20, 75]); + const raw = Buffer.concat(Array.from({ length: 59 }, () => pixel)).subarray(0, 13 * 9 * 4); + const source = await sharp(raw, { raw: { width: 13, height: 9, channels: 4 } }).png().toBuffer(); + const expanded = await sharp({ create: { width: targetWidth, height: targetHeight, channels: 3, background: "#d8bc9e" } }).png().toBuffer(); + const composed = await composeFiboExpansion(source, expanded, geometry); + const area = await sharp(composed).extract({ left, top, width: 13, height: 9 }).raw().toBuffer(); + expect(area.equals(raw)).toBe(true); + expect([...await sharp(composed).extract({ left: 0, top: 0, width: 1, height: 1 }).raw().toBuffer()]).toEqual([216, 188, 158, 255]); + await expect(composeFiboExpansion(source, source, geometry)).rejects.toThrow("尺寸不正确"); +}); + +test("a 4K source keeps its target output size while the model works within its limit", () => { + const g = getFiboGeometry(4096, 4096, 1.5, 1.5); + expect([g.targetWidth, g.targetHeight, g.left, g.top]).toEqual([6144, 6144, 1024, 1024]); + expect(g.modelWidth * g.modelHeight).toBeLessThan(25_000_000); + expect(g.modelLeft * 2 + g.sourceWidth).toBeCloseTo(g.modelWidth, -1); });