lunanting/ai-chatbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

扩图保留原图主体和文字 #64

Merged
WuYongJie merged 1 commits from codex/canvas-fibo-outpaint-20261110 into main 2026-11-10 14:18:33 +08:00
Conversation 0 Commits 1 Files Changed 7 +162 -35
7 changed files with 162 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
app/(admin)/settings/_components/local-ai-sections/image-pipeline-sections.tsx
View File

@ -2377,7 +2377,9 @@ export function LocalAiImagePipelineSections({
items={[
{
label:
extendConfig.provider === "vip98k"
extendConfig.provider === "fal"
? "Bria 扩图服务"
: extendConfig.provider === "vip98k"
? "VIP98K 服务商"
: "阿里云服务商",
tone: "enabled",
@ -2391,7 +2393,11 @@ export function LocalAiImagePipelineSections({
}
title="图像拓展API配置"
>
{extendConfig.provider === "vip98k" ? (
{extendConfig.provider === "fal" ? (
<div className="rounded-lg border bg-muted/40 p-4 text-sm">
使用 Bria 生成新增区域并保留原图主体和文字。密钥从已启用的 fal 官方服务商读取。
</div>
) : extendConfig.provider === "vip98k" ? (
<div className="rounded-lg border bg-muted/40 p-4 text-sm">
使用 VIP98K 的 image-expansion 模型。API Key 从“AI 模型设置”中的
VIP98K 服务商读取，仅在这里维护扩展比例。

2
app/(admin)/settings/_lib/types.ts
View File

@ -274,7 +274,7 @@ export type EnhanceConfig = {
};
export type ExtendConfig = {
provider: "vip98k" | "dashscope";
provider: "vip98k" | "dashscope" | "fal";
xScale: string;
yScale: string;
};

6
app/(admin)/settings/admin-settings-client.tsx
View File

@ -2576,7 +2576,7 @@ export function AdminSettingsClient() {
// 图像拓展API配置状态
const [extendConfig, setExtendConfig] = useState({
provider: "dashscope" as "vip98k" | "dashscope",
provider: "dashscope" as "vip98k" | "dashscope" | "fal",
xScale: "1.5", // X轴放大倍数
yScale: "1.5", // Y轴放大倍数
});
@ -2891,12 +2891,12 @@ export function AdminSettingsClient() {
useEffect(() => {
if (extendConfigData?.setting?.value) {
const config = extendConfigData.setting.value as {
provider?: "vip98k" | "dashscope";
provider?: "vip98k" | "dashscope" | "fal";
xScale?: number;
yScale?: number;
};
setExtendConfig({
provider: config.provider === "vip98k" ? "vip98k" : "dashscope",
provider: config.provider === "vip98k" || config.provider === "fal" ? config.provider : "dashscope",
xScale: config.xScale ? String(config.xScale) : "1.5",
yScale: config.yScale ? String(config.yScale) : "1.5",
});

13
lib/ai/extend/config.ts
View File

@ -3,6 +3,19 @@ export type ExtendProviderCredential = {
updatedAt?: Date | string | null;
};
export type ExtendProvider = "vip98k" | "dashscope" | "fal";
export function resolveFalExtendApiKey(providers: Array<ExtendProviderCredential & { baseUrl?: string | null }>) {
for (const { apiKey, baseUrl } of providers) {
if (!apiKey || !baseUrl) continue;
const url = URL.canParse(baseUrl) ? new URL(baseUrl) : null;
if (url?.protocol === "https:" && ["queue.fal.run", "fal.run", "fal.ai", "api.fal.ai"].includes(url.hostname)) {
return apiKey.trim().replace(/^(?:Bearer|Key)\s+/i, "");
}
}
return null;
}
function getUpdatedAtTimestamp(value: Date | string | null | undefined): number {
if (!value) return 0;
const timestamp = value instanceof Date ? value.getTime() : Date.parse(value);

74
lib/ai/extend/fibo.ts Normal file
View File

@ -0,0 +1,74 @@
import { createFalClient } from "@fal-ai/client";
import sharp from "sharp";
export const FIBO_EXPAND_MODEL = "bria/fibo-edit-1.5/expand";
export const FIBO_EXPAND_PROMPT = "Continue the same scene edge to edge, matching the existing lighting, colors, texture, perspective and cut lines. Preserve the original subject and all lettering; continue any caption background that reaches the side edges.";
export function getFiboGeometry(width: number, height: number, xScale: number, yScale: number) {
const targetWidth = Math.round(width * xScale);
const targetHeight = Math.round(height * yScale);
const ratio = Math.min(1, Math.sqrt(24_000_000 / (targetWidth * targetHeight)));
const modelWidth = Math.max(1, Math.floor(targetWidth * ratio));
const modelHeight = Math.max(1, Math.floor(targetHeight * ratio));
const sourceWidth = Math.max(1, Math.round(width * modelWidth / targetWidth));
const sourceHeight = Math.max(1, Math.round(height * modelHeight / targetHeight));
const left = Math.floor((targetWidth - width) / 2);
const top = Math.floor((targetHeight - height) / 2);
return {
targetWidth, targetHeight, left, top, modelWidth, modelHeight, sourceWidth, sourceHeight,
modelLeft: Math.round(left * modelWidth / targetWidth),
modelTop: Math.round(top * modelHeight / targetHeight),
};
}
export async function composeFiboExpansion(source: Buffer, expanded: Buffer, geometry: ReturnType<typeof getFiboGeometry>) {
const { targetWidth, targetHeight, left, top, modelWidth, modelHeight } = geometry;
const modelImage = sharp(expanded);
const info = await modelImage.metadata();
if (info.width !== modelWidth || info.height !== modelHeight) {
throw new Error("扩图服务返回的图片尺寸不正确");
}
// Copy the source rectangle, including transparent pixels, without blending or resizing it.
const { data: original, info: sourceInfo } = await sharp(source).toColourspace("srgb").ensureAlpha().raw().toBuffer({ resolveWithObject: true });
const result = await modelImage.resize(targetWidth, targetHeight, { fit: "fill" }).toColourspace("srgb").ensureAlpha().raw().toBuffer();
const stride = sourceInfo.width * 4;
for (let row = 0; row < sourceInfo.height; row++) {
original.copy(result, ((top + row) * targetWidth + left) * 4, row * stride, (row + 1) * stride);
}
return sharp(result, { raw: { width: targetWidth, height: targetHeight, channels: 4 } }).png().toBuffer();
}
export async function expandWithFibo({ imageUrl, apiKey, xScale, yScale }: {
imageUrl: string;
apiKey: string;
xScale: number;
yScale: number;
}) {
const response = await fetch(imageUrl, { signal: AbortSignal.timeout(45_000) });
if (!response.ok) throw new Error("无法读取待扩展的原图");
const { data: source, info } = await sharp(Buffer.from(await response.arrayBuffer()))
.rotate().png().toBuffer({ resolveWithObject: true });
const geometry = getFiboGeometry(info.width, info.height, xScale, yScale);
if (geometry.targetWidth * geometry.targetHeight > 64_000_000) {
throw new Error("扩图后的尺寸过大，请降低扩展倍数");
}
const fal = createFalClient({ credentials: apiKey });
const image = await fal.storage.upload(new Blob([new Uint8Array(source)], { type: "image/png" }));
const { data } = await fal.subscribe(FIBO_EXPAND_MODEL, {
input: {
image_url: image,
canvas_size: [geometry.modelWidth, geometry.modelHeight],
original_image_size: [geometry.sourceWidth, geometry.sourceHeight],
original_image_location: [geometry.modelLeft, geometry.modelTop],
retain_original_quality: true,
prompt: FIBO_EXPAND_PROMPT,
},
timeout: 90_000,
});
const outputUrl = (data as { image?: { url?: string } }).image?.url;
if (!outputUrl) throw new Error("扩图服务未返回图片");
const output = await fetch(outputUrl, { signal: AbortSignal.timeout(25_000) });
if (!output.ok) throw new Error("无法读取扩图结果");
return composeFiboExpansion(source, Buffer.from(await output.arrayBuffer()), geometry);
}

60
lib/ai/extend/service.ts
View File

@ -5,14 +5,15 @@ import {
getSystemSetting,
upsertMediaAsset,
} from "@/lib/db/queries";
import { resolveAlibabaExtendApiKey } from "@/lib/ai/extend/config";
import { type ExtendProvider, resolveAlibabaExtendApiKey, resolveFalExtendApiKey } from "@/lib/ai/extend/config";
import { expandWithFibo, FIBO_EXPAND_MODEL } from "@/lib/ai/extend/fibo";
import { executeConfiguredVip98kTask } from "@/lib/ai/vip98k-runtime";
import { db } from "@/lib/db";
import { generatedImage } from "@/lib/db/schema";
import { uploadFile } from "@/lib/storage/service";
type ExtendConfig = {
provider?: "vip98k" | "dashscope";
provider?: ExtendProvider;
apiKey?: string;
xScale?: number;
yScale?: number;
@ -83,12 +84,12 @@ async function createThumbnail(buffer: Buffer): Promise<Buffer> {
}
}
async function uploadBufferPairToS3(buffer: Buffer): Promise<{ thumbnail: string; original: string }> {
async function uploadBufferPairToS3(buffer: Buffer, originalExtension = "jpg"): Promise<{ thumbnail: string; original: string }> {
try {
const timestamp = Date.now();
const randomStr = Math.random().toString(36).slice(2, 8);
const thumbnailFilename = `extend-thumb-${timestamp}-${randomStr}.jpg`;
const originalFilename = `extend-orig-${timestamp}-${randomStr}.jpg`;
const originalFilename = `extend-orig-${timestamp}-${randomStr}.${originalExtension}`;
const thumbnailBuffer = await createThumbnail(buffer);
const thumbnailArrayBuffer = thumbnailBuffer.buffer.slice(
thumbnailBuffer.byteOffset,
@ -473,6 +474,15 @@ async function getValidatedExtendConfig(): Promise<ExtendConfig> {
if (config.provider === "vip98k") {
return config;
}
if (config.provider === "fal") {
const [imageProviders, videoProviders] = await Promise.all([
getAiProvidersByType({ type: "fal-image", enabledOnly: true }),
getAiProvidersByType({ type: "fal-video", enabledOnly: true }),
]);
const apiKey = resolveFalExtendApiKey([...imageProviders, ...videoProviders]);
if (!apiKey) throw new ExtendServiceError("扩图服务未配置可用的 fal 密钥", 503);
return { ...config, apiKey };
}
const alibabaProviders = await getAiProvidersByType({
type: "alibaba-image",
@ -522,35 +532,29 @@ export async function executeExtendJob(
throw new ExtendServiceError("无法获取输入图片URL", 400);
}
const preparedInputImageUrl =
config.provider === "vip98k"
? inputImageUrl
: await normalizeDashscopeExpansionInput(inputImageUrl);
const startTime = Date.now();
const extendedImageUrl = await extendImage(
preparedInputImageUrl,
config,
xScale,
yScale,
requestId,
);
const imageResponse = await fetch(extendedImageUrl);
if (!imageResponse.ok) {
throw new ExtendServiceError(
"下载拓展图片失败",
500,
`${imageResponse.status} ${imageResponse.statusText}`,
);
let buffer: Buffer;
let modelId: string;
if (config.provider === "fal") {
buffer = await expandWithFibo({ imageUrl: inputImageUrl, apiKey: config.apiKey!, xScale, yScale });
modelId = FIBO_EXPAND_MODEL;
} else {
const prepared = config.provider === "vip98k" ? inputImageUrl : await normalizeDashscopeExpansionInput(inputImageUrl);
const url = await extendImage(prepared, config, xScale, yScale, requestId);
const imageResponse = await fetch(url);
if (!imageResponse.ok) {
throw new ExtendServiceError("下载拓展图片失败", 500, `${imageResponse.status} ${imageResponse.statusText}`);
}
buffer = Buffer.from(await imageResponse.arrayBuffer());
modelId = config.provider === "vip98k" ? "image-expansion" : "extend-1.0";
}
const buffer = Buffer.from(await imageResponse.arrayBuffer());
const imageBase64Result = includeBase64 ? buffer.toString("base64") : undefined;
const uploadResult = await uploadBufferPairToS3(buffer);
const uploadResult = await uploadBufferPairToS3(buffer, config.provider === "fal" ? "png" : "jpg");
console.log("[Extend Service] Result uploaded:", {
requestId,
includeBase64,
inputImageUrl: inputImageUrl.slice(0, 120),
extendedImageUrl: extendedImageUrl.slice(0, 120),
modelId,
outputBytes: buffer.length,
});
let mediaAssetId: string | null = null;
@ -567,7 +571,7 @@ export async function executeExtendJob(
thumbnailUrl: uploadResult.thumbnail,
originalUrl: uploadResult.original,
prompt,
modelId: config.provider === "vip98k" ? "image-expansion" : "extend-1.0",
modelId,
imageSize: "2K",
})
.returning();
@ -591,7 +595,7 @@ export async function executeExtendJob(
thumbnailUrl: uploadResult.thumbnail,
displayName: prompt.slice(0, 80),
prompt,
modelId: config.provider === "vip98k" ? "image-expansion" : "extend-1.0",
modelId,
sourceMessageId: messageId,
sourceTable: "GeneratedImage",
sourceRowId: generatedImageId,

32
tests/routes/extend-provider-config.test.ts
View File

@ -1,5 +1,7 @@
import { expect, test } from "@playwright/test";
import { resolveAlibabaExtendApiKey } from "@/lib/ai/extend/config";
import sharp from "sharp";
import { resolveAlibabaExtendApiKey, resolveFalExtendApiKey } from "@/lib/ai/extend/config";
import { composeFiboExpansion, getFiboGeometry } from "@/lib/ai/extend/fibo";
test.describe("image extension provider configuration", () => {
test("uses the most recently updated Alibaba image provider key", () => {
@ -23,4 +25,32 @@ test.describe("image extension provider configuration", () => {
).toBe("legacy-key");
expect(resolveAlibabaExtendApiKey([], "")).toBeNull();
});
test("uses an official fal provider key, not a key configured for a relay", () => {
expect(resolveFalExtendApiKey([
{ baseUrl: "https://relay.example.test", apiKey: "relay-key" },
{ baseUrl: "https://queue.fal.run", apiKey: "Key official-key" },
])).toBe("official-key");
});
});
test("outpainting restores every original pixel in place and keeps the generated border", async () => {
const geometry = getFiboGeometry(13, 9, 1.5, 1.5);
const { targetWidth, targetHeight, left, top } = geometry;
const pixel = Buffer.from([42, 93, 190, 255, 30, 80, 20, 75]);
const raw = Buffer.concat(Array.from({ length: 59 }, () => pixel)).subarray(0, 13 * 9 * 4);
const source = await sharp(raw, { raw: { width: 13, height: 9, channels: 4 } }).png().toBuffer();
const expanded = await sharp({ create: { width: targetWidth, height: targetHeight, channels: 3, background: "#d8bc9e" } }).png().toBuffer();
const composed = await composeFiboExpansion(source, expanded, geometry);
const area = await sharp(composed).extract({ left, top, width: 13, height: 9 }).raw().toBuffer();
expect(area.equals(raw)).toBe(true);
expect([...await sharp(composed).extract({ left: 0, top: 0, width: 1, height: 1 }).raw().toBuffer()]).toEqual([216, 188, 158, 255]);
await expect(composeFiboExpansion(source, source, geometry)).rejects.toThrow("尺寸不正确");
});
test("a 4K source keeps its target output size while the model works within its limit", () => {
const g = getFiboGeometry(4096, 4096, 1.5, 1.5);
expect([g.targetWidth, g.targetHeight, g.left, g.top]).toEqual([6144, 6144, 1024, 1024]);
expect(g.modelWidth * g.modelHeight).toBeLessThan(25_000_000);
expect(g.modelLeft * 2 + g.sourceWidth).toBeCloseTo(g.modelWidth, -1);
});