Merge pull request '改善画布扩图四周衔接' (#66) from codex/canvas-outpaint-20261110 into main

lib/ai/extend/fal-outpaint.ts

@@ -1,37 +1,52 @@
 import { createFalClient } from "@fal-ai/client";
 import sharp from "sharp";
 
-export const FIBO_EXPAND_MODEL = "bria/fibo-edit-1.5/expand";
+export const FAL_OUTPAINT_MODEL = "fal-ai/flux-2-pro/outpaint";
-export const FIBO_EXPAND_PROMPT = "Continue the same scene edge to edge, matching the existing lighting, colors, texture, perspective and cut lines. Preserve the original subject and all lettering; continue any caption background that reaches the side edges.";
 
-export function getFiboGeometry(width: number, height: number, xScale: number, yScale: number) {
+export function getOutpaintGeometry(width: number, height: number, xScale: number, yScale: number) {
   const targetWidth = Math.round(width * xScale);
   const targetHeight = Math.round(height * yScale);
-  const ratio = Math.min(1, Math.sqrt(24_000_000 / (targetWidth * targetHeight)));
+  const ratio = Math.min(1, 2048 / Math.max(targetWidth, targetHeight));
   const modelWidth = Math.max(1, Math.floor(targetWidth * ratio));
   const modelHeight = Math.max(1, Math.floor(targetHeight * ratio));
   const sourceWidth = Math.max(1, Math.round(width * modelWidth / targetWidth));
   const sourceHeight = Math.max(1, Math.round(height * modelHeight / targetHeight));
   const left = Math.floor((targetWidth - width) / 2);
   const top = Math.floor((targetHeight - height) / 2);
+  const modelLeft = Math.round(left * modelWidth / targetWidth);
+  const modelTop = Math.round(top * modelHeight / targetHeight);
+  // Align the model canvas to its 16-pixel grid, then remove the extra border.
+  const extraWidth = (16 - modelWidth % 16) % 16;
+  const extraHeight = (16 - modelHeight % 16) % 16;
+  const cropLeft = Math.floor(extraWidth / 2);
+  const cropTop = Math.floor(extraHeight / 2);
   return {
     targetWidth, targetHeight, left, top, modelWidth, modelHeight, sourceWidth, sourceHeight,
-    modelLeft: Math.round(left * modelWidth / targetWidth),
+    paddedWidth: modelWidth + extraWidth,
-    modelTop: Math.round(top * modelHeight / targetHeight),
+    paddedHeight: modelHeight + extraHeight,
+    cropLeft,
+    cropTop,
+    expandLeft: modelLeft + cropLeft,
+    expandRight: modelWidth - sourceWidth - modelLeft + extraWidth - cropLeft,
+    expandTop: modelTop + cropTop,
+    expandBottom: modelHeight - sourceHeight - modelTop + extraHeight - cropTop,
   };
 }
 
-export async function composeFiboExpansion(source: Buffer, expanded: Buffer, geometry: ReturnType<typeof getFiboGeometry>) {
+export async function composeOutpaint(source: Buffer, expanded: Buffer, geometry: ReturnType<typeof getOutpaintGeometry>) {
-  const { targetWidth, targetHeight, left, top, modelWidth, modelHeight } = geometry;
+  const { targetWidth, targetHeight, left, top, modelWidth, modelHeight, paddedWidth, paddedHeight, cropLeft, cropTop } = geometry;
   const modelImage = sharp(expanded);
   const info = await modelImage.metadata();
-  if (info.width !== modelWidth || info.height !== modelHeight) {
+  if (info.width !== paddedWidth || info.height !== paddedHeight) {
     throw new Error("扩图服务返回的图片尺寸不正确");
   }
 
   // Copy the source rectangle, including transparent pixels, without blending or resizing it.
   const { data: original, info: sourceInfo } = await sharp(source).toColourspace("srgb").ensureAlpha().raw().toBuffer({ resolveWithObject: true });
-  const result = await modelImage.resize(targetWidth, targetHeight, { fit: "fill" }).toColourspace("srgb").ensureAlpha().raw().toBuffer();
+  const result = await modelImage
+    .extract({ left: cropLeft, top: cropTop, width: modelWidth, height: modelHeight })
+    .resize(targetWidth, targetHeight, { fit: "fill" })
+    .toColourspace("srgb").ensureAlpha().raw().toBuffer();
   const stride = sourceInfo.width * 4;
   for (let row = 0; row < sourceInfo.height; row++) {
     original.copy(result, ((top + row) * targetWidth + left) * 4, row * stride, (row + 1) * stride);
@@ -39,7 +54,7 @@ export async function composeFiboExpansion(source: Buffer, expanded: Buffer, geo
   return sharp(result, { raw: { width: targetWidth, height: targetHeight, channels: 4 } }).png().toBuffer();
 }
 
-export async function expandWithFibo({ imageUrl, apiKey, xScale, yScale }: {
+export async function expandWithFalOutpaint({ imageUrl, apiKey, xScale, yScale }: {
   imageUrl: string;
   apiKey: string;
   xScale: number;
@@ -49,26 +64,30 @@ export async function expandWithFibo({ imageUrl, apiKey, xScale, yScale }: {
   if (!response.ok) throw new Error("无法读取待扩展的原图");
   const { data: source, info } = await sharp(Buffer.from(await response.arrayBuffer()))
     .rotate().png().toBuffer({ resolveWithObject: true });
-  const geometry = getFiboGeometry(info.width, info.height, xScale, yScale);
+  const geometry = getOutpaintGeometry(info.width, info.height, xScale, yScale);
   if (geometry.targetWidth * geometry.targetHeight > 64_000_000) {
     throw new Error("扩图后的尺寸过大，请降低扩展倍数");
   }
   const fal = createFalClient({ credentials: apiKey });
-  const image = await fal.storage.upload(new Blob([new Uint8Array(source)], { type: "image/png" }));
+  const modelSource = geometry.sourceWidth === info.width && geometry.sourceHeight === info.height
-  const { data } = await fal.subscribe(FIBO_EXPAND_MODEL, {
+    ? source
+    : await sharp(source).resize(geometry.sourceWidth, geometry.sourceHeight, { fit: "fill" }).png().toBuffer();
+  const image = await fal.storage.upload(new Blob([new Uint8Array(modelSource)], { type: "image/png" }));
+  const { data } = await fal.subscribe(FAL_OUTPAINT_MODEL, {
     input: {
       image_url: image,
-      canvas_size: [geometry.modelWidth, geometry.modelHeight],
+      expand_left: geometry.expandLeft,
-      original_image_size: [geometry.sourceWidth, geometry.sourceHeight],
+      expand_right: geometry.expandRight,
-      original_image_location: [geometry.modelLeft, geometry.modelTop],
+      expand_top: geometry.expandTop,
-      retain_original_quality: true,
+      expand_bottom: geometry.expandBottom,
-      prompt: FIBO_EXPAND_PROMPT,
+      mode: "high",
+      output_format: "png",
     },
     timeout: 90_000,
   });
-  const outputUrl = (data as { image?: { url?: string } }).image?.url;
+  const outputUrl = (data as { images?: Array<{ url?: string }> }).images?.[0]?.url;
   if (!outputUrl) throw new Error("扩图服务未返回图片");
   const output = await fetch(outputUrl, { signal: AbortSignal.timeout(25_000) });
   if (!output.ok) throw new Error("无法读取扩图结果");
-  return composeFiboExpansion(source, Buffer.from(await output.arrayBuffer()), geometry);
+  return composeOutpaint(source, Buffer.from(await output.arrayBuffer()), geometry);
 }

lib/ai/extend/service.ts

@@ -6,7 +6,7 @@ import {
   upsertMediaAsset,
 } from "@/lib/db/queries";
 import { type ExtendProvider, resolveAlibabaExtendApiKey, resolveFalExtendApiKey } from "@/lib/ai/extend/config";
-import { expandWithFibo, FIBO_EXPAND_MODEL } from "@/lib/ai/extend/fibo";
+import { expandWithFalOutpaint, FAL_OUTPAINT_MODEL } from "@/lib/ai/extend/fal-outpaint";
 import { executeConfiguredVip98kTask } from "@/lib/ai/vip98k-runtime";
 import { db } from "@/lib/db";
 import { generatedImage } from "@/lib/db/schema";
@@ -536,8 +536,8 @@ export async function executeExtendJob(
   let buffer: Buffer;
   let modelId: string;
   if (config.provider === "fal") {
-    buffer = await expandWithFibo({ imageUrl: inputImageUrl, apiKey: config.apiKey!, xScale, yScale });
+    buffer = await expandWithFalOutpaint({ imageUrl: inputImageUrl, apiKey: config.apiKey!, xScale, yScale });
-    modelId = FIBO_EXPAND_MODEL;
+    modelId = FAL_OUTPAINT_MODEL;
   } else {
     const prepared = config.provider === "vip98k" ? inputImageUrl : await normalizeDashscopeExpansionInput(inputImageUrl);
     const url = await extendImage(prepared, config, xScale, yScale, requestId);

tests/routes/extend-provider-config.test.ts

@@ -1,7 +1,7 @@
 import { expect, test } from "@playwright/test";
 import sharp from "sharp";
 import { resolveAlibabaExtendApiKey, resolveFalExtendApiKey } from "@/lib/ai/extend/config";
-import { composeFiboExpansion, getFiboGeometry } from "@/lib/ai/extend/fibo";
+import { composeOutpaint, getOutpaintGeometry } from "@/lib/ai/extend/fal-outpaint";
 
 test.describe("image extension provider configuration", () => {
   test("uses the most recently updated Alibaba image provider key", () => {
@@ -35,22 +35,24 @@ test.describe("image extension provider configuration", () => {
 });
 
 test("outpainting restores every original pixel in place and keeps the generated border", async () => {
-  const geometry = getFiboGeometry(13, 9, 1.5, 1.5);
+  const geometry = getOutpaintGeometry(13, 9, 1.5, 1.5);
-  const { targetWidth, targetHeight, left, top } = geometry;
+  const { paddedWidth, paddedHeight, left, top } = geometry;
   const pixel = Buffer.from([42, 93, 190, 255, 30, 80, 20, 75]);
   const raw = Buffer.concat(Array.from({ length: 59 }, () => pixel)).subarray(0, 13 * 9 * 4);
   const source = await sharp(raw, { raw: { width: 13, height: 9, channels: 4 } }).png().toBuffer();
-  const expanded = await sharp({ create: { width: targetWidth, height: targetHeight, channels: 3, background: "#d8bc9e" } }).png().toBuffer();
+  const expanded = await sharp({ create: { width: paddedWidth, height: paddedHeight, channels: 3, background: "#d8bc9e" } }).png().toBuffer();
-  const composed = await composeFiboExpansion(source, expanded, geometry);
+  const composed = await composeOutpaint(source, expanded, geometry);
   const area = await sharp(composed).extract({ left, top, width: 13, height: 9 }).raw().toBuffer();
   expect(area.equals(raw)).toBe(true);
   expect([...await sharp(composed).extract({ left: 0, top: 0, width: 1, height: 1 }).raw().toBuffer()]).toEqual([216, 188, 158, 255]);
-  await expect(composeFiboExpansion(source, source, geometry)).rejects.toThrow("尺寸不正确");
+  await expect(composeOutpaint(source, source, geometry)).rejects.toThrow("尺寸不正确");
 });
 
 test("a 4K source keeps its target output size while the model works within its limit", () => {
-  const g = getFiboGeometry(4096, 4096, 1.5, 1.5);
+  const g = getOutpaintGeometry(4096, 4096, 1.5, 1.5);
   expect([g.targetWidth, g.targetHeight, g.left, g.top]).toEqual([6144, 6144, 1024, 1024]);
-  expect(g.modelWidth * g.modelHeight).toBeLessThan(25_000_000);
+  expect([g.paddedWidth, g.paddedHeight]).toEqual([2048, 2048]);
-  expect(g.modelLeft * 2 + g.sourceWidth).toBeCloseTo(g.modelWidth, -1);
+  expect(g.expandLeft + g.sourceWidth + g.expandRight).toBe(g.paddedWidth);
+  expect(g.expandTop + g.sourceHeight + g.expandBottom).toBe(g.paddedHeight);
+  expect(Math.abs(g.expandLeft - g.expandRight)).toBeLessThanOrEqual(1);
 });