ai-chatbot/lib/ai/vision-inline.ts

import sharp from "sharp";

type InlineVisionOptions = {
  maxImages?: number;
  maxSide?: number;
  timeoutMs?: number;
  retries?: number;
  loggerPrefix?: string;
  jpegQuality?: number;
  trimTransparentPadding?: boolean;
};

type InlineVisionResult = {
  inlineDataUrls: string[];
  skipped: Array<{ url: string; reason: string }>;
};

const DEFAULT_MAX_IMAGES = 6;
const DEFAULT_MAX_SIDE = 900;
const DEFAULT_TIMEOUT_MS = 12000;
const DEFAULT_RETRIES = 2;
const DEFAULT_JPEG_QUALITY = 82;

async function fetchImageWithRetry(url: string, timeoutMs: number, retries: number) {
  let lastError: unknown;

  for (let attempt = 1; attempt <= retries; attempt++) {
    const controller = new AbortController();
    const timeout = setTimeout(() => controller.abort(), timeoutMs);
    try {
      const response = await fetch(url, {
        method: "GET",
        signal: controller.signal,
      });

      if (!response.ok) {
        throw new Error(`HTTP ${response.status}`);
      }

      const arrayBuffer = await response.arrayBuffer();
      return Buffer.from(arrayBuffer);
    } catch (error) {
      lastError = error;
      if (attempt < retries) {
        await new Promise((resolve) => setTimeout(resolve, 250 * attempt));
      }
    } finally {
      clearTimeout(timeout);
    }
  }

  throw lastError instanceof Error ? lastError : new Error("图片拉取失败");
}

export async function downscaleForVision(
  inputBuffer: Buffer,
  maxSide: number,
  jpegQuality: number,
  trimTransparentPadding = false,
) {
  const image = sharp(inputBuffer, { failOn: "none" });
  const metadata = await image.metadata();
  const hasAlpha = Boolean(metadata.hasAlpha);

  if (hasAlpha && trimTransparentPadding) {
    const trimmed = await sharp(inputBuffer, { failOn: "none" })
      .rotate()
      .trim({
        background: { r: 0, g: 0, b: 0, alpha: 0 },
        threshold: 8,
      })
      .toBuffer({ resolveWithObject: true });
    const padding = Math.max(
      12,
      Math.min(
        48,
        Math.round(Math.max(trimmed.info.width, trimmed.info.height) * 0.04),
      ),
    );
    const output = await sharp(trimmed.data, { failOn: "none" })
      .flatten({ background: "#ffffff" })
      .extend({
        top: padding,
        bottom: padding,
        left: padding,
        right: padding,
        background: "#ffffff",
      })
      .resize({
        width: maxSide,
        height: maxSide,
        fit: "inside",
        withoutEnlargement: true,
      })
      .jpeg({ quality: jpegQuality, mozjpeg: true })
      .toBuffer();
    return { mime: "image/jpeg", buffer: output };
  }

  const resized = image
    .rotate()
    .resize({
      width: maxSide,
      height: maxSide,
      fit: "inside",
      withoutEnlargement: true,
    });

  if (hasAlpha) {
    const output = await resized.png({ compressionLevel: 9 }).toBuffer();
    return { mime: "image/png", buffer: output };
  }

  const output = await resized.jpeg({ quality: jpegQuality, mozjpeg: true }).toBuffer();
  return { mime: "image/jpeg", buffer: output };
}

export async function inlineVisionImages(
  urls: string[],
  options: InlineVisionOptions = {},
): Promise<InlineVisionResult> {
  const maxImages = Math.max(1, options.maxImages ?? DEFAULT_MAX_IMAGES);
  const maxSide = Math.max(320, options.maxSide ?? DEFAULT_MAX_SIDE);
  const timeoutMs = Math.max(3000, options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
  const retries = Math.max(1, options.retries ?? DEFAULT_RETRIES);
  const jpegQuality = Math.max(55, Math.min(95, options.jpegQuality ?? DEFAULT_JPEG_QUALITY));
  const loggerPrefix = options.loggerPrefix ?? "[Vision Inline]";

  const inlineDataUrls: string[] = [];
  const skipped: Array<{ url: string; reason: string }> = [];

  const candidates = urls
    .map((item) => String(item || "").trim())
    .filter((url) => /^https?:\/\//i.test(url))
    .slice(0, maxImages);

  for (const url of candidates) {
    try {
      const buffer = await fetchImageWithRetry(url, timeoutMs, retries);
      const processed = await downscaleForVision(
        buffer,
        maxSide,
        jpegQuality,
        options.trimTransparentPadding === true,
      );
      const dataUrl = `data:${processed.mime};base64,${processed.buffer.toString("base64")}`;
      inlineDataUrls.push(dataUrl);
    } catch (error) {
      const reason = error instanceof Error ? error.message : "unknown_error";
      skipped.push({ url, reason });
      console.warn(`${loggerPrefix} skip reference image:`, {
        url,
        reason,
      });
    }
  }

  return { inlineDataUrls, skipped };
}