1114 lines
32 KiB
TypeScript
1114 lines
32 KiB
TypeScript
import { createHash, timingSafeEqual } from "node:crypto";
|
||
import { sql } from "drizzle-orm";
|
||
import sharp from "sharp";
|
||
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
||
import {
|
||
isImageGenerationModelDescriptor,
|
||
modelSupportsTextToImageDescriptor,
|
||
} from "@/lib/ai/model-classification";
|
||
import { db } from "@/lib/db";
|
||
import { getEnabledAiModelsWithProvider, getSystemSetting } from "@/lib/db/queries";
|
||
import { user } from "@/lib/db/schema";
|
||
import { generateDummyPassword } from "@/lib/db/utils";
|
||
import { createLogger } from "@/lib/observability/logger";
|
||
import {
|
||
DEFAULT_EXTERNAL_IMAGE_API_OWNER_EMAIL,
|
||
EXTERNAL_IMAGE_API_CONFIG_KEY,
|
||
normalizeExternalImageApiConfig,
|
||
type ExternalImageApiAccessKey,
|
||
} from "@/lib/external-image-api/config";
|
||
import {
|
||
extractImageUrlsFromText,
|
||
extractOriginalImageUrlFromAlt,
|
||
inferOriginalImageUrl,
|
||
} from "@/lib/image-url";
|
||
import { uploadFile } from "@/lib/storage/service";
|
||
|
||
type ImageModelEntry = Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>[number];
|
||
type ExternalImageApiGenerateAttempt = {
|
||
modelId: string;
|
||
modelRuntimeId: string;
|
||
displayName: string;
|
||
success: boolean;
|
||
error?: string;
|
||
};
|
||
|
||
export class ExternalImageApiGenerationError extends Error {
|
||
readonly attempts: ExternalImageApiGenerateAttempt[];
|
||
readonly userMessage: string;
|
||
|
||
constructor(
|
||
attempts: ExternalImageApiGenerateAttempt[],
|
||
userMessage = "图片生成失败，请稍后重试",
|
||
) {
|
||
const summary = attempts
|
||
.filter((attempt) => !attempt.success)
|
||
.map((attempt) => attempt.displayName || attempt.modelRuntimeId)
|
||
.join("; ");
|
||
|
||
super(summary || "所有候选模型均调用失败");
|
||
this.name = "ExternalImageApiGenerationError";
|
||
this.attempts = attempts;
|
||
this.userMessage = userMessage;
|
||
}
|
||
|
||
get internalMessage() {
|
||
return this.message;
|
||
}
|
||
}
|
||
|
||
type ExternalImageApiGeneratedImage = {
|
||
id: number;
|
||
url: string;
|
||
originalUrl: string;
|
||
original_url: string;
|
||
thumbnailUrl: string;
|
||
thumbnail_url: string;
|
||
previewUrl: string;
|
||
imageSize: string;
|
||
};
|
||
|
||
type ImageDimensionProbe = {
|
||
width: number;
|
||
height: number;
|
||
bytes: number;
|
||
contentType: string;
|
||
};
|
||
|
||
export const DEFAULT_EXTERNAL_IMAGE_API_IMAGE_SIZE = "1K";
|
||
|
||
const logger = createLogger("external-image-api");
|
||
|
||
function normalizeText(value: unknown) {
|
||
return String(value || "").trim();
|
||
}
|
||
|
||
export function normalizeExternalImageApiImageSize(value?: string | null) {
|
||
return normalizeText(value) || DEFAULT_EXTERNAL_IMAGE_API_IMAGE_SIZE;
|
||
}
|
||
|
||
function mimeTypeToExtension(mimeType: string) {
|
||
const normalized = mimeType.toLowerCase();
|
||
if (normalized.includes("png")) return "png";
|
||
if (normalized.includes("webp")) return "webp";
|
||
if (normalized.includes("gif")) return "gif";
|
||
if (normalized.includes("bmp")) return "bmp";
|
||
return "jpg";
|
||
}
|
||
|
||
function sanitizeUploadFilename(filename: string, fallbackPrefix: string) {
|
||
const normalized = String(filename || "")
|
||
.trim()
|
||
.replace(/[^\w.\-]+/g, "-")
|
||
.replace(/-+/g, "-")
|
||
.replace(/^-|-$/g, "");
|
||
if (normalized) return normalized;
|
||
return `${fallbackPrefix}-${Date.now()}.png`;
|
||
}
|
||
|
||
function getExtensionFromUrl(url: string): string {
|
||
try {
|
||
const pathname = new URL(url).pathname;
|
||
const ext = pathname.split(".").pop()?.toLowerCase();
|
||
if (!ext) return "jpg";
|
||
if (["jpg", "jpeg", "png", "webp", "gif", "bmp"].includes(ext)) {
|
||
return ext;
|
||
}
|
||
} catch {
|
||
// ignore
|
||
}
|
||
return "jpg";
|
||
}
|
||
|
||
function safeTokenCompare(left: string, right: string) {
|
||
const a = Buffer.from(left);
|
||
const b = Buffer.from(right);
|
||
if (a.length !== b.length) return false;
|
||
return timingSafeEqual(a, b);
|
||
}
|
||
|
||
function extractBearerToken(request: Request) {
|
||
const authorization = normalizeText(request.headers.get("authorization"));
|
||
if (/^bearer\s+/i.test(authorization)) {
|
||
return authorization.replace(/^bearer\s+/i, "").trim();
|
||
}
|
||
return normalizeText(request.headers.get("x-api-key"));
|
||
}
|
||
|
||
async function uploadRemoteImageToStorage(imageUrl: string): Promise<string> {
|
||
const response = await fetch(imageUrl, {
|
||
method: "GET",
|
||
signal: AbortSignal.timeout(30_000),
|
||
});
|
||
|
||
if (!response.ok) {
|
||
throw new Error(
|
||
`Remote image fetch failed: ${response.status} ${response.statusText}`,
|
||
);
|
||
}
|
||
|
||
const contentType = response.headers.get("content-type")?.toLowerCase() || "";
|
||
if (contentType && !contentType.startsWith("image/")) {
|
||
throw new Error(`Remote url is not an image: ${contentType}`);
|
||
}
|
||
|
||
const arrayBuffer = await response.arrayBuffer();
|
||
if (!arrayBuffer.byteLength) {
|
||
throw new Error("Remote image content is empty");
|
||
}
|
||
|
||
const ext = contentType
|
||
? mimeTypeToExtension(contentType)
|
||
: getExtensionFromUrl(imageUrl);
|
||
const fileName = `external-image-api-${Date.now()}-${Math.random()
|
||
.toString(36)
|
||
.slice(2, 8)}.${ext}`;
|
||
const uploaded = await uploadFile(fileName, arrayBuffer);
|
||
return uploaded.url;
|
||
}
|
||
|
||
async function uploadDataUrlImage(dataUrl: string): Promise<string> {
|
||
const match = dataUrl.match(/^data:([^;]+);base64,([\s\S]+)$/i);
|
||
if (!match) {
|
||
throw new Error("Invalid data URL image format");
|
||
}
|
||
|
||
const mimeType = match[1] || "image/jpeg";
|
||
const base64Data = normalizeText(match[2]).replace(/\s+/g, "");
|
||
if (!base64Data) {
|
||
throw new Error("Empty base64 image content");
|
||
}
|
||
|
||
const buffer = Buffer.from(base64Data, "base64");
|
||
const fileName = `external-inline-${Date.now()}-${Math.random()
|
||
.toString(36)
|
||
.slice(2, 8)}.${mimeTypeToExtension(mimeType)}`;
|
||
const arrayBuffer = buffer.buffer.slice(
|
||
buffer.byteOffset,
|
||
buffer.byteOffset + buffer.byteLength,
|
||
) as ArrayBuffer;
|
||
const uploaded = await uploadFile(fileName, arrayBuffer);
|
||
return uploaded.url;
|
||
}
|
||
|
||
async function ensureHostedImageUrl(imageUrl: string): Promise<string> {
|
||
const normalizedUrl = normalizeText(imageUrl);
|
||
if (!normalizedUrl) {
|
||
throw new Error("Image URL is empty");
|
||
}
|
||
|
||
if (normalizedUrl.startsWith("data:image")) {
|
||
return uploadDataUrlImage(normalizedUrl);
|
||
}
|
||
|
||
if (!/^https?:\/\//i.test(normalizedUrl)) {
|
||
throw new Error(`Unsupported image URL: ${normalizedUrl}`);
|
||
}
|
||
|
||
if (/\/uploads\//i.test(normalizedUrl)) {
|
||
return normalizedUrl;
|
||
}
|
||
|
||
return uploadRemoteImageToStorage(normalizedUrl);
|
||
}
|
||
|
||
async function probeRemoteImageDimensions(
|
||
imageUrl: string,
|
||
): Promise<ImageDimensionProbe> {
|
||
const normalizedUrl = normalizeText(imageUrl);
|
||
if (!normalizedUrl) {
|
||
throw new Error("图片地址为空");
|
||
}
|
||
|
||
const response = await fetch(normalizedUrl, {
|
||
method: "GET",
|
||
cache: "no-store",
|
||
signal: AbortSignal.timeout(30_000),
|
||
});
|
||
|
||
if (!response.ok) {
|
||
throw new Error(`图片地址不可访问: HTTP ${response.status}`);
|
||
}
|
||
|
||
const contentType = response.headers.get("content-type")?.toLowerCase() || "";
|
||
if (contentType && !contentType.startsWith("image/")) {
|
||
throw new Error(`图片地址返回了非图片内容: ${contentType}`);
|
||
}
|
||
|
||
const buffer = Buffer.from(await response.arrayBuffer());
|
||
if (!buffer.length) {
|
||
throw new Error("图片内容为空");
|
||
}
|
||
|
||
const metadata = await sharp(buffer).metadata();
|
||
if (!metadata.width || !metadata.height) {
|
||
throw new Error("无法识别图片尺寸");
|
||
}
|
||
|
||
return {
|
||
width: metadata.width,
|
||
height: metadata.height,
|
||
bytes: buffer.byteLength,
|
||
contentType,
|
||
};
|
||
}
|
||
|
||
function getMinimumGeneratedOriginalLongestEdge(imageSize?: string | null) {
|
||
const normalized = normalizeText(imageSize).toUpperCase();
|
||
const exactSize = normalized.match(/(\d{3,5})\s*[X×]\s*(\d{3,5})/);
|
||
if (exactSize) {
|
||
const width = Number(exactSize[1]);
|
||
const height = Number(exactSize[2]);
|
||
if (Number.isFinite(width) && Number.isFinite(height)) {
|
||
return Math.max(768, Math.round(Math.max(width, height) * 0.65));
|
||
}
|
||
}
|
||
|
||
if (normalized.includes("4K")) return 1800;
|
||
if (normalized.includes("2K")) return 1000;
|
||
if (normalized.includes("1K")) return 768;
|
||
return 768;
|
||
}
|
||
|
||
async function assertGeneratedOriginalImagesAreLargeEnough(
|
||
images: ExternalImageApiGeneratedImage[],
|
||
imageSize?: string | null,
|
||
) {
|
||
const minimumLongestEdge = getMinimumGeneratedOriginalLongestEdge(imageSize);
|
||
const requestedLabel = normalizeExternalImageApiImageSize(imageSize);
|
||
const probeCache = new Map<string, Promise<ImageDimensionProbe>>();
|
||
|
||
for (const image of images) {
|
||
const originalUrl = normalizeText(image.originalUrl || image.url);
|
||
if (!originalUrl) {
|
||
throw new Error("模型返回结果缺少原图地址");
|
||
}
|
||
|
||
if (!probeCache.has(originalUrl)) {
|
||
probeCache.set(originalUrl, probeRemoteImageDimensions(originalUrl));
|
||
}
|
||
|
||
const dimensions = await probeCache.get(originalUrl)!;
|
||
const longestEdge = Math.max(dimensions.width, dimensions.height);
|
||
if (longestEdge < minimumLongestEdge) {
|
||
throw new Error(
|
||
`模型返回原图尺寸过小: ${dimensions.width}x${dimensions.height}，未达到 ${requestedLabel} 输出要求`,
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
function toRecord(value: unknown): Record<string, unknown> | null {
|
||
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
||
return value as Record<string, unknown>;
|
||
}
|
||
|
||
function getSupportedImageModes(capabilities: unknown) {
|
||
const raw = toRecord(capabilities)?.supportedModes;
|
||
if (!Array.isArray(raw)) return new Set<string>();
|
||
return new Set(
|
||
raw
|
||
.map((item) => normalizeText(item).toLowerCase())
|
||
.filter(Boolean),
|
||
);
|
||
}
|
||
|
||
function getMaxReferenceImages(capabilities: unknown): number | null {
|
||
const raw = toRecord(capabilities)?.maxReferenceImages;
|
||
const parsed = Number(raw);
|
||
if (!Number.isFinite(parsed) || parsed <= 0) return null;
|
||
return Math.round(parsed);
|
||
}
|
||
|
||
function supportsImageEditForReferenceCount(
|
||
entry: ImageModelEntry,
|
||
referenceImageCount: number,
|
||
) {
|
||
if (referenceImageCount <= 0) {
|
||
return modelSupportsTextToImageDescriptor({
|
||
providerType: entry.provider.type,
|
||
modelId: entry.model.modelId,
|
||
capabilities: entry.model.capabilities,
|
||
defaultParams: entry.model.defaultParams,
|
||
supportsImageGen: entry.model.supportsImageGen,
|
||
});
|
||
}
|
||
|
||
const capabilities = toRecord(entry.model.capabilities);
|
||
const providerType = normalizeText(entry.provider.type).toLowerCase();
|
||
const kind = normalizeText(capabilities?.kind).toLowerCase();
|
||
const supportedModes = getSupportedImageModes(entry.model.capabilities);
|
||
const maxReferenceImages = getMaxReferenceImages(entry.model.capabilities);
|
||
const autoRouteEditEndpoint = capabilities?.autoRouteEditEndpoint === true;
|
||
const supportsMultiImageReference =
|
||
capabilities?.supportsMultiImageReference === true;
|
||
const runtimeModelId = normalizeText(entry.model.modelId).toLowerCase();
|
||
const providerImpliesReferenceEdit =
|
||
providerType === "openai-image" ||
|
||
providerType === "grsai-gpt-image" ||
|
||
providerType === "ruxa-image" ||
|
||
providerType === "snapgen-image";
|
||
|
||
const supportsReferenceEdit =
|
||
providerImpliesReferenceEdit ||
|
||
kind === "image-edit" ||
|
||
supportedModes.has("image-edit") ||
|
||
supportedModes.has("reference-images") ||
|
||
autoRouteEditEndpoint ||
|
||
runtimeModelId.endsWith("/edit");
|
||
|
||
if (!supportsReferenceEdit) {
|
||
return false;
|
||
}
|
||
|
||
if (
|
||
maxReferenceImages !== null &&
|
||
referenceImageCount > maxReferenceImages
|
||
) {
|
||
return false;
|
||
}
|
||
|
||
if (referenceImageCount > 1) {
|
||
if (maxReferenceImages !== null) {
|
||
return referenceImageCount <= maxReferenceImages;
|
||
}
|
||
return (
|
||
supportsMultiImageReference ||
|
||
autoRouteEditEndpoint ||
|
||
providerImpliesReferenceEdit
|
||
);
|
||
}
|
||
|
||
return true;
|
||
}
|
||
|
||
function getNormalizedModelCandidates(entry: ImageModelEntry) {
|
||
return [
|
||
entry.model.displayName,
|
||
entry.model.modelId,
|
||
entry.model.id,
|
||
]
|
||
.map((item) =>
|
||
normalizeText(item)
|
||
.toLowerCase()
|
||
.replace(/[\s_]+/g, "-"),
|
||
)
|
||
.filter(Boolean);
|
||
}
|
||
|
||
function isPreferredGptImage2Model(entry: ImageModelEntry) {
|
||
const candidates = getNormalizedModelCandidates(entry);
|
||
|
||
return candidates.some(
|
||
(item) =>
|
||
item.includes("gpt-image-2") ||
|
||
item.includes("gptimage2") ||
|
||
item.includes("gpt-image2"),
|
||
);
|
||
}
|
||
|
||
function getExternalImageApiPriority(entry: ImageModelEntry) {
|
||
const providerType = normalizeText(entry.provider.type).toLowerCase();
|
||
const candidates = getNormalizedModelCandidates(entry);
|
||
const isGptImage2 = isPreferredGptImage2Model(entry);
|
||
const isVip = candidates.some((item) => item.includes("vip"));
|
||
|
||
if (isGptImage2 && providerType === "openai-image") {
|
||
return 400;
|
||
}
|
||
|
||
if (isGptImage2 && providerType === "grsai-gpt-image") {
|
||
return isVip ? 350 : 340;
|
||
}
|
||
|
||
if (isGptImage2 && providerType === "fal-image") {
|
||
return 200;
|
||
}
|
||
|
||
if (isGptImage2) {
|
||
return 100;
|
||
}
|
||
|
||
return 0;
|
||
}
|
||
|
||
function isExternalImageApiPreferredProvider(entry: ImageModelEntry) {
|
||
return getExternalImageApiPriority(entry) >= 340;
|
||
}
|
||
|
||
function sortExternalImageApiModels(
|
||
models: ImageModelEntry[],
|
||
referenceImageCount: number,
|
||
) {
|
||
return [...models].sort((left, right) => {
|
||
const leftPriority = getExternalImageApiPriority(left);
|
||
const rightPriority = getExternalImageApiPriority(right);
|
||
if (leftPriority !== rightPriority) {
|
||
return rightPriority - leftPriority;
|
||
}
|
||
|
||
const leftRefSupport = supportsImageEditForReferenceCount(
|
||
left,
|
||
referenceImageCount,
|
||
)
|
||
? 1
|
||
: 0;
|
||
const rightRefSupport = supportsImageEditForReferenceCount(
|
||
right,
|
||
referenceImageCount,
|
||
)
|
||
? 1
|
||
: 0;
|
||
if (leftRefSupport !== rightRefSupport) {
|
||
return rightRefSupport - leftRefSupport;
|
||
}
|
||
|
||
return (left.model.sortOrder ?? 0) - (right.model.sortOrder ?? 0);
|
||
});
|
||
}
|
||
|
||
function findExternalImageApiModelMatch(
|
||
models: ImageModelEntry[],
|
||
modelRef?: string | null,
|
||
) {
|
||
const normalizedRef = normalizeText(modelRef).toLowerCase();
|
||
if (!normalizedRef) return null;
|
||
|
||
return (
|
||
models.find((entry) => {
|
||
const candidates = [
|
||
entry.model.id,
|
||
entry.model.modelId,
|
||
entry.model.displayName,
|
||
]
|
||
.map((item) => normalizeText(item).toLowerCase())
|
||
.filter(Boolean);
|
||
return candidates.includes(normalizedRef);
|
||
}) || null
|
||
);
|
||
}
|
||
|
||
function normalizeImageRatioForCandidateFilter(value: unknown) {
|
||
const normalized = normalizeText(value).toLowerCase();
|
||
if (!normalized || normalized === "auto") return null;
|
||
const normalizedParts = normalized
|
||
.split(":")
|
||
.map((part) => part.trim())
|
||
.filter(Boolean);
|
||
|
||
if (normalizedParts.length !== 2) {
|
||
return null;
|
||
}
|
||
|
||
const [widthRaw, heightRaw] = normalizedParts;
|
||
if (!/^\d+$/.test(widthRaw) || !/^\d+$/.test(heightRaw)) {
|
||
return null;
|
||
}
|
||
|
||
return `${Number(widthRaw)}:${Number(heightRaw)}`;
|
||
}
|
||
|
||
function shouldFilterCandidateForExternalImageApiRequest(
|
||
entry: ImageModelEntry,
|
||
params: {
|
||
referenceImageCount: number;
|
||
aspectRatio?: string | null;
|
||
imageSize?: string | null;
|
||
},
|
||
) {
|
||
const normalizedReferenceCount = Math.max(0, params.referenceImageCount);
|
||
const providerType = normalizeText(entry.provider.type).toLowerCase();
|
||
const modelRuntimeId = normalizeText(entry.model.modelId).toLowerCase();
|
||
const modelDisplayName = normalizeText(entry.model.displayName).toLowerCase();
|
||
const normalizedAspectRatio = normalizeImageRatioForCandidateFilter(params.aspectRatio);
|
||
|
||
const supportsReferenceImageEdit = supportsImageEditForReferenceCount(
|
||
entry,
|
||
normalizedReferenceCount,
|
||
);
|
||
if (!supportsReferenceImageEdit) {
|
||
return { filtered: false };
|
||
}
|
||
|
||
if (normalizedReferenceCount <= 0) {
|
||
return { filtered: false };
|
||
}
|
||
|
||
if (
|
||
providerType === "grsai-gpt-image" &&
|
||
(modelRuntimeId.includes("gpt-image-2") || modelDisplayName.includes("gpt-image-2"))
|
||
) {
|
||
if (normalizedAspectRatio === "1:1") {
|
||
return {
|
||
filtered: true,
|
||
reason:
|
||
"gpt-image-2 在某些 grsai 网关不兼容 1:1 参数，改走其他候选模型",
|
||
};
|
||
}
|
||
}
|
||
|
||
return { filtered: false };
|
||
}
|
||
|
||
function filterCompatibleCandidates(
|
||
candidates: ImageModelEntry[],
|
||
params: {
|
||
referenceImageCount: number;
|
||
aspectRatio?: string | null;
|
||
imageSize?: string | null;
|
||
},
|
||
) {
|
||
if (!candidates.length) {
|
||
return candidates;
|
||
}
|
||
|
||
const acceptedCandidates = [] as ImageModelEntry[];
|
||
const filteredCandidates: ImageModelEntry[] = [];
|
||
let reason = "";
|
||
|
||
for (const candidate of candidates) {
|
||
const filterDecision = shouldFilterCandidateForExternalImageApiRequest(
|
||
candidate,
|
||
params,
|
||
);
|
||
if (filterDecision.filtered) {
|
||
filteredCandidates.push(candidate);
|
||
if (!reason) {
|
||
reason = filterDecision.reason || "参数兼容性过滤";
|
||
}
|
||
continue;
|
||
}
|
||
acceptedCandidates.push(candidate);
|
||
}
|
||
|
||
if (acceptedCandidates.length === 0) {
|
||
logger.debug(
|
||
[
|
||
"[External Image API] No compatible candidate after request-aware filtering,",
|
||
`fallback to unfiltered candidates (referenceImageCount=${params.referenceImageCount},`,
|
||
`aspectRatio=${params.aspectRatio || "auto"}, imageSize=${params.imageSize || "default"},`,
|
||
`filteredCount=${filteredCandidates.length}, reason=${reason || "none"})`,
|
||
].join(" "),
|
||
);
|
||
return candidates;
|
||
}
|
||
|
||
if (filteredCandidates.length > 0) {
|
||
logger.debug(
|
||
`[External Image API] Filtered ${filteredCandidates.length} request-incompatible candidates (referenceImageCount=${params.referenceImageCount}, aspectRatio=${params.aspectRatio || "auto"}, imageSize=${params.imageSize || "default"}, reason=${reason || "compatibility"})`,
|
||
);
|
||
}
|
||
|
||
return acceptedCandidates;
|
||
}
|
||
|
||
function parseGeneratedImageMarkdownMeta(alt: string, fallbackUrl = "") {
|
||
const normalizedAlt = normalizeText(alt);
|
||
const fallbackOriginalUrl = inferOriginalImageUrl(normalizeText(fallbackUrl));
|
||
const originalUrl =
|
||
extractOriginalImageUrlFromAlt(normalizedAlt, fallbackUrl) || fallbackOriginalUrl;
|
||
let imageSize = DEFAULT_EXTERNAL_IMAGE_API_IMAGE_SIZE;
|
||
|
||
if (normalizedAlt.includes("thumb|||")) {
|
||
const parts = normalizedAlt
|
||
.substring(normalizedAlt.indexOf("thumb|||") + 8)
|
||
.split("|||");
|
||
imageSize = normalizeText(parts[1]) || imageSize;
|
||
} else if (normalizedAlt.includes("thumb$$")) {
|
||
const parts = normalizedAlt
|
||
.substring(normalizedAlt.indexOf("thumb$$") + 7)
|
||
.split("$$");
|
||
imageSize = normalizeText(parts[1]) || imageSize;
|
||
}
|
||
|
||
return {
|
||
originalUrl: normalizeText(originalUrl) || fallbackOriginalUrl,
|
||
imageSize,
|
||
};
|
||
}
|
||
|
||
export async function ensureHostedImageMarkdown(
|
||
markdown: string,
|
||
imageSize = DEFAULT_EXTERNAL_IMAGE_API_IMAGE_SIZE,
|
||
) {
|
||
const pattern = /!\[([^\]]*)\]\(([^)]+)\)/gim;
|
||
const matches = [...markdown.matchAll(pattern)];
|
||
|
||
if (!matches.length) {
|
||
const plainUrls = extractImageUrlsFromText(markdown);
|
||
if (!plainUrls.length) {
|
||
return markdown;
|
||
}
|
||
|
||
const hostedUrls = await Promise.all(
|
||
plainUrls.map((url) => ensureHostedImageUrl(url)),
|
||
);
|
||
return hostedUrls
|
||
.map((url) => ``)
|
||
.join("\n");
|
||
}
|
||
|
||
let rebuilt = "";
|
||
let cursor = 0;
|
||
|
||
for (const match of matches) {
|
||
const raw = match[0];
|
||
const alt = match[1] || "";
|
||
const thumbnailSourceUrl = normalizeText(match[2]);
|
||
const metadata = parseGeneratedImageMarkdownMeta(alt, thumbnailSourceUrl);
|
||
const originalSourceUrl =
|
||
metadata.originalUrl || inferOriginalImageUrl(thumbnailSourceUrl);
|
||
const resolvedImageSize = metadata?.imageSize || imageSize;
|
||
const start = match.index ?? -1;
|
||
if (start < 0) continue;
|
||
|
||
rebuilt += markdown.slice(cursor, start);
|
||
const hostedOriginalUrl = await ensureHostedImageUrl(originalSourceUrl);
|
||
const hostedThumbnailUrl =
|
||
thumbnailSourceUrl && thumbnailSourceUrl !== originalSourceUrl
|
||
? await ensureHostedImageUrl(thumbnailSourceUrl)
|
||
: hostedOriginalUrl;
|
||
rebuilt += ``;
|
||
cursor = start + raw.length;
|
||
}
|
||
|
||
rebuilt += markdown.slice(cursor);
|
||
return rebuilt;
|
||
}
|
||
|
||
export function extractGeneratedImagesFromMarkdown(markdown: string) {
|
||
const pattern = /!\[([^\]]*)\]\(([^)]+)\)/gim;
|
||
const matches = [...markdown.matchAll(pattern)];
|
||
|
||
return matches.map((match, index) => {
|
||
const alt = match[1] || "";
|
||
const thumbnailUrl = match[2] || "";
|
||
const metadata = parseGeneratedImageMarkdownMeta(alt, thumbnailUrl);
|
||
const originalUrl =
|
||
metadata.originalUrl || inferOriginalImageUrl(thumbnailUrl) || thumbnailUrl;
|
||
const imageSize = metadata.imageSize || DEFAULT_EXTERNAL_IMAGE_API_IMAGE_SIZE;
|
||
return {
|
||
id: index + 1,
|
||
url: originalUrl,
|
||
originalUrl,
|
||
original_url: originalUrl,
|
||
thumbnailUrl,
|
||
thumbnail_url: thumbnailUrl,
|
||
previewUrl: thumbnailUrl,
|
||
imageSize,
|
||
};
|
||
});
|
||
}
|
||
|
||
export async function getExternalImageApiConfig() {
|
||
const setting = await getSystemSetting({
|
||
key: EXTERNAL_IMAGE_API_CONFIG_KEY,
|
||
}).catch(() => null);
|
||
return normalizeExternalImageApiConfig(setting?.value);
|
||
}
|
||
|
||
export async function authenticateExternalImageApiRequest(request: Request) {
|
||
const config = await getExternalImageApiConfig();
|
||
const token = extractBearerToken(request);
|
||
|
||
if (!config.enabled) {
|
||
return {
|
||
config,
|
||
accessKey: null,
|
||
token,
|
||
error: "第三方生图 API 未启用",
|
||
status: 403 as const,
|
||
};
|
||
}
|
||
|
||
if (!token) {
|
||
return {
|
||
config,
|
||
accessKey: null,
|
||
token,
|
||
error: "缺少 API Key",
|
||
status: 401 as const,
|
||
};
|
||
}
|
||
|
||
const accessKey =
|
||
config.apiKeys.find(
|
||
(item) => item.enabled && safeTokenCompare(item.apiKey, token),
|
||
) || null;
|
||
|
||
if (!accessKey) {
|
||
return {
|
||
config,
|
||
accessKey,
|
||
token,
|
||
error: "API Key 无效或已禁用",
|
||
status: 401 as const,
|
||
};
|
||
}
|
||
|
||
return {
|
||
config,
|
||
accessKey,
|
||
token,
|
||
error: null,
|
||
status: 200 as const,
|
||
};
|
||
}
|
||
|
||
export async function getExternalImageApiModels() {
|
||
const models = await getEnabledAiModelsWithProvider(false, false);
|
||
|
||
return sortExternalImageApiModels(
|
||
models.filter(
|
||
(entry) =>
|
||
isImageGenerationModelDescriptor({
|
||
providerType: entry.provider.type,
|
||
modelId: entry.model.modelId,
|
||
capabilities: entry.model.capabilities,
|
||
supportsImageGen: entry.model.supportsImageGen,
|
||
}) && supportsImageEditForReferenceCount(entry, 1),
|
||
),
|
||
1,
|
||
);
|
||
}
|
||
|
||
export function serializeExternalImageApiModel(entry: ImageModelEntry) {
|
||
const capabilities = toRecord(entry.model.capabilities);
|
||
const maxReferenceImages = getMaxReferenceImages(entry.model.capabilities);
|
||
return {
|
||
id: entry.model.id,
|
||
modelId: entry.model.modelId,
|
||
displayName: entry.model.displayName,
|
||
description: entry.model.description,
|
||
providerType: entry.provider.type,
|
||
providerName: entry.provider.name,
|
||
avatarUrl: entry.model.avatarUrl,
|
||
imageGenConfig: entry.model.imageGenConfig,
|
||
supportsReferenceImages: supportsImageEditForReferenceCount(entry, 1),
|
||
supportsMultiImageReference:
|
||
capabilities?.supportsMultiImageReference === true ||
|
||
(maxReferenceImages !== null && maxReferenceImages > 1),
|
||
maxReferenceImages,
|
||
preferred: isExternalImageApiPreferredProvider(entry),
|
||
};
|
||
}
|
||
|
||
export async function resolveExternalImageApiModel(
|
||
modelRef?: string | null,
|
||
options?: {
|
||
referenceImageCount?: number;
|
||
},
|
||
) {
|
||
const referenceImageCount = Math.max(0, options?.referenceImageCount ?? 1);
|
||
const models = sortExternalImageApiModels(
|
||
(
|
||
await getEnabledAiModelsWithProvider(false, false)
|
||
).filter((entry) =>
|
||
isImageGenerationModelDescriptor({
|
||
providerType: entry.provider.type,
|
||
modelId: entry.model.modelId,
|
||
capabilities: entry.model.capabilities,
|
||
supportsImageGen: entry.model.supportsImageGen,
|
||
}),
|
||
),
|
||
referenceImageCount,
|
||
);
|
||
if (!models.length) {
|
||
throw new Error("当前没有启用可用于生图的模型");
|
||
}
|
||
|
||
const matched = findExternalImageApiModelMatch(models, modelRef);
|
||
if (!modelRef) {
|
||
const preferred = models.find((entry) =>
|
||
supportsImageEditForReferenceCount(entry, referenceImageCount),
|
||
);
|
||
if (!preferred) {
|
||
throw new Error("当前没有启用可用于图生图的模型");
|
||
}
|
||
return preferred;
|
||
}
|
||
|
||
if (!matched) {
|
||
throw new Error("未找到匹配的生图模型");
|
||
}
|
||
|
||
if (!supportsImageEditForReferenceCount(matched, referenceImageCount)) {
|
||
const maxReferenceImages = getMaxReferenceImages(matched.model.capabilities);
|
||
if (
|
||
referenceImageCount > 1 &&
|
||
maxReferenceImages !== null &&
|
||
referenceImageCount > maxReferenceImages
|
||
) {
|
||
throw new Error(
|
||
`模型「${matched.model.displayName}」最多支持 ${maxReferenceImages} 张参考图`,
|
||
);
|
||
}
|
||
throw new Error(`模型「${matched.model.displayName}」不支持当前图生图调用`);
|
||
}
|
||
|
||
return matched;
|
||
}
|
||
|
||
export async function buildExternalImageApiModelCandidates(params?: {
|
||
modelRef?: string | null;
|
||
referenceImageCount?: number;
|
||
aspectRatio?: string | null;
|
||
imageSize?: string | null;
|
||
}) {
|
||
const referenceImageCount = Math.max(0, params?.referenceImageCount ?? 1);
|
||
const allModels = sortExternalImageApiModels(
|
||
(
|
||
await getEnabledAiModelsWithProvider(false, false)
|
||
).filter((entry) =>
|
||
isImageGenerationModelDescriptor({
|
||
providerType: entry.provider.type,
|
||
modelId: entry.model.modelId,
|
||
capabilities: entry.model.capabilities,
|
||
supportsImageGen: entry.model.supportsImageGen,
|
||
}),
|
||
),
|
||
referenceImageCount,
|
||
);
|
||
|
||
const supportedModels = allModels.filter((entry) =>
|
||
supportsImageEditForReferenceCount(entry, referenceImageCount),
|
||
);
|
||
|
||
const compatibleModels = filterCompatibleCandidates(supportedModels, {
|
||
referenceImageCount,
|
||
aspectRatio: params?.aspectRatio || null,
|
||
imageSize: params?.imageSize || null,
|
||
});
|
||
|
||
if (!supportedModels.length) {
|
||
throw new Error("当前没有启用可用于图生图的模型");
|
||
}
|
||
|
||
const explicitMatch = findExternalImageApiModelMatch(allModels, params?.modelRef);
|
||
if (!params?.modelRef) {
|
||
return compatibleModels;
|
||
}
|
||
|
||
if (!explicitMatch) {
|
||
throw new Error("未找到匹配的生图模型");
|
||
}
|
||
|
||
if (!supportsImageEditForReferenceCount(explicitMatch, referenceImageCount)) {
|
||
const maxReferenceImages = getMaxReferenceImages(explicitMatch.model.capabilities);
|
||
if (
|
||
referenceImageCount > 1 &&
|
||
maxReferenceImages !== null &&
|
||
referenceImageCount > maxReferenceImages
|
||
) {
|
||
throw new Error(
|
||
`模型「${explicitMatch.model.displayName}」最多支持 ${maxReferenceImages} 张参考图`,
|
||
);
|
||
}
|
||
throw new Error(`模型「${explicitMatch.model.displayName}」不支持当前图生图调用`);
|
||
}
|
||
|
||
return [
|
||
explicitMatch,
|
||
...supportedModels.filter((entry) => entry.model.id !== explicitMatch.model.id),
|
||
];
|
||
}
|
||
|
||
export async function resolveExternalImageApiOwnerUserId(ownerEmail?: string | null) {
|
||
const normalizedEmail = normalizeText(ownerEmail).toLowerCase() ||
|
||
DEFAULT_EXTERNAL_IMAGE_API_OWNER_EMAIL;
|
||
|
||
const existing = await db
|
||
.select({ id: user.id })
|
||
.from(user)
|
||
.where(sql`lower(${user.email}) = ${normalizedEmail}`)
|
||
.limit(1);
|
||
|
||
if (existing[0]?.id) {
|
||
return existing[0].id;
|
||
}
|
||
|
||
const [created] = await db
|
||
.insert(user)
|
||
.values({
|
||
email: normalizedEmail,
|
||
password: generateDummyPassword(),
|
||
role: "guest",
|
||
name: "第三方生图 API",
|
||
isActive: true,
|
||
})
|
||
.returning({ id: user.id });
|
||
|
||
if (!created?.id) {
|
||
throw new Error("创建第三方 API 挂载用户失败");
|
||
}
|
||
|
||
return created.id;
|
||
}
|
||
|
||
export async function generateImageWithModel(params: {
|
||
modelDbId: string;
|
||
prompt: string;
|
||
negativePrompt?: string | null;
|
||
imageCount: number;
|
||
aspectRatio?: string | null;
|
||
imageSize?: string | null;
|
||
referenceImageUrls?: string[];
|
||
}) {
|
||
const languageModel = await getLanguageModelById(params.modelDbId);
|
||
if (!languageModel || typeof (languageModel as any).doGenerate !== "function") {
|
||
throw new Error("当前模型暂不支持第三方生图调用");
|
||
}
|
||
|
||
const composedPrompt = [
|
||
normalizeText(params.prompt),
|
||
normalizeText(params.negativePrompt)
|
||
? `Negative prompt:\n${normalizeText(params.negativePrompt)}`
|
||
: "",
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n\n");
|
||
|
||
const result = await (languageModel as any).doGenerate({
|
||
messages: [
|
||
{
|
||
role: "user",
|
||
content: [
|
||
{ type: "text", text: composedPrompt },
|
||
...(params.referenceImageUrls || []).map((imageUrl) => ({
|
||
type: "image",
|
||
image: imageUrl,
|
||
})),
|
||
],
|
||
},
|
||
],
|
||
imageParams: {
|
||
imageCount: params.imageCount,
|
||
aspectRatio: normalizeText(params.aspectRatio) || undefined,
|
||
imageSize: normalizeText(params.imageSize) || undefined,
|
||
},
|
||
});
|
||
|
||
if (result?.finishReason === "error") {
|
||
throw new Error(result.text || "图片生成失败");
|
||
}
|
||
|
||
const text = normalizeText(result?.text);
|
||
if (!text) {
|
||
throw new Error("模型未返回生成图片");
|
||
}
|
||
|
||
return text;
|
||
}
|
||
|
||
export async function generateImagesWithFailover(params: {
|
||
candidates: ImageModelEntry[];
|
||
prompt: string;
|
||
negativePrompt?: string | null;
|
||
imageCount: number;
|
||
aspectRatio?: string | null;
|
||
imageSize?: string | null;
|
||
referenceImageUrls?: string[];
|
||
}): Promise<{
|
||
markdown: string;
|
||
images: ExternalImageApiGeneratedImage[];
|
||
usedModel: ImageModelEntry;
|
||
attempts: ExternalImageApiGenerateAttempt[];
|
||
}> {
|
||
const attempts: ExternalImageApiGenerateAttempt[] = [];
|
||
|
||
for (const candidate of params.candidates) {
|
||
try {
|
||
const markdown = await generateImageWithModel({
|
||
modelDbId: candidate.model.id,
|
||
prompt: params.prompt,
|
||
negativePrompt: params.negativePrompt || null,
|
||
imageCount: params.imageCount,
|
||
aspectRatio: params.aspectRatio || null,
|
||
imageSize: params.imageSize || null,
|
||
referenceImageUrls: params.referenceImageUrls || [],
|
||
});
|
||
|
||
const hostedMarkdown = await ensureHostedImageMarkdown(
|
||
markdown,
|
||
normalizeExternalImageApiImageSize(params.imageSize),
|
||
);
|
||
const images = extractGeneratedImagesFromMarkdown(hostedMarkdown);
|
||
|
||
if (!images.length) {
|
||
throw new Error("模型返回结果里未解析到图片");
|
||
}
|
||
|
||
await assertGeneratedOriginalImagesAreLargeEnough(
|
||
images,
|
||
normalizeExternalImageApiImageSize(params.imageSize),
|
||
);
|
||
|
||
attempts.push({
|
||
modelId: candidate.model.id,
|
||
modelRuntimeId: candidate.model.modelId,
|
||
displayName: candidate.model.displayName,
|
||
success: true,
|
||
});
|
||
|
||
return {
|
||
markdown: hostedMarkdown,
|
||
images,
|
||
usedModel: candidate,
|
||
attempts,
|
||
};
|
||
} catch (error) {
|
||
attempts.push({
|
||
modelId: candidate.model.id,
|
||
modelRuntimeId: candidate.model.modelId,
|
||
displayName: candidate.model.displayName,
|
||
success: false,
|
||
error: error instanceof Error ? error.message : "图片生成失败",
|
||
});
|
||
}
|
||
}
|
||
|
||
throw new ExternalImageApiGenerationError(
|
||
attempts,
|
||
"外部生图服务暂无可用模型返回，请稍后重试",
|
||
);
|
||
}
|
||
|
||
export async function ensureHostedReferenceImageUrls(imageUrls: string[]) {
|
||
const deduped = imageUrls
|
||
.map((item) => normalizeText(item))
|
||
.filter(Boolean)
|
||
.filter((item, index, list) => list.indexOf(item) === index);
|
||
|
||
return Promise.all(deduped.map((url) => ensureHostedImageUrl(url)));
|
||
}
|
||
|
||
export async function uploadExternalImageApiFile(file: File) {
|
||
if (!(file instanceof File)) {
|
||
throw new Error("无效的图片文件");
|
||
}
|
||
|
||
const arrayBuffer = await file.arrayBuffer();
|
||
if (!arrayBuffer.byteLength) {
|
||
throw new Error("上传的图片文件为空");
|
||
}
|
||
|
||
const filename = sanitizeUploadFilename(file.name, "external-reference");
|
||
const uploaded = await uploadFile(filename, arrayBuffer);
|
||
return uploaded.url;
|
||
}
|
||
|
||
export function buildExternalImageApiKeyFingerprint(
|
||
accessKey: ExternalImageApiAccessKey | null,
|
||
) {
|
||
if (!accessKey?.apiKey) return null;
|
||
return createHash("sha256").update(accessKey.apiKey).digest("hex").slice(0, 16);
|
||
}
|
||
|
||
export function getRequesterIp(request: Request) {
|
||
const forwarded = normalizeText(request.headers.get("x-forwarded-for"));
|
||
if (forwarded) {
|
||
return forwarded.split(",")[0]?.trim() || null;
|
||
}
|
||
return normalizeText(request.headers.get("x-real-ip")) || null;
|
||
}
|