956 lines
27 KiB
TypeScript
956 lines
27 KiB
TypeScript
import { createHash, timingSafeEqual } from "node:crypto";
|
|
import { sql } from "drizzle-orm";
|
|
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
|
import { isImageGenerationModelDescriptor } from "@/lib/ai/model-classification";
|
|
import { db } from "@/lib/db";
|
|
import { getEnabledAiModelsWithProvider, getSystemSetting } from "@/lib/db/queries";
|
|
import { user } from "@/lib/db/schema";
|
|
import { generateDummyPassword } from "@/lib/db/utils";
|
|
import { createLogger } from "@/lib/observability/logger";
|
|
import {
|
|
DEFAULT_EXTERNAL_IMAGE_API_OWNER_EMAIL,
|
|
EXTERNAL_IMAGE_API_CONFIG_KEY,
|
|
normalizeExternalImageApiConfig,
|
|
type ExternalImageApiAccessKey,
|
|
} from "@/lib/external-image-api/config";
|
|
import { extractImageUrlsFromText } from "@/lib/image-url";
|
|
import { uploadFile } from "@/lib/storage/service";
|
|
|
|
type ImageModelEntry = Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>[number];
|
|
type ExternalImageApiGenerateAttempt = {
|
|
modelId: string;
|
|
modelRuntimeId: string;
|
|
displayName: string;
|
|
success: boolean;
|
|
error?: string;
|
|
};
|
|
|
|
export class ExternalImageApiGenerationError extends Error {
|
|
readonly attempts: ExternalImageApiGenerateAttempt[];
|
|
readonly userMessage: string;
|
|
|
|
constructor(
|
|
attempts: ExternalImageApiGenerateAttempt[],
|
|
userMessage = "图片生成失败，请稍后重试",
|
|
) {
|
|
const summary = attempts
|
|
.filter((attempt) => !attempt.success)
|
|
.map((attempt) => attempt.displayName || attempt.modelRuntimeId)
|
|
.join("; ");
|
|
|
|
super(summary || "所有候选模型均调用失败");
|
|
this.name = "ExternalImageApiGenerationError";
|
|
this.attempts = attempts;
|
|
this.userMessage = userMessage;
|
|
}
|
|
|
|
get internalMessage() {
|
|
return this.message;
|
|
}
|
|
}
|
|
|
|
type ExternalImageApiGeneratedImage = {
|
|
id: number;
|
|
originalUrl: string;
|
|
thumbnailUrl: string;
|
|
imageSize: string;
|
|
};
|
|
|
|
const logger = createLogger("external-image-api");
|
|
|
|
function normalizeText(value: unknown) {
|
|
return String(value || "").trim();
|
|
}
|
|
|
|
function mimeTypeToExtension(mimeType: string) {
|
|
const normalized = mimeType.toLowerCase();
|
|
if (normalized.includes("png")) return "png";
|
|
if (normalized.includes("webp")) return "webp";
|
|
if (normalized.includes("gif")) return "gif";
|
|
if (normalized.includes("bmp")) return "bmp";
|
|
return "jpg";
|
|
}
|
|
|
|
function sanitizeUploadFilename(filename: string, fallbackPrefix: string) {
|
|
const normalized = String(filename || "")
|
|
.trim()
|
|
.replace(/[^\w.\-]+/g, "-")
|
|
.replace(/-+/g, "-")
|
|
.replace(/^-|-$/g, "");
|
|
if (normalized) return normalized;
|
|
return `${fallbackPrefix}-${Date.now()}.png`;
|
|
}
|
|
|
|
function getExtensionFromUrl(url: string): string {
|
|
try {
|
|
const pathname = new URL(url).pathname;
|
|
const ext = pathname.split(".").pop()?.toLowerCase();
|
|
if (!ext) return "jpg";
|
|
if (["jpg", "jpeg", "png", "webp", "gif", "bmp"].includes(ext)) {
|
|
return ext;
|
|
}
|
|
} catch {
|
|
// ignore
|
|
}
|
|
return "jpg";
|
|
}
|
|
|
|
function safeTokenCompare(left: string, right: string) {
|
|
const a = Buffer.from(left);
|
|
const b = Buffer.from(right);
|
|
if (a.length !== b.length) return false;
|
|
return timingSafeEqual(a, b);
|
|
}
|
|
|
|
function extractBearerToken(request: Request) {
|
|
const authorization = normalizeText(request.headers.get("authorization"));
|
|
if (/^bearer\s+/i.test(authorization)) {
|
|
return authorization.replace(/^bearer\s+/i, "").trim();
|
|
}
|
|
return normalizeText(request.headers.get("x-api-key"));
|
|
}
|
|
|
|
async function uploadRemoteImageToStorage(imageUrl: string): Promise<string> {
|
|
const response = await fetch(imageUrl, {
|
|
method: "GET",
|
|
signal: AbortSignal.timeout(30_000),
|
|
});
|
|
|
|
if (!response.ok) {
|
|
throw new Error(
|
|
`Remote image fetch failed: ${response.status} ${response.statusText}`,
|
|
);
|
|
}
|
|
|
|
const contentType = response.headers.get("content-type")?.toLowerCase() || "";
|
|
if (contentType && !contentType.startsWith("image/")) {
|
|
throw new Error(`Remote url is not an image: ${contentType}`);
|
|
}
|
|
|
|
const arrayBuffer = await response.arrayBuffer();
|
|
if (!arrayBuffer.byteLength) {
|
|
throw new Error("Remote image content is empty");
|
|
}
|
|
|
|
const ext = contentType
|
|
? mimeTypeToExtension(contentType)
|
|
: getExtensionFromUrl(imageUrl);
|
|
const fileName = `external-image-api-${Date.now()}-${Math.random()
|
|
.toString(36)
|
|
.slice(2, 8)}.${ext}`;
|
|
const uploaded = await uploadFile(fileName, arrayBuffer);
|
|
return uploaded.url;
|
|
}
|
|
|
|
async function uploadDataUrlImage(dataUrl: string): Promise<string> {
|
|
const match = dataUrl.match(/^data:([^;]+);base64,([\s\S]+)$/i);
|
|
if (!match) {
|
|
throw new Error("Invalid data URL image format");
|
|
}
|
|
|
|
const mimeType = match[1] || "image/jpeg";
|
|
const base64Data = normalizeText(match[2]).replace(/\s+/g, "");
|
|
if (!base64Data) {
|
|
throw new Error("Empty base64 image content");
|
|
}
|
|
|
|
const buffer = Buffer.from(base64Data, "base64");
|
|
const fileName = `external-inline-${Date.now()}-${Math.random()
|
|
.toString(36)
|
|
.slice(2, 8)}.${mimeTypeToExtension(mimeType)}`;
|
|
const arrayBuffer = buffer.buffer.slice(
|
|
buffer.byteOffset,
|
|
buffer.byteOffset + buffer.byteLength,
|
|
) as ArrayBuffer;
|
|
const uploaded = await uploadFile(fileName, arrayBuffer);
|
|
return uploaded.url;
|
|
}
|
|
|
|
async function ensureHostedImageUrl(imageUrl: string): Promise<string> {
|
|
const normalizedUrl = normalizeText(imageUrl);
|
|
if (!normalizedUrl) {
|
|
throw new Error("Image URL is empty");
|
|
}
|
|
|
|
if (normalizedUrl.startsWith("data:image")) {
|
|
return uploadDataUrlImage(normalizedUrl);
|
|
}
|
|
|
|
if (!/^https?:\/\//i.test(normalizedUrl)) {
|
|
throw new Error(`Unsupported image URL: ${normalizedUrl}`);
|
|
}
|
|
|
|
if (/\/uploads\//i.test(normalizedUrl)) {
|
|
return normalizedUrl;
|
|
}
|
|
|
|
return uploadRemoteImageToStorage(normalizedUrl);
|
|
}
|
|
|
|
function toRecord(value: unknown): Record<string, unknown> | null {
|
|
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
|
return value as Record<string, unknown>;
|
|
}
|
|
|
|
function getSupportedImageModes(capabilities: unknown) {
|
|
const raw = toRecord(capabilities)?.supportedModes;
|
|
if (!Array.isArray(raw)) return new Set<string>();
|
|
return new Set(
|
|
raw
|
|
.map((item) => normalizeText(item).toLowerCase())
|
|
.filter(Boolean),
|
|
);
|
|
}
|
|
|
|
function getMaxReferenceImages(capabilities: unknown): number | null {
|
|
const raw = toRecord(capabilities)?.maxReferenceImages;
|
|
const parsed = Number(raw);
|
|
if (!Number.isFinite(parsed) || parsed <= 0) return null;
|
|
return Math.round(parsed);
|
|
}
|
|
|
|
function supportsImageEditForReferenceCount(
|
|
entry: ImageModelEntry,
|
|
referenceImageCount: number,
|
|
) {
|
|
if (referenceImageCount <= 0) {
|
|
return true;
|
|
}
|
|
|
|
const capabilities = toRecord(entry.model.capabilities);
|
|
const providerType = normalizeText(entry.provider.type).toLowerCase();
|
|
const kind = normalizeText(capabilities?.kind).toLowerCase();
|
|
const supportedModes = getSupportedImageModes(entry.model.capabilities);
|
|
const maxReferenceImages = getMaxReferenceImages(entry.model.capabilities);
|
|
const autoRouteEditEndpoint = capabilities?.autoRouteEditEndpoint === true;
|
|
const supportsMultiImageReference =
|
|
capabilities?.supportsMultiImageReference === true;
|
|
const runtimeModelId = normalizeText(entry.model.modelId).toLowerCase();
|
|
const providerImpliesReferenceEdit =
|
|
providerType === "openai-image" ||
|
|
providerType === "grsai-gpt-image" ||
|
|
providerType === "ruxa-image";
|
|
|
|
const supportsReferenceEdit =
|
|
providerImpliesReferenceEdit ||
|
|
kind === "image-edit" ||
|
|
supportedModes.has("image-edit") ||
|
|
supportedModes.has("reference-images") ||
|
|
autoRouteEditEndpoint ||
|
|
runtimeModelId.endsWith("/edit");
|
|
|
|
if (!supportsReferenceEdit) {
|
|
return false;
|
|
}
|
|
|
|
if (
|
|
maxReferenceImages !== null &&
|
|
referenceImageCount > maxReferenceImages
|
|
) {
|
|
return false;
|
|
}
|
|
|
|
if (referenceImageCount > 1) {
|
|
if (maxReferenceImages !== null) {
|
|
return referenceImageCount <= maxReferenceImages;
|
|
}
|
|
return (
|
|
supportsMultiImageReference ||
|
|
autoRouteEditEndpoint ||
|
|
providerImpliesReferenceEdit
|
|
);
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
function getNormalizedModelCandidates(entry: ImageModelEntry) {
|
|
return [
|
|
entry.model.displayName,
|
|
entry.model.modelId,
|
|
entry.model.id,
|
|
]
|
|
.map((item) =>
|
|
normalizeText(item)
|
|
.toLowerCase()
|
|
.replace(/[\s_]+/g, "-"),
|
|
)
|
|
.filter(Boolean);
|
|
}
|
|
|
|
function isPreferredGptImage2Model(entry: ImageModelEntry) {
|
|
const candidates = getNormalizedModelCandidates(entry);
|
|
|
|
return candidates.some(
|
|
(item) =>
|
|
item.includes("gpt-image-2") ||
|
|
item.includes("gptimage2") ||
|
|
item.includes("gpt-image2"),
|
|
);
|
|
}
|
|
|
|
function getExternalImageApiPriority(entry: ImageModelEntry) {
|
|
const providerType = normalizeText(entry.provider.type).toLowerCase();
|
|
const candidates = getNormalizedModelCandidates(entry);
|
|
const isGptImage2 = isPreferredGptImage2Model(entry);
|
|
const isVip = candidates.some((item) => item.includes("vip"));
|
|
|
|
if (isGptImage2 && providerType === "openai-image") {
|
|
return 400;
|
|
}
|
|
|
|
if (isGptImage2 && providerType === "grsai-gpt-image") {
|
|
return isVip ? 350 : 340;
|
|
}
|
|
|
|
if (isGptImage2 && providerType === "fal-image") {
|
|
return 200;
|
|
}
|
|
|
|
if (isGptImage2) {
|
|
return 100;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
function isExternalImageApiPreferredProvider(entry: ImageModelEntry) {
|
|
return getExternalImageApiPriority(entry) >= 340;
|
|
}
|
|
|
|
function sortExternalImageApiModels(
|
|
models: ImageModelEntry[],
|
|
referenceImageCount: number,
|
|
) {
|
|
return [...models].sort((left, right) => {
|
|
const leftPriority = getExternalImageApiPriority(left);
|
|
const rightPriority = getExternalImageApiPriority(right);
|
|
if (leftPriority !== rightPriority) {
|
|
return rightPriority - leftPriority;
|
|
}
|
|
|
|
const leftRefSupport = supportsImageEditForReferenceCount(
|
|
left,
|
|
referenceImageCount,
|
|
)
|
|
? 1
|
|
: 0;
|
|
const rightRefSupport = supportsImageEditForReferenceCount(
|
|
right,
|
|
referenceImageCount,
|
|
)
|
|
? 1
|
|
: 0;
|
|
if (leftRefSupport !== rightRefSupport) {
|
|
return rightRefSupport - leftRefSupport;
|
|
}
|
|
|
|
return (left.model.sortOrder ?? 0) - (right.model.sortOrder ?? 0);
|
|
});
|
|
}
|
|
|
|
function findExternalImageApiModelMatch(
|
|
models: ImageModelEntry[],
|
|
modelRef?: string | null,
|
|
) {
|
|
const normalizedRef = normalizeText(modelRef).toLowerCase();
|
|
if (!normalizedRef) return null;
|
|
|
|
return (
|
|
models.find((entry) => {
|
|
const candidates = [
|
|
entry.model.id,
|
|
entry.model.modelId,
|
|
entry.model.displayName,
|
|
]
|
|
.map((item) => normalizeText(item).toLowerCase())
|
|
.filter(Boolean);
|
|
return candidates.includes(normalizedRef);
|
|
}) || null
|
|
);
|
|
}
|
|
|
|
function normalizeImageRatioForCandidateFilter(value: unknown) {
|
|
const normalized = normalizeText(value).toLowerCase();
|
|
if (!normalized || normalized === "auto") return null;
|
|
const normalizedParts = normalized
|
|
.split(":")
|
|
.map((part) => part.trim())
|
|
.filter(Boolean);
|
|
|
|
if (normalizedParts.length !== 2) {
|
|
return null;
|
|
}
|
|
|
|
const [widthRaw, heightRaw] = normalizedParts;
|
|
if (!/^\d+$/.test(widthRaw) || !/^\d+$/.test(heightRaw)) {
|
|
return null;
|
|
}
|
|
|
|
return `${Number(widthRaw)}:${Number(heightRaw)}`;
|
|
}
|
|
|
|
function shouldFilterCandidateForExternalImageApiRequest(
|
|
entry: ImageModelEntry,
|
|
params: {
|
|
referenceImageCount: number;
|
|
aspectRatio?: string | null;
|
|
imageSize?: string | null;
|
|
},
|
|
) {
|
|
const normalizedReferenceCount = Math.max(0, params.referenceImageCount);
|
|
const providerType = normalizeText(entry.provider.type).toLowerCase();
|
|
const modelRuntimeId = normalizeText(entry.model.modelId).toLowerCase();
|
|
const modelDisplayName = normalizeText(entry.model.displayName).toLowerCase();
|
|
const normalizedAspectRatio = normalizeImageRatioForCandidateFilter(params.aspectRatio);
|
|
|
|
const supportsReferenceImageEdit = supportsImageEditForReferenceCount(
|
|
entry,
|
|
normalizedReferenceCount,
|
|
);
|
|
if (!supportsReferenceImageEdit) {
|
|
return { filtered: false };
|
|
}
|
|
|
|
if (normalizedReferenceCount <= 0) {
|
|
return { filtered: false };
|
|
}
|
|
|
|
if (
|
|
providerType === "grsai-gpt-image" &&
|
|
(modelRuntimeId.includes("gpt-image-2") || modelDisplayName.includes("gpt-image-2"))
|
|
) {
|
|
if (normalizedAspectRatio === "1:1") {
|
|
return {
|
|
filtered: true,
|
|
reason:
|
|
"gpt-image-2 在某些 grsai 网关不兼容 1:1 参数，改走其他候选模型",
|
|
};
|
|
}
|
|
}
|
|
|
|
return { filtered: false };
|
|
}
|
|
|
|
function filterCompatibleCandidates(
|
|
candidates: ImageModelEntry[],
|
|
params: {
|
|
referenceImageCount: number;
|
|
aspectRatio?: string | null;
|
|
imageSize?: string | null;
|
|
},
|
|
) {
|
|
if (!candidates.length) {
|
|
return candidates;
|
|
}
|
|
|
|
const acceptedCandidates = [] as ImageModelEntry[];
|
|
const filteredCandidates: ImageModelEntry[] = [];
|
|
let reason = "";
|
|
|
|
for (const candidate of candidates) {
|
|
const filterDecision = shouldFilterCandidateForExternalImageApiRequest(
|
|
candidate,
|
|
params,
|
|
);
|
|
if (filterDecision.filtered) {
|
|
filteredCandidates.push(candidate);
|
|
if (!reason) {
|
|
reason = filterDecision.reason || "参数兼容性过滤";
|
|
}
|
|
continue;
|
|
}
|
|
acceptedCandidates.push(candidate);
|
|
}
|
|
|
|
if (acceptedCandidates.length === 0) {
|
|
logger.debug(
|
|
[
|
|
"[External Image API] No compatible candidate after request-aware filtering,",
|
|
`fallback to unfiltered candidates (referenceImageCount=${params.referenceImageCount},`,
|
|
`aspectRatio=${params.aspectRatio || "auto"}, imageSize=${params.imageSize || "default"},`,
|
|
`filteredCount=${filteredCandidates.length}, reason=${reason || "none"})`,
|
|
].join(" "),
|
|
);
|
|
return candidates;
|
|
}
|
|
|
|
if (filteredCandidates.length > 0) {
|
|
logger.debug(
|
|
`[External Image API] Filtered ${filteredCandidates.length} request-incompatible candidates (referenceImageCount=${params.referenceImageCount}, aspectRatio=${params.aspectRatio || "auto"}, imageSize=${params.imageSize || "default"}, reason=${reason || "compatibility"})`,
|
|
);
|
|
}
|
|
|
|
return acceptedCandidates;
|
|
}
|
|
|
|
export async function ensureHostedImageMarkdown(
|
|
markdown: string,
|
|
imageSize = "2K",
|
|
) {
|
|
const pattern = /!\[([^\]]*)\]\(([^)]+)\)/gim;
|
|
const matches = [...markdown.matchAll(pattern)];
|
|
|
|
if (!matches.length) {
|
|
const plainUrls = extractImageUrlsFromText(markdown);
|
|
if (!plainUrls.length) {
|
|
return markdown;
|
|
}
|
|
|
|
const hostedUrls = await Promise.all(
|
|
plainUrls.map((url) => ensureHostedImageUrl(url)),
|
|
);
|
|
return hostedUrls
|
|
.map((url) => ``)
|
|
.join("\n");
|
|
}
|
|
|
|
let rebuilt = "";
|
|
let cursor = 0;
|
|
|
|
for (const match of matches) {
|
|
const raw = match[0];
|
|
const sourceUrl = match[2] || "";
|
|
const start = match.index ?? -1;
|
|
if (start < 0) continue;
|
|
|
|
rebuilt += markdown.slice(cursor, start);
|
|
const hostedUrl = await ensureHostedImageUrl(sourceUrl);
|
|
rebuilt += ``;
|
|
cursor = start + raw.length;
|
|
}
|
|
|
|
rebuilt += markdown.slice(cursor);
|
|
return rebuilt;
|
|
}
|
|
|
|
export function extractGeneratedImagesFromMarkdown(markdown: string) {
|
|
const pattern = /!\[([^\]]*)\]\(([^)]+)\)/gim;
|
|
const matches = [...markdown.matchAll(pattern)];
|
|
|
|
return matches.map((match, index) => {
|
|
const alt = match[1] || "";
|
|
const thumbnailUrl = match[2] || "";
|
|
let originalUrl = thumbnailUrl;
|
|
let imageSize = "2K";
|
|
const metaMatch = alt.match(/thumb\|\|\|([^|]+)\|\|\|(.+)$/i);
|
|
if (metaMatch) {
|
|
originalUrl = normalizeText(metaMatch[1]) || thumbnailUrl;
|
|
imageSize = normalizeText(metaMatch[2]) || imageSize;
|
|
}
|
|
return {
|
|
id: index + 1,
|
|
originalUrl,
|
|
thumbnailUrl,
|
|
imageSize,
|
|
};
|
|
});
|
|
}
|
|
|
|
export async function getExternalImageApiConfig() {
|
|
const setting = await getSystemSetting({
|
|
key: EXTERNAL_IMAGE_API_CONFIG_KEY,
|
|
}).catch(() => null);
|
|
return normalizeExternalImageApiConfig(setting?.value);
|
|
}
|
|
|
|
export async function authenticateExternalImageApiRequest(request: Request) {
|
|
const config = await getExternalImageApiConfig();
|
|
const token = extractBearerToken(request);
|
|
|
|
if (!config.enabled) {
|
|
return {
|
|
config,
|
|
accessKey: null,
|
|
token,
|
|
error: "第三方生图 API 未启用",
|
|
status: 403 as const,
|
|
};
|
|
}
|
|
|
|
if (!token) {
|
|
return {
|
|
config,
|
|
accessKey: null,
|
|
token,
|
|
error: "缺少 API Key",
|
|
status: 401 as const,
|
|
};
|
|
}
|
|
|
|
const accessKey =
|
|
config.apiKeys.find(
|
|
(item) => item.enabled && safeTokenCompare(item.apiKey, token),
|
|
) || null;
|
|
|
|
if (!accessKey) {
|
|
return {
|
|
config,
|
|
accessKey,
|
|
token,
|
|
error: "API Key 无效或已禁用",
|
|
status: 401 as const,
|
|
};
|
|
}
|
|
|
|
return {
|
|
config,
|
|
accessKey,
|
|
token,
|
|
error: null,
|
|
status: 200 as const,
|
|
};
|
|
}
|
|
|
|
export async function getExternalImageApiModels() {
|
|
const models = await getEnabledAiModelsWithProvider(false, false);
|
|
|
|
return sortExternalImageApiModels(
|
|
models.filter(
|
|
(entry) =>
|
|
isImageGenerationModelDescriptor({
|
|
providerType: entry.provider.type,
|
|
modelId: entry.model.modelId,
|
|
capabilities: entry.model.capabilities,
|
|
supportsImageGen: entry.model.supportsImageGen,
|
|
}) && supportsImageEditForReferenceCount(entry, 1),
|
|
),
|
|
1,
|
|
);
|
|
}
|
|
|
|
export function serializeExternalImageApiModel(entry: ImageModelEntry) {
|
|
const capabilities = toRecord(entry.model.capabilities);
|
|
const maxReferenceImages = getMaxReferenceImages(entry.model.capabilities);
|
|
return {
|
|
id: entry.model.id,
|
|
modelId: entry.model.modelId,
|
|
displayName: entry.model.displayName,
|
|
description: entry.model.description,
|
|
providerType: entry.provider.type,
|
|
providerName: entry.provider.name,
|
|
avatarUrl: entry.model.avatarUrl,
|
|
imageGenConfig: entry.model.imageGenConfig,
|
|
supportsReferenceImages: supportsImageEditForReferenceCount(entry, 1),
|
|
supportsMultiImageReference:
|
|
capabilities?.supportsMultiImageReference === true ||
|
|
(maxReferenceImages !== null && maxReferenceImages > 1),
|
|
maxReferenceImages,
|
|
preferred: isExternalImageApiPreferredProvider(entry),
|
|
};
|
|
}
|
|
|
|
export async function resolveExternalImageApiModel(
|
|
modelRef?: string | null,
|
|
options?: {
|
|
referenceImageCount?: number;
|
|
},
|
|
) {
|
|
const referenceImageCount = Math.max(0, options?.referenceImageCount ?? 1);
|
|
const models = sortExternalImageApiModels(
|
|
(
|
|
await getEnabledAiModelsWithProvider(false, false)
|
|
).filter((entry) =>
|
|
isImageGenerationModelDescriptor({
|
|
providerType: entry.provider.type,
|
|
modelId: entry.model.modelId,
|
|
capabilities: entry.model.capabilities,
|
|
supportsImageGen: entry.model.supportsImageGen,
|
|
}),
|
|
),
|
|
referenceImageCount,
|
|
);
|
|
if (!models.length) {
|
|
throw new Error("当前没有启用可用于生图的模型");
|
|
}
|
|
|
|
const matched = findExternalImageApiModelMatch(models, modelRef);
|
|
if (!modelRef) {
|
|
const preferred = models.find((entry) =>
|
|
supportsImageEditForReferenceCount(entry, referenceImageCount),
|
|
);
|
|
if (!preferred) {
|
|
throw new Error("当前没有启用可用于图生图的模型");
|
|
}
|
|
return preferred;
|
|
}
|
|
|
|
if (!matched) {
|
|
throw new Error("未找到匹配的生图模型");
|
|
}
|
|
|
|
if (!supportsImageEditForReferenceCount(matched, referenceImageCount)) {
|
|
const maxReferenceImages = getMaxReferenceImages(matched.model.capabilities);
|
|
if (
|
|
referenceImageCount > 1 &&
|
|
maxReferenceImages !== null &&
|
|
referenceImageCount > maxReferenceImages
|
|
) {
|
|
throw new Error(
|
|
`模型「${matched.model.displayName}」最多支持 ${maxReferenceImages} 张参考图`,
|
|
);
|
|
}
|
|
throw new Error(`模型「${matched.model.displayName}」不支持当前图生图调用`);
|
|
}
|
|
|
|
return matched;
|
|
}
|
|
|
|
export async function buildExternalImageApiModelCandidates(params?: {
|
|
modelRef?: string | null;
|
|
referenceImageCount?: number;
|
|
aspectRatio?: string | null;
|
|
imageSize?: string | null;
|
|
}) {
|
|
const referenceImageCount = Math.max(0, params?.referenceImageCount ?? 1);
|
|
const allModels = sortExternalImageApiModels(
|
|
(
|
|
await getEnabledAiModelsWithProvider(false, false)
|
|
).filter((entry) =>
|
|
isImageGenerationModelDescriptor({
|
|
providerType: entry.provider.type,
|
|
modelId: entry.model.modelId,
|
|
capabilities: entry.model.capabilities,
|
|
supportsImageGen: entry.model.supportsImageGen,
|
|
}),
|
|
),
|
|
referenceImageCount,
|
|
);
|
|
|
|
const supportedModels = allModels.filter((entry) =>
|
|
supportsImageEditForReferenceCount(entry, referenceImageCount),
|
|
);
|
|
|
|
const compatibleModels = filterCompatibleCandidates(supportedModels, {
|
|
referenceImageCount,
|
|
aspectRatio: params?.aspectRatio || null,
|
|
imageSize: params?.imageSize || null,
|
|
});
|
|
|
|
if (!supportedModels.length) {
|
|
throw new Error("当前没有启用可用于图生图的模型");
|
|
}
|
|
|
|
const explicitMatch = findExternalImageApiModelMatch(allModels, params?.modelRef);
|
|
if (!params?.modelRef) {
|
|
return compatibleModels;
|
|
}
|
|
|
|
if (!explicitMatch) {
|
|
throw new Error("未找到匹配的生图模型");
|
|
}
|
|
|
|
if (!supportsImageEditForReferenceCount(explicitMatch, referenceImageCount)) {
|
|
const maxReferenceImages = getMaxReferenceImages(explicitMatch.model.capabilities);
|
|
if (
|
|
referenceImageCount > 1 &&
|
|
maxReferenceImages !== null &&
|
|
referenceImageCount > maxReferenceImages
|
|
) {
|
|
throw new Error(
|
|
`模型「${explicitMatch.model.displayName}」最多支持 ${maxReferenceImages} 张参考图`,
|
|
);
|
|
}
|
|
throw new Error(`模型「${explicitMatch.model.displayName}」不支持当前图生图调用`);
|
|
}
|
|
|
|
return [
|
|
explicitMatch,
|
|
...supportedModels.filter((entry) => entry.model.id !== explicitMatch.model.id),
|
|
];
|
|
}
|
|
|
|
export async function resolveExternalImageApiOwnerUserId(ownerEmail?: string | null) {
|
|
const normalizedEmail = normalizeText(ownerEmail).toLowerCase() ||
|
|
DEFAULT_EXTERNAL_IMAGE_API_OWNER_EMAIL;
|
|
|
|
const existing = await db
|
|
.select({ id: user.id })
|
|
.from(user)
|
|
.where(sql`lower(${user.email}) = ${normalizedEmail}`)
|
|
.limit(1);
|
|
|
|
if (existing[0]?.id) {
|
|
return existing[0].id;
|
|
}
|
|
|
|
const [created] = await db
|
|
.insert(user)
|
|
.values({
|
|
email: normalizedEmail,
|
|
password: generateDummyPassword(),
|
|
role: "guest",
|
|
name: "第三方生图 API",
|
|
isActive: true,
|
|
})
|
|
.returning({ id: user.id });
|
|
|
|
if (!created?.id) {
|
|
throw new Error("创建第三方 API 挂载用户失败");
|
|
}
|
|
|
|
return created.id;
|
|
}
|
|
|
|
export async function generateImageWithModel(params: {
|
|
modelDbId: string;
|
|
prompt: string;
|
|
negativePrompt?: string | null;
|
|
imageCount: number;
|
|
aspectRatio?: string | null;
|
|
imageSize?: string | null;
|
|
referenceImageUrls?: string[];
|
|
}) {
|
|
const languageModel = await getLanguageModelById(params.modelDbId);
|
|
if (!languageModel || typeof (languageModel as any).doGenerate !== "function") {
|
|
throw new Error("当前模型暂不支持第三方生图调用");
|
|
}
|
|
|
|
const composedPrompt = [
|
|
normalizeText(params.prompt),
|
|
normalizeText(params.negativePrompt)
|
|
? `Negative prompt:\n${normalizeText(params.negativePrompt)}`
|
|
: "",
|
|
]
|
|
.filter(Boolean)
|
|
.join("\n\n");
|
|
|
|
const result = await (languageModel as any).doGenerate({
|
|
messages: [
|
|
{
|
|
role: "user",
|
|
content: [
|
|
{ type: "text", text: composedPrompt },
|
|
...(params.referenceImageUrls || []).map((imageUrl) => ({
|
|
type: "image",
|
|
image: imageUrl,
|
|
})),
|
|
],
|
|
},
|
|
],
|
|
imageParams: {
|
|
imageCount: params.imageCount,
|
|
aspectRatio: normalizeText(params.aspectRatio) || undefined,
|
|
imageSize: normalizeText(params.imageSize) || undefined,
|
|
},
|
|
});
|
|
|
|
if (result?.finishReason === "error") {
|
|
throw new Error(result.text || "图片生成失败");
|
|
}
|
|
|
|
const text = normalizeText(result?.text);
|
|
if (!text) {
|
|
throw new Error("模型未返回生成图片");
|
|
}
|
|
|
|
return text;
|
|
}
|
|
|
|
export async function generateImagesWithFailover(params: {
|
|
candidates: ImageModelEntry[];
|
|
prompt: string;
|
|
negativePrompt?: string | null;
|
|
imageCount: number;
|
|
aspectRatio?: string | null;
|
|
imageSize?: string | null;
|
|
referenceImageUrls?: string[];
|
|
}): Promise<{
|
|
markdown: string;
|
|
images: ExternalImageApiGeneratedImage[];
|
|
usedModel: ImageModelEntry;
|
|
attempts: ExternalImageApiGenerateAttempt[];
|
|
}> {
|
|
const attempts: ExternalImageApiGenerateAttempt[] = [];
|
|
|
|
for (const candidate of params.candidates) {
|
|
try {
|
|
const markdown = await generateImageWithModel({
|
|
modelDbId: candidate.model.id,
|
|
prompt: params.prompt,
|
|
negativePrompt: params.negativePrompt || null,
|
|
imageCount: params.imageCount,
|
|
aspectRatio: params.aspectRatio || null,
|
|
imageSize: params.imageSize || null,
|
|
referenceImageUrls: params.referenceImageUrls || [],
|
|
});
|
|
|
|
const hostedMarkdown = await ensureHostedImageMarkdown(
|
|
markdown,
|
|
params.imageSize || "2K",
|
|
);
|
|
const images = extractGeneratedImagesFromMarkdown(hostedMarkdown);
|
|
|
|
if (!images.length) {
|
|
throw new Error("模型返回结果里未解析到图片");
|
|
}
|
|
|
|
attempts.push({
|
|
modelId: candidate.model.id,
|
|
modelRuntimeId: candidate.model.modelId,
|
|
displayName: candidate.model.displayName,
|
|
success: true,
|
|
});
|
|
|
|
return {
|
|
markdown: hostedMarkdown,
|
|
images,
|
|
usedModel: candidate,
|
|
attempts,
|
|
};
|
|
} catch (error) {
|
|
attempts.push({
|
|
modelId: candidate.model.id,
|
|
modelRuntimeId: candidate.model.modelId,
|
|
displayName: candidate.model.displayName,
|
|
success: false,
|
|
error: error instanceof Error ? error.message : "图片生成失败",
|
|
});
|
|
}
|
|
}
|
|
|
|
throw new ExternalImageApiGenerationError(
|
|
attempts,
|
|
"外部生图服务暂无可用模型返回，请稍后重试",
|
|
);
|
|
}
|
|
|
|
export async function ensureHostedReferenceImageUrls(imageUrls: string[]) {
|
|
const deduped = imageUrls
|
|
.map((item) => normalizeText(item))
|
|
.filter(Boolean)
|
|
.filter((item, index, list) => list.indexOf(item) === index);
|
|
|
|
return Promise.all(deduped.map((url) => ensureHostedImageUrl(url)));
|
|
}
|
|
|
|
export async function uploadExternalImageApiFile(file: File) {
|
|
if (!(file instanceof File)) {
|
|
throw new Error("无效的图片文件");
|
|
}
|
|
|
|
const arrayBuffer = await file.arrayBuffer();
|
|
if (!arrayBuffer.byteLength) {
|
|
throw new Error("上传的图片文件为空");
|
|
}
|
|
|
|
const filename = sanitizeUploadFilename(file.name, "external-reference");
|
|
const uploaded = await uploadFile(filename, arrayBuffer);
|
|
return uploaded.url;
|
|
}
|
|
|
|
export function buildExternalImageApiKeyFingerprint(
|
|
accessKey: ExternalImageApiAccessKey | null,
|
|
) {
|
|
if (!accessKey?.apiKey) return null;
|
|
return createHash("sha256").update(accessKey.apiKey).digest("hex").slice(0, 16);
|
|
}
|
|
|
|
export function getRequesterIp(request: Request) {
|
|
const forwarded = normalizeText(request.headers.get("x-forwarded-for"));
|
|
if (forwarded) {
|
|
return forwarded.split(",")[0]?.trim() || null;
|
|
}
|
|
return normalizeText(request.headers.get("x-real-ip")) || null;
|
|
}
|