1090 lines
35 KiB
TypeScript
1090 lines
35 KiB
TypeScript
import "server-only";
|
||
|
||
import sharp from "sharp";
|
||
import { getEnabledAiModelsWithProvider } from "@/lib/db/queries";
|
||
import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service";
|
||
import { createGeminiImage } from "@/lib/ai/providers/gemini-image";
|
||
import { createAlibabaImage } from "@/lib/ai/providers/alibaba-image";
|
||
import { createBflImage } from "@/lib/ai/providers/bfl-image";
|
||
import { createRuxaImage } from "@/lib/ai/providers/ruxa-image";
|
||
import { createFalImage } from "@/lib/ai/providers/fal-image";
|
||
import { createOpenAIImage } from "@/lib/ai/providers/openai-image";
|
||
import { createStepfunImage } from "@/lib/ai/providers/stepfun-image";
|
||
import { createMidjourney } from "@/lib/ai/providers/midjourney";
|
||
import { createRunway } from "@/lib/ai/providers/runway";
|
||
import { createKrea } from "@/lib/ai/providers/krea";
|
||
import { createVolcanoVision } from "@/lib/ai/providers/volcano-vision";
|
||
import { resolveModelRuntimeId } from "@/lib/ai/model-runtime";
|
||
import { filterReachableRemoteUrls } from "@/lib/net/url-access";
|
||
import {
|
||
acquireCanvasImageModelSlot,
|
||
getCanvasImageModelQueueStatus,
|
||
type CanvasImageModelQueueStatus,
|
||
normalizeCanvasImageModelKey,
|
||
recordCanvasImageModelExecutionDuration,
|
||
} from "./image-concurrency";
|
||
import {
|
||
recordAiModelHealthFailure,
|
||
recordAiModelHealthSuccess,
|
||
shouldFailoverAiModelError,
|
||
} from "@/lib/ai/runtime/model-health";
|
||
import { uploadFile } from "@/lib/storage/service";
|
||
|
||
export interface CanvasLocalEditCompositeRegion {
|
||
x: number;
|
||
y: number;
|
||
width: number;
|
||
height: number;
|
||
}
|
||
|
||
export interface CanvasLocalEditCompositeConfig {
|
||
sourceImageUrl: string;
|
||
regions: CanvasLocalEditCompositeRegion[];
|
||
}
|
||
|
||
export interface CanvasGenerateImageRequest {
|
||
prompt: string;
|
||
negativePrompt?: string;
|
||
model?: string;
|
||
aspectRatio?: string;
|
||
imageSize?: string;
|
||
imageCount?: number;
|
||
promptExtend?: boolean;
|
||
referenceImage?: string;
|
||
referenceImages?: string[];
|
||
billingFeature?: "multi-view" | "repaint";
|
||
ownerKey?: string;
|
||
debugLabel?: string;
|
||
debugMeta?: Record<string, unknown>;
|
||
localEditComposite?: CanvasLocalEditCompositeConfig;
|
||
allowFailover?: boolean;
|
||
internalFailoverTriedModelRefs?: string[];
|
||
onQueueStatusChange?: (
|
||
status: CanvasImageModelQueueStatus & {
|
||
modelName?: string | null;
|
||
modelRef?: string | null;
|
||
}
|
||
) => void | Promise<void>;
|
||
}
|
||
|
||
export interface ParsedImage {
|
||
thumbnailUrl: string;
|
||
originalUrl: string;
|
||
}
|
||
|
||
export interface CanvasGenerateImageResult {
|
||
images: ParsedImage[];
|
||
model: string;
|
||
modelRef: string;
|
||
prompt: string;
|
||
}
|
||
|
||
const SUPPORTED_CANVAS_PROVIDER_TYPES = new Set([
|
||
"gemini",
|
||
"grsai-gpt-image",
|
||
"alibaba-image",
|
||
"bfl-image",
|
||
"ruxa-image",
|
||
"fal-image",
|
||
"openai-image",
|
||
"stepfun-image",
|
||
"midjourney",
|
||
"runway",
|
||
"krea",
|
||
"volcano-vision",
|
||
]);
|
||
|
||
function summarizeReferenceImageHosts(urls: string[]) {
|
||
return Array.from(
|
||
new Set(
|
||
urls
|
||
.map((value) => {
|
||
try {
|
||
return new URL(value).host;
|
||
} catch {
|
||
return "";
|
||
}
|
||
})
|
||
.filter(Boolean)
|
||
)
|
||
);
|
||
}
|
||
|
||
function parseImagesFromMarkdown(markdown: string): ParsedImage[] {
|
||
const imageRegex = /!\[([^\]]*)\]\(([^)]+)\)/g;
|
||
const matches = [...markdown.matchAll(imageRegex)];
|
||
const parsed: ParsedImage[] = [];
|
||
|
||
for (const match of matches) {
|
||
const altText = match[1] || "";
|
||
const thumbnailUrl = match[2];
|
||
let originalUrl = thumbnailUrl;
|
||
|
||
if (altText.includes("thumb|||")) {
|
||
const content = altText.substring(altText.indexOf("thumb|||") + 8);
|
||
const parts = content.split("|||");
|
||
originalUrl = parts[0] || thumbnailUrl;
|
||
} else if (altText.includes("thumb$$")) {
|
||
const content = altText.substring(altText.indexOf("thumb$$") + 7);
|
||
const parts = content.split("$$");
|
||
originalUrl = parts[0] || thumbnailUrl;
|
||
} else if (altText.includes("thumb|")) {
|
||
const content = altText.substring(altText.indexOf("thumb|") + 6);
|
||
const parts = content.split("|");
|
||
originalUrl = parts[0] || thumbnailUrl;
|
||
}
|
||
|
||
parsed.push({ thumbnailUrl, originalUrl });
|
||
}
|
||
|
||
return parsed;
|
||
}
|
||
|
||
function bufferToArrayBuffer(buffer: Buffer): ArrayBuffer {
|
||
return buffer.buffer.slice(
|
||
buffer.byteOffset,
|
||
buffer.byteOffset + buffer.byteLength
|
||
) as ArrayBuffer;
|
||
}
|
||
|
||
function clampRatio(value: number) {
|
||
if (!Number.isFinite(value)) return 0;
|
||
return Math.min(1, Math.max(0, value));
|
||
}
|
||
|
||
function normalizeLocalEditCompositeConfig(
|
||
value: CanvasGenerateImageRequest["localEditComposite"]
|
||
): CanvasLocalEditCompositeConfig | null {
|
||
if (!value || typeof value !== "object") return null;
|
||
const sourceImageUrl = String(value.sourceImageUrl || "").trim();
|
||
if (!/^https?:\/\//i.test(sourceImageUrl)) return null;
|
||
if (!Array.isArray(value.regions) || value.regions.length === 0) return null;
|
||
|
||
const regions = value.regions
|
||
.map((region) => {
|
||
const x = clampRatio(Number(region?.x));
|
||
const y = clampRatio(Number(region?.y));
|
||
const width = clampRatio(Number(region?.width));
|
||
const height = clampRatio(Number(region?.height));
|
||
return {
|
||
x,
|
||
y,
|
||
width: Math.min(width, Math.max(0, 1 - x)),
|
||
height: Math.min(height, Math.max(0, 1 - y)),
|
||
};
|
||
})
|
||
.filter((region) => region.width > 0 && region.height > 0);
|
||
|
||
return regions.length > 0 ? { sourceImageUrl, regions } : null;
|
||
}
|
||
|
||
function buildLocalEditCompositeMaskSvg(params: {
|
||
width: number;
|
||
height: number;
|
||
regions: CanvasLocalEditCompositeRegion[];
|
||
}) {
|
||
const { width, height, regions } = params;
|
||
const rects = regions
|
||
.map((region) => {
|
||
const left = Math.max(0, Math.round(region.x * width));
|
||
const top = Math.max(0, Math.round(region.y * height));
|
||
const rectWidth = Math.min(
|
||
width - left,
|
||
Math.max(1, Math.round(region.width * width))
|
||
);
|
||
const rectHeight = Math.min(
|
||
height - top,
|
||
Math.max(1, Math.round(region.height * height))
|
||
);
|
||
return `<rect x="${left}" y="${top}" width="${rectWidth}" height="${rectHeight}" fill="#ffffff" />`;
|
||
})
|
||
.join("");
|
||
|
||
return Buffer.from(`
|
||
<svg width="${width}" height="${height}" viewBox="0 0 ${width} ${height}" xmlns="http://www.w3.org/2000/svg">
|
||
<rect x="0" y="0" width="${width}" height="${height}" fill="rgba(0,0,0,0)" />
|
||
${rects}
|
||
</svg>
|
||
`);
|
||
}
|
||
|
||
async function applyLocalEditCompositeToImages(params: {
|
||
images: ParsedImage[];
|
||
composite: CanvasLocalEditCompositeConfig | null;
|
||
debugLabel: string;
|
||
}) {
|
||
const { images, composite, debugLabel } = params;
|
||
if (!composite || images.length === 0) return images;
|
||
|
||
const sourceBuffer = await sharp(
|
||
await fetchImageBufferFromUrl(composite.sourceImageUrl),
|
||
{ failOn: "none" }
|
||
)
|
||
.rotate()
|
||
.png({ compressionLevel: 6 })
|
||
.toBuffer();
|
||
const sourceMetadata = await sharp(sourceBuffer, { failOn: "none" }).metadata();
|
||
const sourceWidth = Number(sourceMetadata.width || 0);
|
||
const sourceHeight = Number(sourceMetadata.height || 0);
|
||
if (sourceWidth <= 0 || sourceHeight <= 0) {
|
||
throw new Error("局部编辑结果合成失败：无法识别原图尺寸");
|
||
}
|
||
|
||
const maskSvg = buildLocalEditCompositeMaskSvg({
|
||
height: sourceHeight,
|
||
regions: composite.regions,
|
||
width: sourceWidth,
|
||
});
|
||
|
||
const mergedImages: ParsedImage[] = [];
|
||
for (const [index, image] of images.entries()) {
|
||
const generatedUrl = image.originalUrl || image.thumbnailUrl;
|
||
const generatedBuffer = await fetchImageBufferFromUrl(generatedUrl);
|
||
const overlayBuffer = await sharp(generatedBuffer, { failOn: "none" })
|
||
.rotate()
|
||
.resize(sourceWidth, sourceHeight, { fit: "cover", position: "center" })
|
||
.ensureAlpha()
|
||
.composite([{ input: maskSvg, blend: "dest-in" }])
|
||
.png({ compressionLevel: 6 })
|
||
.toBuffer();
|
||
const mergedBuffer = await sharp(sourceBuffer, { failOn: "none" })
|
||
.ensureAlpha()
|
||
.composite([{ input: overlayBuffer, left: 0, top: 0 }])
|
||
.png({ compressionLevel: 6 })
|
||
.toBuffer();
|
||
const thumbnailBuffer = await sharp(mergedBuffer, { failOn: "none" })
|
||
.resize(512, 512, { fit: "inside", withoutEnlargement: true })
|
||
.png({ compressionLevel: 6 })
|
||
.toBuffer();
|
||
const suffix = `${Date.now()}-${index + 1}-${Math.random().toString(36).slice(2, 8)}`;
|
||
const [originalUpload, thumbnailUpload] = await Promise.all([
|
||
uploadFile(
|
||
`canvas-local-edit-composited-${suffix}.png`,
|
||
bufferToArrayBuffer(mergedBuffer)
|
||
),
|
||
uploadFile(
|
||
`canvas-local-edit-composited-thumb-${suffix}.png`,
|
||
bufferToArrayBuffer(thumbnailBuffer)
|
||
),
|
||
]);
|
||
mergedImages.push({
|
||
originalUrl: originalUpload.url,
|
||
thumbnailUrl: thumbnailUpload.url,
|
||
});
|
||
}
|
||
|
||
console.info("[Canvas Generate Image] Local edit composite applied:", {
|
||
debugLabel,
|
||
imageCount: mergedImages.length,
|
||
regionCount: composite.regions.length,
|
||
sourceHeight,
|
||
sourceWidth,
|
||
});
|
||
|
||
return mergedImages;
|
||
}
|
||
|
||
function getProviderBatchLimit(providerType: string) {
|
||
switch (providerType) {
|
||
case "ruxa-image":
|
||
case "bfl-image":
|
||
case "openai-image":
|
||
case "stepfun-image":
|
||
case "midjourney":
|
||
case "runway":
|
||
return 1;
|
||
case "krea":
|
||
return 4;
|
||
case "gemini":
|
||
case "grsai-gpt-image":
|
||
return 2;
|
||
case "alibaba-image":
|
||
case "fal-image":
|
||
return 6;
|
||
default:
|
||
return 2;
|
||
}
|
||
}
|
||
|
||
type CanvasModelWithProvider = Awaited<
|
||
ReturnType<typeof getEnabledAiModelsWithProvider>
|
||
>[number];
|
||
|
||
type CanvasImageSizeValue = "1K" | "2K" | "4K";
|
||
const CANVAS_FAILOVER_SAME_NAME_THRESHOLD = 0.5;
|
||
|
||
interface CanvasImageGenConfig {
|
||
maxImages?: number;
|
||
supports1K?: boolean;
|
||
supports2K?: boolean;
|
||
supports4K?: boolean;
|
||
}
|
||
|
||
function normalizeCanvasImageSizeValue(
|
||
value?: string,
|
||
fallback: CanvasImageSizeValue = "2K"
|
||
): CanvasImageSizeValue {
|
||
const normalized = String(value || "").trim().toUpperCase();
|
||
if (normalized === "1K" || normalized === "2K" || normalized === "4K") {
|
||
return normalized;
|
||
}
|
||
return fallback;
|
||
}
|
||
|
||
function isCanvasImageSizeSupported(
|
||
config: CanvasImageGenConfig | null | undefined,
|
||
size: CanvasImageSizeValue
|
||
) {
|
||
if (!config) return true;
|
||
if (size === "1K") return config.supports1K !== false;
|
||
if (size === "2K") return config.supports2K !== false;
|
||
if (size === "4K") return config.supports4K !== false;
|
||
return true;
|
||
}
|
||
|
||
function pickSupportedCanvasImageSize(
|
||
config: CanvasImageGenConfig | null | undefined,
|
||
requestedSize: CanvasImageSizeValue
|
||
): CanvasImageSizeValue {
|
||
if (isCanvasImageSizeSupported(config, requestedSize)) return requestedSize;
|
||
|
||
const preferredFallbacks: Record<CanvasImageSizeValue, CanvasImageSizeValue[]> = {
|
||
"4K": ["2K", "1K"],
|
||
"2K": ["1K"],
|
||
"1K": ["2K", "4K"],
|
||
};
|
||
|
||
return (
|
||
preferredFallbacks[requestedSize].find((candidate) =>
|
||
isCanvasImageSizeSupported(config, candidate)
|
||
) ||
|
||
(["1K", "2K", "4K"] as const).find((candidate) =>
|
||
isCanvasImageSizeSupported(config, candidate)
|
||
) ||
|
||
"2K"
|
||
);
|
||
}
|
||
|
||
function normalizeCanvasImageParamsByModel(params: {
|
||
imageCount: number;
|
||
imageSize: string;
|
||
imageGenConfig: CanvasImageGenConfig | null | undefined;
|
||
}) {
|
||
const maxImages =
|
||
typeof params.imageGenConfig?.maxImages === "number" &&
|
||
Number.isFinite(params.imageGenConfig.maxImages)
|
||
? Math.max(1, Math.floor(params.imageGenConfig.maxImages))
|
||
: 12;
|
||
const imageCount = Math.max(1, Math.min(maxImages, Math.floor(params.imageCount)));
|
||
|
||
const requestedSize = normalizeCanvasImageSizeValue(params.imageSize);
|
||
const imageSize = pickSupportedCanvasImageSize(params.imageGenConfig, requestedSize);
|
||
|
||
return { imageCount, imageSize };
|
||
}
|
||
|
||
function getCanvasModelRefKeys(item: CanvasModelWithProvider) {
|
||
return [
|
||
String(item.model.id || "").trim(),
|
||
String(item.model.modelId || "").trim(),
|
||
].filter(Boolean);
|
||
}
|
||
|
||
function normalizeModelNameForSimilarity(value: unknown) {
|
||
return String(value || "")
|
||
.trim()
|
||
.toLowerCase()
|
||
.replace(/[「」『』【】[\]()（）{}]/g, " ")
|
||
.replace(/[_/|:：,，.。+]+/g, "-")
|
||
.replace(/\s+/g, " ");
|
||
}
|
||
|
||
function tokenizeModelNameForSimilarity(value: unknown) {
|
||
const normalized = normalizeModelNameForSimilarity(value);
|
||
const compact = normalized.replace(/[^a-z0-9\u4e00-\u9fa5]+/g, "");
|
||
const tokens = new Set(
|
||
normalized
|
||
.split(/[^a-z0-9\u4e00-\u9fa5]+/g)
|
||
.map((item) => item.trim())
|
||
.filter((item) => item.length >= 2)
|
||
);
|
||
|
||
if (compact.length >= 2) {
|
||
tokens.add(compact);
|
||
for (let index = 0; index < compact.length - 1; index += 1) {
|
||
tokens.add(compact.slice(index, index + 2));
|
||
}
|
||
}
|
||
|
||
return tokens;
|
||
}
|
||
|
||
function scoreModelNameSimilarity(
|
||
source: CanvasModelWithProvider,
|
||
candidate: CanvasModelWithProvider
|
||
) {
|
||
const sourceNames = [
|
||
source.model.displayName,
|
||
source.model.modelId,
|
||
].filter(Boolean);
|
||
const candidateNames = [
|
||
candidate.model.displayName,
|
||
candidate.model.modelId,
|
||
].filter(Boolean);
|
||
|
||
let bestScore = 0;
|
||
for (const sourceName of sourceNames) {
|
||
const sourceTokens = tokenizeModelNameForSimilarity(sourceName);
|
||
if (sourceTokens.size === 0) continue;
|
||
for (const candidateName of candidateNames) {
|
||
const candidateTokens = tokenizeModelNameForSimilarity(candidateName);
|
||
if (candidateTokens.size === 0) continue;
|
||
let overlap = 0;
|
||
sourceTokens.forEach((token) => {
|
||
if (candidateTokens.has(token)) overlap += 1;
|
||
});
|
||
const score = overlap / Math.max(sourceTokens.size, candidateTokens.size);
|
||
bestScore = Math.max(bestScore, score);
|
||
}
|
||
}
|
||
|
||
return bestScore;
|
||
}
|
||
|
||
function findCanvasFailoverModel(params: {
|
||
selectedModel: CanvasModelWithProvider;
|
||
imageModels: CanvasModelWithProvider[];
|
||
failedModelRefs: string[];
|
||
}) {
|
||
const failedRefs = new Set(
|
||
params.failedModelRefs.map((item) => String(item || "").trim()).filter(Boolean)
|
||
);
|
||
const selectedFallbackRefs = [
|
||
params.selectedModel.model.fallbackModelId1,
|
||
params.selectedModel.model.fallbackModelId2,
|
||
]
|
||
.map((item) => String(item || "").trim())
|
||
.filter(Boolean);
|
||
|
||
const isUsableFallback = (item: CanvasModelWithProvider) => {
|
||
if (!item.model.isEnabled || item.model.isSubModel || item.model.excludeFromFailover) {
|
||
return false;
|
||
}
|
||
if (!item.model.supportsImageGen) return false;
|
||
if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(item.provider.type)) return false;
|
||
return getCanvasModelRefKeys(item).every((key) => !failedRefs.has(key));
|
||
};
|
||
|
||
for (const fallbackRef of selectedFallbackRefs) {
|
||
const configured = params.imageModels.find(
|
||
(item) =>
|
||
isUsableFallback(item) &&
|
||
(item.model.id === fallbackRef || item.model.modelId === fallbackRef)
|
||
);
|
||
if (configured) return configured;
|
||
}
|
||
|
||
const usableModels = params.imageModels.filter(isUsableFallback);
|
||
const scoredModels = usableModels
|
||
.map((item, index) => ({
|
||
item,
|
||
index,
|
||
nameSimilarity: scoreModelNameSimilarity(params.selectedModel, item),
|
||
sameProviderType: item.provider.type === params.selectedModel.provider.type,
|
||
}))
|
||
.sort((left, right) => {
|
||
const leftSameName =
|
||
left.nameSimilarity >= CANVAS_FAILOVER_SAME_NAME_THRESHOLD;
|
||
const rightSameName =
|
||
right.nameSimilarity >= CANVAS_FAILOVER_SAME_NAME_THRESHOLD;
|
||
if (leftSameName !== rightSameName) return leftSameName ? -1 : 1;
|
||
if (left.sameProviderType !== right.sameProviderType) {
|
||
return left.sameProviderType ? -1 : 1;
|
||
}
|
||
if (right.nameSimilarity !== left.nameSimilarity) {
|
||
return right.nameSimilarity - left.nameSimilarity;
|
||
}
|
||
return left.index - right.index;
|
||
});
|
||
|
||
return scoredModels[0]?.item || null;
|
||
}
|
||
|
||
export async function generateCanvasImage(
|
||
request: CanvasGenerateImageRequest
|
||
): Promise<CanvasGenerateImageResult> {
|
||
const startedAt = Date.now();
|
||
const prompt = String(request.prompt || "").trim();
|
||
if (!prompt) {
|
||
throw new Error("请输入提示词");
|
||
}
|
||
|
||
const negativePrompt =
|
||
typeof request.negativePrompt === "string" ? request.negativePrompt : undefined;
|
||
const model = typeof request.model === "string" ? request.model : undefined;
|
||
const aspectRatio =
|
||
typeof request.aspectRatio === "string" && request.aspectRatio.trim()
|
||
? request.aspectRatio.trim()
|
||
: "auto";
|
||
const requestedImageSize =
|
||
typeof request.imageSize === "string" && request.imageSize.trim()
|
||
? request.imageSize.trim()
|
||
: "2K";
|
||
const requestedImageCount =
|
||
typeof request.imageCount === "number" && Number.isFinite(request.imageCount)
|
||
? Math.max(1, Math.min(12, Math.floor(request.imageCount)))
|
||
: 1;
|
||
const promptExtend =
|
||
typeof request.promptExtend === "boolean" ? request.promptExtend : undefined;
|
||
const ownerKey =
|
||
typeof request.ownerKey === "string" && request.ownerKey.trim()
|
||
? request.ownerKey.trim()
|
||
: undefined;
|
||
const referenceImage =
|
||
typeof request.referenceImage === "string" ? request.referenceImage : undefined;
|
||
const referenceImages = Array.isArray(request.referenceImages)
|
||
? request.referenceImages
|
||
: [];
|
||
const debugLabel = String(request.debugLabel || "").trim() || "canvas-image";
|
||
const debugMeta =
|
||
request.debugMeta && typeof request.debugMeta === "object"
|
||
? request.debugMeta
|
||
: undefined;
|
||
const localEditComposite = normalizeLocalEditCompositeConfig(
|
||
request.localEditComposite
|
||
);
|
||
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const imageModels = allModels.filter((m) => m.model.supportsImageGen);
|
||
|
||
if (!imageModels || imageModels.length === 0) {
|
||
throw new Error("暂无可用的图片生成模型");
|
||
}
|
||
|
||
let selectedModel = model
|
||
? imageModels.find((m) => m.model.modelId === model || m.model.id === model)
|
||
: imageModels.find((m) => m.model.isDefault) || imageModels[0];
|
||
|
||
if (!selectedModel) {
|
||
selectedModel = imageModels[0];
|
||
}
|
||
|
||
let provider = selectedModel.provider;
|
||
|
||
if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(provider.type)) {
|
||
if (request.allowFailover === false) {
|
||
throw new Error(
|
||
"当前图片模型类型暂不支持画布生成，请在后台选择 Gemini/Grsai GPT Image/火山 Seedream/阿里/BFL/RUXA/FAL/OpenAI/StepFun/Runway/Krea 图片模型。"
|
||
);
|
||
}
|
||
|
||
let fallbackModel = selectedModel;
|
||
let fallbackProvider = provider;
|
||
for (const candidate of imageModels) {
|
||
const candidateProvider = candidate.provider;
|
||
if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(candidateProvider.type)) continue;
|
||
fallbackModel = candidate;
|
||
fallbackProvider = candidateProvider;
|
||
break;
|
||
}
|
||
if (!SUPPORTED_CANVAS_PROVIDER_TYPES.has(fallbackProvider.type)) {
|
||
throw new Error(
|
||
"当前图片模型类型暂不支持画布生成，请在后台选择 Gemini/Grsai GPT Image/火山 Seedream/阿里/BFL/RUXA/FAL/OpenAI/StepFun/Runway/Krea 图片模型。"
|
||
);
|
||
}
|
||
if (fallbackModel.model.id !== selectedModel.model.id) {
|
||
console.warn("[Canvas Generate Image] Fallback to supported image model:", {
|
||
requestedModel: selectedModel.model.modelId || selectedModel.model.id,
|
||
fallbackModel: fallbackModel.model.modelId || fallbackModel.model.id,
|
||
requestedProviderType: provider.type,
|
||
fallbackProviderType: fallbackProvider.type,
|
||
});
|
||
}
|
||
selectedModel = fallbackModel;
|
||
provider = fallbackProvider;
|
||
}
|
||
|
||
const normalizedImageParams = normalizeCanvasImageParamsByModel({
|
||
imageCount: requestedImageCount,
|
||
imageSize: requestedImageSize,
|
||
imageGenConfig: selectedModel.model.imageGenConfig as CanvasImageGenConfig | null | undefined,
|
||
});
|
||
const imageSize = normalizedImageParams.imageSize;
|
||
const imageCount = normalizedImageParams.imageCount;
|
||
if (imageSize !== requestedImageSize || imageCount !== requestedImageCount) {
|
||
console.info("[Canvas Generate Image] Normalized model image params:", {
|
||
debugLabel,
|
||
requestedImageSize,
|
||
normalizedImageSize: imageSize,
|
||
requestedImageCount,
|
||
normalizedImageCount: imageCount,
|
||
selectedModelRef: selectedModel.model.id,
|
||
selectedModelId: selectedModel.model.modelId,
|
||
providerType: provider.type,
|
||
});
|
||
}
|
||
|
||
const normalizedReferenceImages = Array.from(
|
||
new Set(
|
||
[
|
||
...referenceImages,
|
||
...(referenceImage ? [referenceImage] : []),
|
||
]
|
||
.map((item) => String(item || "").trim())
|
||
.filter((url) => /^https?:\/\//i.test(url))
|
||
)
|
||
).slice(0, 12);
|
||
const { kept: reachableReferenceImages, dropped: unreachableReferenceImages } =
|
||
await filterReachableRemoteUrls(normalizedReferenceImages, { timeoutMs: 3000 });
|
||
const effectiveReferenceImages =
|
||
reachableReferenceImages.length > 0
|
||
? reachableReferenceImages
|
||
: normalizedReferenceImages;
|
||
if (unreachableReferenceImages.length > 0) {
|
||
console.warn("[Canvas Generate Image] Local reachability check failed for some reference images:", {
|
||
count: unreachableReferenceImages.length,
|
||
urls: unreachableReferenceImages.slice(0, 6),
|
||
hosts: summarizeReferenceImageHosts(unreachableReferenceImages),
|
||
});
|
||
}
|
||
if (normalizedReferenceImages.length > 0 && reachableReferenceImages.length === 0) {
|
||
console.warn(
|
||
"[Canvas Generate Image] All reference images failed local reachability check; proceeding with original URLs.",
|
||
{
|
||
count: normalizedReferenceImages.length,
|
||
hosts: summarizeReferenceImageHosts(normalizedReferenceImages),
|
||
debugLabel,
|
||
}
|
||
);
|
||
}
|
||
|
||
console.info("[Canvas Generate Image] Started:", {
|
||
debugLabel,
|
||
promptLength: prompt.length,
|
||
aspectRatio,
|
||
imageSize,
|
||
imageCount,
|
||
requestedReferenceImageCount: normalizedReferenceImages.length,
|
||
reachableReferenceImageCount: reachableReferenceImages.length,
|
||
effectiveReferenceImageCount: effectiveReferenceImages.length,
|
||
referenceImageHosts: summarizeReferenceImageHosts(effectiveReferenceImages),
|
||
selectedModelRef: selectedModel.model.id,
|
||
selectedModelId: selectedModel.model.modelId,
|
||
providerType: provider.type,
|
||
...debugMeta,
|
||
hasLocalEditComposite: Boolean(localEditComposite),
|
||
});
|
||
|
||
const messages = [
|
||
{
|
||
role: "user" as const,
|
||
content:
|
||
effectiveReferenceImages.length > 0
|
||
? [
|
||
{ type: "text" as const, text: prompt },
|
||
...effectiveReferenceImages.map((url) => ({
|
||
type: "image_url" as const,
|
||
image_url: { url },
|
||
})),
|
||
]
|
||
: prompt,
|
||
},
|
||
];
|
||
|
||
const imageParams = {
|
||
aspectRatio,
|
||
imageSize,
|
||
imageCount: 1,
|
||
negativePrompt,
|
||
promptExtend,
|
||
};
|
||
|
||
let executeGenerateBatch:
|
||
| ((batchImageCount: number) => Promise<string>)
|
||
| null = null;
|
||
|
||
if (provider.type === "gemini" || provider.type === "grsai-gpt-image") {
|
||
const geminiProvider = createGeminiImage({
|
||
apiKey: provider.apiKey,
|
||
baseURL: provider.baseUrl,
|
||
});
|
||
const modelInstance = geminiProvider.chat(selectedModel.model.modelId);
|
||
executeGenerateBatch = async (batchImageCount) => {
|
||
const result = await modelInstance.doGenerate({
|
||
messages,
|
||
imageParams: {
|
||
...imageParams,
|
||
imageCount: batchImageCount,
|
||
},
|
||
});
|
||
return result.text || "";
|
||
};
|
||
} else if (provider.type === "alibaba-image") {
|
||
const alibabaProvider = createAlibabaImage({
|
||
apiKey: provider.apiKey,
|
||
baseURL: provider.baseUrl,
|
||
});
|
||
const modelInstance = alibabaProvider.chat(selectedModel.model.modelId);
|
||
executeGenerateBatch = async (batchImageCount) => {
|
||
const result = await modelInstance.doGenerate({
|
||
messages,
|
||
imageParams: {
|
||
...imageParams,
|
||
imageCount: batchImageCount,
|
||
},
|
||
});
|
||
return result.text || "";
|
||
};
|
||
} else if (provider.type === "bfl-image") {
|
||
const bflProvider = createBflImage({
|
||
apiKey: provider.apiKey,
|
||
baseURL: provider.baseUrl,
|
||
});
|
||
const modelInstance = bflProvider.chat(selectedModel.model.modelId);
|
||
executeGenerateBatch = async (_batchImageCount) => {
|
||
const result = await modelInstance.doGenerate({
|
||
messages,
|
||
imageParams,
|
||
});
|
||
return result.text || "";
|
||
};
|
||
} else if (provider.type === "ruxa-image") {
|
||
const ruxaProvider = createRuxaImage({
|
||
apiKey: provider.apiKey,
|
||
baseURL: provider.baseUrl,
|
||
});
|
||
const runtimeModelId = resolveModelRuntimeId(selectedModel.model, {
|
||
providerType: provider.type,
|
||
});
|
||
const modelInstance = ruxaProvider.chat(runtimeModelId);
|
||
executeGenerateBatch = async (batchImageCount) => {
|
||
const result = await modelInstance.doGenerate({
|
||
messages,
|
||
imageParams: {
|
||
...imageParams,
|
||
imageCount: batchImageCount,
|
||
},
|
||
imageGenConfig: selectedModel.model.imageGenConfig || null,
|
||
});
|
||
return result.text || "";
|
||
};
|
||
} else if (provider.type === "fal-image") {
|
||
const falProvider = createFalImage({
|
||
apiKey: provider.apiKey,
|
||
baseURL: provider.baseUrl,
|
||
});
|
||
const modelInstance = falProvider.chat(selectedModel.model.modelId);
|
||
executeGenerateBatch = async (batchImageCount) => {
|
||
const result = await modelInstance.doGenerate({
|
||
messages,
|
||
imageParams: {
|
||
...imageParams,
|
||
imageCount: batchImageCount,
|
||
},
|
||
});
|
||
return result.text || "";
|
||
};
|
||
} else if (provider.type === "openai-image") {
|
||
const openAIImageProvider = createOpenAIImage({
|
||
apiKey: provider.apiKey,
|
||
baseURL: provider.baseUrl,
|
||
});
|
||
const modelInstance = openAIImageProvider.chat(selectedModel.model.modelId);
|
||
executeGenerateBatch = async (batchImageCount) => {
|
||
const result = await modelInstance.doGenerate({
|
||
messages,
|
||
imageParams: {
|
||
...imageParams,
|
||
imageCount: batchImageCount,
|
||
},
|
||
});
|
||
return result.text || "";
|
||
};
|
||
} else if (provider.type === "stepfun-image") {
|
||
const stepfunProvider = createStepfunImage({
|
||
apiKey: provider.apiKey,
|
||
baseURL: provider.baseUrl,
|
||
});
|
||
const modelInstance = stepfunProvider.chat(selectedModel.model.modelId);
|
||
executeGenerateBatch = async (_batchImageCount) => {
|
||
const result = await modelInstance.doGenerate({
|
||
messages,
|
||
imageParams,
|
||
});
|
||
return result.text || "";
|
||
};
|
||
} else if (provider.type === "midjourney") {
|
||
const midjourneyProvider = createMidjourney({
|
||
apiKey: provider.apiKey,
|
||
baseURL: provider.baseUrl,
|
||
});
|
||
const modelInstance = midjourneyProvider.chat(selectedModel.model.modelId);
|
||
executeGenerateBatch = async (batchImageCount) => {
|
||
const result = await modelInstance.doGenerate({
|
||
messages,
|
||
imageParams: {
|
||
...imageParams,
|
||
imageCount: batchImageCount,
|
||
},
|
||
});
|
||
return result.text || "";
|
||
};
|
||
} else if (provider.type === "runway") {
|
||
const runwayProvider = createRunway({
|
||
apiKey: provider.apiKey,
|
||
baseURL: provider.baseUrl,
|
||
});
|
||
const modelInstance = runwayProvider.chat(selectedModel.model.modelId);
|
||
executeGenerateBatch = async (_batchImageCount) => {
|
||
const result = await modelInstance.doGenerate({
|
||
messages,
|
||
imageParams,
|
||
});
|
||
return result.text || "";
|
||
};
|
||
} else if (provider.type === "krea") {
|
||
const kreaProvider = createKrea({
|
||
apiKey: provider.apiKey,
|
||
baseURL: provider.baseUrl,
|
||
});
|
||
const modelInstance = kreaProvider.chat(selectedModel.model.modelId);
|
||
executeGenerateBatch = async (batchImageCount) => {
|
||
const result = await modelInstance.doGenerate({
|
||
messages,
|
||
imageParams: {
|
||
...imageParams,
|
||
imageCount: batchImageCount,
|
||
},
|
||
});
|
||
return result.text || "";
|
||
};
|
||
} else if (provider.type === "volcano-vision") {
|
||
const volcanoVisionProvider = createVolcanoVision({
|
||
apiKey: provider.apiKey,
|
||
baseURL: provider.baseUrl,
|
||
});
|
||
const modelInstance = volcanoVisionProvider.chat(selectedModel.model.modelId);
|
||
executeGenerateBatch = async (batchImageCount) => {
|
||
const result = await modelInstance.doGenerate({
|
||
messages,
|
||
imageParams: {
|
||
...imageParams,
|
||
imageCount: batchImageCount,
|
||
},
|
||
});
|
||
return result.text || "";
|
||
};
|
||
} else {
|
||
throw new Error(
|
||
`不支持的图片生成模型类型: ${provider.type || "unknown"}，请改用 Gemini/Grsai GPT Image/火山 Seedream/阿里/BFL/RUXA/FAL/OpenAI/StepFun/Midjourney/Runway/Krea 图片模型`
|
||
);
|
||
}
|
||
|
||
const providerBatchLimit = getProviderBatchLimit(provider.type);
|
||
const targetImageCount = Math.max(1, imageCount);
|
||
const collectedImages: ParsedImage[] = [];
|
||
const maxBatches = Math.max(1, Math.ceil(targetImageCount / providerBatchLimit));
|
||
const resolvedModelKey = normalizeCanvasImageModelKey(
|
||
selectedModel.model.modelId || selectedModel.model.id || model
|
||
);
|
||
const slot = await acquireCanvasImageModelSlot({
|
||
modelKey: resolvedModelKey,
|
||
ownerKey,
|
||
debugLabel,
|
||
debugMeta: {
|
||
selectedModelId: selectedModel.model.modelId,
|
||
selectedModelRef: selectedModel.model.id,
|
||
providerType: provider.type,
|
||
...debugMeta,
|
||
},
|
||
onQueued: request.onQueueStatusChange
|
||
? async (queueStatus) => {
|
||
await request.onQueueStatusChange?.({
|
||
...queueStatus,
|
||
modelName: selectedModel.model.displayName || null,
|
||
modelRef: selectedModel.model.id || null,
|
||
});
|
||
}
|
||
: undefined,
|
||
});
|
||
|
||
if (slot.waitMs > 0 && request.onQueueStatusChange) {
|
||
const runningStatus = await getCanvasImageModelQueueStatus({
|
||
modelKey: resolvedModelKey,
|
||
});
|
||
await request.onQueueStatusChange({
|
||
...runningStatus,
|
||
status: "running",
|
||
queuePosition: 0,
|
||
estimatedWaitMs: 0,
|
||
modelName: selectedModel.model.displayName || null,
|
||
modelRef: selectedModel.model.id || null,
|
||
});
|
||
}
|
||
|
||
let caughtError: unknown = null;
|
||
|
||
try {
|
||
for (let batchIndex = 0; batchIndex < maxBatches; batchIndex += 1) {
|
||
const remaining = targetImageCount - collectedImages.length;
|
||
if (remaining <= 0) break;
|
||
const batchImageCount = Math.min(providerBatchLimit, remaining);
|
||
const batchStartedAt = Date.now();
|
||
console.info("[Canvas Generate Image] Batch started:", {
|
||
debugLabel,
|
||
batchIndex: batchIndex + 1,
|
||
totalBatches: maxBatches,
|
||
batchImageCount,
|
||
currentCollectedCount: collectedImages.length,
|
||
providerType: provider.type,
|
||
selectedModelId: selectedModel.model.modelId,
|
||
...debugMeta,
|
||
});
|
||
const imageMarkdown = await executeGenerateBatch(batchImageCount);
|
||
const parsedImages = parseImagesFromMarkdown(imageMarkdown);
|
||
console.info("[Canvas Generate Image] Batch completed:", {
|
||
debugLabel,
|
||
batchIndex: batchIndex + 1,
|
||
totalBatches: maxBatches,
|
||
batchImageCount,
|
||
parsedImageCount: parsedImages.length,
|
||
elapsedMs: Date.now() - batchStartedAt,
|
||
...debugMeta,
|
||
});
|
||
if (parsedImages.length === 0) {
|
||
if (collectedImages.length > 0) {
|
||
break;
|
||
}
|
||
throw new Error("图片生成失败");
|
||
}
|
||
collectedImages.push(...parsedImages.slice(0, batchImageCount));
|
||
}
|
||
|
||
if (collectedImages.length === 0) {
|
||
throw new Error("图片生成失败");
|
||
}
|
||
|
||
const finalImages = await applyLocalEditCompositeToImages({
|
||
composite: localEditComposite,
|
||
debugLabel,
|
||
images: collectedImages.slice(0, targetImageCount),
|
||
});
|
||
console.info("[Canvas Generate Image] Completed:", {
|
||
debugLabel,
|
||
elapsedMs: Date.now() - startedAt,
|
||
waitMs: slot.waitMs,
|
||
singleModelConcurrency: slot.maxConcurrent,
|
||
finalImageCount: finalImages.length,
|
||
providerType: provider.type,
|
||
selectedModelRef: selectedModel.model.id,
|
||
selectedModelId: selectedModel.model.modelId,
|
||
...debugMeta,
|
||
});
|
||
|
||
await recordAiModelHealthSuccess({
|
||
model: selectedModel.model,
|
||
}).catch((healthError) => {
|
||
console.warn("[Canvas Generate Image] Record model health success failed:", {
|
||
selectedModelRef: selectedModel.model.id,
|
||
selectedModelId: selectedModel.model.modelId,
|
||
error: healthError instanceof Error ? healthError.message : String(healthError),
|
||
});
|
||
});
|
||
|
||
recordCanvasImageModelExecutionDuration(
|
||
resolvedModelKey,
|
||
Date.now() - startedAt
|
||
);
|
||
|
||
return {
|
||
images: finalImages,
|
||
model: selectedModel.model.displayName,
|
||
modelRef: selectedModel.model.id,
|
||
prompt,
|
||
};
|
||
} catch (error) {
|
||
console.error("[Canvas Generate Image] Failed:", {
|
||
debugLabel,
|
||
elapsedMs: Date.now() - startedAt,
|
||
waitMs: slot.waitMs,
|
||
singleModelConcurrency: slot.maxConcurrent,
|
||
providerType: provider.type,
|
||
selectedModelRef: selectedModel.model.id,
|
||
selectedModelId: selectedModel.model.modelId,
|
||
error: error instanceof Error ? error.message : String(error),
|
||
...debugMeta,
|
||
});
|
||
caughtError = error;
|
||
} finally {
|
||
await slot.release();
|
||
}
|
||
|
||
await recordAiModelHealthFailure({
|
||
model: selectedModel.model,
|
||
provider,
|
||
error: caughtError,
|
||
context: debugLabel,
|
||
elapsedMs: Date.now() - startedAt,
|
||
metadata: {
|
||
aspectRatio,
|
||
imageSize,
|
||
imageCount,
|
||
requestedReferenceImageCount: normalizedReferenceImages.length,
|
||
reachableReferenceImageCount: reachableReferenceImages.length,
|
||
effectiveReferenceImageCount: effectiveReferenceImages.length,
|
||
droppedReferenceImageCount: unreachableReferenceImages.length,
|
||
...debugMeta,
|
||
},
|
||
}).catch((healthError) => {
|
||
console.warn("[Canvas Generate Image] Record model health failure failed:", {
|
||
selectedModelRef: selectedModel.model.id,
|
||
selectedModelId: selectedModel.model.modelId,
|
||
error: healthError instanceof Error ? healthError.message : String(healthError),
|
||
});
|
||
});
|
||
|
||
const failedModelRefs = Array.from(
|
||
new Set([
|
||
...(Array.isArray(request.internalFailoverTriedModelRefs)
|
||
? request.internalFailoverTriedModelRefs
|
||
: []),
|
||
selectedModel.model.id,
|
||
selectedModel.model.modelId,
|
||
].filter(Boolean))
|
||
);
|
||
const allowFailover = request.allowFailover !== false;
|
||
const fallbackModel = allowFailover && shouldFailoverAiModelError(caughtError)
|
||
? findCanvasFailoverModel({
|
||
selectedModel,
|
||
imageModels,
|
||
failedModelRefs,
|
||
})
|
||
: null;
|
||
|
||
if (fallbackModel) {
|
||
console.warn("[Canvas Generate Image] Failover to alternate image model:", {
|
||
debugLabel,
|
||
failedModelRef: selectedModel.model.id,
|
||
failedModelId: selectedModel.model.modelId,
|
||
fallbackModelRef: fallbackModel.model.id,
|
||
fallbackModelId: fallbackModel.model.modelId,
|
||
fallbackProviderType: fallbackModel.provider.type,
|
||
error: caughtError instanceof Error ? caughtError.message : String(caughtError),
|
||
...debugMeta,
|
||
});
|
||
|
||
return await generateCanvasImage({
|
||
...request,
|
||
model: fallbackModel.model.id,
|
||
internalFailoverTriedModelRefs: failedModelRefs,
|
||
debugMeta: {
|
||
...debugMeta,
|
||
failoverFromModelId: selectedModel.model.modelId,
|
||
failoverFromModelRef: selectedModel.model.id,
|
||
},
|
||
});
|
||
}
|
||
|
||
throw caughtError;
|
||
}
|