feat(agent): recover automatic image generation without switching user-pinned models
This commit is contained in:
134
parent
e3a8f0e0a1
commit
28c2a1ceb8
@ -34,3 +34,20 @@ export function agentProviderImageParameters(parameters: AgentImageParameters) {
|
||||
return value !== undefined ? [[key, value]] : [];
|
||||
}));
|
||||
}
|
||||
|
||||
/** Only user controls or an explicit instruction pin a model; the planner's own
|
||||
* modelId and a preset default remain eligible for automatic recovery. */
|
||||
export function resolveUserImageModel(text: string, entries: Array<{ model: { id: string; modelId: string; displayName?: string | null } }>, selectedId?: string) {
|
||||
if (selectedId) return selectedId;
|
||||
const names = entries.flatMap(({ model }) => [model.modelId, model.displayName].filter((name): name is string => Boolean(name && name.length >= 4)).map(name => ({ id: model.id, name })))
|
||||
.sort((a, b) => b.name.length - a.name.length);
|
||||
const matches: Array<{ id: string; index: number; length: number }> = [];
|
||||
for (const { id, name } of names) {
|
||||
const escaped = name.split(/[\s_-]+/).map(part => part.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("[\\s_-]*");
|
||||
const pattern = new RegExp(`(?:使用|选择|指定|通过|用)\\s*(?:模型\\s*)?${escaped}(?![a-z0-9.-])`, "ig");
|
||||
for (const match of text.matchAll(pattern)) {
|
||||
if (!/(?:不要|不|别|禁止|无需)\s*$/.test(text.slice(Math.max(0, match.index! - 8), match.index!))) matches.push({ id, index: match.index!, length: name.length });
|
||||
}
|
||||
}
|
||||
return matches.sort((a, b) => b.index - a.index || b.length - a.length)[0]?.id;
|
||||
}
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import type { AgentVisionExecutor } from "../runtime/agent-vision-failover";
|
||||
import { applyAgentImagePreferences, agentProviderImageParameters, type AgentImagePreferences } from "./image-preferences";
|
||||
import { resolveUserImageModel, applyAgentImagePreferences, agentProviderImageParameters, type AgentImagePreferences } from "./image-preferences";
|
||||
import { tool, type LanguageModel, type ToolSet } from "ai";
|
||||
import { z } from "zod";
|
||||
import { createCanvasCapabilityTools } from "@/lib/canvas/capabilities/ai-sdk-tools";
|
||||
@ -202,7 +202,8 @@ export async function buildCanvasRuntimeTools(params: {
|
||||
intent: sources.length ? "image_edit" : "image_gen",
|
||||
hasImage: sources.length > 0,
|
||||
}).catch(() => null);
|
||||
const requestedModel = params.imagePreferences?.modelId || input.modelId || params.preferredModelId;
|
||||
const pinnedModel = resolveUserImageModel(params.objective || "", models, params.imagePreferences?.modelId);
|
||||
const requestedModel = pinnedModel || input.modelId || params.preferredModelId;
|
||||
const requestedEntry = requestedModel ? models.find(({ model }) => model.healthStatus !== "auto_disabled" && (model.id === requestedModel || model.modelId === requestedModel)) : null;
|
||||
if (requestedModel && !requestedEntry) throw new Error("指定的图片模型不可用，未切换到其他模型收费生成");
|
||||
const chosen = requestedEntry || models.find(
|
||||
@ -224,7 +225,7 @@ export async function buildCanvasRuntimeTools(params: {
|
||||
{ imageSize },
|
||||
) * input.imageCount;
|
||||
if (
|
||||
points.enabled &&
|
||||
pinnedModel && points.enabled &&
|
||||
points.blockOnInsufficient !== false &&
|
||||
!user?.isHighVip &&
|
||||
(user?.pointsBalance ?? 0) < cost
|
||||
@ -240,7 +241,10 @@ export async function buildCanvasRuntimeTools(params: {
|
||||
negativePrompt: input.negativePrompt,
|
||||
promptExtend: input.promptExtend,
|
||||
providerImageParams: agentProviderImageParameters(input),
|
||||
allowFailover: !params.imagePreferences?.modelId,
|
||||
allowFailover: !pinnedModel,
|
||||
agentImageMode: pinnedModel ? "pinned" : "auto",
|
||||
agentImageMaxCreditCost: points.enabled && points.blockOnInsufficient !== false && !user?.isHighVip
|
||||
? Math.max(0, user?.pointsBalance ?? 0) / input.imageCount : undefined,
|
||||
ownerKey: params.userId,
|
||||
referenceImages: sources.map((node) => node.data.src || node.data.url!),
|
||||
debugLabel: "canvas-agent-runtime",
|
||||
@ -269,7 +273,7 @@ export async function buildCanvasRuntimeTools(params: {
|
||||
height: ratio ? Math.round((1024 * Number(ratio[2])) / Number(ratio[1])) : 1024,
|
||||
prompt: input.prompt,
|
||||
sourceNodeIds: input.referenceNodeIds,
|
||||
metadata: { canvasAgentRunId: params.runId, model: result.modelRef },
|
||||
metadata: { canvasAgentRunId: params.runId, model: image.modelRef || result.modelRef },
|
||||
})),
|
||||
},
|
||||
});
|
||||
@ -283,15 +287,21 @@ export async function buildCanvasRuntimeTools(params: {
|
||||
updatedNodeIds: operation.updatedNodeIds,
|
||||
canvasRevision: operation.canvasRevision,
|
||||
});
|
||||
const billingGroups = new Map<string, typeof result.images>();
|
||||
for (const image of result.images) {
|
||||
const ref = image.modelRef || result.modelRef;
|
||||
billingGroups.set(ref, [...(billingGroups.get(ref) || []), image]);
|
||||
}
|
||||
for (const [modelRef, group] of billingGroups) {
|
||||
await saveGeneratedImages(
|
||||
result.images
|
||||
group
|
||||
.map((image) => ``)
|
||||
.join("\n"),
|
||||
params.userId,
|
||||
params.chatId,
|
||||
params.assistantMessageId,
|
||||
input.prompt,
|
||||
result.modelRef,
|
||||
modelRef,
|
||||
{
|
||||
projectId: params.projectId,
|
||||
sourceImageUrls: sources.map((node) => node.data.src || node.data.url!),
|
||||
@ -300,9 +310,10 @@ export async function buildCanvasRuntimeTools(params: {
|
||||
aspectRatio: input.aspectRatio,
|
||||
},
|
||||
);
|
||||
}
|
||||
return {
|
||||
status: "succeeded",
|
||||
appliedImagePreferences: { modelId: chosen.model.id, imageSize, aspectRatio: input.aspectRatio, imageCount: input.imageCount },
|
||||
status: result.images.length < input.imageCount ? "partial" : "succeeded",
|
||||
appliedImagePreferences: { modelId: result.modelRef, imageSize, aspectRatio: input.aspectRatio, imageCount: input.imageCount },
|
||||
createdNodeIds: operation.createdNodeIds,
|
||||
outputs: result.images.map((image) => ({ url: image.originalUrl })),
|
||||
model: result.modelRef,
|
||||
|
||||
16
lib/ai/runtime/agent-image-failover.ts
Normal file
16
lib/ai/runtime/agent-image-failover.ts
Normal file
@ -0,0 +1,16 @@
|
||||
import { agentModelFault, type AgentModelFault } from "./agent-model-failover";
|
||||
|
||||
export function agentImageFailurePolicy(error: unknown): { retry: boolean; pause: AgentModelFault | null } {
|
||||
const value = error as { message?: string; providerOutcomeUnknown?: boolean; status?: number; statusCode?: number } | null;
|
||||
const message = String(value?.message || "").toLowerCase();
|
||||
// A submitted image job may still succeed. Retain its task and reconcile it;
|
||||
// never submit another paid job just because a poll/connection timed out.
|
||||
const unknown = value?.providerOutcomeUnknown || /request_outcome_unknown|may still be running|outcome.unknown|timed out|timeout|超时|fetch failed|econnreset|socket hang up/.test(message);
|
||||
if (unknown && !/task failed|task.*(?:failed|cancelled|canceled)|任务.*(?:失败|取消)/.test(message)) return { retry: false, pause: null };
|
||||
if (/content.?policy|moderation|safety|违规|敏感|积分不足|配额不足|user cancel|user abort/.test(message)) return { retry: false, pause: null };
|
||||
const fault = agentModelFault(error);
|
||||
if (fault) return { retry: true, pause: fault };
|
||||
if (/task failed|image generation failed|图片生成失败|未返回.*图片|empty.result|empty.response|provider.error|上游.*错误|模型.*维护/.test(message))
|
||||
return { retry: true, pause: { reason: "image_generation_failed", status: Number(value?.statusCode || value?.status) || null } };
|
||||
return { retry: false, pause: null };
|
||||
}
|
||||
@ -4,10 +4,9 @@ import { db } from "@/lib/db";
|
||||
import { aiModelHealthEvent, systemSettings } from "@/lib/db/schema";
|
||||
import { AGENT_MODEL_COOLDOWN_MS, type AgentCircuitStore } from "./agent-model-failover";
|
||||
|
||||
const prefix = "agent_multimodal_circuit:";
|
||||
// PostgreSQL is shared by every process and release. Do not reuse the general
|
||||
// image-model health ban (which has a different, two-hour recovery policy).
|
||||
export const agentModelCircuitStore: AgentCircuitStore = {
|
||||
export function createAgentCircuitStore(prefix: string, context: string): AgentCircuitStore { return {
|
||||
async isPaused(entry) {
|
||||
const [row] = await db.select({ value: systemSettings.value, now: sql<string>`clock_timestamp()` })
|
||||
.from(systemSettings).where(eq(systemSettings.key, prefix + entry.model.id)).limit(1);
|
||||
@ -26,9 +25,12 @@ export const agentModelCircuitStore: AgentCircuitStore = {
|
||||
await tx.insert(aiModelHealthEvent).values({
|
||||
modelId: entry.model.id, providerId: entry.provider.id, modelRuntimeId: entry.model.modelId,
|
||||
modelDisplayName: entry.model.displayName, providerType: entry.provider.type, providerName: entry.provider.name,
|
||||
eventType: "agent_circuit_open", severity: "warning", context: "canvas-agent-multimodal",
|
||||
eventType: "agent_circuit_open", severity: "warning", context,
|
||||
counted: false, errorMessage: fault.reason, metadata: { ...value, cooldownMs: AGENT_MODEL_COOLDOWN_MS },
|
||||
});
|
||||
});
|
||||
},
|
||||
};
|
||||
}; }
|
||||
|
||||
export const agentModelCircuitStore = createAgentCircuitStore("agent_multimodal_circuit:", "canvas-agent-multimodal");
|
||||
export const agentImageCircuitStore = createAgentCircuitStore("agent_image_circuit:", "canvas-agent-image");
|
||||
|
||||
@ -56,7 +56,7 @@ export async function runZcodeCanvasTask(ports: Ports): Promise<CanvasAgentRunSt
|
||||
"你正在处理本站 Agent 任务。本站已授权的图片、节点、界面及项目业务能力通过 canvas MCP 提供；视频画布暂未授权，不得绕过限制；先读实际画布，不猜测节点 ID。",
|
||||
`可用业务工具：${tools.join("、")}。工具名称带 mcp__canvas__ 前缀。`,
|
||||
"原生文件、Bash、记忆及子任务能力仅作用于此任务的私有工作区，不能通过 shell 直接访问模型服务商或本站后台；业务操作使用 MCP。",
|
||||
"imagePreferences 是用户在输入框选择的生图模型和图片参数，与当前规划模型不同。生成和编辑图片优先用 generateCanvasImages；用户选定的模型不得被技能默认值、你选的其他模型或重试覆盖。比例、尺寸、数量、反向提示词等控件必须传入；不要通过其他生成接口绕过用户选择。分析、问答、排版等不需要出图的任务无需生成图片。",
|
||||
"imagePreferences 是用户在输入框选择的生图模型和图片参数，与当前规划模型不同。生成和编辑图片优先用 generateCanvasImages；用户选定的模型不得被技能默认值、你选的其他模型或重试覆盖。自动选择生图模型时，图片工具内部会在明确失败后切换备用模型并补齐结果，不要自行重放付费操作；用户明确指定模型时禁止故障切换。比例、尺寸、数量、反向提示词等控件必须传入；不要通过其他生成接口绕过用户选择。分析、问答、排版等不需要出图的任务无需生成图片。",
|
||||
"查询项目级能力先用 listProjectServices；技能先 loadProjectSkill。项目工具的任务受理、文件选择与后台缺失不等于处理完成。",
|
||||
"需要澄清时调用 requestCanvasUserInput；不要使用终端交互。工具返回的 userUpdates 是用户的新补充，必须从下一步纳入。",
|
||||
"工具错误涉及结果未确认时停止收费动作，不通过更换调用 ID 或另一个工具重复生成。用户明确要求修改的目标才能删除或覆盖。",
|
||||
|
||||
@ -100,6 +100,10 @@ export async function createCanvasCapabilityTools(params: {
|
||||
? applyAgentImagePreferences(rawArgs, params.imagePreferences) : rawArgs;
|
||||
const args = {
|
||||
...inheritedArgs,
|
||||
...(["image.prompt_regenerate", "image.ai_wash"].includes(capability.id) ? {
|
||||
agentImageMode: params.imagePreferences?.modelId ? "pinned" : "auto",
|
||||
...(params.imagePreferences?.modelId ? { modelId: params.imagePreferences.modelId } : {}),
|
||||
} : {}),
|
||||
...(capability.modelPolicy === "agent-selected" &&
|
||||
params.defaultModelId &&
|
||||
!("modelId" in inheritedArgs)
|
||||
|
||||
@ -433,6 +433,10 @@ async function executeForTarget(
|
||||
const result = await generateCanvasImage({
|
||||
prompt,
|
||||
model: model.model.id,
|
||||
...(command.source === "agent" ? {
|
||||
agentImageMode: command.args.agentImageMode === "pinned" ? "pinned" as const : "auto" as const,
|
||||
allowFailover: command.args.agentImageMode !== "pinned",
|
||||
} : {}),
|
||||
ownerKey: params.userId,
|
||||
imageCount: 1,
|
||||
imageSize: String(command.args.imageSize || "2K"),
|
||||
|
||||
@ -1,6 +1,9 @@
|
||||
import "server-only";
|
||||
|
||||
import sharp from "sharp";
|
||||
import { agentImageCircuitStore } from "@/lib/ai/runtime/agent-model-circuit-store";
|
||||
import { getModelGenerationCreditCost, type DiscountableImageGenConfigLike } from "@/lib/ai/image-model-pricing";
|
||||
import { agentImageFailurePolicy } from "@/lib/ai/runtime/agent-image-failover";
|
||||
import { getEnabledAiModelsWithProvider } from "@/lib/db/queries";
|
||||
import { fetchImageBufferFromUrl } from "@/lib/ai/rmbg/service";
|
||||
import { createGeminiImage } from "@/lib/ai/providers/gemini-image";
|
||||
@ -188,6 +191,8 @@ export interface CanvasGenerateImageRequest {
|
||||
retryVariants?: CanvasGenerateImageRetryVariant[];
|
||||
internalRetryVariantIndex?: number;
|
||||
allowFailover?: boolean;
|
||||
agentImageMode?: "auto" | "pinned";
|
||||
agentImageMaxCreditCost?: number;
|
||||
internalFailoverTriedModelRefs?: string[];
|
||||
internalFailoverTriedProviderTypes?: string[];
|
||||
onQueueStatusChange?: (
|
||||
@ -201,6 +206,7 @@ export interface CanvasGenerateImageRequest {
|
||||
export interface ParsedImage {
|
||||
thumbnailUrl: string;
|
||||
originalUrl: string;
|
||||
modelRef?: string;
|
||||
}
|
||||
|
||||
export interface CanvasGenerateImageResult {
|
||||
@ -387,13 +393,7 @@ async function withCanvasProviderBatchTimeout<T>(
|
||||
operation,
|
||||
new Promise<T>((_, reject) => {
|
||||
timeoutId = setTimeout(() => {
|
||||
reject(
|
||||
new Error(
|
||||
`${params.providerType} 图片生成超时（${Math.round(
|
||||
params.timeoutMs / 1000
|
||||
)}秒），将自动切换备用模型`
|
||||
)
|
||||
);
|
||||
reject(Object.assign(new Error(`${params.providerType} 图片生成超时（${Math.round(params.timeoutMs / 1000)}秒），请稍后确认结果`), { providerOutcomeUnknown: true }));
|
||||
}, params.timeoutMs);
|
||||
}),
|
||||
]);
|
||||
@ -4507,6 +4507,7 @@ function findCanvasFailoverModel(params: {
|
||||
}
|
||||
if (
|
||||
!item.model.isEnabled ||
|
||||
item.model.healthStatus === "auto_disabled" ||
|
||||
item.model.isSubModel ||
|
||||
(item.model.excludeFromFailover && !isSameNameScaleAdjustFallback)
|
||||
) {
|
||||
@ -4628,6 +4629,7 @@ export async function generateCanvasImage(
|
||||
request: CanvasGenerateImageRequest
|
||||
): Promise<CanvasGenerateImageResult> {
|
||||
const startedAt = Date.now();
|
||||
if (request.agentImageMode === "pinned") request = { ...request, allowFailover: false };
|
||||
const prompt = String(request.prompt || "").trim();
|
||||
if (!prompt) {
|
||||
throw new Error("请输入提示词");
|
||||
@ -4898,7 +4900,13 @@ export async function generateCanvasImage(
|
||||
);
|
||||
|
||||
const allModels = await getEnabledAiModelsWithProvider();
|
||||
const imageModels = allModels.filter((m) => m.model.supportsImageGen);
|
||||
let imageModels = allModels.filter((m) => m.model.supportsImageGen);
|
||||
if (request.agentImageMode === "auto") {
|
||||
const states = await Promise.all(imageModels.map(async entry => ({ entry, paused: await agentImageCircuitStore.isPaused(entry) })));
|
||||
imageModels = states.filter(item => !item.paused && item.entry.model.healthStatus !== "auto_disabled" &&
|
||||
isCanvasImageSizeSupported(item.entry.model.imageGenConfig as CanvasImageGenConfig | null, normalizeCanvasImageSizeValue(request.imageSize)) &&
|
||||
(request.agentImageMaxCreditCost === undefined || getModelGenerationCreditCost({ imageGenConfig: item.entry.model.imageGenConfig as DiscountableImageGenConfigLike | null }, "image", { imageSize: request.imageSize }) <= request.agentImageMaxCreditCost)).map(item => item.entry);
|
||||
}
|
||||
|
||||
if (!imageModels || imageModels.length === 0) {
|
||||
throw new Error("暂无可用的图片生成模型");
|
||||
@ -4909,8 +4917,12 @@ export async function generateCanvasImage(
|
||||
: imageModels.find((m) => m.model.isDefault) || imageModels[0];
|
||||
|
||||
if (!selectedModel) {
|
||||
selectedModel = imageModels[0];
|
||||
if (model && request.allowFailover === false) throw new Error("指定的图片模型不可用，未切换到其他模型");
|
||||
selectedModel = request.agentImageMode === "auto" ? imageModels.find(entry => !entry.model.excludeFromFailover) : imageModels[0];
|
||||
if (!selectedModel) throw new Error("当前没有可用的备用图片模型，请稍后重试");
|
||||
}
|
||||
if (request.agentImageMode === "pinned" && !isCanvasImageSizeSupported(selectedModel.model.imageGenConfig as CanvasImageGenConfig | null, normalizeCanvasImageSizeValue(request.imageSize)))
|
||||
throw new Error("指定的图片模型不支持所选尺寸，请调整图片尺寸");
|
||||
|
||||
const normalizedReferenceImages = Array.from(
|
||||
new Set(
|
||||
@ -5064,7 +5076,7 @@ export async function generateCanvasImage(
|
||||
imageCount: requestedImageCount,
|
||||
imageSize: requestedImageSize,
|
||||
imageGenConfig: selectedModel.model.imageGenConfig as CanvasImageGenConfig | null | undefined,
|
||||
ignoreModelMaxImages: request.ignoreModelMaxImages === true,
|
||||
ignoreModelMaxImages: Boolean(request.agentImageMode) || request.ignoreModelMaxImages === true,
|
||||
});
|
||||
const imageSize = normalizedImageParams.imageSize;
|
||||
const imageCount =
|
||||
@ -5451,7 +5463,9 @@ export async function generateCanvasImage(
|
||||
);
|
||||
}
|
||||
|
||||
const providerBatchLimit = getProviderBatchLimit(provider.type);
|
||||
const providerBatchLimit = request.agentImageMode
|
||||
? Math.min(getProviderBatchLimit(provider.type), Math.max(1, Number((selectedModel.model.imageGenConfig as CanvasImageGenConfig | null)?.maxImages) || 12))
|
||||
: getProviderBatchLimit(provider.type);
|
||||
const providerBatchTimeoutMs = getCanvasProviderBatchTimeoutMs({
|
||||
debugLabel,
|
||||
debugMeta,
|
||||
@ -5536,12 +5550,11 @@ export async function generateCanvasImage(
|
||||
...debugMeta,
|
||||
});
|
||||
if (parsedImages.length === 0) {
|
||||
if (collectedImages.length > 0) {
|
||||
break;
|
||||
}
|
||||
if (collectedImages.length > 0 && request.agentImageMode !== "auto") break;
|
||||
throw new Error("图片生成失败");
|
||||
}
|
||||
collectedImages.push(...parsedImages.slice(0, batchImageCount));
|
||||
collectedImages.push(...parsedImages.slice(0, batchImageCount).map(image => request.agentImageMode ? { ...image, modelRef: selectedModel.model.id } : image));
|
||||
if (request.agentImageMode === "auto" && parsedImages.length < batchImageCount) throw new Error("图片生成失败：模型返回数量不足");
|
||||
}
|
||||
|
||||
if (collectedImages.length === 0) {
|
||||
@ -5634,7 +5647,7 @@ export async function generateCanvasImage(
|
||||
? request.internalRetryVariantIndex + 1
|
||||
: 0;
|
||||
const retryVariant = retryVariants[nextRetryVariantIndex];
|
||||
if (retryVariant) {
|
||||
if (retryVariant && !request.agentImageMode) {
|
||||
const {
|
||||
debugMeta: retryVariantDebugMeta,
|
||||
label: retryVariantLabel,
|
||||
@ -5687,6 +5700,9 @@ export async function generateCanvasImage(
|
||||
selectedModelId: selectedModel.model.modelId,
|
||||
error: caughtError instanceof Error ? caughtError.message : String(caughtError),
|
||||
});
|
||||
} else if (request.agentImageMode === "auto") {
|
||||
const failure = agentImageFailurePolicy(caughtError);
|
||||
if (failure.pause) await agentImageCircuitStore.fail(selectedModel, failure.pause);
|
||||
} else {
|
||||
await recordAiModelHealthFailure({
|
||||
model: selectedModel.model,
|
||||
@ -5719,22 +5735,21 @@ export async function generateCanvasImage(
|
||||
? request.internalFailoverTriedModelRefs
|
||||
: []),
|
||||
selectedModel.model.id,
|
||||
selectedModel.model.modelId,
|
||||
...(request.agentImageMode === "auto" ? [] : [selectedModel.model.modelId]),
|
||||
].filter(Boolean))
|
||||
);
|
||||
const allowFailover = request.allowFailover !== false;
|
||||
const shouldFailover =
|
||||
shouldFailoverAiModelError(caughtError) ||
|
||||
(!scaleAdjustRejected &&
|
||||
Boolean(debugMeta?.detailFixedModelApplied) &&
|
||||
Boolean(caughtError));
|
||||
const shouldFailover = request.agentImageMode === "auto"
|
||||
? agentImageFailurePolicy(caughtError).retry
|
||||
: shouldFailoverAiModelError(caughtError) ||
|
||||
(!scaleAdjustRejected && Boolean(debugMeta?.detailFixedModelApplied) && Boolean(caughtError));
|
||||
const failureClassification = classifyAiModelHealthFailure(caughtError);
|
||||
const failedProviderTypes = Array.from(
|
||||
new Set([
|
||||
...(Array.isArray(request.internalFailoverTriedProviderTypes)
|
||||
? request.internalFailoverTriedProviderTypes
|
||||
: []),
|
||||
...(shouldFailover &&
|
||||
...(request.agentImageMode !== "auto" && shouldFailover &&
|
||||
failureClassification.reason !== "non_provider_failure" &&
|
||||
failureClassification.reason !== "user_cancelled"
|
||||
? [provider.type]
|
||||
@ -5769,8 +5784,9 @@ export async function generateCanvasImage(
|
||||
...debugMeta,
|
||||
});
|
||||
|
||||
return await generateCanvasImage({
|
||||
const fallbackResult = await generateCanvasImage({
|
||||
...request,
|
||||
...(request.agentImageMode === "auto" ? { imageCount: Math.max(1, targetImageCount - collectedImages.length) } : {}),
|
||||
model: fallbackModel.model.id,
|
||||
internalFailoverTriedModelRefs: failedModelRefs,
|
||||
internalFailoverTriedProviderTypes: failedProviderTypes,
|
||||
@ -5779,9 +5795,19 @@ export async function generateCanvasImage(
|
||||
failoverFromModelId: selectedModel.model.modelId,
|
||||
failoverFromModelRef: selectedModel.model.id,
|
||||
},
|
||||
}).catch(error => {
|
||||
if (request.agentImageMode === "auto" && collectedImages.length) return { images: [], model: selectedModel.model.displayName, modelRef: selectedModel.model.id, prompt: resultPrompt };
|
||||
throw error;
|
||||
});
|
||||
return request.agentImageMode === "auto"
|
||||
? { ...fallbackResult, images: [...collectedImages, ...fallbackResult.images].slice(0, targetImageCount) }
|
||||
: fallbackResult;
|
||||
}
|
||||
|
||||
if (request.agentImageMode && collectedImages.length) return {
|
||||
images: collectedImages, model: selectedModel.model.displayName, modelRef: selectedModel.model.id, prompt: resultPrompt,
|
||||
};
|
||||
|
||||
if (scaleAdjustRejection?.images?.length) {
|
||||
console.warn("[Canvas Generate Image] Rejecting scale adjust candidate because no effective edit was detected:", {
|
||||
debugLabel,
|
||||
|
||||
@ -2,7 +2,7 @@ import { defineConfig } from "@playwright/test";
|
||||
|
||||
export default defineConfig({
|
||||
testDir: "./tests/routes",
|
||||
testMatch: /(?:canvas-agent-runtime(?:-store|-review|-video|-image-preferences|-progress)?|canvas-agent-project-tools|agent-capability-access)\.test\.tsx?/,
|
||||
testMatch: /(?:canvas-agent-runtime(?:-store|-review|-video|-image-preferences|-image-failover|-progress)?|canvas-agent-project-tools|agent-capability-access)\.test\.tsx?/,
|
||||
fullyParallel: true,
|
||||
retries: 0,
|
||||
workers: 2,
|
||||
|
||||
26
tests/helpers/agent-image-provider-harness.ts
Normal file
26
tests/helpers/agent-image-provider-harness.ts
Normal file
@ -0,0 +1,26 @@
|
||||
import { serverLoader } from "./canvas-download-server-loader";
|
||||
|
||||
export function agentImageProviderHarness(options: { generate?: (id: string, count: number, call: number) => string; models?: any[] } = {}) {
|
||||
const calls: any[] = [], paused = new Set<string>();
|
||||
const models = options.models || ["primary", "backup"].map((id, i) => ({ model: { id, modelId: id, displayName: id, isEnabled: true, supportsImageGen: true, supportsVision: true, isDefault: !i, imageGenConfig: { maxImages: 12 }, capabilities: { kind: "image" } }, provider: { id: "same-provider", type: "openai-image", apiKey: "synthetic", isEnabled: true } }));
|
||||
const mocks: Record<string, any> = {
|
||||
"@/lib/db/queries": { getEnabledAiModelsWithProvider: async () => models },
|
||||
"@/lib/ai/rmbg/service": {},
|
||||
"@/lib/net/url-access": { filterReachableRemoteUrls: async (urls: string[]) => ({ kept: urls, dropped: [] }) },
|
||||
"@/lib/canvas/image-concurrency": { acquireCanvasImageModelSlot: async () => ({ waitMs: 0, maxConcurrent: 999, release: async () => {} }), normalizeCanvasImageModelKey: (v: string) => v, recordCanvasImageModelExecutionDuration() {} },
|
||||
"@/lib/ai/runtime/model-health": { recordAiModelHealthSuccess: async () => {}, recordAiModelHealthFailure: async () => {}, classifyAiModelHealthFailure: () => ({ reason: "provider_error" }), shouldFailoverAiModelError: () => false },
|
||||
"@/lib/ai/runtime/agent-model-circuit-store": { agentImageCircuitStore: { isPaused: async (e: any) => paused.has(e.model.id), fail: async (e: any) => { paused.add(e.model.id); } } },
|
||||
"@/lib/ai/image-model-pricing": { getModelGenerationCreditCost: ({ imageGenConfig }: any) => imageGenConfig?.creditCost || 0 },
|
||||
"@/lib/storage/service": {},
|
||||
};
|
||||
for (const name of ["gemini-image", "alibaba-image", "bfl-image", "ruxa-image", "fal-image", "snapgen-image", "openai-image", "bananaimage", "visionary", "atlascloud", "vip98k", "stepfun-image", "midjourney", "runway", "krea", "volcano-vision"]) mocks[`@/lib/ai/providers/${name}`] = {};
|
||||
mocks["@/lib/ai/providers/openai-image"] = { createOpenAIImage: () => ({ chat: (id: string) => ({ doGenerate: async (params: any) => {
|
||||
calls.push({ id, ...params });
|
||||
const text = options.generate?.(id, params.imageParams.imageCount, calls.length) || ``;
|
||||
return { text };
|
||||
} }) }) };
|
||||
const generate = serverLoader(mocks)("lib/canvas/generate-image.ts").generateCanvasImage;
|
||||
const request = { prompt: "a cat", model: "primary", imageCount: 1, imageSize: "2K", aspectRatio: "3:4", agentImageMode: "auto", allowFailover: true, referenceImages: ["https://fixture.invalid/source.png"], negativePrompt: "no text", providerImageParams: { seed: 0, watermark: false } };
|
||||
return { generate, request, calls, paused };
|
||||
}
|
||||
|
||||
63
tests/routes/canvas-agent-runtime-image-failover.test.ts
Normal file
63
tests/routes/canvas-agent-runtime-image-failover.test.ts
Normal file
@ -0,0 +1,63 @@
|
||||
import { expect, test } from "@playwright/test";
|
||||
import { agentImageProviderHarness as harness } from "../helpers/agent-image-provider-harness";
|
||||
import { agentImageFailurePolicy } from "@/lib/ai/runtime/agent-image-failover";
|
||||
|
||||
|
||||
test("automatic image generation switches within the same provider and skips a paused model on the next task", async () => {
|
||||
const h = harness({ generate: id => { if (id === "primary") throw Object.assign(new Error("channel unavailable"), { status: 503 }); return ""; } });
|
||||
const result = await h.generate(h.request);
|
||||
expect(result.modelRef).toBe("backup"); expect(h.calls.map(c => c.id)).toEqual(["primary", "backup"]);
|
||||
expect([...h.paused]).toEqual(["primary"]);
|
||||
expect(h.calls[1].imageParams).toMatchObject({ imageSize: "2K", aspectRatio: "3:4", negativePrompt: "no text", seed: 0, watermark: false });
|
||||
await h.generate(h.request);
|
||||
expect(h.calls.map(c => c.id)).toEqual(["primary", "backup", "backup"]);
|
||||
});
|
||||
|
||||
test("a pinned model failure never calls any backup or silently substitutes a missing model", async () => {
|
||||
const h = harness({ generate: () => { throw Object.assign(new Error("invalid authentication"), { status: 401 }); } });
|
||||
await expect(h.generate({ ...h.request, agentImageMode: "pinned", allowFailover: true })).rejects.toThrow();
|
||||
expect(h.calls.map(c => c.id)).toEqual(["primary"]);
|
||||
await expect(h.generate({ ...h.request, agentImageMode: "pinned", model: "missing" })).rejects.toThrow("指定的图片模型不可用");
|
||||
expect(h.calls).toHaveLength(1);
|
||||
});
|
||||
|
||||
test("a partly successful batch keeps its delivered images and only generates the missing count", async () => {
|
||||
const h = harness({ generate: (id, _n, call) => { if (id === "primary" && call === 2) throw new Error("VIP98K task failed"); return ``; } });
|
||||
const result = await h.generate({ ...h.request, imageCount: 3 });
|
||||
expect(h.calls.map(c => c.id)).toEqual(["primary", "primary", "backup", "backup"]);
|
||||
expect(result.images).toHaveLength(3);
|
||||
expect(result.images.map((i: any) => i.modelRef)).toEqual(["primary", "backup", "backup"]);
|
||||
expect(result.images[0].originalUrl).toContain("primary-1");
|
||||
});
|
||||
|
||||
test("if every backup fails, previously delivered images are retained", async () => {
|
||||
const h = harness({ generate: (_id, _n, call) => { if (call > 1) throw new Error("VIP98K task failed"); return ""; } });
|
||||
const result = await h.generate({ ...h.request, imageCount: 3 });
|
||||
expect(result.images).toHaveLength(1); expect(result.images[0].originalUrl).toContain("keep.png");
|
||||
});
|
||||
|
||||
test("a submitted task with an unknown outcome is not submitted again through a backup", async () => {
|
||||
const h = harness({ generate: () => { throw new Error("VIP98K task timed out; it may still be running"); } });
|
||||
await expect(h.generate(h.request)).rejects.toThrow();
|
||||
expect(h.calls.map(c => c.id)).toEqual(["primary"]); expect(h.paused.size).toBe(0);
|
||||
expect(agentImageFailurePolicy(Object.assign(new Error("timeout"), { providerOutcomeUnknown: true })).retry).toBe(false);
|
||||
expect(agentImageFailurePolicy(new Error("VIP98K task failed: timeout")).retry).toBe(true);
|
||||
});
|
||||
|
||||
test("content restrictions and user input problems cannot be bypassed by switching services", () => {
|
||||
for (const error of [new Error("content policy rejection"), Object.assign(new Error("bad input"), { status: 400 }), new Error("用户积分不足")]) expect(agentImageFailurePolicy(error)).toEqual({ retry: false, pause: null });
|
||||
});
|
||||
|
||||
|
||||
test("automatic fallback respects requested dimensions, disabled candidates and the user's available credit", async () => {
|
||||
const models = [
|
||||
{ id: "primary", creditCost: 1, supportedImageSizes: ["2K"] },
|
||||
{ id: "wrong-size", creditCost: 1, supportedImageSizes: ["1K"] },
|
||||
{ id: "too-expensive", creditCost: 10, supportedImageSizes: ["2K"] },
|
||||
{ id: "disabled-health", creditCost: 1, supportedImageSizes: ["2K"], healthStatus: "auto_disabled" },
|
||||
{ id: "backup", creditCost: 2, supportedImageSizes: ["2K"] },
|
||||
].map(({ id, healthStatus, ...config }) => ({ model: { id, modelId: id, displayName: id, isEnabled: true, healthStatus, supportsImageGen: true, supportsVision: true, imageGenConfig: config }, provider: { type: "openai-image", isEnabled: true } }));
|
||||
const h = harness({ models, generate: id => { if (id === "primary") throw new Error("VIP98K task failed"); return ""; } });
|
||||
const result = await h.generate({ ...h.request, agentImageMaxCreditCost: 3 });
|
||||
expect(result.modelRef).toBe("backup"); expect(h.calls.map(c => c.id)).toEqual(["primary", "backup"]);
|
||||
});
|
||||
@ -7,15 +7,16 @@ const preferences = {
|
||||
parameters: { imageCount: 3, aspectRatio: "3:4", imageSize: "4K", negativePrompt: "不要文字",
|
||||
quality: "high" as const, size: "1536*2048", promptExtend: false, watermark: false, seed: 0 },
|
||||
};
|
||||
function harness() {
|
||||
function harness(results?: any[]) {
|
||||
const saved: any[] = [];
|
||||
const generated: any[] = [];
|
||||
const operations: any[] = [];
|
||||
const mock = serverLoader({
|
||||
"@/lib/canvas/capabilities/ai-sdk-tools": { createCanvasCapabilityTools: async () => ({ editTargetImage: {} }) },
|
||||
"@/lib/canvas/capabilities/registry": { CANVAS_CAPABILITY_REGISTRY: [{ id: "image.prompt_regenerate", title: "重绘", risk: "external", agent: { toolName: "editTargetImage" } }] },
|
||||
"@/lib/canvas/agent-mutations": { applyCanvasOperationPlan: async (value: any) => { operations.push(value); return { createdNodeIds: ["new-image"] }; } },
|
||||
"@/lib/canvas/generate-image": { generateCanvasImage: async (value: any) => { generated.push(value); return { modelRef: value.model, images: [{ originalUrl: "https://test.example/result.png", thumbnailUrl: "https://test.example/thumb.png" }] }; } },
|
||||
"@/lib/ai/runtime/generated-image-persistence": { saveGeneratedImages: async () => {} },
|
||||
"@/lib/canvas/generate-image": { generateCanvasImage: async (value: any) => { generated.push(value); return { modelRef: results ? "backup" : value.model, images: results || [{ originalUrl: "https://test.example/result.png", thumbnailUrl: "https://test.example/thumb.png" }] }; } },
|
||||
"@/lib/ai/runtime/generated-image-persistence": { saveGeneratedImages: async (...args: any[]) => { saved.push(args); } },
|
||||
"@/lib/ai/model-router": { routeToModel: async () => ({ id: "system-default" }) },
|
||||
"@/lib/db/queries": { checkUserImageQuota: async () => ({ allowed: true, remaining: 10 }), getPointsSystemConfig: async () => ({ enabled: false }), getUserById: async () => ({}) },
|
||||
"@/lib/style-pro/model-options": {
|
||||
@ -31,7 +32,7 @@ function harness() {
|
||||
request: new Request("https://test.example/api/canvas/agent-chat"), model: {}, supportsVision: true,
|
||||
preferredModelId: "system-default", imagePreferences, emit() {},
|
||||
});
|
||||
return { build, generated, operations };
|
||||
return { build, generated, operations, saved };
|
||||
}
|
||||
test("explicit controls override conflicting planner settings without losing false or zero values", () => {
|
||||
const resolved = applyAgentImagePreferences({ modelId: "planner-choice", imageCount: 1, imageSize: "1K", aspectRatio: "1:1", prompt: "产品图" }, preferences);
|
||||
@ -73,3 +74,27 @@ test("planning uses the image count for generation tasks but does not turn analy
|
||||
expect(generation).toMatchObject({ expectedImageCount: 3, aspectRatio: "3:4" });
|
||||
expect(h.generated).toHaveLength(0);
|
||||
});
|
||||
|
||||
test("an explicit model in the user's words pins generation, while the planner's own choice does not", async () => {
|
||||
const { resolveUserImageModel } = await import("@/lib/ai/canvas-agent-runtime/image-preferences");
|
||||
const entries = [{ model: { id: "pinned", modelId: "nano-banana-pro", displayName: "Banana Pro" } }];
|
||||
expect(resolveUserImageModel("请使用 nano banana pro 画一只小猫", entries)).toBe("pinned");
|
||||
expect(resolveUserImageModel("不要用nano-banana-pro，自动画图", entries)).toBeUndefined();
|
||||
expect(resolveUserImageModel("不用nano-banana-pro，自动画图", entries)).toBeUndefined();
|
||||
expect(resolveUserImageModel("先使用nano-banana-pro，改用other-image", [...entries, { model: { id: "later", modelId: "other-image" } }])).toBe("later");
|
||||
expect(resolveUserImageModel("画一只小猫", entries)).toBeUndefined();
|
||||
expect(resolveUserImageModel("使用nano-banana-pro", entries, "manual-control")).toBe("manual-control");
|
||||
});
|
||||
|
||||
|
||||
test("mixed-model batches bill and label only the delivered images using their actual models", async () => {
|
||||
const h = harness([{ originalUrl: "https://test.example/first.png", thumbnailUrl: "https://test.example/first.png", modelRef: "system-default" }, { originalUrl: "https://test.example/second.png", thumbnailUrl: "https://test.example/second.png", modelRef: "backup" }]);
|
||||
const tools = await h.build({ parameters: { imageCount: 2, imageSize: "2K" } });
|
||||
const result = await tools.executable.generateCanvasImages.execute({ prompt: "画图" }, "mixed");
|
||||
expect(h.saved).toHaveLength(2);
|
||||
expect(h.saved.map(call => call[5])).toEqual(["system-default", "backup"]);
|
||||
expect(h.saved[0][0]).toContain("first.png"); expect(h.saved[0][0]).not.toContain("second.png");
|
||||
expect(h.saved[1][0]).toContain("second.png");
|
||||
expect(h.operations[0].plan.mediaItems.map((item: any) => item.metadata.model)).toEqual(["system-default", "backup"]);
|
||||
expect(result.appliedImagePreferences.modelId).toBe("backup");
|
||||
});
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user