fix(ai): fail over forbidden image requests
This commit is contained in:
parent
ba232f3385
commit
e1c155be9e
@ -91,6 +91,11 @@ export function shouldCooldownImageFallbackModel(errorMessage: string): boolean
|
|||||||
msg.includes("invalid api key") ||
|
msg.includes("invalid api key") ||
|
||||||
msg.includes("unauthorized") ||
|
msg.includes("unauthorized") ||
|
||||||
/(?:^|\D)401(?:\D|$)/.test(msg) ||
|
/(?:^|\D)401(?:\D|$)/.test(msg) ||
|
||||||
|
msg === "forbidden" ||
|
||||||
|
msg.includes("access forbidden") ||
|
||||||
|
msg.includes("access denied") ||
|
||||||
|
msg.includes("permission denied") ||
|
||||||
|
/(?:^|\D)403(?:\D|$)/.test(msg) ||
|
||||||
msg.includes("vip98k task failed") ||
|
msg.includes("vip98k task failed") ||
|
||||||
msg.includes("cannot convert argument to a bytestring") ||
|
msg.includes("cannot convert argument to a bytestring") ||
|
||||||
msg.includes("model maintenance") ||
|
msg.includes("model maintenance") ||
|
||||||
|
|||||||
@ -9635,6 +9635,11 @@ export async function POST(request: Request) {
|
|||||||
alibabaModelConfig?.fallbackModelId2,
|
alibabaModelConfig?.fallbackModelId2,
|
||||||
].filter(Boolean) as string[];
|
].filter(Boolean) as string[];
|
||||||
const allModels = await getEnabledAiModelsWithProvider();
|
const allModels = await getEnabledAiModelsWithProvider();
|
||||||
|
const currentModelEntry = resolveModelEntryByRuntimeModelId(
|
||||||
|
allModels,
|
||||||
|
actualModelId,
|
||||||
|
finalModelId,
|
||||||
|
);
|
||||||
const fallbackCandidates = buildImageFallbackCandidates(allModels, {
|
const fallbackCandidates = buildImageFallbackCandidates(allModels, {
|
||||||
currentProviderId: alibabaModelConfig?.providerId,
|
currentProviderId: alibabaModelConfig?.providerId,
|
||||||
currentModelId: alibabaModelConfig?.id,
|
currentModelId: alibabaModelConfig?.id,
|
||||||
@ -9880,6 +9885,7 @@ export async function POST(request: Request) {
|
|||||||
|
|
||||||
let imageMarkdown = "";
|
let imageMarkdown = "";
|
||||||
let usedModelId = actualModelId;
|
let usedModelId = actualModelId;
|
||||||
|
const generationStartedAt = Date.now();
|
||||||
|
|
||||||
// 🆕 图片生成真正开始时发出标记(用于前端显示占位图)
|
// 🆕 图片生成真正开始时发出标记(用于前端显示占位图)
|
||||||
writeAssistantTextChunk(
|
writeAssistantTextChunk(
|
||||||
@ -9906,14 +9912,37 @@ export async function POST(request: Request) {
|
|||||||
} catch (primaryError) {
|
} catch (primaryError) {
|
||||||
chatApiLogger.error("[Chat API] Alibaba primary model failed:", primaryError);
|
chatApiLogger.error("[Chat API] Alibaba primary model failed:", primaryError);
|
||||||
|
|
||||||
|
await recordAiModelHealthFailure({
|
||||||
|
model: currentModelEntry?.model || alibabaModelConfig,
|
||||||
|
provider: currentModelEntry?.provider,
|
||||||
|
error: primaryError,
|
||||||
|
context: "chat_image_primary",
|
||||||
|
elapsedMs: Date.now() - generationStartedAt,
|
||||||
|
metadata: {
|
||||||
|
runtimeModelId: actualModelId,
|
||||||
|
finalModelId,
|
||||||
|
providerType: (languageModel as any).provider,
|
||||||
|
aspectRatio: effectiveImageGenerationParams.aspectRatio,
|
||||||
|
imageSize: effectiveImageGenerationParams.imageSize,
|
||||||
|
imageCount:
|
||||||
|
effectiveImageGenerationParams.imageCount ||
|
||||||
|
defaultImageQualityParams.imageCount ||
|
||||||
|
1,
|
||||||
|
},
|
||||||
|
}).catch((healthError) => {
|
||||||
|
chatApiLogger.warn("[Chat API] Record image primary health failure failed:", healthError);
|
||||||
|
});
|
||||||
|
|
||||||
if (!shouldFailoverAiModelError(primaryError)) {
|
if (!shouldFailoverAiModelError(primaryError)) {
|
||||||
chatApiLogger.info("[Chat API] Alibaba primary error is not eligible for image failover; returning original error.");
|
chatApiLogger.info("[Chat API] Alibaba primary error is not eligible for image failover; returning original error.");
|
||||||
throw primaryError;
|
throw primaryError;
|
||||||
}
|
}
|
||||||
|
|
||||||
let fallbackSuccess = false;
|
let fallbackSuccess = false;
|
||||||
|
const triedFallbackModelIds = new Set<string>();
|
||||||
for (const candidate of fallbackCandidates) {
|
for (const candidate of fallbackCandidates) {
|
||||||
try {
|
try {
|
||||||
|
triedFallbackModelIds.add(String(candidate.model.id || ""));
|
||||||
chatApiLogger.info(`[Chat API] 🔄 Alibaba fallback model: ${candidate.model.displayName} (${candidate.provider.type})`);
|
chatApiLogger.info(`[Chat API] 🔄 Alibaba fallback model: ${candidate.model.displayName} (${candidate.provider.type})`);
|
||||||
const fallbackResult = await generateImageWithFailoverCandidate({
|
const fallbackResult = await generateImageWithFailoverCandidate({
|
||||||
candidate,
|
candidate,
|
||||||
@ -9938,6 +9967,22 @@ export async function POST(request: Request) {
|
|||||||
if (shouldCooldownImageFallbackModel(fallbackErrorMessage)) {
|
if (shouldCooldownImageFallbackModel(fallbackErrorMessage)) {
|
||||||
setImageFallbackModelCooldown(candidate.model.id, fallbackErrorMessage);
|
setImageFallbackModelCooldown(candidate.model.id, fallbackErrorMessage);
|
||||||
}
|
}
|
||||||
|
await recordAiModelHealthFailure({
|
||||||
|
model: candidate.model,
|
||||||
|
provider: candidate.provider,
|
||||||
|
error: fallbackError,
|
||||||
|
context: "chat_image_fallback",
|
||||||
|
elapsedMs: Date.now() - generationStartedAt,
|
||||||
|
metadata: {
|
||||||
|
primaryModelId: actualModelId,
|
||||||
|
fallbackModelId: candidate.model.modelId,
|
||||||
|
providerType: candidate.provider.type,
|
||||||
|
aspectRatio: effectiveImageGenerationParams.aspectRatio,
|
||||||
|
imageSize: effectiveImageGenerationParams.imageSize,
|
||||||
|
},
|
||||||
|
}).catch((healthError) => {
|
||||||
|
chatApiLogger.warn("[Chat API] Record image fallback health failure failed:", healthError);
|
||||||
|
});
|
||||||
chatApiLogger.error(`[Chat API] ❌ Alibaba fallback model ${candidate.model.displayName} (${candidate.provider.type}) failed:`, fallbackError);
|
chatApiLogger.error(`[Chat API] ❌ Alibaba fallback model ${candidate.model.displayName} (${candidate.provider.type}) failed:`, fallbackError);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -9966,10 +10011,11 @@ export async function POST(request: Request) {
|
|||||||
currentModelKey: actualModelId,
|
currentModelKey: actualModelId,
|
||||||
currentProviderId: alibabaModelConfig?.providerId,
|
currentProviderId: alibabaModelConfig?.providerId,
|
||||||
providerPriority,
|
providerPriority,
|
||||||
});
|
}).filter((candidate) => !triedFallbackModelIds.has(String(candidate.model.id || "")));
|
||||||
|
|
||||||
for (const candidate of crossProviderCandidates) {
|
for (const candidate of crossProviderCandidates) {
|
||||||
try {
|
try {
|
||||||
|
triedFallbackModelIds.add(String(candidate.model.id || ""));
|
||||||
chatApiLogger.info(`[Chat API] 🔁 Cross-provider failover attempt: ${candidate.model.displayName} (${candidate.provider.type})`);
|
chatApiLogger.info(`[Chat API] 🔁 Cross-provider failover attempt: ${candidate.model.displayName} (${candidate.provider.type})`);
|
||||||
if (
|
if (
|
||||||
candidate.provider.type === "fal-image" &&
|
candidate.provider.type === "fal-image" &&
|
||||||
@ -10053,6 +10099,22 @@ export async function POST(request: Request) {
|
|||||||
if (shouldCooldownImageFallbackModel(crossErrorMessage)) {
|
if (shouldCooldownImageFallbackModel(crossErrorMessage)) {
|
||||||
setImageFallbackModelCooldown(candidate.model.id, crossErrorMessage);
|
setImageFallbackModelCooldown(candidate.model.id, crossErrorMessage);
|
||||||
}
|
}
|
||||||
|
await recordAiModelHealthFailure({
|
||||||
|
model: candidate.model,
|
||||||
|
provider: candidate.provider,
|
||||||
|
error: crossError,
|
||||||
|
context: "chat_image_cross_fallback",
|
||||||
|
elapsedMs: Date.now() - generationStartedAt,
|
||||||
|
metadata: {
|
||||||
|
primaryModelId: actualModelId,
|
||||||
|
fallbackModelId: candidate.model.modelId,
|
||||||
|
providerType: candidate.provider.type,
|
||||||
|
aspectRatio: effectiveImageGenerationParams.aspectRatio,
|
||||||
|
imageSize: effectiveImageGenerationParams.imageSize,
|
||||||
|
},
|
||||||
|
}).catch((healthError) => {
|
||||||
|
chatApiLogger.warn("[Chat API] Record image cross-provider health failure failed:", healthError);
|
||||||
|
});
|
||||||
chatApiLogger.error(`[Chat API] ❌ Cross-provider failover failed: ${candidate.model.displayName} (${candidate.provider.type})`, crossError);
|
chatApiLogger.error(`[Chat API] ❌ Cross-provider failover failed: ${candidate.model.displayName} (${candidate.provider.type})`, crossError);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -10066,6 +10128,19 @@ export async function POST(request: Request) {
|
|||||||
chatApiLogger.info("[Chat API] Alibaba generated, length:", imageMarkdown.length);
|
chatApiLogger.info("[Chat API] Alibaba generated, length:", imageMarkdown.length);
|
||||||
chatApiLogger.info("[Chat API] Alibaba used model:", usedModelId);
|
chatApiLogger.info("[Chat API] Alibaba used model:", usedModelId);
|
||||||
|
|
||||||
|
const usedModelEntry = resolveModelEntryByRuntimeModelId(
|
||||||
|
allModels,
|
||||||
|
usedModelId,
|
||||||
|
usedModelId === actualModelId ? finalModelId : undefined,
|
||||||
|
);
|
||||||
|
await recordAiModelHealthSuccess({
|
||||||
|
model:
|
||||||
|
usedModelEntry?.model ||
|
||||||
|
(usedModelId === actualModelId ? alibabaModelConfig : null),
|
||||||
|
}).catch((healthError) => {
|
||||||
|
chatApiLogger.warn("[Chat API] Record image health success failed:", healthError);
|
||||||
|
});
|
||||||
|
|
||||||
const usedModelDisplayName =
|
const usedModelDisplayName =
|
||||||
resolveModelDisplayNameByModelId(allModels, usedModelId) ||
|
resolveModelDisplayNameByModelId(allModels, usedModelId) ||
|
||||||
alibabaModelConfig?.displayName ||
|
alibabaModelConfig?.displayName ||
|
||||||
|
|||||||
@ -103,7 +103,7 @@ function normalizeResolution(imageSize?: string, apiFormat?: SnapgenApiFormat) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function toSnapgenResolutionParam(resolution: string) {
|
function toSnapgenResolutionParam(resolution: string) {
|
||||||
return resolution.trim().toLowerCase();
|
return resolution.trim().toUpperCase();
|
||||||
}
|
}
|
||||||
|
|
||||||
function normalizeOutputFormat(outputFormat?: string): "png" | "jpeg" {
|
function normalizeOutputFormat(outputFormat?: string): "png" | "jpeg" {
|
||||||
|
|||||||
@ -139,6 +139,22 @@ export function classifyAiModelHealthFailure(error: unknown): {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
httpStatus === 403 ||
|
||||||
|
/(?:^|\D)403(?:\D|$)/.test(normalized) ||
|
||||||
|
normalized === "forbidden" ||
|
||||||
|
normalized.includes("access forbidden") ||
|
||||||
|
normalized.includes("access denied") ||
|
||||||
|
normalized.includes("permission denied")
|
||||||
|
) {
|
||||||
|
return {
|
||||||
|
eventType: "unavailable",
|
||||||
|
severity: "critical",
|
||||||
|
counted: true,
|
||||||
|
reason: "provider_access_forbidden",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
if (
|
if (
|
||||||
normalized.includes("生成结果未通过替换质检") ||
|
normalized.includes("生成结果未通过替换质检") ||
|
||||||
normalized.includes("output_moderation") ||
|
normalized.includes("output_moderation") ||
|
||||||
@ -323,7 +339,10 @@ export function shouldFailoverAiModelError(error: unknown) {
|
|||||||
normalized.includes("no task id") ||
|
normalized.includes("no task id") ||
|
||||||
normalized.includes("missing task id");
|
normalized.includes("missing task id");
|
||||||
|
|
||||||
if (classification.reason === "provider_authentication_failed") {
|
if (
|
||||||
|
classification.reason === "provider_authentication_failed" ||
|
||||||
|
classification.reason === "provider_access_forbidden"
|
||||||
|
) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user