fix(agent): fail over rejected credentials and allow local image downloads

This commit is contained in:
134 2026-10-19 17:55:29 +08:00
parent 02fad667f9
commit 45c554fc27
9 changed files with 334 additions and 8 deletions

View File

@ -193,6 +193,7 @@ import {
} from "@/lib/ai/runtime/agent-media-markdown";
import { extractImageUrlsFromText } from "@/lib/image-url";
import { saveGeneratedImages } from "@/lib/ai/runtime/generated-image-persistence";
import { withAuthenticationFailover } from "@/lib/ai/runtime/authentication-failover";
import {
recordAiModelHealthFailure,
recordAiModelHealthSuccess,
@ -1222,6 +1223,14 @@ export async function POST(request: Request) {
}
}
if (shouldRunAgentConversation && preferredAgentModelId && !shouldKeepSelectedModel) {
const preferred = await getAiModelById({ id: preferredAgentModelId }).catch(() => null);
if (!preferred?.isEnabled || preferred.healthStatus === "auto_disabled") {
preferredAgentModelId = undefined;
configuredAgentConversationModelId = undefined;
}
}
if (shouldRunAgentConversation && !preferredAgentModelId) {
const configuredAgentModel = await getAgentAiModelWithProvider().catch((agentModelError) => {
chatApiLogger.warn("[Chat API] Failed to load configured agent model:", agentModelError);
@ -8842,7 +8851,7 @@ export async function POST(request: Request) {
.filter(Boolean)
.join("\n\n");
const activeRuntimeModelDescriptor = await resolveAgentModelSlotDescriptor(
let activeRuntimeModelDescriptor = await resolveAgentModelSlotDescriptor(
resolvedRuntimeModelLookupId,
).catch(() => null);
if (shouldRunAgentConversation && activeRuntimeModelDescriptor) {
@ -12222,8 +12231,70 @@ export async function POST(request: Request) {
const shouldDisableSmoothStream =
resolvedModelSupportsImageGen || isVolcanoVideoModel;
let activeConversationModel = currentModelConfig;
let activeConversationProvider = currentProviderConfig;
const recordedAuthFailures = new Set<string>();
const recordConversationAuthFailure = async (error: unknown) => {
const key = activeConversationModel?.id;
if (!key || recordedAuthFailures.has(key)) return;
recordedAuthFailures.add(key);
recordProviderHealthFailure({
providerKey: activeConversationProvider?.id,
providerType: activeConversationProvider?.type,
error,
});
await recordAiModelHealthFailure({
model: activeConversationModel,
provider: activeConversationProvider,
error,
context: "canvas_agent_conversation_auth",
}).catch((healthError) => chatApiLogger.warn("[Chat API] Could not persist model authentication failure:", healthError));
};
const needsVisionFallback = convertedMessages.some((item: any) =>
Array.isArray(item.content) && item.content.some((part: any) =>
part.type === "image" || (part.type === "file" && String(part.mediaType || "").startsWith("image/"))
)
);
const streamModel = shouldRunAgentConversation && !shouldKeepSelectedModel &&
currentModelConfig?.excludeFromFailover !== true &&
!resolvedModelSupportsImageGen && !isVolcanoVideoModel
? withAuthenticationFailover({
primary: languageModel as any,
onAuthenticationFailure: recordConversationAuthFailure,
resolveFallback: async () => {
const enabled = await getEnabledAiModelsWithProvider(false);
const preferredIds = [currentModelConfig?.fallbackModelId1, currentModelConfig?.fallbackModelId2];
const candidates = enabled.filter(({ model, provider }) =>
model.id !== activeConversationModel?.id && model.isEnabled && provider.isEnabled &&
!model.excludeFromFailover && !model.isSubModel && model.healthStatus !== "auto_disabled" &&
!isImageGenerationModelType(provider.type, model) && !isVideoProviderType(provider.type, model) &&
(!needsVisionFallback || model.supportsVision) &&
(isAiModelAgentAuto(model) || preferredIds.includes(model.id)) &&
!shouldAvoidProvider(getProviderHealthSnapshot(provider.id))
).sort((a, b) => {
const rank = (id: string, isDefault: boolean) => {
const index = preferredIds.indexOf(id);
return index >= 0 ? index : isDefault ? 2 : 3;
};
return rank(a.model.id, a.model.isDefault) - rank(b.model.id, b.model.isDefault);
});
for (const candidate of candidates) {
const fallback = await getLanguageModelById(candidate.model.id);
if (!fallback || typeof fallback === "string") continue;
activeConversationModel = candidate.model;
activeConversationProvider = candidate.provider;
finalModelId = candidate.model.id;
actualModelId = candidate.model.modelId;
activeRuntimeModelDescriptor = await resolveAgentModelSlotDescriptor(candidate.model.id).catch(() => null);
return fallback as any;
}
return null;
},
})
: languageModel;
const result = streamText({
model: languageModel,
model: streamModel,
maxRetries: shouldRunAgentConversation ? 0 : undefined,
system: effectiveConversationSystemPrompt,
messages: convertedMessages,
stopWhen: stepCountIs(shouldUseCanvasCapabilityToolLoop ? 8 : 5),

View File

@ -0,0 +1,52 @@
import type { LanguageModelV2 } from "@ai-sdk/provider";
export function isProviderAuthenticationFailure(error: unknown, depth = 0): boolean {
if (!error || depth > 3) return false;
const value = typeof error === "object" ? error as Record<string, unknown> : {};
const status = Number(value.statusCode || value.status || value.httpStatus);
const message = String(value.message || (typeof error === "string" ? error : "")).toLowerCase();
return status === 401 || /\b401\b/.test(message) ||
/invalid authentication|invalid[ _]api[ _]key|authentication[_ ](?:failed|error)|密钥鉴权失败/.test(message) ||
isProviderAuthenticationFailure(value.cause, depth + 1);
}
export function isCredentialFailureBan(reason?: string | null) {
return /^provider_(?:authentication_failed|access_forbidden)\b/.test(String(reason || ""));
}
// Retry only an explicitly rejected authentication request, before the provider
// returns a stream. Never replay accepted streams, tools, timeouts or 5xx POSTs.
export function withAuthenticationFailover(params: {
primary: LanguageModelV2;
onAuthenticationFailure: (error: unknown) => Promise<void>;
resolveFallback: () => Promise<LanguageModelV2 | null>;
}): LanguageModelV2 {
let active = params.primary;
let recovery: Promise<LanguageModelV2 | null> | undefined;
const invoke = async <T>(call: (model: LanguageModelV2) => PromiseLike<T>, signal?: AbortSignal): Promise<T> => {
const attempted = active;
try {
return await call(attempted);
} catch (error) {
if (signal?.aborted || attempted !== params.primary || !isProviderAuthenticationFailure(error)) throw error;
recovery ||= (async () => {
await params.onAuthenticationFailure(error);
const fallback = await params.resolveFallback();
if (fallback && fallback !== params.primary) active = fallback;
return fallback;
})();
const fallback = await recovery;
if (!fallback || fallback === attempted) throw error;
signal?.throwIfAborted();
return await call(fallback);
}
};
return {
specificationVersion: "v2",
get provider() { return active.provider; },
get modelId() { return active.modelId; },
get supportedUrls() { return active.supportedUrls; },
doGenerate: (options) => invoke((model) => model.doGenerate(options), options.abortSignal),
doStream: (options) => invoke((model) => model.doStream(options), options.abortSignal),
};
}

View File

@ -1,4 +1,5 @@
import "server-only";
import { isProviderAuthenticationFailure, isCredentialFailureBan } from "./authentication-failover";
import { and, desc, eq, inArray, sql } from "drizzle-orm";
import { db } from "@/lib/db";
@ -127,7 +128,7 @@ export function classifyAiModelHealthFailure(error: unknown): {
}
if (
httpStatus === 401 ||
isProviderAuthenticationFailure(error) ||
/(?:^|\D)401(?:\D|$)/.test(normalized) ||
normalized.includes("unauthorized") ||
normalized.includes("invalid api key") ||
@ -475,6 +476,7 @@ export async function reviveAiModelsFromCooldown(params?: { modelId?: string })
healthConsecutiveFailures: aiModel.healthConsecutiveFailures,
healthLastEventType: aiModel.healthLastEventType,
healthAutoDisabledAt: aiModel.healthAutoDisabledAt,
healthAutoDisabledReason: aiModel.healthAutoDisabledReason,
healthLastFailureAt: aiModel.healthLastFailureAt,
healthUpdatedAt: aiModel.healthUpdatedAt,
})
@ -482,6 +484,7 @@ export async function reviveAiModelsFromCooldown(params?: { modelId?: string })
.where(and(...conditions));
const dueModels = candidates.filter((model) => {
if (isCredentialFailureBan(model.healthAutoDisabledReason)) return false;
const cooldownStartedAt =
model.healthAutoDisabledAt ||
model.healthLastFailureAt ||
@ -663,7 +666,8 @@ export async function recordAiModelHealthFailure(params: {
classification.counted &&
!existing.excludeFromFailover &&
!existing.excludeFromHealthBan &&
((isObservation && existing.isEnabled) ||
(classification.reason === "provider_authentication_failed" ||
(isObservation && existing.isEnabled) ||
(existing.healthStatus !== "auto_disabled" &&
nextConsecutiveFailures >= AUTO_DISABLE_FAILURE_THRESHOLD));
const nextStatus = shouldAutoDisable

View File

@ -1,6 +1,7 @@
import "server-only";
import { isProviderAuthenticationFailure } from "./authentication-failover";
export type ProviderHealthEventType = "success" | "timeout" | "quota" | "error";
export type ProviderHealthEventType = "success" | "timeout" | "quota" | "auth" | "error";
export interface ProviderHealthSnapshot {
providerKey: string;
@ -19,6 +20,7 @@ function normalizeErrorMessage(error: unknown) {
}
export function classifyProviderHealthEvent(error: unknown): ProviderHealthEventType {
if (isProviderAuthenticationFailure(error)) return "auth";
const normalized = normalizeErrorMessage(error).toLowerCase();
if (
@ -93,7 +95,7 @@ export function shouldAvoidProvider(snapshot?: ProviderHealthSnapshot | null) {
if (!snapshot) return false;
const ageMs = Date.now() - snapshot.updatedAt;
if (snapshot.lastEventType === "quota") {
if (snapshot.lastEventType === "quota" || snapshot.lastEventType === "auth") {
return ageMs <= 15 * 60 * 1000;
}

View File

@ -28,6 +28,7 @@ export async function resolveCanvasCapabilityContext(params: {
command: CanvasCapabilityClientCommand;
projectId: string;
requiresRemoteImage?: boolean;
allowUnsavedDownloadTargets?: boolean;
userId: string;
}) {
const [project] = await db
@ -57,9 +58,14 @@ export async function resolveCanvasCapabilityContext(params: {
const nodeById = new Map<string, Record<string, any>>(
nodes.map((node: Record<string, any>) => [String(node?.id || ""), node])
);
const targets = params.command.nodeIds.map((nodeId) => {
const targets = params.command.nodeIds.map((nodeId): CanvasCapabilityTarget | null => {
const node = nodeById.get(nodeId);
if (!node) {
// Downloading uses the browser's current image URL. A freshly generated
// node can exist there before autosave; it does not need a server target.
if (params.allowUnsavedDownloadTargets && params.command.capabilityId === "image.download") {
return null;
}
throw new CanvasCapabilityRuntimeError(
`选中节点不存在:${nodeId}`,
404,
@ -95,6 +101,6 @@ export async function resolveCanvasCapabilityContext(params: {
return {
project,
canvasData,
targets,
targets: targets.filter((target): target is CanvasCapabilityTarget => target !== null),
};
}

View File

@ -200,6 +200,8 @@ export async function executeCanvasCapabilityRuntime(params: {
command,
projectId: params.projectId,
requiresRemoteImage: capability.requiresRemoteImage,
allowUnsavedDownloadTargets:
capability.id === "image.download" && capability.executionMode === "local" && capability.risk === "read",
userId: params.userId,
});
let task = await createOrRecoverCanvasCapabilityTask({

View File

@ -0,0 +1,61 @@
import assert from "node:assert/strict";
import { test } from "node:test";
import type { LanguageModelV2, LanguageModelV2CallOptions } from "@ai-sdk/provider";
import { isCredentialFailureBan, isProviderAuthenticationFailure, withAuthenticationFailover } from "@/lib/ai/runtime/authentication-failover";
const options = { prompt: [] } as unknown as LanguageModelV2CallOptions;
function model(id: string, generate: LanguageModelV2["doGenerate"], stream: LanguageModelV2["doStream"]): LanguageModelV2 {
return { specificationVersion: "v2", provider: "test-provider", modelId: id, supportedUrls: {}, doGenerate: generate, doStream: stream };
}
const authError = () => Object.assign(new Error("Invalid Authentication"), { statusCode: 401 });
const unused = async (): Promise<never> => { throw new Error("Unexpected model method"); };
test("an explicit 401 before streaming switches model once and preserves request tools", async () => {
let failures = 0; let lookups = 0; let primaryCalls = 0; let fallbackCalls = 0;
const stream = new ReadableStream({ start(controller) { controller.close(); } });
const fallback = model("working-vision", unused, async (received) => { fallbackCalls++; assert.equal(received, options); return { stream }; });
const wrapped = withAuthenticationFailover({
primary: model("rejected", unused, async () => { primaryCalls++; throw authError(); }),
onAuthenticationFailure: async () => { failures++; },
resolveFallback: async () => { lookups++; return fallback; },
});
assert.equal((await wrapped.doStream(options)).stream, stream);
await wrapped.doStream(options);
assert.equal(wrapped.modelId, "working-vision");
assert.deepEqual({ failures, lookups, primaryCalls, fallbackCalls }, { failures: 1, lookups: 1, primaryCalls: 1, fallbackCalls: 2 });
});
test("timeouts, 5xx and outcome-unknown failures never replay paid requests", async () => {
for (const error of [Object.assign(new Error("request_outcome_unknown"), { statusCode: 503 }), new Error("Request timed out"), Object.assign(new Error("Too many requests"), { statusCode: 429 })]) {
const wrapped = withAuthenticationFailover({ primary: model("primary", unused, async () => { throw error; }), onAuthenticationFailure: unused, resolveFallback: unused });
await assert.rejects(async () => await wrapped.doStream(options), (caught) => caught === error);
}
});
test("once a stream has been accepted, an error inside it is not retried", async () => {
const stream = new ReadableStream({ start(controller) { controller.enqueue({ type: "error", error: authError() }); controller.close(); } });
const wrapped = withAuthenticationFailover({ primary: model("accepted", unused, async () => ({ stream })), onAuthenticationFailure: unused, resolveFallback: unused });
assert.equal((await wrapped.doStream(options)).stream, stream);
});
test("a failing fallback does not start another retry loop", async () => {
let fallbackCalls = 0;
const fallback = model("also-rejected", unused, async () => { fallbackCalls++; throw authError(); });
const wrapped = withAuthenticationFailover({ primary: model("primary", unused, async () => { throw authError(); }), onAuthenticationFailure: async () => {}, resolveFallback: async () => fallback });
await assert.rejects(async () => await wrapped.doStream(options), /Invalid Authentication/);
assert.equal(fallbackCalls, 1);
});
test("user cancellation prevents fallback submission", async () => {
const controller = new AbortController(); controller.abort();
const wrapped = withAuthenticationFailover({ primary: model("cancelled", unused, async () => { throw authError(); }), onAuthenticationFailure: unused, resolveFallback: unused });
await assert.rejects(async () => await wrapped.doStream({ ...options, abortSignal: controller.signal }), /Invalid Authentication/);
});
test("credential failures are recognized separately from temporary outages", () => {
assert(isProviderAuthenticationFailure(new Error("Invalid Authentication")));
assert(isProviderAuthenticationFailure({ cause: { status: 401 } }));
assert(!isProviderAuthenticationFailure({ status: 503, message: "Request timed out" }));
assert(isCredentialFailureBan("provider_authentication_failed: replace API key"));
assert(!isCredentialFailureBan("request_timeout: upstream busy"));
});

View File

@ -0,0 +1,50 @@
import assert from "node:assert/strict";
import { randomUUID } from "node:crypto";
import { after, before, test } from "node:test";
import { eq } from "drizzle-orm";
import { db } from "@/lib/db";
import { aiModel, aiProvider } from "@/lib/db/schema";
import { recordAiModelHealthFailure, reviveAiModelsFromCooldown } from "@/lib/ai/runtime/model-health";
const database = new URL(process.env.POSTGRES_URL || "http://missing");
if (database.hostname !== "127.0.0.1" || !/_test$/.test(database.pathname)) throw new Error("Use a disposable local database");
const providerId = randomUUID();
const authModelId = randomUUID();
const transientModelId = randomUUID();
before(async () => {
await db.insert(aiProvider).values({ id: providerId, name: "Synthetic credential test", type: "custom", baseUrl: "https://invalid.example", apiKey: "synthetic-unused-test-value" });
await db.insert(aiModel).values([
{ id: authModelId, providerId, modelId: "auth-test", displayName: "Auth test" },
{ id: transientModelId, providerId, modelId: "transient-test", displayName: "Transient test" },
]);
});
after(async () => {
await db.delete(aiProvider).where(eq(aiProvider.id, providerId));
await db.$client.end({ timeout: 2 });
});
test("a confirmed authentication rejection disables the invalid model immediately", async () => {
await recordAiModelHealthFailure({ model: { id: authModelId }, error: Object.assign(new Error("Invalid Authentication"), { statusCode: 401 }) });
const [model] = await db.select().from(aiModel).where(eq(aiModel.id, authModelId));
assert.equal(model.isEnabled, false);
assert.equal(model.healthStatus, "auto_disabled");
assert.equal(model.healthConsecutiveFailures, 1);
assert.match(model.healthAutoDisabledReason || "", /^provider_authentication_failed/);
});
test("waiting for cooldown cannot reactivate a model with invalid credentials", async () => {
await db.update(aiModel).set({ healthAutoDisabledAt: new Date(Date.now() - 3 * 60 * 60_000) }).where(eq(aiModel.id, authModelId));
assert.deepEqual(await reviveAiModelsFromCooldown({ modelId: authModelId }), []);
const [model] = await db.select().from(aiModel).where(eq(aiModel.id, authModelId));
assert.equal(model.isEnabled, false);
});
test("temporary outages can still enter the normal observation pool after cooldown", async () => {
for (let i = 0; i < 3; i++) await recordAiModelHealthFailure({ model: { id: transientModelId }, error: Object.assign(new Error("Service unavailable"), { statusCode: 503 }) });
await db.update(aiModel).set({ healthAutoDisabledAt: new Date(Date.now() - 3 * 60 * 60_000) }).where(eq(aiModel.id, transientModelId));
const revived = await reviveAiModelsFromCooldown({ modelId: transientModelId });
assert.equal(revived.length, 1);
assert.equal(revived[0].isEnabled, true);
assert.equal(revived[0].healthStatus, "degraded");
});

View File

@ -0,0 +1,78 @@
import assert from "node:assert/strict";
import { randomUUID } from "node:crypto";
import { after, before, test } from "node:test";
import { eq } from "drizzle-orm";
import { db } from "@/lib/db";
import { canvasProject, user } from "@/lib/db/schema";
import { resolveCanvasCapabilityContext } from "@/lib/canvas/capabilities/context";
import { createCanvasCapabilityClientCommand } from "@/lib/canvas/capabilities/client-command";
import { executeCanvasCapabilityRuntime } from "@/lib/canvas/capabilities/runtime";
import { downloadCanvasImageAsPng } from "@/lib/client/canvas-image-download";
const database = new URL(process.env.POSTGRES_URL || "http://missing");
if (!["127.0.0.1", "localhost"].includes(database.hostname) || !/_test$/.test(database.pathname)) {
throw new Error("Use a disposable local test database");
}
const owner = randomUUID();
const stranger = randomUUID();
const projectId = randomUUID();
const unsavedNodeId = randomUUID();
const textNodeId = randomUUID();
const download = () => createCanvasCapabilityClientCommand({ capabilityId: "image.download", nodeIds: [unsavedNodeId], projectId, source: "toolbar" });
before(async () => {
await db.insert(user).values([{ id: owner, email: `${owner}@download.invalid` }, { id: stranger, email: `${stranger}@download.invalid` }]);
await db.insert(canvasProject).values({ id: projectId, userId: owner, title: "Local download regression", canvasData: { nodes: [{ id: textNodeId, type: "text", data: { text: "Not an image" } }], edges: [] } });
});
after(async () => {
await db.delete(user).where(eq(user.id, owner));
await db.delete(user).where(eq(user.id, stranger));
await db.$client.end({ timeout: 2 });
});
test("download dispatch succeeds while a generated node is waiting for autosave", async () => {
const result = await executeCanvasCapabilityRuntime({ command: download(), projectId, userId: owner, requestHeaders: new Headers(), requestUrl: "https://test.invalid" });
assert.equal(result.mode, "interactive");
assert.equal(result.clientCommand?.capabilityId, "image.download");
assert.deepEqual(result.clientCommand?.nodeIds, [unsavedNodeId]);
assert.equal(result.task.status, "succeeded");
});
test("unsaved download permission never bypasses project ownership", async () => {
await assert.rejects(resolveCanvasCapabilityContext({ command: download(), projectId, userId: stranger, allowUnsavedDownloadTargets: true }), { code: "CANVAS_CAPABILITY_PROJECT_NOT_FOUND" });
});
test("other image operations still require a persisted node", async () => {
const command = createCanvasCapabilityClientCommand({ capabilityId: "image.enhance", nodeIds: [unsavedNodeId], projectId, source: "toolbar" });
await assert.rejects(resolveCanvasCapabilityContext({ command, projectId, userId: owner, allowUnsavedDownloadTargets: true }), { code: "CANVAS_CAPABILITY_NODE_NOT_FOUND" });
});
test("existing non-image nodes cannot masquerade as downloadable images", async () => {
const command = createCanvasCapabilityClientCommand({ capabilityId: "image.download", nodeIds: [textNodeId], projectId, source: "toolbar" });
await assert.rejects(resolveCanvasCapabilityContext({ command, projectId, userId: owner, allowUnsavedDownloadTargets: true }), { code: "CANVAS_CAPABILITY_INVALID_NODE_TYPE" });
});
test("the browser download uses the current image URL without node persistence or a model call", async () => {
const original = { fetch: globalThis.fetch, document: globalThis.document, window: globalThis.window };
const requests: Array<{ path: string; body: Record<string, unknown> }> = [];
const frames: Array<{ src?: string }> = [];
try {
globalThis.fetch = async (input, init) => {
requests.push({ path: String(input), body: JSON.parse(String(init?.body)) });
return Response.json({ delivery: "cdn", downloadUrl: "https://cdn.example/download.png", filename: "original.png", bytes: 123 });
};
globalThis.document = { createElement: () => ({}), body: { appendChild: (frame: { src?: string }) => frames.push(frame) } } as unknown as Document;
globalThis.window = { setTimeout: () => 1 } as unknown as Window & typeof globalThis;
const result = await downloadCanvasImageAsPng({ url: "https://provider.example/new-unsaved-image.png", name: "Original" });
assert.equal(result.filename, "original.png");
assert.equal(requests.length, 1);
assert.equal(requests[0].path, "/api/canvas/download-image");
assert.equal(requests[0].body.url, "https://provider.example/new-unsaved-image.png");
assert.equal(requests[0].body.nodeIds, undefined);
assert.equal(frames[0].src, "https://cdn.example/download.png");
} finally {
globalThis.fetch = original.fetch;
globalThis.document = original.document;
globalThis.window = original.window;
}
});