From 3a087d3392c810064460197b867ee03665582dec Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 5 Apr 2026 01:28:28 +0800 Subject: [PATCH] fix: stabilize canvas agent mode for kimi --- .../settings/admin-settings-client.tsx | 11 +- app/(chat)/api/chat/route.ts | 240 +++++++++++++++++- lib/ai/auto-agent-graph.ts | 13 +- lib/ai/model-compat.ts | 81 ++++++ lib/ai/prompt-optimizer.ts | 3 +- lib/ai/smart-analyzer.ts | 11 +- 6 files changed, 334 insertions(+), 25 deletions(-) create mode 100644 lib/ai/model-compat.ts diff --git a/app/(admin)/settings/admin-settings-client.tsx b/app/(admin)/settings/admin-settings-client.tsx index 4edf47d4f..61273b9bf 100644 --- a/app/(admin)/settings/admin-settings-client.tsx +++ b/app/(admin)/settings/admin-settings-client.tsx @@ -8310,6 +8310,12 @@ export function AdminSettingsClient() { /> +
+ 无限画布里“对话框”的 Agent 对话/识图默认模型，不在本区域设置； + 它取自 AI 模型管理里被标记为“Agent模式”的多模态模型。 + 这里的配置只影响 AI详情生成专用模型，以及 Agent模式自动出图时的默认生图模型。 +
+
@@ -8465,7 +8470,7 @@ export function AdminSettingsClient() {
- 当用户选择"Agent模式"时，系统实际使用的生图模型。"系统智能选择"会根据用户意图（如抠图、换背景等）自动路由到最佳模型。 + 当用户选择"Agent模式"并进入执行出图时，系统实际使用的默认生图模型。"系统智能选择"会根据用户意图（如抠图、换背景等）自动路由到最佳模型；这里不控制无限画布对话框里的 Agent 对话/识图模型。
diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index f25ba5c4e..b30d1f390 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -45,6 +45,7 @@ import { } from "@/lib/ai/rmbg/service"; import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; import { resolveDetailAgentLocalMoonshotModel } from "@/lib/ai/detail-agent-model-runtime"; +import { shouldForceInlineVisionInputs } from "@/lib/ai/model-compat"; import { inlineVisionImages } from "@/lib/ai/vision-inline"; import { createDocument } from "@/lib/ai/tools/create-document"; import { getWeather } from "@/lib/ai/tools/get-weather"; @@ -140,6 +141,7 @@ import { resolveClarificationBypass, resolveClarificationMessage, } from "@/lib/ai/runtime/clarification-interruption"; +import { resolveCanvasAgentClarificationGate } from "@/lib/ai/runtime/middlewares/canvas-agent-clarification-gate"; import { pruneTaskBoundaryContext, pruneTaskBoundaryHistoryMessages, @@ -2089,18 +2091,188 @@ function ensureImageAnalysisSummarySection(message: string, summary: string): st * 注意：gemini-image 和 huanke-gemini 都需要 URL 格式（不是 Base64） * 因为这些 provider 内部会自行处理图片上传到 S3 */ -async function convertMessagesWithImages(messages: ChatMessage[], providerType: string) { +async function convertMessagesWithImages( + messages: ChatMessage[], + providerType: + | string + | { + providerType?: string; + providerBaseUrl?: string | null; + modelId?: string | null; + displayName?: string | null; + forceInlineImageUrls?: boolean; + loggerPrefix?: string; + }, +) { + type ImageConversionTarget = { + providerType?: string; + providerBaseUrl?: string | null; + modelId?: string | null; + displayName?: string | null; + forceInlineImageUrls?: boolean; + loggerPrefix?: string; + }; + + const normalizeTarget = ( + target: string | ImageConversionTarget, + ): Required> & + Omit => { + if (typeof target === "string") { + return { + providerType: target, + providerBaseUrl: null, + modelId: null, + displayName: null, + forceInlineImageUrls: false, + loggerPrefix: "[Chat API] Vision inline", + }; + } + + return { + providerType: String(target.providerType || ""), + providerBaseUrl: target.providerBaseUrl || null, + modelId: target.modelId || null, + displayName: target.displayName || null, + forceInlineImageUrls: target.forceInlineImageUrls === true, + loggerPrefix: target.loggerPrefix || "[Chat API] Vision inline", + }; + }; + + const extractImagePartUrl = (part: unknown): string | null => { + const partAny = part as any; + const candidateUrl = + partAny?.type === "file" + ? partAny.url || partAny.data + : partAny?.type === "image" + ? partAny.image || partAny.url + : partAny?.type === "image_url" + ? partAny.image_url?.url + : undefined; + + if (typeof candidateUrl !== "string") return null; + const normalized = candidateUrl.trim(); + if (!/^https?:\/\//i.test(normalized)) return null; + return normalized; + }; + + const replaceImagePartUrl = (part: unknown, nextUrl: string): unknown => { + const partAny = part as any; + if (partAny?.type === "file") { + return { + ...partAny, + url: nextUrl, + data: nextUrl, + mediaType: String(partAny.mediaType || "").startsWith("image/") + ? partAny.mediaType + : "image/jpeg", + mimeType: String(partAny.mimeType || "").startsWith("image/") + ? partAny.mimeType + : "image/jpeg", + }; + } + if (partAny?.type === "image") { + return { + ...partAny, + image: nextUrl, + url: nextUrl, + }; + } + if (partAny?.type === "image_url") { + return { + ...partAny, + image_url: { + ...(partAny.image_url || {}), + url: nextUrl, + }, + }; + } + return part; + }; + + const inlineImageUrlsInMessages = async ( + sourceMessages: ChatMessage[], + target: ReturnType, + ) => { + const uniqueUrls = Array.from( + new Set( + sourceMessages + .flatMap((message) => message.parts || []) + .map((part) => extractImagePartUrl(part)) + .filter((url): url is string => Boolean(url)), + ), + ); + + if (uniqueUrls.length === 0) { + return sourceMessages; + } + + const { inlineDataUrls } = await inlineVisionImages(uniqueUrls, { + maxImages: Math.min(8, uniqueUrls.length), + maxSide: 1200, + timeoutMs: 12000, + retries: 2, + loggerPrefix: target.loggerPrefix, + }).catch((error) => { + console.warn(`${target.loggerPrefix} failed to inline message images:`, error); + return { inlineDataUrls: [], skipped: [] }; + }); + + if (inlineDataUrls.length === 0) { + return sourceMessages; + } + + const inlineMap = new Map(); + for (let index = 0; index < inlineDataUrls.length; index += 1) { + const originalUrl = uniqueUrls[index]; + const inlineUrl = inlineDataUrls[index]; + if (originalUrl && inlineUrl) { + inlineMap.set(originalUrl, inlineUrl); + } + } + + if (inlineMap.size === 0) { + return sourceMessages; + } + + console.log(`${target.loggerPrefix} inlined chat message images:`, { + total: uniqueUrls.length, + inlined: inlineMap.size, + providerType: target.providerType, + modelId: target.modelId, + displayName: target.displayName, + }); + + return sourceMessages.map((message) => ({ + ...message, + parts: (message.parts || []).map((part) => { + const originalUrl = extractImagePartUrl(part); + if (!originalUrl) return part; + const inlineUrl = inlineMap.get(originalUrl); + return inlineUrl ? replaceImagePartUrl(part, inlineUrl) : part; + }) as typeof message.parts, + })) as ChatMessage[]; + }; + + const target = normalizeTarget(providerType); + let preparedMessages = messages; + + if (target.forceInlineImageUrls) { + preparedMessages = await inlineImageUrlsInMessages(preparedMessages, target); + } + + const normalizedProviderType = target.providerType; + if ( - providerType === "bfl-image" || - providerType === "ruxa-image" || - providerType === "fal-image" || - providerType === "openai-image" || - providerType === "midjourney" || - providerType === "runway" + normalizedProviderType === "bfl-image" || + normalizedProviderType === "ruxa-image" || + normalizedProviderType === "fal-image" || + normalizedProviderType === "openai-image" || + normalizedProviderType === "midjourney" || + normalizedProviderType === "runway" ) { const convertedMessages = []; - for (const message of messages) { + for (const message of preparedMessages) { const content: any[] = []; for (const part of message.parts) { @@ -2149,8 +2321,8 @@ async function convertMessagesWithImages(messages: ChatMessage[], providerType: // 所有 provider 都使用 URL 格式传输图片 // gemini-image 和 huanke-gemini 内部会将 Base64 转换为 S3 URL // 这里不再进行 Base64 转换，直接返回标准格式 - if (providerType !== "gemini") { - return convertToModelMessages(messages); + if (normalizedProviderType !== "gemini") { + return convertToModelMessages(preparedMessages); } // 🆕 检测是否有标注图片 @@ -2162,7 +2334,7 @@ async function convertMessagesWithImages(messages: ChatMessage[], providerType: // 对于 Gemini，需要特殊处理图片 const convertedMessages = []; - for (const message of messages) { + for (const message of preparedMessages) { const content: any[] = []; // 🆕 如果是用户消息且有标注图片，在文本前添加系统提示 @@ -6407,6 +6579,34 @@ export async function POST(request: Request) { } } + if (shouldRunAgentConversation) { + const canvasClarificationGate = resolveCanvasAgentClarificationGate({ + text: requestTextContent, + composerMode, + contextPayload: effectiveContextPayload, + mediaInput: requestResolvedMediaInput, + targetResolution: requestTargetResolution, + taskBoundaryDecision, + executionMode: smartAutoExecution?.mode, + actionKind: resolvedConversationAction.kind, + }); + + if (canvasClarificationGate.shouldClarify) { + smartAutoNeedsClarification = true; + smartAutoClarificationQuestion = + canvasClarificationGate.clarificationMessage || smartAutoClarificationQuestion; + smartAutoFriendlyMessage = + canvasClarificationGate.clarificationMessage || smartAutoFriendlyMessage; + console.log("[Smart Auto Mode] 🧭 Canvas clarification gate triggered:", { + reason: canvasClarificationGate.reason, + executionMode: smartAutoExecution?.mode, + resolutionMode: requestTargetResolution?.resolutionMode, + selectedCount: requestResolvedMediaInput?.selectedImageRefs.length || 0, + candidateCount: requestTargetResolution?.candidateRefs.length || 0, + }); + } + } + if (shouldRunAgentConversation && smartAutoNeedsClarification) { const clarificationMessage = resolveClarificationMessage({ clarificationQuestion: smartAutoClarificationQuestion, @@ -9054,6 +9254,10 @@ export async function POST(request: Request) { const currentModelConfig = isUuidModelId ? await getAiModelById({ id: finalModelId }).catch(() => undefined) : undefined; + const currentProviderConfig = + currentModelConfig?.providerId + ? await getAiProviderById({ id: currentModelConfig.providerId }).catch(() => null) + : null; const runtimeProviderType = String( resolvedProviderType || (languageModel as any).provider || "" ); @@ -10878,7 +11082,19 @@ export async function POST(request: Request) { // 转换消息，对于 Gemini 和千问视频将图片 URL 转换为 base64 const convertedMessages = await convertMessagesWithImages( injectHistoryImagesToLastUserMessage(uiMessagesForModel, smartAutoHistoryImages), - (languageModel as any).provider || "" + { + providerType: (languageModel as any).provider || "", + providerBaseUrl: currentProviderConfig?.baseUrl || null, + modelId: currentModelConfig?.modelId || actualModelId, + displayName: currentModelConfig?.displayName || actualModelId, + forceInlineImageUrls: shouldForceInlineVisionInputs({ + providerType: resolvedProviderType || currentProviderConfig?.type || runtimeProviderType, + providerBaseUrl: currentProviderConfig?.baseUrl || null, + modelId: currentModelConfig?.modelId || actualModelId, + displayName: currentModelConfig?.displayName || actualModelId, + }), + loggerPrefix: "[Chat API] Stream text vision inline", + } ); console.log("[Chat API] Messages converted, count:", convertedMessages.length); // 调试日志 diff --git a/lib/ai/auto-agent-graph.ts b/lib/ai/auto-agent-graph.ts index fabe9e3d6..7f37dbd2f 100644 --- a/lib/ai/auto-agent-graph.ts +++ b/lib/ai/auto-agent-graph.ts @@ -1,6 +1,7 @@ import { Annotation, StateGraph, START, END } from "@langchain/langgraph"; import { generateText } from "ai"; import sharp from "sharp"; +import { getCompatibleTextGenerationTemperature } from "@/lib/ai/model-compat"; import type { AnalysisResult } from "@/lib/ai/smart-analyzer"; import type { RouteResult } from "@/lib/ai/model-router"; import type { AgentSkillRouteResult } from "@/lib/ai/skill-router"; @@ -1359,7 +1360,7 @@ async function buildExecutionPlanWithAgent( ] .filter(Boolean) .join("\n\n"), - temperature: 0.1, + temperature: getCompatibleTextGenerationTemperature(plannerModelConfig, 0.1), maxTokens: 420, abortSignal: AbortSignal.timeout(AGENT_OPERATION_TIMEOUT_MS), } as any); @@ -1482,7 +1483,7 @@ async function generateAgentConversation( content: [{ type: "text" as const, text: visionPrompt }, ...visionInputs], }, ], - temperature: 0.2, + temperature: getCompatibleTextGenerationTemperature(visionModelConfig, 0.2), maxTokens: 360, abortSignal: AbortSignal.timeout(AGENT_OPERATION_TIMEOUT_MS), } as any); @@ -1544,7 +1545,7 @@ async function generateAgentConversation( model: reasoningModel, system: systemPrompt, prompt, - temperature: 0.35, + temperature: getCompatibleTextGenerationTemperature(reasoningModelConfig, 0.35), maxTokens: 520, abortSignal: AbortSignal.timeout(AGENT_OPERATION_TIMEOUT_MS), } as any); @@ -1626,7 +1627,7 @@ async function recommendImageParamsWithAgent( - 若用户未指定比例，优先 auto。 - negativePrompt 仅在明显需要避坑时填写。`, prompt: requestText, - temperature: 0.15, + temperature: getCompatibleTextGenerationTemperature(plannerModelConfig, 0.15), maxTokens: 220, abortSignal: AbortSignal.timeout(AGENT_OPERATION_TIMEOUT_MS), } as any); @@ -1719,7 +1720,7 @@ async function recommendVideoPlanWithAgent( - 有参考图时优先 first-frame 或 first-last-frame。 - 避免抽象空话，必须可执行。`, prompt, - temperature: 0.2, + temperature: getCompatibleTextGenerationTemperature(plannerModelConfig, 0.2), maxTokens: 720, abortSignal: AbortSignal.timeout(AGENT_OPERATION_TIMEOUT_MS), } as any); @@ -1799,7 +1800,7 @@ async function refineRouteWithAgent( 意图：${state.analysis.intent}/${state.analysis.subIntent || "none"} 候选模型： ${candidatesText}`, - temperature: 0.1, + temperature: getCompatibleTextGenerationTemperature(plannerModelConfig, 0.1), maxTokens: 120, abortSignal: AbortSignal.timeout(AGENT_OPERATION_TIMEOUT_MS), } as any); diff --git a/lib/ai/model-compat.ts b/lib/ai/model-compat.ts new file mode 100644 index 000000000..167e6eecc --- /dev/null +++ b/lib/ai/model-compat.ts @@ -0,0 +1,81 @@ +type ModelCompatDescriptor = { + providerType?: string | null; + providerBaseUrl?: string | null; + modelId?: string | null; + displayName?: string | null; + provider?: { + type?: string | null; + baseUrl?: string | null; + } | null; + model?: { + modelId?: string | null; + displayName?: string | null; + } | null; +}; + +const KIMI_MODEL_PATTERN = /\bkimi(?:[-_.\s]?[a-z0-9.]+)?\b/i; + +function normalizeProviderType(input: ModelCompatDescriptor): string { + return String(input.providerType || input.provider?.type || "") + .trim() + .toLowerCase(); +} + +function normalizeProviderBaseUrl(input: ModelCompatDescriptor): string { + return String(input.providerBaseUrl || input.provider?.baseUrl || "") + .trim() + .toLowerCase(); +} + +function normalizeModelText(input: ModelCompatDescriptor): string { + return `${String(input.displayName || input.model?.displayName || "")} ${String(input.modelId || input.model?.modelId || "")}` + .trim() + .toLowerCase(); +} + +export function isMoonshotKimiModel( + input?: ModelCompatDescriptor | null, +): boolean { + if (!input) { + return false; + } + + const providerType = normalizeProviderType(input); + const providerBaseUrl = normalizeProviderBaseUrl(input); + const modelText = normalizeModelText(input); + + if (providerType.includes("moonshot")) { + return true; + } + + if (providerBaseUrl.includes("moonshot.cn")) { + return true; + } + + if (modelText.includes("moonshot")) { + return true; + } + + if (KIMI_MODEL_PATTERN.test(modelText)) { + return true; + } + + return false; +} + +export function getCompatibleTextGenerationTemperature( + input: ModelCompatDescriptor | null | undefined, + preferredTemperature?: number, +): number | undefined { + if (isMoonshotKimiModel(input)) { + return 1; + } + + return preferredTemperature; +} + +export function shouldForceInlineVisionInputs( + input: ModelCompatDescriptor | null | undefined, +): boolean { + return isMoonshotKimiModel(input); +} diff --git a/lib/ai/prompt-optimizer.ts b/lib/ai/prompt-optimizer.ts index f288bb9fb..679c41866 100644 --- a/lib/ai/prompt-optimizer.ts +++ b/lib/ai/prompt-optimizer.ts @@ -170,6 +170,7 @@ const EDIT_TEMPLATES: Record< }; import { generateText } from "ai"; +import { getCompatibleTextGenerationTemperature } from "@/lib/ai/model-compat"; function inferStyleTransferTarget(originalPrompt: string, extracted?: string): string { const text = (originalPrompt || "").trim(); @@ -674,7 +675,7 @@ export async function optimizePromptWithLLM(params: { model, system: optimizerSystemPrompt, prompt: finalUserMessage, - temperature: 0.2, + temperature: getCompatibleTextGenerationTemperature(selectedOptimizerModel, 0.2), maxTokens: 200, } as any); diff --git a/lib/ai/smart-analyzer.ts b/lib/ai/smart-analyzer.ts index 97373af36..00dbc7256 100644 --- a/lib/ai/smart-analyzer.ts +++ b/lib/ai/smart-analyzer.ts @@ -6,6 +6,7 @@ */ import { generateText } from "ai"; +import { getCompatibleTextGenerationTemperature } from "@/lib/ai/model-compat"; import sharp from "sharp"; import { isAiModelMultimodal } from "@/lib/db/ai-model-flags"; // 使用动态导入避免循环依赖问题 @@ -1505,6 +1506,7 @@ export async function summarizeImageForExecutionReport(params: { if (fallbackVisionImages.length === 0) continue; try { + const compatibleTemperature = getCompatibleTextGenerationTemperature(candidate, 0.1); const { text: summaryText } = await generateText({ model, messages: [ @@ -1519,7 +1521,9 @@ export async function summarizeImageForExecutionReport(params: { ], }, ], - temperature: 0.1, + ...(compatibleTemperature !== undefined + ? { temperature: compatibleTemperature } + : {}), maxTokens: 120, abortSignal: AbortSignal.timeout(ANALYZER_TIMEOUT_MS), } as any); @@ -2107,6 +2111,7 @@ ${historyText} ); // 调用模型分析（有图时携带图片，确保真实视觉分析） + const analyzerTemperature = getCompatibleTextGenerationTemperature(selectedAnalyzerModel, 0.1); const requestPayload = hasImage && visionImages.length > 0 && !cachedVisualSummary ? { @@ -2121,7 +2126,7 @@ ${historyText} ], }, ], - temperature: 0.1, + ...(analyzerTemperature !== undefined ? { temperature: analyzerTemperature } : {}), maxTokens: 360, abortSignal: AbortSignal.timeout(ANALYZER_TIMEOUT_MS), } @@ -2129,7 +2134,7 @@ ${historyText} model, system: ANALYZER_SYSTEM_PROMPT, prompt: userMessage, - temperature: 0.1, + ...(analyzerTemperature !== undefined ? { temperature: analyzerTemperature } : {}), maxTokens: 300, abortSignal: AbortSignal.timeout(ANALYZER_TIMEOUT_MS), };