fix: stabilize canvas agent mode for kimi
Some checks failed
Lint / build (20) (push) Has been cancelled
Playwright Tests / test (push) Has been cancelled

This commit is contained in:
Your Name 2026-04-05 01:28:28 +08:00
parent 2a322feea9
commit 3a087d3392
6 changed files with 334 additions and 25 deletions

View File

@ -8310,6 +8310,12 @@ export function AdminSettingsClient() {
/>
</div>
<div className="rounded-lg border border-amber-200 bg-amber-50 px-3 py-2 text-xs leading-5 text-amber-900">
无限画布里“对话框”的 Agent 对话/识图默认模型,不在本区域设置;
它取自 AI 模型管理里被标记为“Agent模式”的多模态模型。
这里的配置只影响 AI详情生成专用模型,以及 Agent模式自动出图时的默认生图模型。
</div>
<div className="space-y-2 mt-4">
<Label htmlFor="canvasDetailAgentModelId">
AI详情生成 - Agent专用模型
@ -8352,8 +8358,7 @@ export function AdminSettingsClient() {
</Select>
<div className="text-xs text-muted-foreground">
用于 AI详情生成 的分镜规划与排版复审。未设置时沿用
Agent 对话默认模型;建议选择官方 Kimi K2.5
多模态模型。
Agent 对话默认模型;仅影响 AI详情生成,不影响普通无限画布对话框的 Agent 对话。建议选择官方 Kimi K2.5 多模态模型。
</div>
</div>
@ -8465,7 +8470,7 @@ export function AdminSettingsClient() {
</SelectContent>
</Select>
<div className="text-xs text-muted-foreground">
当用户选择"Agent模式"时,系统实际使用的生图模型。"系统智能选择"会根据用户意图(如抠图、换背景等)自动路由到最佳模型。
当用户选择"Agent模式"并进入执行出图时,系统实际使用的默认生图模型。"系统智能选择"会根据用户意图(如抠图、换背景等)自动路由到最佳模型;这里不控制无限画布对话框里的 Agent 对话/识图模型。
</div>
</div>
</>

View File

@ -45,6 +45,7 @@ import {
} from "@/lib/ai/rmbg/service";
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
import { resolveDetailAgentLocalMoonshotModel } from "@/lib/ai/detail-agent-model-runtime";
import { shouldForceInlineVisionInputs } from "@/lib/ai/model-compat";
import { inlineVisionImages } from "@/lib/ai/vision-inline";
import { createDocument } from "@/lib/ai/tools/create-document";
import { getWeather } from "@/lib/ai/tools/get-weather";
@ -140,6 +141,7 @@ import {
resolveClarificationBypass,
resolveClarificationMessage,
} from "@/lib/ai/runtime/clarification-interruption";
import { resolveCanvasAgentClarificationGate } from "@/lib/ai/runtime/middlewares/canvas-agent-clarification-gate";
import {
pruneTaskBoundaryContext,
pruneTaskBoundaryHistoryMessages,
@ -2089,18 +2091,188 @@ function ensureImageAnalysisSummarySection(message: string, summary: string): st
* 注意:gemini-image 和 huanke-gemini 都需要 URL 格式(不是 Base64)
* 因为这些 provider 内部会自行处理图片上传到 S3
*/
async function convertMessagesWithImages(messages: ChatMessage[], providerType: string) {
async function convertMessagesWithImages(
messages: ChatMessage[],
providerType:
| string
| {
providerType?: string;
providerBaseUrl?: string | null;
modelId?: string | null;
displayName?: string | null;
forceInlineImageUrls?: boolean;
loggerPrefix?: string;
},
) {
type ImageConversionTarget = {
providerType?: string;
providerBaseUrl?: string | null;
modelId?: string | null;
displayName?: string | null;
forceInlineImageUrls?: boolean;
loggerPrefix?: string;
};
const normalizeTarget = (
target: string | ImageConversionTarget,
): Required<Pick<ImageConversionTarget, "providerType" | "forceInlineImageUrls" | "loggerPrefix">> &
Omit<ImageConversionTarget, "providerType" | "forceInlineImageUrls" | "loggerPrefix"> => {
if (typeof target === "string") {
return {
providerType: target,
providerBaseUrl: null,
modelId: null,
displayName: null,
forceInlineImageUrls: false,
loggerPrefix: "[Chat API] Vision inline",
};
}
return {
providerType: String(target.providerType || ""),
providerBaseUrl: target.providerBaseUrl || null,
modelId: target.modelId || null,
displayName: target.displayName || null,
forceInlineImageUrls: target.forceInlineImageUrls === true,
loggerPrefix: target.loggerPrefix || "[Chat API] Vision inline",
};
};
const extractImagePartUrl = (part: unknown): string | null => {
const partAny = part as any;
const candidateUrl =
partAny?.type === "file"
? partAny.url || partAny.data
: partAny?.type === "image"
? partAny.image || partAny.url
: partAny?.type === "image_url"
? partAny.image_url?.url
: undefined;
if (typeof candidateUrl !== "string") return null;
const normalized = candidateUrl.trim();
if (!/^https?:\/\//i.test(normalized)) return null;
return normalized;
};
const replaceImagePartUrl = (part: unknown, nextUrl: string): unknown => {
const partAny = part as any;
if (partAny?.type === "file") {
return {
...partAny,
url: nextUrl,
data: nextUrl,
mediaType: String(partAny.mediaType || "").startsWith("image/")
? partAny.mediaType
: "image/jpeg",
mimeType: String(partAny.mimeType || "").startsWith("image/")
? partAny.mimeType
: "image/jpeg",
};
}
if (partAny?.type === "image") {
return {
...partAny,
image: nextUrl,
url: nextUrl,
};
}
if (partAny?.type === "image_url") {
return {
...partAny,
image_url: {
...(partAny.image_url || {}),
url: nextUrl,
},
};
}
return part;
};
const inlineImageUrlsInMessages = async (
sourceMessages: ChatMessage[],
target: ReturnType<typeof normalizeTarget>,
) => {
const uniqueUrls = Array.from(
new Set(
sourceMessages
.flatMap((message) => message.parts || [])
.map((part) => extractImagePartUrl(part))
.filter((url): url is string => Boolean(url)),
),
);
if (uniqueUrls.length === 0) {
return sourceMessages;
}
const { inlineDataUrls } = await inlineVisionImages(uniqueUrls, {
maxImages: Math.min(8, uniqueUrls.length),
maxSide: 1200,
timeoutMs: 12000,
retries: 2,
loggerPrefix: target.loggerPrefix,
}).catch((error) => {
console.warn(`${target.loggerPrefix} failed to inline message images:`, error);
return { inlineDataUrls: [], skipped: [] };
});
if (inlineDataUrls.length === 0) {
return sourceMessages;
}
const inlineMap = new Map<string, string>();
for (let index = 0; index < inlineDataUrls.length; index += 1) {
const originalUrl = uniqueUrls[index];
const inlineUrl = inlineDataUrls[index];
if (originalUrl && inlineUrl) {
inlineMap.set(originalUrl, inlineUrl);
}
}
if (inlineMap.size === 0) {
return sourceMessages;
}
console.log(`${target.loggerPrefix} inlined chat message images:`, {
total: uniqueUrls.length,
inlined: inlineMap.size,
providerType: target.providerType,
modelId: target.modelId,
displayName: target.displayName,
});
return sourceMessages.map((message) => ({
...message,
parts: (message.parts || []).map((part) => {
const originalUrl = extractImagePartUrl(part);
if (!originalUrl) return part;
const inlineUrl = inlineMap.get(originalUrl);
return inlineUrl ? replaceImagePartUrl(part, inlineUrl) : part;
}) as typeof message.parts,
})) as ChatMessage[];
};
const target = normalizeTarget(providerType);
let preparedMessages = messages;
if (target.forceInlineImageUrls) {
preparedMessages = await inlineImageUrlsInMessages(preparedMessages, target);
}
const normalizedProviderType = target.providerType;
if (
providerType === "bfl-image" ||
providerType === "ruxa-image" ||
providerType === "fal-image" ||
providerType === "openai-image" ||
providerType === "midjourney" ||
providerType === "runway"
normalizedProviderType === "bfl-image" ||
normalizedProviderType === "ruxa-image" ||
normalizedProviderType === "fal-image" ||
normalizedProviderType === "openai-image" ||
normalizedProviderType === "midjourney" ||
normalizedProviderType === "runway"
) {
const convertedMessages = [];
for (const message of messages) {
for (const message of preparedMessages) {
const content: any[] = [];
for (const part of message.parts) {
@ -2149,8 +2321,8 @@ async function convertMessagesWithImages(messages: ChatMessage[], providerType:
// 所有 provider 都使用 URL 格式传输图片
// gemini-image 和 huanke-gemini 内部会将 Base64 转换为 S3 URL
// 这里不再进行 Base64 转换,直接返回标准格式
if (providerType !== "gemini") {
return convertToModelMessages(messages);
if (normalizedProviderType !== "gemini") {
return convertToModelMessages(preparedMessages);
}
// 🆕 检测是否有标注图片
@ -2162,7 +2334,7 @@ async function convertMessagesWithImages(messages: ChatMessage[], providerType:
// 对于 Gemini,需要特殊处理图片
const convertedMessages = [];
for (const message of messages) {
for (const message of preparedMessages) {
const content: any[] = [];
// 🆕 如果是用户消息且有标注图片,在文本前添加系统提示
@ -6407,6 +6579,34 @@ export async function POST(request: Request) {
}
}
if (shouldRunAgentConversation) {
const canvasClarificationGate = resolveCanvasAgentClarificationGate({
text: requestTextContent,
composerMode,
contextPayload: effectiveContextPayload,
mediaInput: requestResolvedMediaInput,
targetResolution: requestTargetResolution,
taskBoundaryDecision,
executionMode: smartAutoExecution?.mode,
actionKind: resolvedConversationAction.kind,
});
if (canvasClarificationGate.shouldClarify) {
smartAutoNeedsClarification = true;
smartAutoClarificationQuestion =
canvasClarificationGate.clarificationMessage || smartAutoClarificationQuestion;
smartAutoFriendlyMessage =
canvasClarificationGate.clarificationMessage || smartAutoFriendlyMessage;
console.log("[Smart Auto Mode] 🧭 Canvas clarification gate triggered:", {
reason: canvasClarificationGate.reason,
executionMode: smartAutoExecution?.mode,
resolutionMode: requestTargetResolution?.resolutionMode,
selectedCount: requestResolvedMediaInput?.selectedImageRefs.length || 0,
candidateCount: requestTargetResolution?.candidateRefs.length || 0,
});
}
}
if (shouldRunAgentConversation && smartAutoNeedsClarification) {
const clarificationMessage = resolveClarificationMessage({
clarificationQuestion: smartAutoClarificationQuestion,
@ -9054,6 +9254,10 @@ export async function POST(request: Request) {
const currentModelConfig = isUuidModelId
? await getAiModelById({ id: finalModelId }).catch(() => undefined)
: undefined;
const currentProviderConfig =
currentModelConfig?.providerId
? await getAiProviderById({ id: currentModelConfig.providerId }).catch(() => null)
: null;
const runtimeProviderType = String(
resolvedProviderType || (languageModel as any).provider || ""
);
@ -10878,7 +11082,19 @@ export async function POST(request: Request) {
// 转换消息,对于 Gemini 和千问视频将图片 URL 转换为 base64
const convertedMessages = await convertMessagesWithImages(
injectHistoryImagesToLastUserMessage(uiMessagesForModel, smartAutoHistoryImages),
(languageModel as any).provider || ""
{
providerType: (languageModel as any).provider || "",
providerBaseUrl: currentProviderConfig?.baseUrl || null,
modelId: currentModelConfig?.modelId || actualModelId,
displayName: currentModelConfig?.displayName || actualModelId,
forceInlineImageUrls: shouldForceInlineVisionInputs({
providerType: resolvedProviderType || currentProviderConfig?.type || runtimeProviderType,
providerBaseUrl: currentProviderConfig?.baseUrl || null,
modelId: currentModelConfig?.modelId || actualModelId,
displayName: currentModelConfig?.displayName || actualModelId,
}),
loggerPrefix: "[Chat API] Stream text vision inline",
}
);
console.log("[Chat API] Messages converted, count:", convertedMessages.length); // 调试日志

View File

@ -1,6 +1,7 @@
import { Annotation, StateGraph, START, END } from "@langchain/langgraph";
import { generateText } from "ai";
import sharp from "sharp";
import { getCompatibleTextGenerationTemperature } from "@/lib/ai/model-compat";
import type { AnalysisResult } from "@/lib/ai/smart-analyzer";
import type { RouteResult } from "@/lib/ai/model-router";
import type { AgentSkillRouteResult } from "@/lib/ai/skill-router";
@ -1359,7 +1360,7 @@ async function buildExecutionPlanWithAgent(
]
.filter(Boolean)
.join("\n\n"),
temperature: 0.1,
temperature: getCompatibleTextGenerationTemperature(plannerModelConfig, 0.1),
maxTokens: 420,
abortSignal: AbortSignal.timeout(AGENT_OPERATION_TIMEOUT_MS),
} as any);
@ -1482,7 +1483,7 @@ async function generateAgentConversation(
content: [{ type: "text" as const, text: visionPrompt }, ...visionInputs],
},
],
temperature: 0.2,
temperature: getCompatibleTextGenerationTemperature(visionModelConfig, 0.2),
maxTokens: 360,
abortSignal: AbortSignal.timeout(AGENT_OPERATION_TIMEOUT_MS),
} as any);
@ -1544,7 +1545,7 @@ async function generateAgentConversation(
model: reasoningModel,
system: systemPrompt,
prompt,
temperature: 0.35,
temperature: getCompatibleTextGenerationTemperature(reasoningModelConfig, 0.35),
maxTokens: 520,
abortSignal: AbortSignal.timeout(AGENT_OPERATION_TIMEOUT_MS),
} as any);
@ -1626,7 +1627,7 @@ async function recommendImageParamsWithAgent(
- 若用户未指定比例,优先 auto。
- negativePrompt 仅在明显需要避坑时填写。`,
prompt: requestText,
temperature: 0.15,
temperature: getCompatibleTextGenerationTemperature(plannerModelConfig, 0.15),
maxTokens: 220,
abortSignal: AbortSignal.timeout(AGENT_OPERATION_TIMEOUT_MS),
} as any);
@ -1719,7 +1720,7 @@ async function recommendVideoPlanWithAgent(
- 有参考图时优先 first-frame 或 first-last-frame。
- 避免抽象空话,必须可执行。`,
prompt,
temperature: 0.2,
temperature: getCompatibleTextGenerationTemperature(plannerModelConfig, 0.2),
maxTokens: 720,
abortSignal: AbortSignal.timeout(AGENT_OPERATION_TIMEOUT_MS),
} as any);
@ -1799,7 +1800,7 @@ async function refineRouteWithAgent(
意图:${state.analysis.intent}/${state.analysis.subIntent || "none"}
候选模型:
${candidatesText}`,
temperature: 0.1,
temperature: getCompatibleTextGenerationTemperature(plannerModelConfig, 0.1),
maxTokens: 120,
abortSignal: AbortSignal.timeout(AGENT_OPERATION_TIMEOUT_MS),
} as any);

81
lib/ai/model-compat.ts Normal file
View File

@ -0,0 +1,81 @@
type ModelCompatDescriptor = {
providerType?: string | null;
providerBaseUrl?: string | null;
modelId?: string | null;
displayName?: string | null;
provider?: {
type?: string | null;
baseUrl?: string | null;
} | null;
model?: {
modelId?: string | null;
displayName?: string | null;
} | null;
};
const KIMI_MODEL_PATTERN = /\bkimi(?:[-_.\s]?[a-z0-9.]+)?\b/i;
function normalizeProviderType(input: ModelCompatDescriptor): string {
return String(input.providerType || input.provider?.type || "")
.trim()
.toLowerCase();
}
function normalizeProviderBaseUrl(input: ModelCompatDescriptor): string {
return String(input.providerBaseUrl || input.provider?.baseUrl || "")
.trim()
.toLowerCase();
}
function normalizeModelText(input: ModelCompatDescriptor): string {
return `${String(input.displayName || input.model?.displayName || "")} ${String(input.modelId || input.model?.modelId || "")}`
.trim()
.toLowerCase();
}
export function isMoonshotKimiModel(
input?: ModelCompatDescriptor | null,
): boolean {
if (!input) {
return false;
}
const providerType = normalizeProviderType(input);
const providerBaseUrl = normalizeProviderBaseUrl(input);
const modelText = normalizeModelText(input);
if (providerType.includes("moonshot")) {
return true;
}
if (providerBaseUrl.includes("moonshot.cn")) {
return true;
}
if (modelText.includes("moonshot")) {
return true;
}
if (KIMI_MODEL_PATTERN.test(modelText)) {
return true;
}
return false;
}
export function getCompatibleTextGenerationTemperature(
input: ModelCompatDescriptor | null | undefined,
preferredTemperature?: number,
): number | undefined {
if (isMoonshotKimiModel(input)) {
return 1;
}
return preferredTemperature;
}
export function shouldForceInlineVisionInputs(
input: ModelCompatDescriptor | null | undefined,
): boolean {
return isMoonshotKimiModel(input);
}

View File

@ -170,6 +170,7 @@ const EDIT_TEMPLATES: Record<
};
import { generateText } from "ai";
import { getCompatibleTextGenerationTemperature } from "@/lib/ai/model-compat";
function inferStyleTransferTarget(originalPrompt: string, extracted?: string): string {
const text = (originalPrompt || "").trim();
@ -674,7 +675,7 @@ export async function optimizePromptWithLLM(params: {
model,
system: optimizerSystemPrompt,
prompt: finalUserMessage,
temperature: 0.2,
temperature: getCompatibleTextGenerationTemperature(selectedOptimizerModel, 0.2),
maxTokens: 200,
} as any);

View File

@ -6,6 +6,7 @@
*/
import { generateText } from "ai";
import { getCompatibleTextGenerationTemperature } from "@/lib/ai/model-compat";
import sharp from "sharp";
import { isAiModelMultimodal } from "@/lib/db/ai-model-flags";
// 使用动态导入避免循环依赖问题
@ -1505,6 +1506,7 @@ export async function summarizeImageForExecutionReport(params: {
if (fallbackVisionImages.length === 0) continue;
try {
const compatibleTemperature = getCompatibleTextGenerationTemperature(candidate, 0.1);
const { text: summaryText } = await generateText({
model,
messages: [
@ -1519,7 +1521,9 @@ export async function summarizeImageForExecutionReport(params: {
],
},
],
temperature: 0.1,
...(compatibleTemperature !== undefined
? { temperature: compatibleTemperature }
: {}),
maxTokens: 120,
abortSignal: AbortSignal.timeout(ANALYZER_TIMEOUT_MS),
} as any);
@ -2107,6 +2111,7 @@ ${historyText}
);
// 调用模型分析(有图时携带图片,确保真实视觉分析)
const analyzerTemperature = getCompatibleTextGenerationTemperature(selectedAnalyzerModel, 0.1);
const requestPayload = hasImage && visionImages.length > 0
&& !cachedVisualSummary
? {
@ -2121,7 +2126,7 @@ ${historyText}
],
},
],
temperature: 0.1,
...(analyzerTemperature !== undefined ? { temperature: analyzerTemperature } : {}),
maxTokens: 360,
abortSignal: AbortSignal.timeout(ANALYZER_TIMEOUT_MS),
}
@ -2129,7 +2134,7 @@ ${historyText}
model,
system: ANALYZER_SYSTEM_PROMPT,
prompt: userMessage,
temperature: 0.1,
...(analyzerTemperature !== undefined ? { temperature: analyzerTemperature } : {}),
maxTokens: 300,
abortSignal: AbortSignal.timeout(ANALYZER_TIMEOUT_MS),
};