diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index 8f47fe4dd..d8a528425 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -183,6 +183,63 @@ function applyOptimizedPromptToLatestUserMessage( return cloned; } +type ChatImageParams = NonNullable; +type AgentRecommendedImageParams = { + aspectRatio?: string; + imageSize?: string; + imageCount?: number; + negativePrompt?: string; + promptExtend?: boolean; +}; + +function hasUserCustomizedImageParams(params?: PostRequestBody["imageParams"]): boolean { + if (!params) return false; + const customizedAspectRatio = + typeof params.aspectRatio === "string" && params.aspectRatio.trim() && params.aspectRatio !== "auto"; + const customizedImageSize = + typeof params.imageSize === "string" && params.imageSize.trim() && params.imageSize.toUpperCase() !== "2K"; + const customizedImageCount = typeof params.imageCount === "number" && params.imageCount !== 1; + const customizedAdvanced = Boolean(params.size || params.seed !== undefined || params.n || params.negativePrompt); + return customizedAspectRatio || customizedImageSize || customizedImageCount || customizedAdvanced; +} + +function mergeImageParamsWithAgentRecommendation(params: { + incoming?: PostRequestBody["imageParams"]; + recommended?: AgentRecommendedImageParams; + intent?: "image_edit" | "image_gen" | "vision" | "video_gen" | "chat"; + hasUserCustomSettings: boolean; +}): PostRequestBody["imageParams"] { + const { incoming, recommended, intent, hasUserCustomSettings } = params; + const base: ChatImageParams = { ...(incoming || {}) }; + if (!recommended) { + return Object.keys(base).length > 0 ? base : undefined; + } + + const merged: ChatImageParams = { + ...base, + negativePrompt: + typeof recommended.negativePrompt === "string" && recommended.negativePrompt.trim() + ? recommended.negativePrompt.trim() + : base.negativePrompt, + promptExtend: + typeof recommended.promptExtend === "boolean" + ? recommended.promptExtend + : base.promptExtend, + }; + + if (!hasUserCustomSettings) { + if (recommended.aspectRatio) merged.aspectRatio = recommended.aspectRatio; + if (recommended.imageSize) merged.imageSize = recommended.imageSize; + if (typeof recommended.imageCount === "number") merged.imageCount = recommended.imageCount; + } + + if (intent === "image_edit") { + merged.imageCount = 1; + } + + return Object.keys(merged).length > 0 ? merged : undefined; +} + function guessImageMediaType(url: string): string { const cleanUrl = url.split("?")[0]?.toLowerCase() || ""; if (cleanUrl.endsWith(".jpg") || cleanUrl.endsWith(".jpeg")) return "image/jpeg"; @@ -818,6 +875,8 @@ export async function POST(request: Request) { // 🆕 存储智能模式的相关信息 let smartAutoFriendlyMessage: string | undefined; // AI的自然对话回复 let smartAutoOptimizedPrompt: string | undefined; // 优化后的提示词（不修改用户消息，只用于API调用） + let smartAutoRecommendedImageParams: AgentRecommendedImageParams | undefined; + let smartAutoDetectedIntent: "image_edit" | "vision" | "image_gen" | "video_gen" | "chat" | undefined; let smartAutoHistoryImages: string[] = []; // 从历史对话中提取的图片URL（支持多张） let smartAutoNeedsClarification = false; // 是否需要先追问用户 let smartAutoClarificationQuestion: string | undefined; @@ -827,23 +886,56 @@ export async function POST(request: Request) { if (!shouldRunAgentConversation && requestedModelId !== "auto") { const selectedModelIsUUID = - /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test( - requestedModelId - ); + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(requestedModelId); + let selectedModelConfig = selectedModelIsUUID + ? await getAiModelById({ id: requestedModelId }) + : null; + let selectedModelProviderType: string | undefined; - if (selectedModelIsUUID) { - const selectedModelConfig = await getAiModelById({ id: requestedModelId }); - if (selectedModelConfig && isAiModelAgentAuto(selectedModelConfig)) { + if (!selectedModelConfig) { + const enabledModels = await getEnabledAiModelsWithProvider(false); + const matchedModel = enabledModels.find( + ({ model }) => model.id === requestedModelId || model.modelId === requestedModelId + ); + if (matchedModel) { + selectedModelConfig = matchedModel.model; + selectedModelProviderType = matchedModel.provider.type as string; + } + } + + if (selectedModelConfig && !selectedModelProviderType) { + const selectedProvider = await getAiProviderById({ + id: selectedModelConfig.providerId, + }); + selectedModelProviderType = selectedProvider?.type as string | undefined; + } + + if (selectedModelConfig) { + const fixedImageProviderTypes = new Set([ + "gemini", + "huanke-gemini", + "alibaba-image", + "ruxa-image", + "volcano-vision", + ]); + const isFixedImageModel = + !selectedModelConfig.isSubModel && + (selectedModelConfig.supportsImageGen || + (selectedModelProviderType + ? fixedImageProviderTypes.has(selectedModelProviderType) + : false)); + + if (isAiModelAgentAuto(selectedModelConfig)) { shouldRunAgentConversation = true; shouldKeepSelectedModel = false; console.log( `[Smart Auto Mode] 🧠 Agent-only model detected, force auto routing: ${selectedModelConfig.displayName}` ); - } else if (selectedModelConfig?.supportsImageGen && !selectedModelConfig?.isSubModel) { + } else if (isFixedImageModel) { shouldRunAgentConversation = true; shouldKeepSelectedModel = true; console.log( - `[Smart Auto Mode] 🎯 Agent conversation enabled for fixed image model: ${selectedModelConfig.displayName}` + `[Smart Auto Mode] 🎯 Agent conversation enabled for fixed image model: ${selectedModelConfig.displayName} (${selectedModelProviderType || "unknown"})` ); } } @@ -982,6 +1074,7 @@ export async function POST(request: Request) { needsClarification: autoGraphResult.analysis.needsClarification, }); + smartAutoDetectedIntent = autoGraphResult.analysis.intent; smartAutoNeedsClarification = Boolean(autoGraphResult.analysis.needsClarification); smartAutoClarificationQuestion = autoGraphResult.analysis.clarifyingQuestions?.[0]; } @@ -1023,6 +1116,10 @@ export async function POST(request: Request) { optimized: autoGraphResult.optimizedPrompt.substring(0, 100), }); } + if (autoGraphResult.recommendedImageParams) { + smartAutoRecommendedImageParams = autoGraphResult.recommendedImageParams; + console.log("[Smart Auto Mode] 🎛️ Agent recommended image params:", smartAutoRecommendedImageParams); + } if (autoGraphResult.route && !shouldKeepSelectedModel) { finalModelId = autoGraphResult.route.id; @@ -1092,6 +1189,50 @@ export async function POST(request: Request) { } // ========== End Smart Auto Mode Logic ========== + if ( + shouldRunAgentConversation && + !smartAutoNeedsClarification && + !smartAutoFriendlyMessage?.trim() + ) { + const latestUserText = message.parts + .filter((part): part is { type: "text"; text: string } => part.type === "text") + .map((part) => part.text) + .join(" "); + const hasCurrentImageInput = message.parts.some((part: any) => + part?.type === "file" || part?.type === "image" || part?.type === "image_url" + ); + const hasHistoryImageInput = smartAutoHistoryImages.length > 0; + smartAutoFriendlyMessage = buildAutoModeFallbackMessage({ + text: latestUserText, + hasImage: hasCurrentImageInput || hasHistoryImageInput, + fromHistory: !hasCurrentImageInput && hasHistoryImageInput, + }); + console.log("[Smart Auto Mode] 🧩 Backfilled conversational reply with fallback"); + } + + const hasUserCustomImageParams = hasUserCustomizedImageParams(requestBody.imageParams); + const shouldApplyAgentImageParams = + shouldRunAgentConversation && + !smartAutoNeedsClarification && + (smartAutoDetectedIntent === "image_gen" || smartAutoDetectedIntent === "image_edit"); + const effectiveImageParams = shouldApplyAgentImageParams + ? mergeImageParamsWithAgentRecommendation({ + incoming: requestBody.imageParams, + recommended: smartAutoRecommendedImageParams, + intent: smartAutoDetectedIntent, + hasUserCustomSettings: hasUserCustomImageParams, + }) + : requestBody.imageParams; + + if (shouldApplyAgentImageParams) { + console.log("[Smart Auto Mode] 🎯 Effective image params:", { + incoming: requestBody.imageParams, + recommended: smartAutoRecommendedImageParams, + merged: effectiveImageParams, + hasUserCustomImageParams, + }); + } + const session = await auth(); if (!session?.user) { @@ -1359,7 +1500,7 @@ export async function POST(request: Request) { const isGeminiImageProvider = ["gemini-image", "huanke-gemini"].includes((languageModel as any).provider); if (isGeminiImageProvider && typeof (languageModel as any).doGenerate === "function") { console.log("[Chat API] Using Gemini image generation, provider:", (languageModel as any).provider); - console.log("[Chat API] Image params:", requestBody.imageParams); + console.log("[Chat API] Image params:", effectiveImageParams); // 🆕 获取备用模型配置 const currentModelConfig = await getAiModelById({ id: finalModelId }); @@ -1445,7 +1586,7 @@ export async function POST(request: Request) { // 调用 doGenerate 获取图片（如果是备用模型，传入 overrideModelId） const result = await (languageModel as any).doGenerate({ messages: modelMessages, - imageParams: requestBody.imageParams || { aspectRatio: "auto", imageSize: "2K" }, + imageParams: effectiveImageParams || { aspectRatio: "auto", imageSize: "2K" }, overrideModelId: isRetry ? modelIdToUse : undefined, // 🆕 使用备用模型 ID }); @@ -1721,7 +1862,7 @@ export async function POST(request: Request) { const isAlibabaLikeImageProvider = ["alibaba-image", "ruxa-image"].includes((languageModel as any).provider); if (isAlibabaLikeImageProvider && typeof (languageModel as any).doGenerate === "function") { console.log("[Chat API] Using image generation provider:", (languageModel as any).provider); - console.log("[Chat API] Image params:", requestBody.imageParams); + console.log("[Chat API] Image params:", effectiveImageParams); // 🆕 获取模型显示名称，用于错误提示 const alibabaModelConfig = await getAiModelById({ id: finalModelId }); @@ -1811,7 +1952,7 @@ export async function POST(request: Request) { // 调用 doGenerate 获取图片，传递模型显示名称 const result = await (languageModel as any).doGenerate({ messages: modelMessages, - imageParams: requestBody.imageParams || { aspectRatio: "auto", imageSize: "2K", imageCount: 1 }, + imageParams: effectiveImageParams || { aspectRatio: "auto", imageSize: "2K", imageCount: 1 }, modelDisplayName, // 🆕 传递模型显示名称 overrideModelId: isRetry ? modelIdToUse : undefined, }); @@ -1909,7 +2050,7 @@ export async function POST(request: Request) { const candidateResult = await (candidateLanguageModel as any).doGenerate({ messages: candidateMessages, - imageParams: requestBody.imageParams || { aspectRatio: "auto", imageSize: "2K", imageCount: 1 }, + imageParams: effectiveImageParams || { aspectRatio: "auto", imageSize: "2K", imageCount: 1 }, modelDisplayName: candidate.model.displayName, overrideModelId: candidate.model.modelId, }); @@ -2476,7 +2617,7 @@ export async function POST(request: Request) { const rawText = generated.text?.trim() || "图片生成失败，请稍后重试"; const normalizedText = await normalizeAssistantImageMarkdown( rawText, - requestBody.imageParams?.imageSize || "2K" + effectiveImageParams?.imageSize || "2K" ); const messageId = generateUUID(); diff --git a/components/messages.tsx b/components/messages.tsx index 4d8a0cabe..fa4293ad8 100644 --- a/components/messages.tsx +++ b/components/messages.tsx @@ -360,10 +360,7 @@ function PureMessages({ // 🆕 智能自动模式：确保文字可见后再显示生图占位图 useEffect(() => { - const lastUserMessage = [...messages].reverse().find(m => m.role === "user"); - const isAutoModeWithImage = selectedModelId === "auto" && - lastUserMessage?.parts?.some((p: any) => p.type === "file" || p.type === "image"); - + const isAutoMode = selectedModelId === "auto"; const lastUserIndex = messages.findLastIndex((m) => m.role === "user"); const assistantMessagesAfterUser = lastUserIndex >= 0 ? messages.slice(lastUserIndex + 1).filter((m) => m.role === "assistant") @@ -375,18 +372,27 @@ function PureMessages({ }) ); - let hasImageContent = false; - if (isAutoModeWithImage) { - hasImageContent = assistantMessagesAfterUser.some(msg => - msg.parts?.some((part: any) => part.type === "text" && part.text && ( - part.text.includes("![thumb|") || - part.text.includes("![thumb|||") || - part.text.includes("![Generated Image") - )) - ); - } + const hasImageContent = assistantMessagesAfterUser.some((msg) => + msg.parts?.some( + (part: any) => + part.type === "text" && + part.text && + (part.text.includes("![thumb|") || + part.text.includes("![thumb|||") || + part.text.includes("![Generated Image")) + ) + ); + const hasGenerationStart = assistantMessagesAfterUser.some((msg) => + msg.parts?.some( + (part: any) => + part.type === "text" && + part.text?.includes("") + ) + ); + const isAutoImageFlow = + isAutoMode && (isGeneratingImageFlag || hasGenerationStart || hasImageContent); - if (!isAutoModeWithImage || !isGeneratingImageFlag || !hasAssistantTextReplyWithContent || hasImageContent) { + if (!isAutoImageFlow || !isGeneratingImageFlag || !hasAssistantTextReplyWithContent || hasImageContent) { if (autoImagePlaceholderTimerRef.current) { clearTimeout(autoImagePlaceholderTimerRef.current); autoImagePlaceholderTimerRef.current = null; @@ -577,10 +583,7 @@ function PureMessages({ const showAlibabaImage = modelType === "alibaba-image" || (!modelType && isAlibabaImageModel); const showVideo = modelType === "volcano-video" || modelType === "qianwen-video" || (!modelType && (isQianwenVideoModel || isVideoModel)); - // 🆕 智能自动模式：检查用户是否发送了图片 const isAutoMode = selectedModelId === "auto"; - const lastUserMsg = [...messages].reverse().find(m => m.role === "user"); - const userMessageHasImage = lastUserMsg?.parts?.some((p: any) => p.type === "file" || p.type === "image"); // 🆕 检查是否收到了 GENERATING_IMAGE 标记（表示对话已发送，开始生图） const lastUserIndexForMarkers = messages.findLastIndex(m => m.role === "user"); @@ -598,6 +601,16 @@ function PureMessages({ const assistantMessagesAfterUser = lastUserIndexForMarkers >= 0 ? messages.slice(lastUserIndexForMarkers + 1).filter((msg) => msg.role === "assistant") : messages.filter((msg) => msg.role === "assistant"); + const hasImageContent = assistantMessagesAfterUser.some((msg) => + msg.parts?.some( + (part: any) => + part.type === "text" && + part.text && + (part.text.includes("![thumb|") || + part.text.includes("![thumb|||") || + part.text.includes("![Generated Image")) + ) + ); const hasAssistantTextReplyWithContent = assistantMessagesAfterUser.some((msg) => msg.parts?.some((part: any) => { if (part.type !== "text" || !part.text) return false; @@ -605,9 +618,11 @@ function PureMessages({ }) ); const shouldWaitForAgentReplyBeforeImagePlaceholder = - userMessageHasImage && (isAutoMode || showGemini || showVolcano || showAlibabaImage) && !hasAssistantTextReplyWithContent; + const isAutoImageFlow = + isAutoMode && + (isGeneratingImageFlag || hasGeneratingMarker || hasGenerationEnd || hasImageContent); if (showVideo) return ; if (shouldWaitForAgentReplyBeforeImagePlaceholder) { @@ -633,7 +648,7 @@ function PureMessages({ } // 🆕 智能自动模式占位图 - if (isAutoMode && userMessageHasImage) { + if (isAutoImageFlow) { // 🆕 只用 isGeneratingImageFlag 来控制是否显示生图占位图 // isGeneratingImageFlag 由 chat.tsx 控制：延迟 3 秒后设置为 true，收到图片后设置为 false if ( diff --git a/lib/ai/auto-agent-graph.ts b/lib/ai/auto-agent-graph.ts index e4a434ca1..c8e38a73d 100644 --- a/lib/ai/auto-agent-graph.ts +++ b/lib/ai/auto-agent-graph.ts @@ -8,6 +8,14 @@ interface HistoryMessage { content: string; } +export interface AgentRecommendedImageParams { + aspectRatio?: string; + imageSize?: string; + imageCount?: number; + negativePrompt?: string; + promptExtend?: boolean; +} + export interface AutoAgentInput { text: string; hasImage: boolean; @@ -21,6 +29,7 @@ export interface AutoAgentResult { analysis?: AnalysisResult; friendlyMessage?: string; optimizedPrompt?: string; + recommendedImageParams?: AgentRecommendedImageParams; route?: RouteResult; } @@ -34,6 +43,7 @@ const AutoAgentState = Annotation.Root({ analysis: Annotation, friendlyMessage: Annotation, optimizedPrompt: Annotation, + recommendedImageParams: Annotation, route: Annotation, }); @@ -45,6 +55,98 @@ function compactText(text: string, max = 48) { return `${normalized.slice(0, max)}...`; } +function normalizeAspectRatio(value?: string): string | undefined { + if (!value) return undefined; + const normalized = value.trim().toLowerCase(); + if (normalized === "auto") return "auto"; + const raw = value.trim(); + const allowed = new Set([ + "1:1", + "2:3", + "3:2", + "3:4", + "4:3", + "4:5", + "5:4", + "9:16", + "16:9", + "21:9", + ]); + return allowed.has(raw) ? raw : undefined; +} + +function normalizeImageSize(value?: string): string | undefined { + if (!value) return undefined; + const normalized = value.trim().toUpperCase(); + if (normalized === "1K" || normalized === "2K" || normalized === "4K") { + return normalized; + } + return undefined; +} + +function sanitizeRecommendedImageParams( + params: AgentRecommendedImageParams | undefined, + intent?: AnalysisResult["intent"], +): AgentRecommendedImageParams | undefined { + if (!params) return undefined; + const safe: AgentRecommendedImageParams = {}; + const normalizedAspectRatio = normalizeAspectRatio(params.aspectRatio); + const normalizedImageSize = normalizeImageSize(params.imageSize); + if (normalizedAspectRatio) safe.aspectRatio = normalizedAspectRatio; + if (normalizedImageSize) safe.imageSize = normalizedImageSize; + + if (typeof params.imageCount === "number" && Number.isFinite(params.imageCount)) { + const min = 1; + const max = intent === "image_edit" ? 1 : 4; + safe.imageCount = Math.min(Math.max(Math.round(params.imageCount), min), max); + } + if (intent === "image_edit") { + safe.imageCount = 1; + } + + if (typeof params.negativePrompt === "string" && params.negativePrompt.trim()) { + safe.negativePrompt = params.negativePrompt.trim().slice(0, 500); + } + if (typeof params.promptExtend === "boolean") { + safe.promptExtend = params.promptExtend; + } + + return Object.keys(safe).length > 0 ? safe : undefined; +} + +function heuristicImageParams( + text: string, + analysis: AnalysisResult, +): AgentRecommendedImageParams | undefined { + const normalized = text.toLowerCase(); + const params: AgentRecommendedImageParams = { + imageSize: "2K", + promptExtend: true, + }; + + if (analysis.intent === "image_edit") { + params.imageCount = 1; + } else if (/多张|四张|4张|三张|3张/.test(normalized)) { + params.imageCount = /四张|4张/.test(normalized) ? 4 : 3; + } else if (/两张|2张|双图/.test(normalized)) { + params.imageCount = 2; + } else { + params.imageCount = 1; + } + + if (/横图|横版|宽屏|封面|banner|海报/.test(normalized)) { + params.aspectRatio = "16:9"; + } else if (/竖图|竖版|手机壁纸|全身|海报竖版/.test(normalized)) { + params.aspectRatio = "9:16"; + } else if (/方图|方形|头像|logo|电商主图/.test(normalized)) { + params.aspectRatio = "1:1"; + } else { + params.aspectRatio = "auto"; + } + + return sanitizeRecommendedImageParams(params, analysis.intent); +} + function buildStrategicFriendlyMessage( analysis: AnalysisResult, state: AutoAgentStateType, @@ -60,6 +162,58 @@ function buildStrategicFriendlyMessage( return generateAgentConversation(analysis, state); } +type EnabledModelWithProvider = Awaited< + ReturnType +>[number]; + +function pickQwenVisionModel(models: EnabledModelWithProvider[]) { + const visionModels = models.filter((item) => item.model.supportsVision && item.model.isEnabled); + const qwenByName = visionModels.find( + (item) => + item.provider.type === "alibaba" && + /(qwen|通义|vl)/i.test(`${item.model.displayName} ${item.model.modelId}`) + ); + if (qwenByName) return qwenByName; + + const alibabaVision = visionModels.find((item) => item.provider.type === "alibaba"); + if (alibabaVision) return alibabaVision; + return visionModels[0]; +} + +function pickAgentReasoningModel( + agentModel: EnabledModelWithProvider | undefined, + models: EnabledModelWithProvider[], +) { + const isReasoningCompatibleProvider = (providerType: string) => + ["zhipu", "alibaba", "openai", "custom", "local"].includes(providerType); + + if (agentModel && isReasoningCompatibleProvider(agentModel.provider.type)) return agentModel; + const zhipuModel = models.find( + (item) => + item.provider.type === "zhipu" && + item.model.isEnabled && + !item.model.isSubModel && + isReasoningCompatibleProvider(item.provider.type) + ); + if (zhipuModel) return zhipuModel; + return models.find( + (item) => + item.model.isEnabled && + !item.model.isSubModel && + isReasoningCompatibleProvider(item.provider.type) + ); +} + +function parseJsonBlock>(text: string): T | null { + const match = text.match(/\{[\s\S]*\}/); + if (!match) return null; + try { + return JSON.parse(match[0]) as T; + } catch { + return null; + } +} + async function generateAgentConversation( analysis: AnalysisResult, state: AutoAgentStateType @@ -72,25 +226,12 @@ async function generateAgentConversation( const { getLanguageModelById } = await import("./dynamic-provider"); const allModels = await getEnabledAiModelsWithProvider(); const agentModel = await getAgentAiModelWithProvider(); - const canUseAgentModel = Boolean( - agentModel && (!state.hasImage || agentModel.model.supportsVision) - ); - - const preferredVisionModel = allModels.find( - (m) => m.provider.type === "alibaba" && m.model.supportsVision && m.model.isEnabled - ); - const candidateModel = - (canUseAgentModel ? agentModel : undefined) || - (state.hasImage - ? preferredVisionModel || allModels.find((m) => m.model.supportsVision && m.model.isEnabled) - : allModels.find((m) => m.model.isEnabled && !m.model.isSubModel)); - - if (!candidateModel) { + const reasoningModelConfig = pickAgentReasoningModel(agentModel, allModels); + if (!reasoningModelConfig) { return analysis.conversationalReply || null; } - - const model = await getLanguageModelById(candidateModel.model.id); - if (!model) { + const reasoningModel = await getLanguageModelById(reasoningModelConfig.model.id); + if (!reasoningModel) { return analysis.conversationalReply || null; } @@ -110,6 +251,15 @@ async function generateAgentConversation( let visualAnalysis = ""; if (state.hasImage && validImageUrls.length > 0) { try { + const visionModelConfig = pickQwenVisionModel(allModels); + if (!visionModelConfig) { + throw new Error("missing_vision_model"); + } + const visionModel = await getLanguageModelById(visionModelConfig.model.id); + if (!visionModel) { + throw new Error("missing_vision_runtime"); + } + const visionPrompt = `请分析以下参考图片，给出可用于生图/修图执行前决策的结果。 要求： 1) 每张图输出 1 行，格式：图X：主体...；风格...；构图...；光影与色调...；材质/氛围... @@ -118,7 +268,7 @@ async function generateAgentConversation( const visionInputs = validImageUrls.map((url) => ({ type: "image" as const, image: url })); const { text: visionText } = await generateText({ - model, + model: visionModel, messages: [ { role: "user" as const, @@ -130,26 +280,34 @@ async function generateAgentConversation( } as any); visualAnalysis = visionText.trim(); + console.log( + `[Auto Agent] Vision analysis model: ${visionModelConfig.model.displayName} (${visionModelConfig.provider.type})` + ); } catch (error) { + console.warn("[Auto Agent] Vision analysis failed, continue without visual summary"); visualAnalysis = ""; } } - const systemPrompt = `你是一个真实执行中的 AI 生图/修图 Agent。 -请根据用户需求和本轮输入图片，给出“执行前”的分析汇报。 + const systemPrompt = `你是一个真实执行中的 AI 生图/修图 Agent。你的核心目标是提升用户一次生成成功率。 -要求： -1) 不要说“我看不到图片”或“无法访问图片”，必须基于给定图片做真实分析。 -2) 语气自然、专业，不要模板腔，不要空话。 -3) 不暴露内部实现、模型路由、算法细节。 -4) 输出 Markdown，控制在 8-14 行。 -5) 有图片时，必须按顺序输出以下 Markdown H3 标题（逐字一致）： - - ### 需求理解 - - ### 图片分析 - - ### 执行计划 - - ### 输出设置 -6) “图片分析”要结合每张参考图差异，给出风格/构图/材质/光影等具体观察。 -7) 只有明确拿到“原图分辨率”时，才能写具体像素尺寸；否则只写宽高比/清晰度目标，不要编造 335x500 这类预览尺寸。`; +你必须按以下 AGENT SKILLS 执行： +1) 需求拆解：把用户目标拆成可执行指令，优先满足用户明确要求。 +2) 风险预判：主动识别可能导致失败的歧义（主体数量、视角、风格冲突、文字内容）。 +3) 提示词策略：给出可落地的正向约束和必要的负向约束，避免空泛描述。 +4) 参数策略：在输出设置中明确建议宽高比、清晰度档位、出图数量（若用户未指定）。 + +输出要求： +- 不要说“看不到图片/无法访问图片”。 +- 不暴露内部实现、模型路由、算法细节。 +- 输出 Markdown，控制在 8-14 行。 +- 有图片时必须按顺序输出以下 Markdown H3 标题（逐字一致）： + ### 需求理解 + ### 图片分析 + ### 执行计划 + ### 输出设置 +- “图片分析”必须基于视觉分析结果写具体观察，不允许写泛化空话。 +- 只有明确拿到原图分辨率时，才能写具体像素尺寸；否则只写宽高比/清晰度目标。`; const prompt = [ `【用户原始需求】${state.text || "（无文本）"}`, @@ -165,44 +323,167 @@ async function generateAgentConversation( .filter(Boolean) .join("\n\n"); - const imageInputs = validImageUrls.map((url) => ({ type: "image" as const, image: url })); - - const requestPayload = state.hasImage && imageInputs.length > 0 - ? { - model, - system: systemPrompt, - messages: [ - { - role: "user" as const, - content: [ - { type: "text" as const, text: prompt }, - ...imageInputs, - ], - }, - ], - temperature: 0.35, - maxTokens: 520, - } - : { - model, - system: systemPrompt, - prompt, - temperature: 0.35, - maxTokens: 420, - }; - - const { text } = await generateText(requestPayload as any); + const { text } = await generateText({ + model: reasoningModel, + system: systemPrompt, + prompt, + temperature: 0.35, + maxTokens: 520, + } as any); const cleaned = text.trim(); if (cleaned.length < 10) { return analysis.conversationalReply || null; } + console.log( + `[Auto Agent] Conversation model: ${reasoningModelConfig.model.displayName} (${reasoningModelConfig.provider.type})` + ); return cleaned; } catch (error) { return analysis.conversationalReply || null; } } +async function recommendImageParamsWithAgent( + analysis: AnalysisResult, + state: AutoAgentStateType, +): Promise { + if (analysis.intent !== "image_edit" && analysis.intent !== "image_gen") { + return undefined; + } + + try { + const { + getEnabledAiModelsWithProvider, + getAgentAiModelWithProvider, + } = await import("@/lib/db/queries"); + const { getLanguageModelById } = await import("./dynamic-provider"); + const allModels = await getEnabledAiModelsWithProvider(); + const agentModel = await getAgentAiModelWithProvider(); + const plannerModelConfig = pickAgentReasoningModel(agentModel, allModels); + if (!plannerModelConfig) { + return heuristicImageParams(state.text, analysis); + } + const plannerModel = await getLanguageModelById(plannerModelConfig.model.id); + if (!plannerModel) { + return heuristicImageParams(state.text, analysis); + } + + const history = state.historyMessages + .slice(-4) + .map((msg) => `${msg.role === "user" ? "用户" : "助手"}: ${compactText(msg.content, 120)}`) + .join("\n"); + + const requestText = [ + `用户请求：${state.text || "（无文本）"}`, + `识别意图：${analysis.intent}/${analysis.subIntent || "none"}`, + `是否有参考图：${state.hasImage ? "是" : "否"}`, + history ? `最近对话：\n${history}` : "", + "请给出本次最稳妥的生图参数建议。", + ] + .filter(Boolean) + .join("\n\n"); + + const { text } = await generateText({ + model: plannerModel, + system: `你是生图参数规划 Agent。目标是提升首图命中率和稳定性。 +只输出 JSON，不要输出解释。格式如下： +{ + "aspectRatio": "1:1|2:3|3:2|3:4|4:3|4:5|5:4|9:16|16:9|21:9|auto", + "imageSize": "1K|2K|4K", + "imageCount": 1, + "negativePrompt": "可选，简洁", + "promptExtend": true +} +规则： +- image_edit 场景强制 imageCount=1。 +- 若用户未指定比例，优先 auto。 +- negativePrompt 仅在明显需要避坑时填写。`, + prompt: requestText, + temperature: 0.15, + maxTokens: 220, + } as any); + + const parsed = parseJsonBlock(text); + const sanitized = sanitizeRecommendedImageParams(parsed || undefined, analysis.intent); + if (sanitized) { + console.log( + `[Auto Agent] Parameter planner model: ${plannerModelConfig.model.displayName} (${plannerModelConfig.provider.type})` + ); + return sanitized; + } + return heuristicImageParams(state.text, analysis); + } catch { + return heuristicImageParams(state.text, analysis); + } +} + +async function refineRouteWithAgent( + state: AutoAgentStateType, + baseRoute: RouteResult, +): Promise { + if (!state.analysis || (state.analysis.intent !== "image_edit" && state.analysis.intent !== "image_gen")) { + return baseRoute; + } + + try { + const { + getEnabledAiModelsWithProvider, + getAgentAiModelWithProvider, + } = await import("@/lib/db/queries"); + const { getLanguageModelById } = await import("./dynamic-provider"); + const { isAiModelAgentAuto } = await import("@/lib/db/ai-model-flags"); + const allModelsRaw = await getEnabledAiModelsWithProvider(false); + const allModels = allModelsRaw.filter((item) => !isAiModelAgentAuto(item.model)); + const imageCandidates = allModels + .filter((item) => item.model.supportsImageGen && item.model.isEnabled && !item.model.isSubModel) + .slice(0, 12); + if (imageCandidates.length <= 1) return baseRoute; + + const agentModel = await getAgentAiModelWithProvider(); + const plannerModelConfig = pickAgentReasoningModel(agentModel, allModelsRaw); + if (!plannerModelConfig) return baseRoute; + const plannerModel = await getLanguageModelById(plannerModelConfig.model.id); + if (!plannerModel) return baseRoute; + + const candidatesText = imageCandidates + .map((item, index) => `${index + 1}. id=${item.model.id}; 名称=${item.model.displayName}; provider=${item.provider.type}`) + .join("\n"); + + const { text } = await generateText({ + model: plannerModel, + system: `你是生图模型选择 Agent。目标是优先提高一次成功率。 +只输出 JSON： +{ + "modelId": "候选里的id", + "reason": "一句话中文理由，不超过24字" +} +如果无法判断，也必须给出候选中的 modelId。`, + prompt: `用户请求：${state.text || "（无文本）"} +意图：${state.analysis.intent}/${state.analysis.subIntent || "none"} +候选模型： +${candidatesText}`, + temperature: 0.1, + maxTokens: 120, + } as any); + + const parsed = parseJsonBlock<{ modelId?: string; reason?: string }>(text); + if (!parsed?.modelId) return baseRoute; + const chosen = imageCandidates.find((item) => item.model.id === parsed.modelId); + if (!chosen) return baseRoute; + + return { + id: chosen.model.id, + displayName: chosen.model.displayName, + providerType: chosen.provider.type, + modelId: chosen.model.modelId, + reason: `Agent规划: ${(parsed.reason || chosen.model.displayName).slice(0, 40)}`, + }; + } catch { + return baseRoute; + } +} + const analyzeNode = async (state: AutoAgentStateType) => { const { analyzeIntent } = await import("./smart-analyzer"); const analysis = await analyzeIntent({ @@ -243,7 +524,8 @@ const optimizeNode = async (state: AutoAgentStateType) => { return {}; } if (state.analysis.intent === "image_edit" && !state.analysis.subIntent) { - return {}; + const recommendedImageParams = await recommendImageParamsWithAgent(state.analysis, state); + return { recommendedImageParams }; } if (state.analysis.intent !== "image_edit" && state.analysis.intent !== "image_gen") { return {}; @@ -256,16 +538,31 @@ const optimizeNode = async (state: AutoAgentStateType) => { historyMessages: state.historyMessages, context: state.context ?? undefined, }); - return { optimizedPrompt: optimized.optimizedPrompt }; + const mergedParams = sanitizeRecommendedImageParams( + { + ...optimized.recommendedImageParams, + negativePrompt: + optimized.negativePrompt && optimized.negativePrompt.trim().length > 0 + ? optimized.negativePrompt + : optimized.recommendedImageParams?.negativePrompt, + }, + state.analysis.intent, + ) ?? (await recommendImageParamsWithAgent(state.analysis, state)); + + return { + optimizedPrompt: optimized.optimizedPrompt, + recommendedImageParams: mergedParams, + }; }; const routeNode = async (state: AutoAgentStateType) => { const { routeToModel } = await import("./model-router"); - const route = await routeToModel({ + const baseRoute = await routeToModel({ intent: state.analysis?.intent || "chat", subIntent: state.analysis?.subIntent, hasImage: state.hasImage, }); + const route = await refineRouteWithAgent(state, baseRoute); return { route }; }; @@ -305,6 +602,7 @@ export async function runAutoAgentGraph(params: AutoAgentInput): Promise = {}; + if (normalizedAspectRatio && allowedAspectRatios.has(normalizedAspectRatio)) { + output.aspectRatio = normalizedAspectRatio; + } + if (normalizedSize && allowedSizes.has(normalizedSize)) { + output.imageSize = normalizedSize; + } + if (normalizedCount) { + output.imageCount = normalizedCount; + } + if (typeof input.promptExtend === "boolean") { + output.promptExtend = input.promptExtend; + } + return Object.keys(output).length > 0 ? output : undefined; +} + function fallbackImageGenPrompt(originalPrompt: string): OptimizeResult { const normalized = originalPrompt.replace(/\s+/g, " ").trim(); const optimizedPrompt = [ @@ -262,6 +328,12 @@ function fallbackImageGenPrompt(originalPrompt: string): OptimizeResult { originalPrompt, optimizedPrompt, negativePrompt: "低清晰度，构图混乱，比例畸形，重复主体，手部异常，文字错乱，水印，噪点，过曝，过度锐化", + recommendedImageParams: { + aspectRatio: "auto", + imageSize: "2K", + imageCount: 1, + promptExtend: true, + }, }; } @@ -300,16 +372,25 @@ export async function optimizePromptWithLLM(params: { } = await import("@/lib/db/queries"); const { getLanguageModelById } = await import("./dynamic-provider"); - // 获取千问3 VL 模型 + // 提示词优化优先使用 Agent 模型（建议智普），失败时回退 const allModels = await getEnabledAiModelsWithProvider(); const agentModel = await getAgentAiModelWithProvider(); + const isReasoningCompatibleProvider = (providerType: string) => + ["zhipu", "alibaba", "openai", "custom", "local"].includes(providerType); + const usableAgentModel = + agentModel && isReasoningCompatibleProvider(agentModel.provider.type) + ? agentModel + : undefined; + const zhipuModel = allModels.find( + (m) => m.provider.type === "zhipu" && m.model.isEnabled && !m.model.isSubModel + ); const qwenModel = allModels.find( (m) => m.provider.type === "alibaba" && m.model.supportsVision ); - const selectedOptimizerModel = agentModel || qwenModel; + const selectedOptimizerModel = usableAgentModel || zhipuModel || qwenModel; if (!selectedOptimizerModel) { - console.log("[Prompt Optimizer LLM] No Qwen3 VL model, falling back to template"); + console.log("[Prompt Optimizer LLM] No optimizer model available, falling back to template"); return optimizePrompt({ originalPrompt, intent, subIntent }); } @@ -340,7 +421,11 @@ export async function optimizePromptWithLLM(params: { const finalUserMessage = `${contextMessage}## 当前用户请求：\n"${originalPrompt}"\n\n## 识别的意图：${intentLabel}\n\n请生成优化后的提示词：`; - console.log("[Prompt Optimizer LLM] Calling Qwen3 VL for prompt optimization"); + console.log( + "[Prompt Optimizer LLM] Calling optimizer model:", + selectedOptimizerModel.model.displayName, + `(${selectedOptimizerModel.provider.type})` + ); const { text: responseText } = await generateText({ model, @@ -358,6 +443,7 @@ export async function optimizePromptWithLLM(params: { originalPrompt, optimizedPrompt: parsedResult.optimizedPrompt || originalPrompt, negativePrompt: parsedResult.negativePrompt || "模糊，低质量，变形，失真", + recommendedImageParams: sanitizeSuggestedParams(parsedResult.suggestedParams), friendlyMessage: intent === "image_edit" ? FRIENDLY_MESSAGES[subIntent || ""]?.template : undefined, }; } diff --git a/lib/ai/smart-analyzer.ts b/lib/ai/smart-analyzer.ts index 71aa78b70..da6166e1c 100644 --- a/lib/ai/smart-analyzer.ts +++ b/lib/ai/smart-analyzer.ts @@ -212,19 +212,32 @@ async function analyzWithLLM( } = await import("@/lib/db/queries"); const { getLanguageModelById } = await import("./dynamic-provider"); - // 获取千问3 VL 模型 + // 图片识别固定走千问多模态；纯文本理解优先走 Agent 模型（建议智普） const allModels = await getEnabledAiModelsWithProvider(); const agentModel = await getAgentAiModelWithProvider(); - const qwenModel = allModels.find( - (m) => m.provider.type === "alibaba" && m.model.supportsVision + const isReasoningCompatibleProvider = (providerType: string) => + ["zhipu", "alibaba", "openai", "custom", "local"].includes(providerType); + const qwenVisionModel = allModels.find( + (m) => + m.provider.type === "alibaba" && + m.model.supportsVision && + /(qwen|通义|vl)/i.test(`${m.model.displayName} ${m.model.modelId}`) ); - const selectedAnalyzerModel = - (agentModel && (!hasImage || agentModel.model.supportsVision) + const fallbackVisionModel = + qwenVisionModel || + allModels.find((m) => m.provider.type === "alibaba" && m.model.supportsVision); + const usableAgentModel = + agentModel && isReasoningCompatibleProvider(agentModel.provider.type) ? agentModel - : undefined) || qwenModel; + : undefined; + const selectedAnalyzerModel = hasImage + ? fallbackVisionModel + : usableAgentModel || + allModels.find((m) => m.provider.type === "zhipu" && !m.model.isSubModel) || + fallbackVisionModel; if (!selectedAnalyzerModel) { - console.log("[Smart Analyzer] No Qwen3 VL model found"); + console.log("[Smart Analyzer] No available analyzer model found"); return null; } @@ -261,7 +274,13 @@ ${historyText} ? `${contextMessage}用户上传了图片，请求内容："${text}"` : `${contextMessage}用户请求（无图片）："${text}"`; - console.log("[Smart Analyzer] Calling Qwen3 VL for analysis with", historyMessages?.length || 0, "history messages"); + console.log( + "[Smart Analyzer] Calling analyzer model:", + selectedAnalyzerModel.model.displayName, + `(${selectedAnalyzerModel.provider.type})`, + "history=", + historyMessages?.length || 0 + ); const visionImages = (imageUrls || []) .filter((url) => /^https?:\/\//i.test(url)) diff --git a/lib/db/ai-model-flags.ts b/lib/db/ai-model-flags.ts index 6ddde95b5..1d9b3e0d2 100644 --- a/lib/db/ai-model-flags.ts +++ b/lib/db/ai-model-flags.ts @@ -10,6 +10,10 @@ function toRecord(value: unknown): Record | null { export function isAiModelAgentAuto(input: { defaultParams?: unknown; capabilities?: unknown; + modelId?: string | null; + displayName?: string | null; + supportsVision?: boolean | null; + supportsImageGen?: boolean | null; }): boolean { const defaultParams = toRecord(input.defaultParams); if (typeof defaultParams?.[AGENT_AUTO_MODEL_KEY] === "boolean") { @@ -21,6 +25,24 @@ export function isAiModelAgentAuto(input: { return capabilities[AGENT_AUTO_MODEL_KEY] as boolean; } + const supportsVision = Boolean(input.supportsVision); + const supportsImageGen = Boolean(input.supportsImageGen); + if (!supportsVision || supportsImageGen) { + return false; + } + + const modelText = `${input.modelId || ""} ${input.displayName || ""}`.toLowerCase(); + const isQwenLikeModel = + modelText.includes("qwen") || modelText.includes("通义"); + const isVisionLikeModel = + modelText.includes("vl") || + modelText.includes("vision") || + modelText.includes("多模态"); + + if (isQwenLikeModel && isVisionLikeModel) { + return true; + } + return false; }