diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index 877729843..16d7d13ad 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -798,6 +798,45 @@ function buildAutoModeFallbackMessage(params: { ].join("\n"); } +function extractMessageImageUrls(parts: any[]): string[] { + return (parts || []) + .map((part: any) => { + if (!part) return null; + if (part.type === "file") return part.url || part.data || null; + if (part.type === "image") return part.url || part.image || null; + if (part.type === "image_url") return part.image_url?.url || null; + return null; + }) + .filter((url: any): url is string => typeof url === "string" && /^https?:\/\//i.test(url)); +} + +function shouldEnrichImageAnalysisText(message?: string): boolean { + if (!message) return false; + if (!/###\s*图片分析/.test(message)) return false; + return /(基于当前上传图片处理|基于你选中的原图处理|基于上一轮图片继续处理)/.test(message); +} + +function injectImageAnalysisSummary(message: string, summary: string): string { + const normalizedSummary = summary.replace(/\s+/g, " ").trim(); + if (!normalizedSummary) return message; + const lines = message.split("\n"); + const analysisHeaderIndex = lines.findIndex((line) => /^###\s*图片分析/.test(line.trim())); + if (analysisHeaderIndex < 0) return message; + + for (let i = analysisHeaderIndex + 1; i < lines.length; i += 1) { + const current = lines[i]?.trim() || ""; + if (/^###\s*/.test(current)) break; + if (/^- /.test(current)) { + lines[i] = `- ${normalizedSummary}`; + return lines.join("\n"); + } + } + + const insertIndex = analysisHeaderIndex + 1; + lines.splice(insertIndex, 0, `- ${normalizedSummary}`); + return lines.join("\n"); +} + /** * 自定义消息转换函数 * 注意:gemini-image 和 huanke-gemini 都需要 URL 格式(不是 Base64) @@ -2099,6 +2138,44 @@ export async function POST(request: Request) { }); } + if ( + shouldRunAgentConversation && + !smartAutoNeedsClarification && + smartAutoFriendlyMessage && + shouldEnrichImageAnalysisText(smartAutoFriendlyMessage) + ) { + try { + const currentImageUrls = extractMessageImageUrls(message.parts as any[]); + const summaryImageUrls = + currentImageUrls.length > 0 ? currentImageUrls : smartAutoHistoryImages; + + if (summaryImageUrls.length > 0) { + const { summarizeImageForExecutionReport } = await import("@/lib/ai/smart-analyzer"); + const summary = await withOperationTimeout( + summarizeImageForExecutionReport({ + text: message.parts + .filter((part): part is { type: "text"; text: string } => part.type === "text") + .map((part) => part.text) + .join(" "), + imageUrls: summaryImageUrls, + context: typeof autoContext === "string" ? autoContext : undefined, + }), + "enrichImageAnalysisSummary", + 8000, + ); + if (summary?.trim()) { + smartAutoFriendlyMessage = injectImageAnalysisSummary( + smartAutoFriendlyMessage, + summary.trim(), + ); + console.log("[Smart Auto Mode] ✅ Enriched image analysis summary"); + } + } + } catch (enrichError) { + console.warn("[Smart Auto Mode] Skip image analysis enrichment:", enrichError); + } + } + const session = await auth(); if (!session?.user) { diff --git a/lib/ai/smart-analyzer.ts b/lib/ai/smart-analyzer.ts index 19133a96e..88d3e3fc5 100644 --- a/lib/ai/smart-analyzer.ts +++ b/lib/ai/smart-analyzer.ts @@ -636,6 +636,47 @@ export async function summarizeImageForExecutionReport(params: { if (summary) return summary; } + try { + const selectedVisionModel = + (agentModel && isLikelyVisionCapableModel(agentModel) ? agentModel : undefined) || + allModels.find((m) => isLikelyVisionCapableModel(m)); + if (!selectedVisionModel) return null; + + const { getLanguageModelById } = await import("./dynamic-provider"); + const model = await getLanguageModelById(selectedVisionModel.model.id); + if (!model) return null; + + const fallbackVisionImages = await buildAnalyzerVisionImagesByMode({ + imageUrls, + preferRemoteUrls: shouldPreferRemoteVisionUrls(selectedVisionModel), + }); + if (fallbackVisionImages.length === 0) return null; + + const { text: summaryText } = await generateText({ + model, + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: `${promptText}\n只输出一行,格式:画面要点:主体+场景+风格+光线(不超过45字)。`, + }, + ...fallbackVisionImages, + ], + }, + ], + temperature: 0.1, + maxTokens: 120, + abortSignal: AbortSignal.timeout(8000), + } as any); + + const normalized = summaryText?.replace(/\s+/g, " ").trim(); + return normalized || null; + } catch (error) { + console.warn("[Smart Analyzer] summarizeImageForExecutionReport fallback failed:", error); + } + return null; }