diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index ab38bc3a6..cc590ef4e 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -1698,13 +1698,20 @@ function buildPromptOnlyReply(params: { optimizedPrompt: string; }): string { const demand = sanitizePromptOnlyUserIntent(params.userText) || "你希望我先分析这张图并整理可直接使用的提示词"; - const summary = String(params.imageSummary || "").replace(/\s+/g, " ").trim(); + const summaryLines = String(params.imageSummary || "") + .split(/\r?\n/) + .map((line) => line.replace(/\s+/g, " ").trim()) + .filter(Boolean); + const formattedSummary = + summaryLines.length > 0 + ? summaryLines.map((line) => `- ${line}`).join("\n") + : "- 我已基于当前图片梳理主体、场景、风格与光线特征。"; return [ "【需求理解】", `- ${demand}`, "【图片分析】", - summary ? `- ${summary}` : "- 我已基于当前图片梳理主体、场景、风格与光线特征。", + formattedSummary, "【优化后提示词】", `- ${params.optimizedPrompt.replace(/\s+/g, " ").trim()}`, "【使用建议】", @@ -1712,6 +1719,37 @@ function buildPromptOnlyReply(params: { ].join("\n"); } +function isConfirmedPromptGenerationRequest(text: string): boolean { + const normalized = String(text || "").toLowerCase().replace(/\s+/g, ""); + if (!normalized) return false; + + const refersPrompt = + /根据这个提示词|按这个提示词|就按这个提示词|按刚才的提示词|按上面的提示词|用这个提示词|用刚才的提示词|按该提示词/.test( + normalized, + ); + const wantsImageGeneration = + /生图|生成图片|出图|画出来|生成一张|来一张|开始生成|按此生成|按这个生成/.test(normalized); + + return refersPrompt && wantsImageGeneration; +} + +function extractConfirmedPromptFromWorkingState( + workingState?: { + lastTurnSummary?: Record | null; + } | null, +): string | undefined { + const summary = workingState?.lastTurnSummary; + if (!summary || typeof summary !== "object") return undefined; + + const resolutionMode = + typeof summary.resolutionMode === "string" ? summary.resolutionMode.trim() : ""; + const optimizedPrompt = + typeof summary.optimizedPrompt === "string" ? summary.optimizedPrompt.trim() : ""; + + if (resolutionMode !== "prompt_only" || !optimizedPrompt) return undefined; + return optimizedPrompt; +} + function hasStructuredAgentSections(message?: string): boolean { if (!message) return false; return /(?:###\s*需求理解|【\s*需求理解\s*】|###\s*图片分析|【\s*图片分析\s*】|###\s*执行计划|【\s*执行计划\s*】|###\s*优化后提示词|【\s*优化后提示词\s*】)/.test( @@ -1785,8 +1823,11 @@ function shouldEnrichImageAnalysisText(message?: string): boolean { } function injectImageAnalysisSummary(message: string, summary: string): string { - const normalizedSummary = summary.replace(/\s+/g, " ").trim(); - if (!normalizedSummary) return message; + const normalizedSummaryLines = summary + .split(/\r?\n/) + .map((line) => line.replace(/\s+/g, " ").trim()) + .filter(Boolean); + if (normalizedSummaryLines.length === 0) return message; const lines = message.split("\n"); const analysisHeaderIndex = lines.findIndex((line) => /^(###\s*图片分析|【\s*图片分析\s*】)/.test(line.trim()) @@ -1797,27 +1838,31 @@ function injectImageAnalysisSummary(message: string, summary: string): string { const current = lines[i]?.trim() || ""; if (/^(###\s*|【\s*[^】]+\s*】)/.test(current)) break; if (/^- /.test(current)) { - lines[i] = `- ${normalizedSummary}`; + lines.splice(i, 1, ...normalizedSummaryLines.map((line) => `- ${line}`)); return lines.join("\n"); } } const insertIndex = analysisHeaderIndex + 1; - lines.splice(insertIndex, 0, `- ${normalizedSummary}`); + lines.splice(insertIndex, 0, ...normalizedSummaryLines.map((line) => `- ${line}`)); return lines.join("\n"); } function ensureImageAnalysisSummarySection(message: string, summary: string): string { - const normalizedSummary = summary.replace(/\s+/g, " ").trim(); - if (!normalizedSummary) return message; + const normalizedSummaryLines = summary + .split(/\r?\n/) + .map((line) => line.replace(/\s+/g, " ").trim()) + .filter(Boolean); + if (normalizedSummaryLines.length === 0) return message; if (/(###\s*图片分析|【\s*图片分析\s*】)/.test(message)) { - return injectImageAnalysisSummary(message, normalizedSummary); + return injectImageAnalysisSummary(message, normalizedSummaryLines.join("\n")); } const trimmed = message.trim(); - if (!trimmed) return `【图片分析】\n- ${normalizedSummary}`; - return `${trimmed}\n【图片分析】\n- ${normalizedSummary}`; + const summaryBlock = normalizedSummaryLines.map((line) => `- ${line}`).join("\n"); + if (!trimmed) return `【图片分析】\n${summaryBlock}`; + return `${trimmed}\n【图片分析】\n${summaryBlock}`; } /** @@ -2814,6 +2859,7 @@ export async function POST(request: Request) { let requestWorkingState: | Awaited> | null = null; + let smartAutoHistoryMessagesFromDb: DBMessage[] = []; let preferredAgentModelId: string | undefined; const deepThinkingOn = deepThinkingEnabled !== false; let shouldRunAgentConversation = finalModelId === "auto"; @@ -2915,10 +2961,12 @@ export async function POST(request: Request) { let autoTextForFallback = ""; let autoFromHistoryForFallback = false; let autoCurrentImageUrlsForFallback: string[] = []; + let confirmedPromptForGeneration: string | undefined; + let forcePromptDrivenImageGeneration = false; try { // 🆕 首先获取历史消息（在 Smart Auto Mode 内部单独获取，避免依赖后面的 messagesFromDb） - const smartAutoHistoryMessagesFromDb = await getMessagesByChatId({ id }); + smartAutoHistoryMessagesFromDb = await getMessagesByChatId({ id }); console.log("[Smart Auto Mode] Loaded", smartAutoHistoryMessagesFromDb.length, "messages from DB"); autoTextForFallback = requestTextContent; @@ -2927,6 +2975,10 @@ export async function POST(request: Request) { chatId: id, }); } + confirmedPromptForGeneration = extractConfirmedPromptFromWorkingState(requestWorkingState); + forcePromptDrivenImageGeneration = + isConfirmedPromptGenerationRequest(requestTextContent) && + Boolean(confirmedPromptForGeneration); if (!requestResolvedMediaInput) { requestResolvedMediaInput = resolveChatMediaInput({ messageParts: message.parts as any[], @@ -2968,6 +3020,17 @@ export async function POST(request: Request) { autoFromHistoryForFallback = currentMessageImageInputs.length === 0 && imageUrls.length > 0; + if (forcePromptDrivenImageGeneration) { + imageUrls = []; + hasImage = false; + autoCurrentImageUrlsForFallback = []; + smartAutoHistoryImages = []; + autoFromHistoryForFallback = false; + autoTextForFallback = confirmedPromptForGeneration || requestTextContent; + smartAutoOptimizedPrompt = confirmedPromptForGeneration; + console.log("[Smart Auto Mode] 🎯 Using confirmed prompt-only result for image generation"); + } + console.log("[Smart Auto Mode] Input:", { textContent: requestTextContent.substring(0, 100), hasImage, @@ -3041,6 +3104,28 @@ export async function POST(request: Request) { smartAutoPlan = autoGraphResult.plan; smartAutoExecution = autoGraphResult.execution; + if (forcePromptDrivenImageGeneration && confirmedPromptForGeneration) { + smartAutoDetectedIntent = "image_gen"; + smartAutoOptimizedPrompt = confirmedPromptForGeneration; + smartAutoExecution = { + mode: "generate_image", + routeIntent: "image_gen", + routeSubIntent: undefined, + targetSource: "none", + preferredCapability: "generate_image", + shouldUseStructuredReply: true, + shouldOptimizePrompt: true, + shouldApplyImagePipeline: true, + shouldApplyVideoPipeline: false, + shouldStayConversational: false, + shouldPublishToCanvas: false, + toolCalls: [], + availableTools: [], + }; + smartAutoFriendlyMessage = + smartAutoFriendlyMessage || "收到，我会按你刚确认的提示词直接生图。"; + } + if (smartAutoPlan) { console.log("[Smart Auto Mode] Plan result:", { goal: smartAutoPlan.goal, @@ -4092,15 +4177,23 @@ export async function POST(request: Request) { const streamId = generateUUID(); await createStreamId({ streamId, chatId: id }); - const wantsPromptOnlyFromCurrentImage = + const promptOnlySourceImageUrls = + requestResolvedMediaInput.currentMessageImageUrls.length > 0 + ? requestResolvedMediaInput.currentMessageImageUrls.slice(0, 4) + : requestTargetResolution?.targetImageUrls.slice(0, 4) || smartAutoHistoryImages.slice(0, 4); + const promptOnlySourceAssetIds = + requestTargetResolution?.targetAssetIds?.length + ? requestTargetResolution.targetAssetIds + : requestResolvedMediaInput.uploadedAssetIds; + const wantsPromptOnlyFromImageContext = shouldRunAgentConversation && !smartAutoNeedsClarification && - requestResolvedMediaInput.currentMessageImageUrls.length > 0 && + promptOnlySourceImageUrls.length > 0 && ( await import("@/lib/ai/smart-analyzer") ).isExplicitPromptOptimizationRequest(requestTextContent); - if (wantsPromptOnlyFromCurrentImage) { + if (wantsPromptOnlyFromImageContext) { let promptOnlyImageSummary = ""; try { const { summarizeImageForExecutionReport } = await import("@/lib/ai/smart-analyzer"); @@ -4119,10 +4212,11 @@ export async function POST(request: Request) { const summary = await withOperationTimeout( summarizeImageForExecutionReport({ text: requestTextContent, - imageUrls: requestResolvedMediaInput.currentMessageImageUrls.slice(0, 4), + imageUrls: promptOnlySourceImageUrls, context: typeof autoContext === "string" ? autoContext : undefined, preferredModelId: preferredAgentModelId, - assetIds: summaryAssetIds, + assetIds: + promptOnlySourceAssetIds.length > 0 ? promptOnlySourceAssetIds : summaryAssetIds, }), "promptOnlyImageSummary", summaryTimeoutMs, @@ -4202,12 +4296,20 @@ export async function POST(request: Request) { chatId: id, projectId: contextPayload?.projectId || null, currentIntent: "vision", + currentFocusAssetIds: + promptOnlySourceAssetIds.length > 0 ? promptOnlySourceAssetIds : null, + currentFocusImageUrls: promptOnlySourceImageUrls, + lastResolvedTargetAssetIds: + promptOnlySourceAssetIds.length > 0 ? promptOnlySourceAssetIds : null, + lastResolvedTargetImageUrls: promptOnlySourceImageUrls, lastMessageId: messageId, lastTurnSummary: { text: requestTextContent.slice(0, 500), resolutionMode: "prompt_only", confidence: 90, optimizedPrompt: promptOnlyOptimizedPrompt.slice(0, 1000), + imageSummary: promptOnlyImageSummary.slice(0, 1200), + sourceImageUrls: promptOnlySourceImageUrls, }, }); } catch (promptOnlyMemoryError) { diff --git a/lib/ai/smart-analyzer.ts b/lib/ai/smart-analyzer.ts index 6f3abfa79..e9868ee79 100644 --- a/lib/ai/smart-analyzer.ts +++ b/lib/ai/smart-analyzer.ts @@ -456,7 +456,7 @@ function getEnvInt(name: string, fallback: number, min: number, max: number): nu return Math.max(min, Math.min(max, parsed)); } -const ANALYZER_MAX_IMAGE_COUNT = 2; +const ANALYZER_MAX_IMAGE_COUNT = 4; const ANALYZER_IMAGE_MAX_SIDE = 500; const ANALYZER_TIMEOUT_MS = getEnvInt( "AGENT_ANALYZER_TIMEOUT_MS", @@ -1324,6 +1324,12 @@ async function persistVisualSummarySnapshot(params: { } } +function stripVisualSummaryImageLabel(summary: string): string { + return String(summary || "") + .replace(/^\s*图\s*\d+\s*[:：]\s*/i, "") + .trim(); +} + export async function summarizeImageForExecutionReport(params: { text?: string; imageUrls?: string[]; @@ -1332,10 +1338,28 @@ export async function summarizeImageForExecutionReport(params: { assetIds?: string[]; }): Promise { const { text, imageUrls, context, preferredModelId, assetIds } = params; + const normalizedInputUrls = uniqueNonEmptyStrings(imageUrls).slice(0, ANALYZER_MAX_IMAGE_COUNT); + if (normalizedInputUrls.length > 1) { + const mergedLines: string[] = []; + for (const [index, imageUrl] of normalizedInputUrls.entries()) { + const singleSummary = await summarizeImageForExecutionReport({ + text, + imageUrls: [imageUrl], + context, + preferredModelId, + assetIds: assetIds?.[index] ? [assetIds[index] as string] : [], + }); + if (singleSummary?.trim()) { + mergedLines.push(`图${index + 1}：${stripVisualSummaryImageLabel(singleSummary)}`); + } + } + return mergedLines.length > 0 ? mergedLines.join("\n") : null; + } + const uniqueAssetIds = uniqueNonEmptyStrings(assetIds); const cachedSummary = await getCachedVisualSummaryByAssetIds(uniqueAssetIds); if (cachedSummary) return cachedSummary; - const visionImages = await buildAnalyzerVisionImages(imageUrls); + const visionImages = await buildAnalyzerVisionImages(normalizedInputUrls); if (visionImages.length === 0) return null; const finalizeVisualSummary = async (summary?: string | null): Promise => { @@ -1399,7 +1423,8 @@ export async function summarizeImageForExecutionReport(params: { const promptText = [ "只根据图片中可见内容做客观观察，不要推测用户意图。", "禁止出现“选中/待处理/按需求/处理/修改/转换”等执行词。", - "输出一行：画面：主体与关键元素；场景：环境；风格：视觉风格；光线：明暗与色温。", + "如果只有1张图，输出一行：画面：主体与关键元素；场景：环境；风格：视觉风格；光线：明暗与色温。", + "如果有多张图，逐张输出“图1：...”“图2：...”格式，每张各一行。", text ? `用户目标（仅辅助理解，不要复述）：${text.slice(0, 120)}` : "", context ? `画布上下文（仅辅助理解，不要复述）：${context.slice(0, 120)}` : "", ].join("\n"); @@ -1428,13 +1453,13 @@ export async function summarizeImageForExecutionReport(params: { } if (volcanoProvider?.apiKey && volcanoProvider?.baseUrl) { - const summary = await summarizeWithVolcanoOfficialApi({ - apiKey: volcanoProvider.apiKey, - baseUrl: volcanoProvider.baseUrl, - promptText, - imageUrls: (imageUrls || []).filter((url) => isHttpUrl(url)).slice(0, ANALYZER_MAX_IMAGE_COUNT), - modelId: agentModel?.model?.modelId, - }); + const summary = await summarizeWithVolcanoOfficialApi({ + apiKey: volcanoProvider.apiKey, + baseUrl: volcanoProvider.baseUrl, + promptText, + imageUrls: normalizedInputUrls.filter((url) => isHttpUrl(url)).slice(0, ANALYZER_MAX_IMAGE_COUNT), + modelId: agentModel?.model?.modelId, + }); if (summary) return await finalizeVisualSummary(summary); } @@ -1468,7 +1493,7 @@ export async function summarizeImageForExecutionReport(params: { content: [ { type: "text", - text: `${promptText}\n仅输出一行，不超过60字。`, + text: `${promptText}\n单图时仅输出一行，不超过60字；多图时每张一行，总计不超过6行。`, }, ...fallbackVisionImages, ], @@ -1518,9 +1543,16 @@ export async function summarizeImageForExecutionReport(params: { function normalizeVisualSummaryText(text?: string | null): string { if (!text) return ""; return text - .replace(/\s+/g, " ") - .replace(/^[-*•]\s*/, "") - .replace(/^(画面要点|画面描述|图片分析|分析结果)[:：]\s*/i, "") + .split(/\r?\n/) + .map((line) => + line + .replace(/\s+/g, " ") + .replace(/^[-*•]\s*/, "") + .replace(/^(画面要点|画面描述|图片分析|分析结果)[:：]\s*/i, "") + .trim(), + ) + .filter(Boolean) + .join("\n") .trim(); }