persist prompt-only flows and support multi-image analysis
This commit is contained in:
Your Name
parent
c05eca89f2
commit
6de541c1fb
@ -1698,13 +1698,20 @@ function buildPromptOnlyReply(params: {
|
||||
optimizedPrompt: string;
|
||||
}): string {
|
||||
const demand = sanitizePromptOnlyUserIntent(params.userText) || "你希望我先分析这张图并整理可直接使用的提示词";
|
||||
const summary = String(params.imageSummary || "").replace(/\s+/g, " ").trim();
|
||||
const summaryLines = String(params.imageSummary || "")
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.replace(/\s+/g, " ").trim())
|
||||
.filter(Boolean);
|
||||
const formattedSummary =
|
||||
summaryLines.length > 0
|
||||
? summaryLines.map((line) => `- ${line}`).join("\n")
|
||||
: "- 我已基于当前图片梳理主体、场景、风格与光线特征。";
|
||||
|
||||
return [
|
||||
"【需求理解】",
|
||||
`- ${demand}`,
|
||||
"【图片分析】",
|
||||
summary ? `- ${summary}` : "- 我已基于当前图片梳理主体、场景、风格与光线特征。",
|
||||
formattedSummary,
|
||||
"【优化后提示词】",
|
||||
`- ${params.optimizedPrompt.replace(/\s+/g, " ").trim()}`,
|
||||
"【使用建议】",
|
||||
@ -1712,6 +1719,37 @@ function buildPromptOnlyReply(params: {
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function isConfirmedPromptGenerationRequest(text: string): boolean {
|
||||
const normalized = String(text || "").toLowerCase().replace(/\s+/g, "");
|
||||
if (!normalized) return false;
|
||||
|
||||
const refersPrompt =
|
||||
/根据这个提示词|按这个提示词|就按这个提示词|按刚才的提示词|按上面的提示词|用这个提示词|用刚才的提示词|按该提示词/.test(
|
||||
normalized,
|
||||
);
|
||||
const wantsImageGeneration =
|
||||
/生图|生成图片|出图|画出来|生成一张|来一张|开始生成|按此生成|按这个生成/.test(normalized);
|
||||
|
||||
return refersPrompt && wantsImageGeneration;
|
||||
}
|
||||
|
||||
function extractConfirmedPromptFromWorkingState(
|
||||
workingState?: {
|
||||
lastTurnSummary?: Record<string, unknown> | null;
|
||||
} | null,
|
||||
): string | undefined {
|
||||
const summary = workingState?.lastTurnSummary;
|
||||
if (!summary || typeof summary !== "object") return undefined;
|
||||
|
||||
const resolutionMode =
|
||||
typeof summary.resolutionMode === "string" ? summary.resolutionMode.trim() : "";
|
||||
const optimizedPrompt =
|
||||
typeof summary.optimizedPrompt === "string" ? summary.optimizedPrompt.trim() : "";
|
||||
|
||||
if (resolutionMode !== "prompt_only" || !optimizedPrompt) return undefined;
|
||||
return optimizedPrompt;
|
||||
}
|
||||
|
||||
function hasStructuredAgentSections(message?: string): boolean {
|
||||
if (!message) return false;
|
||||
return /(?:###\s*需求理解|【\s*需求理解\s*】|###\s*图片分析|【\s*图片分析\s*】|###\s*执行计划|【\s*执行计划\s*】|###\s*优化后提示词|【\s*优化后提示词\s*】)/.test(
|
||||
@ -1785,8 +1823,11 @@ function shouldEnrichImageAnalysisText(message?: string): boolean {
|
||||
}
|
||||
|
||||
function injectImageAnalysisSummary(message: string, summary: string): string {
|
||||
const normalizedSummary = summary.replace(/\s+/g, " ").trim();
|
||||
if (!normalizedSummary) return message;
|
||||
const normalizedSummaryLines = summary
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.replace(/\s+/g, " ").trim())
|
||||
.filter(Boolean);
|
||||
if (normalizedSummaryLines.length === 0) return message;
|
||||
const lines = message.split("\n");
|
||||
const analysisHeaderIndex = lines.findIndex((line) =>
|
||||
/^(###\s*图片分析|【\s*图片分析\s*】)/.test(line.trim())
|
||||
@ -1797,27 +1838,31 @@ function injectImageAnalysisSummary(message: string, summary: string): string {
|
||||
const current = lines[i]?.trim() || "";
|
||||
if (/^(###\s*|【\s*[^】]+\s*】)/.test(current)) break;
|
||||
if (/^- /.test(current)) {
|
||||
lines[i] = `- ${normalizedSummary}`;
|
||||
lines.splice(i, 1, ...normalizedSummaryLines.map((line) => `- ${line}`));
|
||||
return lines.join("\n");
|
||||
}
|
||||
}
|
||||
|
||||
const insertIndex = analysisHeaderIndex + 1;
|
||||
lines.splice(insertIndex, 0, `- ${normalizedSummary}`);
|
||||
lines.splice(insertIndex, 0, ...normalizedSummaryLines.map((line) => `- ${line}`));
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
function ensureImageAnalysisSummarySection(message: string, summary: string): string {
|
||||
const normalizedSummary = summary.replace(/\s+/g, " ").trim();
|
||||
if (!normalizedSummary) return message;
|
||||
const normalizedSummaryLines = summary
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.replace(/\s+/g, " ").trim())
|
||||
.filter(Boolean);
|
||||
if (normalizedSummaryLines.length === 0) return message;
|
||||
|
||||
if (/(###\s*图片分析|【\s*图片分析\s*】)/.test(message)) {
|
||||
return injectImageAnalysisSummary(message, normalizedSummary);
|
||||
return injectImageAnalysisSummary(message, normalizedSummaryLines.join("\n"));
|
||||
}
|
||||
|
||||
const trimmed = message.trim();
|
||||
if (!trimmed) return `【图片分析】\n- ${normalizedSummary}`;
|
||||
return `${trimmed}\n【图片分析】\n- ${normalizedSummary}`;
|
||||
const summaryBlock = normalizedSummaryLines.map((line) => `- ${line}`).join("\n");
|
||||
if (!trimmed) return `【图片分析】\n${summaryBlock}`;
|
||||
return `${trimmed}\n【图片分析】\n${summaryBlock}`;
|
||||
}
|
||||
|
||||
/**
|
||||
@ -2814,6 +2859,7 @@ export async function POST(request: Request) {
|
||||
let requestWorkingState:
|
||||
| Awaited<ReturnType<typeof getConversationWorkingStateByChatId>>
|
||||
| null = null;
|
||||
let smartAutoHistoryMessagesFromDb: DBMessage[] = [];
|
||||
let preferredAgentModelId: string | undefined;
|
||||
const deepThinkingOn = deepThinkingEnabled !== false;
|
||||
let shouldRunAgentConversation = finalModelId === "auto";
|
||||
@ -2915,10 +2961,12 @@ export async function POST(request: Request) {
|
||||
let autoTextForFallback = "";
|
||||
let autoFromHistoryForFallback = false;
|
||||
let autoCurrentImageUrlsForFallback: string[] = [];
|
||||
let confirmedPromptForGeneration: string | undefined;
|
||||
let forcePromptDrivenImageGeneration = false;
|
||||
|
||||
try {
|
||||
// 🆕 首先获取历史消息（在 Smart Auto Mode 内部单独获取，避免依赖后面的 messagesFromDb）
|
||||
const smartAutoHistoryMessagesFromDb = await getMessagesByChatId({ id });
|
||||
smartAutoHistoryMessagesFromDb = await getMessagesByChatId({ id });
|
||||
console.log("[Smart Auto Mode] Loaded", smartAutoHistoryMessagesFromDb.length, "messages from DB");
|
||||
|
||||
autoTextForFallback = requestTextContent;
|
||||
@ -2927,6 +2975,10 @@ export async function POST(request: Request) {
|
||||
chatId: id,
|
||||
});
|
||||
}
|
||||
confirmedPromptForGeneration = extractConfirmedPromptFromWorkingState(requestWorkingState);
|
||||
forcePromptDrivenImageGeneration =
|
||||
isConfirmedPromptGenerationRequest(requestTextContent) &&
|
||||
Boolean(confirmedPromptForGeneration);
|
||||
if (!requestResolvedMediaInput) {
|
||||
requestResolvedMediaInput = resolveChatMediaInput({
|
||||
messageParts: message.parts as any[],
|
||||
@ -2968,6 +3020,17 @@ export async function POST(request: Request) {
|
||||
autoFromHistoryForFallback =
|
||||
currentMessageImageInputs.length === 0 && imageUrls.length > 0;
|
||||
|
||||
if (forcePromptDrivenImageGeneration) {
|
||||
imageUrls = [];
|
||||
hasImage = false;
|
||||
autoCurrentImageUrlsForFallback = [];
|
||||
smartAutoHistoryImages = [];
|
||||
autoFromHistoryForFallback = false;
|
||||
autoTextForFallback = confirmedPromptForGeneration || requestTextContent;
|
||||
smartAutoOptimizedPrompt = confirmedPromptForGeneration;
|
||||
console.log("[Smart Auto Mode] 🎯 Using confirmed prompt-only result for image generation");
|
||||
}
|
||||
|
||||
console.log("[Smart Auto Mode] Input:", {
|
||||
textContent: requestTextContent.substring(0, 100),
|
||||
hasImage,
|
||||
@ -3041,6 +3104,28 @@ export async function POST(request: Request) {
|
||||
smartAutoPlan = autoGraphResult.plan;
|
||||
smartAutoExecution = autoGraphResult.execution;
|
||||
|
||||
if (forcePromptDrivenImageGeneration && confirmedPromptForGeneration) {
|
||||
smartAutoDetectedIntent = "image_gen";
|
||||
smartAutoOptimizedPrompt = confirmedPromptForGeneration;
|
||||
smartAutoExecution = {
|
||||
mode: "generate_image",
|
||||
routeIntent: "image_gen",
|
||||
routeSubIntent: undefined,
|
||||
targetSource: "none",
|
||||
preferredCapability: "generate_image",
|
||||
shouldUseStructuredReply: true,
|
||||
shouldOptimizePrompt: true,
|
||||
shouldApplyImagePipeline: true,
|
||||
shouldApplyVideoPipeline: false,
|
||||
shouldStayConversational: false,
|
||||
shouldPublishToCanvas: false,
|
||||
toolCalls: [],
|
||||
availableTools: [],
|
||||
};
|
||||
smartAutoFriendlyMessage =
|
||||
smartAutoFriendlyMessage || "收到，我会按你刚确认的提示词直接生图。";
|
||||
}
|
||||
|
||||
if (smartAutoPlan) {
|
||||
console.log("[Smart Auto Mode] Plan result:", {
|
||||
goal: smartAutoPlan.goal,
|
||||
@ -4092,15 +4177,23 @@ export async function POST(request: Request) {
|
||||
const streamId = generateUUID();
|
||||
await createStreamId({ streamId, chatId: id });
|
||||
|
||||
const wantsPromptOnlyFromCurrentImage =
|
||||
const promptOnlySourceImageUrls =
|
||||
requestResolvedMediaInput.currentMessageImageUrls.length > 0
|
||||
? requestResolvedMediaInput.currentMessageImageUrls.slice(0, 4)
|
||||
: requestTargetResolution?.targetImageUrls.slice(0, 4) || smartAutoHistoryImages.slice(0, 4);
|
||||
const promptOnlySourceAssetIds =
|
||||
requestTargetResolution?.targetAssetIds?.length
|
||||
? requestTargetResolution.targetAssetIds
|
||||
: requestResolvedMediaInput.uploadedAssetIds;
|
||||
const wantsPromptOnlyFromImageContext =
|
||||
shouldRunAgentConversation &&
|
||||
!smartAutoNeedsClarification &&
|
||||
requestResolvedMediaInput.currentMessageImageUrls.length > 0 &&
|
||||
promptOnlySourceImageUrls.length > 0 &&
|
||||
(
|
||||
await import("@/lib/ai/smart-analyzer")
|
||||
).isExplicitPromptOptimizationRequest(requestTextContent);
|
||||
|
||||
if (wantsPromptOnlyFromCurrentImage) {
|
||||
if (wantsPromptOnlyFromImageContext) {
|
||||
let promptOnlyImageSummary = "";
|
||||
try {
|
||||
const { summarizeImageForExecutionReport } = await import("@/lib/ai/smart-analyzer");
|
||||
@ -4119,10 +4212,11 @@ export async function POST(request: Request) {
|
||||
const summary = await withOperationTimeout(
|
||||
summarizeImageForExecutionReport({
|
||||
text: requestTextContent,
|
||||
imageUrls: requestResolvedMediaInput.currentMessageImageUrls.slice(0, 4),
|
||||
imageUrls: promptOnlySourceImageUrls,
|
||||
context: typeof autoContext === "string" ? autoContext : undefined,
|
||||
preferredModelId: preferredAgentModelId,
|
||||
assetIds: summaryAssetIds,
|
||||
assetIds:
|
||||
promptOnlySourceAssetIds.length > 0 ? promptOnlySourceAssetIds : summaryAssetIds,
|
||||
}),
|
||||
"promptOnlyImageSummary",
|
||||
summaryTimeoutMs,
|
||||
@ -4202,12 +4296,20 @@ export async function POST(request: Request) {
|
||||
chatId: id,
|
||||
projectId: contextPayload?.projectId || null,
|
||||
currentIntent: "vision",
|
||||
currentFocusAssetIds:
|
||||
promptOnlySourceAssetIds.length > 0 ? promptOnlySourceAssetIds : null,
|
||||
currentFocusImageUrls: promptOnlySourceImageUrls,
|
||||
lastResolvedTargetAssetIds:
|
||||
promptOnlySourceAssetIds.length > 0 ? promptOnlySourceAssetIds : null,
|
||||
lastResolvedTargetImageUrls: promptOnlySourceImageUrls,
|
||||
lastMessageId: messageId,
|
||||
lastTurnSummary: {
|
||||
text: requestTextContent.slice(0, 500),
|
||||
resolutionMode: "prompt_only",
|
||||
confidence: 90,
|
||||
optimizedPrompt: promptOnlyOptimizedPrompt.slice(0, 1000),
|
||||
imageSummary: promptOnlyImageSummary.slice(0, 1200),
|
||||
sourceImageUrls: promptOnlySourceImageUrls,
|
||||
},
|
||||
});
|
||||
} catch (promptOnlyMemoryError) {
|
||||
|
||||
@ -456,7 +456,7 @@ function getEnvInt(name: string, fallback: number, min: number, max: number): nu
|
||||
return Math.max(min, Math.min(max, parsed));
|
||||
}
|
||||
|
||||
const ANALYZER_MAX_IMAGE_COUNT = 2;
|
||||
const ANALYZER_MAX_IMAGE_COUNT = 4;
|
||||
const ANALYZER_IMAGE_MAX_SIDE = 500;
|
||||
const ANALYZER_TIMEOUT_MS = getEnvInt(
|
||||
"AGENT_ANALYZER_TIMEOUT_MS",
|
||||
@ -1324,6 +1324,12 @@ async function persistVisualSummarySnapshot(params: {
|
||||
}
|
||||
}
|
||||
|
||||
function stripVisualSummaryImageLabel(summary: string): string {
|
||||
return String(summary || "")
|
||||
.replace(/^\s*图\s*\d+\s*[:：]\s*/i, "")
|
||||
.trim();
|
||||
}
|
||||
|
||||
export async function summarizeImageForExecutionReport(params: {
|
||||
text?: string;
|
||||
imageUrls?: string[];
|
||||
@ -1332,10 +1338,28 @@ export async function summarizeImageForExecutionReport(params: {
|
||||
assetIds?: string[];
|
||||
}): Promise<string | null> {
|
||||
const { text, imageUrls, context, preferredModelId, assetIds } = params;
|
||||
const normalizedInputUrls = uniqueNonEmptyStrings(imageUrls).slice(0, ANALYZER_MAX_IMAGE_COUNT);
|
||||
if (normalizedInputUrls.length > 1) {
|
||||
const mergedLines: string[] = [];
|
||||
for (const [index, imageUrl] of normalizedInputUrls.entries()) {
|
||||
const singleSummary = await summarizeImageForExecutionReport({
|
||||
text,
|
||||
imageUrls: [imageUrl],
|
||||
context,
|
||||
preferredModelId,
|
||||
assetIds: assetIds?.[index] ? [assetIds[index] as string] : [],
|
||||
});
|
||||
if (singleSummary?.trim()) {
|
||||
mergedLines.push(`图${index + 1}：${stripVisualSummaryImageLabel(singleSummary)}`);
|
||||
}
|
||||
}
|
||||
return mergedLines.length > 0 ? mergedLines.join("\n") : null;
|
||||
}
|
||||
|
||||
const uniqueAssetIds = uniqueNonEmptyStrings(assetIds);
|
||||
const cachedSummary = await getCachedVisualSummaryByAssetIds(uniqueAssetIds);
|
||||
if (cachedSummary) return cachedSummary;
|
||||
const visionImages = await buildAnalyzerVisionImages(imageUrls);
|
||||
const visionImages = await buildAnalyzerVisionImages(normalizedInputUrls);
|
||||
if (visionImages.length === 0) return null;
|
||||
|
||||
const finalizeVisualSummary = async (summary?: string | null): Promise<string | null> => {
|
||||
@ -1399,7 +1423,8 @@ export async function summarizeImageForExecutionReport(params: {
|
||||
const promptText = [
|
||||
"只根据图片中可见内容做客观观察，不要推测用户意图。",
|
||||
"禁止出现“选中/待处理/按需求/处理/修改/转换”等执行词。",
|
||||
"输出一行：画面：主体与关键元素；场景：环境；风格：视觉风格；光线：明暗与色温。",
|
||||
"如果只有1张图，输出一行：画面：主体与关键元素；场景：环境；风格：视觉风格；光线：明暗与色温。",
|
||||
"如果有多张图，逐张输出“图1：...”“图2：...”格式，每张各一行。",
|
||||
text ? `用户目标（仅辅助理解，不要复述）：${text.slice(0, 120)}` : "",
|
||||
context ? `画布上下文（仅辅助理解，不要复述）：${context.slice(0, 120)}` : "",
|
||||
].join("\n");
|
||||
@ -1428,13 +1453,13 @@ export async function summarizeImageForExecutionReport(params: {
|
||||
}
|
||||
|
||||
if (volcanoProvider?.apiKey && volcanoProvider?.baseUrl) {
|
||||
const summary = await summarizeWithVolcanoOfficialApi({
|
||||
apiKey: volcanoProvider.apiKey,
|
||||
baseUrl: volcanoProvider.baseUrl,
|
||||
promptText,
|
||||
imageUrls: (imageUrls || []).filter((url) => isHttpUrl(url)).slice(0, ANALYZER_MAX_IMAGE_COUNT),
|
||||
modelId: agentModel?.model?.modelId,
|
||||
});
|
||||
const summary = await summarizeWithVolcanoOfficialApi({
|
||||
apiKey: volcanoProvider.apiKey,
|
||||
baseUrl: volcanoProvider.baseUrl,
|
||||
promptText,
|
||||
imageUrls: normalizedInputUrls.filter((url) => isHttpUrl(url)).slice(0, ANALYZER_MAX_IMAGE_COUNT),
|
||||
modelId: agentModel?.model?.modelId,
|
||||
});
|
||||
if (summary) return await finalizeVisualSummary(summary);
|
||||
}
|
||||
|
||||
@ -1468,7 +1493,7 @@ export async function summarizeImageForExecutionReport(params: {
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: `${promptText}\n仅输出一行，不超过60字。`,
|
||||
text: `${promptText}\n单图时仅输出一行，不超过60字；多图时每张一行，总计不超过6行。`,
|
||||
},
|
||||
...fallbackVisionImages,
|
||||
],
|
||||
@ -1518,9 +1543,16 @@ export async function summarizeImageForExecutionReport(params: {
|
||||
function normalizeVisualSummaryText(text?: string | null): string {
|
||||
if (!text) return "";
|
||||
return text
|
||||
.replace(/\s+/g, " ")
|
||||
.replace(/^[-*•]\s*/, "")
|
||||
.replace(/^(画面要点|画面描述|图片分析|分析结果)[:：]\s*/i, "")
|
||||
.split(/\r?\n/)
|
||||
.map((line) =>
|
||||
line
|
||||
.replace(/\s+/g, " ")
|
||||
.replace(/^[-*•]\s*/, "")
|
||||
.replace(/^(画面要点|画面描述|图片分析|分析结果)[:：]\s*/i, "")
|
||||
.trim(),
|
||||
)
|
||||
.filter(Boolean)
|
||||
.join("\n")
|
||||
.trim();
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user