fix(agent): enforce concrete visual summary content
Some checks failed
Lint / build (20) (push) Has been cancelled
Playwright Tests / test (push) Has been cancelled

This commit is contained in:
Your Name 2026-02-19 04:03:55 +08:00
parent e5c4b3780c
commit 484c1807e3
2 changed files with 74 additions and 35 deletions

View File

@ -2233,7 +2233,10 @@ export async function POST(request: Request) {
imageSummary: summary.trim(),
});
smartAutoFriendlyMessage = ensureImageAnalysisSummarySection(baseMessage, summary.trim());
console.log("[Smart Auto Mode] ✅ Image analysis summary ready");
console.log(
"[Smart Auto Mode] ✅ Image analysis summary ready:",
summary.trim().slice(0, 160),
);
} else if (smartAutoFriendlyMessage && shouldEnrichImageAnalysisText(smartAutoFriendlyMessage)) {
console.warn("[Smart Auto Mode] ⚠️ Image summary unavailable, keep original analysis text");
}

View File

@ -557,7 +557,7 @@ async function summarizeWithZhipuOfficialApi(params: {
{
role: "system",
content:
"你是图像观察助手。请根据图片给出执行前说明,输出严格两行中文:第1行“画面:...”,第2行“关键要素:...”。不要输出其他内容。",
"你是图像观察助手。只描述图片可见内容,不要提执行动作或用户意图。输出严格一行中文。",
},
{
role: "user",
@ -581,9 +581,10 @@ async function summarizeWithZhipuOfficialApi(params: {
const message = payload?.choices?.[0]?.message;
const responseText = String(message?.content || message?.reasoning_content || "").trim();
if (responseText) {
const normalized = normalizeVisualSummaryText(responseText);
if (normalized && !isWeakVisualSummary(normalized)) {
console.log(`[Smart Analyzer] Zhipu official summary model used: ${modelId}`);
return responseText;
return normalized;
}
} catch {
continue;
@ -620,12 +621,10 @@ export async function summarizeImageForExecutionReport(params: {
const zhipuProvider = zhipuProviderFromAgent || zhipuProviderFallback;
const promptText = [
text ? `用户需求:${text}` : "",
context ? `画布上下文:${context.slice(0, 220)}` : "",
"请给出当前画面的客观描述,便于执行前说明。",
]
.filter(Boolean)
.join("\n");
"只根据图片中可见内容做客观观察,不要推测用户意图。",
"禁止出现“选中/待处理/按需求/处理/修改/转换”等执行词。",
"输出一行:画面:主体与关键元素;场景:环境;风格:视觉风格;光线:明暗与色温。",
].join("\n");
if (zhipuProvider?.apiKey && zhipuProvider?.baseUrl) {
const summary = await summarizeWithZhipuOfficialApi({
@ -647,33 +646,40 @@ export async function summarizeImageForExecutionReport(params: {
const model = await getLanguageModelById(selectedVisionModel.model.id);
if (!model) return null;
const fallbackVisionImages = await buildAnalyzerVisionImagesByMode({
imageUrls,
preferRemoteUrls: shouldPreferRemoteVisionUrls(selectedVisionModel),
});
if (fallbackVisionImages.length === 0) return null;
const preferRemote = shouldPreferRemoteVisionUrls(selectedVisionModel);
const modeAttempts = Array.from(new Set([preferRemote, !preferRemote]));
const { text: summaryText } = await generateText({
model,
messages: [
{
role: "user",
content: [
{
type: "text",
text: `${promptText}\n只输出一行,格式:画面要点:主体+场景+风格+光线(不超过45字)。`,
},
...fallbackVisionImages,
],
},
],
temperature: 0.1,
maxTokens: 120,
abortSignal: AbortSignal.timeout(ANALYZER_TIMEOUT_MS),
} as any);
for (const preferRemoteUrls of modeAttempts) {
const fallbackVisionImages = await buildAnalyzerVisionImagesByMode({
imageUrls,
preferRemoteUrls,
});
if (fallbackVisionImages.length === 0) continue;
const normalized = summaryText?.replace(/\s+/g, " ").trim();
return normalized || null;
const { text: summaryText } = await generateText({
model,
messages: [
{
role: "user",
content: [
{
type: "text",
text: `${promptText}\n仅输出一行,不超过60字。`,
},
...fallbackVisionImages,
],
},
],
temperature: 0.1,
maxTokens: 120,
abortSignal: AbortSignal.timeout(ANALYZER_TIMEOUT_MS),
} as any);
const normalized = normalizeVisualSummaryText(summaryText);
if (normalized && !isWeakVisualSummary(normalized)) {
return normalized;
}
}
} catch (error) {
console.warn("[Smart Analyzer] summarizeImageForExecutionReport fallback failed:", error);
}
@ -681,6 +687,36 @@ export async function summarizeImageForExecutionReport(params: {
return null;
}
function normalizeVisualSummaryText(text?: string | null): string {
if (!text) return "";
return text
.replace(/\s+/g, " ")
.replace(/^[-*•]\s*/, "")
.replace(/^(画面要点|画面描述|图片分析|分析结果)[::]\s*/i, "")
.trim();
}
function isWeakVisualSummary(summary: string): boolean {
const normalized = summary.replace(/\s+/g, " ").trim();
if (!normalized || normalized.length < 14) return true;
const weakPatterns = [
/待处理图/,
/选中.*位置/,
/当前上传图片/,
/基于.*原图处理/,
/按.*需求执行/,
/仅做.*调整/,
/不改动原有内容结构/,
/转换为.*场景/,
];
if (weakPatterns.some((pattern) => pattern.test(normalized))) {
return true;
}
return false;
}
// 历史消息类型
interface HistoryMessage {
role: "user" | "assistant";