fix(agent): enforce concrete visual summary content
This commit is contained in:
parent
e5c4b3780c
commit
484c1807e3
@ -2233,7 +2233,10 @@ export async function POST(request: Request) {
|
||||
imageSummary: summary.trim(),
|
||||
});
|
||||
smartAutoFriendlyMessage = ensureImageAnalysisSummarySection(baseMessage, summary.trim());
|
||||
console.log("[Smart Auto Mode] ✅ Image analysis summary ready");
|
||||
console.log(
|
||||
"[Smart Auto Mode] ✅ Image analysis summary ready:",
|
||||
summary.trim().slice(0, 160),
|
||||
);
|
||||
} else if (smartAutoFriendlyMessage && shouldEnrichImageAnalysisText(smartAutoFriendlyMessage)) {
|
||||
console.warn("[Smart Auto Mode] ⚠️ Image summary unavailable, keep original analysis text");
|
||||
}
|
||||
|
||||
@ -557,7 +557,7 @@ async function summarizeWithZhipuOfficialApi(params: {
|
||||
{
|
||||
role: "system",
|
||||
content:
|
||||
"你是图像观察助手。请根据图片给出执行前说明,输出严格两行中文:第1行“画面:...”,第2行“关键要素:...”。不要输出其他内容。",
|
||||
"你是图像观察助手。只描述图片可见内容,不要提执行动作或用户意图。输出严格一行中文。",
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
@ -581,9 +581,10 @@ async function summarizeWithZhipuOfficialApi(params: {
|
||||
|
||||
const message = payload?.choices?.[0]?.message;
|
||||
const responseText = String(message?.content || message?.reasoning_content || "").trim();
|
||||
if (responseText) {
|
||||
const normalized = normalizeVisualSummaryText(responseText);
|
||||
if (normalized && !isWeakVisualSummary(normalized)) {
|
||||
console.log(`[Smart Analyzer] Zhipu official summary model used: ${modelId}`);
|
||||
return responseText;
|
||||
return normalized;
|
||||
}
|
||||
} catch {
|
||||
continue;
|
||||
@ -620,12 +621,10 @@ export async function summarizeImageForExecutionReport(params: {
|
||||
const zhipuProvider = zhipuProviderFromAgent || zhipuProviderFallback;
|
||||
|
||||
const promptText = [
|
||||
text ? `用户需求:${text}` : "",
|
||||
context ? `画布上下文:${context.slice(0, 220)}` : "",
|
||||
"请给出当前画面的客观描述,便于执行前说明。",
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
"只根据图片中可见内容做客观观察,不要推测用户意图。",
|
||||
"禁止出现“选中/待处理/按需求/处理/修改/转换”等执行词。",
|
||||
"输出一行:画面:主体与关键元素;场景:环境;风格:视觉风格;光线:明暗与色温。",
|
||||
].join("\n");
|
||||
|
||||
if (zhipuProvider?.apiKey && zhipuProvider?.baseUrl) {
|
||||
const summary = await summarizeWithZhipuOfficialApi({
|
||||
@ -647,33 +646,40 @@ export async function summarizeImageForExecutionReport(params: {
|
||||
const model = await getLanguageModelById(selectedVisionModel.model.id);
|
||||
if (!model) return null;
|
||||
|
||||
const fallbackVisionImages = await buildAnalyzerVisionImagesByMode({
|
||||
imageUrls,
|
||||
preferRemoteUrls: shouldPreferRemoteVisionUrls(selectedVisionModel),
|
||||
});
|
||||
if (fallbackVisionImages.length === 0) return null;
|
||||
const preferRemote = shouldPreferRemoteVisionUrls(selectedVisionModel);
|
||||
const modeAttempts = Array.from(new Set([preferRemote, !preferRemote]));
|
||||
|
||||
const { text: summaryText } = await generateText({
|
||||
model,
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: `${promptText}\n只输出一行,格式:画面要点:主体+场景+风格+光线(不超过45字)。`,
|
||||
},
|
||||
...fallbackVisionImages,
|
||||
],
|
||||
},
|
||||
],
|
||||
temperature: 0.1,
|
||||
maxTokens: 120,
|
||||
abortSignal: AbortSignal.timeout(ANALYZER_TIMEOUT_MS),
|
||||
} as any);
|
||||
for (const preferRemoteUrls of modeAttempts) {
|
||||
const fallbackVisionImages = await buildAnalyzerVisionImagesByMode({
|
||||
imageUrls,
|
||||
preferRemoteUrls,
|
||||
});
|
||||
if (fallbackVisionImages.length === 0) continue;
|
||||
|
||||
const normalized = summaryText?.replace(/\s+/g, " ").trim();
|
||||
return normalized || null;
|
||||
const { text: summaryText } = await generateText({
|
||||
model,
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: `${promptText}\n仅输出一行,不超过60字。`,
|
||||
},
|
||||
...fallbackVisionImages,
|
||||
],
|
||||
},
|
||||
],
|
||||
temperature: 0.1,
|
||||
maxTokens: 120,
|
||||
abortSignal: AbortSignal.timeout(ANALYZER_TIMEOUT_MS),
|
||||
} as any);
|
||||
|
||||
const normalized = normalizeVisualSummaryText(summaryText);
|
||||
if (normalized && !isWeakVisualSummary(normalized)) {
|
||||
return normalized;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn("[Smart Analyzer] summarizeImageForExecutionReport fallback failed:", error);
|
||||
}
|
||||
@ -681,6 +687,36 @@ export async function summarizeImageForExecutionReport(params: {
|
||||
return null;
|
||||
}
|
||||
|
||||
function normalizeVisualSummaryText(text?: string | null): string {
|
||||
if (!text) return "";
|
||||
return text
|
||||
.replace(/\s+/g, " ")
|
||||
.replace(/^[-*•]\s*/, "")
|
||||
.replace(/^(画面要点|画面描述|图片分析|分析结果)[::]\s*/i, "")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function isWeakVisualSummary(summary: string): boolean {
|
||||
const normalized = summary.replace(/\s+/g, " ").trim();
|
||||
if (!normalized || normalized.length < 14) return true;
|
||||
|
||||
const weakPatterns = [
|
||||
/待处理图/,
|
||||
/选中.*位置/,
|
||||
/当前上传图片/,
|
||||
/基于.*原图处理/,
|
||||
/按.*需求执行/,
|
||||
/仅做.*调整/,
|
||||
/不改动原有内容结构/,
|
||||
/转换为.*场景/,
|
||||
];
|
||||
if (weakPatterns.some((pattern) => pattern.test(normalized))) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
// 历史消息类型
|
||||
interface HistoryMessage {
|
||||
role: "user" | "assistant";
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user