fix: force qwen3-vl fallback for multimodal-only agent vision
This commit is contained in:
parent
86e914709f
commit
acd37e8c57
@ -880,6 +880,7 @@ export async function summarizeImageForExecutionReport(params: {
|
|||||||
},
|
},
|
||||||
)?.provider;
|
)?.provider;
|
||||||
const volcanoProvider = volcanoProviderFromAgent || volcanoProviderFallback;
|
const volcanoProvider = volcanoProviderFromAgent || volcanoProviderFallback;
|
||||||
|
const forceQwenSummary = shouldForceQwen3VlForImageAnalysis(agentModel);
|
||||||
|
|
||||||
const promptText = [
|
const promptText = [
|
||||||
"只根据图片中可见内容做客观观察,不要推测用户意图。",
|
"只根据图片中可见内容做客观观察,不要推测用户意图。",
|
||||||
@ -889,6 +890,18 @@ export async function summarizeImageForExecutionReport(params: {
|
|||||||
context ? `画布上下文(仅辅助理解,不要复述):${context.slice(0, 120)}` : "",
|
context ? `画布上下文(仅辅助理解,不要复述):${context.slice(0, 120)}` : "",
|
||||||
].join("\n");
|
].join("\n");
|
||||||
|
|
||||||
|
if (forceQwenSummary) {
|
||||||
|
console.warn(
|
||||||
|
"[Smart Analyzer] Force qwen3-vl visual summary due multimodal-flag-only agent model",
|
||||||
|
);
|
||||||
|
const forcedSummary = await summarizeWithHardcodedQwen3VlFallback({
|
||||||
|
text,
|
||||||
|
context,
|
||||||
|
imageUrls,
|
||||||
|
});
|
||||||
|
if (forcedSummary) return forcedSummary;
|
||||||
|
}
|
||||||
|
|
||||||
if (zhipuProvider?.apiKey && zhipuProvider?.baseUrl) {
|
if (zhipuProvider?.apiKey && zhipuProvider?.baseUrl) {
|
||||||
const summary = await summarizeWithZhipuOfficialApi({
|
const summary = await summarizeWithZhipuOfficialApi({
|
||||||
apiKey: zhipuProvider.apiKey,
|
apiKey: zhipuProvider.apiKey,
|
||||||
@ -1110,6 +1123,30 @@ function isLikelyVisionCapableModel(modelWithProvider: {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function shouldForceQwen3VlForImageAnalysis(modelWithProvider?: {
|
||||||
|
model: {
|
||||||
|
supportsVision?: boolean;
|
||||||
|
displayName?: string | null;
|
||||||
|
modelId?: string | null;
|
||||||
|
capabilities?: unknown;
|
||||||
|
};
|
||||||
|
provider: {
|
||||||
|
type?: string | null;
|
||||||
|
};
|
||||||
|
} | null): boolean {
|
||||||
|
if (!modelWithProvider) return false;
|
||||||
|
if (modelWithProvider.model.supportsVision) return false;
|
||||||
|
if (!isAiModelMultimodal(modelWithProvider.model)) return false;
|
||||||
|
|
||||||
|
const providerType = String(modelWithProvider.provider?.type || "");
|
||||||
|
if (!["custom", "openai", "local"].includes(providerType)) return false;
|
||||||
|
|
||||||
|
const modelText =
|
||||||
|
`${modelWithProvider.model.displayName || ""} ${modelWithProvider.model.modelId || ""}`.toLowerCase();
|
||||||
|
|
||||||
|
return /(doubao[-_.]?seed[-_.]?2[-_.]?0|seed[-_.]?2[-_.]?0)/i.test(modelText);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 分析用户意图
|
* 分析用户意图
|
||||||
*/
|
*/
|
||||||
@ -1246,6 +1283,20 @@ async function analyzWithLLM(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (hasImage && shouldForceQwen3VlForImageAnalysis(selectedAnalyzerModel)) {
|
||||||
|
console.warn(
|
||||||
|
"[Smart Analyzer] Force hardcoded qwen3-vl for image analysis due multimodal-flag-only primary model",
|
||||||
|
);
|
||||||
|
return analyzeWithHardcodedQwen3VlFallback({
|
||||||
|
text,
|
||||||
|
hasImage,
|
||||||
|
imageUrls,
|
||||||
|
historyMessages,
|
||||||
|
context,
|
||||||
|
reason: "force_qwen3vl_multimodal_flag_only",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
const model = await getLanguageModelById(selectedAnalyzerModel.model.id);
|
const model = await getLanguageModelById(selectedAnalyzerModel.model.id);
|
||||||
if (!model) {
|
if (!model) {
|
||||||
console.log("[Smart Analyzer] Failed to get language model");
|
console.log("[Smart Analyzer] Failed to get language model");
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user