fix: keep multimodal agent vision analysis on volcano
This commit is contained in:
parent
8329268dea
commit
3badf859b9
@ -14,6 +14,7 @@ type DynamicModel = {
|
|||||||
providerType?: string;
|
providerType?: string;
|
||||||
supportsVision: boolean;
|
supportsVision: boolean;
|
||||||
supportsImageGen?: boolean;
|
supportsImageGen?: boolean;
|
||||||
|
isMultimodal?: boolean;
|
||||||
isDefault: boolean;
|
isDefault: boolean;
|
||||||
providerName: string;
|
providerName: string;
|
||||||
isSubModel?: boolean; // 🆕 是否为子模型
|
isSubModel?: boolean; // 🆕 是否为子模型
|
||||||
@ -1594,7 +1595,7 @@ function PureModelSelectorCompact({
|
|||||||
|
|
||||||
// 优先选择支持视觉的模型(如果有附件)
|
// 优先选择支持视觉的模型(如果有附件)
|
||||||
if (hasAttachments) {
|
if (hasAttachments) {
|
||||||
const visionModel = dynamicModels.find((m) => m.supportsVision);
|
const visionModel = dynamicModels.find((m) => m.supportsVision || m.isMultimodal);
|
||||||
if (visionModel) return visionModel;
|
if (visionModel) return visionModel;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -5,6 +5,8 @@
|
|||||||
* 核心逻辑:图片编辑场景优先使用 alibaba-image (灵魂4.5)
|
* 核心逻辑:图片编辑场景优先使用 alibaba-image (灵魂4.5)
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
import { isAiModelMultimodal } from "@/lib/db/ai-model-flags";
|
||||||
|
|
||||||
// 使用动态导入避免循环依赖
|
// 使用动态导入避免循环依赖
|
||||||
// import { getEnabledAiModelsWithProvider } from "@/lib/db/queries";
|
// import { getEnabledAiModelsWithProvider } from "@/lib/db/queries";
|
||||||
|
|
||||||
@ -17,6 +19,14 @@ export interface RouteResult {
|
|||||||
reason: string;
|
reason: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function hasVisionCapability(model: {
|
||||||
|
supportsVision?: boolean | null;
|
||||||
|
capabilities?: unknown;
|
||||||
|
defaultParams?: unknown;
|
||||||
|
}) {
|
||||||
|
return Boolean(model.supportsVision) || isAiModelMultimodal(model);
|
||||||
|
}
|
||||||
|
|
||||||
// 意图到模型的映射配置
|
// 意图到模型的映射配置
|
||||||
// providerTypes 按优先级排序,第一个是首选
|
// providerTypes 按优先级排序,第一个是首选
|
||||||
const INTENT_MODEL_MAP: Record<
|
const INTENT_MODEL_MAP: Record<
|
||||||
@ -144,7 +154,7 @@ export async function routeToModel(params: {
|
|||||||
|
|
||||||
if (configuredModel) {
|
if (configuredModel) {
|
||||||
const isImageIntent = intent === "image_edit" || intent === "image_gen";
|
const isImageIntent = intent === "image_edit" || intent === "image_gen";
|
||||||
const matchesVision = !config.requireVision || configuredModel.model.supportsVision;
|
const matchesVision = !config.requireVision || hasVisionCapability(configuredModel.model);
|
||||||
const matchesImageGen = !config.requireImageGen || configuredModel.model.supportsImageGen;
|
const matchesImageGen = !config.requireImageGen || configuredModel.model.supportsImageGen;
|
||||||
const allowImageIntentWithImageGenOnly =
|
const allowImageIntentWithImageGenOnly =
|
||||||
isImageIntent && configuredModel.model.supportsImageGen;
|
isImageIntent && configuredModel.model.supportsImageGen;
|
||||||
@ -181,7 +191,7 @@ export async function routeToModel(params: {
|
|||||||
if (m.provider.type !== providerType) return false;
|
if (m.provider.type !== providerType) return false;
|
||||||
|
|
||||||
// 检查能力要求
|
// 检查能力要求
|
||||||
if (config.requireVision && !m.model.supportsVision) return false;
|
if (config.requireVision && !hasVisionCapability(m.model)) return false;
|
||||||
if (config.requireImageGen && !m.model.supportsImageGen) return false;
|
if (config.requireImageGen && !m.model.supportsImageGen) return false;
|
||||||
|
|
||||||
// 排除子模型(只选主模型)
|
// 排除子模型(只选主模型)
|
||||||
|
|||||||
@ -681,6 +681,7 @@ async function summarizeWithVolcanoOfficialApi(params: {
|
|||||||
baseUrl: string;
|
baseUrl: string;
|
||||||
promptText: string;
|
promptText: string;
|
||||||
imageUrls: string[];
|
imageUrls: string[];
|
||||||
|
modelId?: string;
|
||||||
}): Promise<string | null> {
|
}): Promise<string | null> {
|
||||||
const endpoint = normalizeVolcanoResponsesEndpoint(params.baseUrl);
|
const endpoint = normalizeVolcanoResponsesEndpoint(params.baseUrl);
|
||||||
const rewrittenUrls = await Promise.all(
|
const rewrittenUrls = await Promise.all(
|
||||||
@ -695,7 +696,7 @@ async function summarizeWithVolcanoOfficialApi(params: {
|
|||||||
Authorization: `Bearer ${params.apiKey}`,
|
Authorization: `Bearer ${params.apiKey}`,
|
||||||
},
|
},
|
||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
model: "doubao-seed-2-0-pro-260215",
|
model: String(params.modelId || "").trim() || "doubao-seed-2-0-pro-260215",
|
||||||
input: [
|
input: [
|
||||||
{
|
{
|
||||||
role: "user",
|
role: "user",
|
||||||
@ -737,6 +738,91 @@ async function summarizeWithVolcanoOfficialApi(params: {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function analyzeWithVolcanoOfficialApi(params: {
|
||||||
|
apiKey: string;
|
||||||
|
baseUrl: string;
|
||||||
|
userMessage: string;
|
||||||
|
imageUrls: string[];
|
||||||
|
userText: string;
|
||||||
|
hasImage: boolean;
|
||||||
|
modelId?: string;
|
||||||
|
}): Promise<AnalysisResult | null> {
|
||||||
|
const endpoint = normalizeVolcanoResponsesEndpoint(params.baseUrl);
|
||||||
|
const rewrittenUrls = (
|
||||||
|
await Promise.all(
|
||||||
|
params.imageUrls.map((url) => rewriteUploadPublicUrlForVolcanoSummary(url)),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.filter((url) => isHttpUrl(url))
|
||||||
|
.slice(0, ANALYZER_MAX_IMAGE_COUNT);
|
||||||
|
|
||||||
|
if (rewrittenUrls.length === 0) return null;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch(endpoint, {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
Authorization: `Bearer ${params.apiKey}`,
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
model: String(params.modelId || "").trim() || "doubao-seed-2-0-pro-260215",
|
||||||
|
input: [
|
||||||
|
{
|
||||||
|
role: "system",
|
||||||
|
content: [
|
||||||
|
{
|
||||||
|
type: "input_text",
|
||||||
|
text: ANALYZER_SYSTEM_PROMPT,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
role: "user",
|
||||||
|
content: [
|
||||||
|
{
|
||||||
|
type: "input_text",
|
||||||
|
text: params.userMessage,
|
||||||
|
},
|
||||||
|
...rewrittenUrls.map((url) => ({ type: "input_image", image_url: url })),
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
const payload = await response.json().catch(() => ({}));
|
||||||
|
if (!response.ok || payload?.error) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const directText = String(payload?.output_text || "").trim();
|
||||||
|
const output = Array.isArray(payload?.output) ? payload.output : [];
|
||||||
|
const chunks: string[] = [];
|
||||||
|
for (const item of output) {
|
||||||
|
const content = Array.isArray(item?.content) ? item.content : [];
|
||||||
|
for (const part of content) {
|
||||||
|
if (typeof part?.text === "string" && part.text.trim()) {
|
||||||
|
chunks.push(part.text.trim());
|
||||||
|
} else if (typeof part?.output_text === "string" && part.output_text.trim()) {
|
||||||
|
chunks.push(part.output_text.trim());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const responseText = directText || chunks.join("\n").trim();
|
||||||
|
if (!responseText) return null;
|
||||||
|
|
||||||
|
return parseAnalysisResultFromModelText(
|
||||||
|
responseText,
|
||||||
|
params.userText,
|
||||||
|
params.hasImage,
|
||||||
|
);
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function normalizeZhipuChatEndpoint(baseUrl: string): string {
|
function normalizeZhipuChatEndpoint(baseUrl: string): string {
|
||||||
const cleaned = baseUrl.replace(/\/+$/, "");
|
const cleaned = baseUrl.replace(/\/+$/, "");
|
||||||
if (/\/chat\/completions$/i.test(cleaned)) return cleaned;
|
if (/\/chat\/completions$/i.test(cleaned)) return cleaned;
|
||||||
@ -1137,6 +1223,7 @@ export async function summarizeImageForExecutionReport(params: {
|
|||||||
baseUrl: volcanoProvider.baseUrl,
|
baseUrl: volcanoProvider.baseUrl,
|
||||||
promptText,
|
promptText,
|
||||||
imageUrls: (imageUrls || []).filter((url) => isHttpUrl(url)).slice(0, ANALYZER_MAX_IMAGE_COUNT),
|
imageUrls: (imageUrls || []).filter((url) => isHttpUrl(url)).slice(0, ANALYZER_MAX_IMAGE_COUNT),
|
||||||
|
modelId: agentModel?.model?.modelId,
|
||||||
});
|
});
|
||||||
if (summary) return await finalizeVisualSummary(summary);
|
if (summary) return await finalizeVisualSummary(summary);
|
||||||
}
|
}
|
||||||
@ -1339,9 +1426,11 @@ function isLikelyVisionCapableModel(modelWithProvider: {
|
|||||||
const model = modelWithProvider.model;
|
const model = modelWithProvider.model;
|
||||||
const providerType = String(modelWithProvider.provider?.type || "");
|
const providerType = String(modelWithProvider.provider?.type || "");
|
||||||
if (!model?.isEnabled || model.isSubModel) return false;
|
if (!model?.isEnabled || model.isSubModel) return false;
|
||||||
|
if (isAiModelMultimodal(model)) return true;
|
||||||
const modelText = `${model.displayName || ""} ${model.modelId || ""}`.toLowerCase();
|
const modelText = `${model.displayName || ""} ${model.modelId || ""}`.toLowerCase();
|
||||||
const likelyImageGenerationOnly =
|
const likelyImageGenerationOnly =
|
||||||
!model.supportsVision &&
|
!model.supportsVision &&
|
||||||
|
!isAiModelMultimodal(model) &&
|
||||||
Boolean(model.supportsImageGen) &&
|
Boolean(model.supportsImageGen) &&
|
||||||
((["custom", "openai", "local"].includes(providerType) &&
|
((["custom", "openai", "local"].includes(providerType) &&
|
||||||
/(doubao[-_.]?seed[-_.]?2[-_.]?0|seed[-_.]?2[-_.]?0|seedream|gpt[-_.]?image|nano[-_.]?banana|flux|imagen|image[-_.]?edit)/i.test(
|
/(doubao[-_.]?seed[-_.]?2[-_.]?0|seed[-_.]?2[-_.]?0|seedream|gpt[-_.]?image|nano[-_.]?banana|flux|imagen|image[-_.]?edit)/i.test(
|
||||||
@ -1352,7 +1441,6 @@ function isLikelyVisionCapableModel(modelWithProvider: {
|
|||||||
));
|
));
|
||||||
if (likelyImageGenerationOnly) return false;
|
if (likelyImageGenerationOnly) return false;
|
||||||
if (model.supportsVision) return true;
|
if (model.supportsVision) return true;
|
||||||
if (isAiModelMultimodal(model)) return true;
|
|
||||||
|
|
||||||
if (providerType === "volcano-vision" || providerType === "volcano-multimodal") return true;
|
if (providerType === "volcano-vision" || providerType === "volcano-multimodal") return true;
|
||||||
|
|
||||||
@ -1382,6 +1470,18 @@ function isLikelyVisionCapableModel(modelWithProvider: {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getPreferredVolcanoMultimodalAnalyzerCandidate(
|
||||||
|
...candidates: Array<VisionSummaryModel | null | undefined>
|
||||||
|
): VisionSummaryModel | null {
|
||||||
|
for (const candidate of candidates) {
|
||||||
|
if (!candidate?.provider?.apiKey || !candidate?.provider?.baseUrl) continue;
|
||||||
|
if (!isLikelyVolcanoEndpoint(candidate.provider.baseUrl)) continue;
|
||||||
|
if (!isAiModelMultimodal(candidate.model)) continue;
|
||||||
|
return candidate;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
function shouldForceQwen3VlForImageAnalysis(modelWithProvider?: {
|
function shouldForceQwen3VlForImageAnalysis(modelWithProvider?: {
|
||||||
model: {
|
model: {
|
||||||
supportsVision?: boolean;
|
supportsVision?: boolean;
|
||||||
@ -1694,6 +1794,36 @@ ${historyText}
|
|||||||
? `${contextMessage}用户上传了图片,请求内容:"${text}"`
|
? `${contextMessage}用户上传了图片,请求内容:"${text}"`
|
||||||
: `${contextMessage}用户请求(无图片):"${text}"`;
|
: `${contextMessage}用户请求(无图片):"${text}"`;
|
||||||
|
|
||||||
|
const volcanoAnalyzerCandidate = getPreferredVolcanoMultimodalAnalyzerCandidate(
|
||||||
|
selectedAnalyzerModel,
|
||||||
|
preferredModel,
|
||||||
|
agentModel,
|
||||||
|
);
|
||||||
|
if (
|
||||||
|
hasImage &&
|
||||||
|
visionImages.length > 0 &&
|
||||||
|
volcanoAnalyzerCandidate?.provider?.apiKey &&
|
||||||
|
volcanoAnalyzerCandidate?.provider?.baseUrl
|
||||||
|
) {
|
||||||
|
const volcanoOfficialResult = await analyzeWithVolcanoOfficialApi({
|
||||||
|
apiKey: volcanoAnalyzerCandidate.provider.apiKey,
|
||||||
|
baseUrl: volcanoAnalyzerCandidate.provider.baseUrl,
|
||||||
|
modelId: volcanoAnalyzerCandidate.model.modelId,
|
||||||
|
userMessage,
|
||||||
|
userText: text,
|
||||||
|
hasImage,
|
||||||
|
imageUrls: (imageUrls || [])
|
||||||
|
.filter((url) => isHttpUrl(url))
|
||||||
|
.slice(0, ANALYZER_MAX_IMAGE_COUNT),
|
||||||
|
});
|
||||||
|
if (volcanoOfficialResult) {
|
||||||
|
console.log(
|
||||||
|
`[Smart Analyzer] Volcano official analyzer used: ${volcanoAnalyzerCandidate.model.displayName} (${volcanoAnalyzerCandidate.model.modelId})`,
|
||||||
|
);
|
||||||
|
return volcanoOfficialResult;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const zhipuProviderFromAgent =
|
const zhipuProviderFromAgent =
|
||||||
usableAgentModel &&
|
usableAgentModel &&
|
||||||
usableAgentModel.provider &&
|
usableAgentModel.provider &&
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user