fix: require explicit multimodal flags
This commit is contained in:
parent
7cd29fe2b5
commit
d6a06e0cd0
@ -175,6 +175,7 @@ import { convertToUIMessages, generateUUID } from "@/lib/utils";
|
|||||||
import { isAiModelAgentAuto } from "@/lib/db/ai-model-flags";
|
import { isAiModelAgentAuto } from "@/lib/db/ai-model-flags";
|
||||||
import { generateTitleFromUserMessage } from "../../actions";
|
import { generateTitleFromUserMessage } from "../../actions";
|
||||||
import { type PostRequestBody, postRequestBodySchema } from "./schema";
|
import { type PostRequestBody, postRequestBodySchema } from "./schema";
|
||||||
|
import { isAiModelMultimodal } from "@/lib/db/ai-model-flags";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 将图片 URL 转换为 base64 格式
|
* 将图片 URL 转换为 base64 格式
|
||||||
@ -3206,7 +3207,7 @@ export async function POST(request: Request) {
|
|||||||
: false));
|
: false));
|
||||||
const isConversationMultimodalModel =
|
const isConversationMultimodalModel =
|
||||||
!selectedModelConfig.isSubModel &&
|
!selectedModelConfig.isSubModel &&
|
||||||
selectedModelConfig.supportsVision &&
|
isAiModelMultimodal(selectedModelConfig) &&
|
||||||
!selectedModelConfig.supportsImageGen &&
|
!selectedModelConfig.supportsImageGen &&
|
||||||
!isVideoProviderType(selectedModelProviderType);
|
!isVideoProviderType(selectedModelProviderType);
|
||||||
requestedModelIsFixedVisual = isFixedVisualModel;
|
requestedModelIsFixedVisual = isFixedVisualModel;
|
||||||
|
|||||||
@ -964,38 +964,7 @@ function isAgentGenerateTextProviderType(providerType: string): boolean {
|
|||||||
|
|
||||||
function isLikelyVisionCapableModel(item: EnabledModelWithProvider) {
|
function isLikelyVisionCapableModel(item: EnabledModelWithProvider) {
|
||||||
if (!item.model.isEnabled || item.model.isSubModel) return false;
|
if (!item.model.isEnabled || item.model.isSubModel) return false;
|
||||||
if (item.model.supportsVision) return true;
|
return Boolean(item.model.supportsVision) || isAiModelMultimodal(item.model);
|
||||||
if (isAiModelMultimodal(item.model)) return true;
|
|
||||||
|
|
||||||
const providerType = String(item.provider.type || "");
|
|
||||||
const modelText = `${item.model.displayName || ""} ${item.model.modelId || ""}`.toLowerCase();
|
|
||||||
|
|
||||||
if (providerType === "volcano-multimodal") return true;
|
|
||||||
|
|
||||||
if (
|
|
||||||
providerType === "zhipu" &&
|
|
||||||
/(glm[-_.]?4v|glm[-_.]?4\.?6v|vision|多模态)/i.test(modelText)
|
|
||||||
) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (
|
|
||||||
providerType === "alibaba" &&
|
|
||||||
/(qwen|vl|vision|通义|多模态)/i.test(modelText)
|
|
||||||
) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (
|
|
||||||
["custom", "openai", "local"].includes(providerType) &&
|
|
||||||
/(doubao[-_.]?seed[-_.]?2[-_.]?0|seed[-_.]?2[-_.]?0|qwen[-_.]?vl|glm[-_.]?4v|gpt[-_.]?4o|vision|vl|多模态)/i.test(
|
|
||||||
modelText,
|
|
||||||
)
|
|
||||||
) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function getAgentModelIncompatibleReason(params: {
|
function getAgentModelIncompatibleReason(params: {
|
||||||
|
|||||||
@ -78,7 +78,7 @@ function isConversationalCandidate(
|
|||||||
const hasVision = hasVisionCapability(item.model);
|
const hasVision = hasVisionCapability(item.model);
|
||||||
|
|
||||||
if (matchesPreferredProvider) {
|
if (matchesPreferredProvider) {
|
||||||
return requireVision ? hasVision || providerType === "volcano-multimodal" : true;
|
return requireVision ? hasVision : true;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (IMAGE_ONLY_PROVIDER_TYPES.has(providerType)) {
|
if (IMAGE_ONLY_PROVIDER_TYPES.has(providerType)) {
|
||||||
|
|||||||
@ -1686,50 +1686,8 @@ function isLikelyVisionCapableModel(modelWithProvider: {
|
|||||||
};
|
};
|
||||||
}) {
|
}) {
|
||||||
const model = modelWithProvider.model;
|
const model = modelWithProvider.model;
|
||||||
const providerType = String(modelWithProvider.provider?.type || "");
|
|
||||||
if (!model?.isEnabled || model.isSubModel) return false;
|
if (!model?.isEnabled || model.isSubModel) return false;
|
||||||
if (isAiModelMultimodal(model)) return true;
|
return Boolean(model.supportsVision) || isAiModelMultimodal(model);
|
||||||
const modelText = `${model.displayName || ""} ${model.modelId || ""}`.toLowerCase();
|
|
||||||
const likelyImageGenerationOnly =
|
|
||||||
!model.supportsVision &&
|
|
||||||
!isAiModelMultimodal(model) &&
|
|
||||||
Boolean(model.supportsImageGen) &&
|
|
||||||
((["custom", "openai", "local"].includes(providerType) &&
|
|
||||||
/(doubao[-_.]?seed[-_.]?2[-_.]?0|seed[-_.]?2[-_.]?0|seedream|gpt[-_.]?image|nano[-_.]?banana|flux|imagen|image[-_.]?edit)/i.test(
|
|
||||||
modelText,
|
|
||||||
)) ||
|
|
||||||
["gemini", "huanke-gemini", "alibaba-image", "ruxa-image", "fal-image"].includes(
|
|
||||||
providerType,
|
|
||||||
));
|
|
||||||
if (likelyImageGenerationOnly) return false;
|
|
||||||
if (model.supportsVision) return true;
|
|
||||||
|
|
||||||
if (providerType === "volcano-vision" || providerType === "volcano-multimodal") return true;
|
|
||||||
|
|
||||||
if (
|
|
||||||
providerType === "zhipu" &&
|
|
||||||
/(glm[-_.]?4v|glm[-_.]?4\.?6v|vision|多模态)/i.test(modelText)
|
|
||||||
) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (
|
|
||||||
providerType === "alibaba" &&
|
|
||||||
/(qwen|vl|vision|通义|多模态)/i.test(modelText)
|
|
||||||
) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (
|
|
||||||
["custom", "openai", "local"].includes(providerType) &&
|
|
||||||
/(doubao[-_.]?seed[-_.]?2[-_.]?0|seed[-_.]?2[-_.]?0|qwen[-_.]?vl|glm[-_.]?4v|gpt[-_.]?4o|vision|vl|多模态)/i.test(
|
|
||||||
modelText,
|
|
||||||
)
|
|
||||||
) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function getPreferredVolcanoMultimodalAnalyzerCandidate(
|
function getPreferredVolcanoMultimodalAnalyzerCandidate(
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user