fix(agent): fallback image analysis to hardcoded qwen3-vl
This commit is contained in:
Your Name
parent
1e7c7bb2e9
commit
75e139c167
@ -340,7 +340,12 @@ function shouldPreferRemoteVisionUrls(modelWithProvider?: {
|
||||
const baseUrl = String(modelWithProvider.provider?.baseUrl || "").toLowerCase();
|
||||
const modelText = `${modelWithProvider.model?.displayName || ""} ${modelWithProvider.model?.modelId || ""}`.toLowerCase();
|
||||
|
||||
if (providerType === "volcano-multimodal" || providerType === "doubao-video" || providerType === "volcano-video") {
|
||||
if (
|
||||
providerType === "volcano-vision" ||
|
||||
providerType === "volcano-multimodal" ||
|
||||
providerType === "doubao-video" ||
|
||||
providerType === "volcano-video"
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
if (baseUrl.includes("ark.cn-beijing.volces.com")) {
|
||||
@ -366,6 +371,121 @@ async function buildAnalyzerVisionImagesByMode(params: {
|
||||
return buildAnalyzerVisionImages(urls);
|
||||
}
|
||||
|
||||
async function getQwen3VlFallbackImageInput(imageUrls?: string[]): Promise<string | null> {
|
||||
const firstImageUrl = (imageUrls || []).find((url) => isHttpUrl(url));
|
||||
if (!firstImageUrl) return null;
|
||||
const inlineDataUrl = await toAnalyzerDataUrl(firstImageUrl);
|
||||
return inlineDataUrl || firstImageUrl;
|
||||
}
|
||||
|
||||
async function analyzeWithHardcodedQwen3VlFallback(params: {
|
||||
text: string;
|
||||
hasImage: boolean;
|
||||
imageUrls?: string[];
|
||||
historyMessages?: HistoryMessage[];
|
||||
context?: string;
|
||||
reason: string;
|
||||
}): Promise<AnalysisResult | null> {
|
||||
if (!params.hasImage) return null;
|
||||
|
||||
const imageInput = await getQwen3VlFallbackImageInput(params.imageUrls);
|
||||
if (!imageInput) {
|
||||
console.warn(
|
||||
`[Smart Analyzer] qwen3-vl fallback skipped: no valid image url (${params.reason})`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
const historySnippet = (params.historyMessages || [])
|
||||
.slice(-6)
|
||||
.map((msg) => `${msg.role === "user" ? "用户" : "AI"}: ${msg.content.slice(0, 100)}`)
|
||||
.join("\n");
|
||||
|
||||
const prompt = [
|
||||
"你是图片需求分析助手。请结合图片与用户意图，输出严格 JSON，不要输出其他内容。",
|
||||
"intent 只能是 image_edit / vision / image_gen / video_gen / chat。",
|
||||
"subIntent 可选：remove_background / replace_background / enhance / extend / inpaint / style_transfer / color_adjust / redo / resize。",
|
||||
"如果用户提到视频、首帧、尾帧、分镜，intent=video_gen。",
|
||||
"有图片且用户在修改图片时，优先 intent=image_edit。",
|
||||
"JSON 字段：intent, subIntent, confidence, conversationalReply, needsClarification, clarifyingQuestions。",
|
||||
"confidence 为 0-1 数字；needsClarification 默认 false；clarifyingQuestions 默认空数组。",
|
||||
params.context ? `画布上下文（仅理解，不要复述）: ${params.context.slice(0, 180)}` : "",
|
||||
historySnippet ? `近期对话（仅理解，不要复述）:\n${historySnippet}` : "",
|
||||
`当前用户请求: ${params.text}`,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
|
||||
try {
|
||||
const { callQwen3Vl } = await import("./qwen3-vl");
|
||||
console.warn(
|
||||
`[Smart Analyzer] Primary analysis failed, fallback to hardcoded qwen3-vl (${params.reason})`,
|
||||
);
|
||||
const { text: fallbackText, model } = await callQwen3Vl({
|
||||
imageUrl: imageInput,
|
||||
prompt,
|
||||
temperature: 0.1,
|
||||
maxTokens: 360,
|
||||
timeoutMs: ANALYZER_TIMEOUT_MS,
|
||||
retries: 1,
|
||||
});
|
||||
const parsed = parseAnalysisResultFromModelText(
|
||||
fallbackText,
|
||||
params.text,
|
||||
params.hasImage,
|
||||
);
|
||||
if (parsed) {
|
||||
console.log(`[Smart Analyzer] qwen3-vl fallback success: ${model}`);
|
||||
return parsed;
|
||||
}
|
||||
console.warn("[Smart Analyzer] qwen3-vl fallback returned unparsable content");
|
||||
return null;
|
||||
} catch (error) {
|
||||
console.warn("[Smart Analyzer] qwen3-vl fallback failed:", error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function summarizeWithHardcodedQwen3VlFallback(params: {
|
||||
text?: string;
|
||||
context?: string;
|
||||
imageUrls?: string[];
|
||||
}): Promise<string | null> {
|
||||
const imageInput = await getQwen3VlFallbackImageInput(params.imageUrls);
|
||||
if (!imageInput) return null;
|
||||
|
||||
const prompt = [
|
||||
"你是图像观察助手。只描述图片可见内容，不要写执行过程，不要推测用户意图。",
|
||||
"输出严格一行中文：画面：主体与关键元素；场景：环境；风格：视觉风格；光线：明暗与色温。",
|
||||
params.text ? `用户目标（仅理解，不要复述）: ${params.text.slice(0, 120)}` : "",
|
||||
params.context ? `画布上下文（仅理解，不要复述）: ${params.context.slice(0, 120)}` : "",
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
|
||||
try {
|
||||
const { callQwen3Vl } = await import("./qwen3-vl");
|
||||
console.warn("[Smart Analyzer] Visual summary fallback to hardcoded qwen3-vl");
|
||||
const { text, model } = await callQwen3Vl({
|
||||
imageUrl: imageInput,
|
||||
prompt,
|
||||
temperature: 0.1,
|
||||
maxTokens: 140,
|
||||
timeoutMs: ANALYZER_TIMEOUT_MS,
|
||||
retries: 1,
|
||||
});
|
||||
const normalized = normalizeVisualSummaryText(text);
|
||||
if (normalized && !isWeakVisualSummary(normalized)) {
|
||||
console.log(`[Smart Analyzer] qwen3-vl summary fallback success: ${model}`);
|
||||
return normalized;
|
||||
}
|
||||
return null;
|
||||
} catch (error) {
|
||||
console.warn("[Smart Analyzer] qwen3-vl summary fallback failed:", error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function parseAnalysisResultFromModelText(
|
||||
responseText: string,
|
||||
userText: string,
|
||||
@ -728,9 +848,17 @@ export async function summarizeImageForExecutionReport(params: {
|
||||
? agentModel.provider
|
||||
: null;
|
||||
const volcanoProviderFallback = allModels.find(
|
||||
(m) =>
|
||||
m.provider?.apiKey &&
|
||||
(m.provider.type === "volcano-multimodal" || isLikelyVolcanoEndpoint(m.provider.baseUrl)),
|
||||
(m) => {
|
||||
const providerType = String(m.provider?.type || "");
|
||||
return (
|
||||
m.provider?.apiKey &&
|
||||
(
|
||||
providerType === "volcano-vision" ||
|
||||
providerType === "volcano-multimodal" ||
|
||||
isLikelyVolcanoEndpoint(m.provider?.baseUrl)
|
||||
)
|
||||
);
|
||||
},
|
||||
)?.provider;
|
||||
const volcanoProvider = volcanoProviderFromAgent || volcanoProviderFallback;
|
||||
|
||||
@ -767,60 +895,60 @@ export async function summarizeImageForExecutionReport(params: {
|
||||
agentModel,
|
||||
allModels,
|
||||
});
|
||||
if (candidateModels.length === 0) return null;
|
||||
if (candidateModels.length > 0) {
|
||||
const { getLanguageModelById } = await import("./dynamic-provider");
|
||||
for (const candidate of candidateModels) {
|
||||
const model = await getLanguageModelById(candidate.model.id);
|
||||
if (!model) continue;
|
||||
|
||||
const { getLanguageModelById } = await import("./dynamic-provider");
|
||||
for (const candidate of candidateModels) {
|
||||
const model = await getLanguageModelById(candidate.model.id);
|
||||
if (!model) continue;
|
||||
const preferRemote = shouldPreferRemoteVisionUrls(candidate);
|
||||
const modeAttempts = Array.from(new Set([preferRemote, !preferRemote]));
|
||||
|
||||
const preferRemote = shouldPreferRemoteVisionUrls(candidate);
|
||||
const modeAttempts = Array.from(new Set([preferRemote, !preferRemote]));
|
||||
for (const preferRemoteUrls of modeAttempts) {
|
||||
const fallbackVisionImages = await buildAnalyzerVisionImagesByMode({
|
||||
imageUrls,
|
||||
preferRemoteUrls,
|
||||
});
|
||||
if (fallbackVisionImages.length === 0) continue;
|
||||
|
||||
for (const preferRemoteUrls of modeAttempts) {
|
||||
const fallbackVisionImages = await buildAnalyzerVisionImagesByMode({
|
||||
imageUrls,
|
||||
preferRemoteUrls,
|
||||
});
|
||||
if (fallbackVisionImages.length === 0) continue;
|
||||
try {
|
||||
const { text: summaryText } = await generateText({
|
||||
model,
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: `${promptText}\n仅输出一行，不超过60字。`,
|
||||
},
|
||||
...fallbackVisionImages,
|
||||
],
|
||||
},
|
||||
],
|
||||
temperature: 0.1,
|
||||
maxTokens: 120,
|
||||
abortSignal: AbortSignal.timeout(ANALYZER_TIMEOUT_MS),
|
||||
} as any);
|
||||
|
||||
try {
|
||||
const { text: summaryText } = await generateText({
|
||||
model,
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: `${promptText}\n仅输出一行，不超过60字。`,
|
||||
},
|
||||
...fallbackVisionImages,
|
||||
],
|
||||
},
|
||||
],
|
||||
temperature: 0.1,
|
||||
maxTokens: 120,
|
||||
abortSignal: AbortSignal.timeout(ANALYZER_TIMEOUT_MS),
|
||||
} as any);
|
||||
|
||||
const normalized = normalizeVisualSummaryText(summaryText);
|
||||
if (normalized && !isWeakVisualSummary(normalized)) {
|
||||
console.log(
|
||||
`[Smart Analyzer] Visual summary model used: ${candidate.model.displayName} (${candidate.provider.type}), preferRemote=${preferRemoteUrls}`,
|
||||
);
|
||||
return normalized;
|
||||
}
|
||||
if (normalized) {
|
||||
const normalized = normalizeVisualSummaryText(summaryText);
|
||||
if (normalized && !isWeakVisualSummary(normalized)) {
|
||||
console.log(
|
||||
`[Smart Analyzer] Visual summary model used: ${candidate.model.displayName} (${candidate.provider.type}), preferRemote=${preferRemoteUrls}`,
|
||||
);
|
||||
return normalized;
|
||||
}
|
||||
if (normalized) {
|
||||
console.warn(
|
||||
`[Smart Analyzer] Weak visual summary ignored: ${normalized.slice(0, 120)}`,
|
||||
);
|
||||
}
|
||||
} catch (attemptError) {
|
||||
console.warn(
|
||||
`[Smart Analyzer] Weak visual summary ignored: ${normalized.slice(0, 120)}`,
|
||||
`[Smart Analyzer] Visual summary attempt failed: ${candidate.model.displayName} (${candidate.provider.type}), preferRemote=${preferRemoteUrls}`,
|
||||
attemptError,
|
||||
);
|
||||
}
|
||||
} catch (attemptError) {
|
||||
console.warn(
|
||||
`[Smart Analyzer] Visual summary attempt failed: ${candidate.model.displayName} (${candidate.provider.type}), preferRemote=${preferRemoteUrls}`,
|
||||
attemptError,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -828,6 +956,13 @@ export async function summarizeImageForExecutionReport(params: {
|
||||
console.warn("[Smart Analyzer] summarizeImageForExecutionReport fallback failed:", error);
|
||||
}
|
||||
|
||||
const qwenFallbackSummary = await summarizeWithHardcodedQwen3VlFallback({
|
||||
text,
|
||||
context,
|
||||
imageUrls,
|
||||
});
|
||||
if (qwenFallbackSummary) return qwenFallbackSummary;
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
@ -928,7 +1063,7 @@ function isLikelyVisionCapableModel(modelWithProvider: {
|
||||
|
||||
const modelText = `${model.displayName || ""} ${model.modelId || ""}`.toLowerCase();
|
||||
|
||||
if (providerType === "volcano-multimodal") return true;
|
||||
if (providerType === "volcano-vision" || providerType === "volcano-multimodal") return true;
|
||||
|
||||
if (
|
||||
providerType === "zhipu" &&
|
||||
@ -1082,13 +1217,27 @@ async function analyzWithLLM(
|
||||
|
||||
if (!selectedAnalyzerModel) {
|
||||
console.log("[Smart Analyzer] No available analyzer model found");
|
||||
return null;
|
||||
return analyzeWithHardcodedQwen3VlFallback({
|
||||
text,
|
||||
hasImage,
|
||||
imageUrls,
|
||||
historyMessages,
|
||||
context,
|
||||
reason: "no_available_model",
|
||||
});
|
||||
}
|
||||
|
||||
const model = await getLanguageModelById(selectedAnalyzerModel.model.id);
|
||||
if (!model) {
|
||||
console.log("[Smart Analyzer] Failed to get language model");
|
||||
return null;
|
||||
return analyzeWithHardcodedQwen3VlFallback({
|
||||
text,
|
||||
hasImage,
|
||||
imageUrls,
|
||||
historyMessages,
|
||||
context,
|
||||
reason: "model_runtime_unavailable",
|
||||
});
|
||||
}
|
||||
|
||||
// 🆕 构建包含历史上下文的消息
|
||||
@ -1202,9 +1351,22 @@ ${historyText}
|
||||
abortSignal: AbortSignal.timeout(ANALYZER_TIMEOUT_MS),
|
||||
};
|
||||
|
||||
const { text: responseText } = await generateText(requestPayload as any);
|
||||
try {
|
||||
const { text: responseText } = await generateText(requestPayload as any);
|
||||
const parsed = parseAnalysisResultFromModelText(responseText, text, hasImage);
|
||||
if (parsed) return parsed;
|
||||
} catch (error) {
|
||||
console.warn("[Smart Analyzer] Primary analyzer model failed:", error);
|
||||
}
|
||||
|
||||
return parseAnalysisResultFromModelText(responseText, text, hasImage);
|
||||
return analyzeWithHardcodedQwen3VlFallback({
|
||||
text,
|
||||
hasImage,
|
||||
imageUrls,
|
||||
historyMessages,
|
||||
context,
|
||||
reason: "primary_analysis_failed",
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user