diff --git a/lib/ai/smart-analyzer.ts b/lib/ai/smart-analyzer.ts index dca1d439c..4621112b8 100644 --- a/lib/ai/smart-analyzer.ts +++ b/lib/ai/smart-analyzer.ts @@ -424,6 +424,110 @@ function isLikelyZhipuEndpoint(baseUrl?: string | null): boolean { return /open\.bigmodel\.cn/i.test(baseUrl); } +function isLikelyVolcanoEndpoint(baseUrl?: string | null): boolean { + if (!baseUrl) return false; + return /ark\.cn-beijing\.volces\.com/i.test(baseUrl); +} + +function normalizeVolcanoResponsesEndpoint(baseUrl: string): string { + const cleaned = baseUrl.replace(/\/+$/, ""); + if (!cleaned) return "https://ark.cn-beijing.volces.com/api/v3/responses"; + if (/\/responses$/i.test(cleaned)) return cleaned; + if (/\/api\/v3$/i.test(cleaned)) return `${cleaned}/responses`; + if (/ark\.cn-beijing\.volces\.com/i.test(cleaned)) { + return `${cleaned}/api/v3/responses`.replace(/\/api\/v3\/api\/v3\//, "/api/v3/"); + } + return `${cleaned}/responses`; +} + +async function rewriteUploadPublicUrlForVolcanoSummary(url: string): Promise { + if (!/^https?:\/\//i.test(url)) return url; + try { + const { getDefaultStorageConfig } = await import("@/lib/db/queries"); + const storageConfig = await getDefaultStorageConfig(); + if ( + !storageConfig || + storageConfig.type !== "s3" || + !storageConfig.endpoint || + !storageConfig.bucket || + !storageConfig.publicUrl + ) { + return url; + } + + const parsed = new URL(url); + const publicOrigin = new URL(storageConfig.publicUrl).origin; + if (parsed.origin !== publicOrigin) return url; + if (!parsed.pathname.startsWith("/uploads/")) return url; + + const endpointBase = storageConfig.endpoint.replace(/\/+$/, ""); + return `${endpointBase}/${storageConfig.bucket}${parsed.pathname}${parsed.search}`; + } catch { + return url; + } +} + +async function summarizeWithVolcanoOfficialApi(params: { + apiKey: string; + baseUrl: string; + promptText: string; + imageUrls: string[]; +}): Promise { + const endpoint = normalizeVolcanoResponsesEndpoint(params.baseUrl); + const rewrittenUrls = await Promise.all( + params.imageUrls.map((url) => rewriteUploadPublicUrlForVolcanoSummary(url)), + ); + + try { + const response = await fetch(endpoint, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${params.apiKey}`, + }, + body: JSON.stringify({ + model: "doubao-seed-2-0-pro-260215", + input: [ + { + role: "user", + content: [ + ...rewrittenUrls.map((url) => ({ type: "input_image", image_url: url })), + { type: "input_text", text: params.promptText }, + ], + }, + ], + }), + }); + + const payload = await response.json().catch(() => ({})); + if (!response.ok || payload?.error) { + return null; + } + + const directText = String(payload?.output_text || "").trim(); + if (directText) { + const normalized = normalizeVisualSummaryText(directText); + if (normalized && !isWeakVisualSummary(normalized)) return normalized; + } + + const output = Array.isArray(payload?.output) ? payload.output : []; + const chunks: string[] = []; + for (const item of output) { + const content = Array.isArray(item?.content) ? item.content : []; + for (const part of content) { + if (typeof part?.text === "string" && part.text.trim()) { + chunks.push(part.text.trim()); + } + } + } + const merged = normalizeVisualSummaryText(chunks.join(" ").trim()); + if (merged && !isWeakVisualSummary(merged)) return merged; + } catch { + return null; + } + return null; +} + function normalizeZhipuChatEndpoint(baseUrl: string): string { const cleaned = baseUrl.replace(/\/+$/, ""); if (/\/chat\/completions$/i.test(cleaned)) return cleaned; @@ -619,6 +723,16 @@ export async function summarizeImageForExecutionReport(params: { (m.provider.type === "zhipu" || isLikelyZhipuEndpoint(m.provider.baseUrl)), )?.provider; const zhipuProvider = zhipuProviderFromAgent || zhipuProviderFallback; + const volcanoProviderFromAgent = + agentModel?.provider?.apiKey && isLikelyVolcanoEndpoint(agentModel.provider.baseUrl) + ? agentModel.provider + : null; + const volcanoProviderFallback = allModels.find( + (m) => + m.provider?.apiKey && + (m.provider.type === "volcano-multimodal" || isLikelyVolcanoEndpoint(m.provider.baseUrl)), + )?.provider; + const volcanoProvider = volcanoProviderFromAgent || volcanoProviderFallback; const promptText = [ "只根据图片中可见内容做客观观察,不要推测用户意图。", @@ -638,6 +752,16 @@ export async function summarizeImageForExecutionReport(params: { if (summary) return summary; } + if (volcanoProvider?.apiKey && volcanoProvider?.baseUrl) { + const summary = await summarizeWithVolcanoOfficialApi({ + apiKey: volcanoProvider.apiKey, + baseUrl: volcanoProvider.baseUrl, + promptText, + imageUrls: (imageUrls || []).filter((url) => isHttpUrl(url)).slice(0, ANALYZER_MAX_IMAGE_COUNT), + }); + if (summary) return summary; + } + try { const candidateModels = collectVisionSummaryCandidates({ agentModel,