fix: hardcode canvas vision routes to qwen2.5-vl with alibaba-image key
This commit is contained in:
Your Name
parent
4222e723b0
commit
5c2fd46534
@ -1,52 +1,6 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { auth } from "@/app/(auth)/auth";
|
||||
import { generateText } from "ai";
|
||||
import { getEnabledAiModelsWithProvider } from "@/lib/db/queries";
|
||||
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
||||
|
||||
const BLOCKED_PROVIDER_TYPES = new Set([
|
||||
"fal-video",
|
||||
"qianwen-video",
|
||||
"volcano-video",
|
||||
"doubao-video",
|
||||
"alibaba-image",
|
||||
"ruxa-image",
|
||||
"fal-image",
|
||||
"volcano-vision",
|
||||
]);
|
||||
|
||||
type VisionModelItem = Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>[number];
|
||||
|
||||
function isQwenVisionModel(item: VisionModelItem) {
|
||||
const providerType = String(item.provider?.type || "");
|
||||
if (BLOCKED_PROVIDER_TYPES.has(providerType)) return false;
|
||||
const modelText = `${item.model.modelId || ""} ${item.model.displayName || ""}`.toLowerCase();
|
||||
return /qwen.*vl|通义.*vl|千问.*vl/.test(modelText);
|
||||
}
|
||||
|
||||
function isQwen25VisionModel(item: VisionModelItem) {
|
||||
if (!isQwenVisionModel(item)) return false;
|
||||
const modelText = `${item.model.modelId || ""} ${item.model.displayName || ""}`.toLowerCase();
|
||||
return /qwen[-_. ]?2\.?5[-_. ]?vl|qwen2\.5[-_. ]?vl/.test(modelText);
|
||||
}
|
||||
|
||||
function scoreVisionCandidate(item: VisionModelItem) {
|
||||
let score = 0;
|
||||
const providerType = String(item.provider?.type || "");
|
||||
if (isQwen25VisionModel(item)) score += 400;
|
||||
if (item.model.supportsVision) score += 120;
|
||||
if (providerType === "alibaba") score += 80;
|
||||
if (providerType === "custom") score += 40;
|
||||
if (item.model.isDefault) score += 20;
|
||||
score -= Math.max(0, item.model.sortOrder ?? 0);
|
||||
return score;
|
||||
}
|
||||
|
||||
function pickAutoTitleModel(allModels: VisionModelItem[]) {
|
||||
const candidates = allModels.filter((item) => isQwenVisionModel(item));
|
||||
if (candidates.length === 0) return null;
|
||||
return [...candidates].sort((a, b) => scoreVisionCandidate(b) - scoreVisionCandidate(a))[0];
|
||||
}
|
||||
import { callQwen25Vl } from "@/lib/ai/qwen25-vl";
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
console.log("[Auto-title] API called");
|
||||
@ -65,56 +19,23 @@ export async function POST(request: NextRequest) {
|
||||
return NextResponse.json({ error: "Image URL is required" }, { status: 400 });
|
||||
}
|
||||
|
||||
// 固定使用 Qwen2.5-VL（若未配置则降级到其它 Qwen-VL）
|
||||
const modelsData = await getEnabledAiModelsWithProvider();
|
||||
|
||||
console.log("[Auto-title] Available models count:", modelsData.length);
|
||||
console.log("[Auto-title] All model IDs:", modelsData.map(m => m.model.modelId));
|
||||
|
||||
const visionModelData = pickAutoTitleModel(modelsData);
|
||||
|
||||
console.log("[Auto-title] Selected vision model:", {
|
||||
modelId: visionModelData?.model.modelId,
|
||||
displayName: visionModelData?.model.displayName,
|
||||
providerType: visionModelData?.provider?.type,
|
||||
isQwen25: Boolean(visionModelData && isQwen25VisionModel(visionModelData)),
|
||||
});
|
||||
|
||||
if (!visionModelData) {
|
||||
console.log("[Auto-title] No Qwen VL model found, using default title");
|
||||
return NextResponse.json({ title: "未命名画布" });
|
||||
}
|
||||
|
||||
const visionModel = visionModelData.model;
|
||||
const model = await getLanguageModelById(visionModel.id);
|
||||
if (!model) {
|
||||
console.log("[Auto-title] Failed to load vision model");
|
||||
return NextResponse.json({ title: "未命名画布" });
|
||||
}
|
||||
console.log("[Auto-title] 使用硬编码模型: qwen2.5-vl");
|
||||
|
||||
// 构造 Prompt
|
||||
const prompt = "请用2-4个字概括这张图片的主体内容，作为文件标题。直接返回标题，不要包含任何标点符号或其他文字。";
|
||||
|
||||
const { text } = await generateText({
|
||||
model,
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: prompt },
|
||||
{ type: 'image', image: imageUrl },
|
||||
],
|
||||
},
|
||||
],
|
||||
const { text, model } = await callQwen25Vl({
|
||||
imageUrl,
|
||||
prompt,
|
||||
temperature: 0.2,
|
||||
maxOutputTokens: 50,
|
||||
maxTokens: 50,
|
||||
});
|
||||
|
||||
let title = (text || "").trim();
|
||||
title = title.replace(/[\s\p{P}\p{S}]+/gu, "");
|
||||
if (title.length > 4) title = title.slice(0, 4);
|
||||
if (title.length < 2) title = "新画布";
|
||||
console.log("[Auto-title] Generated title:", title);
|
||||
console.log("[Auto-title] Generated title:", title, "model:", model);
|
||||
return NextResponse.json({ title });
|
||||
|
||||
} catch (error) {
|
||||
|
||||
@ -1,8 +1,6 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { generateText } from "ai";
|
||||
import { getEnabledAiModelsWithProvider } from "@/lib/db/queries";
|
||||
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
||||
import { uploadFile } from "@/lib/storage/service";
|
||||
import { callQwen25Vl } from "@/lib/ai/qwen25-vl";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
export const maxDuration = 30;
|
||||
@ -32,45 +30,6 @@ interface PickObjectResult {
|
||||
};
|
||||
}
|
||||
|
||||
const BLOCKED_PROVIDER_TYPES = new Set([
|
||||
"fal-video",
|
||||
"qianwen-video",
|
||||
"volcano-video",
|
||||
"doubao-video",
|
||||
]);
|
||||
|
||||
type VisionModelItem = Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>[number];
|
||||
|
||||
function isQwenVisionModel(item: VisionModelItem) {
|
||||
if (BLOCKED_PROVIDER_TYPES.has(String(item.provider?.type || ""))) return false;
|
||||
const modelText = `${item.model.displayName || ""} ${item.model.modelId || ""}`.toLowerCase();
|
||||
return /qwen.*vl|通义.*vl|千问.*vl/.test(modelText);
|
||||
}
|
||||
|
||||
function isQwen25VisionModel(item: VisionModelItem) {
|
||||
if (!isQwenVisionModel(item)) return false;
|
||||
const modelText = `${item.model.displayName || ""} ${item.model.modelId || ""}`.toLowerCase();
|
||||
return /qwen[-_. ]?2\.?5[-_. ]?vl|qwen2\.5[-_. ]?vl/.test(modelText);
|
||||
}
|
||||
|
||||
function scoreQwenVisionCandidate(item: VisionModelItem) {
|
||||
let score = 0;
|
||||
if (isQwen25VisionModel(item)) score += 400;
|
||||
if (item.model.supportsVision) score += 120;
|
||||
const providerType = String(item.provider?.type || "");
|
||||
if (providerType === "alibaba") score += 80;
|
||||
if (providerType === "custom") score += 40;
|
||||
if (item.model.isDefault) score += 20;
|
||||
score -= Math.max(0, item.model.sortOrder ?? 0);
|
||||
return score;
|
||||
}
|
||||
|
||||
function pickObjectVisionModel(allModels: VisionModelItem[]) {
|
||||
const candidates = allModels.filter((item) => isQwenVisionModel(item));
|
||||
if (candidates.length === 0) return null;
|
||||
return [...candidates].sort((a, b) => scoreQwenVisionCandidate(b) - scoreQwenVisionCandidate(a))[0];
|
||||
}
|
||||
|
||||
function inferFileExtFromMime(mimeType: string) {
|
||||
if (mimeType.includes("png")) return "png";
|
||||
if (mimeType.includes("webp")) return "webp";
|
||||
@ -124,36 +83,7 @@ export async function POST(request: Request) {
|
||||
|
||||
const normalizedImage = await normalizeImageInput(imageData);
|
||||
|
||||
// 固定使用 Qwen2.5-VL（若未配置则降级到其它 Qwen-VL）
|
||||
const allModels = await getEnabledAiModelsWithProvider();
|
||||
const selectedModel = pickObjectVisionModel(allModels);
|
||||
|
||||
if (!selectedModel) {
|
||||
console.log("[PickObject] No Qwen VL model found");
|
||||
return NextResponse.json({
|
||||
success: false,
|
||||
error: '未配置可用的 Qwen-VL 模型（请配置 qwen2.5-vl）',
|
||||
}, { status: 500 });
|
||||
}
|
||||
|
||||
console.log(
|
||||
"[PickObject] 使用模型:",
|
||||
selectedModel.model.modelId,
|
||||
selectedModel.model.displayName,
|
||||
selectedModel.provider.type,
|
||||
{
|
||||
isQwen25: isQwen25VisionModel(selectedModel),
|
||||
}
|
||||
);
|
||||
|
||||
const model = await getLanguageModelById(selectedModel.model.id);
|
||||
if (!model) {
|
||||
console.log("[PickObject] Failed to get language model");
|
||||
return NextResponse.json({
|
||||
success: false,
|
||||
error: '无法加载模型',
|
||||
}, { status: 500 });
|
||||
}
|
||||
console.log("[PickObject] 使用硬编码模型: qwen2.5-vl");
|
||||
|
||||
// 构建 prompt - AI 通过红色标记来识别用户选择的物品
|
||||
const prompt = `
|
||||
@ -183,28 +113,14 @@ ${POSITION_OPTIONS.join('、')}
|
||||
console.log('[PickObject] 调用视觉模型...');
|
||||
|
||||
// 调用视觉模型，发送带红色标记的图片
|
||||
const { text: responseText } = await generateText({
|
||||
model,
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'image',
|
||||
image: normalizedImage,
|
||||
},
|
||||
{
|
||||
type: 'text',
|
||||
text: prompt,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
const { text: responseText, model } = await callQwen25Vl({
|
||||
imageUrl: normalizedImage,
|
||||
prompt,
|
||||
temperature: 0.1,
|
||||
maxTokens: 300,
|
||||
} as any);
|
||||
});
|
||||
|
||||
console.log('[PickObject] 模型返回内容:', responseText?.substring(0, 200));
|
||||
console.log('[PickObject] 模型返回内容:', model, responseText?.substring(0, 200));
|
||||
|
||||
if (!responseText) {
|
||||
return NextResponse.json({
|
||||
@ -279,9 +195,9 @@ ${POSITION_OPTIONS.join('、')}
|
||||
success: true,
|
||||
object: result,
|
||||
model: {
|
||||
id: selectedModel.model.modelId,
|
||||
name: selectedModel.model.displayName,
|
||||
provider: selectedModel.provider.type,
|
||||
id: model,
|
||||
name: model,
|
||||
provider: "alibaba-image-key",
|
||||
},
|
||||
});
|
||||
|
||||
|
||||
116
lib/ai/qwen25-vl.ts
Normal file
116
lib/ai/qwen25-vl.ts
Normal file
@ -0,0 +1,116 @@
|
||||
import "server-only";
|
||||
|
||||
import { getEnabledAiProviders } from "@/lib/db/queries";
|
||||
|
||||
const QWEN25_VL_MODELS = [
|
||||
"qwen2.5-vl-72b-instruct",
|
||||
"qwen2.5-vl-7b-instruct",
|
||||
] as const;
|
||||
|
||||
const QWEN_COMPAT_BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1";
|
||||
|
||||
function parseAssistantText(payload: any): string {
|
||||
const choices = Array.isArray(payload?.choices) ? payload.choices : [];
|
||||
const content = choices[0]?.message?.content;
|
||||
|
||||
if (typeof content === "string") {
|
||||
return content.trim();
|
||||
}
|
||||
|
||||
if (Array.isArray(content)) {
|
||||
const chunks = content
|
||||
.map((part: any) => (typeof part?.text === "string" ? part.text.trim() : ""))
|
||||
.filter(Boolean);
|
||||
return chunks.join("\n").trim();
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
||||
|
||||
async function getQwenImageProviderApiKey() {
|
||||
const providers = await getEnabledAiProviders();
|
||||
const provider = providers.find(
|
||||
(item) => item.type === "alibaba-image" && typeof item.apiKey === "string" && item.apiKey.trim(),
|
||||
);
|
||||
|
||||
if (!provider?.apiKey) {
|
||||
throw new Error("未配置可用的千问生图 Provider API Key");
|
||||
}
|
||||
|
||||
return provider.apiKey.trim();
|
||||
}
|
||||
|
||||
export async function callQwen25Vl(params: {
|
||||
imageUrl: string;
|
||||
prompt: string;
|
||||
temperature?: number;
|
||||
maxTokens?: number;
|
||||
}) {
|
||||
const apiKey = await getQwenImageProviderApiKey();
|
||||
const endpoint = `${QWEN_COMPAT_BASE_URL}/chat/completions`;
|
||||
|
||||
let lastError: Error | null = null;
|
||||
|
||||
for (const model of QWEN25_VL_MODELS) {
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 30000);
|
||||
|
||||
try {
|
||||
const response = await fetch(endpoint, {
|
||||
method: "POST",
|
||||
signal: controller.signal,
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model,
|
||||
temperature: params.temperature ?? 0.1,
|
||||
max_tokens: params.maxTokens ?? 300,
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{ type: "text", text: params.prompt },
|
||||
{
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
url: params.imageUrl,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}),
|
||||
});
|
||||
|
||||
const payload = await response.json().catch(() => ({}));
|
||||
if (!response.ok) {
|
||||
const message =
|
||||
payload?.error?.message || payload?.message || `HTTP ${response.status}`;
|
||||
throw new Error(message);
|
||||
}
|
||||
|
||||
const text = parseAssistantText(payload);
|
||||
if (!text) {
|
||||
throw new Error("Qwen2.5-VL 未返回文本内容");
|
||||
}
|
||||
|
||||
return { text, model };
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
} catch (error) {
|
||||
lastError = error instanceof Error ? error : new Error(String(error));
|
||||
const errorText = String(lastError.message || "");
|
||||
const isModelNotFound =
|
||||
/model.*not.*found|invalid model|not support/i.test(errorText);
|
||||
if (!isModelNotFound) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError || new Error("Qwen2.5-VL 调用失败");
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user