import { NextResponse } from "next/server"; import { generateText } from "ai"; import { getEnabledAiModelsWithProvider } from "@/lib/db/queries"; import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; import { uploadFile } from "@/lib/storage/service"; export const runtime = "nodejs"; export const maxDuration = 30; // 位置词库 const POSITION_OPTIONS = [ '左上区域', '中央顶部', '右上区域', '左侧中部', '中央区域', '右侧中部', '左下区域', '中央底部', '右下区域' ] as const; interface PickObjectRequest { imageData: string; // 带红色标记的图片 base64 clickX: number; // 点击位置 X (0-1) clickY: number; // 点击位置 Y (0-1) } interface PickObjectResult { name: string; position: string; confidence: number; boundingBox: { x: number; y: number; width: number; height: number; }; } const BLOCKED_PROVIDER_TYPES = new Set([ "fal-video", "qianwen-video", "volcano-video", "doubao-video", ]); type VisionModelItem = Awaited>[number]; function isQwenVisionModel(item: VisionModelItem) { if (BLOCKED_PROVIDER_TYPES.has(String(item.provider?.type || ""))) return false; const modelText = `${item.model.displayName || ""} ${item.model.modelId || ""}`.toLowerCase(); return /qwen.*vl|通义.*vl|千问.*vl/.test(modelText); } function isQwen25VisionModel(item: VisionModelItem) { if (!isQwenVisionModel(item)) return false; const modelText = `${item.model.displayName || ""} ${item.model.modelId || ""}`.toLowerCase(); return /qwen[-_. ]?2\.?5[-_. ]?vl|qwen2\.5[-_. ]?vl/.test(modelText); } function scoreQwenVisionCandidate(item: VisionModelItem) { let score = 0; if (isQwen25VisionModel(item)) score += 400; if (item.model.supportsVision) score += 120; const providerType = String(item.provider?.type || ""); if (providerType === "alibaba") score += 80; if (providerType === "custom") score += 40; if (item.model.isDefault) score += 20; score -= Math.max(0, item.model.sortOrder ?? 0); return score; } function pickObjectVisionModel(allModels: VisionModelItem[]) { const candidates = allModels.filter((item) => isQwenVisionModel(item)); if (candidates.length === 0) return null; return [...candidates].sort((a, b) => scoreQwenVisionCandidate(b) - scoreQwenVisionCandidate(a))[0]; } function inferFileExtFromMime(mimeType: string) { if (mimeType.includes("png")) return "png"; if (mimeType.includes("webp")) return "webp"; if (mimeType.includes("gif")) return "gif"; return "jpg"; } async function normalizeImageInput(imageData: string): Promise { const raw = String(imageData || "").trim(); if (!raw) throw new Error("缺少图片数据"); if (/^https?:\/\//i.test(raw)) { return raw; } const base64Match = raw.match(/^data:(image\/[a-zA-Z0-9.+-]+);base64,(.+)$/); if (!base64Match?.[1] || !base64Match?.[2]) { throw new Error("图片格式不支持"); } const mimeType = base64Match[1]; const base64Body = base64Match[2]; const buffer = Buffer.from(base64Body, "base64"); const ext = inferFileExtFromMime(mimeType); const filename = `pick-object-${Date.now()}.${ext}`; const uploaded = await uploadFile(filename, buffer as any); return uploaded.url; } export async function POST(request: Request) { try { const body: PickObjectRequest = await request.json(); const { imageData, clickX, clickY } = body; console.log('[PickObject] 收到请求:', { imageDataLength: imageData?.length, clickX, clickY }); // 验证参数 if (!imageData) { return NextResponse.json({ success: false, error: '缺少图片数据', }, { status: 400 }); } if (typeof clickX !== 'number' || typeof clickY !== 'number') { return NextResponse.json({ success: false, error: '缺少点击坐标', }, { status: 400 }); } const normalizedImage = await normalizeImageInput(imageData); // 固定使用 Qwen2.5-VL(若未配置则降级到其它 Qwen-VL) const allModels = await getEnabledAiModelsWithProvider(); const selectedModel = pickObjectVisionModel(allModels); if (!selectedModel) { console.log("[PickObject] No Qwen VL model found"); return NextResponse.json({ success: false, error: '未配置可用的 Qwen-VL 模型(请配置 qwen2.5-vl)', }, { status: 500 }); } console.log( "[PickObject] 使用模型:", selectedModel.model.modelId, selectedModel.model.displayName, selectedModel.provider.type, { isQwen25: isQwen25VisionModel(selectedModel), } ); const model = await getLanguageModelById(selectedModel.model.id); if (!model) { console.log("[PickObject] Failed to get language model"); return NextResponse.json({ success: false, error: '无法加载模型', }, { status: 500 }); } // 构建 prompt - AI 通过红色标记来识别用户选择的物品 const prompt = ` 图片中有一个红色圆形标记(带十字线),请识别这个红色标记所指向的具体物品。 【重要】 - 只识别红色标记正中心位置所属的“完整产品/主体” - 如果标记落在局部部件上(如椅子扶手、桌子腿、沙发靠背),也要返回完整物体名称(如“椅子/桌子/沙发”),不要返回部件名称 - 例如:标记在花瓶上 → 返回"花瓶"(不是"桌子"或"屏风");标记在椅子扶手 → 返回"椅子"(不是"扶手") 【输出要求 - 严格遵守】 返回纯 JSON,无其他文字: { "name": "物品名称", "position": "位置描述", "confidence": 0.95 } 【name 格式】≤6字,简洁具体 正确示例:橙色花瓶、白色杯子、木质椅子、吊灯、抱枕、花盆 错误示例:屏风(太大)、一个漂亮的花瓶(太长) 【position 格式】固定4字,从以下选择: ${POSITION_OPTIONS.join('、')} `.trim(); console.log('[PickObject] 调用视觉模型...'); // 调用视觉模型,发送带红色标记的图片 const { text: responseText } = await generateText({ model, messages: [ { role: 'user', content: [ { type: 'image', image: normalizedImage, }, { type: 'text', text: prompt, }, ], }, ], temperature: 0.1, maxTokens: 300, } as any); console.log('[PickObject] 模型返回内容:', responseText?.substring(0, 200)); if (!responseText) { return NextResponse.json({ success: false, error: 'AI 未返回结果', }, { status: 500 }); } // 解析 JSON 结果 let result: PickObjectResult; try { // 尝试从返回内容中提取 JSON const jsonMatch = responseText.match(/\{[\s\S]*\}/); if (!jsonMatch) { throw new Error('未找到 JSON'); } const parsed = JSON.parse(jsonMatch[0]); // 验证和修正名称 let name = parsed.name || '物品'; if (name.length > 8) { name = name.substring(0, 6); } // 验证位置是否在词库中 let position = parsed.position; if (!POSITION_OPTIONS.includes(position as any)) { // 根据点击位置推算 const posX = clickX < 0.33 ? '左' : clickX > 0.66 ? '右' : '中央'; const posY = clickY < 0.33 ? '上' : clickY > 0.66 ? '下' : '中'; if (posX === '中央') { position = posY === '中' ? '中央区域' : posY === '上' ? '中央顶部' : '中央底部'; } else { position = posY === '中' ? `${posX}侧中部` : `${posX}${posY}区域`; } } // 确保 confidence 有效 let confidence = parsed.confidence; if (typeof confidence !== 'number' || confidence < 0 || confidence > 1) { confidence = 0.85; } result = { name, position, confidence, boundingBox: { x: clickX, y: clickY, width: 0, height: 0 }, // 不再使用,保持兼容 }; } catch (parseError) { console.error('[PickObject] JSON parse error:', parseError, 'Content:', responseText); // 返回默认结果 const posX = clickX < 0.33 ? '左' : clickX > 0.66 ? '右' : '中央'; const posY = clickY < 0.33 ? '上' : clickY > 0.66 ? '下' : '中'; const position = posX === '中央' ? (posY === '中' ? '中央区域' : posY === '上' ? '中央顶部' : '中央底部') : (posY === '中' ? `${posX}侧中部` : `${posX}${posY}区域`); result = { name: '物品', position, confidence: 0.5, boundingBox: { x: clickX, y: clickY, width: 0, height: 0 }, }; } console.log('[PickObject] 识别结果:', result); return NextResponse.json({ success: true, object: result, model: { id: selectedModel.model.modelId, name: selectedModel.model.displayName, provider: selectedModel.provider.type, }, }); } catch (error) { console.error('[PickObject] Error:', error); return NextResponse.json({ success: false, error: `识别失败: ${error instanceof Error ? error.message : '未知错误'}`, }, { status: 500 }); } }