309 lines
9.7 KiB
TypeScript
309 lines
9.7 KiB
TypeScript
import { NextResponse } from "next/server";
|
||
import { generateText } from "ai";
|
||
import { getAgentAiModelWithProvider, getEnabledAiModelsWithProvider } from "@/lib/db/queries";
|
||
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
||
import { isAiModelMultimodal } from "@/lib/db/ai-model-flags";
|
||
import { uploadFile } from "@/lib/storage/service";
|
||
|
||
export const runtime = "nodejs";
|
||
export const maxDuration = 30;
|
||
|
||
// 位置词库
|
||
const POSITION_OPTIONS = [
|
||
'左上区域', '中央顶部', '右上区域',
|
||
'左侧中部', '中央区域', '右侧中部',
|
||
'左下区域', '中央底部', '右下区域'
|
||
] as const;
|
||
|
||
interface PickObjectRequest {
|
||
imageData: string; // 带红色标记的图片 base64
|
||
clickX: number; // 点击位置 X (0-1)
|
||
clickY: number; // 点击位置 Y (0-1)
|
||
}
|
||
|
||
interface PickObjectResult {
|
||
name: string;
|
||
position: string;
|
||
confidence: number;
|
||
boundingBox: {
|
||
x: number;
|
||
y: number;
|
||
width: number;
|
||
height: number;
|
||
};
|
||
}
|
||
|
||
const BLOCKED_PROVIDER_TYPES = new Set([
|
||
"fal-video",
|
||
"qianwen-video",
|
||
"volcano-video",
|
||
"doubao-video",
|
||
]);
|
||
|
||
function isVisionCandidate(item: Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>[number]) {
|
||
if (BLOCKED_PROVIDER_TYPES.has(String(item.provider?.type || ""))) return false;
|
||
|
||
if (item.model.supportsVision) return true;
|
||
if (isAiModelMultimodal(item.model)) return true;
|
||
|
||
const modelText = `${item.model.displayName || ""} ${item.model.modelId || ""}`.toLowerCase();
|
||
if (/vl|vision|多模态|glm-4v|gpt-4o|doubao-seed-2-0|seed-2-0/.test(modelText)) {
|
||
return true;
|
||
}
|
||
|
||
return false;
|
||
}
|
||
|
||
function scoreVisionCandidate(item: Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>[number]) {
|
||
let score = 0;
|
||
if (isAiModelMultimodal(item.model)) score += 200;
|
||
if (item.model.supportsVision) score += 160;
|
||
const providerType = String(item.provider?.type || "");
|
||
if (providerType === "volcano-multimodal") score += 80;
|
||
if (providerType === "zhipu") score += 40;
|
||
if (providerType === "alibaba") score += 30;
|
||
if (item.model.isDefault) score += 20;
|
||
score -= Math.max(0, item.model.sortOrder ?? 0);
|
||
return score;
|
||
}
|
||
|
||
function pickObjectVisionModel(
|
||
allModels: Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>,
|
||
agentModel: Awaited<ReturnType<typeof getAgentAiModelWithProvider>>,
|
||
) {
|
||
if (agentModel && isVisionCandidate(agentModel)) {
|
||
return agentModel;
|
||
}
|
||
|
||
const candidates = allModels.filter((item) => isVisionCandidate(item));
|
||
if (candidates.length === 0) return null;
|
||
|
||
return [...candidates].sort((a, b) => scoreVisionCandidate(b) - scoreVisionCandidate(a))[0];
|
||
}
|
||
|
||
function inferFileExtFromMime(mimeType: string) {
|
||
if (mimeType.includes("png")) return "png";
|
||
if (mimeType.includes("webp")) return "webp";
|
||
if (mimeType.includes("gif")) return "gif";
|
||
return "jpg";
|
||
}
|
||
|
||
async function normalizeImageInput(imageData: string): Promise<string> {
|
||
const raw = String(imageData || "").trim();
|
||
if (!raw) throw new Error("缺少图片数据");
|
||
|
||
if (/^https?:\/\//i.test(raw)) {
|
||
return raw;
|
||
}
|
||
|
||
const base64Match = raw.match(/^data:(image\/[a-zA-Z0-9.+-]+);base64,(.+)$/);
|
||
if (!base64Match?.[1] || !base64Match?.[2]) {
|
||
throw new Error("图片格式不支持");
|
||
}
|
||
|
||
const mimeType = base64Match[1];
|
||
const base64Body = base64Match[2];
|
||
const buffer = Buffer.from(base64Body, "base64");
|
||
const ext = inferFileExtFromMime(mimeType);
|
||
const filename = `pick-object-${Date.now()}.${ext}`;
|
||
const uploaded = await uploadFile(filename, buffer as any);
|
||
return uploaded.url;
|
||
}
|
||
|
||
export async function POST(request: Request) {
|
||
try {
|
||
const body: PickObjectRequest = await request.json();
|
||
const { imageData, clickX, clickY } = body;
|
||
|
||
console.log('[PickObject] 收到请求:', { imageDataLength: imageData?.length, clickX, clickY });
|
||
|
||
// 验证参数
|
||
if (!imageData) {
|
||
return NextResponse.json({
|
||
success: false,
|
||
error: '缺少图片数据',
|
||
}, { status: 400 });
|
||
}
|
||
|
||
if (typeof clickX !== 'number' || typeof clickY !== 'number') {
|
||
return NextResponse.json({
|
||
success: false,
|
||
error: '缺少点击坐标',
|
||
}, { status: 400 });
|
||
}
|
||
|
||
const normalizedImage = await normalizeImageInput(imageData);
|
||
|
||
// 优先使用「AGENT + 多模态/视觉」模型
|
||
const allModels = await getEnabledAiModelsWithProvider();
|
||
const agentModel = await getAgentAiModelWithProvider();
|
||
const selectedModel = pickObjectVisionModel(allModels, agentModel);
|
||
|
||
if (!selectedModel) {
|
||
console.log("[PickObject] No vision model found");
|
||
return NextResponse.json({
|
||
success: false,
|
||
error: '未配置可用的视觉/多模态模型',
|
||
}, { status: 500 });
|
||
}
|
||
|
||
console.log(
|
||
"[PickObject] 使用模型:",
|
||
selectedModel.model.modelId,
|
||
selectedModel.model.displayName,
|
||
selectedModel.provider.type,
|
||
{
|
||
isAgentModel:
|
||
agentModel?.model?.id === selectedModel.model.id,
|
||
isMultimodal: isAiModelMultimodal(selectedModel.model),
|
||
}
|
||
);
|
||
|
||
const model = await getLanguageModelById(selectedModel.model.id);
|
||
if (!model) {
|
||
console.log("[PickObject] Failed to get language model");
|
||
return NextResponse.json({
|
||
success: false,
|
||
error: '无法加载模型',
|
||
}, { status: 500 });
|
||
}
|
||
|
||
// 构建 prompt - AI 通过红色标记来识别用户选择的物品
|
||
const prompt = `
|
||
图片中有一个红色圆形标记（带十字线），请识别这个红色标记所指向的具体物品。
|
||
|
||
【重要】
|
||
- 只识别红色标记正中心位置所属的“完整产品/主体”
|
||
- 如果标记落在局部部件上（如椅子扶手、桌子腿、沙发靠背），也要返回完整物体名称（如“椅子/桌子/沙发”），不要返回部件名称
|
||
- 例如：标记在花瓶上 → 返回"花瓶"（不是"桌子"或"屏风"）；标记在椅子扶手 → 返回"椅子"（不是"扶手"）
|
||
|
||
【输出要求 - 严格遵守】
|
||
返回纯 JSON，无其他文字：
|
||
{
|
||
"name": "物品名称",
|
||
"position": "位置描述",
|
||
"confidence": 0.95
|
||
}
|
||
|
||
【name 格式】≤6字，简洁具体
|
||
正确示例：橙色花瓶、白色杯子、木质椅子、吊灯、抱枕、花盆
|
||
错误示例：屏风（太大）、一个漂亮的花瓶（太长）
|
||
|
||
【position 格式】固定4字，从以下选择：
|
||
${POSITION_OPTIONS.join('、')}
|
||
`.trim();
|
||
|
||
console.log('[PickObject] 调用视觉模型...');
|
||
|
||
// 调用视觉模型，发送带红色标记的图片
|
||
const { text: responseText } = await generateText({
|
||
model,
|
||
messages: [
|
||
{
|
||
role: 'user',
|
||
content: [
|
||
{
|
||
type: 'image',
|
||
image: normalizedImage,
|
||
},
|
||
{
|
||
type: 'text',
|
||
text: prompt,
|
||
},
|
||
],
|
||
},
|
||
],
|
||
temperature: 0.1,
|
||
maxTokens: 300,
|
||
} as any);
|
||
|
||
console.log('[PickObject] 模型返回内容:', responseText?.substring(0, 200));
|
||
|
||
if (!responseText) {
|
||
return NextResponse.json({
|
||
success: false,
|
||
error: 'AI 未返回结果',
|
||
}, { status: 500 });
|
||
}
|
||
|
||
// 解析 JSON 结果
|
||
let result: PickObjectResult;
|
||
try {
|
||
// 尝试从返回内容中提取 JSON
|
||
const jsonMatch = responseText.match(/\{[\s\S]*\}/);
|
||
if (!jsonMatch) {
|
||
throw new Error('未找到 JSON');
|
||
}
|
||
const parsed = JSON.parse(jsonMatch[0]);
|
||
|
||
// 验证和修正名称
|
||
let name = parsed.name || '物品';
|
||
if (name.length > 8) {
|
||
name = name.substring(0, 6);
|
||
}
|
||
|
||
// 验证位置是否在词库中
|
||
let position = parsed.position;
|
||
if (!POSITION_OPTIONS.includes(position as any)) {
|
||
// 根据点击位置推算
|
||
const posX = clickX < 0.33 ? '左' : clickX > 0.66 ? '右' : '中央';
|
||
const posY = clickY < 0.33 ? '上' : clickY > 0.66 ? '下' : '中';
|
||
if (posX === '中央') {
|
||
position = posY === '中' ? '中央区域' : posY === '上' ? '中央顶部' : '中央底部';
|
||
} else {
|
||
position = posY === '中' ? `${posX}侧中部` : `${posX}${posY}区域`;
|
||
}
|
||
}
|
||
|
||
// 确保 confidence 有效
|
||
let confidence = parsed.confidence;
|
||
if (typeof confidence !== 'number' || confidence < 0 || confidence > 1) {
|
||
confidence = 0.85;
|
||
}
|
||
|
||
result = {
|
||
name,
|
||
position,
|
||
confidence,
|
||
boundingBox: { x: clickX, y: clickY, width: 0, height: 0 }, // 不再使用，保持兼容
|
||
};
|
||
|
||
} catch (parseError) {
|
||
console.error('[PickObject] JSON parse error:', parseError, 'Content:', responseText);
|
||
|
||
// 返回默认结果
|
||
const posX = clickX < 0.33 ? '左' : clickX > 0.66 ? '右' : '中央';
|
||
const posY = clickY < 0.33 ? '上' : clickY > 0.66 ? '下' : '中';
|
||
const position = posX === '中央'
|
||
? (posY === '中' ? '中央区域' : posY === '上' ? '中央顶部' : '中央底部')
|
||
: (posY === '中' ? `${posX}侧中部` : `${posX}${posY}区域`);
|
||
|
||
result = {
|
||
name: '物品',
|
||
position,
|
||
confidence: 0.5,
|
||
boundingBox: { x: clickX, y: clickY, width: 0, height: 0 },
|
||
};
|
||
}
|
||
|
||
console.log('[PickObject] 识别结果:', result);
|
||
|
||
return NextResponse.json({
|
||
success: true,
|
||
object: result,
|
||
model: {
|
||
id: selectedModel.model.modelId,
|
||
name: selectedModel.model.displayName,
|
||
provider: selectedModel.provider.type,
|
||
},
|
||
});
|
||
|
||
} catch (error) {
|
||
console.error('[PickObject] Error:', error);
|
||
return NextResponse.json({
|
||
success: false,
|
||
error: `识别失败: ${error instanceof Error ? error.message : '未知错误'}`,
|
||
}, { status: 500 });
|
||
}
|
||
}
|