import { NextResponse } from "next/server"; import { auth } from "@/app/(auth)/auth"; import { callQwen3Vl } from "@/lib/ai/qwen3-vl"; import { getSystemSetting } from "@/lib/db/queries"; import { createLogger } from "@/lib/observability/logger"; import { uploadFile } from "@/lib/storage/service"; import { VIDEO_CANVAS_FUNCTION_CONFIG_KEY, defaultVideoCanvasFunctionConfig, normalizeVideoCanvasFunctionConfig, } from "@/lib/video-canvas/config"; import sharp from "sharp"; export const runtime = "nodejs"; export const maxDuration = 30; const logger = createLogger("pick-object"); // 位置词库 const POSITION_OPTIONS = [ '左上区域', '中央顶部', '右上区域', '左侧中部', '中央区域', '右侧中部', '左下区域', '中央底部', '右下区域' ] as const; interface PickObjectRequest { imageData: string; // 带红色标记的图片 base64 clickX: number; // 点击位置 X (0-1) clickY: number; // 点击位置 Y (0-1) mode?: "object" | "video-canvas-mark"; } interface PickObjectResult { name: string; labels?: string[]; position: string; confidence: number; boundingBox: { x: number; y: number; width: number; height: number; }; } const MAX_VISION_SIDE = 500; const PICK_OBJECT_FETCH_TIMEOUT_MS = 15000; const MAX_PICK_OBJECT_IMAGE_BYTES = 10 * 1024 * 1024; function isBlockedImageHost(hostname: string) { const host = hostname.toLowerCase(); if ( host === "localhost" || host === "127.0.0.1" || host === "::1" || host.endsWith(".local") || host.endsWith(".internal") ) { return true; } const parts = host.split(".").map((part) => Number(part)); if (parts.length === 4 && parts.every((part) => Number.isInteger(part) && part >= 0 && part <= 255)) { const [a, b] = parts; return ( a === 10 || a === 127 || (a === 172 && b >= 16 && b <= 31) || (a === 192 && b === 168) || (a === 169 && b === 254) ); } return host.startsWith("fc") || host.startsWith("fd") || host.startsWith("fe80"); } function ensureImageSizeAllowed(byteLength: number) { if (byteLength > MAX_PICK_OBJECT_IMAGE_BYTES) { throw new Error("图片过大,请上传 10MB 以内的图片"); } } async function downscaleImageForVision(buffer: Buffer) { const source = sharp(buffer, { failOn: "none" }); const meta = await source.metadata(); const hasAlpha = Boolean(meta.hasAlpha); const width = meta.width || 0; const height = meta.height || 0; const resized = source .rotate() .resize({ width: MAX_VISION_SIDE, height: MAX_VISION_SIDE, fit: "inside", withoutEnlargement: true, }); if (hasAlpha) { return { buffer: await resized.png({ compressionLevel: 9 }).toBuffer(), ext: "png", width, height, }; } return { buffer: await resized.jpeg({ quality: 84, mozjpeg: true }).toBuffer(), ext: "jpg", width, height, }; } async function uploadVisionAnalysisImage(buffer: Buffer, filenamePrefix: string) { const { buffer: processedBuffer, ext, width, height } = await downscaleImageForVision(buffer); const filename = `${filenamePrefix}-${Date.now()}.${ext}`; const uploaded = await uploadFile(filename, processedBuffer as any); logger.info("vision image prepared", { originalWidth: width, originalHeight: height, maxSide: MAX_VISION_SIDE, outputExt: ext, outputBytes: processedBuffer.length, }); return uploaded.url; } async function prepareMarkedVisionAnalysisImage( buffer: Buffer, clickX: number, clickY: number ) { const prepared = await downscaleImageForVision(buffer); const metadata = await sharp(prepared.buffer, { failOn: "none" }).metadata(); const width = Math.max(1, metadata.width || prepared.width || MAX_VISION_SIDE); const height = Math.max(1, metadata.height || prepared.height || MAX_VISION_SIDE); const markerX = Math.max(0, Math.min(width, clickX * width)); const markerY = Math.max(0, Math.min(height, clickY * height)); const radius = Math.max(10, Math.min(width, height) * 0.025); const overlay = Buffer.from(` `); const composited = sharp(prepared.buffer, { failOn: "none" }).composite([ { input: overlay, left: 0, top: 0 }, ]); const markedBuffer = prepared.ext === "png" ? await composited.png({ compressionLevel: 9 }).toBuffer() : await composited.jpeg({ quality: 84, mozjpeg: true }).toBuffer(); logger.info("marked vision image prepared", { clickX, clickY, outputBytes: markedBuffer.length, outputExt: prepared.ext, }); const mimeType = prepared.ext === "png" ? "image/png" : "image/jpeg"; return `data:${mimeType};base64,${markedBuffer.toString("base64")}`; } async function normalizeImageInput( imageData: string, options?: { clickX: number; clickY: number; markPoint: boolean } ): Promise { const raw = String(imageData || "").trim(); if (!raw) throw new Error("缺少图片数据"); if (/^https?:\/\//i.test(raw)) { const url = new URL(raw); if (!["http:", "https:"].includes(url.protocol) || isBlockedImageHost(url.hostname)) { throw new Error("图片地址不允许访问"); } const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), PICK_OBJECT_FETCH_TIMEOUT_MS); try { const response = await fetch(raw, { method: "GET", signal: controller.signal, }); if (!response.ok) { throw new Error(`拉取图片失败: HTTP ${response.status}`); } const contentLength = Number(response.headers.get("content-length") || "0"); if (contentLength) { ensureImageSizeAllowed(contentLength); } const arrayBuffer = await response.arrayBuffer(); ensureImageSizeAllowed(arrayBuffer.byteLength); const buffer = Buffer.from(arrayBuffer); return options?.markPoint ? prepareMarkedVisionAnalysisImage(buffer, options.clickX, options.clickY) : uploadVisionAnalysisImage(buffer, "pick-object-url"); } finally { clearTimeout(timeout); } } const base64Match = raw.match(/^data:(image\/[a-zA-Z0-9.+-]+);base64,(.+)$/); if (!base64Match?.[1] || !base64Match?.[2]) { throw new Error("图片格式不支持"); } const base64Body = base64Match[2]; ensureImageSizeAllowed(Math.ceil((base64Body.length * 3) / 4)); const buffer = Buffer.from(base64Body, "base64"); ensureImageSizeAllowed(buffer.byteLength); return options?.markPoint ? prepareMarkedVisionAnalysisImage(buffer, options.clickX, options.clickY) : uploadVisionAnalysisImage(buffer, "pick-object-base64"); } export async function POST(request: Request) { try { const session = await auth(); if (!session?.user?.id) { return NextResponse.json({ success: false, error: "未授权,请先登录", }, { status: 401 }); } const body: PickObjectRequest = await request.json(); const { imageData, clickX, clickY, mode = "object" } = body; logger.info("request received", { imageDataLength: imageData?.length, clickX, clickY, }); // 验证参数 if (!imageData) { return NextResponse.json({ success: false, error: '缺少图片数据', }, { status: 400 }); } if (typeof clickX !== 'number' || typeof clickY !== 'number') { return NextResponse.json({ success: false, error: '缺少点击坐标', }, { status: 400 }); } if (clickX < 0 || clickX > 1 || clickY < 0 || clickY > 1) { return NextResponse.json({ success: false, error: '点击坐标超出范围', }, { status: 400 }); } const normalizedImage = await normalizeImageInput(imageData, { clickX, clickY, markPoint: mode === "video-canvas-mark", }); logger.info("vision model selected", { model: "qwen3-vl" }); const videoCanvasConfig = mode === "video-canvas-mark" ? normalizeVideoCanvasFunctionConfig( ( await getSystemSetting({ key: VIDEO_CANVAS_FUNCTION_CONFIG_KEY }).catch( () => null ) )?.value || defaultVideoCanvasFunctionConfig ) : null; const elementMarkDefinition = videoCanvasConfig?.functions["element-mark"]; if (mode === "video-canvas-mark" && elementMarkDefinition?.enabled === false) { return NextResponse.json( { success: false, error: "视频元素标记功能当前未启用" }, { status: 403 } ); } // AI 通过红色标记识别用户点中的主体;视频画布规则来自系统设置。 const defaultObjectPrompt = ` 图片中有一个红色圆形标记(带十字线),请识别这个红色标记所指向的具体物品。 【重要】 - 只识别红色标记正中心位置所属的“完整产品/主体” - 如果标记落在局部部件上(如椅子扶手、桌子腿、沙发靠背),也要返回完整物体名称(如“椅子/桌子/沙发”),不要返回部件名称 - 例如:标记在花瓶上 → 返回"花瓶"(不是"桌子"或"屏风");标记在椅子扶手 → 返回"椅子"(不是"扶手") 【输出要求 - 严格遵守】 返回纯 JSON,无其他文字: { "name": "物品名称", "position": "位置描述", "confidence": 0.95 } 【name 格式】≤6字,简洁具体 正确示例:橙色花瓶、白色杯子、木质椅子、吊灯、抱枕、花盆 错误示例:屏风(太大)、一个漂亮的花瓶(太长) 【position 格式】固定4字,从以下选择: ${POSITION_OPTIONS.join('、')} `.trim(); const prompt = mode === "video-canvas-mark" ? String( elementMarkDefinition?.prompt || defaultVideoCanvasFunctionConfig.functions["element-mark"]?.prompt || defaultObjectPrompt ).replaceAll("{{positionOptions}}", POSITION_OPTIONS.join("、")) : defaultObjectPrompt; logger.info("vision model call started"); // 调用视觉模型,发送带红色标记的图片 const { text: responseText, model } = await callQwen3Vl({ imageUrl: normalizedImage, model: mode === "video-canvas-mark" ? elementMarkDefinition?.model || videoCanvasConfig?.defaultMultimodalModel : undefined, prompt, temperature: 0.1, maxTokens: 300, }); logger.info("vision model call completed", { model, responseLength: responseText?.length || 0, }); if (!responseText) { return NextResponse.json({ success: false, error: 'AI 未返回结果', }, { status: 500 }); } // 解析 JSON 结果 let result: PickObjectResult; try { // 尝试从返回内容中提取 JSON const jsonMatch = responseText.match(/\{[\s\S]*\}/); if (!jsonMatch) { throw new Error('未找到 JSON'); } const parsed = JSON.parse(jsonMatch[0]); // 验证和修正名称 const labels = mode === "video-canvas-mark" && Array.isArray(parsed.labels) ? Array.from( new Set( parsed.labels .map((item: unknown) => String(item || "").trim().slice(0, 8)) .filter(Boolean) ) ).slice(0, 4) : []; let name = labels[0] || parsed.name || '物品'; if (name.length > 8) { name = name.substring(0, 6); } // 验证位置是否在词库中 let position = parsed.position; if (!POSITION_OPTIONS.includes(position as any)) { // 根据点击位置推算 const posX = clickX < 0.33 ? '左' : clickX > 0.66 ? '右' : '中央'; const posY = clickY < 0.33 ? '上' : clickY > 0.66 ? '下' : '中'; if (posX === '中央') { position = posY === '中' ? '中央区域' : posY === '上' ? '中央顶部' : '中央底部'; } else { position = posY === '中' ? `${posX}侧中部` : `${posX}${posY}区域`; } } // 确保 confidence 有效 let confidence = parsed.confidence; if (typeof confidence !== 'number' || confidence < 0 || confidence > 1) { confidence = 0.85; } result = { name, labels: mode === "video-canvas-mark" ? (labels.length ? labels : [name]) : undefined, position, confidence, boundingBox: { x: clickX, y: clickY, width: 0, height: 0 }, // 不再使用,保持兼容 }; } catch (parseError) { logger.warn("model response parse failed", { error: parseError, responseLength: responseText.length, }); // 返回默认结果 const posX = clickX < 0.33 ? '左' : clickX > 0.66 ? '右' : '中央'; const posY = clickY < 0.33 ? '上' : clickY > 0.66 ? '下' : '中'; const position = posX === '中央' ? (posY === '中' ? '中央区域' : posY === '上' ? '中央顶部' : '中央底部') : (posY === '中' ? `${posX}侧中部` : `${posX}${posY}区域`); result = { name: '物品', labels: mode === "video-canvas-mark" ? ['物品'] : undefined, position, confidence: 0.5, boundingBox: { x: clickX, y: clickY, width: 0, height: 0 }, }; } logger.info("object picked", { nameLength: result.name.length, position: result.position, confidence: result.confidence, }); return NextResponse.json({ success: true, object: result, model: { id: model, name: model, provider: "alibaba-image-key", }, }); } catch (error) { logger.error("pick object failed", error); return NextResponse.json({ success: false, error: `识别失败: ${error instanceof Error ? error.message : '未知错误'}`, }, { status: 500 }); } }