448 lines
14 KiB
TypeScript
448 lines
14 KiB
TypeScript
import { NextResponse } from "next/server";
|
||
import { auth } from "@/app/(auth)/auth";
|
||
import { callQwen3Vl } from "@/lib/ai/qwen3-vl";
|
||
import { getSystemSetting } from "@/lib/db/queries";
|
||
import { createLogger } from "@/lib/observability/logger";
|
||
import { uploadFile } from "@/lib/storage/service";
|
||
import {
|
||
VIDEO_CANVAS_FUNCTION_CONFIG_KEY,
|
||
defaultVideoCanvasFunctionConfig,
|
||
normalizeVideoCanvasFunctionConfig,
|
||
} from "@/lib/video-canvas/config";
|
||
import sharp from "sharp";
|
||
|
||
export const runtime = "nodejs";
|
||
export const maxDuration = 30;
|
||
|
||
const logger = createLogger("pick-object");
|
||
|
||
// 位置词库
|
||
const POSITION_OPTIONS = [
|
||
'左上区域', '中央顶部', '右上区域',
|
||
'左侧中部', '中央区域', '右侧中部',
|
||
'左下区域', '中央底部', '右下区域'
|
||
] as const;
|
||
|
||
interface PickObjectRequest {
|
||
imageData: string; // 带红色标记的图片 base64
|
||
clickX: number; // 点击位置 X (0-1)
|
||
clickY: number; // 点击位置 Y (0-1)
|
||
mode?: "object" | "video-canvas-mark";
|
||
}
|
||
|
||
interface PickObjectResult {
|
||
name: string;
|
||
labels?: string[];
|
||
position: string;
|
||
confidence: number;
|
||
boundingBox: {
|
||
x: number;
|
||
y: number;
|
||
width: number;
|
||
height: number;
|
||
};
|
||
}
|
||
|
||
const MAX_VISION_SIDE = 500;
|
||
const PICK_OBJECT_FETCH_TIMEOUT_MS = 15000;
|
||
const MAX_PICK_OBJECT_IMAGE_BYTES = 10 * 1024 * 1024;
|
||
|
||
function isBlockedImageHost(hostname: string) {
|
||
const host = hostname.toLowerCase();
|
||
if (
|
||
host === "localhost" ||
|
||
host === "127.0.0.1" ||
|
||
host === "::1" ||
|
||
host.endsWith(".local") ||
|
||
host.endsWith(".internal")
|
||
) {
|
||
return true;
|
||
}
|
||
|
||
const parts = host.split(".").map((part) => Number(part));
|
||
if (parts.length === 4 && parts.every((part) => Number.isInteger(part) && part >= 0 && part <= 255)) {
|
||
const [a, b] = parts;
|
||
return (
|
||
a === 10 ||
|
||
a === 127 ||
|
||
(a === 172 && b >= 16 && b <= 31) ||
|
||
(a === 192 && b === 168) ||
|
||
(a === 169 && b === 254)
|
||
);
|
||
}
|
||
|
||
return host.startsWith("fc") || host.startsWith("fd") || host.startsWith("fe80");
|
||
}
|
||
|
||
function ensureImageSizeAllowed(byteLength: number) {
|
||
if (byteLength > MAX_PICK_OBJECT_IMAGE_BYTES) {
|
||
throw new Error("图片过大，请上传 10MB 以内的图片");
|
||
}
|
||
}
|
||
|
||
async function downscaleImageForVision(buffer: Buffer) {
|
||
const source = sharp(buffer, { failOn: "none" });
|
||
const meta = await source.metadata();
|
||
const hasAlpha = Boolean(meta.hasAlpha);
|
||
const width = meta.width || 0;
|
||
const height = meta.height || 0;
|
||
|
||
const resized = source
|
||
.rotate()
|
||
.resize({
|
||
width: MAX_VISION_SIDE,
|
||
height: MAX_VISION_SIDE,
|
||
fit: "inside",
|
||
withoutEnlargement: true,
|
||
});
|
||
|
||
if (hasAlpha) {
|
||
return {
|
||
buffer: await resized.png({ compressionLevel: 9 }).toBuffer(),
|
||
ext: "png",
|
||
width,
|
||
height,
|
||
};
|
||
}
|
||
|
||
return {
|
||
buffer: await resized.jpeg({ quality: 84, mozjpeg: true }).toBuffer(),
|
||
ext: "jpg",
|
||
width,
|
||
height,
|
||
};
|
||
}
|
||
|
||
async function uploadVisionAnalysisImage(buffer: Buffer, filenamePrefix: string) {
|
||
const { buffer: processedBuffer, ext, width, height } = await downscaleImageForVision(buffer);
|
||
const filename = `${filenamePrefix}-${Date.now()}.${ext}`;
|
||
const uploaded = await uploadFile(filename, processedBuffer as any);
|
||
logger.info("vision image prepared", {
|
||
originalWidth: width,
|
||
originalHeight: height,
|
||
maxSide: MAX_VISION_SIDE,
|
||
outputExt: ext,
|
||
outputBytes: processedBuffer.length,
|
||
});
|
||
return uploaded.url;
|
||
}
|
||
|
||
async function prepareMarkedVisionAnalysisImage(
|
||
buffer: Buffer,
|
||
clickX: number,
|
||
clickY: number
|
||
) {
|
||
const prepared = await downscaleImageForVision(buffer);
|
||
const metadata = await sharp(prepared.buffer, { failOn: "none" }).metadata();
|
||
const width = Math.max(1, metadata.width || prepared.width || MAX_VISION_SIDE);
|
||
const height = Math.max(1, metadata.height || prepared.height || MAX_VISION_SIDE);
|
||
const markerX = Math.max(0, Math.min(width, clickX * width));
|
||
const markerY = Math.max(0, Math.min(height, clickY * height));
|
||
const radius = Math.max(10, Math.min(width, height) * 0.025);
|
||
const overlay = Buffer.from(`
|
||
<svg width="${width}" height="${height}" xmlns="http://www.w3.org/2000/svg">
|
||
<circle cx="${markerX}" cy="${markerY}" r="${radius + 3}" fill="#fff" />
|
||
<circle cx="${markerX}" cy="${markerY}" r="${radius}" fill="#f00" />
|
||
<g stroke="#f00" stroke-width="2">
|
||
<line x1="${markerX - radius * 2}" y1="${markerY}" x2="${markerX - radius - 2}" y2="${markerY}" />
|
||
<line x1="${markerX + radius + 2}" y1="${markerY}" x2="${markerX + radius * 2}" y2="${markerY}" />
|
||
<line x1="${markerX}" y1="${markerY - radius * 2}" x2="${markerX}" y2="${markerY - radius - 2}" />
|
||
<line x1="${markerX}" y1="${markerY + radius + 2}" x2="${markerX}" y2="${markerY + radius * 2}" />
|
||
</g>
|
||
</svg>
|
||
`);
|
||
const composited = sharp(prepared.buffer, { failOn: "none" }).composite([
|
||
{ input: overlay, left: 0, top: 0 },
|
||
]);
|
||
const markedBuffer =
|
||
prepared.ext === "png"
|
||
? await composited.png({ compressionLevel: 9 }).toBuffer()
|
||
: await composited.jpeg({ quality: 84, mozjpeg: true }).toBuffer();
|
||
logger.info("marked vision image prepared", {
|
||
clickX,
|
||
clickY,
|
||
outputBytes: markedBuffer.length,
|
||
outputExt: prepared.ext,
|
||
});
|
||
const mimeType = prepared.ext === "png" ? "image/png" : "image/jpeg";
|
||
return `data:${mimeType};base64,${markedBuffer.toString("base64")}`;
|
||
}
|
||
|
||
async function normalizeImageInput(
|
||
imageData: string,
|
||
options?: { clickX: number; clickY: number; markPoint: boolean }
|
||
): Promise<string> {
|
||
const raw = String(imageData || "").trim();
|
||
if (!raw) throw new Error("缺少图片数据");
|
||
|
||
if (/^https?:\/\//i.test(raw)) {
|
||
const url = new URL(raw);
|
||
if (!["http:", "https:"].includes(url.protocol) || isBlockedImageHost(url.hostname)) {
|
||
throw new Error("图片地址不允许访问");
|
||
}
|
||
|
||
const controller = new AbortController();
|
||
const timeout = setTimeout(() => controller.abort(), PICK_OBJECT_FETCH_TIMEOUT_MS);
|
||
try {
|
||
const response = await fetch(raw, {
|
||
method: "GET",
|
||
signal: controller.signal,
|
||
});
|
||
if (!response.ok) {
|
||
throw new Error(`拉取图片失败: HTTP ${response.status}`);
|
||
}
|
||
const contentLength = Number(response.headers.get("content-length") || "0");
|
||
if (contentLength) {
|
||
ensureImageSizeAllowed(contentLength);
|
||
}
|
||
const arrayBuffer = await response.arrayBuffer();
|
||
ensureImageSizeAllowed(arrayBuffer.byteLength);
|
||
const buffer = Buffer.from(arrayBuffer);
|
||
return options?.markPoint
|
||
? prepareMarkedVisionAnalysisImage(buffer, options.clickX, options.clickY)
|
||
: uploadVisionAnalysisImage(buffer, "pick-object-url");
|
||
} finally {
|
||
clearTimeout(timeout);
|
||
}
|
||
}
|
||
|
||
const base64Match = raw.match(/^data:(image\/[a-zA-Z0-9.+-]+);base64,(.+)$/);
|
||
if (!base64Match?.[1] || !base64Match?.[2]) {
|
||
throw new Error("图片格式不支持");
|
||
}
|
||
|
||
const base64Body = base64Match[2];
|
||
ensureImageSizeAllowed(Math.ceil((base64Body.length * 3) / 4));
|
||
const buffer = Buffer.from(base64Body, "base64");
|
||
ensureImageSizeAllowed(buffer.byteLength);
|
||
return options?.markPoint
|
||
? prepareMarkedVisionAnalysisImage(buffer, options.clickX, options.clickY)
|
||
: uploadVisionAnalysisImage(buffer, "pick-object-base64");
|
||
}
|
||
|
||
export async function POST(request: Request) {
|
||
try {
|
||
const session = await auth();
|
||
if (!session?.user?.id) {
|
||
return NextResponse.json({
|
||
success: false,
|
||
error: "未授权，请先登录",
|
||
}, { status: 401 });
|
||
}
|
||
|
||
const body: PickObjectRequest = await request.json();
|
||
const { imageData, clickX, clickY, mode = "object" } = body;
|
||
|
||
logger.info("request received", {
|
||
imageDataLength: imageData?.length,
|
||
clickX,
|
||
clickY,
|
||
});
|
||
|
||
// 验证参数
|
||
if (!imageData) {
|
||
return NextResponse.json({
|
||
success: false,
|
||
error: '缺少图片数据',
|
||
}, { status: 400 });
|
||
}
|
||
|
||
if (typeof clickX !== 'number' || typeof clickY !== 'number') {
|
||
return NextResponse.json({
|
||
success: false,
|
||
error: '缺少点击坐标',
|
||
}, { status: 400 });
|
||
}
|
||
|
||
if (clickX < 0 || clickX > 1 || clickY < 0 || clickY > 1) {
|
||
return NextResponse.json({
|
||
success: false,
|
||
error: '点击坐标超出范围',
|
||
}, { status: 400 });
|
||
}
|
||
|
||
const normalizedImage = await normalizeImageInput(imageData, {
|
||
clickX,
|
||
clickY,
|
||
markPoint: mode === "video-canvas-mark",
|
||
});
|
||
|
||
logger.info("vision model selected", { model: "qwen3-vl" });
|
||
|
||
const videoCanvasConfig =
|
||
mode === "video-canvas-mark"
|
||
? normalizeVideoCanvasFunctionConfig(
|
||
(
|
||
await getSystemSetting({ key: VIDEO_CANVAS_FUNCTION_CONFIG_KEY }).catch(
|
||
() => null
|
||
)
|
||
)?.value || defaultVideoCanvasFunctionConfig
|
||
)
|
||
: null;
|
||
const elementMarkDefinition = videoCanvasConfig?.functions["element-mark"];
|
||
if (mode === "video-canvas-mark" && elementMarkDefinition?.enabled === false) {
|
||
return NextResponse.json(
|
||
{ success: false, error: "视频元素标记功能当前未启用" },
|
||
{ status: 403 }
|
||
);
|
||
}
|
||
|
||
// AI 通过红色标记识别用户点中的主体；视频画布规则来自系统设置。
|
||
const defaultObjectPrompt = `
|
||
图片中有一个红色圆形标记（带十字线），请识别这个红色标记所指向的具体物品。
|
||
|
||
【重要】
|
||
- 只识别红色标记正中心位置所属的“完整产品/主体”
|
||
- 如果标记落在局部部件上（如椅子扶手、桌子腿、沙发靠背），也要返回完整物体名称（如“椅子/桌子/沙发”），不要返回部件名称
|
||
- 例如：标记在花瓶上 → 返回"花瓶"（不是"桌子"或"屏风"）；标记在椅子扶手 → 返回"椅子"（不是"扶手"）
|
||
|
||
【输出要求 - 严格遵守】
|
||
返回纯 JSON，无其他文字：
|
||
{
|
||
"name": "物品名称",
|
||
"position": "位置描述",
|
||
"confidence": 0.95
|
||
}
|
||
|
||
【name 格式】≤6字，简洁具体
|
||
正确示例：橙色花瓶、白色杯子、木质椅子、吊灯、抱枕、花盆
|
||
错误示例：屏风（太大）、一个漂亮的花瓶（太长）
|
||
|
||
【position 格式】固定4字，从以下选择：
|
||
${POSITION_OPTIONS.join('、')}
|
||
`.trim();
|
||
const prompt =
|
||
mode === "video-canvas-mark"
|
||
? String(
|
||
elementMarkDefinition?.prompt ||
|
||
defaultVideoCanvasFunctionConfig.functions["element-mark"]?.prompt ||
|
||
defaultObjectPrompt
|
||
).replaceAll("{{positionOptions}}", POSITION_OPTIONS.join("、"))
|
||
: defaultObjectPrompt;
|
||
|
||
logger.info("vision model call started");
|
||
|
||
// 调用视觉模型，发送带红色标记的图片
|
||
const { text: responseText, model } = await callQwen3Vl({
|
||
imageUrl: normalizedImage,
|
||
model:
|
||
mode === "video-canvas-mark"
|
||
? elementMarkDefinition?.model || videoCanvasConfig?.defaultMultimodalModel
|
||
: undefined,
|
||
prompt,
|
||
temperature: 0.1,
|
||
maxTokens: 300,
|
||
});
|
||
|
||
logger.info("vision model call completed", {
|
||
model,
|
||
responseLength: responseText?.length || 0,
|
||
});
|
||
|
||
if (!responseText) {
|
||
return NextResponse.json({
|
||
success: false,
|
||
error: 'AI 未返回结果',
|
||
}, { status: 500 });
|
||
}
|
||
|
||
// 解析 JSON 结果
|
||
let result: PickObjectResult;
|
||
try {
|
||
// 尝试从返回内容中提取 JSON
|
||
const jsonMatch = responseText.match(/\{[\s\S]*\}/);
|
||
if (!jsonMatch) {
|
||
throw new Error('未找到 JSON');
|
||
}
|
||
const parsed = JSON.parse(jsonMatch[0]);
|
||
|
||
// 验证和修正名称
|
||
const labels =
|
||
mode === "video-canvas-mark" && Array.isArray(parsed.labels)
|
||
? Array.from(
|
||
new Set(
|
||
parsed.labels
|
||
.map((item: unknown) => String(item || "").trim().slice(0, 8))
|
||
.filter(Boolean)
|
||
)
|
||
).slice(0, 4)
|
||
: [];
|
||
let name = labels[0] || parsed.name || '物品';
|
||
if (name.length > 8) {
|
||
name = name.substring(0, 6);
|
||
}
|
||
|
||
// 验证位置是否在词库中
|
||
let position = parsed.position;
|
||
if (!POSITION_OPTIONS.includes(position as any)) {
|
||
// 根据点击位置推算
|
||
const posX = clickX < 0.33 ? '左' : clickX > 0.66 ? '右' : '中央';
|
||
const posY = clickY < 0.33 ? '上' : clickY > 0.66 ? '下' : '中';
|
||
if (posX === '中央') {
|
||
position = posY === '中' ? '中央区域' : posY === '上' ? '中央顶部' : '中央底部';
|
||
} else {
|
||
position = posY === '中' ? `${posX}侧中部` : `${posX}${posY}区域`;
|
||
}
|
||
}
|
||
|
||
// 确保 confidence 有效
|
||
let confidence = parsed.confidence;
|
||
if (typeof confidence !== 'number' || confidence < 0 || confidence > 1) {
|
||
confidence = 0.85;
|
||
}
|
||
|
||
result = {
|
||
name,
|
||
labels: mode === "video-canvas-mark" ? (labels.length ? labels : [name]) : undefined,
|
||
position,
|
||
confidence,
|
||
boundingBox: { x: clickX, y: clickY, width: 0, height: 0 }, // 不再使用，保持兼容
|
||
};
|
||
|
||
} catch (parseError) {
|
||
logger.warn("model response parse failed", {
|
||
error: parseError,
|
||
responseLength: responseText.length,
|
||
});
|
||
|
||
// 返回默认结果
|
||
const posX = clickX < 0.33 ? '左' : clickX > 0.66 ? '右' : '中央';
|
||
const posY = clickY < 0.33 ? '上' : clickY > 0.66 ? '下' : '中';
|
||
const position = posX === '中央'
|
||
? (posY === '中' ? '中央区域' : posY === '上' ? '中央顶部' : '中央底部')
|
||
: (posY === '中' ? `${posX}侧中部` : `${posX}${posY}区域`);
|
||
|
||
result = {
|
||
name: '物品',
|
||
labels: mode === "video-canvas-mark" ? ['物品'] : undefined,
|
||
position,
|
||
confidence: 0.5,
|
||
boundingBox: { x: clickX, y: clickY, width: 0, height: 0 },
|
||
};
|
||
}
|
||
|
||
logger.info("object picked", {
|
||
nameLength: result.name.length,
|
||
position: result.position,
|
||
confidence: result.confidence,
|
||
});
|
||
|
||
return NextResponse.json({
|
||
success: true,
|
||
object: result,
|
||
model: {
|
||
id: model,
|
||
name: model,
|
||
provider: "alibaba-image-key",
|
||
},
|
||
});
|
||
|
||
} catch (error) {
|
||
logger.error("pick object failed", error);
|
||
return NextResponse.json({
|
||
success: false,
|
||
error: `识别失败: ${error instanceof Error ? error.message : '未知错误'}`,
|
||
}, { status: 500 });
|
||
}
|
||
}
|