feat: optimize canvas resize flow and object picking model routing
This commit is contained in:
Your Name
parent
ea2885769c
commit
4524bc6d9f
@ -1,7 +1,9 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { generateText } from "ai";
|
||||
import { getEnabledAiModelsWithProvider } from "@/lib/db/queries";
|
||||
import { getAgentAiModelWithProvider, getEnabledAiModelsWithProvider } from "@/lib/db/queries";
|
||||
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
||||
import { isAiModelMultimodal } from "@/lib/db/ai-model-flags";
|
||||
import { uploadFile } from "@/lib/storage/service";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
export const maxDuration = 30;
|
||||
@ -31,6 +33,83 @@ interface PickObjectResult {
|
||||
};
|
||||
}
|
||||
|
||||
const BLOCKED_PROVIDER_TYPES = new Set([
|
||||
"fal-video",
|
||||
"qianwen-video",
|
||||
"volcano-video",
|
||||
"doubao-video",
|
||||
]);
|
||||
|
||||
function isVisionCandidate(item: Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>[number]) {
|
||||
if (BLOCKED_PROVIDER_TYPES.has(String(item.provider?.type || ""))) return false;
|
||||
|
||||
if (item.model.supportsVision) return true;
|
||||
if (isAiModelMultimodal(item.model)) return true;
|
||||
|
||||
const modelText = `${item.model.displayName || ""} ${item.model.modelId || ""}`.toLowerCase();
|
||||
if (/vl|vision|多模态|glm-4v|gpt-4o|doubao-seed-2-0|seed-2-0/.test(modelText)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
function scoreVisionCandidate(item: Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>[number]) {
|
||||
let score = 0;
|
||||
if (isAiModelMultimodal(item.model)) score += 200;
|
||||
if (item.model.supportsVision) score += 160;
|
||||
const providerType = String(item.provider?.type || "");
|
||||
if (providerType === "volcano-multimodal") score += 80;
|
||||
if (providerType === "zhipu") score += 40;
|
||||
if (providerType === "alibaba") score += 30;
|
||||
if (item.model.isDefault) score += 20;
|
||||
score -= Math.max(0, item.model.sortOrder ?? 0);
|
||||
return score;
|
||||
}
|
||||
|
||||
function pickObjectVisionModel(
|
||||
allModels: Awaited<ReturnType<typeof getEnabledAiModelsWithProvider>>,
|
||||
agentModel: Awaited<ReturnType<typeof getAgentAiModelWithProvider>>,
|
||||
) {
|
||||
if (agentModel && isVisionCandidate(agentModel)) {
|
||||
return agentModel;
|
||||
}
|
||||
|
||||
const candidates = allModels.filter((item) => isVisionCandidate(item));
|
||||
if (candidates.length === 0) return null;
|
||||
|
||||
return [...candidates].sort((a, b) => scoreVisionCandidate(b) - scoreVisionCandidate(a))[0];
|
||||
}
|
||||
|
||||
function inferFileExtFromMime(mimeType: string) {
|
||||
if (mimeType.includes("png")) return "png";
|
||||
if (mimeType.includes("webp")) return "webp";
|
||||
if (mimeType.includes("gif")) return "gif";
|
||||
return "jpg";
|
||||
}
|
||||
|
||||
async function normalizeImageInput(imageData: string): Promise<string> {
|
||||
const raw = String(imageData || "").trim();
|
||||
if (!raw) throw new Error("缺少图片数据");
|
||||
|
||||
if (/^https?:\/\//i.test(raw)) {
|
||||
return raw;
|
||||
}
|
||||
|
||||
const base64Match = raw.match(/^data:(image\/[a-zA-Z0-9.+-]+);base64,(.+)$/);
|
||||
if (!base64Match?.[1] || !base64Match?.[2]) {
|
||||
throw new Error("图片格式不支持");
|
||||
}
|
||||
|
||||
const mimeType = base64Match[1];
|
||||
const base64Body = base64Match[2];
|
||||
const buffer = Buffer.from(base64Body, "base64");
|
||||
const ext = inferFileExtFromMime(mimeType);
|
||||
const filename = `pick-object-${Date.now()}.${ext}`;
|
||||
const uploaded = await uploadFile(filename, buffer as any);
|
||||
return uploaded.url;
|
||||
}
|
||||
|
||||
export async function POST(request: Request) {
|
||||
try {
|
||||
const body: PickObjectRequest = await request.json();
|
||||
@ -53,23 +132,34 @@ export async function POST(request: Request) {
|
||||
}, { status: 400 });
|
||||
}
|
||||
|
||||
// 获取 Qwen VL 模型（复用现有逻辑）
|
||||
const normalizedImage = await normalizeImageInput(imageData);
|
||||
|
||||
// 优先使用「AGENT + 多模态/视觉」模型
|
||||
const allModels = await getEnabledAiModelsWithProvider();
|
||||
const qwenModel = allModels.find(
|
||||
(m) => m.provider.type === "alibaba" && m.model.supportsVision
|
||||
);
|
||||
const agentModel = await getAgentAiModelWithProvider();
|
||||
const selectedModel = pickObjectVisionModel(allModels, agentModel);
|
||||
|
||||
if (!qwenModel) {
|
||||
console.log("[PickObject] No Qwen VL model found");
|
||||
if (!selectedModel) {
|
||||
console.log("[PickObject] No vision model found");
|
||||
return NextResponse.json({
|
||||
success: false,
|
||||
error: 'Qwen VL 模型未配置',
|
||||
error: '未配置可用的视觉/多模态模型',
|
||||
}, { status: 500 });
|
||||
}
|
||||
|
||||
console.log('[PickObject] 使用模型:', qwenModel.model.modelId, qwenModel.model.displayName);
|
||||
console.log(
|
||||
"[PickObject] 使用模型:",
|
||||
selectedModel.model.modelId,
|
||||
selectedModel.model.displayName,
|
||||
selectedModel.provider.type,
|
||||
{
|
||||
isAgentModel:
|
||||
agentModel?.model?.id === selectedModel.model.id,
|
||||
isMultimodal: isAiModelMultimodal(selectedModel.model),
|
||||
}
|
||||
);
|
||||
|
||||
const model = await getLanguageModelById(qwenModel.model.id);
|
||||
const model = await getLanguageModelById(selectedModel.model.id);
|
||||
if (!model) {
|
||||
console.log("[PickObject] Failed to get language model");
|
||||
return NextResponse.json({
|
||||
@ -103,9 +193,9 @@ export async function POST(request: Request) {
|
||||
${POSITION_OPTIONS.join('、')}
|
||||
`.trim();
|
||||
|
||||
console.log('[PickObject] 调用 Qwen VL 模型...');
|
||||
console.log('[PickObject] 调用视觉模型...');
|
||||
|
||||
// 调用 Qwen VL 模型，发送带红色标记的图片
|
||||
// 调用视觉模型，发送带红色标记的图片
|
||||
const { text: responseText } = await generateText({
|
||||
model,
|
||||
messages: [
|
||||
@ -114,7 +204,7 @@ ${POSITION_OPTIONS.join('、')}
|
||||
content: [
|
||||
{
|
||||
type: 'image',
|
||||
image: imageData, // 带红色标记的 base64 图片
|
||||
image: normalizedImage,
|
||||
},
|
||||
{
|
||||
type: 'text',
|
||||
@ -127,7 +217,7 @@ ${POSITION_OPTIONS.join('、')}
|
||||
maxTokens: 300,
|
||||
} as any);
|
||||
|
||||
console.log('[PickObject] Qwen 返回内容:', responseText?.substring(0, 200));
|
||||
console.log('[PickObject] 模型返回内容:', responseText?.substring(0, 200));
|
||||
|
||||
if (!responseText) {
|
||||
return NextResponse.json({
|
||||
@ -201,6 +291,11 @@ ${POSITION_OPTIONS.join('、')}
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
object: result,
|
||||
model: {
|
||||
id: selectedModel.model.modelId,
|
||||
name: selectedModel.model.displayName,
|
||||
provider: selectedModel.provider.type,
|
||||
},
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
|
||||
@ -234,10 +234,8 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) {
|
||||
const [isProcessing, setIsProcessing] = useState(false);
|
||||
const [angleDialogOpen, setAngleDialogOpen] = useState(false);
|
||||
const [sceneFusionDialogOpen, setSceneFusionDialogOpen] = useState(false);
|
||||
const [resizeDialogOpen, setResizeDialogOpen] = useState(false);
|
||||
const [resizeAnnotatorOpen, setResizeAnnotatorOpen] = useState(false);
|
||||
const [promptDialogOpen, setPromptDialogOpen] = useState(false);
|
||||
const [resizeWidth, setResizeWidth] = useState(1024);
|
||||
const [resizeHeight, setResizeHeight] = useState(1024);
|
||||
const [regeneratePrompt, setRegeneratePrompt] = useState("");
|
||||
const [isUploading, setIsUploading] = useState(false);
|
||||
const [annotatorOpen, setAnnotatorOpen] = useState(false);
|
||||
@ -1028,7 +1026,7 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) {
|
||||
return;
|
||||
}
|
||||
if (action === 'resize') {
|
||||
setResizeDialogOpen(true);
|
||||
setResizeAnnotatorOpen(true);
|
||||
return;
|
||||
}
|
||||
if (action === 'annotate') {
|
||||
@ -1227,77 +1225,79 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) {
|
||||
toast.success('标注已保存！发送对话时将重点处理标注区域');
|
||||
};
|
||||
|
||||
// 尺寸调整执行
|
||||
const handleResize = async () => {
|
||||
if (!data.src) return;
|
||||
if (isProcessing) {
|
||||
toast.info("正在处理中，请稍候...");
|
||||
return;
|
||||
}
|
||||
|
||||
setResizeDialogOpen(false);
|
||||
setIsProcessing(true);
|
||||
|
||||
// 创建处理中节点
|
||||
const newNodeId = addNode({
|
||||
type: 'image',
|
||||
position: {
|
||||
x: node.position.x + node.size.width + 20,
|
||||
y: node.position.y
|
||||
},
|
||||
size: { width: resizeWidth / 3, height: resizeHeight / 3 }, // 按比例显示
|
||||
data: {
|
||||
src: '',
|
||||
status: 'generating',
|
||||
prompt: `尺寸调整 ${resizeWidth}x${resizeHeight}`,
|
||||
title: `${displayTitle}-${resizeWidth}x${resizeHeight}`,
|
||||
originalSrc: data.src,
|
||||
},
|
||||
connections: [node.id],
|
||||
const uploadBase64ToStorage = useCallback(async (base64Url: string) => {
|
||||
const response = await fetch(base64Url);
|
||||
const blob = await response.blob();
|
||||
const formData = new FormData();
|
||||
formData.append("file", blob, `canvas-resize-${Date.now()}.png`);
|
||||
|
||||
const uploadResponse = await fetch("/api/files/upload", {
|
||||
method: "POST",
|
||||
body: formData,
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await imageGenerationService.resizeImage(data.src, resizeWidth, resizeHeight);
|
||||
|
||||
if (result?.success && result.data?.url) {
|
||||
updateNode(newNodeId, {
|
||||
data: {
|
||||
src: result.data.url,
|
||||
status: 'completed',
|
||||
prompt: `尺寸调整 ${resizeWidth}x${resizeHeight}`,
|
||||
title: `${displayTitle}-${resizeWidth}x${resizeHeight}`,
|
||||
originalSrc: data.src,
|
||||
},
|
||||
});
|
||||
toast.success('尺寸调整完成');
|
||||
} else {
|
||||
updateNode(newNodeId, {
|
||||
data: {
|
||||
src: '',
|
||||
status: 'failed',
|
||||
prompt: '尺寸调整失败',
|
||||
title: `${displayTitle}-尺寸调整失败`,
|
||||
originalSrc: data.src,
|
||||
},
|
||||
});
|
||||
toast.error(result?.error || '尺寸调整失败');
|
||||
if (!uploadResponse.ok) {
|
||||
throw new Error("上传失败");
|
||||
}
|
||||
|
||||
const result = await uploadResponse.json();
|
||||
return result.url as string;
|
||||
}, []);
|
||||
|
||||
const handleResizeEditorComplete = useCallback(
|
||||
async (resizedImageUrl: string) => {
|
||||
setResizeAnnotatorOpen(false);
|
||||
let finalUrl = resizedImageUrl;
|
||||
|
||||
if (resizedImageUrl.startsWith("data:")) {
|
||||
try {
|
||||
toast.info("正在上传尺寸调整结果...");
|
||||
finalUrl = await uploadBase64ToStorage(resizedImageUrl);
|
||||
} catch (error) {
|
||||
console.error("上传尺寸调整图片失败:", error);
|
||||
toast.error("尺寸调整结果上传失败");
|
||||
return;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('尺寸调整失败:', error);
|
||||
updateNode(newNodeId, {
|
||||
data: {
|
||||
src: '',
|
||||
status: 'failed',
|
||||
prompt: '尺寸调整失败',
|
||||
title: `${displayTitle}-尺寸调整失败`,
|
||||
|
||||
const title = `${displayTitle}-尺寸调整`;
|
||||
addNode({
|
||||
type: "image",
|
||||
position: {
|
||||
x: node.position.x + node.size.width + 20,
|
||||
y: node.position.y,
|
||||
},
|
||||
size: { width: node.size.width, height: node.size.height },
|
||||
data: {
|
||||
src: finalUrl,
|
||||
status: "completed",
|
||||
prompt: "尺寸调整结果",
|
||||
title,
|
||||
originalSrc: data.src,
|
||||
},
|
||||
connections: [node.id],
|
||||
});
|
||||
toast.error('尺寸调整失败');
|
||||
} finally {
|
||||
setIsProcessing(false);
|
||||
}
|
||||
};
|
||||
|
||||
addPendingChatAttachment({
|
||||
url: finalUrl,
|
||||
name: title,
|
||||
contentType: "image/jpeg",
|
||||
});
|
||||
toast.success("尺寸调整结果已发送到对话");
|
||||
},
|
||||
[
|
||||
addNode,
|
||||
addPendingChatAttachment,
|
||||
data.src,
|
||||
displayTitle,
|
||||
node.id,
|
||||
node.position.x,
|
||||
node.position.y,
|
||||
node.size.height,
|
||||
node.size.width,
|
||||
uploadBase64ToStorage,
|
||||
],
|
||||
);
|
||||
|
||||
// 基于当前图片+新提示词重绘
|
||||
const handlePromptRegenerate = async () => {
|
||||
@ -1689,55 +1689,6 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) {
|
||||
onOpenChange={setSceneFusionDialogOpen}
|
||||
/>
|
||||
|
||||
{/* 尺寸调整对话框 */}
|
||||
<Dialog open={resizeDialogOpen} onOpenChange={setResizeDialogOpen}>
|
||||
<DialogContent className="sm:max-w-[400px]">
|
||||
<DialogHeader>
|
||||
<DialogTitle className="flex items-center gap-2">
|
||||
<Maximize2 className="h-5 w-5" />
|
||||
尺寸调整
|
||||
</DialogTitle>
|
||||
</DialogHeader>
|
||||
<div className="grid gap-4 py-4">
|
||||
<div className="grid grid-cols-2 gap-4">
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="width">宽度 (px)</Label>
|
||||
<Input
|
||||
id="width"
|
||||
type="number"
|
||||
value={resizeWidth}
|
||||
onChange={(e) => setResizeWidth(parseInt(e.target.value) || 1024)}
|
||||
min={100}
|
||||
max={4096}
|
||||
/>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="height">高度 (px)</Label>
|
||||
<Input
|
||||
id="height"
|
||||
type="number"
|
||||
value={resizeHeight}
|
||||
onChange={(e) => setResizeHeight(parseInt(e.target.value) || 1024)}
|
||||
min={100}
|
||||
max={4096}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex flex-wrap gap-2">
|
||||
<Button variant="outline" size="sm" onClick={() => { setResizeWidth(512); setResizeHeight(512); }}>512x512</Button>
|
||||
<Button variant="outline" size="sm" onClick={() => { setResizeWidth(1024); setResizeHeight(1024); }}>1024x1024</Button>
|
||||
<Button variant="outline" size="sm" onClick={() => { setResizeWidth(1920); setResizeHeight(1080); }}>1920x1080</Button>
|
||||
<Button variant="outline" size="sm" onClick={() => { setResizeWidth(1080); setResizeHeight(1920); }}>1080x1920</Button>
|
||||
<Button variant="outline" size="sm" onClick={() => { setResizeWidth(2048); setResizeHeight(2048); }}>2048x2048</Button>
|
||||
</div>
|
||||
</div>
|
||||
<DialogFooter>
|
||||
<Button variant="outline" onClick={() => setResizeDialogOpen(false)}>取消</Button>
|
||||
<Button onClick={handleResize}>确定</Button>
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
|
||||
{/* 提示词重绘对话框 */}
|
||||
<Dialog open={promptDialogOpen} onOpenChange={setPromptDialogOpen}>
|
||||
<DialogContent className="sm:max-w-[560px]">
|
||||
@ -2166,6 +2117,16 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) {
|
||||
onCancel={() => setAnnotatorOpen(false)}
|
||||
/>
|
||||
)}
|
||||
|
||||
{resizeAnnotatorOpen && data.src && (
|
||||
<ImageAnnotator
|
||||
imageUrl={data.src}
|
||||
initialResizeMode
|
||||
confirmButtonText="发送到对话"
|
||||
onConfirm={handleResizeEditorComplete}
|
||||
onCancel={() => setResizeAnnotatorOpen(false)}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* 双击放大预览弹窗 */}
|
||||
{fullscreenPreviewOpen && data.src && createPortal(
|
||||
|
||||
@ -33,6 +33,7 @@ interface ImageAnnotatorProps {
|
||||
onCancel: () => void;
|
||||
onHomeClick?: () => void;
|
||||
initialResizeMode?: boolean; // 是否直接进入尺寸调整模式
|
||||
confirmButtonText?: string;
|
||||
}
|
||||
|
||||
const COLORS = [
|
||||
@ -49,7 +50,14 @@ const COLORS = [
|
||||
|
||||
const LINE_WIDTHS = [2, 4, 6, 8];
|
||||
|
||||
export function ImageAnnotator({ imageUrl, onConfirm, onCancel, onHomeClick, initialResizeMode = false }: ImageAnnotatorProps) {
|
||||
export function ImageAnnotator({
|
||||
imageUrl,
|
||||
onConfirm,
|
||||
onCancel,
|
||||
onHomeClick,
|
||||
initialResizeMode = false,
|
||||
confirmButtonText = "发送到新对话",
|
||||
}: ImageAnnotatorProps) {
|
||||
const canvasRef = useRef<HTMLCanvasElement>(null);
|
||||
const containerRef = useRef<HTMLDivElement>(null);
|
||||
const [ctx, setCtx] = useState<CanvasRenderingContext2D | null>(null);
|
||||
@ -1510,7 +1518,7 @@ export function ImageAnnotator({ imageUrl, onConfirm, onCancel, onHomeClick, ini
|
||||
{!showSceneFusion && (
|
||||
<Button size="sm" onClick={handleConfirm} className="bg-primary text-primary-foreground hover:bg-primary/90">
|
||||
<Check className="mr-1 h-4 w-4" />
|
||||
发送到新对话
|
||||
{confirmButtonText}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user