From 4524bc6d9f2f011973165ee23c7b22dcc629c611 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 20 Feb 2026 21:51:39 +0800 Subject: [PATCH] feat: optimize canvas resize flow and object picking model routing --- app/api/pick-object/route.ts | 123 ++++++++++++++-- components/canvas/nodes/ImageNode.tsx | 197 +++++++++++--------------- components/image-annotator.tsx | 12 +- 3 files changed, 198 insertions(+), 134 deletions(-) diff --git a/app/api/pick-object/route.ts b/app/api/pick-object/route.ts index b7c291379..6572d477b 100644 --- a/app/api/pick-object/route.ts +++ b/app/api/pick-object/route.ts @@ -1,7 +1,9 @@ import { NextResponse } from "next/server"; import { generateText } from "ai"; -import { getEnabledAiModelsWithProvider } from "@/lib/db/queries"; +import { getAgentAiModelWithProvider, getEnabledAiModelsWithProvider } from "@/lib/db/queries"; import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; +import { isAiModelMultimodal } from "@/lib/db/ai-model-flags"; +import { uploadFile } from "@/lib/storage/service"; export const runtime = "nodejs"; export const maxDuration = 30; @@ -31,6 +33,83 @@ interface PickObjectResult { }; } +const BLOCKED_PROVIDER_TYPES = new Set([ + "fal-video", + "qianwen-video", + "volcano-video", + "doubao-video", +]); + +function isVisionCandidate(item: Awaited>[number]) { + if (BLOCKED_PROVIDER_TYPES.has(String(item.provider?.type || ""))) return false; + + if (item.model.supportsVision) return true; + if (isAiModelMultimodal(item.model)) return true; + + const modelText = `${item.model.displayName || ""} ${item.model.modelId || ""}`.toLowerCase(); + if (/vl|vision|多模态|glm-4v|gpt-4o|doubao-seed-2-0|seed-2-0/.test(modelText)) { + return true; + } + + return false; +} + +function scoreVisionCandidate(item: Awaited>[number]) { + let score = 0; + if (isAiModelMultimodal(item.model)) score += 200; + if (item.model.supportsVision) score += 160; + const providerType = String(item.provider?.type || ""); + if (providerType === "volcano-multimodal") score += 80; + if (providerType === "zhipu") score += 40; + if (providerType === "alibaba") score += 30; + if (item.model.isDefault) score += 20; + score -= Math.max(0, item.model.sortOrder ?? 0); + return score; +} + +function pickObjectVisionModel( + allModels: Awaited>, + agentModel: Awaited>, +) { + if (agentModel && isVisionCandidate(agentModel)) { + return agentModel; + } + + const candidates = allModels.filter((item) => isVisionCandidate(item)); + if (candidates.length === 0) return null; + + return [...candidates].sort((a, b) => scoreVisionCandidate(b) - scoreVisionCandidate(a))[0]; +} + +function inferFileExtFromMime(mimeType: string) { + if (mimeType.includes("png")) return "png"; + if (mimeType.includes("webp")) return "webp"; + if (mimeType.includes("gif")) return "gif"; + return "jpg"; +} + +async function normalizeImageInput(imageData: string): Promise { + const raw = String(imageData || "").trim(); + if (!raw) throw new Error("缺少图片数据"); + + if (/^https?:\/\//i.test(raw)) { + return raw; + } + + const base64Match = raw.match(/^data:(image\/[a-zA-Z0-9.+-]+);base64,(.+)$/); + if (!base64Match?.[1] || !base64Match?.[2]) { + throw new Error("图片格式不支持"); + } + + const mimeType = base64Match[1]; + const base64Body = base64Match[2]; + const buffer = Buffer.from(base64Body, "base64"); + const ext = inferFileExtFromMime(mimeType); + const filename = `pick-object-${Date.now()}.${ext}`; + const uploaded = await uploadFile(filename, buffer as any); + return uploaded.url; +} + export async function POST(request: Request) { try { const body: PickObjectRequest = await request.json(); @@ -53,23 +132,34 @@ export async function POST(request: Request) { }, { status: 400 }); } - // 获取 Qwen VL 模型（复用现有逻辑） + const normalizedImage = await normalizeImageInput(imageData); + + // 优先使用「AGENT + 多模态/视觉」模型 const allModels = await getEnabledAiModelsWithProvider(); - const qwenModel = allModels.find( - (m) => m.provider.type === "alibaba" && m.model.supportsVision - ); + const agentModel = await getAgentAiModelWithProvider(); + const selectedModel = pickObjectVisionModel(allModels, agentModel); - if (!qwenModel) { - console.log("[PickObject] No Qwen VL model found"); + if (!selectedModel) { + console.log("[PickObject] No vision model found"); return NextResponse.json({ success: false, - error: 'Qwen VL 模型未配置', + error: '未配置可用的视觉/多模态模型', }, { status: 500 }); } - console.log('[PickObject] 使用模型:', qwenModel.model.modelId, qwenModel.model.displayName); + console.log( + "[PickObject] 使用模型:", + selectedModel.model.modelId, + selectedModel.model.displayName, + selectedModel.provider.type, + { + isAgentModel: + agentModel?.model?.id === selectedModel.model.id, + isMultimodal: isAiModelMultimodal(selectedModel.model), + } + ); - const model = await getLanguageModelById(qwenModel.model.id); + const model = await getLanguageModelById(selectedModel.model.id); if (!model) { console.log("[PickObject] Failed to get language model"); return NextResponse.json({ @@ -103,9 +193,9 @@ export async function POST(request: Request) { ${POSITION_OPTIONS.join('、')} `.trim(); - console.log('[PickObject] 调用 Qwen VL 模型...'); + console.log('[PickObject] 调用视觉模型...'); - // 调用 Qwen VL 模型，发送带红色标记的图片 + // 调用视觉模型，发送带红色标记的图片 const { text: responseText } = await generateText({ model, messages: [ @@ -114,7 +204,7 @@ ${POSITION_OPTIONS.join('、')} content: [ { type: 'image', - image: imageData, // 带红色标记的 base64 图片 + image: normalizedImage, }, { type: 'text', @@ -127,7 +217,7 @@ ${POSITION_OPTIONS.join('、')} maxTokens: 300, } as any); - console.log('[PickObject] Qwen 返回内容:', responseText?.substring(0, 200)); + console.log('[PickObject] 模型返回内容:', responseText?.substring(0, 200)); if (!responseText) { return NextResponse.json({ @@ -201,6 +291,11 @@ ${POSITION_OPTIONS.join('、')} return NextResponse.json({ success: true, object: result, + model: { + id: selectedModel.model.modelId, + name: selectedModel.model.displayName, + provider: selectedModel.provider.type, + }, }); } catch (error) { diff --git a/components/canvas/nodes/ImageNode.tsx b/components/canvas/nodes/ImageNode.tsx index 239cb4973..89de3e5e1 100644 --- a/components/canvas/nodes/ImageNode.tsx +++ b/components/canvas/nodes/ImageNode.tsx @@ -234,10 +234,8 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) { const [isProcessing, setIsProcessing] = useState(false); const [angleDialogOpen, setAngleDialogOpen] = useState(false); const [sceneFusionDialogOpen, setSceneFusionDialogOpen] = useState(false); - const [resizeDialogOpen, setResizeDialogOpen] = useState(false); + const [resizeAnnotatorOpen, setResizeAnnotatorOpen] = useState(false); const [promptDialogOpen, setPromptDialogOpen] = useState(false); - const [resizeWidth, setResizeWidth] = useState(1024); - const [resizeHeight, setResizeHeight] = useState(1024); const [regeneratePrompt, setRegeneratePrompt] = useState(""); const [isUploading, setIsUploading] = useState(false); const [annotatorOpen, setAnnotatorOpen] = useState(false); @@ -1028,7 +1026,7 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) { return; } if (action === 'resize') { - setResizeDialogOpen(true); + setResizeAnnotatorOpen(true); return; } if (action === 'annotate') { @@ -1227,77 +1225,79 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) { toast.success('标注已保存！发送对话时将重点处理标注区域'); }; - // 尺寸调整执行 - const handleResize = async () => { - if (!data.src) return; - if (isProcessing) { - toast.info("正在处理中，请稍候..."); - return; - } - - setResizeDialogOpen(false); - setIsProcessing(true); - - // 创建处理中节点 - const newNodeId = addNode({ - type: 'image', - position: { - x: node.position.x + node.size.width + 20, - y: node.position.y - }, - size: { width: resizeWidth / 3, height: resizeHeight / 3 }, // 按比例显示 - data: { - src: '', - status: 'generating', - prompt: `尺寸调整 ${resizeWidth}x${resizeHeight}`, - title: `${displayTitle}-${resizeWidth}x${resizeHeight}`, - originalSrc: data.src, - }, - connections: [node.id], + const uploadBase64ToStorage = useCallback(async (base64Url: string) => { + const response = await fetch(base64Url); + const blob = await response.blob(); + const formData = new FormData(); + formData.append("file", blob, `canvas-resize-${Date.now()}.png`); + + const uploadResponse = await fetch("/api/files/upload", { + method: "POST", + body: formData, }); - try { - const result = await imageGenerationService.resizeImage(data.src, resizeWidth, resizeHeight); - - if (result?.success && result.data?.url) { - updateNode(newNodeId, { - data: { - src: result.data.url, - status: 'completed', - prompt: `尺寸调整 ${resizeWidth}x${resizeHeight}`, - title: `${displayTitle}-${resizeWidth}x${resizeHeight}`, - originalSrc: data.src, - }, - }); - toast.success('尺寸调整完成'); - } else { - updateNode(newNodeId, { - data: { - src: '', - status: 'failed', - prompt: '尺寸调整失败', - title: `${displayTitle}-尺寸调整失败`, - originalSrc: data.src, - }, - }); - toast.error(result?.error || '尺寸调整失败'); + if (!uploadResponse.ok) { + throw new Error("上传失败"); + } + + const result = await uploadResponse.json(); + return result.url as string; + }, []); + + const handleResizeEditorComplete = useCallback( + async (resizedImageUrl: string) => { + setResizeAnnotatorOpen(false); + let finalUrl = resizedImageUrl; + + if (resizedImageUrl.startsWith("data:")) { + try { + toast.info("正在上传尺寸调整结果..."); + finalUrl = await uploadBase64ToStorage(resizedImageUrl); + } catch (error) { + console.error("上传尺寸调整图片失败:", error); + toast.error("尺寸调整结果上传失败"); + return; + } } - } catch (error) { - console.error('尺寸调整失败:', error); - updateNode(newNodeId, { - data: { - src: '', - status: 'failed', - prompt: '尺寸调整失败', - title: `${displayTitle}-尺寸调整失败`, + + const title = `${displayTitle}-尺寸调整`; + addNode({ + type: "image", + position: { + x: node.position.x + node.size.width + 20, + y: node.position.y, + }, + size: { width: node.size.width, height: node.size.height }, + data: { + src: finalUrl, + status: "completed", + prompt: "尺寸调整结果", + title, originalSrc: data.src, }, + connections: [node.id], }); - toast.error('尺寸调整失败'); - } finally { - setIsProcessing(false); - } - }; + + addPendingChatAttachment({ + url: finalUrl, + name: title, + contentType: "image/jpeg", + }); + toast.success("尺寸调整结果已发送到对话"); + }, + [ + addNode, + addPendingChatAttachment, + data.src, + displayTitle, + node.id, + node.position.x, + node.position.y, + node.size.height, + node.size.width, + uploadBase64ToStorage, + ], + ); // 基于当前图片+新提示词重绘 const handlePromptRegenerate = async () => { @@ -1689,55 +1689,6 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) { onOpenChange={setSceneFusionDialogOpen} /> - {/* 尺寸调整对话框 */} - - - - - - 尺寸调整 - - -
-
-
- - setResizeWidth(parseInt(e.target.value) || 1024)} - min={100} - max={4096} - /> -
-
- - setResizeHeight(parseInt(e.target.value) || 1024)} - min={100} - max={4096} - /> -
-
-
- - - - - -
-
- - - - - - - {/* 提示词重绘对话框 */} @@ -2166,6 +2117,16 @@ export function ImageNode({ node, isSelected }: ImageNodeProps) { onCancel={() => setAnnotatorOpen(false)} /> )} + + {resizeAnnotatorOpen && data.src && ( + setResizeAnnotatorOpen(false)} + /> + )} {/* 双击放大预览弹窗 */} {fullscreenPreviewOpen && data.src && createPortal( diff --git a/components/image-annotator.tsx b/components/image-annotator.tsx index 76972b211..68b693360 100644 --- a/components/image-annotator.tsx +++ b/components/image-annotator.tsx @@ -33,6 +33,7 @@ interface ImageAnnotatorProps { onCancel: () => void; onHomeClick?: () => void; initialResizeMode?: boolean; // 是否直接进入尺寸调整模式 + confirmButtonText?: string; } const COLORS = [ @@ -49,7 +50,14 @@ const COLORS = [ const LINE_WIDTHS = [2, 4, 6, 8]; -export function ImageAnnotator({ imageUrl, onConfirm, onCancel, onHomeClick, initialResizeMode = false }: ImageAnnotatorProps) { +export function ImageAnnotator({ + imageUrl, + onConfirm, + onCancel, + onHomeClick, + initialResizeMode = false, + confirmButtonText = "发送到新对话", +}: ImageAnnotatorProps) { const canvasRef = useRef(null); const containerRef = useRef(null); const [ctx, setCtx] = useState(null); @@ -1510,7 +1518,7 @@ export function ImageAnnotator({ imageUrl, onConfirm, onCancel, onHomeClick, ini {!showSceneFusion && ( )}