From 449dd19d5fa68bf01c86885e983431925cae8827 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 7 Mar 2026 16:59:16 +0800 Subject: [PATCH] feat: add specialist direct executor --- app/(chat)/api/chat/route.ts | 697 +++++++++++++++++++++++++++-- app/api/canvas/box-cutout/route.ts | 166 +------ lib/ai/auto-agent-graph.ts | 54 ++- lib/ai/box-cutout/service.ts | 210 +++++++++ lib/ai/model-router.ts | 151 ++++++- lib/ai/skill-router.ts | 243 ++++++++++ lib/ai/specialists/index.ts | 300 +++++++++++++ lib/ai/tools/generate-video.ts | 212 ++++++--- lib/canvas/generate-image.ts | 2 + 9 files changed, 1746 insertions(+), 289 deletions(-) create mode 100644 lib/ai/box-cutout/service.ts create mode 100644 lib/ai/skill-router.ts create mode 100644 lib/ai/specialists/index.ts diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index 41d72d010..c8ac8a40f 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -20,6 +20,10 @@ import { getUsage } from "tokenlens/helpers"; import { auth, type UserType } from "@/app/(auth)/auth"; import type { VisibilityType } from "@/components/visibility-selector"; import { entitlementsByUserType } from "@/lib/ai/entitlements"; +import { + BoxCutoutServiceError, + executeBoxCutoutJob, +} from "@/lib/ai/box-cutout/service"; import { executeExtendJob, ExtendServiceError, @@ -43,7 +47,10 @@ import { createDocument } from "@/lib/ai/tools/create-document"; import { getWeather } from "@/lib/ai/tools/get-weather"; import { requestSuggestions } from "@/lib/ai/tools/request-suggestions"; import { updateDocument } from "@/lib/ai/tools/update-document"; -import { generateVideo } from "@/lib/ai/tools/generate-video"; +import { + executeDirectVideoGeneration, + generateVideo, +} from "@/lib/ai/tools/generate-video"; import { getEnabledTools } from "@/lib/ai/tools-registry"; import { isProductionEnvironment } from "@/lib/constants"; import { @@ -74,6 +81,7 @@ import { import { db } from "@/lib/db"; import type { DBMessage } from "@/lib/db/schema"; import { generatedImage, message as messageTable } from "@/lib/db/schema"; +import { generateCanvasImage } from "@/lib/canvas/generate-image"; import { resolveChatMediaInput } from "@/lib/ai/context-memory/media-input"; import { resolveTargetAssets } from "@/lib/ai/context-memory/target-resolution"; import type { @@ -784,42 +792,139 @@ type AgentPreparedExecution = { availableTools: string[]; }; -type DirectAgentToolCapability = "remove_background" | "resize" | "extend"; +type DirectAgentToolCapability = + | "remove_background" + | "resize" + | "extend" + | "box_cutout" + | "generate_image" + | "generate_video"; -function resolveDirectAgentToolCapability( +type DirectAgentToolName = + | "removeBackground" + | "resizeImage" + | "extendImage" + | "boxCutout" + | "generateImage" + | "generateVideo"; + +type DirectAgentSourceToolName = + | "useSelectedCanvasImage" + | "useLatestResultImage" + | "useOriginalUploadImage"; + +type ResolvedDirectAgentToolExecution = { + toolName: DirectAgentToolName; + capability: DirectAgentToolCapability; + sourceTools: DirectAgentSourceToolName[]; + publishToCanvas: boolean; +}; + +function collectAgentToolCalls( execution?: AgentPreparedExecution, plan?: AgentExecutionPlan, -): DirectAgentToolCapability | null { - const toolNames = new Set([ - ...(execution?.toolCalls || []).map((tool) => tool.name), - ...(plan?.toolCalls || []).map((tool) => tool.name), - ]); +): AgentPlannedToolCall[] { + const merged = [...(execution?.toolCalls || []), ...(plan?.toolCalls || [])]; + const seen = new Set(); + const result: AgentPlannedToolCall[] = []; - if ( - execution?.preferredCapability === "remove_background" || - execution?.routeSubIntent === "remove_background" || - toolNames.has("removeBackground") - ) { - return "remove_background"; + for (const tool of merged) { + const name = String(tool?.name || "").trim(); + if (!name || seen.has(name)) continue; + seen.add(name); + result.push(tool); } - if ( - execution?.preferredCapability === "resize" || - execution?.routeSubIntent === "resize" || - toolNames.has("resizeImage") - ) { - return "resize"; + return result; +} + +function mapDirectToolNameToCapability( + toolName: DirectAgentToolName, +): DirectAgentToolCapability { + switch (toolName) { + case "removeBackground": + return "remove_background"; + case "resizeImage": + return "resize"; + case "extendImage": + return "extend"; + case "boxCutout": + return "box_cutout"; + case "generateImage": + return "generate_image"; + case "generateVideo": + return "generate_video"; + default: + return "generate_image"; + } +} + +function resolveDirectAgentToolExecution( + execution?: AgentPreparedExecution, + plan?: AgentExecutionPlan, +): ResolvedDirectAgentToolExecution | null { + const mergedToolCalls = collectAgentToolCalls(execution, plan); + const toolNames = mergedToolCalls.map((tool) => tool.name); + const sourceTools = toolNames.filter((toolName): toolName is DirectAgentSourceToolName => + toolName === "useSelectedCanvasImage" || + toolName === "useLatestResultImage" || + toolName === "useOriginalUploadImage", + ); + + const directToolName = + toolNames.find((toolName): toolName is DirectAgentToolName => + toolName === "removeBackground" || + toolName === "resizeImage" || + toolName === "extendImage" || + toolName === "boxCutout" || + toolName === "generateImage" || + toolName === "generateVideo", + ) || + (() => { + if ( + execution?.preferredCapability === "remove_background" || + execution?.routeSubIntent === "remove_background" + ) { + return "removeBackground"; + } + if ( + execution?.preferredCapability === "resize" || + execution?.routeSubIntent === "resize" + ) { + return "resizeImage"; + } + if ( + execution?.preferredCapability === "extend" || + execution?.routeSubIntent === "extend" + ) { + return "extendImage"; + } + if ( + execution?.preferredCapability === "box_cutout" || + execution?.routeSubIntent === "box_cutout" + ) { + return "boxCutout"; + } + if (execution?.mode === "generate_image") { + return "generateImage"; + } + if (execution?.mode === "generate_video") { + return "generateVideo"; + } + return null; + })(); + + if (!directToolName) { + return null; } - if ( - execution?.preferredCapability === "extend" || - execution?.routeSubIntent === "extend" || - toolNames.has("extendImage") - ) { - return "extend"; - } - - return null; + return { + toolName: directToolName, + capability: mapDirectToolNameToCapability(directToolName), + sourceTools, + publishToCanvas: + execution?.shouldPublishToCanvas === true || toolNames.includes("publishToCanvas"), + }; } function getDirectAgentToolLabel(capability: DirectAgentToolCapability): string { @@ -830,6 +935,12 @@ function getDirectAgentToolLabel(capability: DirectAgentToolCapability): string return "Agent工具：尺寸调整"; case "extend": return "Agent工具：图像拓展"; + case "box_cutout": + return "Agent工具：框选抠图"; + case "generate_image": + return "Agent工具：图片生成"; + case "generate_video": + return "Agent工具：视频生成"; default: return "Agent工具"; } @@ -864,6 +975,12 @@ function getDirectAgentToolSuccessMessage(capability: DirectAgentToolCapability) return "已完成尺寸调整，结果如下："; case "extend": return "已完成图像拓展，结果如下："; + case "box_cutout": + return "已完成框选抠图，结果如下："; + case "generate_image": + return "已完成图片生成，结果如下："; + case "generate_video": + return "已完成视频生成，结果如下："; default: return "已完成处理，结果如下："; } @@ -879,6 +996,12 @@ function getDirectAgentToolTargetMissingMessage( return "请先上传或选中一张图片，我才能继续帮你调整尺寸。"; case "extend": return "请先上传或选中一张图片，我才能继续帮你做图像拓展。"; + case "box_cutout": + return "请先选中一张图片并提供框选区域，我才能继续帮你做框选抠图。"; + case "generate_image": + return "请先告诉我你想生成什么，或者提供参考图。"; + case "generate_video": + return "请先告诉我你想生成什么视频，必要时再提供参考图。"; default: return "请先提供要处理的图片。"; } @@ -903,6 +1026,12 @@ function getDirectAgentToolErrorMessage( return `尺寸调整暂时失败：${detail}`; case "extend": return `图像拓展暂时失败：${detail}`; + case "box_cutout": + return `框选抠图暂时失败：${detail}`; + case "generate_image": + return `图片生成暂时失败：${detail}`; + case "generate_video": + return `视频生成暂时失败：${detail}`; default: return `处理暂时失败：${detail}`; } @@ -919,6 +1048,75 @@ function buildAgentToolImageMarkdown(params: { return `![thumb|||${originalUrl}|||${imageSize}](${thumbnailUrl})`; } +function buildAgentToolImagesMarkdown( + items: Array<{ originalUrl: string; thumbnailUrl?: string | null; imageSize?: string }>, +) { + return items + .map((item) => + buildAgentToolImageMarkdown({ + originalUrl: item.originalUrl, + thumbnailUrl: item.thumbnailUrl, + imageSize: item.imageSize, + }), + ) + .join("\n"); +} + +function buildAgentToolVideoMarkdown(videoUrl: string) { + const normalizedUrl = String(videoUrl || "").trim(); + if (!normalizedUrl) return ""; + return [ + ``, + `[下载视频](${normalizedUrl})`, + ].join("\n\n"); +} + +function pickAgentSourceImageUrls(params: { + sourceTools: DirectAgentSourceToolName[]; + requestResolvedMediaInput?: ResolvedChatMediaInput | null; + requestTargetResolution?: ResolvedTargetAssets | null; + requestWorkingState?: { + lastResolvedTargetImageUrls?: string[] | null; + lastGeneratedImageUrls?: string[] | null; + lastUploadedImageUrls?: string[] | null; + } | null; +}): string[] { + const preferredBuckets: string[][] = []; + const sourceTools = params.sourceTools || []; + + if (sourceTools.includes("useSelectedCanvasImage")) { + preferredBuckets.push(params.requestResolvedMediaInput?.selectedImageUrls || []); + } + if (sourceTools.includes("useLatestResultImage")) { + preferredBuckets.push(params.requestWorkingState?.lastGeneratedImageUrls || []); + preferredBuckets.push(params.requestResolvedMediaInput?.historyAssistantImageUrls || []); + } + if (sourceTools.includes("useOriginalUploadImage")) { + preferredBuckets.push(params.requestResolvedMediaInput?.currentMessageImageUrls || []); + preferredBuckets.push(params.requestWorkingState?.lastUploadedImageUrls || []); + preferredBuckets.push(params.requestResolvedMediaInput?.historyUserImageUrls || []); + } + + preferredBuckets.push( + params.requestTargetResolution?.targetImageUrls || [], + params.requestResolvedMediaInput?.selectedImageUrls || [], + params.requestResolvedMediaInput?.currentMessageImageUrls || [], + params.requestWorkingState?.lastResolvedTargetImageUrls || [], + params.requestWorkingState?.lastGeneratedImageUrls || [], + params.requestResolvedMediaInput?.historyAssistantImageUrls || [], + params.requestResolvedMediaInput?.historyUserImageUrls || [], + ); + + return Array.from( + new Set( + preferredBuckets + .flat() + .map((url) => String(url || "").trim()) + .filter((url) => /^https?:\/\//i.test(url)), + ), + ); +} + const IMAGE_FAILOVER_PROVIDER_ALLOWLIST = new Set([ "alibaba-image", "ruxa-image", @@ -3788,22 +3986,395 @@ export async function POST(request: Request) { return createSseResponse(clarificationStream); } - const directAgentToolCapability = shouldRunAgentConversation - ? resolveDirectAgentToolCapability(smartAutoExecution, smartAutoPlan) + const directAgentExecution = shouldRunAgentConversation + ? resolveDirectAgentToolExecution(smartAutoExecution, smartAutoPlan) : null; - const directAgentToolTargetImageUrl = - requestTargetResolution?.targetImageUrls?.[0] || - requestResolvedMediaInput?.selectedImageUrls?.[0] || - requestResolvedMediaInput?.currentMessageImageUrls?.[0] || - requestWorkingState?.lastResolvedTargetImageUrls?.[0] || - requestWorkingState?.lastGeneratedImageUrls?.[0] || - null; + const directAgentToolTargetImageUrls = directAgentExecution + ? pickAgentSourceImageUrls({ + sourceTools: directAgentExecution.sourceTools, + requestResolvedMediaInput, + requestTargetResolution, + requestWorkingState, + }) + : []; + const directAgentToolTargetImageUrl = directAgentToolTargetImageUrls[0] || null; if ( shouldRunAgentConversation && !smartAutoNeedsClarification && - directAgentToolCapability + directAgentExecution?.toolName === "generateImage" ) { + const generationPrompt = + String(smartAutoOptimizedPrompt || requestTextContent || "").trim() || "生成图片"; + const directImageStream = createUIMessageStream({ + execute: async ({ writer }) => { + writeAssistantTextChunk( + writer, + "", + "agent:generate_image:start", + ); + if (contextPayload?.source === "canvas") { + writeImageGenerationStatusChunk( + writer, + { + phase: "processing", + providerType: requestedModelProviderType, + modelName: finalModelId, + imageCount: + effectiveImageParams?.imageCount || + defaultImageQualityParams.imageCount || + 1, + }, + "agent:generate_image:status-processing", + ); + } + + const assistantMessageId = generateUUID(); + try { + const result = await generateCanvasImage({ + prompt: generationPrompt, + negativePrompt: effectiveImageParams?.negativePrompt, + model: finalModelId, + aspectRatio: + effectiveImageParams?.aspectRatio || + defaultImageQualityParams.aspectRatio, + imageSize: + effectiveImageParams?.imageSize || + defaultImageQualityParams.imageSize, + imageCount: + effectiveImageParams?.imageCount || + defaultImageQualityParams.imageCount, + referenceImages: directAgentToolTargetImageUrls, + }); + + const imageMarkdown = buildAgentToolImagesMarkdown( + result.images.map((image) => ({ + originalUrl: image.originalUrl, + thumbnailUrl: image.thumbnailUrl, + imageSize: + effectiveImageParams?.imageSize || + defaultImageQualityParams.imageSize, + })), + ); + const assistantText = prependImageModelBadge( + [ + getDirectAgentToolSuccessMessage("generate_image"), + imageMarkdown, + ] + .filter(Boolean) + .join("\n\n"), + result.model, + ); + + await saveMessages({ + messages: [ + { + id: assistantMessageId, + chatId: id, + role: "assistant", + parts: [{ type: "text", text: assistantText }], + attachments: [], + createdAt: new Date(), + videoStorageStatus: null, + temporaryVideoUrl: null, + permanentVideoUrl: null, + archivedAt: null, + videoMetadata: null, + }, + ], + }); + + await saveGeneratedImages( + assistantText, + session.user.id, + id, + assistantMessageId, + generationPrompt, + result.modelRef || finalModelId, + { + sourceImageUrls: directAgentToolTargetImageUrls, + relationType: "generated_from", + sourceType: + directAgentToolTargetImageUrls.length > 0 + ? "reference_generate" + : "generated", + projectId: contextPayload?.projectId || null, + }, + ); + + writeAssistantTextChunk( + writer, + assistantText, + "agent:generate_image:result", + assistantMessageId, + ); + if (contextPayload?.source === "canvas") { + writeCanvasMediaChunk( + writer, + { + messageId: assistantMessageId, + imageUrls: result.images.map((image) => image.originalUrl), + prompt: generationPrompt, + sourceImageUrls: directAgentToolTargetImageUrls, + }, + "agent:generate_image:canvas-media", + ); + writeImageGenerationStatusChunk( + writer, + { + phase: "completed", + providerType: requestedModelProviderType, + modelName: result.model, + imageCount: result.images.length, + }, + "agent:generate_image:status-completed", + ); + } + writeAssistantTextChunk( + writer, + "", + "agent:generate_image:end", + ); + } catch (directToolError) { + console.error("[Chat API] Direct agent image generation failed:", directToolError); + const errorMessage = getDirectAgentToolErrorMessage( + "generate_image", + directToolError, + ); + await saveMessages({ + messages: [ + { + id: assistantMessageId, + chatId: id, + role: "assistant", + parts: [{ type: "text", text: errorMessage }], + attachments: [], + createdAt: new Date(), + videoStorageStatus: null, + temporaryVideoUrl: null, + permanentVideoUrl: null, + archivedAt: null, + videoMetadata: null, + }, + ], + }); + writeAssistantTextChunk( + writer, + errorMessage, + "agent:generate_image:error", + assistantMessageId, + ); + if (contextPayload?.source === "canvas") { + writeImageGenerationStatusChunk( + writer, + { + phase: "failed", + providerType: requestedModelProviderType, + modelName: finalModelId, + }, + "agent:generate_image:status-failed", + ); + } + writeAssistantTextChunk( + writer, + "", + "agent:generate_image:end-error", + ); + } + }, + }); + + return createSseResponse(directImageStream); + } + + if ( + shouldRunAgentConversation && + !smartAutoNeedsClarification && + directAgentExecution?.toolName === "generateVideo" + ) { + const generationPrompt = + String(smartAutoOptimizedPrompt || requestTextContent || "").trim() || "生成视频"; + const directVideoStream = createUIMessageStream({ + execute: async ({ writer }) => { + const assistantMessageId = generateUUID(); + try { + const plannedArgs = findAgentToolCallArgs( + "generateVideo", + smartAutoExecution, + smartAutoPlan, + ); + const requestedMode = String( + plannedArgs?.mode || effectiveVideoParams?.mode || "", + ).trim() as + | "text-to-video" + | "first-frame" + | "first-last-frame" + | "reference-images"; + const mode = + requestedMode || + (directAgentToolTargetImageUrls.length >= 2 + ? "first-last-frame" + : directAgentToolTargetImageUrls.length >= 1 + ? "first-frame" + : "text-to-video"); + const images = + mode === "first-last-frame" + ? directAgentToolTargetImageUrls.slice(0, 2) + : mode === "first-frame" + ? directAgentToolTargetImageUrls.slice(0, 1) + : mode === "reference-images" + ? directAgentToolTargetImageUrls.slice(0, 4) + : []; + const directVideoRatio = + effectiveVideoParams?.ratio === "16:9" || + effectiveVideoParams?.ratio === "4:3" || + effectiveVideoParams?.ratio === "1:1" || + effectiveVideoParams?.ratio === "3:4" || + effectiveVideoParams?.ratio === "9:16" || + effectiveVideoParams?.ratio === "21:9" || + effectiveVideoParams?.ratio === "adaptive" + ? effectiveVideoParams.ratio + : undefined; + const result = await executeDirectVideoGeneration({ + prompt: generationPrompt, + mode, + images, + resolution: effectiveVideoParams?.resolution, + ratio: directVideoRatio, + duration: effectiveVideoParams?.duration, + preferredModelRef: finalModelId, + }); + + const videoUrl = String(result.videoUrl || "").trim(); + if (!videoUrl) { + throw new Error("视频生成完成但未返回视频地址"); + } + + const assistantText = prependImageModelBadge( + [ + getDirectAgentToolSuccessMessage("generate_video"), + buildAgentToolVideoMarkdown(videoUrl), + ] + .filter(Boolean) + .join("\n\n"), + result.modelDisplayName, + ); + + await saveMessages({ + messages: [ + { + id: assistantMessageId, + chatId: id, + role: "assistant", + parts: [{ type: "text", text: assistantText }], + attachments: [], + createdAt: new Date(), + videoStorageStatus: "temporary", + temporaryVideoUrl: videoUrl, + permanentVideoUrl: null, + archivedAt: null, + videoMetadata: (result.metadata || {}) as any, + }, + ], + }); + + try { + await upsertConversationWorkingState({ + userId: session.user.id, + chatId: id, + projectId: contextPayload?.projectId || null, + currentIntent: "video_gen", + lastMessageId: assistantMessageId, + }); + } catch (memoryError) { + console.warn( + "[Chat API] Failed to update working state after direct video generation:", + memoryError, + ); + } + + await consumeGeneratedVideoPoints({ + userId: session.user.id, + chatId: id, + messageId: assistantMessageId, + modelRefs: [result.modelRef || finalModelId], + }); + + if (videoUrl) { + scheduleVideoArchive({ + chatId: id, + messageId: assistantMessageId, + text: assistantText, + temporaryVideoUrl: videoUrl, + taskId: assistantMessageId, + metadata: result.metadata, + }); + } + + writeAssistantTextChunk( + writer, + assistantText, + "agent:generate_video:result", + assistantMessageId, + ); + if (contextPayload?.source === "canvas") { + writeCanvasMediaChunk( + writer, + { + messageId: assistantMessageId, + videoUrls: [videoUrl], + prompt: generationPrompt, + sourceImageUrls: directAgentToolTargetImageUrls, + }, + "agent:generate_video:canvas-media", + ); + } + } catch (directToolError) { + console.error("[Chat API] Direct agent video generation failed:", directToolError); + const errorMessage = getDirectAgentToolErrorMessage( + "generate_video", + directToolError, + ); + await saveMessages({ + messages: [ + { + id: assistantMessageId, + chatId: id, + role: "assistant", + parts: [{ type: "text", text: errorMessage }], + attachments: [], + createdAt: new Date(), + videoStorageStatus: null, + temporaryVideoUrl: null, + permanentVideoUrl: null, + archivedAt: null, + videoMetadata: null, + }, + ], + }); + writeAssistantTextChunk( + writer, + errorMessage, + "agent:generate_video:error", + assistantMessageId, + ); + } + }, + }); + + return createSseResponse(directVideoStream); + } + + if ( + shouldRunAgentConversation && + !smartAutoNeedsClarification && + directAgentExecution && + (directAgentExecution.toolName === "removeBackground" || + directAgentExecution.toolName === "resizeImage" || + directAgentExecution.toolName === "extendImage" || + directAgentExecution.toolName === "boxCutout") + ) { + const directAgentToolCapability = directAgentExecution.capability; if (!directAgentToolTargetImageUrl) { const missingTargetMessage = getDirectAgentToolTargetMissingMessage( directAgentToolCapability, @@ -3954,6 +4525,47 @@ export async function POST(request: Request) { resultThumbnailUrl = result.thumbnail_url; resultImageSize = "2K"; resultAssetId = result.mediaAssetId || null; + } else if (directAgentToolCapability === "box_cutout") { + const plannedArgs = findAgentToolCallArgs( + "boxCutout", + smartAutoExecution, + smartAutoPlan, + ); + const cropRect = + plannedArgs?.cropRect && typeof plannedArgs.cropRect === "object" + ? (plannedArgs.cropRect as Record) + : undefined; + + if (!cropRect) { + throw new BoxCutoutServiceError( + "缺少框选区域，暂时无法执行框选抠图。", + 400, + ); + } + + const result = await executeBoxCutoutJob({ + userId: session.user.id, + imageUrl: directAgentToolTargetImageUrl, + cropRect: { + x: toFiniteNumber(cropRect.x), + y: toFiniteNumber(cropRect.y), + width: toFiniteNumber(cropRect.width), + height: toFiniteNumber(cropRect.height), + }, + paddingRatio: toFiniteNumber(plannedArgs?.paddingRatio), + prompt: requestTextContent || "框选抠图处理", + chatId: id, + messageId: assistantMessageId, + sourceNodeId: + typeof plannedArgs?.sourceNodeId === "string" + ? plannedArgs.sourceNodeId + : undefined, + }); + + resultOriginalUrl = result.original_url; + resultThumbnailUrl = result.thumbnail_url || null; + resultImageSize = "2K"; + resultAssetId = result.mediaAssetId || null; } const assistantText = prependImageModelBadge( @@ -4000,7 +4612,7 @@ export async function POST(request: Request) { ); } - if (smartAutoExecution?.shouldPublishToCanvas && contextPayload?.source === "canvas") { + if (directAgentExecution.publishToCanvas && contextPayload?.source === "canvas") { console.log( `[Chat API] Agent publishToCanvas satisfied by assistant media auto-sync (${directAgentToolCapability})`, ); @@ -4021,7 +4633,7 @@ export async function POST(request: Request) { messageId: assistantMessageId, imageUrls: resultOriginalUrl ? [resultOriginalUrl] : [], prompt: requestTextContent || getDirectAgentToolLabel(directAgentToolCapability), - sourceImageUrls: requestTargetResolution?.targetImageUrls || [], + sourceImageUrls: directAgentToolTargetImageUrls, }, `agent:${directAgentToolCapability}:canvas-media`, ); @@ -4044,7 +4656,8 @@ export async function POST(request: Request) { if ( !(directToolError instanceof RmbgServiceError) && !(directToolError instanceof ResizeServiceError) && - !(directToolError instanceof ExtendServiceError) + !(directToolError instanceof ExtendServiceError) && + !(directToolError instanceof BoxCutoutServiceError) ) { console.warn( `[Chat API] Direct agent tool ${directAgentToolCapability} threw unexpected error type`, diff --git a/app/api/canvas/box-cutout/route.ts b/app/api/canvas/box-cutout/route.ts index 84c16a2f2..5201fbdc8 100644 --- a/app/api/canvas/box-cutout/route.ts +++ b/app/api/canvas/box-cutout/route.ts @@ -1,26 +1,23 @@ import { NextRequest, NextResponse } from "next/server"; -import sharp from "sharp"; import { auth } from "@/app/(auth)/auth"; import { - executeRmbgJob, - fetchImageBufferFromUrl, - RmbgServiceError, -} from "@/lib/ai/rmbg/service"; + BoxCutoutServiceError, + executeBoxCutoutJob, +} from "@/lib/ai/box-cutout/service"; +import { RmbgServiceError } from "@/lib/ai/rmbg/service"; export const runtime = "nodejs"; export const maxDuration = 60; -type NormalizedCropRect = { - x: number; - y: number; - width: number; - height: number; -}; - type BoxCutoutRequest = { imageUrl?: string; - cropRect?: Partial; + cropRect?: { + x?: number; + y?: number; + width?: number; + height?: number; + }; paddingRatio?: number; prompt?: string; chatId?: string | null; @@ -29,85 +26,6 @@ type BoxCutoutRequest = { sourceNodeId?: string; }; -const MIN_CROP_EDGE_PX = 24; -const DEFAULT_PADDING_RATIO = 0.08; - -function clamp(value: number, min: number, max: number) { - return Math.min(max, Math.max(min, value)); -} - -function sanitizeNormalizedCropRect(input: Partial | undefined): NormalizedCropRect | null { - if (!input) return null; - - const x = Number(input.x); - const y = Number(input.y); - const width = Number(input.width); - const height = Number(input.height); - - if (![x, y, width, height].every((value) => Number.isFinite(value))) { - return null; - } - - const normalized: NormalizedCropRect = { - x: clamp(x, 0, 1), - y: clamp(y, 0, 1), - width: clamp(width, 0, 1), - height: clamp(height, 0, 1), - }; - - if (normalized.width <= 0 || normalized.height <= 0) { - return null; - } - - normalized.width = Math.min(normalized.width, 1 - normalized.x); - normalized.height = Math.min(normalized.height, 1 - normalized.y); - - if (normalized.width <= 0 || normalized.height <= 0) { - return null; - } - - return normalized; -} - -function normalizePaddingRatio(value: unknown) { - const numeric = Number(value); - if (!Number.isFinite(numeric)) return DEFAULT_PADDING_RATIO; - return clamp(numeric, 0, 0.25); -} - -function toPixelRect( - rect: NormalizedCropRect, - sourceWidth: number, - sourceHeight: number -) { - const left = clamp(Math.round(rect.x * sourceWidth), 0, Math.max(sourceWidth - 1, 0)); - const top = clamp(Math.round(rect.y * sourceHeight), 0, Math.max(sourceHeight - 1, 0)); - const width = clamp(Math.round(rect.width * sourceWidth), 1, sourceWidth - left); - const height = clamp(Math.round(rect.height * sourceHeight), 1, sourceHeight - top); - - return { left, top, width, height }; -} - -function applyPadding( - rect: NormalizedCropRect, - paddingRatio: number -): NormalizedCropRect { - const padX = rect.width * paddingRatio; - const padY = rect.height * paddingRatio; - - const x = clamp(rect.x - padX, 0, 1); - const y = clamp(rect.y - padY, 0, 1); - const right = clamp(rect.x + rect.width + padX, 0, 1); - const bottom = clamp(rect.y + rect.height + padY, 0, 1); - - return { - x, - y, - width: clamp(right - x, 0, 1), - height: clamp(bottom - y, 0, 1), - }; -} - export async function POST(request: NextRequest) { try { const session = await auth(); @@ -117,78 +35,32 @@ export async function POST(request: NextRequest) { const body = (await request.json()) as BoxCutoutRequest; const rawImageUrl = String(body.imageUrl || "").trim(); - const cropRect = sanitizeNormalizedCropRect(body.cropRect); - if (!rawImageUrl) { return NextResponse.json({ success: false, error: "缺少 imageUrl 参数" }, { status: 400 }); } - if (!cropRect) { - return NextResponse.json({ success: false, error: "缺少有效的 cropRect 参数" }, { status: 400 }); - } - const resolvedImageUrl = new URL(rawImageUrl, request.url).toString(); - const paddingRatio = normalizePaddingRatio(body.paddingRatio); - - const sourceBuffer = await sharp(await fetchImageBufferFromUrl(resolvedImageUrl), { failOn: "none" }) - .rotate() - .toBuffer(); - const sourceImage = sharp(sourceBuffer, { failOn: "none" }); - const sourceMetadata = await sourceImage.metadata(); - const sourceWidth = Number(sourceMetadata.width || 0); - const sourceHeight = Number(sourceMetadata.height || 0); - - if (sourceWidth <= 0 || sourceHeight <= 0) { - return NextResponse.json({ success: false, error: "无法识别原图尺寸" }, { status: 400 }); - } - - const cropRectPixels = toPixelRect(cropRect, sourceWidth, sourceHeight); - if (cropRectPixels.width < MIN_CROP_EDGE_PX || cropRectPixels.height < MIN_CROP_EDGE_PX) { - return NextResponse.json( - { success: false, error: "框选区域过小，请重新框选" }, - { status: 400 } - ); - } - - const paddedCropRect = applyPadding(cropRect, paddingRatio); - const paddedCropRectPixels = toPixelRect(paddedCropRect, sourceWidth, sourceHeight); - const croppedBuffer = await sourceImage - .extract({ - left: paddedCropRectPixels.left, - top: paddedCropRectPixels.top, - width: paddedCropRectPixels.width, - height: paddedCropRectPixels.height, - }) - .toBuffer(); - - const rmbgResult = await executeRmbgJob({ + const result = await executeBoxCutoutJob({ userId: session.user.id, - imageBuffer: croppedBuffer, - imageBase64: croppedBuffer.toString("base64"), - sourceImageUrl: resolvedImageUrl, + imageUrl: resolvedImageUrl, + cropRect: body.cropRect || {}, + paddingRatio: body.paddingRatio, prompt: body.prompt || "框选抠图", chatId: body.chatId || null, messageId: body.messageId || null, skipGallerySave: body.skipGallerySave === true, + sourceNodeId: body.sourceNodeId, }); - return NextResponse.json({ - success: true, - ...rmbgResult, - cropRectApplied: cropRect, - paddedCropRectApplied: paddedCropRect, - cropRectPixels, - paddedCropRectPixels, - sourceNodeId: body.sourceNodeId || null, - }); + return NextResponse.json(result); } catch (error) { - if (error instanceof RmbgServiceError) { + if (error instanceof BoxCutoutServiceError || error instanceof RmbgServiceError) { return NextResponse.json( { success: false, error: error.detail || error.message, }, - { status: error.status } + { status: error.status }, ); } @@ -198,7 +70,7 @@ export async function POST(request: NextRequest) { success: false, error: error instanceof Error ? error.message : "框选抠图失败", }, - { status: 500 } + { status: 500 }, ); } } diff --git a/lib/ai/auto-agent-graph.ts b/lib/ai/auto-agent-graph.ts index d8452d8c6..d8fb4ad44 100644 --- a/lib/ai/auto-agent-graph.ts +++ b/lib/ai/auto-agent-graph.ts @@ -3,6 +3,7 @@ import { generateText } from "ai"; import sharp from "sharp"; import type { AnalysisResult } from "@/lib/ai/smart-analyzer"; import type { RouteResult } from "@/lib/ai/model-router"; +import type { AgentSkillRouteResult } from "@/lib/ai/skill-router"; import { getEnabledAgentToolDefinitionsForContext, type AgentProjectToolDefinition, @@ -1210,6 +1211,7 @@ async function buildExecutionPlanWithAgent( state: AutoAgentStateType, analysis: AnalysisResult, availableTools: AgentProjectToolDefinition[], + skillRoute?: AgentSkillRouteResult, ): Promise { const fallbackPlan = buildFallbackPlan({ analysis, @@ -1275,6 +1277,7 @@ async function buildExecutionPlanWithAgent( 5. shouldUseStructuredReply 只在 edit_image / generate_image / generate_video 时设为 true。 6. targetSource 要忠实反映当前目标图来源：explicit_selection/current_upload/text_reference/working_memory/latest_result/clarification/none。 7. preferredCapability 要尽量具体，例如 remove_background / enhance / extend / resize / box_cutout / publish_canvas。 +8. 必须遵守当前专家路由约束；如果约束要求纯对话，则绝不能输出 generate_image / edit_image / generate_video。 输出格式： { @@ -1306,6 +1309,12 @@ async function buildExecutionPlanWithAgent( `历史图数量：${state.historyImageCount}`, `画布选中图数量：${state.selectedImageCount}`, `基础分析：intent=${analysis.intent}; subIntent=${analysis.subIntent || "none"}; confidence=${analysis.confidence}`, + skillRoute + ? `专家路由：${skillRoute.primary.specialistId}｜原因：${skillRoute.reason}` + : "", + skillRoute?.planningHints?.length + ? `专家约束：\n${skillRoute.planningHints.map((item, index) => `${index + 1}. ${item}`).join("\n")}` + : "", state.context ? `画布上下文：${compactText(state.context, 600)}` : "", historyContext ? `最近对话：\n${historyContext}` : "", `可用工具：\n${toolCatalog}`, @@ -1318,7 +1327,23 @@ async function buildExecutionPlanWithAgent( } as any); const parsed = parseJsonBlock>(text); - const planned = sanitizeAgentPlan(parsed, fallbackPlan, availableTools); + let planned = sanitizeAgentPlan(parsed, fallbackPlan, availableTools); + if (skillRoute) { + const { refinePlanWithSpecialist } = await import("./specialists"); + for (const specialist of [skillRoute.primary, ...skillRoute.overlays]) { + planned = refinePlanWithSpecialist(specialist, { + text: state.text, + analysis, + plan: planned, + availableTools, + hasImage: state.hasImage, + hasCanvasSelection: state.hasCanvasSelection || state.selectedImageCount > 0, + targetResolutionMode: state.targetResolutionMode, + mediaSource: state.mediaSource, + selectedImageCount: state.selectedImageCount, + }); + } + } console.log( `[Auto Agent] Planner model: ${plannerModelConfig.model.displayName} (${plannerModelConfig.provider.type})`, ); @@ -1766,13 +1791,36 @@ const analyzeNode = async (state: AutoAgentStateType) => { const planNode = async (state: AutoAgentStateType) => { if (!state.analysis) return {}; - const availableTools = await getEnabledAgentToolDefinitionsForContext({ + const baseAvailableTools = await getEnabledAgentToolDefinitionsForContext({ hasImage: state.hasImage, hasCanvasSelection: state.hasCanvasSelection || state.selectedImageCount > 0, hasHistoryImage: state.historyImageCount > 0 || state.imageFromHistory, hasCurrentUpload: state.currentUploadImageCount > 0, }); - const plan = await buildExecutionPlanWithAgent(state, state.analysis, availableTools); + const { filterToolsForSkillRoute, routeAgentSkill } = await import("./skill-router"); + const skillRoute = routeAgentSkill({ + text: state.text, + analysis: state.analysis, + hasImage: state.hasImage, + hasCanvasSelection: state.hasCanvasSelection || state.selectedImageCount > 0, + targetResolutionMode: state.targetResolutionMode, + mediaSource: state.mediaSource, + selectedImageCount: state.selectedImageCount, + }); + const availableTools = filterToolsForSkillRoute(baseAvailableTools, skillRoute); + console.log("[Auto Agent] Skill route:", { + specialist: skillRoute.primary.specialistId, + domain: skillRoute.primary.domain, + overlays: skillRoute.overlays.map((item) => item.specialistId), + allowlist: skillRoute.toolAllowlist, + reason: skillRoute.reason, + }); + const plan = await buildExecutionPlanWithAgent( + state, + state.analysis, + availableTools, + skillRoute, + ); const alignedAnalysis = alignAnalysisWithPlan(state.analysis, plan); return { diff --git a/lib/ai/box-cutout/service.ts b/lib/ai/box-cutout/service.ts new file mode 100644 index 000000000..2c7888821 --- /dev/null +++ b/lib/ai/box-cutout/service.ts @@ -0,0 +1,210 @@ +import sharp from "sharp"; + +import { + executeRmbgJob, + fetchImageBufferFromUrl, +} from "@/lib/ai/rmbg/service"; + +export type NormalizedCropRect = { + x: number; + y: number; + width: number; + height: number; +}; + +export interface ExecuteBoxCutoutJobParams { + userId: string; + imageUrl: string; + cropRect: Partial; + paddingRatio?: number; + prompt?: string; + chatId?: string | null; + messageId?: string | null; + skipGallerySave?: boolean; + sourceNodeId?: string; +} + +export interface ExecuteBoxCutoutJobResult { + success: true; + original_url: string; + thumbnail_url?: string | null; + generatedImageId?: string | null; + mediaAssetId?: string | null; + cropRectApplied: NormalizedCropRect; + paddedCropRectApplied: NormalizedCropRect; + cropRectPixels: { + left: number; + top: number; + width: number; + height: number; + }; + paddedCropRectPixels: { + left: number; + top: number; + width: number; + height: number; + }; + sourceNodeId?: string | null; +} + +export class BoxCutoutServiceError extends Error { + status: number; + detail?: string; + + constructor(message: string, status = 400, detail?: string) { + super(message); + this.name = "BoxCutoutServiceError"; + this.status = status; + this.detail = detail; + } +} + +const MIN_CROP_EDGE_PX = 24; +const DEFAULT_PADDING_RATIO = 0.08; + +function clamp(value: number, min: number, max: number) { + return Math.min(max, Math.max(min, value)); +} + +export function sanitizeNormalizedCropRect( + input: Partial | undefined, +): NormalizedCropRect | null { + if (!input) return null; + + const x = Number(input.x); + const y = Number(input.y); + const width = Number(input.width); + const height = Number(input.height); + + if (![x, y, width, height].every((value) => Number.isFinite(value))) { + return null; + } + + const normalized: NormalizedCropRect = { + x: clamp(x, 0, 1), + y: clamp(y, 0, 1), + width: clamp(width, 0, 1), + height: clamp(height, 0, 1), + }; + + if (normalized.width <= 0 || normalized.height <= 0) { + return null; + } + + normalized.width = Math.min(normalized.width, 1 - normalized.x); + normalized.height = Math.min(normalized.height, 1 - normalized.y); + + if (normalized.width <= 0 || normalized.height <= 0) { + return null; + } + + return normalized; +} + +export function normalizeBoxCutoutPaddingRatio(value: unknown) { + const numeric = Number(value); + if (!Number.isFinite(numeric)) return DEFAULT_PADDING_RATIO; + return clamp(numeric, 0, 0.25); +} + +function toPixelRect( + rect: NormalizedCropRect, + sourceWidth: number, + sourceHeight: number, +) { + const left = clamp(Math.round(rect.x * sourceWidth), 0, Math.max(sourceWidth - 1, 0)); + const top = clamp(Math.round(rect.y * sourceHeight), 0, Math.max(sourceHeight - 1, 0)); + const width = clamp(Math.round(rect.width * sourceWidth), 1, sourceWidth - left); + const height = clamp(Math.round(rect.height * sourceHeight), 1, sourceHeight - top); + + return { left, top, width, height }; +} + +function applyPadding( + rect: NormalizedCropRect, + paddingRatio: number, +): NormalizedCropRect { + const padX = rect.width * paddingRatio; + const padY = rect.height * paddingRatio; + + const x = clamp(rect.x - padX, 0, 1); + const y = clamp(rect.y - padY, 0, 1); + const right = clamp(rect.x + rect.width + padX, 0, 1); + const bottom = clamp(rect.y + rect.height + padY, 0, 1); + + return { + x, + y, + width: clamp(right - x, 0, 1), + height: clamp(bottom - y, 0, 1), + }; +} + +export async function executeBoxCutoutJob( + params: ExecuteBoxCutoutJobParams, +): Promise { + const imageUrl = String(params.imageUrl || "").trim(); + if (!imageUrl) { + throw new BoxCutoutServiceError("缺少 imageUrl 参数", 400); + } + + const cropRect = sanitizeNormalizedCropRect(params.cropRect); + if (!cropRect) { + throw new BoxCutoutServiceError("缺少有效的 cropRect 参数", 400); + } + + const paddingRatio = normalizeBoxCutoutPaddingRatio(params.paddingRatio); + const sourceBuffer = await sharp(await fetchImageBufferFromUrl(imageUrl), { + failOn: "none", + }) + .rotate() + .toBuffer(); + const sourceImage = sharp(sourceBuffer, { failOn: "none" }); + const sourceMetadata = await sourceImage.metadata(); + const sourceWidth = Number(sourceMetadata.width || 0); + const sourceHeight = Number(sourceMetadata.height || 0); + + if (sourceWidth <= 0 || sourceHeight <= 0) { + throw new BoxCutoutServiceError("无法识别原图尺寸", 400); + } + + const cropRectPixels = toPixelRect(cropRect, sourceWidth, sourceHeight); + if (cropRectPixels.width < MIN_CROP_EDGE_PX || cropRectPixels.height < MIN_CROP_EDGE_PX) { + throw new BoxCutoutServiceError("框选区域过小，请重新框选", 400); + } + + const paddedCropRect = applyPadding(cropRect, paddingRatio); + const paddedCropRectPixels = toPixelRect(paddedCropRect, sourceWidth, sourceHeight); + const croppedBuffer = await sourceImage + .extract({ + left: paddedCropRectPixels.left, + top: paddedCropRectPixels.top, + width: paddedCropRectPixels.width, + height: paddedCropRectPixels.height, + }) + .toBuffer(); + + const result = await executeRmbgJob({ + userId: params.userId, + imageBuffer: croppedBuffer, + imageBase64: croppedBuffer.toString("base64"), + sourceImageUrl: imageUrl, + prompt: params.prompt || "框选抠图", + chatId: params.chatId || null, + messageId: params.messageId || null, + skipGallerySave: params.skipGallerySave === true, + }); + + return { + success: true, + original_url: result.original_url, + thumbnail_url: result.thumbnail_url, + generatedImageId: result.generatedImageId, + mediaAssetId: result.mediaAssetId, + cropRectApplied: cropRect, + paddedCropRectApplied: paddedCropRect, + cropRectPixels, + paddedCropRectPixels, + sourceNodeId: params.sourceNodeId || null, + }; +} diff --git a/lib/ai/model-router.ts b/lib/ai/model-router.ts index 8b9cdfb51..67d3024b8 100644 --- a/lib/ai/model-router.ts +++ b/lib/ai/model-router.ts @@ -7,6 +7,10 @@ import { isAiModelMultimodal } from "@/lib/db/ai-model-flags"; +type EnabledModelWithProvider = Awaited< + ReturnType +>[number]; + // 使用动态导入避免循环依赖 // import { getEnabledAiModelsWithProvider } from "@/lib/db/queries"; @@ -27,6 +31,94 @@ function hasVisionCapability(model: { return Boolean(model.supportsVision) || isAiModelMultimodal(model); } +const CHAT_PROVIDER_TYPES = [ + "alibaba", + "zhipu", + "openai", + "custom", + "local", + "gemini", + "huanke-gemini", + "volcano-multimodal", +] as const; + +const VIDEO_PROVIDER_TYPES = new Set([ + "doubao-video", + "volcano-video", + "qianwen-video", + "fal-video", + "ruxa-video", +]); + +const IMAGE_ONLY_PROVIDER_TYPES = new Set([ + "alibaba-image", + "ruxa-image", + "fal-image", + "openai-image", + "midjourney", + "gemini-image", +]); + +function isVideoProviderType(providerType: string) { + return VIDEO_PROVIDER_TYPES.has(providerType); +} + +function isConversationalCandidate( + item: EnabledModelWithProvider, + options?: { requireVision?: boolean }, +) { + const providerType = String(item.provider.type || ""); + if (!item.model.isEnabled || item.model.isSubModel) return false; + if (isVideoProviderType(providerType)) return false; + + const requireVision = Boolean(options?.requireVision); + const matchesPreferredProvider = CHAT_PROVIDER_TYPES.includes( + providerType as (typeof CHAT_PROVIDER_TYPES)[number], + ); + const hasVision = hasVisionCapability(item.model); + + if (matchesPreferredProvider) { + return requireVision ? hasVision || providerType === "volcano-multimodal" : true; + } + + if (IMAGE_ONLY_PROVIDER_TYPES.has(providerType)) { + return false; + } + + if (requireVision) { + return hasVision; + } + + return !item.model.supportsImageGen; +} + +function matchesIntentRequirements( + item: EnabledModelWithProvider, + params: { + intent: string; + requireVision?: boolean; + requireImageGen?: boolean; + }, +) { + if (!item.model.isEnabled || item.model.isSubModel) return false; + + if (params.intent === "chat") { + return isConversationalCandidate(item); + } + + if (params.intent === "vision") { + return ( + !isVideoProviderType(String(item.provider.type || "")) && + hasVisionCapability(item.model) + ); + } + + if (params.requireVision && !hasVisionCapability(item.model)) return false; + if (params.requireImageGen && !item.model.supportsImageGen) return false; + + return true; +} + // 意图到模型的映射配置 // providerTypes 按优先级排序，第一个是首选 const INTENT_MODEL_MAP: Record< @@ -48,7 +140,7 @@ const INTENT_MODEL_MAP: Record< // 视觉理解：优先千问3 VL（成本低效果好） vision: { - providerTypes: ["alibaba", "gemini", "huanke-gemini"], + providerTypes: ["alibaba", "zhipu", "openai", "custom", "gemini", "huanke-gemini", "volcano-multimodal"], requireVision: true, description: "图片理解", }, @@ -68,7 +160,7 @@ const INTENT_MODEL_MAP: Record< // 普通对话：优先千问3 chat: { - providerTypes: ["alibaba", "openai", "local"], + providerTypes: ["alibaba", "zhipu", "openai", "custom", "local", "gemini", "huanke-gemini", "volcano-multimodal"], description: "对话", }, }; @@ -147,7 +239,7 @@ export async function routeToModel(params: { const [settings] = await db.select().from(siteSettings).limit(1); const configuredDefaultModelId = settings?.canvasAutoModeDefaultModelId; - if (configuredDefaultModelId) { + if (configuredDefaultModelId && intent !== "chat") { const configuredModel = allModels.find( (m) => m.model.id === configuredDefaultModelId && m.model.isEnabled && !m.model.isSubModel ); @@ -182,6 +274,8 @@ export async function routeToModel(params: { } else { console.log("[Model Router] ⚠️ Configured default model not found or disabled, falling back to intent routing"); } + } else if (configuredDefaultModelId && intent === "chat") { + console.log("[Model Router] ℹ️ Skip configured default image model for chat intent"); } // 按优先级查找匹配的模型 @@ -190,17 +284,11 @@ export async function routeToModel(params: { // 匹配 provider 类型 if (m.provider.type !== providerType) return false; - // 检查能力要求 - if (config.requireVision && !hasVisionCapability(m.model)) return false; - if (config.requireImageGen && !m.model.supportsImageGen) return false; - - // 排除子模型（只选主模型） - if (m.model.isSubModel) return false; - - // 必须启用 - if (!m.model.isEnabled) return false; - - return true; + return matchesIntentRequirements(m, { + intent, + requireVision: config.requireVision, + requireImageGen: config.requireImageGen, + }); }); if (candidates.length > 0) { @@ -223,17 +311,24 @@ export async function routeToModel(params: { } } - // 兜底逻辑：返回任意可用的图片编辑模型 - const fallbackCandidates = allModels.filter( - (m) => - !m.model.isSubModel && - m.model.isEnabled && - (m.model.supportsImageGen || m.model.supportsVision) - ); + const fallbackCandidates = allModels.filter((item) => { + if (intent === "chat") { + return isConversationalCandidate(item, { requireVision: false }); + } + if (intent === "vision") { + return isConversationalCandidate(item, { requireVision: true }); + } + return ( + !item.model.isSubModel && + item.model.isEnabled && + !isVideoProviderType(String(item.provider.type || "")) && + (item.model.supportsImageGen || hasVisionCapability(item.model)) + ); + }); if (fallbackCandidates.length > 0) { const fallback = fallbackCandidates[0]; - console.log("[Model Router] Using fallback (with image capability):", fallback.model.displayName); + console.log("[Model Router] Using intent-safe fallback:", fallback.model.displayName); return { id: fallback.model.id, displayName: fallback.model.displayName, @@ -244,7 +339,17 @@ export async function routeToModel(params: { } // 最终兜底：返回任意可用模型 - const anyModel = allModels.find((m) => !m.model.isSubModel && m.model.isEnabled); + const anyModel = + (intent === "chat" + ? allModels.find( + (m) => + !m.model.isSubModel && + m.model.isEnabled && + !isVideoProviderType(String(m.provider.type || "")) && + !m.model.supportsImageGen, + ) + : undefined) || + allModels.find((m) => !m.model.isSubModel && m.model.isEnabled); if (anyModel) { console.log("[Model Router] Using any available model:", anyModel.model.displayName); return { diff --git a/lib/ai/skill-router.ts b/lib/ai/skill-router.ts new file mode 100644 index 000000000..d9f49e0ed --- /dev/null +++ b/lib/ai/skill-router.ts @@ -0,0 +1,243 @@ +import type { + AgentProjectToolDefinition, + AgentProjectToolName, +} from "@/lib/ai/tools-registry"; +import type { + AgentSpecialistContext, + AgentSpecialistRoute, +} from "@/lib/ai/specialists"; + +export interface AgentSkillRouteResult { + primary: AgentSpecialistRoute; + overlays: AgentSpecialistRoute[]; + toolAllowlist: AgentProjectToolName[]; + planningHints: string[]; + reason: string; +} + +function normalizeCompactText(text: string) { + return String(text || "").toLowerCase().replace(/\s+/g, "").trim(); +} + +function dedupeToolNames(toolNames: AgentProjectToolName[]) { + return Array.from(new Set(toolNames)); +} + +function isPureGreeting(text: string) { + const normalized = normalizeCompactText(text); + if (!normalized) return false; + return /^(你好|您好|hello|hi|嗨|哈喽|在吗|早上好|中午好|下午好|晚上好|深夜好|谢谢|感谢|收到|好的|ok|okay|明白了|了解了)$/.test( + normalized, + ); +} + +function hasCanvasContext(context: AgentSpecialistContext) { + return ( + context.hasCanvasSelection || + (context.selectedImageCount || 0) > 0 || + context.mediaSource === "canvas_selection" || + context.targetResolutionMode === "explicit_selection" + ); +} + +function mentionsCanvas(text: string) { + return /画布|发布到画布|同步到画布|发送到画布|放到画布/.test( + String(text || "").toLowerCase(), + ); +} + +function createRoute( + route: AgentSpecialistRoute, + toolAllowlist: AgentProjectToolName[], + planningHints: string[], +): AgentSpecialistRoute { + return { + ...route, + toolAllowlist: dedupeToolNames(toolAllowlist), + planningHints, + }; +} + +export function routeAgentSkill( + context: AgentSpecialistContext, +): AgentSkillRouteResult { + const greetingTurn = isPureGreeting(context.text); + + const conversationRoute = createRoute( + { + domain: "conversation", + specialistId: "conversation-coordinator", + reason: greetingTurn ? "纯打招呼/寒暄" : "普通对话回合", + planningHints: [], + }, + ["respondDirect"], + [ + "当前回合是纯对话，不要调用生图、生视频或修图工具。", + "即使当前有图片，也只能先自然回复，不能直接执行。", + ], + ); + + const visionRoute = createRoute( + { + domain: "vision", + specialistId: "vision-analyst", + reason: "当前回合以看图分析为主", + planningHints: [], + }, + [ + "analyzeImage", + "respondDirect", + "useSelectedCanvasImage", + "useLatestResultImage", + "useOriginalUploadImage", + ], + [ + "当前回合重点是识图、分析、解释或给建议，不要误切到生图。", + "如有目标图，优先围绕目标图做分析，再给用户结论。", + ], + ); + + const imageEditRoute = createRoute( + { + domain: "image_edit", + specialistId: "image-edit-specialist", + reason: "当前回合是修图/抠图/增强类任务", + planningHints: [], + }, + [ + "editImage", + "removeBackground", + "enhanceImage", + "extendImage", + "resizeImage", + "boxCutout", + "useSelectedCanvasImage", + "useLatestResultImage", + "useOriginalUploadImage", + "publishToCanvas", + ], + [ + "当前回合必须收敛到修图链路，优先选择最具体的修图工具。", + "除非用户明确要求生成新图，否则不要把修图误判成生图。", + ], + ); + + const imageGenerateRoute = createRoute( + { + domain: "image_generate", + specialistId: "image-generate-specialist", + reason: "当前回合是生图任务", + planningHints: [], + }, + [ + "generateImage", + "useSelectedCanvasImage", + "useLatestResultImage", + "useOriginalUploadImage", + "publishToCanvas", + ], + [ + "当前回合是生成新图，聚焦生成链路，不要退回修图工具。", + "如用户带参考图，仅把参考图作为风格/构图条件，不要误当纯分析。", + ], + ); + + const videoRoute = createRoute( + { + domain: "video_generate", + specialistId: "video-specialist", + reason: "当前回合是视频生成任务", + planningHints: [], + }, + [ + "generateVideo", + "useSelectedCanvasImage", + "useLatestResultImage", + "useOriginalUploadImage", + ], + [ + "当前回合是视频任务，只规划视频生成相关动作。", + "如果信息不足可以先澄清，但不要误切回图片生成。", + ], + ); + + const canvasOverlay = createRoute( + { + domain: "canvas", + specialistId: "canvas-specialist", + reason: "检测到画布上下文，需要优先绑定目标图并回传结果", + planningHints: [], + }, + [ + "useSelectedCanvasImage", + "useLatestResultImage", + "useOriginalUploadImage", + "publishToCanvas", + ], + [ + "如果用户在无限画布中操作，优先绑定选中图或最近结果图。", + "修图/生图结果优先同步回画布，避免只停留在聊天气泡。", + ], + ); + + let primary = conversationRoute; + + if (!greetingTurn) { + switch (context.analysis.intent) { + case "vision": + primary = visionRoute; + break; + case "image_edit": + primary = imageEditRoute; + break; + case "image_gen": + primary = imageGenerateRoute; + break; + case "video_gen": + primary = videoRoute; + break; + case "chat": + default: + primary = conversationRoute; + break; + } + } + + const overlays = + primary.domain === "conversation" + ? [] + : hasCanvasContext(context) || mentionsCanvas(context.text) + ? [canvasOverlay] + : []; + + const toolAllowlist = dedupeToolNames([ + ...(primary.toolAllowlist || []), + ...overlays.flatMap((item) => item.toolAllowlist || []), + ]); + + const planningHints = [ + ...primary.planningHints, + ...overlays.flatMap((item) => item.planningHints), + ]; + + const reason = [primary.reason, ...overlays.map((item) => item.reason)] + .filter(Boolean) + .join("；"); + + return { + primary, + overlays, + toolAllowlist, + planningHints, + reason, + }; +} + +export function filterToolsForSkillRoute( + availableTools: AgentProjectToolDefinition[], + route: AgentSkillRouteResult, +) { + if (!route.toolAllowlist.length) return availableTools; + const allowSet = new Set(route.toolAllowlist); + return availableTools.filter((tool) => allowSet.has(tool.name)); +} diff --git a/lib/ai/specialists/index.ts b/lib/ai/specialists/index.ts new file mode 100644 index 000000000..6979450c0 --- /dev/null +++ b/lib/ai/specialists/index.ts @@ -0,0 +1,300 @@ +import type { AnalysisResult } from "@/lib/ai/smart-analyzer"; +import type { + AgentExecutionPlan, + AgentPlanNextAction, + AgentPlanResponseMode, + AgentPlannedToolCall, + AgentPreferredCapability, + AgentTargetResolutionMode, +} from "@/lib/ai/auto-agent-graph"; +import type { + AgentProjectToolDefinition, + AgentProjectToolName, +} from "@/lib/ai/tools-registry"; + +export type AgentSpecialistDomain = + | "conversation" + | "vision" + | "image_edit" + | "image_generate" + | "video_generate" + | "canvas"; + +export type AgentSpecialistId = + | "conversation-coordinator" + | "vision-analyst" + | "image-edit-specialist" + | "image-generate-specialist" + | "video-specialist" + | "canvas-specialist"; + +export interface AgentSpecialistRoute { + domain: AgentSpecialistDomain; + specialistId: AgentSpecialistId; + reason: string; + planningHints: string[]; + toolAllowlist?: AgentProjectToolName[]; +} + +export interface AgentSpecialistContext { + text: string; + analysis: AnalysisResult; + hasImage: boolean; + hasCanvasSelection: boolean; + targetResolutionMode: AgentTargetResolutionMode; + mediaSource?: "current_message" | "canvas_selection" | "history" | "none"; + selectedImageCount?: number; +} + +export interface AgentPlanRefineInput extends AgentSpecialistContext { + plan: AgentExecutionPlan; + availableTools: AgentProjectToolDefinition[]; +} + +function hasTool( + availableTools: AgentProjectToolDefinition[], + name: AgentProjectToolName, +) { + return availableTools.some((tool) => tool.name === name); +} + +function upsertTool( + toolCalls: AgentPlannedToolCall[], + name: AgentProjectToolName, + reason: string, + options?: { + prepend?: boolean; + args?: Record; + }, +) { + const existing = toolCalls.find((tool) => tool.name === name); + if (existing) { + if (!existing.reason) { + existing.reason = reason; + } + if (!existing.args && options?.args) { + existing.args = options.args; + } + return toolCalls; + } + + const nextTool = { + name, + reason, + args: options?.args, + } satisfies AgentPlannedToolCall; + + return options?.prepend ? [nextTool, ...toolCalls] : [...toolCalls, nextTool]; +} + +function forcePlanShape(params: { + plan: AgentExecutionPlan; + nextAction: AgentPlanNextAction; + intent: AnalysisResult["intent"]; + responseMode?: AgentPlanResponseMode; + preferredCapability?: AgentPreferredCapability; + subIntent?: string; + toolCalls: AgentPlannedToolCall[]; + shouldUseStructuredReply?: boolean; + targetSource?: AgentTargetResolutionMode; +}): AgentExecutionPlan { + return { + ...params.plan, + nextAction: params.nextAction, + intent: params.intent, + responseMode: + params.responseMode ?? params.plan.responseMode, + preferredCapability: + params.preferredCapability ?? params.plan.preferredCapability, + subIntent: + params.intent === "image_edit" + ? params.subIntent ?? params.plan.subIntent + : undefined, + toolCalls: params.toolCalls, + shouldUseStructuredReply: + params.shouldUseStructuredReply ?? params.plan.shouldUseStructuredReply, + targetSource: params.targetSource ?? params.plan.targetSource, + }; +} + +function normalizeEditCapability( + analysis: AnalysisResult, + text: string, +): { + capability: AgentPreferredCapability; + toolName: AgentProjectToolName; + subIntent?: string; +} { + const normalizedText = String(text || "").toLowerCase(); + const subIntent = String(analysis.subIntent || "").trim().toLowerCase(); + + if (subIntent === "remove_background" || /抠图|去背|去背景|透明底/.test(normalizedText)) { + return { + capability: "remove_background", + toolName: "removeBackground", + subIntent: "remove_background", + }; + } + if (subIntent === "extend" || /扩图|拓图|补边|延展|加宽|加高/.test(normalizedText)) { + return { + capability: "extend", + toolName: "extendImage", + subIntent: "extend", + }; + } + if (subIntent === "resize" || /尺寸|分辨率|改成.*x|改为.*x|比例|画布/.test(normalizedText)) { + return { + capability: "resize", + toolName: "resizeImage", + subIntent: "resize", + }; + } + if (subIntent === "box_cutout" || /框选抠图|框选/.test(normalizedText)) { + return { + capability: "box_cutout", + toolName: "boxCutout", + subIntent: "box_cutout", + }; + } + if (subIntent === "enhance" || /增强|放大|高清|超清|锐化|修复/.test(normalizedText)) { + return { + capability: "enhance", + toolName: "enhanceImage", + subIntent: "enhance", + }; + } + return { + capability: "generic_edit", + toolName: "editImage", + subIntent: analysis.subIntent || undefined, + }; +} + +export function refinePlanWithSpecialist( + specialist: AgentSpecialistRoute, + input: AgentPlanRefineInput, +): AgentExecutionPlan { + const { plan, availableTools, analysis, text } = input; + let toolCalls = [...plan.toolCalls]; + + if (specialist.domain === "conversation") { + const shouldClarify = plan.nextAction === "clarify"; + if (hasTool(availableTools, "respondDirect")) { + toolCalls = [{ name: "respondDirect", reason: "纯对话由对话协调专家直接回复" }]; + } + return forcePlanShape({ + plan, + nextAction: shouldClarify ? "clarify" : "respond", + intent: "chat", + responseMode: shouldClarify ? "clarify" : "direct_answer", + preferredCapability: "respond", + toolCalls, + shouldUseStructuredReply: false, + targetSource: "none", + }); + } + + if (specialist.domain === "vision") { + if (hasTool(availableTools, "analyzeImage")) { + toolCalls = [{ name: "analyzeImage", reason: "视觉分析任务由看图专家处理" }]; + } else if (hasTool(availableTools, "respondDirect")) { + toolCalls = [{ name: "respondDirect", reason: "缺少看图工具时退回直接回答" }]; + } + return forcePlanShape({ + plan, + nextAction: "analyze_image", + intent: "vision", + responseMode: "tool_then_answer", + preferredCapability: "analyze", + toolCalls, + shouldUseStructuredReply: false, + }); + } + + if (specialist.domain === "image_edit") { + const editTarget = normalizeEditCapability(analysis, text); + if (hasTool(availableTools, editTarget.toolName)) { + toolCalls = [{ name: editTarget.toolName, reason: "修图专家选择最匹配的编辑能力" }]; + } else if (hasTool(availableTools, "editImage")) { + toolCalls = [{ name: "editImage", reason: "修图专家回退到通用编辑能力" }]; + } + return forcePlanShape({ + plan, + nextAction: "edit_image", + intent: "image_edit", + subIntent: editTarget.subIntent, + responseMode: "tool_then_execute", + preferredCapability: editTarget.capability, + toolCalls, + shouldUseStructuredReply: true, + }); + } + + if (specialist.domain === "image_generate") { + if (hasTool(availableTools, "generateImage")) { + toolCalls = [{ name: "generateImage", reason: "生图专家执行图片生成" }]; + } + return forcePlanShape({ + plan, + nextAction: "generate_image", + intent: "image_gen", + responseMode: "tool_then_execute", + preferredCapability: "generate_image", + toolCalls, + shouldUseStructuredReply: true, + }); + } + + if (specialist.domain === "video_generate") { + if (hasTool(availableTools, "generateVideo")) { + toolCalls = [{ name: "generateVideo", reason: "视频专家执行视频生成" }]; + } + return forcePlanShape({ + plan, + nextAction: "generate_video", + intent: "video_gen", + responseMode: "tool_then_execute", + preferredCapability: "generate_video", + toolCalls, + shouldUseStructuredReply: true, + }); + } + + if (specialist.domain === "canvas") { + const isExecutionAction = + plan.nextAction === "edit_image" || + plan.nextAction === "generate_image"; + const normalizedTargetSource = + input.hasCanvasSelection && plan.targetSource === "none" + ? "explicit_selection" + : plan.targetSource; + + if (input.hasCanvasSelection && hasTool(availableTools, "useSelectedCanvasImage")) { + toolCalls = upsertTool( + toolCalls, + "useSelectedCanvasImage", + "画布专家优先使用当前选中的画布图片", + { prepend: true }, + ); + } + + if ( + isExecutionAction && + hasTool(availableTools, "publishToCanvas") + ) { + toolCalls = upsertTool( + toolCalls, + "publishToCanvas", + "画布专家要求结果同步回无限画布", + ); + } + + return { + ...plan, + toolCalls, + targetSource: normalizedTargetSource, + }; + } + + return plan; +} diff --git a/lib/ai/tools/generate-video.ts b/lib/ai/tools/generate-video.ts index ff264b9e8..43a98d8d1 100644 --- a/lib/ai/tools/generate-video.ts +++ b/lib/ai/tools/generate-video.ts @@ -16,6 +16,30 @@ import type { VideoGenerationOptions, } from "@/types/ai"; +export interface ExecuteDirectVideoGenerationParams { + prompt: string; + mode: VideoGenerationMode; + images?: string[]; + resolution?: "480p" | "720p" | "1080p"; + ratio?: "16:9" | "4:3" | "1:1" | "3:4" | "9:16" | "21:9" | "adaptive"; + duration?: number; + watermark?: boolean; + cameraFixed?: boolean; + seed?: number; + returnLastFrame?: boolean; + preferredModelRef?: string; + onProgress?: (chunk: string) => void | Promise; +} + +export interface ExecuteDirectVideoGenerationResult { + success: boolean; + videoUrl?: string; + metadata?: Record | null; + modelDisplayName: string; + modelId: string; + modelRef: string; +} + function inferSupportedModesFromModelId(modelId: string): VideoGenerationMode[] { const normalized = modelId.toLowerCase(); @@ -232,6 +256,103 @@ function buildVideoOptions( return options; } +async function resolvePreferredVideoParentModelId(preferredModelRef?: string) { + const normalizedRef = String(preferredModelRef || "").trim(); + if (!normalizedRef) return undefined; + + const models = await getEnabledAiModelsWithProvider(); + const matched = models.find( + (item) => + item.provider.type === "volcano-video" && + (item.model.id === normalizedRef || item.model.modelId === normalizedRef), + ); + + if (!matched) return undefined; + if (matched.model.isSubModel && matched.model.parentModelId) { + return matched.model.parentModelId; + } + return matched.model.id; +} + +export async function executeDirectVideoGeneration( + params: ExecuteDirectVideoGenerationParams, +): Promise { + const prompt = String(params.prompt || "").trim(); + if (!prompt) { + throw new Error("缺少视频生成提示词"); + } + + const preferredParentModelId = await resolvePreferredVideoParentModelId( + params.preferredModelRef, + ); + const allModels = await getEnabledAiModelsWithProvider(); + const fallbackParentModel = + allModels.find( + (item) => + item.provider.type === "volcano-video" && + !item.model.isSubModel && + item.model.modelId === "volcano-video-master", + ) || + allModels.find( + (item) => item.provider.type === "volcano-video" && !item.model.isSubModel, + ); + + const parentModelId = preferredParentModelId || fallbackParentModel?.model.id; + if (!parentModelId) { + throw new Error("未找到可用的视频生成主模型"); + } + + const generationParams = { + mode: params.mode, + resolution: params.resolution || "720p", + ratio: params.ratio || "16:9", + duration: params.duration || 5, + watermark: params.watermark ?? true, + cameraFixed: params.cameraFixed ?? false, + seed: params.seed, + returnLastFrame: params.returnLastFrame ?? false, + referenceImages: params.images, + } satisfies Partial; + + const { model, capabilities } = await getCompatibleModel( + params.mode, + parentModelId, + generationParams, + ); + + validateParamsAgainstCapabilities(generationParams, capabilities); + + const options = buildVideoOptions( + params.mode, + params.images, + prompt, + generationParams, + model.model.modelId, + ); + + let videoMetadata: Record | null = null; + for await (const chunk of doGenerateStream(model.provider.apiKey, model.provider.baseUrl, options)) { + await params.onProgress?.(chunk); + const metadataMatch = chunk.match(//); + if (metadataMatch) { + try { + videoMetadata = JSON.parse(metadataMatch[1]); + } catch (error) { + console.error("[Generate Video Tool] 解析元数据失败:", error); + } + } + } + + return { + success: true, + videoUrl: videoMetadata?.temporaryVideoUrl || undefined, + metadata: videoMetadata, + modelDisplayName: String(model.model.displayName || model.model.modelId || "视频模型"), + modelId: String(model.model.modelId || model.model.id), + modelRef: String(model.model.id || model.model.modelId), + }; +} + /** * 视频生成工具 */ @@ -276,20 +397,18 @@ export function generateVideo({ dataStream }: { dataStream: any }) { console.log("[Generate Video Tool] 模式:", mode); console.log("[Generate Video Tool] 提示词:", prompt); - // 1. 🆕 智能选择最佳子模型 - // 先获取火山引擎视频主模型 - const allModels = await getEnabledAiModelsWithProvider(); - const volcanoMasterModel = allModels.find( - m => m.provider.type === "volcano-video" && m.model.modelId === "volcano-video-master" - ); + dataStream.writeData({ + type: "video-generation-start", + data: { + mode, + model: "智能选择中", + }, + }); - if (!volcanoMasterModel) { - throw new Error("未找到火山引擎视频主模型，请在系统设置中添加"); - } - - // 准备参数用于智能选择 - const params = { + const result = await executeDirectVideoGeneration({ + prompt, mode, + images, resolution, ratio, duration, @@ -297,75 +416,20 @@ export function generateVideo({ dataStream }: { dataStream: any }) { cameraFixed, seed, returnLastFrame, - referenceImages: images, - }; - - // 根据用户参数智能选择最佳子模型 - const { model, capabilities } = await selectBestSubModel( - volcanoMasterModel.model.id, - mode, - params - ); - - console.log( - "[Generate Video Tool] ✅ 智能选择模型:", - model.model.displayName, - `(${model.model.modelId})` - ); - - // 2. 验证参数是否符合模型能力 - validateParamsAgainstCapabilities(params, capabilities); - - // 3. 构建视频生成选项 - const options = buildVideoOptions( - mode, - images, - prompt, - params, - model.model.modelId - ); - - // 4. 调用视频生成（流式输出） - dataStream.writeData({ - type: "video-generation-start", - data: { - mode, - model: model.model.displayName, + onProgress: async (chunk) => { + dataStream.writeMessageAnnotation({ + type: "video-progress", + content: chunk, + }); }, }); - const apiKey = model.provider.apiKey; - const baseURL = model.provider.baseUrl; - - let videoMetadata: any = null; - - // 使用异步生成器进行流式输出 - for await (const chunk of doGenerateStream(apiKey, baseURL, options)) { - dataStream.writeMessageAnnotation({ - type: "video-progress", - content: chunk, - }); - - // 提取视频元数据（从注释中） - const metadataMatch = chunk.match( - // - ); - if (metadataMatch) { - try { - videoMetadata = JSON.parse(metadataMatch[1]); - } catch (e) { - console.error("[Generate Video Tool] 解析元数据失败:", e); - } - } - } - console.log("[Generate Video Tool] ✅ 视频生成完成"); - // 5. 返回元数据（用于保存到数据库） return { success: true, - videoUrl: videoMetadata?.temporaryVideoUrl, - metadata: videoMetadata, + videoUrl: result.videoUrl, + metadata: result.metadata, }; } catch (error: any) { console.error("[Generate Video Tool] ❌ 生成失败:", error); diff --git a/lib/canvas/generate-image.ts b/lib/canvas/generate-image.ts index dbeda3222..33db7cfb5 100644 --- a/lib/canvas/generate-image.ts +++ b/lib/canvas/generate-image.ts @@ -28,6 +28,7 @@ export interface ParsedImage { export interface CanvasGenerateImageResult { images: ParsedImage[]; model: string; + modelRef: string; prompt: string; } @@ -258,6 +259,7 @@ export async function generateCanvasImage( return { images: parsedImages, model: selectedModel.model.displayName, + modelRef: selectedModel.model.id, prompt, }; }