feat(canvas): add capability runtime and MCP tools
This commit is contained in:
134
parent
5a7121dceb
commit
1b70c1d678
@ -7,6 +7,7 @@ import type {
|
||||
ResolvedTargetAssets,
|
||||
} from "@/lib/ai/context-memory/types";
|
||||
import type { CanvasViewOperationPlan } from "@/types/canvas/agent-protocol";
|
||||
import type { CanvasCapabilityClientCommand } from "@/lib/canvas/capabilities/client-command";
|
||||
import { generateUUID } from "@/lib/utils";
|
||||
|
||||
const chatApiLogger = createLogger("chat.stream");
|
||||
@ -422,6 +423,19 @@ export function writeCanvasViewOperationChunk(
|
||||
);
|
||||
}
|
||||
|
||||
export function writeCanvasCapabilityCommandChunk(
|
||||
writer: StreamWriter,
|
||||
command: CanvasCapabilityClientCommand,
|
||||
label: string
|
||||
) {
|
||||
writeStructuredDataChunk(
|
||||
writer,
|
||||
"data-canvasCapabilityCommand",
|
||||
command,
|
||||
label
|
||||
);
|
||||
}
|
||||
|
||||
type AgentTargetLockRefPreview = {
|
||||
url: string;
|
||||
title?: string;
|
||||
|
||||
@ -90,6 +90,10 @@ import {
|
||||
hasCanvasAgentPanViewIntent,
|
||||
} from "@/lib/canvas/agent-view-pan";
|
||||
import { buildCanvasAgentDirectWritebackPlan } from "@/lib/canvas/agent-direct-writeback";
|
||||
import {
|
||||
createCanvasCapabilityTools,
|
||||
getCanvasCapabilityToolNames,
|
||||
} from "@/lib/canvas/capabilities/ai-sdk-tools";
|
||||
import {
|
||||
getCanvasAgentOperationErrorMessage,
|
||||
normalizeCanvasAgentOperationErrorCode,
|
||||
@ -392,6 +396,7 @@ import {
|
||||
writeAgentTargetLockChunk,
|
||||
writeAssistantTextChunk,
|
||||
writeCanvasMediaChunk,
|
||||
writeCanvasCapabilityCommandChunk,
|
||||
writeCanvasOperationChunk,
|
||||
writeCanvasViewOperationChunk,
|
||||
writeImageGenerationStatusChunk,
|
||||
@ -7716,6 +7721,17 @@ export async function POST(request: Request) {
|
||||
const directAgentExecution = directAgentToolContext?.execution || null;
|
||||
const directAgentToolTargetImageUrls = directAgentToolContext?.targetImageUrls || [];
|
||||
const directAgentToolTargetImageUrl = directAgentToolContext?.targetImageUrl || null;
|
||||
const selectedCanvasImageNodeIds = (
|
||||
requestCanvasAgentContext?.selectedNodes || []
|
||||
)
|
||||
.filter((node) => node.type === "image")
|
||||
.map((node) => node.nodeId)
|
||||
.filter(Boolean);
|
||||
const shouldUseCanvasCapabilityToolLoop = Boolean(
|
||||
contextPayload?.source === "canvas" &&
|
||||
contextPayload.projectId &&
|
||||
selectedCanvasImageNodeIds.length > 0
|
||||
);
|
||||
|
||||
if (
|
||||
shouldRunAgentConversation &&
|
||||
@ -8265,6 +8281,7 @@ export async function POST(request: Request) {
|
||||
shouldRunAgentConversation &&
|
||||
!smartAutoNeedsClarification &&
|
||||
directAgentExecution &&
|
||||
!shouldUseCanvasCapabilityToolLoop &&
|
||||
isDirectAgentImageEditToolName(directAgentExecution.toolName)
|
||||
) {
|
||||
const directAgentToolCapability = directAgentExecution.capability;
|
||||
@ -8711,6 +8728,14 @@ export async function POST(request: Request) {
|
||||
agentIdentityInstruction,
|
||||
recentDialogueSystemContext,
|
||||
baseConversationSystemPrompt,
|
||||
shouldUseCanvasCapabilityToolLoop
|
||||
? [
|
||||
"当前请求来自无限画布，且用户已明确选中图片节点。",
|
||||
"涉及选中图片的识别、编辑、生成、3D、改名、批注或删除时，必须调用已提供的画布能力工具，不得只用文字声称完成。",
|
||||
"可以连续调用多个工具；下一步要处理上一步结果时，把上一步返回的 createdNodeIds 作为 nodeIds。",
|
||||
"只有工具返回成功后才能告诉用户已完成；若工具返回 interactive，说明已在画布打开对应交互界面并等待用户操作。",
|
||||
].join("\n")
|
||||
: "",
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n\n");
|
||||
@ -10948,6 +10973,52 @@ export async function POST(request: Request) {
|
||||
chatApiLogger.info("[Smart Auto Mode] 🚫 generateVideo tool disabled (manual video mode not selected)");
|
||||
}
|
||||
|
||||
if (
|
||||
contextPayload?.source === "canvas" &&
|
||||
contextPayload.projectId &&
|
||||
selectedCanvasImageNodeIds.length > 0
|
||||
) {
|
||||
const canvasCapabilityTools = await createCanvasCapabilityTools({
|
||||
projectId: contextPayload.projectId,
|
||||
selectedNodeIds: selectedCanvasImageNodeIds,
|
||||
userId: session.user.id,
|
||||
requestHeaders: request.headers,
|
||||
requestUrl: request.url,
|
||||
defaultModelId: finalModelId,
|
||||
destructiveConfirmed: isCanvasOperationConfirmText(requestTextContent),
|
||||
callbacks: {
|
||||
onClientCommand: (command, capability) => {
|
||||
writeCanvasCapabilityCommandChunk(
|
||||
dataStream as unknown as StreamWriter,
|
||||
command,
|
||||
`canvas-capability:${capability.id}:client-command`
|
||||
);
|
||||
},
|
||||
onResult: (runtimeResult, capability) => {
|
||||
if (!runtimeResult.operation) return;
|
||||
writeCanvasOperationChunk(
|
||||
dataStream as unknown as StreamWriter,
|
||||
{
|
||||
projectId: contextPayload.projectId!,
|
||||
operationId: runtimeResult.task.requestId,
|
||||
operation: capability.id,
|
||||
status: "applied",
|
||||
createdNodeIds: runtimeResult.operation.createdNodeIds,
|
||||
createdNodeIdsByMediaItemId:
|
||||
runtimeResult.operation.createdNodeIdsByMediaItemId,
|
||||
updatedNodeIds: runtimeResult.operation.updatedNodeIds,
|
||||
canvasRevision: runtimeResult.operation.canvasRevision,
|
||||
message: `${capability.title}已完成`,
|
||||
},
|
||||
`canvas-capability:${capability.id}:operation`
|
||||
);
|
||||
},
|
||||
},
|
||||
});
|
||||
Object.assign(tools, canvasCapabilityTools);
|
||||
activeTools.push(...getCanvasCapabilityToolNames(canvasCapabilityTools));
|
||||
}
|
||||
|
||||
chatApiLogger.info("[Chat API] Calling streamText with model:", (languageModel as any).modelId, (languageModel as any).provider); // 调试日志
|
||||
chatApiLogger.info("[Chat API] Provider type:", (languageModel as any).provider); // 调试日志
|
||||
|
||||
@ -11938,9 +12009,13 @@ export async function POST(request: Request) {
|
||||
model: languageModel,
|
||||
system: effectiveConversationSystemPrompt,
|
||||
messages: convertedMessages,
|
||||
stopWhen: stepCountIs(5),
|
||||
stopWhen: stepCountIs(shouldUseCanvasCapabilityToolLoop ? 8 : 5),
|
||||
experimental_activeTools:
|
||||
finalModelId === "chat-model-reasoning" || isVolcanoVideoModel ? [] : activeTools,
|
||||
(finalModelId === "chat-model-reasoning" &&
|
||||
!shouldUseCanvasCapabilityToolLoop) ||
|
||||
isVolcanoVideoModel
|
||||
? []
|
||||
: activeTools,
|
||||
experimental_transform: shouldDisableSmoothStream
|
||||
? undefined
|
||||
: smoothStream({ chunking: "word" }),
|
||||
|
||||
91
app/api/canvas/[id]/capabilities/route.ts
Normal file
91
app/api/canvas/[id]/capabilities/route.ts
Normal file
@ -0,0 +1,91 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { ZodError } from "zod";
|
||||
|
||||
import { auth } from "@/app/(auth)/auth";
|
||||
import { canvasCapabilityClientCommandSchema } from "@/lib/canvas/capabilities/client-command";
|
||||
import { CanvasCapabilityRuntimeError } from "@/lib/canvas/capabilities/context";
|
||||
import { getCanvasCapabilityManifest } from "@/lib/canvas/capabilities/registry";
|
||||
import { executeCanvasCapabilityRuntime } from "@/lib/canvas/capabilities/runtime";
|
||||
import { getCanvasCapabilityTask } from "@/lib/canvas/capabilities/task-store";
|
||||
import { CanvasAgentMutationError } from "@/lib/canvas/agent-mutations";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
export const maxDuration = 600;
|
||||
|
||||
export async function GET(
|
||||
request: Request,
|
||||
{ params }: { params: Promise<{ id: string }> }
|
||||
) {
|
||||
const session = await auth();
|
||||
if (!session?.user?.id) {
|
||||
return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
|
||||
}
|
||||
const { id } = await params;
|
||||
const requestId = new URL(request.url).searchParams.get("requestId");
|
||||
if (!requestId) {
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
capabilities: getCanvasCapabilityManifest(),
|
||||
});
|
||||
}
|
||||
const task = await getCanvasCapabilityTask({
|
||||
requestId,
|
||||
userId: session.user.id,
|
||||
projectId: id,
|
||||
});
|
||||
if (!task) {
|
||||
return NextResponse.json(
|
||||
{ success: false, error: "能力任务不存在" },
|
||||
{ status: 404 }
|
||||
);
|
||||
}
|
||||
return NextResponse.json({ success: true, task });
|
||||
}
|
||||
|
||||
export async function POST(
|
||||
request: Request,
|
||||
{ params }: { params: Promise<{ id: string }> }
|
||||
) {
|
||||
const session = await auth();
|
||||
if (!session?.user?.id) {
|
||||
return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
|
||||
}
|
||||
|
||||
try {
|
||||
const { id } = await params;
|
||||
const command = canvasCapabilityClientCommandSchema.parse(await request.json());
|
||||
const result = await executeCanvasCapabilityRuntime({
|
||||
command,
|
||||
projectId: id,
|
||||
requestHeaders: request.headers,
|
||||
requestUrl: request.url,
|
||||
userId: session.user.id,
|
||||
});
|
||||
return NextResponse.json({ success: true, ...result });
|
||||
} catch (error) {
|
||||
if (
|
||||
error instanceof CanvasCapabilityRuntimeError ||
|
||||
error instanceof CanvasAgentMutationError
|
||||
) {
|
||||
return NextResponse.json(
|
||||
{ success: false, error: error.message, code: error.code },
|
||||
{ status: error.status }
|
||||
);
|
||||
}
|
||||
if (error instanceof ZodError) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: "画布能力参数无效",
|
||||
issues: error.issues,
|
||||
},
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
const message = error instanceof Error ? error.message : "执行画布能力失败";
|
||||
return NextResponse.json(
|
||||
{ success: false, error: message },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
38
app/api/mcp/canvas/[id]/route.ts
Normal file
38
app/api/mcp/canvas/[id]/route.ts
Normal file
@ -0,0 +1,38 @@
|
||||
import { WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js";
|
||||
import { NextResponse } from "next/server";
|
||||
|
||||
import { auth } from "@/app/(auth)/auth";
|
||||
import { createCanvasCapabilityMcpServer } from "@/lib/canvas/capabilities/mcp-server";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
export const maxDuration = 600;
|
||||
|
||||
async function handleMcpRequest(
|
||||
request: Request,
|
||||
{ params }: { params: Promise<{ id: string }> }
|
||||
) {
|
||||
const session = await auth();
|
||||
if (!session?.user?.id) {
|
||||
return NextResponse.json(
|
||||
{ error: "未登录，无法访问画布 MCP 服务" },
|
||||
{ status: 401 }
|
||||
);
|
||||
}
|
||||
|
||||
const { id: projectId } = await params;
|
||||
const transport = new WebStandardStreamableHTTPServerTransport({
|
||||
enableJsonResponse: true,
|
||||
});
|
||||
const server = await createCanvasCapabilityMcpServer({
|
||||
projectId,
|
||||
requestHeaders: request.headers,
|
||||
requestUrl: request.url,
|
||||
userId: session.user.id,
|
||||
});
|
||||
await server.connect(transport);
|
||||
return transport.handleRequest(request);
|
||||
}
|
||||
|
||||
export const POST = handleMcpRequest;
|
||||
export const GET = handleMcpRequest;
|
||||
export const DELETE = handleMcpRequest;
|
||||
@ -151,6 +151,7 @@ import {
|
||||
type AgentTargetLockStreamPayload,
|
||||
buildAgentProgressNodeSteps,
|
||||
type CanvasOperationStreamPayload,
|
||||
type CanvasCapabilityCommandStreamPayload,
|
||||
type CanvasViewOperationStreamPayload,
|
||||
type CanvasMediaStreamPayload,
|
||||
filterCanvasMediaPayloadForUnhandledItems,
|
||||
@ -165,6 +166,7 @@ import {
|
||||
shouldRetryCanvasViewOperationAfterRefresh,
|
||||
shouldSkipCanvasMediaPayloadForHandledItems,
|
||||
} from "@/components/canvas/chat-panel/canvas-agent-stream";
|
||||
import { canvasCapabilityClientCommandSchema } from "@/lib/canvas/capabilities/client-command";
|
||||
import {
|
||||
mergePendingCanvasAgentTargetNodeIds,
|
||||
resolvePendingCanvasAgentTargetNode,
|
||||
@ -3232,6 +3234,9 @@ export function CanvasChatPanel({
|
||||
// 🆕 物品拾取
|
||||
const pendingPickedObjects = useCanvasStore((state) => state.pendingPickedObjects);
|
||||
const clearPendingPickedObjects = useCanvasStore((state) => state.clearPendingPickedObjects);
|
||||
const dispatchCanvasCapabilityCommand = useCanvasStore(
|
||||
(state) => state.dispatchCanvasCapabilityCommand
|
||||
);
|
||||
|
||||
const clearLatestGeneratedImageHighlights = useCallback(() => {
|
||||
const currentNodes = useCanvasStore.getState().nodes;
|
||||
@ -6626,6 +6631,24 @@ export function CanvasChatPanel({
|
||||
return;
|
||||
}
|
||||
|
||||
if (dataPart.type === "data-canvasCapabilityCommand") {
|
||||
try {
|
||||
const rawPayload: CanvasCapabilityCommandStreamPayload =
|
||||
typeof dataPart.data === "string"
|
||||
? JSON.parse(dataPart.data)
|
||||
: dataPart.data;
|
||||
const command = canvasCapabilityClientCommandSchema.parse(rawPayload);
|
||||
dispatchCanvasCapabilityCommand(command);
|
||||
} catch (error) {
|
||||
console.error(
|
||||
"[CanvasChat] Failed to parse data-canvasCapabilityCommand:",
|
||||
error
|
||||
);
|
||||
toast.error("无法打开画布交互工具");
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (dataPart.type === "data-canvasOperation") {
|
||||
try {
|
||||
const payload: CanvasOperationStreamPayload =
|
||||
|
||||
@ -1,4 +1,7 @@
|
||||
import type { CanvasViewOperationPlan } from "@/types/canvas/agent-protocol";
|
||||
import type { CanvasCapabilityClientCommand } from "@/lib/canvas/capabilities/client-command";
|
||||
|
||||
export type CanvasCapabilityCommandStreamPayload = CanvasCapabilityClientCommand;
|
||||
|
||||
export type CanvasMediaStreamPayload = {
|
||||
messageId?: string;
|
||||
|
||||
@ -220,6 +220,12 @@ import {
|
||||
} from "@/lib/ai-wash/config";
|
||||
import { normalizeOptionalHexColor } from "@/lib/canvas/text-edit-colors";
|
||||
import { isCanvasImageUploadPending } from "@/lib/canvas/local-upload-state";
|
||||
import type {
|
||||
CanvasCapabilityClientAction,
|
||||
SelectedImageToolbarActionKey,
|
||||
} from "@/lib/canvas/capabilities/types";
|
||||
import { createCanvasCapabilityClientCommand } from "@/lib/canvas/capabilities/client-command";
|
||||
import { getCanvasCapabilityByClientAction } from "@/lib/canvas/capabilities/registry";
|
||||
import { cn, fetcher } from "@/lib/utils";
|
||||
import type {
|
||||
CanvasTextEditPayloadItem,
|
||||
@ -282,16 +288,6 @@ interface CanvasNodeAnnotationDisplayItem {
|
||||
createdByAgent?: boolean;
|
||||
}
|
||||
|
||||
interface LayerSplitBounds {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
originalWidth: number;
|
||||
originalHeight: number;
|
||||
coverage: number;
|
||||
}
|
||||
|
||||
const AI_CLOUD_IMAGE_ACTIONS = new Set([
|
||||
"ai-wash",
|
||||
"rmbg",
|
||||
@ -1477,7 +1473,10 @@ export function ImageNode({
|
||||
isLayerSplitEnabled && imageLayerSplitEnabledByToolbar;
|
||||
const selectedImageInlineActionCount =
|
||||
getSelectedImageInlineActionCount(canvasToolbarConfig);
|
||||
const selectedImageToolbarActionCandidates = [
|
||||
const selectedImageToolbarActionCandidates: ReadonlyArray<{
|
||||
key: SelectedImageToolbarActionKey;
|
||||
enabled: boolean;
|
||||
}> = [
|
||||
{ key: "pickObject", enabled: imagePickObjectEnabled },
|
||||
{ key: "enhance", enabled: imageEnhanceEnabled },
|
||||
{ key: "removeBackground", enabled: imageRemoveBackgroundEnabled },
|
||||
@ -1497,9 +1496,7 @@ export function ImageNode({
|
||||
{ key: "flipRotate", enabled: imageFlipRotateEnabled },
|
||||
{ key: "promptRegenerate", enabled: imagePromptRegenerateEnabled },
|
||||
{ key: "note", enabled: imageNoteEnabled },
|
||||
] as const;
|
||||
type SelectedImageToolbarActionKey =
|
||||
(typeof selectedImageToolbarActionCandidates)[number]["key"];
|
||||
];
|
||||
const enabledSelectedImageToolbarActions =
|
||||
selectedImageToolbarActionCandidates.filter((action) => action.enabled);
|
||||
const selectedImageInlineActionKeys = new Set<SelectedImageToolbarActionKey>(
|
||||
@ -2037,6 +2034,7 @@ export function ImageNode({
|
||||
>({});
|
||||
const localEditRegionsRef = useRef<LocalEditRegion[]>(localEditRegions);
|
||||
const hydratedImageTaskRecoveryRef = useRef<string | null>(null);
|
||||
const processedCapabilityCommandIdsRef = useRef(new Set<string>());
|
||||
|
||||
const updateNode = useCanvasStore((state) => state.updateNode);
|
||||
const deleteNode = useCanvasStore((state) => state.deleteNode);
|
||||
@ -2044,6 +2042,7 @@ export function ImageNode({
|
||||
const selectNode = useCanvasStore((state) => state.selectNode);
|
||||
const projectId = useCanvasStore((state) => state.projectId);
|
||||
const getCanvasState = useCanvasStore((state) => state.getCanvasState);
|
||||
const loadCanvas = useCanvasStore((state) => state.loadCanvas);
|
||||
const addPendingChatAttachment = useCanvasStore(
|
||||
(state) => state.addPendingChatAttachment
|
||||
);
|
||||
@ -6043,35 +6042,6 @@ export function ImageNode({
|
||||
await detectTextItems(true);
|
||||
}, [data.src, data.textEditItems, data.textEditSource, detectTextItems]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!imageNodeActionRequest || imageNodeActionRequest.nodeId !== node.id) {
|
||||
return;
|
||||
}
|
||||
|
||||
clearImageNodeActionRequest();
|
||||
|
||||
if (imageNodeActionRequest.action === "box-cutout") {
|
||||
handleEnterBoxCutoutMode();
|
||||
return;
|
||||
}
|
||||
|
||||
if (imageNodeActionRequest.action === "text-edit") {
|
||||
void handleOpenTextEdit();
|
||||
return;
|
||||
}
|
||||
|
||||
if (imageNodeActionRequest.action === "local-edit") {
|
||||
handleEnterLocalEditMode();
|
||||
}
|
||||
}, [
|
||||
clearImageNodeActionRequest,
|
||||
handleEnterBoxCutoutMode,
|
||||
handleEnterLocalEditMode,
|
||||
handleOpenTextEdit,
|
||||
imageNodeActionRequest,
|
||||
node.id,
|
||||
]);
|
||||
|
||||
const handleTextItemChange = (id: string, value: string) => {
|
||||
setTextEditItems((prev) =>
|
||||
prev.map((item) =>
|
||||
@ -6356,30 +6326,6 @@ export function ImageNode({
|
||||
e.target.value = "";
|
||||
};
|
||||
|
||||
// 获取操作的中文名称
|
||||
const getActionName = (action: string): string => {
|
||||
const names: Record<string, string> = {
|
||||
rmbg: "抠图",
|
||||
watermark: "去印",
|
||||
enhance: "增强",
|
||||
extend: "扩图",
|
||||
crop: "裁剪",
|
||||
adjust: "调整",
|
||||
"flip-rotate": "翻转与旋转",
|
||||
resize: "调尺",
|
||||
angle: "视角",
|
||||
"layer-split": "图层拆分",
|
||||
"scene-fusion": "批量处理",
|
||||
annotate: "标注",
|
||||
"text-edit": "改字",
|
||||
note: "添加批注",
|
||||
"prompt-regenerate": "重绘",
|
||||
"box-cutout": "框选抠图",
|
||||
"local-edit": "局部编辑",
|
||||
};
|
||||
return names[action] || action;
|
||||
};
|
||||
|
||||
const openThreedGenerationDialog = useCallback(
|
||||
(mode: "model" | "world") => {
|
||||
if (!data.src) {
|
||||
@ -6642,8 +6588,55 @@ export function ImageNode({
|
||||
updateNode,
|
||||
]);
|
||||
|
||||
// 处理图片编辑操作 - 集成我们现有的 API
|
||||
const handleAction = async (action: string) => {
|
||||
const executeSharedToolbarCapability = async (
|
||||
action: "rmbg" | "watermark" | "enhance" | "extend" | "layer-split"
|
||||
) => {
|
||||
if (!projectId) {
|
||||
toast.error("画布还在加载，请稍后再试");
|
||||
return;
|
||||
}
|
||||
const capability = getCanvasCapabilityByClientAction(action);
|
||||
if (!capability) {
|
||||
toast.error("画布能力未注册");
|
||||
return;
|
||||
}
|
||||
|
||||
setIsCreatingDetachedResultNode(true);
|
||||
setIsProcessing(true);
|
||||
try {
|
||||
const command = createCanvasCapabilityClientCommand({
|
||||
capabilityId: capability.id,
|
||||
nodeIds: [node.id],
|
||||
projectId,
|
||||
source: "toolbar",
|
||||
});
|
||||
const response = await fetch(`/api/canvas/${projectId}/capabilities`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(command),
|
||||
});
|
||||
const payload = await response.json().catch(() => ({}));
|
||||
if (!response.ok || payload?.success !== true) {
|
||||
throw new Error(payload?.error || `${capability.title}失败`);
|
||||
}
|
||||
|
||||
const projectResult = await canvasService.getProject(projectId);
|
||||
if (!projectResult.success || !projectResult.data?.canvasData) {
|
||||
throw new Error(projectResult.error || "结果已生成，但画布同步失败");
|
||||
}
|
||||
loadCanvas(projectResult.data.canvasData);
|
||||
toast.success(`${capability.title}完成`);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : `${capability.title}失败`;
|
||||
toast.error(message);
|
||||
} finally {
|
||||
setIsCreatingDetachedResultNode(false);
|
||||
setIsProcessing(false);
|
||||
}
|
||||
};
|
||||
|
||||
// 处理图片编辑操作
|
||||
const handleAction = async (action: CanvasCapabilityClientAction | "scene-fusion" | "annotate") => {
|
||||
if (!data.src) {
|
||||
toast.error("请先上传图片");
|
||||
return;
|
||||
@ -6746,34 +6739,7 @@ export function ImageNode({
|
||||
toast.info("图层拆分功能未开启");
|
||||
return;
|
||||
}
|
||||
setIsCreatingDetachedResultNode(true);
|
||||
setIsProcessing(true);
|
||||
try {
|
||||
const result = await imageGenerationService.splitImageLayers(data.src);
|
||||
const layers = result.data?.layers || [];
|
||||
|
||||
if (!result.success || layers.length === 0) {
|
||||
toast.error(result.error || "图层拆分失败");
|
||||
return;
|
||||
}
|
||||
|
||||
const addedLayerCount = handleLayerSplitComplete(
|
||||
layers.map((layer) => ({
|
||||
originalUrl: layer.originalUrl,
|
||||
title: layer.title,
|
||||
layerIndex: layer.layerIndex,
|
||||
totalLayers: layer.totalLayers,
|
||||
bounds: layer.bounds ?? null,
|
||||
}))
|
||||
);
|
||||
toast.success(`图层拆分完成，已新增 ${addedLayerCount} 个图层结果`);
|
||||
} catch (error) {
|
||||
console.error("图层拆分失败:", error);
|
||||
toast.error("图层拆分失败");
|
||||
} finally {
|
||||
setIsCreatingDetachedResultNode(false);
|
||||
setIsProcessing(false);
|
||||
}
|
||||
await executeSharedToolbarCapability("layer-split");
|
||||
return;
|
||||
}
|
||||
if (action === "box-cutout") {
|
||||
@ -6835,102 +6801,13 @@ export function ImageNode({
|
||||
return;
|
||||
}
|
||||
|
||||
const actionName = getActionName(action);
|
||||
|
||||
setImageAdjustPanelOpen(false);
|
||||
setCropPanelOpen(false);
|
||||
setCropInteraction(null);
|
||||
setFlipRotatePanelOpen(false);
|
||||
setAnglePanelOpen(false);
|
||||
setIsCreatingDetachedResultNode(true);
|
||||
setIsProcessing(true);
|
||||
// 先创建一个"处理中"状态的节点，显示在原图右侧
|
||||
const newNodeId = addNode({
|
||||
type: "image",
|
||||
position: {
|
||||
x: node.position.x + node.size.width + 20,
|
||||
y: node.position.y,
|
||||
},
|
||||
size: { width: node.size.width, height: node.size.height },
|
||||
data: {
|
||||
src: "", // 暂时没有图片
|
||||
status: "generating",
|
||||
prompt: `${actionName}处理中...`,
|
||||
title: `${displayTitle}-${actionName}`,
|
||||
originalSrc: data.src,
|
||||
},
|
||||
connections: [node.id],
|
||||
});
|
||||
|
||||
let result;
|
||||
|
||||
try {
|
||||
switch (action) {
|
||||
case "rmbg":
|
||||
result = await imageGenerationService.removeBackground(data.src);
|
||||
break;
|
||||
case "watermark":
|
||||
result = await imageGenerationService.removeWatermark(data.src);
|
||||
break;
|
||||
case "enhance":
|
||||
result = await imageGenerationService.enhanceImage(data.src);
|
||||
break;
|
||||
case "extend":
|
||||
result = await imageGenerationService.extendImage(data.src);
|
||||
break;
|
||||
default:
|
||||
// 删除创建的节点
|
||||
if (newNodeId) deleteNode(newNodeId);
|
||||
return;
|
||||
}
|
||||
|
||||
if (result?.success && result.data?.url) {
|
||||
// 更新节点为完成状态
|
||||
if (newNodeId) {
|
||||
updateNode(newNodeId, {
|
||||
data: {
|
||||
src: result.data.url,
|
||||
status: "completed",
|
||||
prompt: `${actionName}结果`,
|
||||
title: `${displayTitle}-${actionName}`,
|
||||
originalSrc: data.src,
|
||||
},
|
||||
});
|
||||
}
|
||||
toast.success(`${actionName}处理完成`);
|
||||
} else {
|
||||
// 更新节点为失败状态
|
||||
if (newNodeId) {
|
||||
updateNode(newNodeId, {
|
||||
data: {
|
||||
src: "",
|
||||
status: "failed",
|
||||
prompt: `${actionName}失败`,
|
||||
title: `${displayTitle}-${actionName}-失败`,
|
||||
originalSrc: data.src,
|
||||
},
|
||||
});
|
||||
}
|
||||
toast.error(result?.error || "处理失败");
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`${action} 处理失败:`, error);
|
||||
// 更新节点为失败状态
|
||||
if (newNodeId) {
|
||||
updateNode(newNodeId, {
|
||||
data: {
|
||||
src: "",
|
||||
status: "failed",
|
||||
prompt: `${actionName}失败`,
|
||||
title: `${displayTitle}-${actionName}-失败`,
|
||||
originalSrc: data.src,
|
||||
},
|
||||
});
|
||||
}
|
||||
toast.error("处理失败，请重试");
|
||||
} finally {
|
||||
setIsCreatingDetachedResultNode(false);
|
||||
setIsProcessing(false);
|
||||
if (
|
||||
action === "rmbg" ||
|
||||
action === "watermark" ||
|
||||
action === "enhance" ||
|
||||
action === "extend"
|
||||
) {
|
||||
await executeSharedToolbarCapability(action);
|
||||
}
|
||||
};
|
||||
|
||||
@ -6974,7 +6851,16 @@ export function ImageNode({
|
||||
const action = customEvent.detail?.action;
|
||||
|
||||
if (!targetNodeId || !action || targetNodeId !== node.id) return;
|
||||
void handleAction(action);
|
||||
if (action === "scene-fusion" || action === "annotate") {
|
||||
void handleAction(action);
|
||||
return;
|
||||
}
|
||||
const capability = getCanvasCapabilityByClientAction(
|
||||
action as CanvasCapabilityClientAction
|
||||
);
|
||||
if (capability) {
|
||||
void handleAction(capability.clientAction);
|
||||
}
|
||||
};
|
||||
|
||||
window.addEventListener(
|
||||
@ -7036,105 +6922,6 @@ export function ImageNode({
|
||||
]
|
||||
);
|
||||
|
||||
const handleLayerSplitComplete = useCallback(
|
||||
(
|
||||
layers: Array<{
|
||||
originalUrl: string;
|
||||
title?: string;
|
||||
layerIndex: number;
|
||||
totalLayers: number;
|
||||
bounds?: {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
originalWidth: number;
|
||||
originalHeight: number;
|
||||
coverage: number;
|
||||
} | null;
|
||||
}>
|
||||
) => {
|
||||
const validLayers = layers.filter(
|
||||
(item) =>
|
||||
typeof item?.originalUrl === "string" &&
|
||||
item.originalUrl.trim().length > 0
|
||||
);
|
||||
|
||||
if (validLayers.length === 0) {
|
||||
throw new Error("没有可添加到画布的图层结果");
|
||||
}
|
||||
|
||||
const isFullFrameLayer = (bounds?: LayerSplitBounds | null) => {
|
||||
if (!bounds) return false;
|
||||
return (
|
||||
bounds.coverage >= 0.92 &&
|
||||
bounds.width >= bounds.originalWidth - 2 &&
|
||||
bounds.height >= bounds.originalHeight - 2
|
||||
);
|
||||
};
|
||||
|
||||
const backgroundLayers = validLayers
|
||||
.filter((layer) => isFullFrameLayer(layer.bounds))
|
||||
.sort(
|
||||
(left, right) => (left.layerIndex || 0) - (right.layerIndex || 0)
|
||||
);
|
||||
const subjectLayers = validLayers
|
||||
.filter((layer) => !isFullFrameLayer(layer.bounds))
|
||||
.sort(
|
||||
(left, right) => (left.layerIndex || 0) - (right.layerIndex || 0)
|
||||
);
|
||||
const orderedLayers = [...backgroundLayers, ...subjectLayers];
|
||||
const groupGap = 28;
|
||||
const groupOffsetX = node.size.width + groupGap;
|
||||
const groupOffsetY = 0;
|
||||
|
||||
orderedLayers.forEach((layer, index) => {
|
||||
const layerTitle = `${displayTitle}-${layer.title || `图层${layer.layerIndex || index + 1}`}`;
|
||||
const bounds = layer.bounds;
|
||||
const fullFrameLayer = isFullFrameLayer(bounds);
|
||||
const targetPosition = {
|
||||
x: node.position.x + groupOffsetX,
|
||||
y: node.position.y + groupOffsetY,
|
||||
};
|
||||
const targetSize = { width: node.size.width, height: node.size.height };
|
||||
|
||||
addNode({
|
||||
type: "image",
|
||||
position: targetPosition,
|
||||
size: targetSize,
|
||||
data: {
|
||||
src: layer.originalUrl,
|
||||
status: "completed",
|
||||
prompt: `图层拆分结果 ${layer.layerIndex}/${layer.totalLayers}`,
|
||||
title: layerTitle,
|
||||
originalSrc: data.src,
|
||||
metadata: {
|
||||
layerIndex: layer.layerIndex,
|
||||
totalLayers: layer.totalLayers,
|
||||
bounds: layer.bounds,
|
||||
allowOverlap: true,
|
||||
placementMode: "overlay",
|
||||
layerSplitRole: fullFrameLayer ? "background" : "subject",
|
||||
},
|
||||
},
|
||||
connections: [node.id],
|
||||
});
|
||||
});
|
||||
|
||||
return orderedLayers.length;
|
||||
},
|
||||
[
|
||||
addNode,
|
||||
data.src,
|
||||
displayTitle,
|
||||
node.id,
|
||||
node.position.x,
|
||||
node.position.y,
|
||||
node.size.height,
|
||||
node.size.width,
|
||||
]
|
||||
);
|
||||
|
||||
// 图片标注完成回调
|
||||
const handleAnnotateComplete = async (
|
||||
annotatedImageUrl: string,
|
||||
@ -8364,6 +8151,78 @@ export function ImageNode({
|
||||
setDownloadProgress(null);
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
if (!imageNodeActionRequest || imageNodeActionRequest.nodeId !== node.id) {
|
||||
return;
|
||||
}
|
||||
if (
|
||||
processedCapabilityCommandIdsRef.current.has(
|
||||
imageNodeActionRequest.requestId
|
||||
)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
processedCapabilityCommandIdsRef.current.add(imageNodeActionRequest.requestId);
|
||||
|
||||
const executeCommand = async () => {
|
||||
const { action, args } = imageNodeActionRequest;
|
||||
|
||||
if (action === "add-to-chat") {
|
||||
await handleAddToChat();
|
||||
return;
|
||||
}
|
||||
if (action === "pick-object") {
|
||||
handleStartPicking();
|
||||
return;
|
||||
}
|
||||
if (action === "rename") {
|
||||
const requestedTitle = normalizeImageTitle(String(args.title || ""));
|
||||
if (requestedTitle) {
|
||||
updateNode(node.id, {
|
||||
data: { ...data, title: requestedTitle },
|
||||
});
|
||||
setDraftTitle(requestedTitle);
|
||||
toast.success("图片名称已更新");
|
||||
} else {
|
||||
setDraftTitle(displayTitle);
|
||||
setIsEditingTitle(true);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (action === "download") {
|
||||
await handleDownload();
|
||||
return;
|
||||
}
|
||||
if (action === "delete") {
|
||||
if (imageNodeActionRequest.confirmed === true) {
|
||||
deleteNode(node.id);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
await handleAction(action);
|
||||
};
|
||||
|
||||
void executeCommand().finally(() => {
|
||||
processedCapabilityCommandIdsRef.current.delete(
|
||||
imageNodeActionRequest.requestId
|
||||
);
|
||||
clearImageNodeActionRequest();
|
||||
});
|
||||
}, [
|
||||
clearImageNodeActionRequest,
|
||||
data,
|
||||
deleteNode,
|
||||
displayTitle,
|
||||
handleAction,
|
||||
handleAddToChat,
|
||||
handleDownload,
|
||||
handleStartPicking,
|
||||
imageNodeActionRequest,
|
||||
node.id,
|
||||
updateNode,
|
||||
]);
|
||||
|
||||
const imageAdjustOverlayStyle = useMemo<React.CSSProperties>(() => {
|
||||
if (boxCutoutMetrics) {
|
||||
return {
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
"use client";
|
||||
|
||||
import React, { useMemo, useState } from "react";
|
||||
import React, { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import {
|
||||
AlertCircle,
|
||||
Box,
|
||||
@ -43,6 +43,7 @@ import {
|
||||
type CanvasToolbarConfig,
|
||||
} from "@/lib/canvas-toolbar-config";
|
||||
import { useCanvasStore } from "@/store/canvas";
|
||||
import { canvasService, model3DGenerationService } from "@/services/canvas";
|
||||
import type { CanvasNode, Model3DNodeData } from "@/types/canvas";
|
||||
import { cn, fetcher } from "@/lib/utils";
|
||||
|
||||
@ -61,6 +62,15 @@ const CANVAS_MODEL_VIEWER_SETTINGS: Model3DViewerSettings = {
|
||||
wireframe: false,
|
||||
};
|
||||
|
||||
const MODEL_3D_RECOVERY_INTERVAL_MS = 5000;
|
||||
const MODEL_3D_RECOVERY_MAX_ATTEMPTS = 360;
|
||||
|
||||
function isTerminalModelFailureStatus(status: string) {
|
||||
return ["failed", "error", "banned", "expired", "cancelled", "canceled"].includes(
|
||||
status.trim().toLowerCase()
|
||||
);
|
||||
}
|
||||
|
||||
function inferModelFileName(modelUrl: string | null, nodeId: string) {
|
||||
const raw = String(modelUrl || "").trim();
|
||||
if (!raw) return `model3d-${nodeId.slice(0, 6)}.glb`;
|
||||
@ -93,7 +103,12 @@ export function Model3DNode({
|
||||
}: Model3DNodeProps) {
|
||||
const data = node.data as Model3DNodeData;
|
||||
const [isHovered, setIsHovered] = useState(false);
|
||||
const recoveryAttemptRef = useRef(0);
|
||||
const recoveryTaskRef = useRef<string | null>(null);
|
||||
const deleteNode = useCanvasStore((state) => state.deleteNode);
|
||||
const updateNode = useCanvasStore((state) => state.updateNode);
|
||||
const projectId = useCanvasStore((state) => state.projectId);
|
||||
const getCanvasState = useCanvasStore((state) => state.getCanvasState);
|
||||
const model3dOpen3dItem = getCanvasToolbarItem(
|
||||
canvasToolbarConfig,
|
||||
"model3dOpen3d"
|
||||
@ -127,6 +142,17 @@ export function Model3DNode({
|
||||
"model3dDelete"
|
||||
);
|
||||
const viewportZoom = useCanvasStore((state) => state.viewport.zoom);
|
||||
const taskId = String(data.metadata?.taskId || "").trim();
|
||||
const assetPayload = readWorldAssetPayload(data);
|
||||
const providerId = String(
|
||||
data.metadata?.providerId || assetPayload?.providerId || ""
|
||||
).trim();
|
||||
const providerName = String(
|
||||
data.metadata?.providerName || assetPayload?.providerName || ""
|
||||
).trim();
|
||||
const sourceImageUrl = String(
|
||||
data.metadata?.sourceImageUrl || data.previewImage || ""
|
||||
).trim();
|
||||
const { data: modelAliasConfigResponse } = useSWR<{
|
||||
success?: boolean;
|
||||
data?: PublicThreeDModelAliasConfig;
|
||||
@ -199,6 +225,224 @@ export function Model3DNode({
|
||||
[placeholderViewportScale]
|
||||
);
|
||||
|
||||
const persistCanvasStateSnapshot = useCallback(() => {
|
||||
if (!projectId || typeof window === "undefined") return;
|
||||
window.setTimeout(() => {
|
||||
void canvasService.saveCanvasState(projectId, getCanvasState());
|
||||
}, 0);
|
||||
}, [getCanvasState, projectId]);
|
||||
|
||||
const persistCanvas3DAsset = useCallback(
|
||||
async (asset: Record<string, unknown>) => {
|
||||
if (!projectId) return;
|
||||
try {
|
||||
await fetch(`/api/canvas/${projectId}/model3d-assets`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ asset }),
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("[Model3DNode] 保存 3D 资产失败:", error);
|
||||
}
|
||||
},
|
||||
[projectId]
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (data.status !== "generating" || !taskId) return;
|
||||
const provider = String(data.metadata?.provider || "tripo-3d").trim();
|
||||
if (provider !== "tripo-3d") return;
|
||||
|
||||
if (recoveryTaskRef.current !== taskId) {
|
||||
recoveryTaskRef.current = taskId;
|
||||
recoveryAttemptRef.current = 0;
|
||||
}
|
||||
|
||||
let cancelled = false;
|
||||
let timer: ReturnType<typeof setTimeout> | null = null;
|
||||
const prompt = String(data.prompt || "图生 3D 模型").trim();
|
||||
const modelVersion = String(data.modelVersion || "tripo-standard").trim();
|
||||
const previewImage = String(data.previewImage || sourceImageUrl).trim();
|
||||
|
||||
const currentData = () => {
|
||||
const currentNode = getCanvasState().nodes.find((item) => item.id === node.id);
|
||||
return (currentNode?.data || data) as Model3DNodeData;
|
||||
};
|
||||
|
||||
const persistTerminalAsset = async (params: {
|
||||
error: string | null;
|
||||
modelUrl: string | null;
|
||||
previewImageUrl: string;
|
||||
progress: number | null;
|
||||
status: "failed" | "success";
|
||||
taskAssetPayload?: Record<string, unknown> | null;
|
||||
}) => {
|
||||
await persistCanvas3DAsset({
|
||||
taskId,
|
||||
provider: "tripo-3d",
|
||||
status: params.status,
|
||||
mode: "image",
|
||||
prompt,
|
||||
modelVersion,
|
||||
progress: params.progress,
|
||||
modelUrl: params.modelUrl,
|
||||
previewImageUrl: params.previewImageUrl,
|
||||
error: params.error,
|
||||
assetPayload: {
|
||||
...(params.taskAssetPayload || {}),
|
||||
providerId: providerId || null,
|
||||
providerName: providerName || null,
|
||||
sourceImageUrl,
|
||||
},
|
||||
createdAt: Date.now(),
|
||||
});
|
||||
persistCanvasStateSnapshot();
|
||||
};
|
||||
|
||||
const markFailed = async (message: string, progress: number | null) => {
|
||||
const latest = currentData();
|
||||
updateNode(node.id, {
|
||||
data: {
|
||||
...latest,
|
||||
src: "",
|
||||
previewImage: previewImage || sourceImageUrl,
|
||||
status: "failed",
|
||||
metadata: {
|
||||
...(latest.metadata || {}),
|
||||
taskId,
|
||||
provider: "tripo-3d",
|
||||
mode: "image",
|
||||
sourceImageUrl,
|
||||
progress,
|
||||
error: message,
|
||||
},
|
||||
} as Model3DNodeData,
|
||||
});
|
||||
await persistTerminalAsset({
|
||||
error: message,
|
||||
modelUrl: null,
|
||||
previewImageUrl: previewImage || sourceImageUrl,
|
||||
progress,
|
||||
status: "failed",
|
||||
});
|
||||
toast.error(message);
|
||||
};
|
||||
|
||||
const scheduleNextPoll = () => {
|
||||
if (cancelled) return;
|
||||
if (recoveryAttemptRef.current >= MODEL_3D_RECOVERY_MAX_ATTEMPTS) {
|
||||
void markFailed("3D 模型任务同步超时，请稍后重试", null);
|
||||
return;
|
||||
}
|
||||
timer = setTimeout(() => {
|
||||
void pollTask();
|
||||
}, MODEL_3D_RECOVERY_INTERVAL_MS);
|
||||
};
|
||||
|
||||
async function pollTask() {
|
||||
if (cancelled) return;
|
||||
recoveryAttemptRef.current += 1;
|
||||
const result = await model3DGenerationService.getTask(taskId, {
|
||||
providerId: providerId || null,
|
||||
});
|
||||
if (cancelled) return;
|
||||
if (!result.success || !result.data) {
|
||||
scheduleNextPoll();
|
||||
return;
|
||||
}
|
||||
|
||||
const status = String(result.data.status || "unknown").toLowerCase();
|
||||
const progress = result.data.progress ?? null;
|
||||
if (status === "success") {
|
||||
const modelUrl = String(result.data.modelUrl || "").trim();
|
||||
const nextPreviewImage = String(
|
||||
result.data.previewImageUrl || previewImage || sourceImageUrl
|
||||
).trim();
|
||||
if (!modelUrl) {
|
||||
scheduleNextPoll();
|
||||
return;
|
||||
}
|
||||
const taskAssetPayload =
|
||||
result.data.assetPayload && typeof result.data.assetPayload === "object"
|
||||
? (result.data.assetPayload as Record<string, unknown>)
|
||||
: {};
|
||||
const latest = currentData();
|
||||
updateNode(node.id, {
|
||||
data: {
|
||||
...latest,
|
||||
src: modelUrl,
|
||||
previewImage: nextPreviewImage,
|
||||
format: inferModelFormat(modelUrl),
|
||||
status: "completed",
|
||||
metadata: {
|
||||
...(latest.metadata || {}),
|
||||
taskId,
|
||||
provider: "tripo-3d",
|
||||
mode: "image",
|
||||
sourceImageUrl,
|
||||
progress: 100,
|
||||
output: result.data.output || null,
|
||||
assetPayload: {
|
||||
...taskAssetPayload,
|
||||
providerId: providerId || taskAssetPayload.providerId || null,
|
||||
providerName: providerName || taskAssetPayload.providerName || null,
|
||||
sourceImageUrl,
|
||||
},
|
||||
recoveredAt: new Date().toISOString(),
|
||||
},
|
||||
} as Model3DNodeData,
|
||||
});
|
||||
await persistTerminalAsset({
|
||||
error: null,
|
||||
modelUrl,
|
||||
previewImageUrl: nextPreviewImage,
|
||||
progress: 100,
|
||||
status: "success",
|
||||
taskAssetPayload,
|
||||
});
|
||||
toast.success("3D 模型已生成到画布");
|
||||
return;
|
||||
}
|
||||
|
||||
if (isTerminalModelFailureStatus(status)) {
|
||||
await markFailed(result.data.errorMessage || "3D 模型生成失败", progress);
|
||||
return;
|
||||
}
|
||||
|
||||
const latest = currentData();
|
||||
updateNode(node.id, {
|
||||
data: {
|
||||
...latest,
|
||||
metadata: {
|
||||
...(latest.metadata || {}),
|
||||
progress,
|
||||
},
|
||||
} as Model3DNodeData,
|
||||
});
|
||||
scheduleNextPoll();
|
||||
}
|
||||
|
||||
void pollTask();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
if (timer) clearTimeout(timer);
|
||||
};
|
||||
}, [
|
||||
data.modelVersion,
|
||||
data.previewImage,
|
||||
data.prompt,
|
||||
data.status,
|
||||
getCanvasState,
|
||||
node.id,
|
||||
persistCanvas3DAsset,
|
||||
persistCanvasStateSnapshot,
|
||||
providerId,
|
||||
providerName,
|
||||
sourceImageUrl,
|
||||
taskId,
|
||||
updateNode,
|
||||
]);
|
||||
|
||||
const handleOpen = () => {
|
||||
if (!primaryOpenUrl) {
|
||||
toast.error("暂无可打开的地址");
|
||||
|
||||
61
docs/无限画布-Capability-Runtime与MCP接入说明.md
Normal file
61
docs/无限画布-Capability-Runtime与MCP接入说明.md
Normal file
@ -0,0 +1,61 @@
|
||||
# 无限画布 Capability Runtime 与 MCP 接入说明
|
||||
|
||||
## 目标
|
||||
|
||||
无限画布的图片工具栏、Agent 和 MCP 共用同一套能力定义与执行器。MCP 只负责协议适配，不复制抠图、增强、3D 等业务逻辑。
|
||||
|
||||
## 调用链
|
||||
|
||||
```text
|
||||
图片工具栏 ─┐
|
||||
AI SDK Tools ├─> Canvas Capability Registry ─> Runtime/Queue ─> 画布写回
|
||||
MCP Tools ───┘ └──────────────> Client Command（交互面板）
|
||||
```
|
||||
|
||||
- 能力注册：`lib/canvas/capabilities/registry.ts`
|
||||
- 参数与命令协议：`lib/canvas/capabilities/client-command.ts`
|
||||
- 统一运行时：`lib/canvas/capabilities/runtime.ts`
|
||||
- 后台执行器：`lib/canvas/capabilities/headless-executor.ts`
|
||||
- 持久化任务：`lib/canvas/capabilities/task-store.ts`
|
||||
- 项目内顺序队列：`lib/canvas/capabilities/task-queue.ts`
|
||||
- AI SDK 工具：`lib/canvas/capabilities/ai-sdk-tools.ts`
|
||||
- MCP 适配器：`lib/canvas/capabilities/mcp-server.ts`
|
||||
- 前端命令总线：`store/canvas/index.ts`
|
||||
|
||||
## HTTP 入口
|
||||
|
||||
- 应用能力接口：`GET/POST /api/canvas/:projectId/capabilities`
|
||||
- MCP Streamable HTTP：`GET/POST/DELETE /api/mcp/canvas/:projectId`
|
||||
|
||||
两个入口都使用当前登录会话鉴权，运行时会再次校验画布归属和节点类型。MCP 调用必须传 `nodeIds`；连续调用时可将上一步返回的 `createdNodeIds` 传给下一步。
|
||||
|
||||
## 执行模式
|
||||
|
||||
- `headless`：服务端直接执行，完成后写入新节点。
|
||||
- `interactive`：通过 SSE 下发客户端命令，打开裁剪、调色、局部编辑等现有面板。
|
||||
- `hybrid`：参数完整时后台执行，否则打开交互面板。
|
||||
- 删除等危险能力必须明确确认，并使用请求号保证幂等。
|
||||
|
||||
Tripo 3D 与 World Labs 3D 世界由 Agent/MCP 直接创建任务和结果节点。节点保存任务 ID，页面刷新后继续轮询并恢复最终结果。
|
||||
|
||||
## 部署
|
||||
|
||||
部署新版本前必须执行：
|
||||
|
||||
```bash
|
||||
pnpm install --frozen-lockfile
|
||||
pnpm db:migrate
|
||||
pnpm build
|
||||
```
|
||||
|
||||
迁移 `0090_add_canvas_capability_tasks.sql` 新增持久化任务表。未执行迁移时，能力任务无法启动。
|
||||
|
||||
## 增加服务商或能力
|
||||
|
||||
1. 在 Registry 新增或扩展能力 Schema、风险、模型策略和工具名称。
|
||||
2. 在共享后台执行器中接入服务商路由；不要在 React 组件或 MCP 回调中直接实现服务商逻辑。
|
||||
3. 结果统一返回媒体/3D 节点描述，由运行时写回画布。
|
||||
4. 需要用户框选或配置时，复用现有 `clientAction` 和前端面板。
|
||||
5. 添加能力契约、MCP 工具发现、写回和失败恢复测试。
|
||||
|
||||
服务商切换应由系统模型配置和现有服务层完成。Agent/MCP 只传模型别名或能力参数，不持有服务商密钥。
|
||||
@ -104,7 +104,10 @@ function findNode(nodes: any[], nodeId?: string) {
|
||||
}
|
||||
|
||||
function mediaNodeType(item: CanvasMediaItem) {
|
||||
return item.type === "video" ? "video" : "image";
|
||||
if (item.type === "video") return "video";
|
||||
if (item.type === "model3d") return "model3d";
|
||||
if (item.type === "world3d") return "world3d";
|
||||
return "image";
|
||||
}
|
||||
|
||||
function buildMediaNode(params: {
|
||||
@ -138,6 +141,13 @@ function buildMediaNode(params: {
|
||||
const primarySourceNodeId = String(
|
||||
itemMetadata.sourceNodeId || planMetadata.sourceNodeId || params.sourceNodeIds[0] || ""
|
||||
).trim();
|
||||
const modelVersion = String(
|
||||
itemMetadata.modelVersion || planMetadata.modelVersion || ""
|
||||
).trim();
|
||||
const taskStatus =
|
||||
itemMetadata.status === "completed" || itemMetadata.status === "failed"
|
||||
? itemMetadata.status
|
||||
: "generating";
|
||||
const metadata = {
|
||||
...planMetadata,
|
||||
...itemMetadata,
|
||||
@ -160,29 +170,57 @@ function buildMediaNode(params: {
|
||||
: undefined,
|
||||
};
|
||||
|
||||
let data: Record<string, unknown>;
|
||||
if (type === "video") {
|
||||
data = {
|
||||
src: params.item.url,
|
||||
assetId: params.item.assetId,
|
||||
status: "completed",
|
||||
prompt: params.item.prompt,
|
||||
title: params.item.title || "Agent 视频结果",
|
||||
metadata,
|
||||
};
|
||||
} else if (type === "model3d") {
|
||||
data = {
|
||||
src: String(itemMetadata.modelUrl || "").trim(),
|
||||
previewImage: params.item.url,
|
||||
prompt: params.item.prompt,
|
||||
title: params.item.title || "Agent 3D 模型结果",
|
||||
modelVersion: modelVersion || "tripo-standard",
|
||||
format: String(itemMetadata.format || "glb"),
|
||||
status: taskStatus,
|
||||
metadata,
|
||||
};
|
||||
} else if (type === "world3d") {
|
||||
data = {
|
||||
imageUrl: params.item.url,
|
||||
previewImage: params.item.url,
|
||||
prompt: params.item.prompt,
|
||||
title: params.item.title || "Agent 3D 世界结果",
|
||||
modelVersion: modelVersion || "world-standard",
|
||||
status: taskStatus,
|
||||
activeView: "image",
|
||||
firstPerson: false,
|
||||
hotspots: [],
|
||||
metadata,
|
||||
};
|
||||
} else {
|
||||
data = {
|
||||
src: params.item.url,
|
||||
assetId: params.item.assetId,
|
||||
status: "completed",
|
||||
prompt: params.item.prompt,
|
||||
title: params.item.title || "Agent 图片结果",
|
||||
metadata,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
id: generateId(),
|
||||
type,
|
||||
position: params.position,
|
||||
size: params.size,
|
||||
data:
|
||||
type === "video"
|
||||
? {
|
||||
src: params.item.url,
|
||||
assetId: params.item.assetId,
|
||||
status: "completed",
|
||||
prompt: params.item.prompt,
|
||||
title: params.item.title || "Agent 视频结果",
|
||||
metadata,
|
||||
}
|
||||
: {
|
||||
src: params.item.url,
|
||||
assetId: params.item.assetId,
|
||||
status: "completed",
|
||||
prompt: params.item.prompt,
|
||||
title: params.item.title || "Agent 图片结果",
|
||||
metadata,
|
||||
},
|
||||
data,
|
||||
connections: params.sourceNodeIds,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
|
||||
150
lib/canvas/capabilities/ai-sdk-tools.ts
Normal file
150
lib/canvas/capabilities/ai-sdk-tools.ts
Normal file
@ -0,0 +1,150 @@
|
||||
import { tool, type ToolSet } from "ai";
|
||||
import { z } from "zod";
|
||||
|
||||
import { createCanvasCapabilityClientCommand } from "@/lib/canvas/capabilities/client-command";
|
||||
import { CANVAS_CAPABILITY_REGISTRY } from "@/lib/canvas/capabilities/registry";
|
||||
import {
|
||||
executeCanvasCapabilityRuntime,
|
||||
type CanvasCapabilityRuntimeResult,
|
||||
} from "@/lib/canvas/capabilities/runtime";
|
||||
import type { CanvasCapabilityDefinition } from "@/lib/canvas/capabilities/types";
|
||||
|
||||
type CanvasCapabilityToolCallbacks = {
|
||||
onClientCommand?: (
|
||||
command: NonNullable<CanvasCapabilityRuntimeResult["clientCommand"]>,
|
||||
capability: CanvasCapabilityDefinition
|
||||
) => void;
|
||||
onResult?: (
|
||||
result: CanvasCapabilityRuntimeResult,
|
||||
capability: CanvasCapabilityDefinition
|
||||
) => void;
|
||||
};
|
||||
|
||||
function isCapabilityAvailableForSelection(params: {
|
||||
capability: CanvasCapabilityDefinition;
|
||||
selectedNodeIds: string[];
|
||||
}) {
|
||||
const count = params.selectedNodeIds.length;
|
||||
return (
|
||||
params.capability.agent.exposed &&
|
||||
count >= params.capability.selection.min &&
|
||||
(!params.capability.selection.max || count <= params.capability.selection.max)
|
||||
);
|
||||
}
|
||||
|
||||
function buildToolDescription(capability: CanvasCapabilityDefinition) {
|
||||
const modeDescription =
|
||||
capability.executionMode === "interactive"
|
||||
? "该工具会在用户画布中打开交互面板。"
|
||||
: capability.executionMode === "headless"
|
||||
? "该工具会在后台执行并把结果写回画布。"
|
||||
: "参数完整时后台执行，否则在用户画布中打开交互面板。";
|
||||
const aliases = capability.agent.aliases.length
|
||||
? `常见表达：${capability.agent.aliases.join("、")}。`
|
||||
: "";
|
||||
return `${capability.description}${modeDescription}${aliases}默认作用于当前选中图片；连续操作时可把上一步返回的 createdNodeIds 传入 nodeIds。`;
|
||||
}
|
||||
|
||||
export async function createCanvasCapabilityTools(params: {
|
||||
callbacks?: CanvasCapabilityToolCallbacks;
|
||||
defaultModelId?: string;
|
||||
destructiveConfirmed?: boolean;
|
||||
projectId: string;
|
||||
requestHeaders: Headers;
|
||||
requestUrl: string;
|
||||
selectedNodeIds: string[];
|
||||
userId: string;
|
||||
}) {
|
||||
const selectedNodeIds = Array.from(
|
||||
new Set(
|
||||
params.selectedNodeIds
|
||||
.map((id) => String(id || "").trim())
|
||||
.filter(Boolean)
|
||||
)
|
||||
);
|
||||
const tools: ToolSet = {};
|
||||
const { getEnabledCanvasCapabilityIds } = await import(
|
||||
"@/lib/canvas/capabilities/availability"
|
||||
);
|
||||
const enabledCapabilityIds = await getEnabledCanvasCapabilityIds(
|
||||
CANVAS_CAPABILITY_REGISTRY
|
||||
);
|
||||
|
||||
for (const capability of CANVAS_CAPABILITY_REGISTRY) {
|
||||
if (!enabledCapabilityIds.has(capability.id)) continue;
|
||||
if (!isCapabilityAvailableForSelection({ capability, selectedNodeIds })) continue;
|
||||
const toolName = capability.agent.toolName;
|
||||
if (!toolName) continue;
|
||||
|
||||
tools[toolName] = tool({
|
||||
description: buildToolDescription(capability),
|
||||
inputSchema: z.object({
|
||||
nodeIds: z
|
||||
.array(z.string().trim().min(1).max(180))
|
||||
.min(1)
|
||||
.max(50)
|
||||
.optional()
|
||||
.describe("可选。传入上一个工具返回的 createdNodeIds 可链式处理新结果。"),
|
||||
args: capability.inputSchema.optional().describe("能力参数。"),
|
||||
}),
|
||||
execute: async (input) => {
|
||||
const rawArgs =
|
||||
input.args && typeof input.args === "object" && !Array.isArray(input.args)
|
||||
? input.args
|
||||
: {};
|
||||
const args = {
|
||||
...rawArgs,
|
||||
...(capability.modelPolicy === "agent-selected" &&
|
||||
params.defaultModelId &&
|
||||
!("modelId" in rawArgs)
|
||||
? { modelId: params.defaultModelId }
|
||||
: {}),
|
||||
};
|
||||
const command = createCanvasCapabilityClientCommand({
|
||||
capabilityId: capability.id,
|
||||
nodeIds: input.nodeIds?.length ? input.nodeIds : selectedNodeIds,
|
||||
projectId: params.projectId,
|
||||
args,
|
||||
source: "agent",
|
||||
confirmed:
|
||||
capability.risk === "destructive"
|
||||
? params.destructiveConfirmed === true
|
||||
: undefined,
|
||||
});
|
||||
const result = await executeCanvasCapabilityRuntime({
|
||||
command,
|
||||
projectId: params.projectId,
|
||||
requestHeaders: params.requestHeaders,
|
||||
requestUrl: params.requestUrl,
|
||||
userId: params.userId,
|
||||
});
|
||||
|
||||
if (result.clientCommand) {
|
||||
params.callbacks?.onClientCommand?.(result.clientCommand, capability);
|
||||
}
|
||||
params.callbacks?.onResult?.(result, capability);
|
||||
|
||||
return {
|
||||
capabilityId: capability.id,
|
||||
title: capability.title,
|
||||
mode: result.mode,
|
||||
status: result.task.status,
|
||||
taskId: result.task.id,
|
||||
requestId: result.task.requestId,
|
||||
createdNodeIds: result.operation?.createdNodeIds || [],
|
||||
updatedNodeIds: result.operation?.updatedNodeIds || [],
|
||||
outputs: result.outputs || [],
|
||||
clientAction: result.clientCommand
|
||||
? capability.clientAction
|
||||
: undefined,
|
||||
};
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return tools;
|
||||
}
|
||||
|
||||
export function getCanvasCapabilityToolNames(tools: ToolSet) {
|
||||
return Object.keys(tools);
|
||||
}
|
||||
38
lib/canvas/capabilities/availability.ts
Normal file
38
lib/canvas/capabilities/availability.ts
Normal file
@ -0,0 +1,38 @@
|
||||
import "server-only";
|
||||
|
||||
import {
|
||||
CANVAS_TOOLBAR_CONFIG_SETTING_KEY,
|
||||
isCanvasToolbarItemEnabled,
|
||||
normalizeCanvasToolbarConfig,
|
||||
type CanvasToolbarItemKey,
|
||||
} from "@/lib/canvas-toolbar-config";
|
||||
import type { CanvasCapabilityDefinition } from "@/lib/canvas/capabilities/types";
|
||||
import { getSystemSetting } from "@/lib/db/queries";
|
||||
|
||||
export async function isCanvasCapabilityEnabled(
|
||||
capability: CanvasCapabilityDefinition
|
||||
) {
|
||||
const enabledIds = await getEnabledCanvasCapabilityIds([capability]);
|
||||
return enabledIds.has(capability.id);
|
||||
}
|
||||
|
||||
export async function getEnabledCanvasCapabilityIds(
|
||||
capabilities: readonly CanvasCapabilityDefinition[]
|
||||
) {
|
||||
const setting = await getSystemSetting({
|
||||
key: CANVAS_TOOLBAR_CONFIG_SETTING_KEY,
|
||||
}).catch(() => null);
|
||||
const config = normalizeCanvasToolbarConfig(setting?.value);
|
||||
return new Set(
|
||||
capabilities
|
||||
.filter(
|
||||
(capability) =>
|
||||
!capability.toolbarSettingKey ||
|
||||
isCanvasToolbarItemEnabled(
|
||||
config,
|
||||
capability.toolbarSettingKey as CanvasToolbarItemKey
|
||||
)
|
||||
)
|
||||
.map((capability) => capability.id)
|
||||
);
|
||||
}
|
||||
59
lib/canvas/capabilities/client-command.ts
Normal file
59
lib/canvas/capabilities/client-command.ts
Normal file
@ -0,0 +1,59 @@
|
||||
import { z } from "zod";
|
||||
import { getCanvasCapability, parseCanvasCapabilityArgs } from "./registry";
|
||||
import {
|
||||
CANVAS_CAPABILITY_IDS,
|
||||
type CanvasCapabilityCommandSource,
|
||||
type CanvasCapabilityId,
|
||||
} from "./types";
|
||||
|
||||
export const canvasCapabilityClientCommandSchema = z.object({
|
||||
version: z.literal(1),
|
||||
requestId: z.string().trim().min(1).max(180),
|
||||
capabilityId: z.enum(CANVAS_CAPABILITY_IDS),
|
||||
projectId: z.string().trim().min(1).max(180).optional(),
|
||||
nodeIds: z.array(z.string().trim().min(1).max(180)).min(1).max(50),
|
||||
args: z.record(z.unknown()).default({}),
|
||||
source: z.enum(["toolbar", "agent", "mcp", "system"]),
|
||||
confirmed: z.boolean().optional(),
|
||||
createdAt: z.string().datetime(),
|
||||
});
|
||||
|
||||
export type CanvasCapabilityClientCommand = z.infer<
|
||||
typeof canvasCapabilityClientCommandSchema
|
||||
>;
|
||||
|
||||
export function createCanvasCapabilityClientCommand(params: {
|
||||
capabilityId: CanvasCapabilityId;
|
||||
nodeIds: string[];
|
||||
projectId?: string;
|
||||
requestId?: string;
|
||||
args?: unknown;
|
||||
source: CanvasCapabilityCommandSource;
|
||||
confirmed?: boolean;
|
||||
createdAt?: string;
|
||||
}): CanvasCapabilityClientCommand {
|
||||
const capability = getCanvasCapability(params.capabilityId);
|
||||
if (!capability) throw new Error(`Unknown canvas capability: ${params.capabilityId}`);
|
||||
const nodeIds = Array.from(new Set(params.nodeIds.map((id) => id.trim()).filter(Boolean)));
|
||||
if (nodeIds.length < capability.selection.min) {
|
||||
throw new Error(`${capability.title} requires at least ${capability.selection.min} selected node(s)`);
|
||||
}
|
||||
if (capability.selection.max && nodeIds.length > capability.selection.max) {
|
||||
throw new Error(`${capability.title} supports at most ${capability.selection.max} selected node(s)`);
|
||||
}
|
||||
if (capability.confirmation === "always" && params.confirmed !== true) {
|
||||
throw new Error(`${capability.title} requires explicit confirmation`);
|
||||
}
|
||||
|
||||
return canvasCapabilityClientCommandSchema.parse({
|
||||
version: 1,
|
||||
requestId: params.requestId || crypto.randomUUID(),
|
||||
capabilityId: params.capabilityId,
|
||||
projectId: params.projectId,
|
||||
nodeIds,
|
||||
args: parseCanvasCapabilityArgs(params.capabilityId, params.args),
|
||||
source: params.source,
|
||||
confirmed: params.confirmed,
|
||||
createdAt: params.createdAt || new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
100
lib/canvas/capabilities/context.ts
Normal file
100
lib/canvas/capabilities/context.ts
Normal file
@ -0,0 +1,100 @@
|
||||
import { and, eq } from "drizzle-orm";
|
||||
|
||||
import type { CanvasCapabilityClientCommand } from "@/lib/canvas/capabilities/client-command";
|
||||
import { db } from "@/lib/db";
|
||||
import { canvasProject } from "@/lib/db/schema";
|
||||
|
||||
export class CanvasCapabilityRuntimeError extends Error {
|
||||
status: number;
|
||||
code: string;
|
||||
|
||||
constructor(message: string, status = 400, code = "CANVAS_CAPABILITY_ERROR") {
|
||||
super(message);
|
||||
this.name = "CanvasCapabilityRuntimeError";
|
||||
this.status = status;
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
|
||||
export type CanvasCapabilityTarget = {
|
||||
node: Record<string, any>;
|
||||
nodeId: string;
|
||||
sourceImageUrl: string;
|
||||
assetId?: string;
|
||||
title: string;
|
||||
};
|
||||
|
||||
export async function resolveCanvasCapabilityContext(params: {
|
||||
command: CanvasCapabilityClientCommand;
|
||||
projectId: string;
|
||||
requiresRemoteImage?: boolean;
|
||||
userId: string;
|
||||
}) {
|
||||
const [project] = await db
|
||||
.select()
|
||||
.from(canvasProject)
|
||||
.where(
|
||||
and(
|
||||
eq(canvasProject.id, params.projectId),
|
||||
eq(canvasProject.userId, params.userId)
|
||||
)
|
||||
)
|
||||
.limit(1);
|
||||
|
||||
if (!project) {
|
||||
throw new CanvasCapabilityRuntimeError(
|
||||
"画布不存在或无权访问",
|
||||
404,
|
||||
"CANVAS_CAPABILITY_PROJECT_NOT_FOUND"
|
||||
);
|
||||
}
|
||||
|
||||
const canvasData =
|
||||
project.canvasData && typeof project.canvasData === "object"
|
||||
? (project.canvasData as Record<string, any>)
|
||||
: {};
|
||||
const nodes = Array.isArray(canvasData.nodes) ? canvasData.nodes : [];
|
||||
const nodeById = new Map<string, Record<string, any>>(
|
||||
nodes.map((node: Record<string, any>) => [String(node?.id || ""), node])
|
||||
);
|
||||
const targets = params.command.nodeIds.map((nodeId) => {
|
||||
const node = nodeById.get(nodeId);
|
||||
if (!node) {
|
||||
throw new CanvasCapabilityRuntimeError(
|
||||
`选中节点不存在：${nodeId}`,
|
||||
404,
|
||||
"CANVAS_CAPABILITY_NODE_NOT_FOUND"
|
||||
);
|
||||
}
|
||||
if (node.type !== "image") {
|
||||
throw new CanvasCapabilityRuntimeError(
|
||||
`能力仅支持图片节点：${nodeId}`,
|
||||
400,
|
||||
"CANVAS_CAPABILITY_INVALID_NODE_TYPE"
|
||||
);
|
||||
}
|
||||
const data =
|
||||
node.data && typeof node.data === "object" ? node.data : {};
|
||||
const sourceImageUrl = String(data.src || data.url || "").trim();
|
||||
if (params.requiresRemoteImage !== false && !/^https?:\/\//i.test(sourceImageUrl)) {
|
||||
throw new CanvasCapabilityRuntimeError(
|
||||
`图片尚未上传完成：${nodeId}`,
|
||||
409,
|
||||
"CANVAS_CAPABILITY_IMAGE_NOT_REMOTE"
|
||||
);
|
||||
}
|
||||
return {
|
||||
node,
|
||||
nodeId,
|
||||
sourceImageUrl,
|
||||
assetId: String(data.assetId || "").trim() || undefined,
|
||||
title: String(data.title || data.prompt || "图片").trim() || "图片",
|
||||
} satisfies CanvasCapabilityTarget;
|
||||
});
|
||||
|
||||
return {
|
||||
project,
|
||||
canvasData,
|
||||
targets,
|
||||
};
|
||||
}
|
||||
44
lib/canvas/capabilities/execution-mode.ts
Normal file
44
lib/canvas/capabilities/execution-mode.ts
Normal file
@ -0,0 +1,44 @@
|
||||
import type { CanvasCapabilityClientCommand } from "@/lib/canvas/capabilities/client-command";
|
||||
import type { CanvasCapabilityDefinition } from "@/lib/canvas/capabilities/types";
|
||||
|
||||
function hasCompleteRect(args: Record<string, unknown>) {
|
||||
return [args.x, args.y, args.width, args.height].every((value) =>
|
||||
Number.isFinite(Number(value))
|
||||
);
|
||||
}
|
||||
|
||||
export function shouldExecuteCanvasCapabilityHeadlessly(params: {
|
||||
capability: CanvasCapabilityDefinition;
|
||||
command: CanvasCapabilityClientCommand;
|
||||
}) {
|
||||
const { capability, command } = params;
|
||||
if (capability.executionMode === "headless") return true;
|
||||
if (capability.executionMode === "interactive" || capability.executionMode === "local") {
|
||||
return false;
|
||||
}
|
||||
|
||||
switch (capability.id) {
|
||||
case "image.ai_wash":
|
||||
case "image.prompt_regenerate":
|
||||
return Boolean(String(command.args.prompt || "").trim());
|
||||
case "image.box_cutout":
|
||||
return hasCompleteRect(command.args);
|
||||
case "image.resize":
|
||||
return (
|
||||
Number.isFinite(Number(command.args.width)) &&
|
||||
Number.isFinite(Number(command.args.height))
|
||||
);
|
||||
case "image.multi_angle":
|
||||
case "image.generate_3d_model":
|
||||
case "image.generate_3d_world":
|
||||
return command.source === "agent" || command.source === "mcp";
|
||||
case "image.rename":
|
||||
return Boolean(String(command.args.title || "").trim());
|
||||
case "image.note":
|
||||
return Boolean(String(command.args.text || "").trim());
|
||||
case "image.delete":
|
||||
return command.confirmed === true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
471
lib/canvas/capabilities/headless-executor.ts
Normal file
471
lib/canvas/capabilities/headless-executor.ts
Normal file
@ -0,0 +1,471 @@
|
||||
import {
|
||||
executeBoxCutoutJob,
|
||||
} from "@/lib/ai/box-cutout/service";
|
||||
import { executeExtendJob } from "@/lib/ai/extend/service";
|
||||
import { executeLayerSplitJob } from "@/lib/ai/layer-split/service";
|
||||
import { executeResizeJob } from "@/lib/ai/resize/service";
|
||||
import {
|
||||
executeRmbgJob,
|
||||
fetchImageBufferFromUrl,
|
||||
} from "@/lib/ai/rmbg/service";
|
||||
import type { CanvasCapabilityClientCommand } from "@/lib/canvas/capabilities/client-command";
|
||||
import type { CanvasCapabilityTarget } from "@/lib/canvas/capabilities/context";
|
||||
import { generateCanvasImage } from "@/lib/canvas/generate-image";
|
||||
import { requestLongTaskResult } from "@/lib/client/long-task";
|
||||
import {
|
||||
findStyleProImageModelByRef,
|
||||
getStyleProImageModelEntries,
|
||||
sortStyleProImageModelEntries,
|
||||
} from "@/lib/style-pro/model-options";
|
||||
|
||||
export type CanvasCapabilityMediaResult = {
|
||||
assetId?: string;
|
||||
height?: number;
|
||||
metadata?: Record<string, unknown>;
|
||||
nodeType?: "image" | "model3d" | "world3d";
|
||||
prompt?: string;
|
||||
sourceNodeId: string;
|
||||
title: string;
|
||||
url: string;
|
||||
width?: number;
|
||||
};
|
||||
|
||||
type HeadlessExecutorParams = {
|
||||
command: CanvasCapabilityClientCommand;
|
||||
requestHeaders: Headers;
|
||||
requestUrl: string;
|
||||
targets: CanvasCapabilityTarget[];
|
||||
userId: string;
|
||||
};
|
||||
|
||||
function asFiniteNumber(value: unknown) {
|
||||
const number = Number(value);
|
||||
return Number.isFinite(number) ? number : undefined;
|
||||
}
|
||||
|
||||
function getCookieHeaders(headers: Headers) {
|
||||
const cookie = headers.get("cookie") || "";
|
||||
return cookie ? { cookie } : undefined;
|
||||
}
|
||||
|
||||
async function postRuntimeEndpoint(params: {
|
||||
body: Record<string, unknown>;
|
||||
path: string;
|
||||
requestHeaders: Headers;
|
||||
requestUrl: string;
|
||||
timeoutMs?: number;
|
||||
}) {
|
||||
const response = await fetch(new URL(params.path, params.requestUrl), {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
...getCookieHeaders(params.requestHeaders),
|
||||
},
|
||||
body: JSON.stringify(params.body),
|
||||
signal: AbortSignal.timeout(params.timeoutMs || 20 * 60 * 1000),
|
||||
});
|
||||
const payload = await response.json().catch(() => null);
|
||||
if (!response.ok || payload?.success === false || payload?.error) {
|
||||
throw new Error(
|
||||
String(payload?.error || payload?.detail || `能力服务返回 ${response.status}`)
|
||||
);
|
||||
}
|
||||
return payload as Record<string, any>;
|
||||
}
|
||||
|
||||
async function postRuntimeFormData(params: {
|
||||
formData: FormData;
|
||||
path: string;
|
||||
requestHeaders: Headers;
|
||||
requestUrl: string;
|
||||
timeoutMs?: number;
|
||||
}) {
|
||||
const response = await fetch(new URL(params.path, params.requestUrl), {
|
||||
method: "POST",
|
||||
headers: getCookieHeaders(params.requestHeaders),
|
||||
body: params.formData,
|
||||
signal: AbortSignal.timeout(params.timeoutMs || 20 * 60 * 1000),
|
||||
});
|
||||
const payload = await response.json().catch(() => null);
|
||||
if (!response.ok || payload?.success === false || payload?.error) {
|
||||
throw new Error(
|
||||
String(payload?.error || payload?.detail || `能力服务返回 ${response.status}`)
|
||||
);
|
||||
}
|
||||
return payload as Record<string, any>;
|
||||
}
|
||||
|
||||
function inferTripoImageType(imageUrl: string) {
|
||||
try {
|
||||
const extension = new URL(imageUrl).pathname.split(".").pop()?.toLowerCase();
|
||||
if (extension === "png" || extension === "webp") return extension;
|
||||
} catch {}
|
||||
return "jpg";
|
||||
}
|
||||
|
||||
async function resolveImageModel(modelRef?: unknown) {
|
||||
const entries = sortStyleProImageModelEntries(
|
||||
await getStyleProImageModelEntries({ includeHidden: false })
|
||||
);
|
||||
const requested = findStyleProImageModelByRef(entries, String(modelRef || ""));
|
||||
const selected = requested || entries.find((entry) => entry.model.isDefault) || entries[0];
|
||||
if (!selected) throw new Error("系统未配置可用的图片生成模型");
|
||||
return selected;
|
||||
}
|
||||
|
||||
async function executeForTarget(
|
||||
params: HeadlessExecutorParams,
|
||||
target: CanvasCapabilityTarget
|
||||
): Promise<CanvasCapabilityMediaResult[]> {
|
||||
const { command } = params;
|
||||
const prompt = String(command.args.prompt || "").trim();
|
||||
const common = {
|
||||
sourceNodeId: target.nodeId,
|
||||
prompt: prompt || undefined,
|
||||
};
|
||||
|
||||
if (command.capabilityId === "image.remove_background") {
|
||||
const imageBuffer = await fetchImageBufferFromUrl(target.sourceImageUrl);
|
||||
const result = await executeRmbgJob({
|
||||
userId: params.userId,
|
||||
imageBuffer,
|
||||
sourceImageUrl: target.sourceImageUrl,
|
||||
prompt: prompt || "抠图处理",
|
||||
skipGallerySave: false,
|
||||
});
|
||||
return [{
|
||||
...common,
|
||||
assetId: result.mediaAssetId || undefined,
|
||||
title: `${target.title}-抠图`,
|
||||
url: result.original_url,
|
||||
}];
|
||||
}
|
||||
|
||||
if (command.capabilityId === "image.extend") {
|
||||
const result = await executeExtendJob({
|
||||
userId: params.userId,
|
||||
imageUrl: target.sourceImageUrl,
|
||||
sourceImageUrl: target.sourceImageUrl,
|
||||
prompt: prompt || "图像拓展处理",
|
||||
xScale: asFiniteNumber(command.args.xScale),
|
||||
yScale: asFiniteNumber(command.args.yScale),
|
||||
requestId: command.requestId,
|
||||
});
|
||||
return [{
|
||||
...common,
|
||||
assetId: result.mediaAssetId || undefined,
|
||||
title: `${target.title}-扩图`,
|
||||
url: result.original_url,
|
||||
}];
|
||||
}
|
||||
|
||||
if (command.capabilityId === "image.resize") {
|
||||
const width = asFiniteNumber(command.args.width);
|
||||
const height = asFiniteNumber(command.args.height);
|
||||
if (!width || !height) throw new Error("请提供完整的目标宽度和高度");
|
||||
const result = await executeResizeJob({
|
||||
userId: params.userId,
|
||||
imageUrl: target.sourceImageUrl,
|
||||
sourceImageUrl: target.sourceImageUrl,
|
||||
canvasWidth: Math.round(width),
|
||||
canvasHeight: Math.round(height),
|
||||
backgroundColor: String(command.args.backgroundColor || "").trim() || undefined,
|
||||
transparentBackground: command.args.transparentBackground === true,
|
||||
mimeType: String(command.args.mimeType || "").trim() || undefined,
|
||||
});
|
||||
return [{
|
||||
...common,
|
||||
assetId: result.mediaAssetId || undefined,
|
||||
height: result.canvas_height,
|
||||
title: `${target.title}-${result.canvas_width}x${result.canvas_height}`,
|
||||
url: result.original_url,
|
||||
width: result.canvas_width,
|
||||
}];
|
||||
}
|
||||
|
||||
if (command.capabilityId === "image.box_cutout") {
|
||||
const result = await executeBoxCutoutJob({
|
||||
userId: params.userId,
|
||||
imageUrl: target.sourceImageUrl,
|
||||
sourceNodeId: target.nodeId,
|
||||
cropRect: {
|
||||
x: asFiniteNumber(command.args.x),
|
||||
y: asFiniteNumber(command.args.y),
|
||||
width: asFiniteNumber(command.args.width),
|
||||
height: asFiniteNumber(command.args.height),
|
||||
},
|
||||
paddingRatio: asFiniteNumber(command.args.paddingRatio),
|
||||
prompt: prompt || "框选抠图",
|
||||
});
|
||||
return [{
|
||||
...common,
|
||||
assetId: result.mediaAssetId || undefined,
|
||||
metadata: {
|
||||
cropRectApplied: result.cropRectApplied,
|
||||
paddedCropRectApplied: result.paddedCropRectApplied,
|
||||
},
|
||||
title: `${target.title}-框选抠图`,
|
||||
url: result.original_url,
|
||||
}];
|
||||
}
|
||||
|
||||
if (command.capabilityId === "image.layer_split") {
|
||||
const result = await executeLayerSplitJob({
|
||||
userId: params.userId,
|
||||
imageUrl: target.sourceImageUrl,
|
||||
sourceNodeId: target.nodeId,
|
||||
prompt: prompt || undefined,
|
||||
});
|
||||
return result.layers.map((layer) => ({
|
||||
...common,
|
||||
assetId: layer.mediaAssetId || undefined,
|
||||
metadata: {
|
||||
bounds: layer.bounds || null,
|
||||
layerIndex: layer.layerIndex,
|
||||
model: result.model,
|
||||
provider: result.provider,
|
||||
totalLayers: layer.totalLayers,
|
||||
},
|
||||
title: layer.title || `${target.title}-图层${layer.layerIndex + 1}`,
|
||||
url: layer.originalUrl,
|
||||
}));
|
||||
}
|
||||
|
||||
if (command.capabilityId === "image.enhance") {
|
||||
const result = await requestLongTaskResult({
|
||||
endpoint: new URL("/api/enhance", params.requestUrl).toString(),
|
||||
payload: {
|
||||
imageUrl: target.sourceImageUrl,
|
||||
prompt: prompt || "图像增强处理",
|
||||
scaleFactor: asFiniteNumber(command.args.scaleFactor),
|
||||
},
|
||||
pollingParams: {
|
||||
prompt: prompt || "图像增强处理",
|
||||
sourceImageUrl: target.sourceImageUrl,
|
||||
},
|
||||
timeoutMs: 10 * 60 * 1000,
|
||||
headers: getCookieHeaders(params.requestHeaders),
|
||||
});
|
||||
if (!result?.success || !result.original_url) {
|
||||
throw new Error(String(result?.error || result?.detail || "图像增强失败"));
|
||||
}
|
||||
return [{
|
||||
...common,
|
||||
title: `${target.title}-高清增强`,
|
||||
url: String(result.original_url),
|
||||
}];
|
||||
}
|
||||
|
||||
if (command.capabilityId === "image.remove_watermark") {
|
||||
const result = await requestLongTaskResult({
|
||||
endpoint: new URL("/api/watermark-remove", params.requestUrl).toString(),
|
||||
payload: { image_url: target.sourceImageUrl },
|
||||
timeoutMs: 20 * 60 * 1000,
|
||||
headers: getCookieHeaders(params.requestHeaders),
|
||||
});
|
||||
if (!result?.success || !result.original_url) {
|
||||
throw new Error(String(result?.error || result?.detail || "去水印失败"));
|
||||
}
|
||||
return [{
|
||||
...common,
|
||||
title: `${target.title}-去水印`,
|
||||
url: String(result.original_url),
|
||||
}];
|
||||
}
|
||||
|
||||
if (command.capabilityId === "image.multi_angle") {
|
||||
const result = await postRuntimeEndpoint({
|
||||
path: "/api/angle-edit",
|
||||
requestUrl: params.requestUrl,
|
||||
requestHeaders: params.requestHeaders,
|
||||
body: {
|
||||
image_url: target.sourceImageUrl,
|
||||
horizontal_angle: asFiniteNumber(command.args.horizontalAngle) ?? 30,
|
||||
vertical_angle: asFiniteNumber(command.args.verticalAngle) ?? 0,
|
||||
zoom: asFiniteNumber(command.args.zoom) ?? 0,
|
||||
},
|
||||
});
|
||||
const url = String(result.image_url || result.original_url || "").trim();
|
||||
if (!url) throw new Error("多视角服务未返回图片");
|
||||
return [{ ...common, title: `${target.title}-新视角`, url }];
|
||||
}
|
||||
|
||||
if (command.capabilityId === "image.generate_3d_model") {
|
||||
const modelVersion =
|
||||
String(command.args.modelId || command.args.modelVersion || "").trim() ||
|
||||
"tripo-standard";
|
||||
const promptText = prompt || target.title || "图生 3D 模型";
|
||||
const response = await postRuntimeEndpoint({
|
||||
path: "/api/canvas/3d/tasks",
|
||||
requestUrl: params.requestUrl,
|
||||
requestHeaders: params.requestHeaders,
|
||||
timeoutMs: 180_000,
|
||||
body: {
|
||||
taskType: "image_to_model",
|
||||
imageUrl: target.sourceImageUrl,
|
||||
imageType: inferTripoImageType(target.sourceImageUrl),
|
||||
prompt: promptText,
|
||||
modelVersion,
|
||||
faceLimit: 120_000,
|
||||
texture: true,
|
||||
pbr: true,
|
||||
autoSize: false,
|
||||
quad: false,
|
||||
generateParts: false,
|
||||
smartLowPoly: false,
|
||||
geometryQuality: "standard",
|
||||
textureQuality: "standard",
|
||||
textureAlignment: "original_image",
|
||||
orientation: "default",
|
||||
},
|
||||
});
|
||||
const task = response.data as Record<string, unknown> | undefined;
|
||||
const taskId = String(task?.taskId || "").trim();
|
||||
if (!taskId) throw new Error("3D 模型服务未返回任务 ID");
|
||||
const providerId = String(task?.providerId || "").trim() || null;
|
||||
const providerName = String(task?.providerName || "").trim() || null;
|
||||
return [
|
||||
{
|
||||
...common,
|
||||
nodeType: "model3d",
|
||||
title: `${target.title}-3D模型`,
|
||||
url: target.sourceImageUrl,
|
||||
metadata: {
|
||||
taskId,
|
||||
provider: "tripo-3d",
|
||||
mode: "image",
|
||||
status: "generating",
|
||||
sourceImageUrl: target.sourceImageUrl,
|
||||
providerId,
|
||||
providerName,
|
||||
modelVersion,
|
||||
generationStartedAt: new Date().toISOString(),
|
||||
assetPayload: {
|
||||
providerId,
|
||||
providerName,
|
||||
sourceImageUrl: target.sourceImageUrl,
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
if (command.capabilityId === "image.generate_3d_world") {
|
||||
const modelVersion =
|
||||
String(command.args.modelId || command.args.modelVersion || "").trim() ||
|
||||
"world-standard";
|
||||
const promptText = prompt || target.title || "图生 3D 世界";
|
||||
const formData = new FormData();
|
||||
formData.append("sourceUrl", target.sourceImageUrl);
|
||||
formData.append("kind", "image");
|
||||
const uploadResponse = await postRuntimeFormData({
|
||||
formData,
|
||||
path: "/api/canvas/world/upload",
|
||||
requestUrl: params.requestUrl,
|
||||
requestHeaders: params.requestHeaders,
|
||||
timeoutMs: 180_000,
|
||||
});
|
||||
const upload = uploadResponse.data as Record<string, unknown> | undefined;
|
||||
const mediaAssetId = String(upload?.mediaAssetId || "").trim();
|
||||
if (!mediaAssetId) throw new Error("3D 世界服务未返回媒体资源 ID");
|
||||
const providerId = String(upload?.providerId || "").trim() || null;
|
||||
const providerName = String(upload?.providerName || "").trim() || null;
|
||||
const taskResponse = await postRuntimeEndpoint({
|
||||
path: "/api/canvas/world/tasks",
|
||||
requestUrl: params.requestUrl,
|
||||
requestHeaders: params.requestHeaders,
|
||||
timeoutMs: 240_000,
|
||||
body: {
|
||||
mode: "image",
|
||||
mediaAssetId,
|
||||
providerId,
|
||||
prompt: promptText,
|
||||
displayName: target.title || promptText,
|
||||
model: modelVersion,
|
||||
disableRecaption: false,
|
||||
makePublic: false,
|
||||
},
|
||||
});
|
||||
const task = taskResponse.data as Record<string, unknown> | undefined;
|
||||
const taskId = String(task?.taskId || "").trim();
|
||||
if (!taskId) throw new Error("3D 世界服务未返回任务 ID");
|
||||
const taskAssetPayload =
|
||||
task?.assetPayload && typeof task.assetPayload === "object"
|
||||
? (task.assetPayload as Record<string, unknown>)
|
||||
: {};
|
||||
return [
|
||||
{
|
||||
...common,
|
||||
nodeType: "world3d",
|
||||
title: `${target.title}-3D世界`,
|
||||
url: target.sourceImageUrl,
|
||||
metadata: {
|
||||
taskId,
|
||||
provider: "worldlabs-3d",
|
||||
mode: "world-image",
|
||||
status: "generating",
|
||||
sourceImageUrl: target.sourceImageUrl,
|
||||
providerId,
|
||||
providerName,
|
||||
modelVersion,
|
||||
generationStartedAt: new Date().toISOString(),
|
||||
assetPayload: {
|
||||
...taskAssetPayload,
|
||||
providerId,
|
||||
providerName,
|
||||
sourceImageUrl: target.sourceImageUrl,
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
if (
|
||||
command.capabilityId === "image.prompt_regenerate" ||
|
||||
command.capabilityId === "image.ai_wash"
|
||||
) {
|
||||
if (!prompt) throw new Error("请提供重绘提示词");
|
||||
const model = await resolveImageModel(command.args.modelId);
|
||||
const result = await generateCanvasImage({
|
||||
prompt,
|
||||
model: model.model.id,
|
||||
ownerKey: params.userId,
|
||||
imageCount: 1,
|
||||
imageSize: String(command.args.imageSize || "2K"),
|
||||
aspectRatio: String(command.args.aspectRatio || "auto"),
|
||||
promptExtend: command.args.promptExtend === true,
|
||||
referenceImage: target.sourceImageUrl,
|
||||
debugLabel:
|
||||
command.capabilityId === "image.ai_wash"
|
||||
? "canvas-capability-ai-wash"
|
||||
: "canvas-capability-regenerate",
|
||||
debugMeta: {
|
||||
capabilityId: command.capabilityId,
|
||||
requestId: command.requestId,
|
||||
sourceNodeId: target.nodeId,
|
||||
},
|
||||
});
|
||||
const image = result.images[0];
|
||||
if (!image?.originalUrl) throw new Error("图片模型未返回有效结果");
|
||||
return [{
|
||||
...common,
|
||||
title:
|
||||
command.capabilityId === "image.ai_wash"
|
||||
? `${target.title}-AI洗图`
|
||||
: `${target.title}-重绘`,
|
||||
url: image.originalUrl,
|
||||
}];
|
||||
}
|
||||
|
||||
throw new Error(`尚未实现后台执行能力：${command.capabilityId}`);
|
||||
}
|
||||
|
||||
export async function executeCanvasCapabilityHeadlessly(
|
||||
params: HeadlessExecutorParams
|
||||
) {
|
||||
const results: CanvasCapabilityMediaResult[] = [];
|
||||
for (const target of params.targets) {
|
||||
results.push(...(await executeForTarget(params, target)));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
117
lib/canvas/capabilities/mcp-server.ts
Normal file
117
lib/canvas/capabilities/mcp-server.ts
Normal file
@ -0,0 +1,117 @@
|
||||
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
||||
import { z } from "zod";
|
||||
|
||||
import { createCanvasCapabilityClientCommand } from "@/lib/canvas/capabilities/client-command";
|
||||
import { CANVAS_CAPABILITY_REGISTRY } from "@/lib/canvas/capabilities/registry";
|
||||
|
||||
const requestIdSchema = z.string().trim().min(1).max(180).optional();
|
||||
const nodeIdsSchema = z
|
||||
.array(z.string().trim().min(1).max(180))
|
||||
.min(1)
|
||||
.max(50)
|
||||
.describe("要操作的画布图片节点 ID；多步调用可传入上一步返回的 createdNodeIds");
|
||||
|
||||
export async function createCanvasCapabilityMcpServer(params: {
|
||||
filterDisabled?: boolean;
|
||||
projectId: string;
|
||||
requestHeaders: Headers;
|
||||
requestUrl: string;
|
||||
userId: string;
|
||||
}) {
|
||||
const server = new McpServer({
|
||||
name: "buyi-canvas-capabilities",
|
||||
version: "1.0.0",
|
||||
});
|
||||
const enabledCapabilityIds =
|
||||
params.filterDisabled === false
|
||||
? new Set(CANVAS_CAPABILITY_REGISTRY.map((capability) => capability.id))
|
||||
: await import("@/lib/canvas/capabilities/availability").then(
|
||||
({ getEnabledCanvasCapabilityIds }) =>
|
||||
getEnabledCanvasCapabilityIds(CANVAS_CAPABILITY_REGISTRY)
|
||||
);
|
||||
|
||||
for (const capability of CANVAS_CAPABILITY_REGISTRY) {
|
||||
if (!enabledCapabilityIds.has(capability.id)) continue;
|
||||
if (!(capability.mcp.exposed && capability.mcp.toolName)) continue;
|
||||
|
||||
const inputSchema = z.object({
|
||||
nodeIds: nodeIdsSchema,
|
||||
args: capability.inputSchema.optional().describe("能力参数"),
|
||||
requestId: requestIdSchema.describe("幂等请求号；重试时必须复用同一个值"),
|
||||
confirmed: z
|
||||
.boolean()
|
||||
.optional()
|
||||
.describe("危险操作的明确确认；删除等能力必须为 true"),
|
||||
});
|
||||
|
||||
server.registerTool(
|
||||
capability.mcp.toolName,
|
||||
{
|
||||
title: capability.title,
|
||||
description: capability.description,
|
||||
inputSchema,
|
||||
annotations: {
|
||||
destructiveHint: capability.risk === "destructive",
|
||||
idempotentHint: true,
|
||||
openWorldHint: capability.risk === "external",
|
||||
readOnlyHint: capability.risk === "read",
|
||||
},
|
||||
},
|
||||
async (input) => {
|
||||
try {
|
||||
const command = createCanvasCapabilityClientCommand({
|
||||
capabilityId: capability.id,
|
||||
nodeIds: input.nodeIds,
|
||||
projectId: params.projectId,
|
||||
requestId: input.requestId,
|
||||
args: input.args,
|
||||
source: "mcp",
|
||||
confirmed: input.confirmed,
|
||||
});
|
||||
const { executeCanvasCapabilityRuntime } = await import(
|
||||
"@/lib/canvas/capabilities/runtime"
|
||||
);
|
||||
const result = await executeCanvasCapabilityRuntime({
|
||||
command,
|
||||
projectId: params.projectId,
|
||||
requestHeaders: params.requestHeaders,
|
||||
requestUrl: params.requestUrl,
|
||||
userId: params.userId,
|
||||
});
|
||||
const createdNodeIds = result.operation?.createdNodeIds ?? [];
|
||||
const summary =
|
||||
result.mode === "interactive"
|
||||
? `已创建客户端交互命令：${capability.title}`
|
||||
: `已执行：${capability.title}${
|
||||
createdNodeIds.length > 0
|
||||
? `，创建 ${createdNodeIds.length} 个节点`
|
||||
: ""
|
||||
}`;
|
||||
|
||||
return {
|
||||
content: [{ type: "text" as const, text: summary }],
|
||||
structuredContent: {
|
||||
capabilityId: capability.id,
|
||||
mode: result.mode,
|
||||
task: result.task,
|
||||
...(result.operation ? { operation: result.operation } : {}),
|
||||
...(result.outputs ? { outputs: result.outputs } : {}),
|
||||
...(result.clientCommand
|
||||
? { clientCommand: result.clientCommand }
|
||||
: {}),
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
const message =
|
||||
error instanceof Error ? error.message : `执行${capability.title}失败`;
|
||||
return {
|
||||
isError: true,
|
||||
content: [{ type: "text" as const, text: message }],
|
||||
};
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
return server;
|
||||
}
|
||||
485
lib/canvas/capabilities/registry.ts
Normal file
485
lib/canvas/capabilities/registry.ts
Normal file
@ -0,0 +1,485 @@
|
||||
import { z } from "zod";
|
||||
import type {
|
||||
CanvasCapabilityClientAction,
|
||||
CanvasCapabilityDefinition,
|
||||
CanvasCapabilityId,
|
||||
CanvasCapabilityManifestEntry,
|
||||
SelectedImageToolbarActionKey,
|
||||
} from "./types";
|
||||
|
||||
const emptyArgsSchema = z.object({}).passthrough();
|
||||
const promptArgsSchema = z
|
||||
.object({
|
||||
prompt: z.string().trim().min(1).max(8_000).optional(),
|
||||
})
|
||||
.passthrough();
|
||||
const resizeArgsSchema = z
|
||||
.object({
|
||||
width: z.number().int().positive().max(16_384).optional(),
|
||||
height: z.number().int().positive().max(16_384).optional(),
|
||||
aspectRatio: z.string().trim().max(32).optional(),
|
||||
})
|
||||
.passthrough();
|
||||
const extendArgsSchema = z
|
||||
.object({
|
||||
prompt: z.string().trim().min(1).max(8_000).optional(),
|
||||
xScale: z.number().min(1.1).max(4).optional(),
|
||||
yScale: z.number().min(1.1).max(4).optional(),
|
||||
})
|
||||
.passthrough();
|
||||
const angleArgsSchema = z
|
||||
.object({
|
||||
horizontalAngle: z.number().min(0).max(360).optional(),
|
||||
verticalAngle: z.number().min(-30).max(90).optional(),
|
||||
zoom: z.number().min(0).max(10).optional(),
|
||||
})
|
||||
.passthrough();
|
||||
const cropArgsSchema = z
|
||||
.object({
|
||||
x: z.number().min(0).optional(),
|
||||
y: z.number().min(0).optional(),
|
||||
width: z.number().positive().optional(),
|
||||
height: z.number().positive().optional(),
|
||||
})
|
||||
.passthrough();
|
||||
const transformArgsSchema = z
|
||||
.object({
|
||||
rotate: z.number().min(-360).max(360).optional(),
|
||||
flipHorizontal: z.boolean().optional(),
|
||||
flipVertical: z.boolean().optional(),
|
||||
})
|
||||
.passthrough();
|
||||
|
||||
type DefinitionInput = Omit<
|
||||
CanvasCapabilityDefinition,
|
||||
"selection" | "requiresRemoteImage" | "confirmation" | "agent" | "mcp"
|
||||
> & {
|
||||
selection?: Partial<CanvasCapabilityDefinition["selection"]>;
|
||||
requiresRemoteImage?: boolean;
|
||||
confirmation?: CanvasCapabilityDefinition["confirmation"];
|
||||
agent?: Partial<CanvasCapabilityDefinition["agent"]>;
|
||||
mcp?: Partial<CanvasCapabilityDefinition["mcp"]>;
|
||||
};
|
||||
|
||||
function defineCapability(input: DefinitionInput): CanvasCapabilityDefinition {
|
||||
const mcpToolName = input.id.replaceAll(".", "_");
|
||||
return {
|
||||
...input,
|
||||
requiresRemoteImage: input.requiresRemoteImage ?? true,
|
||||
selection: {
|
||||
nodeTypes: ["image"],
|
||||
min: input.selection?.min ?? 1,
|
||||
...(input.selection?.max ? { max: input.selection.max } : {}),
|
||||
},
|
||||
confirmation: input.confirmation ?? "never",
|
||||
agent: {
|
||||
exposed: input.agent?.exposed ?? true,
|
||||
aliases: input.agent?.aliases ?? [],
|
||||
toolName: input.agent?.toolName ?? mcpToolName,
|
||||
},
|
||||
mcp: {
|
||||
exposed: input.mcp?.exposed ?? true,
|
||||
toolName: input.mcp?.toolName ?? mcpToolName,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export const CANVAS_CAPABILITY_REGISTRY = [
|
||||
defineCapability({
|
||||
id: "image.add_to_chat",
|
||||
title: "添加到对话",
|
||||
description: "将选中图片加入当前 Agent 对话上下文。",
|
||||
clientAction: "add-to-chat",
|
||||
toolbarSettingKey: "imageAddToChat",
|
||||
executionMode: "local",
|
||||
risk: "read",
|
||||
modelPolicy: "none",
|
||||
requiresRemoteImage: false,
|
||||
inputSchema: emptyArgsSchema,
|
||||
agent: { exposed: false },
|
||||
mcp: { exposed: false },
|
||||
}),
|
||||
defineCapability({
|
||||
id: "image.pick_object",
|
||||
title: "识物",
|
||||
description: "进入图片点选识物模式，并把识别结果加入对话上下文。",
|
||||
clientAction: "pick-object",
|
||||
toolbarActionKey: "pickObject",
|
||||
toolbarSettingKey: "imagePickObject",
|
||||
executionMode: "interactive",
|
||||
risk: "read",
|
||||
modelPolicy: "feature-setting",
|
||||
confirmation: "interactive",
|
||||
inputSchema: z
|
||||
.object({ x: z.number().min(0).max(1).optional(), y: z.number().min(0).max(1).optional() })
|
||||
.passthrough(),
|
||||
agent: { aliases: ["识物", "识别物品", "选择物体"] },
|
||||
}),
|
||||
defineCapability({
|
||||
id: "image.enhance",
|
||||
title: "高清增强",
|
||||
description: "使用系统配置的增强服务提升图片清晰度。",
|
||||
clientAction: "enhance",
|
||||
toolbarActionKey: "enhance",
|
||||
toolbarSettingKey: "imageEnhance",
|
||||
executionMode: "headless",
|
||||
risk: "external",
|
||||
modelPolicy: "feature-setting",
|
||||
modelSettingKey: "canvas.imageEnhance",
|
||||
inputSchema: z.object({ scaleFactor: z.number().min(1).max(8).optional() }).passthrough(),
|
||||
agent: { toolName: "enhanceImage", aliases: ["高清", "增强", "变清晰"] },
|
||||
}),
|
||||
defineCapability({
|
||||
id: "image.remove_background",
|
||||
title: "抠图去背",
|
||||
description: "调用系统抠图能力移除图片背景。",
|
||||
clientAction: "rmbg",
|
||||
toolbarActionKey: "removeBackground",
|
||||
toolbarSettingKey: "imageRemoveBackground",
|
||||
executionMode: "headless",
|
||||
risk: "external",
|
||||
modelPolicy: "feature-setting",
|
||||
modelSettingKey: "canvas.removeBackground",
|
||||
inputSchema: promptArgsSchema,
|
||||
agent: { toolName: "removeBackground", aliases: ["抠图", "去背景", "透明背景"] },
|
||||
}),
|
||||
defineCapability({
|
||||
id: "image.ai_wash",
|
||||
title: "AI洗图",
|
||||
description: "按管理员配置的预设、档位和模型执行 AI 洗图。",
|
||||
clientAction: "ai-wash",
|
||||
toolbarActionKey: "aiWashQuick",
|
||||
executionMode: "hybrid",
|
||||
risk: "external",
|
||||
modelPolicy: "feature-setting",
|
||||
modelSettingKey: "canvas.aiWash",
|
||||
confirmation: "interactive",
|
||||
inputSchema: z
|
||||
.object({
|
||||
prompt: z.string().trim().min(1).max(8_000).optional(),
|
||||
presetId: z.string().trim().min(1).optional(),
|
||||
modelId: z.string().trim().min(1).optional(),
|
||||
levels: z.record(z.number().int().nonnegative()).optional(),
|
||||
})
|
||||
.passthrough(),
|
||||
agent: { aliases: ["洗图", "AI洗图", "重新洗图"] },
|
||||
}),
|
||||
defineCapability({
|
||||
id: "image.text_edit",
|
||||
title: "图片改字",
|
||||
description: "识别图片文字并打开可编辑文字面板。",
|
||||
clientAction: "text-edit",
|
||||
toolbarActionKey: "textEdit",
|
||||
toolbarSettingKey: "imageTextEdit",
|
||||
executionMode: "hybrid",
|
||||
risk: "external",
|
||||
modelPolicy: "feature-setting",
|
||||
modelSettingKey: "canvas.textEdit",
|
||||
confirmation: "interactive",
|
||||
inputSchema: promptArgsSchema,
|
||||
agent: { aliases: ["改字", "修改图片文字", "替换文案"] },
|
||||
}),
|
||||
defineCapability({
|
||||
id: "image.box_cutout",
|
||||
title: "框选抠图",
|
||||
description: "框选图片区域后提取指定主体。",
|
||||
clientAction: "box-cutout",
|
||||
toolbarActionKey: "boxCutout",
|
||||
toolbarSettingKey: "imageBoxCutout",
|
||||
executionMode: "hybrid",
|
||||
risk: "external",
|
||||
modelPolicy: "feature-setting",
|
||||
confirmation: "interactive",
|
||||
inputSchema: cropArgsSchema,
|
||||
agent: { toolName: "boxCutout", aliases: ["框选抠图", "抠出这部分"] },
|
||||
}),
|
||||
defineCapability({
|
||||
id: "image.local_edit",
|
||||
title: "局部编辑",
|
||||
description: "按框选区域与区域提示词执行局部重绘。",
|
||||
clientAction: "local-edit",
|
||||
toolbarActionKey: "localEdit",
|
||||
toolbarSettingKey: "imageLocalEdit",
|
||||
executionMode: "hybrid",
|
||||
risk: "external",
|
||||
modelPolicy: "agent-selected",
|
||||
confirmation: "interactive",
|
||||
inputSchema: promptArgsSchema,
|
||||
agent: { toolName: "localEditImage", aliases: ["局部编辑", "局部重绘"] },
|
||||
}),
|
||||
defineCapability({
|
||||
id: "image.scale_adjust",
|
||||
title: "比例调整",
|
||||
description: "按目标框等比缩放和移动选中主体。",
|
||||
clientAction: "scale-adjust",
|
||||
toolbarSettingKey: "imageLocalEdit",
|
||||
executionMode: "hybrid",
|
||||
risk: "external",
|
||||
modelPolicy: "feature-setting",
|
||||
confirmation: "interactive",
|
||||
inputSchema: cropArgsSchema,
|
||||
agent: { aliases: ["比例调整", "缩放主体", "调整主体位置"] },
|
||||
}),
|
||||
defineCapability({
|
||||
id: "image.remove_watermark",
|
||||
title: "去水印",
|
||||
description: "使用系统配置的服务移除水印和画面瑕疵。",
|
||||
clientAction: "watermark",
|
||||
toolbarActionKey: "removeWatermark",
|
||||
toolbarSettingKey: "imageRemoveWatermark",
|
||||
executionMode: "headless",
|
||||
risk: "external",
|
||||
modelPolicy: "feature-setting",
|
||||
modelSettingKey: "canvas.removeWatermark",
|
||||
inputSchema: promptArgsSchema,
|
||||
agent: { toolName: "removeWatermark", aliases: ["去水印", "去印", "清理水印"] },
|
||||
}),
|
||||
defineCapability({
|
||||
id: "image.resize",
|
||||
title: "尺寸调整",
|
||||
description: "调整图片尺寸、比例和主体位置。",
|
||||
clientAction: "resize",
|
||||
toolbarActionKey: "resize",
|
||||
toolbarSettingKey: "imageResize",
|
||||
executionMode: "hybrid",
|
||||
risk: "external",
|
||||
modelPolicy: "feature-setting",
|
||||
confirmation: "interactive",
|
||||
inputSchema: resizeArgsSchema,
|
||||
agent: { toolName: "resizeImage", aliases: ["调整尺寸", "改比例", "改分辨率"] },
|
||||
}),
|
||||
defineCapability({
|
||||
id: "image.multi_angle",
|
||||
title: "多视角",
|
||||
description: "按水平角、俯仰角和景别生成主体新视角。",
|
||||
clientAction: "angle",
|
||||
toolbarActionKey: "multiAngle",
|
||||
toolbarSettingKey: "imageMultiAngle",
|
||||
executionMode: "hybrid",
|
||||
risk: "external",
|
||||
modelPolicy: "feature-setting",
|
||||
confirmation: "interactive",
|
||||
inputSchema: angleArgsSchema,
|
||||
agent: { toolName: "angleEditImage", aliases: ["多视角", "换角度", "旋转视角"] },
|
||||
}),
|
||||
defineCapability({
|
||||
id: "image.generate_3d_model",
|
||||
title: "生成3D模型",
|
||||
description: "使用系统配置的 Tripo 模型从图片生成 3D 模型。",
|
||||
clientAction: "generate-3d-model",
|
||||
toolbarActionKey: "generate3dModel",
|
||||
toolbarSettingKey: "imageGenerate3dModel",
|
||||
executionMode: "hybrid",
|
||||
risk: "external",
|
||||
modelPolicy: "feature-setting",
|
||||
modelSettingKey: "canvas.tripo3d",
|
||||
confirmation: "interactive",
|
||||
inputSchema: z
|
||||
.object({
|
||||
modelId: z.string().trim().min(1).optional(),
|
||||
prompt: z.string().trim().min(1).max(8_000).optional(),
|
||||
})
|
||||
.passthrough(),
|
||||
agent: { aliases: ["生成3D模型", "转成3D模型", "Tripo"] },
|
||||
}),
|
||||
defineCapability({
|
||||
id: "image.generate_3d_world",
|
||||
title: "生成3D世界",
|
||||
description: "使用系统配置的 World Labs 模型生成可漫游 3D 世界。",
|
||||
clientAction: "generate-3d-world",
|
||||
toolbarActionKey: "generate3dWorld",
|
||||
toolbarSettingKey: "imageGenerate3dWorld",
|
||||
executionMode: "hybrid",
|
||||
risk: "external",
|
||||
modelPolicy: "feature-setting",
|
||||
modelSettingKey: "canvas.worldLabs3d",
|
||||
confirmation: "interactive",
|
||||
inputSchema: z
|
||||
.object({
|
||||
modelId: z.string().trim().min(1).optional(),
|
||||
prompt: z.string().trim().min(1).max(8_000).optional(),
|
||||
})
|
||||
.passthrough(),
|
||||
agent: { aliases: ["生成3D世界", "生成场景", "World Labs"] },
|
||||
}),
|
||||
defineCapability({
|
||||
id: "image.layer_split",
|
||||
title: "图层拆分",
|
||||
description: "将图片拆分为可独立编辑的图层节点。",
|
||||
clientAction: "layer-split",
|
||||
toolbarActionKey: "layerSplit",
|
||||
toolbarSettingKey: "imageLayerSplit",
|
||||
executionMode: "headless",
|
||||
risk: "external",
|
||||
modelPolicy: "feature-setting",
|
||||
modelSettingKey: "canvas.layerSplit",
|
||||
inputSchema: emptyArgsSchema,
|
||||
agent: { aliases: ["图层拆分", "拆图层", "分层"] },
|
||||
}),
|
||||
defineCapability({
|
||||
id: "image.extend",
|
||||
title: "扩图",
|
||||
description: "向指定方向扩展画面并补全内容。",
|
||||
clientAction: "extend",
|
||||
toolbarActionKey: "extend",
|
||||
toolbarSettingKey: "imageExtend",
|
||||
executionMode: "headless",
|
||||
risk: "external",
|
||||
modelPolicy: "feature-setting",
|
||||
modelSettingKey: "canvas.extend",
|
||||
inputSchema: extendArgsSchema,
|
||||
agent: { toolName: "extendImage", aliases: ["扩图", "补边", "扩展画面"] },
|
||||
}),
|
||||
defineCapability({
|
||||
id: "image.crop",
|
||||
title: "裁剪",
|
||||
description: "打开裁剪面板，或按完整坐标直接裁剪。",
|
||||
clientAction: "crop",
|
||||
toolbarActionKey: "crop",
|
||||
toolbarSettingKey: "imageCrop",
|
||||
executionMode: "hybrid",
|
||||
risk: "write",
|
||||
modelPolicy: "none",
|
||||
requiresRemoteImage: false,
|
||||
confirmation: "interactive",
|
||||
inputSchema: cropArgsSchema,
|
||||
agent: { aliases: ["裁剪", "剪裁", "截取画面"] },
|
||||
}),
|
||||
defineCapability({
|
||||
id: "image.adjust",
|
||||
title: "画面调整",
|
||||
description: "打开亮度、对比度、饱和度等图片调整面板。",
|
||||
clientAction: "adjust",
|
||||
toolbarActionKey: "adjust",
|
||||
toolbarSettingKey: "imageAdjust",
|
||||
executionMode: "interactive",
|
||||
risk: "write",
|
||||
modelPolicy: "none",
|
||||
requiresRemoteImage: false,
|
||||
confirmation: "interactive",
|
||||
inputSchema: z.record(z.number()).optional().default({}),
|
||||
agent: { aliases: ["调色", "调整亮度", "调整对比度"] },
|
||||
}),
|
||||
defineCapability({
|
||||
id: "image.flip_rotate",
|
||||
title: "翻转与旋转",
|
||||
description: "翻转或旋转选中图片。",
|
||||
clientAction: "flip-rotate",
|
||||
toolbarActionKey: "flipRotate",
|
||||
toolbarSettingKey: "imageFlipRotate",
|
||||
executionMode: "hybrid",
|
||||
risk: "write",
|
||||
modelPolicy: "none",
|
||||
requiresRemoteImage: false,
|
||||
confirmation: "interactive",
|
||||
inputSchema: transformArgsSchema,
|
||||
agent: { aliases: ["翻转", "旋转图片", "镜像"] },
|
||||
}),
|
||||
defineCapability({
|
||||
id: "image.prompt_regenerate",
|
||||
title: "提示词重绘",
|
||||
description: "按新的提示词和系统生图模型重绘选中图片。",
|
||||
clientAction: "prompt-regenerate",
|
||||
toolbarActionKey: "promptRegenerate",
|
||||
toolbarSettingKey: "imagePromptRegenerate",
|
||||
executionMode: "hybrid",
|
||||
risk: "external",
|
||||
modelPolicy: "agent-selected",
|
||||
confirmation: "interactive",
|
||||
inputSchema: promptArgsSchema,
|
||||
agent: { toolName: "editTargetImage", aliases: ["重绘", "按提示词重做"] },
|
||||
}),
|
||||
defineCapability({
|
||||
id: "image.note",
|
||||
title: "添加批注",
|
||||
description: "给图片添加可作为 Agent 上下文的批注。",
|
||||
clientAction: "note",
|
||||
toolbarActionKey: "note",
|
||||
toolbarSettingKey: "imageNote",
|
||||
executionMode: "hybrid",
|
||||
risk: "write",
|
||||
modelPolicy: "none",
|
||||
requiresRemoteImage: false,
|
||||
confirmation: "interactive",
|
||||
inputSchema: z.object({ text: z.string().trim().max(1_000).optional() }).passthrough(),
|
||||
agent: { aliases: ["添加批注", "备注", "标注"] },
|
||||
}),
|
||||
defineCapability({
|
||||
id: "image.rename",
|
||||
title: "重命名",
|
||||
description: "修改选中图片节点标题。",
|
||||
clientAction: "rename",
|
||||
toolbarSettingKey: "imageRename",
|
||||
executionMode: "hybrid",
|
||||
risk: "write",
|
||||
modelPolicy: "none",
|
||||
requiresRemoteImage: false,
|
||||
inputSchema: z.object({ title: z.string().trim().min(1).max(120).optional() }).passthrough(),
|
||||
agent: { aliases: ["重命名", "改名", "修改标题"] },
|
||||
}),
|
||||
defineCapability({
|
||||
id: "image.download",
|
||||
title: "下载图片",
|
||||
description: "下载选中图片资源。",
|
||||
clientAction: "download",
|
||||
toolbarSettingKey: "imageDownload",
|
||||
executionMode: "local",
|
||||
risk: "read",
|
||||
modelPolicy: "none",
|
||||
requiresRemoteImage: false,
|
||||
inputSchema: emptyArgsSchema,
|
||||
agent: { exposed: false },
|
||||
mcp: { exposed: false },
|
||||
}),
|
||||
defineCapability({
|
||||
id: "image.delete",
|
||||
title: "删除图片",
|
||||
description: "从当前画布删除选中图片节点。",
|
||||
clientAction: "delete",
|
||||
toolbarSettingKey: "imageDelete",
|
||||
executionMode: "local",
|
||||
risk: "destructive",
|
||||
modelPolicy: "none",
|
||||
requiresRemoteImage: false,
|
||||
confirmation: "always",
|
||||
inputSchema: z.object({ confirmed: z.literal(true).optional() }).passthrough(),
|
||||
agent: { aliases: ["删除图片", "移除图片"] },
|
||||
}),
|
||||
] as const satisfies readonly CanvasCapabilityDefinition[];
|
||||
|
||||
const capabilityById = new Map<CanvasCapabilityId, CanvasCapabilityDefinition>(
|
||||
CANVAS_CAPABILITY_REGISTRY.map((capability) => [capability.id, capability]),
|
||||
);
|
||||
const capabilityByClientAction = new Map<CanvasCapabilityClientAction, CanvasCapabilityDefinition>(
|
||||
CANVAS_CAPABILITY_REGISTRY.map((capability) => [capability.clientAction, capability]),
|
||||
);
|
||||
const capabilityByToolbarAction = new Map<
|
||||
SelectedImageToolbarActionKey,
|
||||
CanvasCapabilityDefinition
|
||||
>(
|
||||
CANVAS_CAPABILITY_REGISTRY.flatMap((capability) =>
|
||||
capability.toolbarActionKey ? [[capability.toolbarActionKey, capability]] : [],
|
||||
),
|
||||
);
|
||||
|
||||
export function getCanvasCapability(id: CanvasCapabilityId) {
|
||||
return capabilityById.get(id) ?? null;
|
||||
}
|
||||
|
||||
export function getCanvasCapabilityByClientAction(action: CanvasCapabilityClientAction) {
|
||||
return capabilityByClientAction.get(action) ?? null;
|
||||
}
|
||||
|
||||
export function getCanvasCapabilityByToolbarAction(action: SelectedImageToolbarActionKey) {
|
||||
return capabilityByToolbarAction.get(action) ?? null;
|
||||
}
|
||||
|
||||
export function parseCanvasCapabilityArgs(id: CanvasCapabilityId, value: unknown) {
|
||||
const capability = getCanvasCapability(id);
|
||||
if (!capability) throw new Error(`Unknown canvas capability: ${id}`);
|
||||
return capability.inputSchema.parse(value ?? {});
|
||||
}
|
||||
|
||||
export function getCanvasCapabilityManifest(): CanvasCapabilityManifestEntry[] {
|
||||
return CANVAS_CAPABILITY_REGISTRY.map(({ inputSchema: _inputSchema, ...capability }) => capability);
|
||||
}
|
||||
301
lib/canvas/capabilities/runtime.ts
Normal file
301
lib/canvas/capabilities/runtime.ts
Normal file
@ -0,0 +1,301 @@
|
||||
import type { CanvasCapabilityClientCommand } from "@/lib/canvas/capabilities/client-command";
|
||||
import { createCanvasCapabilityClientCommand } from "@/lib/canvas/capabilities/client-command";
|
||||
import { isCanvasCapabilityEnabled } from "@/lib/canvas/capabilities/availability";
|
||||
import {
|
||||
CanvasCapabilityRuntimeError,
|
||||
resolveCanvasCapabilityContext,
|
||||
} from "@/lib/canvas/capabilities/context";
|
||||
import { shouldExecuteCanvasCapabilityHeadlessly } from "@/lib/canvas/capabilities/execution-mode";
|
||||
import {
|
||||
executeCanvasCapabilityHeadlessly,
|
||||
type CanvasCapabilityMediaResult,
|
||||
} from "@/lib/canvas/capabilities/headless-executor";
|
||||
import { getCanvasCapability } from "@/lib/canvas/capabilities/registry";
|
||||
import { enqueueCanvasCapabilityExecution } from "@/lib/canvas/capabilities/task-queue";
|
||||
import {
|
||||
createOrRecoverCanvasCapabilityTask,
|
||||
MAX_CANVAS_CAPABILITY_TASK_ATTEMPTS,
|
||||
markCanvasCapabilityTaskFailed,
|
||||
markCanvasCapabilityTaskRunning,
|
||||
markCanvasCapabilityTaskSucceeded,
|
||||
} from "@/lib/canvas/capabilities/task-store";
|
||||
import { applyCanvasOperationPlan } from "@/lib/canvas/agent-mutations";
|
||||
import type { CanvasOperationPlan } from "@/types/canvas/agent-protocol";
|
||||
|
||||
export type CanvasCapabilityRuntimeResult = {
|
||||
clientCommand?: CanvasCapabilityClientCommand;
|
||||
mode: "headless" | "interactive";
|
||||
operation?: {
|
||||
canvasRevision: string;
|
||||
canvasUpdatedAt: string;
|
||||
createdNodeIds: string[];
|
||||
createdNodeIdsByMediaItemId: Record<string, string>;
|
||||
updatedNodeIds: string[];
|
||||
};
|
||||
outputs?: Array<{
|
||||
assetId?: string;
|
||||
sourceNodeId: string;
|
||||
title: string;
|
||||
url: string;
|
||||
}>;
|
||||
task: {
|
||||
attempts: number;
|
||||
error: string | null;
|
||||
id: string;
|
||||
requestId: string;
|
||||
result: unknown;
|
||||
status: string;
|
||||
version: number;
|
||||
};
|
||||
};
|
||||
|
||||
function serializeTask(task: {
|
||||
attempts: number;
|
||||
error: string | null;
|
||||
id: string;
|
||||
requestId: string;
|
||||
result: unknown;
|
||||
status: string;
|
||||
version: number;
|
||||
}) {
|
||||
return {
|
||||
attempts: task.attempts,
|
||||
error: task.error,
|
||||
id: task.id,
|
||||
requestId: task.requestId,
|
||||
result: task.result,
|
||||
status: task.status,
|
||||
version: task.version,
|
||||
};
|
||||
}
|
||||
|
||||
function buildMediaOperationPlan(params: {
|
||||
command: CanvasCapabilityClientCommand;
|
||||
mediaResults: CanvasCapabilityMediaResult[];
|
||||
projectId: string;
|
||||
projectRevision: string;
|
||||
}) {
|
||||
return {
|
||||
version: 1,
|
||||
operationId: params.command.requestId,
|
||||
operation: "insert_beside",
|
||||
projectId: params.projectId,
|
||||
baseCanvasRevision: params.projectRevision,
|
||||
sourceNodeIds: params.command.nodeIds,
|
||||
targetNodeIds: params.command.nodeIds,
|
||||
placement: "right",
|
||||
placementStrategy: "beside_each_source",
|
||||
preserveOriginal: true,
|
||||
layoutStrategy: params.mediaResults.length > 4 ? "grid" : "row",
|
||||
spacing: 24,
|
||||
mediaItems: params.mediaResults.map((item, index) => ({
|
||||
mediaItemId: `${params.command.requestId}:result-${index + 1}`,
|
||||
type: item.nodeType || "image",
|
||||
url: item.url,
|
||||
assetId: item.assetId,
|
||||
width: item.width,
|
||||
height: item.height,
|
||||
title: item.title,
|
||||
prompt: item.prompt,
|
||||
sourceNodeIds: [item.sourceNodeId],
|
||||
metadata: {
|
||||
...(item.metadata || {}),
|
||||
canvasCapabilityId: params.command.capabilityId,
|
||||
canvasCapabilityRequestId: params.command.requestId,
|
||||
canvasCapabilitySource: params.command.source,
|
||||
sourceNodeId: item.sourceNodeId,
|
||||
},
|
||||
})),
|
||||
metadata: {
|
||||
canvasCapabilityId: params.command.capabilityId,
|
||||
canvasCapabilityRequestId: params.command.requestId,
|
||||
canvasCapabilitySource: params.command.source,
|
||||
canvasCapabilityVersion: params.command.version,
|
||||
},
|
||||
} satisfies CanvasOperationPlan;
|
||||
}
|
||||
|
||||
function buildMutationOperationPlan(params: {
|
||||
command: CanvasCapabilityClientCommand;
|
||||
projectId: string;
|
||||
projectRevision: string;
|
||||
}): CanvasOperationPlan | null {
|
||||
const common = {
|
||||
version: 1 as const,
|
||||
operationId: params.command.requestId,
|
||||
projectId: params.projectId,
|
||||
baseCanvasRevision: params.projectRevision,
|
||||
targetNodeIds: params.command.nodeIds,
|
||||
sourceNodeIds: params.command.nodeIds,
|
||||
metadata: {
|
||||
canvasCapabilityId: params.command.capabilityId,
|
||||
canvasCapabilityRequestId: params.command.requestId,
|
||||
canvasCapabilitySource: params.command.source,
|
||||
},
|
||||
};
|
||||
|
||||
if (params.command.capabilityId === "image.rename") {
|
||||
return {
|
||||
...common,
|
||||
operation: "rename_node",
|
||||
textContent: String(params.command.args.title || "").trim(),
|
||||
};
|
||||
}
|
||||
if (params.command.capabilityId === "image.note") {
|
||||
return {
|
||||
...common,
|
||||
operation: "update_note",
|
||||
textContent: String(params.command.args.text || "").trim(),
|
||||
};
|
||||
}
|
||||
if (params.command.capabilityId === "image.delete") {
|
||||
return {
|
||||
...common,
|
||||
operation: "delete_nodes",
|
||||
confirmed: params.command.confirmed === true,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function summarizeOperation(operation: Awaited<ReturnType<typeof applyCanvasOperationPlan>>) {
|
||||
return {
|
||||
canvasRevision: operation.canvasRevision,
|
||||
canvasUpdatedAt: operation.canvasUpdatedAt,
|
||||
createdNodeIds: operation.createdNodeIds,
|
||||
createdNodeIdsByMediaItemId: operation.createdNodeIdsByMediaItemId,
|
||||
updatedNodeIds: operation.updatedNodeIds,
|
||||
};
|
||||
}
|
||||
|
||||
export async function executeCanvasCapabilityRuntime(params: {
|
||||
command: CanvasCapabilityClientCommand;
|
||||
projectId: string;
|
||||
requestHeaders: Headers;
|
||||
requestUrl: string;
|
||||
userId: string;
|
||||
}): Promise<CanvasCapabilityRuntimeResult> {
|
||||
const command = createCanvasCapabilityClientCommand({
|
||||
capabilityId: params.command.capabilityId,
|
||||
nodeIds: params.command.nodeIds,
|
||||
projectId: params.projectId,
|
||||
requestId: params.command.requestId,
|
||||
args: params.command.args,
|
||||
source: params.command.source,
|
||||
confirmed: params.command.confirmed,
|
||||
createdAt: params.command.createdAt,
|
||||
});
|
||||
const capability = getCanvasCapability(command.capabilityId);
|
||||
if (!capability) {
|
||||
throw new CanvasCapabilityRuntimeError("画布能力不存在", 404);
|
||||
}
|
||||
if (!(await isCanvasCapabilityEnabled(capability))) {
|
||||
throw new CanvasCapabilityRuntimeError(
|
||||
`${capability.title}已在系统设置中关闭`,
|
||||
403,
|
||||
"CANVAS_CAPABILITY_DISABLED"
|
||||
);
|
||||
}
|
||||
const context = await resolveCanvasCapabilityContext({
|
||||
command,
|
||||
projectId: params.projectId,
|
||||
requiresRemoteImage: capability.requiresRemoteImage,
|
||||
userId: params.userId,
|
||||
});
|
||||
let task = await createOrRecoverCanvasCapabilityTask({
|
||||
command,
|
||||
userId: params.userId,
|
||||
projectId: params.projectId,
|
||||
});
|
||||
|
||||
if (task.status === "succeeded") {
|
||||
const storedResult =
|
||||
task.result && typeof task.result === "object"
|
||||
? (task.result as Omit<CanvasCapabilityRuntimeResult, "task">)
|
||||
: ({ mode: "headless" } as const);
|
||||
return { ...storedResult, task: serializeTask(task) };
|
||||
}
|
||||
if (task.status === "running") {
|
||||
return {
|
||||
mode: shouldExecuteCanvasCapabilityHeadlessly({ capability, command })
|
||||
? "headless"
|
||||
: "interactive",
|
||||
task: serializeTask(task),
|
||||
};
|
||||
}
|
||||
if (
|
||||
task.status === "failed" &&
|
||||
task.attempts >= MAX_CANVAS_CAPABILITY_TASK_ATTEMPTS
|
||||
) {
|
||||
throw new CanvasCapabilityRuntimeError(
|
||||
task.error || "画布能力任务超过最大重试次数",
|
||||
409,
|
||||
"CANVAS_CAPABILITY_RETRY_EXHAUSTED"
|
||||
);
|
||||
}
|
||||
|
||||
const shouldRunHeadlessly = shouldExecuteCanvasCapabilityHeadlessly({
|
||||
capability,
|
||||
command,
|
||||
});
|
||||
if (!shouldRunHeadlessly) {
|
||||
task = await markCanvasCapabilityTaskRunning(task.id);
|
||||
const result = {
|
||||
mode: "interactive" as const,
|
||||
clientCommand: command,
|
||||
};
|
||||
task = (await markCanvasCapabilityTaskSucceeded(task.id, result)) || task;
|
||||
return { ...result, task: serializeTask(task) };
|
||||
}
|
||||
|
||||
return enqueueCanvasCapabilityExecution({
|
||||
projectId: params.projectId,
|
||||
execute: async () => {
|
||||
task = await markCanvasCapabilityTaskRunning(task.id);
|
||||
try {
|
||||
const projectRevision = context.project.updatedAt.toISOString();
|
||||
const mutationPlan = buildMutationOperationPlan({
|
||||
command,
|
||||
projectId: params.projectId,
|
||||
projectRevision,
|
||||
});
|
||||
const mediaResults = mutationPlan
|
||||
? []
|
||||
: await executeCanvasCapabilityHeadlessly({
|
||||
command,
|
||||
requestHeaders: params.requestHeaders,
|
||||
requestUrl: params.requestUrl,
|
||||
targets: context.targets,
|
||||
userId: params.userId,
|
||||
});
|
||||
const operation = await applyCanvasOperationPlan({
|
||||
userId: params.userId,
|
||||
projectId: params.projectId,
|
||||
plan:
|
||||
mutationPlan ||
|
||||
buildMediaOperationPlan({
|
||||
command,
|
||||
mediaResults,
|
||||
projectId: params.projectId,
|
||||
projectRevision,
|
||||
}),
|
||||
});
|
||||
const result = {
|
||||
mode: "headless" as const,
|
||||
operation: summarizeOperation(operation),
|
||||
outputs: mediaResults.map((item) => ({
|
||||
assetId: item.assetId,
|
||||
sourceNodeId: item.sourceNodeId,
|
||||
title: item.title,
|
||||
url: item.url,
|
||||
})),
|
||||
};
|
||||
task = (await markCanvasCapabilityTaskSucceeded(task.id, result)) || task;
|
||||
return { ...result, task: serializeTask(task) };
|
||||
} catch (error) {
|
||||
task = (await markCanvasCapabilityTaskFailed(task.id, error)) || task;
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
34
lib/canvas/capabilities/task-queue.ts
Normal file
34
lib/canvas/capabilities/task-queue.ts
Normal file
@ -0,0 +1,34 @@
|
||||
type CanvasCapabilityQueueState = {
|
||||
tails: Map<string, Promise<unknown>>;
|
||||
};
|
||||
|
||||
declare global {
|
||||
var __canvasCapabilityQueueState: CanvasCapabilityQueueState | undefined;
|
||||
}
|
||||
|
||||
function getQueueState() {
|
||||
if (!globalThis.__canvasCapabilityQueueState) {
|
||||
globalThis.__canvasCapabilityQueueState = { tails: new Map() };
|
||||
}
|
||||
return globalThis.__canvasCapabilityQueueState;
|
||||
}
|
||||
|
||||
export async function enqueueCanvasCapabilityExecution<T>(params: {
|
||||
projectId: string;
|
||||
execute: () => Promise<T>;
|
||||
}): Promise<T> {
|
||||
const state = getQueueState();
|
||||
const previous = state.tails.get(params.projectId) || Promise.resolve();
|
||||
const execution = previous.catch(() => undefined).then(params.execute);
|
||||
const tail = execution.finally(() => {
|
||||
if (state.tails.get(params.projectId) === tail) {
|
||||
state.tails.delete(params.projectId);
|
||||
}
|
||||
});
|
||||
state.tails.set(params.projectId, tail);
|
||||
return execution;
|
||||
}
|
||||
|
||||
export function getCanvasCapabilityQueueDepth(projectId: string) {
|
||||
return getQueueState().tails.has(projectId) ? 1 : 0;
|
||||
}
|
||||
161
lib/canvas/capabilities/task-store.ts
Normal file
161
lib/canvas/capabilities/task-store.ts
Normal file
@ -0,0 +1,161 @@
|
||||
import { and, eq, lt, sql } from "drizzle-orm";
|
||||
|
||||
import type { CanvasCapabilityClientCommand } from "@/lib/canvas/capabilities/client-command";
|
||||
import { db } from "@/lib/db";
|
||||
import { canvasCapabilityTask } from "@/lib/db/schema";
|
||||
|
||||
const STALE_RUNNING_TASK_MS = 15 * 60 * 1000;
|
||||
export const MAX_CANVAS_CAPABILITY_TASK_ATTEMPTS = 3;
|
||||
|
||||
export type CanvasCapabilityTaskResult = Record<string, unknown>;
|
||||
|
||||
export async function getCanvasCapabilityTask(params: {
|
||||
requestId: string;
|
||||
userId: string;
|
||||
projectId: string;
|
||||
}) {
|
||||
const [task] = await db
|
||||
.select()
|
||||
.from(canvasCapabilityTask)
|
||||
.where(
|
||||
and(
|
||||
eq(canvasCapabilityTask.requestId, params.requestId),
|
||||
eq(canvasCapabilityTask.userId, params.userId),
|
||||
eq(canvasCapabilityTask.projectId, params.projectId)
|
||||
)
|
||||
)
|
||||
.limit(1);
|
||||
return task || null;
|
||||
}
|
||||
|
||||
export async function createOrRecoverCanvasCapabilityTask(params: {
|
||||
command: CanvasCapabilityClientCommand;
|
||||
userId: string;
|
||||
projectId: string;
|
||||
}) {
|
||||
await db
|
||||
.insert(canvasCapabilityTask)
|
||||
.values({
|
||||
requestId: params.command.requestId,
|
||||
userId: params.userId,
|
||||
projectId: params.projectId,
|
||||
capabilityId: params.command.capabilityId,
|
||||
source: params.command.source,
|
||||
request: params.command,
|
||||
})
|
||||
.onConflictDoNothing({ target: canvasCapabilityTask.requestId });
|
||||
|
||||
const taskLookup = {
|
||||
requestId: params.command.requestId,
|
||||
userId: params.userId,
|
||||
projectId: params.projectId,
|
||||
};
|
||||
let task = await getCanvasCapabilityTask(taskLookup);
|
||||
if (!task) {
|
||||
throw new Error("无法创建画布能力任务");
|
||||
}
|
||||
|
||||
const staleBefore = new Date(Date.now() - STALE_RUNNING_TASK_MS);
|
||||
if (
|
||||
task.status === "failed" &&
|
||||
task.attempts < MAX_CANVAS_CAPABILITY_TASK_ATTEMPTS
|
||||
) {
|
||||
await db
|
||||
.update(canvasCapabilityTask)
|
||||
.set({
|
||||
status: "pending",
|
||||
error: null,
|
||||
completedAt: null,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(canvasCapabilityTask.id, task.id));
|
||||
} else if (
|
||||
task.status === "running" &&
|
||||
task.updatedAt < staleBefore &&
|
||||
task.attempts < MAX_CANVAS_CAPABILITY_TASK_ATTEMPTS
|
||||
) {
|
||||
await db
|
||||
.update(canvasCapabilityTask)
|
||||
.set({
|
||||
status: "pending",
|
||||
error: "恢复超时的执行中任务",
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(
|
||||
and(
|
||||
eq(canvasCapabilityTask.id, task.id),
|
||||
lt(canvasCapabilityTask.updatedAt, staleBefore)
|
||||
)
|
||||
);
|
||||
} else if (task.status === "running" && task.updatedAt < staleBefore) {
|
||||
await db
|
||||
.update(canvasCapabilityTask)
|
||||
.set({
|
||||
status: "failed",
|
||||
error: "画布能力任务超过最大重试次数",
|
||||
completedAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(eq(canvasCapabilityTask.id, task.id));
|
||||
}
|
||||
|
||||
task = (await getCanvasCapabilityTask(taskLookup)) || task;
|
||||
return task;
|
||||
}
|
||||
|
||||
export async function markCanvasCapabilityTaskRunning(taskId: string) {
|
||||
const now = new Date();
|
||||
const [task] = await db
|
||||
.update(canvasCapabilityTask)
|
||||
.set({
|
||||
status: "running",
|
||||
attempts: sql`${canvasCapabilityTask.attempts} + 1`,
|
||||
startedAt: now,
|
||||
completedAt: null,
|
||||
error: null,
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(eq(canvasCapabilityTask.id, taskId))
|
||||
.returning();
|
||||
|
||||
if (!task) throw new Error("画布能力任务不存在");
|
||||
return task;
|
||||
}
|
||||
|
||||
export async function markCanvasCapabilityTaskSucceeded(
|
||||
taskId: string,
|
||||
result: CanvasCapabilityTaskResult
|
||||
) {
|
||||
const now = new Date();
|
||||
const [task] = await db
|
||||
.update(canvasCapabilityTask)
|
||||
.set({
|
||||
status: "succeeded",
|
||||
result,
|
||||
error: null,
|
||||
completedAt: now,
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(eq(canvasCapabilityTask.id, taskId))
|
||||
.returning();
|
||||
return task || null;
|
||||
}
|
||||
|
||||
export async function markCanvasCapabilityTaskFailed(
|
||||
taskId: string,
|
||||
error: unknown
|
||||
) {
|
||||
const now = new Date();
|
||||
const message = error instanceof Error ? error.message : String(error || "执行失败");
|
||||
const [task] = await db
|
||||
.update(canvasCapabilityTask)
|
||||
.set({
|
||||
status: "failed",
|
||||
error: message.slice(0, 4000),
|
||||
completedAt: now,
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(eq(canvasCapabilityTask.id, taskId))
|
||||
.returning();
|
||||
return task || null;
|
||||
}
|
||||
138
lib/canvas/capabilities/types.ts
Normal file
138
lib/canvas/capabilities/types.ts
Normal file
@ -0,0 +1,138 @@
|
||||
import type { z } from "zod";
|
||||
|
||||
export const CANVAS_CAPABILITY_IDS = [
|
||||
"image.add_to_chat",
|
||||
"image.pick_object",
|
||||
"image.enhance",
|
||||
"image.remove_background",
|
||||
"image.ai_wash",
|
||||
"image.text_edit",
|
||||
"image.box_cutout",
|
||||
"image.local_edit",
|
||||
"image.scale_adjust",
|
||||
"image.remove_watermark",
|
||||
"image.resize",
|
||||
"image.multi_angle",
|
||||
"image.generate_3d_model",
|
||||
"image.generate_3d_world",
|
||||
"image.layer_split",
|
||||
"image.extend",
|
||||
"image.crop",
|
||||
"image.adjust",
|
||||
"image.flip_rotate",
|
||||
"image.prompt_regenerate",
|
||||
"image.note",
|
||||
"image.rename",
|
||||
"image.download",
|
||||
"image.delete",
|
||||
] as const;
|
||||
|
||||
export type CanvasCapabilityId = (typeof CANVAS_CAPABILITY_IDS)[number];
|
||||
|
||||
export type CanvasCapabilityClientAction =
|
||||
| "add-to-chat"
|
||||
| "pick-object"
|
||||
| "enhance"
|
||||
| "rmbg"
|
||||
| "ai-wash"
|
||||
| "text-edit"
|
||||
| "box-cutout"
|
||||
| "local-edit"
|
||||
| "scale-adjust"
|
||||
| "watermark"
|
||||
| "resize"
|
||||
| "angle"
|
||||
| "generate-3d-model"
|
||||
| "generate-3d-world"
|
||||
| "layer-split"
|
||||
| "extend"
|
||||
| "crop"
|
||||
| "adjust"
|
||||
| "flip-rotate"
|
||||
| "prompt-regenerate"
|
||||
| "note"
|
||||
| "rename"
|
||||
| "download"
|
||||
| "delete";
|
||||
|
||||
export type SelectedImageToolbarActionKey =
|
||||
| "pickObject"
|
||||
| "enhance"
|
||||
| "removeBackground"
|
||||
| "aiWashQuick"
|
||||
| "textEdit"
|
||||
| "boxCutout"
|
||||
| "localEdit"
|
||||
| "removeWatermark"
|
||||
| "resize"
|
||||
| "multiAngle"
|
||||
| "generate3dModel"
|
||||
| "generate3dWorld"
|
||||
| "layerSplit"
|
||||
| "extend"
|
||||
| "crop"
|
||||
| "adjust"
|
||||
| "flipRotate"
|
||||
| "promptRegenerate"
|
||||
| "note";
|
||||
|
||||
export type CanvasCapabilityExecutionMode =
|
||||
| "headless"
|
||||
| "interactive"
|
||||
| "hybrid"
|
||||
| "local";
|
||||
export type CanvasCapabilityRisk = "read" | "write" | "external" | "destructive";
|
||||
export type CanvasCapabilityModelPolicy =
|
||||
| "none"
|
||||
| "agent-selected"
|
||||
| "feature-setting"
|
||||
| "user-selected";
|
||||
export type CanvasCapabilityCommandSource = "toolbar" | "agent" | "mcp" | "system";
|
||||
|
||||
export interface CanvasCapabilityDefinition<TSchema extends z.ZodTypeAny = z.ZodTypeAny> {
|
||||
id: CanvasCapabilityId;
|
||||
title: string;
|
||||
description: string;
|
||||
clientAction: CanvasCapabilityClientAction;
|
||||
toolbarActionKey?: SelectedImageToolbarActionKey;
|
||||
toolbarSettingKey?: string;
|
||||
executionMode: CanvasCapabilityExecutionMode;
|
||||
risk: CanvasCapabilityRisk;
|
||||
modelPolicy: CanvasCapabilityModelPolicy;
|
||||
modelSettingKey?: string;
|
||||
requiresRemoteImage: boolean;
|
||||
selection: {
|
||||
nodeTypes: readonly ["image"];
|
||||
min: number;
|
||||
max?: number;
|
||||
};
|
||||
confirmation: "never" | "interactive" | "always";
|
||||
agent: {
|
||||
exposed: boolean;
|
||||
toolName?: string;
|
||||
aliases: readonly string[];
|
||||
};
|
||||
mcp: {
|
||||
exposed: boolean;
|
||||
toolName: string;
|
||||
};
|
||||
inputSchema: TSchema;
|
||||
}
|
||||
|
||||
export interface CanvasCapabilityManifestEntry {
|
||||
id: CanvasCapabilityId;
|
||||
title: string;
|
||||
description: string;
|
||||
clientAction: CanvasCapabilityClientAction;
|
||||
toolbarActionKey?: SelectedImageToolbarActionKey;
|
||||
toolbarSettingKey?: string;
|
||||
executionMode: CanvasCapabilityExecutionMode;
|
||||
risk: CanvasCapabilityRisk;
|
||||
modelPolicy: CanvasCapabilityModelPolicy;
|
||||
modelSettingKey?: string;
|
||||
requiresRemoteImage: boolean;
|
||||
selection: CanvasCapabilityDefinition["selection"];
|
||||
confirmation: CanvasCapabilityDefinition["confirmation"];
|
||||
agent: CanvasCapabilityDefinition["agent"];
|
||||
mcp: CanvasCapabilityDefinition["mcp"];
|
||||
}
|
||||
24
lib/db/migrations/0091_add_canvas_capability_tasks.sql
Normal file
24
lib/db/migrations/0091_add_canvas_capability_tasks.sql
Normal file
@ -0,0 +1,24 @@
|
||||
CREATE TABLE IF NOT EXISTS "CanvasCapabilityTask" (
|
||||
"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
|
||||
"requestId" varchar(180) NOT NULL UNIQUE,
|
||||
"userId" uuid NOT NULL REFERENCES "User"("id") ON DELETE CASCADE,
|
||||
"projectId" uuid NOT NULL REFERENCES "CanvasProject"("id") ON DELETE CASCADE,
|
||||
"capabilityId" varchar(96) NOT NULL,
|
||||
"source" varchar(16) NOT NULL,
|
||||
"status" varchar(16) NOT NULL DEFAULT 'pending',
|
||||
"version" integer NOT NULL DEFAULT 1,
|
||||
"request" jsonb NOT NULL,
|
||||
"result" jsonb,
|
||||
"error" text,
|
||||
"attempts" integer NOT NULL DEFAULT 0,
|
||||
"startedAt" timestamp,
|
||||
"completedAt" timestamp,
|
||||
"createdAt" timestamp NOT NULL DEFAULT now(),
|
||||
"updatedAt" timestamp NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS "CanvasCapabilityTask_project_status_idx"
|
||||
ON "CanvasCapabilityTask"("projectId", "status", "createdAt");
|
||||
|
||||
CREATE INDEX IF NOT EXISTS "CanvasCapabilityTask_user_created_idx"
|
||||
ON "CanvasCapabilityTask"("userId", "createdAt");
|
||||
@ -540,6 +540,13 @@
|
||||
"when": 1783946203000,
|
||||
"tag": "0090_add_bananaimage_provider_type",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 77,
|
||||
"version": "7",
|
||||
"when": 1783936800000,
|
||||
"tag": "0091_add_canvas_capability_tasks",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@ -947,6 +947,31 @@ export const canvasImageTask = pgTable("CanvasImageTask", {
|
||||
|
||||
export type CanvasImageTask = InferSelectModel<typeof canvasImageTask>;
|
||||
|
||||
export const canvasCapabilityTask = pgTable("CanvasCapabilityTask", {
|
||||
id: uuid("id").primaryKey().notNull().defaultRandom(),
|
||||
requestId: varchar("requestId", { length: 180 }).notNull().unique(),
|
||||
userId: uuid("userId")
|
||||
.notNull()
|
||||
.references(() => user.id, { onDelete: "cascade" }),
|
||||
projectId: uuid("projectId")
|
||||
.notNull()
|
||||
.references(() => canvasProject.id, { onDelete: "cascade" }),
|
||||
capabilityId: varchar("capabilityId", { length: 96 }).notNull(),
|
||||
source: varchar("source", { length: 16 }).notNull(),
|
||||
status: varchar("status", { length: 16 }).notNull().default("pending"),
|
||||
version: integer("version").notNull().default(1),
|
||||
request: jsonb("request").notNull(),
|
||||
result: jsonb("result"),
|
||||
error: text("error"),
|
||||
attempts: integer("attempts").notNull().default(0),
|
||||
startedAt: timestamp("startedAt"),
|
||||
completedAt: timestamp("completedAt"),
|
||||
createdAt: timestamp("createdAt").notNull().defaultNow(),
|
||||
updatedAt: timestamp("updatedAt").notNull().defaultNow(),
|
||||
});
|
||||
|
||||
export type CanvasCapabilityTask = InferSelectModel<typeof canvasCapabilityTask>;
|
||||
|
||||
export const enhanceProTask = pgTable("EnhanceProTask", {
|
||||
id: uuid("id").primaryKey().notNull().defaultRandom(),
|
||||
userId: uuid("userId")
|
||||
|
||||
@ -76,6 +76,7 @@
|
||||
"@icons-pack/react-simple-icons": "^13.7.0",
|
||||
"@langchain/core": "^1.1.19",
|
||||
"@langchain/langgraph": "^1.1.3",
|
||||
"@modelcontextprotocol/sdk": "^1.29.0",
|
||||
"@opentelemetry/api": "^1.9.0",
|
||||
"@opentelemetry/api-logs": "^0.200.0",
|
||||
"@plait-board/markdown-to-drawnix": "0.0.7",
|
||||
|
||||
548
pnpm-lock.yaml
generated
548
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
@ -15,6 +15,19 @@ import {
|
||||
normalizeStyleProPersistedState,
|
||||
type StyleProPersistedState,
|
||||
} from '@/lib/style-pro/config';
|
||||
import {
|
||||
canvasCapabilityClientCommandSchema,
|
||||
createCanvasCapabilityClientCommand,
|
||||
type CanvasCapabilityClientCommand,
|
||||
} from '@/lib/canvas/capabilities/client-command';
|
||||
import {
|
||||
getCanvasCapability,
|
||||
getCanvasCapabilityByClientAction,
|
||||
} from '@/lib/canvas/capabilities/registry';
|
||||
import type {
|
||||
CanvasCapabilityClientAction,
|
||||
CanvasCapabilityCommandSource,
|
||||
} from '@/lib/canvas/capabilities/types';
|
||||
|
||||
// 生成唯一 ID
|
||||
const generateId = () => crypto.randomUUID();
|
||||
@ -73,12 +86,17 @@ export interface PickedObject {
|
||||
};
|
||||
}
|
||||
|
||||
export type ImageNodeActionType = 'box-cutout' | 'text-edit' | 'local-edit';
|
||||
export type ImageNodeActionType = CanvasCapabilityClientAction;
|
||||
|
||||
export interface ImageNodeActionRequest {
|
||||
export interface ImageNodeActionRequest extends CanvasCapabilityClientCommand {
|
||||
nodeId: string;
|
||||
action: ImageNodeActionType;
|
||||
requestId: string;
|
||||
}
|
||||
|
||||
export interface ImageNodeActionOptions {
|
||||
args?: unknown;
|
||||
source?: CanvasCapabilityCommandSource;
|
||||
confirmed?: boolean;
|
||||
}
|
||||
|
||||
type CanvasViewportSource = 'user' | 'agent-transient';
|
||||
@ -120,6 +138,8 @@ interface CanvasStore {
|
||||
pickedObject: PickedObject | null;
|
||||
pendingPickedObjects: PickedObject[];
|
||||
imageNodeActionRequest: ImageNodeActionRequest | null;
|
||||
imageNodeActionQueue: ImageNodeActionRequest[];
|
||||
handledCanvasCapabilityRequestIds: string[];
|
||||
|
||||
// 🆕 新增节点（用于自动聚焦）
|
||||
lastAddedNodeId: string | null;
|
||||
@ -193,7 +213,14 @@ interface CanvasStore {
|
||||
setPickedObject: (obj: PickedObject | null) => void;
|
||||
addPendingPickedObject: (obj: PickedObject) => void;
|
||||
clearPendingPickedObjects: () => void;
|
||||
requestImageNodeAction: (nodeId: string, action: ImageNodeActionType) => void;
|
||||
dispatchCanvasCapabilityCommand: (
|
||||
command: CanvasCapabilityClientCommand
|
||||
) => void;
|
||||
requestImageNodeAction: (
|
||||
nodeId: string,
|
||||
action: ImageNodeActionType,
|
||||
options?: ImageNodeActionOptions
|
||||
) => void;
|
||||
clearImageNodeActionRequest: () => void;
|
||||
|
||||
// 🆕 自动聚焦控制
|
||||
@ -236,11 +263,18 @@ export const useCanvasStore = create<CanvasStore>()(
|
||||
pickedObject: null,
|
||||
pendingPickedObjects: [],
|
||||
imageNodeActionRequest: null,
|
||||
imageNodeActionQueue: [],
|
||||
handledCanvasCapabilityRequestIds: [],
|
||||
lastAddedNodeId: null,
|
||||
agentOperationRefreshUntil: 0,
|
||||
|
||||
// 项目操作
|
||||
setProject: (id, title) => set((state) => {
|
||||
if (state.projectId !== id) {
|
||||
state.imageNodeActionRequest = null;
|
||||
state.imageNodeActionQueue = [];
|
||||
state.handledCanvasCapabilityRequestIds = [];
|
||||
}
|
||||
state.projectId = id;
|
||||
if (title) state.projectTitle = title;
|
||||
}),
|
||||
@ -632,6 +666,9 @@ export const useCanvasStore = create<CanvasStore>()(
|
||||
state.canvasUpdatedAt = null;
|
||||
state.suppressedMediaKeys = [];
|
||||
state.styleProState = { history: [] };
|
||||
state.imageNodeActionRequest = null;
|
||||
state.imageNodeActionQueue = [];
|
||||
state.handledCanvasCapabilityRequestIds = [];
|
||||
state.history = [];
|
||||
state.historyIndex = -1;
|
||||
}),
|
||||
@ -725,16 +762,85 @@ export const useCanvasStore = create<CanvasStore>()(
|
||||
state.pendingPickedObjects = [];
|
||||
}),
|
||||
|
||||
requestImageNodeAction: (nodeId, action) => set((state) => {
|
||||
state.imageNodeActionRequest = {
|
||||
dispatchCanvasCapabilityCommand: (command) => set((state) => {
|
||||
const envelope = canvasCapabilityClientCommandSchema.parse(command);
|
||||
if (
|
||||
state.handledCanvasCapabilityRequestIds.includes(envelope.requestId)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
const parsedCommand = createCanvasCapabilityClientCommand({
|
||||
capabilityId: envelope.capabilityId,
|
||||
nodeIds: envelope.nodeIds,
|
||||
projectId: envelope.projectId,
|
||||
requestId: envelope.requestId,
|
||||
args: envelope.args,
|
||||
source: envelope.source,
|
||||
confirmed: envelope.confirmed,
|
||||
createdAt: envelope.createdAt,
|
||||
});
|
||||
const capability = getCanvasCapability(parsedCommand.capabilityId);
|
||||
if (!capability) return;
|
||||
const availableImageNodeIds = new Set(
|
||||
state.nodes
|
||||
.filter((node) => node.type === 'image')
|
||||
.map((node) => node.id)
|
||||
);
|
||||
const requests = parsedCommand.nodeIds
|
||||
.filter((nodeId) => availableImageNodeIds.has(nodeId))
|
||||
.map((nodeId, index) => ({
|
||||
...parsedCommand,
|
||||
requestId:
|
||||
parsedCommand.nodeIds.length > 1
|
||||
? `${parsedCommand.requestId}:${index + 1}`
|
||||
: parsedCommand.requestId,
|
||||
nodeIds: [nodeId],
|
||||
nodeId,
|
||||
action: capability.clientAction,
|
||||
}));
|
||||
const firstRequest = requests.shift();
|
||||
if (!firstRequest) return;
|
||||
state.handledCanvasCapabilityRequestIds.push(envelope.requestId);
|
||||
if (state.handledCanvasCapabilityRequestIds.length > 200) {
|
||||
state.handledCanvasCapabilityRequestIds.splice(
|
||||
0,
|
||||
state.handledCanvasCapabilityRequestIds.length - 200
|
||||
);
|
||||
}
|
||||
|
||||
if (state.imageNodeActionRequest) {
|
||||
state.imageNodeActionQueue.push(firstRequest, ...requests);
|
||||
} else {
|
||||
state.imageNodeActionRequest = firstRequest;
|
||||
state.imageNodeActionQueue.push(...requests);
|
||||
}
|
||||
}),
|
||||
|
||||
requestImageNodeAction: (nodeId, action, options) => set((state) => {
|
||||
const capability = getCanvasCapabilityByClientAction(action);
|
||||
if (!capability) return;
|
||||
const command = createCanvasCapabilityClientCommand({
|
||||
capabilityId: capability.id,
|
||||
nodeIds: [nodeId],
|
||||
projectId: state.projectId || undefined,
|
||||
args: options?.args,
|
||||
source: options?.source || 'toolbar',
|
||||
confirmed: options?.confirmed,
|
||||
});
|
||||
const request = {
|
||||
...command,
|
||||
nodeId,
|
||||
action,
|
||||
requestId: generateId(),
|
||||
};
|
||||
if (state.imageNodeActionRequest) {
|
||||
state.imageNodeActionQueue.push(request);
|
||||
} else {
|
||||
state.imageNodeActionRequest = request;
|
||||
}
|
||||
}),
|
||||
|
||||
clearImageNodeActionRequest: () => set((state) => {
|
||||
state.imageNodeActionRequest = null;
|
||||
state.imageNodeActionRequest = state.imageNodeActionQueue.shift() || null;
|
||||
}),
|
||||
|
||||
clearLastAddedNodeId: () => set((state) => {
|
||||
|
||||
291
tests/routes/canvas-capability-runtime.test.ts
Normal file
291
tests/routes/canvas-capability-runtime.test.ts
Normal file
@ -0,0 +1,291 @@
|
||||
import { expect, test } from "@playwright/test";
|
||||
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
||||
import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
|
||||
|
||||
import { applyCanvasOperationPlanToCanvasData } from "@/lib/canvas/agent-mutations";
|
||||
import {
|
||||
CANVAS_CAPABILITY_REGISTRY,
|
||||
getCanvasCapability,
|
||||
getCanvasCapabilityManifest,
|
||||
} from "@/lib/canvas/capabilities/registry";
|
||||
import { createCanvasCapabilityClientCommand } from "@/lib/canvas/capabilities/client-command";
|
||||
import { shouldExecuteCanvasCapabilityHeadlessly } from "@/lib/canvas/capabilities/execution-mode";
|
||||
import { createCanvasCapabilityMcpServer } from "@/lib/canvas/capabilities/mcp-server";
|
||||
import { CANVAS_CAPABILITY_IDS } from "@/lib/canvas/capabilities/types";
|
||||
import { useCanvasStore } from "@/store/canvas";
|
||||
|
||||
const SELECTED_IMAGE_TOOLBAR_ACTIONS = [
|
||||
"pickObject",
|
||||
"enhance",
|
||||
"removeBackground",
|
||||
"aiWashQuick",
|
||||
"textEdit",
|
||||
"boxCutout",
|
||||
"localEdit",
|
||||
"removeWatermark",
|
||||
"resize",
|
||||
"multiAngle",
|
||||
"generate3dModel",
|
||||
"generate3dWorld",
|
||||
"layerSplit",
|
||||
"extend",
|
||||
"crop",
|
||||
"adjust",
|
||||
"flipRotate",
|
||||
"promptRegenerate",
|
||||
"note",
|
||||
] as const;
|
||||
|
||||
test.describe("canvas capability runtime contracts", () => {
|
||||
test("registers every selected-image toolbar capability exactly once", () => {
|
||||
const registeredActions = CANVAS_CAPABILITY_REGISTRY.flatMap((capability) =>
|
||||
capability.toolbarActionKey ? [capability.toolbarActionKey] : []
|
||||
);
|
||||
|
||||
expect(new Set(registeredActions).size).toBe(registeredActions.length);
|
||||
expect(registeredActions.sort()).toEqual([...SELECTED_IMAGE_TOOLBAR_ACTIONS].sort());
|
||||
});
|
||||
|
||||
test("keeps capability, client action, Agent, and MCP names unique", () => {
|
||||
const ids = CANVAS_CAPABILITY_REGISTRY.map((capability) => capability.id);
|
||||
const clientActions = CANVAS_CAPABILITY_REGISTRY.map(
|
||||
(capability) => capability.clientAction
|
||||
);
|
||||
const agentTools = CANVAS_CAPABILITY_REGISTRY.flatMap((capability) =>
|
||||
capability.agent.exposed && capability.agent.toolName
|
||||
? [capability.agent.toolName]
|
||||
: []
|
||||
);
|
||||
const mcpTools = CANVAS_CAPABILITY_REGISTRY.flatMap((capability) =>
|
||||
capability.mcp.exposed ? [capability.mcp.toolName] : []
|
||||
);
|
||||
|
||||
expect(ids).toEqual([...CANVAS_CAPABILITY_IDS]);
|
||||
expect(new Set(ids).size).toBe(ids.length);
|
||||
expect(new Set(clientActions).size).toBe(clientActions.length);
|
||||
expect(new Set(agentTools).size).toBe(agentTools.length);
|
||||
expect(new Set(mcpTools).size).toBe(mcpTools.length);
|
||||
});
|
||||
|
||||
test("validates arguments and preserves execution provenance", () => {
|
||||
const command = createCanvasCapabilityClientCommand({
|
||||
capabilityId: "image.enhance",
|
||||
nodeIds: ["image-a", "image-a", "image-b"],
|
||||
projectId: "canvas-1",
|
||||
requestId: "request-1",
|
||||
args: { scaleFactor: 4 },
|
||||
source: "agent",
|
||||
createdAt: "2026-07-13T00:00:00.000Z",
|
||||
});
|
||||
|
||||
expect(command).toMatchObject({
|
||||
capabilityId: "image.enhance",
|
||||
nodeIds: ["image-a", "image-b"],
|
||||
projectId: "canvas-1",
|
||||
requestId: "request-1",
|
||||
args: { scaleFactor: 4 },
|
||||
source: "agent",
|
||||
});
|
||||
expect(() =>
|
||||
createCanvasCapabilityClientCommand({
|
||||
capabilityId: "image.enhance",
|
||||
nodeIds: ["image-a"],
|
||||
args: { scaleFactor: 12 },
|
||||
source: "agent",
|
||||
})
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
test("requires explicit confirmation for destructive capabilities", () => {
|
||||
expect(() =>
|
||||
createCanvasCapabilityClientCommand({
|
||||
capabilityId: "image.delete",
|
||||
nodeIds: ["image-a"],
|
||||
source: "agent",
|
||||
})
|
||||
).toThrow("requires explicit confirmation");
|
||||
|
||||
expect(
|
||||
createCanvasCapabilityClientCommand({
|
||||
capabilityId: "image.delete",
|
||||
nodeIds: ["image-a"],
|
||||
source: "agent",
|
||||
confirmed: true,
|
||||
}).confirmed
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
test("publishes a serializable manifest without runtime schemas", () => {
|
||||
const manifest = getCanvasCapabilityManifest();
|
||||
|
||||
expect(manifest).toHaveLength(CANVAS_CAPABILITY_IDS.length);
|
||||
expect(manifest.every((entry) => !("inputSchema" in entry))).toBe(true);
|
||||
expect(() => JSON.stringify(manifest)).not.toThrow();
|
||||
});
|
||||
|
||||
test("exposes the registry through a real MCP client", async () => {
|
||||
const server = await createCanvasCapabilityMcpServer({
|
||||
filterDisabled: false,
|
||||
projectId: "canvas-1",
|
||||
requestHeaders: new Headers(),
|
||||
requestUrl: "http://localhost/api/mcp/canvas/canvas-1",
|
||||
userId: "user-1",
|
||||
});
|
||||
const client = new Client({ name: "capability-contract-test", version: "1.0.0" });
|
||||
const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair();
|
||||
|
||||
await server.connect(serverTransport);
|
||||
await client.connect(clientTransport);
|
||||
try {
|
||||
const result = await client.listTools();
|
||||
const expectedNames = CANVAS_CAPABILITY_REGISTRY.flatMap((capability) =>
|
||||
capability.mcp.exposed && capability.mcp.toolName
|
||||
? [capability.mcp.toolName]
|
||||
: []
|
||||
);
|
||||
expect(result.tools.map((tool) => tool.name).sort()).toEqual(
|
||||
expectedNames.sort()
|
||||
);
|
||||
expect(
|
||||
result.tools.every(
|
||||
(tool) =>
|
||||
tool.inputSchema.type === "object" &&
|
||||
Object.hasOwn(tool.inputSchema.properties || {}, "nodeIds")
|
||||
)
|
||||
).toBe(true);
|
||||
} finally {
|
||||
await client.close();
|
||||
await server.close();
|
||||
}
|
||||
});
|
||||
|
||||
test("runs 3D capabilities headlessly for Agent and MCP", () => {
|
||||
for (const capabilityId of [
|
||||
"image.generate_3d_model",
|
||||
"image.generate_3d_world",
|
||||
] as const) {
|
||||
const capability = getCanvasCapability(capabilityId);
|
||||
expect(capability).not.toBeNull();
|
||||
if (!capability) continue;
|
||||
|
||||
const agentCommand = createCanvasCapabilityClientCommand({
|
||||
capabilityId,
|
||||
nodeIds: ["image-a"],
|
||||
source: "agent",
|
||||
});
|
||||
const toolbarCommand = createCanvasCapabilityClientCommand({
|
||||
capabilityId,
|
||||
nodeIds: ["image-a"],
|
||||
source: "toolbar",
|
||||
});
|
||||
expect(
|
||||
shouldExecuteCanvasCapabilityHeadlessly({ capability, command: agentCommand })
|
||||
).toBe(true);
|
||||
expect(
|
||||
shouldExecuteCanvasCapabilityHeadlessly({ capability, command: toolbarCommand })
|
||||
).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
test("writes recoverable 3D task nodes through the shared mutation protocol", () => {
|
||||
const sourceNode = {
|
||||
id: "image-a",
|
||||
type: "image",
|
||||
position: { x: 10, y: 20 },
|
||||
size: { width: 320, height: 240 },
|
||||
data: { src: "https://example.com/source.png" },
|
||||
};
|
||||
const cases = [
|
||||
{ type: "model3d" as const, provider: "tripo-3d" },
|
||||
{ type: "world3d" as const, provider: "worldlabs-3d" },
|
||||
];
|
||||
|
||||
for (const item of cases) {
|
||||
const result = applyCanvasOperationPlanToCanvasData({
|
||||
canvasData: { nodes: [sourceNode], edges: [] },
|
||||
plan: {
|
||||
version: 1,
|
||||
operationId: `operation-${item.type}`,
|
||||
operation: "insert_beside",
|
||||
projectId: "canvas-1",
|
||||
sourceNodeIds: [sourceNode.id],
|
||||
targetNodeIds: [sourceNode.id],
|
||||
placement: "right",
|
||||
placementStrategy: "beside_each_source",
|
||||
mediaItems: [
|
||||
{
|
||||
mediaItemId: `result-${item.type}`,
|
||||
type: item.type,
|
||||
url: "https://example.com/source.png",
|
||||
title: `3D result ${item.type}`,
|
||||
sourceNodeIds: [sourceNode.id],
|
||||
metadata: {
|
||||
taskId: `task-${item.type}`,
|
||||
provider: item.provider,
|
||||
status: "generating",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
const resultNodes = (Array.isArray(result.canvasData.nodes)
|
||||
? result.canvasData.nodes
|
||||
: []) as Array<{
|
||||
connections?: string[];
|
||||
data?: { metadata?: { taskId?: string }; status?: string };
|
||||
id: string;
|
||||
type?: string;
|
||||
}>;
|
||||
const createdNode = resultNodes.find(
|
||||
(node) => node.id === result.createdNodeIds[0]
|
||||
);
|
||||
expect(createdNode?.type).toBe(item.type);
|
||||
expect(createdNode?.data?.status).toBe("generating");
|
||||
expect(createdNode?.data?.metadata?.taskId).toBe(`task-${item.type}`);
|
||||
expect(createdNode?.connections).toContain(sourceNode.id);
|
||||
}
|
||||
});
|
||||
|
||||
test("queues client commands once and clears them when switching canvas", () => {
|
||||
useCanvasStore.setState({
|
||||
projectId: "canvas-1",
|
||||
nodes: [
|
||||
{
|
||||
id: "image-a",
|
||||
type: "image",
|
||||
position: { x: 0, y: 0 },
|
||||
size: { width: 320, height: 320 },
|
||||
data: {
|
||||
src: "https://example.com/source.png",
|
||||
status: "completed",
|
||||
},
|
||||
connections: [],
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
},
|
||||
],
|
||||
imageNodeActionRequest: null,
|
||||
imageNodeActionQueue: [],
|
||||
handledCanvasCapabilityRequestIds: [],
|
||||
});
|
||||
const command = createCanvasCapabilityClientCommand({
|
||||
capabilityId: "image.crop",
|
||||
nodeIds: ["image-a"],
|
||||
projectId: "canvas-1",
|
||||
requestId: "interactive-request-1",
|
||||
source: "agent",
|
||||
createdAt: "2026-07-13T00:00:00.000Z",
|
||||
});
|
||||
|
||||
useCanvasStore.getState().dispatchCanvasCapabilityCommand(command);
|
||||
useCanvasStore.getState().dispatchCanvasCapabilityCommand(command);
|
||||
expect(useCanvasStore.getState().imageNodeActionRequest?.requestId).toBe(
|
||||
command.requestId
|
||||
);
|
||||
expect(useCanvasStore.getState().imageNodeActionQueue).toHaveLength(0);
|
||||
|
||||
useCanvasStore.getState().setProject("canvas-2");
|
||||
expect(useCanvasStore.getState().imageNodeActionRequest).toBeNull();
|
||||
expect(useCanvasStore.getState().handledCanvasCapabilityRequestIds).toEqual([]);
|
||||
});
|
||||
});
|
||||
@ -126,7 +126,7 @@ export type CanvasOperationKind =
|
||||
|
||||
export interface CanvasMediaItem {
|
||||
mediaItemId: string;
|
||||
type: "image" | "video";
|
||||
type: "image" | "video" | "model3d" | "world3d";
|
||||
url: string;
|
||||
assetId?: string;
|
||||
width?: number;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user