From 650a94db72f698129c63329c8b62b6fab9616659 Mon Sep 17 00:00:00 2001 From: 134 Date: Tue, 20 Oct 2026 14:54:41 +0800 Subject: [PATCH] feat(canvas-agent): run native ZCode core in isolated task sandboxes --- app/(chat)/api/chat/route.ts | 14 +- app/api/canvas/agent-capabilities/route.ts | 37 ++ app/api/canvas/agent-chat/route.ts | 447 ++++++++++++++ app/api/canvas/agent-runs/route.ts | 65 ++ app/api/canvas/video-agent/route.ts | 78 +++ .../[id]/CanvasEditorLegacyPageClient.tsx | 2 + components/canvas/CanvasChatPanel.tsx | 36 +- .../canvas/LiblibVideoCanvasReplica.tsx | 138 ++++- .../chat-panel/canvas-agent-run-panel.tsx | 181 ++++++ .../chat-panel/canvas-agent-ui-listener.tsx | 78 +++ .../chat-panel/video-canvas-agent-panel.tsx | 324 ++++++++++ docs/specs/zcode-canvas/architecture.md | 37 ++ lib/ai/canvas-agent-runtime/canvas.ts | 33 + .../canvas-agent-runtime/capability-access.ts | 264 ++++++++ lib/ai/canvas-agent-runtime/context.ts | 48 ++ lib/ai/canvas-agent-runtime/conversation.ts | 107 ++++ .../image-access-tools.ts | 225 +++++++ lib/ai/canvas-agent-runtime/loop.ts | 300 +++++++++ lib/ai/canvas-agent-runtime/prompt.ts | 29 + lib/ai/canvas-agent-runtime/protocol.ts | 90 +++ lib/ai/canvas-agent-runtime/requirements.ts | 38 ++ lib/ai/canvas-agent-runtime/results.ts | 44 ++ lib/ai/canvas-agent-runtime/review.ts | 183 ++++++ lib/ai/canvas-agent-runtime/routing.ts | 10 + lib/ai/canvas-agent-runtime/store.ts | 370 +++++++++++ lib/ai/canvas-agent-runtime/tools.ts | 298 +++++++++ lib/ai/canvas-agent-runtime/ui-command.ts | 10 + .../video-access-tools.ts | 322 ++++++++++ .../video-image-functions.ts | 133 ++++ .../video-media-functions.ts | 98 +++ lib/ai/canvas-agent-runtime/video-probe.ts | 29 + lib/ai/canvas-agent-runtime/video-protocol.ts | 171 ++++++ lib/ai/canvas-agent-runtime/video-review.ts | 91 +++ lib/ai/canvas-agent-runtime/video-store.ts | 330 ++++++++++ lib/ai/canvas-agent-runtime/video-tools.ts | 578 ++++++++++++++++++ lib/ai/third-party/zcode/LICENSE | 202 ++++++ lib/ai/third-party/zcode/NOTICE.txt | 11 + lib/ai/third-party/zcode/scheduler.ts | 254 ++++++++ lib/ai/zcode-agent/model-gateway.ts | 76 +++ lib/ai/zcode-agent/task.ts | 182 ++++++ lib/canvas/capabilities/ai-sdk-tools.ts | 45 +- lib/canvas/capabilities/availability.ts | 18 +- lib/canvas/capabilities/registry.ts | 6 +- lib/canvas/capabilities/runtime.ts | 2 +- .../migrations/0100_add_canvas_agent_runs.sql | 16 + lib/db/migrations/meta/_journal.json | 7 + lib/db/schema.ts | 18 + package.json | 2 + playwright.canvas-agent.config.ts | 11 + scripts/zcode-agent/native-fixture.mjs | 81 +++ scripts/zcode-agent/sandbox-fixture.mjs | 54 ++ scripts/zcode-agent/stage-runtime.mjs | 24 + scripts/zcode-agent/task-fixture.ts | 70 +++ services/zcode-agent/NOTICE.md | 18 + services/zcode-agent/README.md | 62 ++ services/zcode-agent/bridge.mjs | 61 ++ services/zcode-agent/broker.mjs | 84 +++ services/zcode-agent/runner.d.mts | 15 + services/zcode-agent/runner.mjs | 190 ++++++ services/zcode-agent/sandbox.mjs | 58 ++ tests/routes/agent-capability-access.test.ts | 326 ++++++++++ .../canvas-agent-runtime-review.test.ts | 131 ++++ .../routes/canvas-agent-runtime-store.test.ts | 332 ++++++++++ .../routes/canvas-agent-runtime-video.test.ts | 375 ++++++++++++ tests/routes/canvas-agent-runtime.test.ts | 510 ++++++++++++++++ 65 files changed, 8443 insertions(+), 36 deletions(-) create mode 100644 app/api/canvas/agent-capabilities/route.ts create mode 100644 app/api/canvas/agent-chat/route.ts create mode 100644 app/api/canvas/agent-runs/route.ts create mode 100644 app/api/canvas/video-agent/route.ts create mode 100644 components/canvas/chat-panel/canvas-agent-run-panel.tsx create mode 100644 components/canvas/chat-panel/canvas-agent-ui-listener.tsx create mode 100644 components/canvas/chat-panel/video-canvas-agent-panel.tsx create mode 100644 docs/specs/zcode-canvas/architecture.md create mode 100644 lib/ai/canvas-agent-runtime/canvas.ts create mode 100644 lib/ai/canvas-agent-runtime/capability-access.ts create mode 100644 lib/ai/canvas-agent-runtime/context.ts create mode 100644 lib/ai/canvas-agent-runtime/conversation.ts create mode 100644 lib/ai/canvas-agent-runtime/image-access-tools.ts create mode 100644 lib/ai/canvas-agent-runtime/loop.ts create mode 100644 lib/ai/canvas-agent-runtime/prompt.ts create mode 100644 lib/ai/canvas-agent-runtime/protocol.ts create mode 100644 lib/ai/canvas-agent-runtime/requirements.ts create mode 100644 lib/ai/canvas-agent-runtime/results.ts create mode 100644 lib/ai/canvas-agent-runtime/review.ts create mode 100644 lib/ai/canvas-agent-runtime/routing.ts create mode 100644 lib/ai/canvas-agent-runtime/store.ts create mode 100644 lib/ai/canvas-agent-runtime/tools.ts create mode 100644 lib/ai/canvas-agent-runtime/ui-command.ts create mode 100644 lib/ai/canvas-agent-runtime/video-access-tools.ts create mode 100644 lib/ai/canvas-agent-runtime/video-image-functions.ts create mode 100644 lib/ai/canvas-agent-runtime/video-media-functions.ts create mode 100644 lib/ai/canvas-agent-runtime/video-probe.ts create mode 100644 lib/ai/canvas-agent-runtime/video-protocol.ts create mode 100644 lib/ai/canvas-agent-runtime/video-review.ts create mode 100644 lib/ai/canvas-agent-runtime/video-store.ts create mode 100644 lib/ai/canvas-agent-runtime/video-tools.ts create mode 100644 lib/ai/third-party/zcode/LICENSE create mode 100644 lib/ai/third-party/zcode/NOTICE.txt create mode 100644 lib/ai/third-party/zcode/scheduler.ts create mode 100644 lib/ai/zcode-agent/model-gateway.ts create mode 100644 lib/ai/zcode-agent/task.ts create mode 100644 lib/db/migrations/0100_add_canvas_agent_runs.sql create mode 100644 playwright.canvas-agent.config.ts create mode 100644 scripts/zcode-agent/native-fixture.mjs create mode 100644 scripts/zcode-agent/sandbox-fixture.mjs create mode 100644 scripts/zcode-agent/stage-runtime.mjs create mode 100644 scripts/zcode-agent/task-fixture.ts create mode 100644 services/zcode-agent/NOTICE.md create mode 100644 services/zcode-agent/README.md create mode 100644 services/zcode-agent/bridge.mjs create mode 100644 services/zcode-agent/broker.mjs create mode 100644 services/zcode-agent/runner.d.mts create mode 100644 services/zcode-agent/runner.mjs create mode 100644 services/zcode-agent/sandbox.mjs create mode 100644 tests/routes/agent-capability-access.test.ts create mode 100644 tests/routes/canvas-agent-runtime-review.test.ts create mode 100644 tests/routes/canvas-agent-runtime-store.test.ts create mode 100644 tests/routes/canvas-agent-runtime-video.test.ts create mode 100644 tests/routes/canvas-agent-runtime.test.ts diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts index ada9bc369..04db2300a 100644 --- a/app/(chat)/api/chat/route.ts +++ b/app/(chat)/api/chat/route.ts @@ -1,3 +1,5 @@ +import { isExplicitDestructiveIntent } from "@/lib/ai/canvas-agent-runtime/capability-access"; +import { inferVideoTaskRequirements } from "@/lib/ai/canvas-agent-runtime/video-protocol"; import { createLogger } from "@/lib/observability/logger"; import { geolocation } from "@vercel/functions"; import { @@ -929,6 +931,15 @@ export async function POST(request: Request) { const json = await request.json(); chatApiLogger.info("[Chat API] Received request body:", JSON.stringify(json, null, 2)); requestBody = postRequestBodySchema.parse(json); + // Agent/Auto Agent has one execution owner: the isolated native ZCode entry. + // This also upgrades existing browser tabs without routing Claw here. + if (requestBody.composerMode === "agent" && requestBody.contextPayload?.source === "canvas" && requestBody.contextPayload.projectId) { + const { POST: nativeAgentPost } = await import("@/app/api/canvas/agent-chat/route"); + return nativeAgentPost(new Request(new URL("/api/canvas/agent-chat", request.url), { + method: "POST", headers: request.headers, body: JSON.stringify(json), + })); + } + } catch (error) { chatApiLogger.error("[Chat API] Schema validation failed:", error); return new ChatSDKError("bad_request:api").toResponse(); @@ -1351,6 +1362,7 @@ export async function POST(request: Request) { } const allowAgentVideoGeneration = + (contextPayload?.source === "canvas" && inferVideoTaskRequirements(requestTextContent).expectsVideo) || Boolean(requestBody.videoParams) || isVideoProviderType(requestedModelProviderType, requestedModelConfig || undefined); let smartAutoWorkflowDecision: @@ -11202,7 +11214,7 @@ export async function POST(request: Request) { requestHeaders: request.headers, requestUrl: request.url, defaultModelId: finalModelId, - destructiveConfirmed: isCanvasOperationConfirmText(requestTextContent), + authorizeDestructive: (nodeIds) => isExplicitDestructiveIntent(requestTextContent) && nodeIds.length > 0 && nodeIds.every((nodeId) => selectedCanvasImageNodeIds.includes(nodeId)), callbacks: { onClientCommand: (command, capability) => { writeCanvasCapabilityCommandChunk( diff --git a/app/api/canvas/agent-capabilities/route.ts b/app/api/canvas/agent-capabilities/route.ts new file mode 100644 index 000000000..3665a4ff7 --- /dev/null +++ b/app/api/canvas/agent-capabilities/route.ts @@ -0,0 +1,37 @@ +import { NextResponse } from "next/server"; +import { z } from "zod"; +import { and, eq } from "drizzle-orm"; +import { auth } from "@/app/(auth)/auth"; +import { db } from "@/lib/db"; +import { canvasProject } from "@/lib/db/schema"; +import { getSystemSetting } from "@/lib/db/queries"; +import { + normalizeVideoCanvasFunctionConfig, + VIDEO_CANVAS_FUNCTION_CONFIG_KEY, +} from "@/lib/video-canvas/config"; +import { getCanvasAgentAccessManifest } from "@/lib/ai/canvas-agent-runtime/capability-access"; +export const dynamic = "force-dynamic"; +export async function GET(request: Request) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); + const parsed = z.string().uuid().safeParse(new URL(request.url).searchParams.get("projectId")); + if (!parsed.success) return NextResponse.json({ error: "画布编号无效" }, { status: 400 }); + const [project] = await db + .select({ id: canvasProject.id, workspace: canvasProject.workspaceType }) + .from(canvasProject) + .where(and(eq(canvasProject.id, parsed.data), eq(canvasProject.userId, session.user.id))) + .limit(1); + if (!project) return NextResponse.json({ error: "画布不存在" }, { status: 404 }); + const config = normalizeVideoCanvasFunctionConfig( + (await getSystemSetting({ key: VIDEO_CANVAS_FUNCTION_CONFIG_KEY }))?.value, + ); + return NextResponse.json( + { + workspace: project.workspace, + capabilities: getCanvasAgentAccessManifest(config).filter( + (item) => item.workspace === (project.workspace === "video" ? "video" : "canvas"), + ), + }, + { headers: { "Cache-Control": "private, no-store" } }, + ); +} diff --git a/app/api/canvas/agent-chat/route.ts b/app/api/canvas/agent-chat/route.ts new file mode 100644 index 000000000..63beb9e14 --- /dev/null +++ b/app/api/canvas/agent-chat/route.ts @@ -0,0 +1,447 @@ +import { + createUIMessageStream, + createUIMessageStreamResponse, + generateText, +} from "ai"; +import { after, NextResponse } from "next/server"; +import { z } from "zod"; +import { db } from "@/lib/db"; +import { siteSettings, canvasProject } from "@/lib/db/schema"; +import { eq } from "drizzle-orm"; +import { auth } from "@/app/(auth)/auth"; +import { postRequestBodySchema } from "@/app/(chat)/api/chat/schema"; +import { entitlementsByUserType } from "@/lib/ai/entitlements"; +import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; +import { + getAgentAiModelWithProvider, + getAccessibleAgentPresetById, + getMultimodalAiModelsWithProvider, + getChatById, + getMessagesByChatId, + getMessageCountByUserId, + getSiteSettings, + saveChat, + saveMessages, +} from "@/lib/db/queries"; +import { runZcodeCanvasTask } from "@/lib/ai/zcode-agent/task"; +import { createZcodeModelGateway } from "@/lib/ai/zcode-agent/model-gateway"; +import { newAgentRunSnapshot, publicAgentError } from "@/lib/ai/canvas-agent-runtime/protocol"; +import { shouldUseCanvasAgentRuntime } from "@/lib/ai/canvas-agent-runtime/routing"; +import { + acquireAgentRun, + agentRunView, + AgentRunError, + checkpointAgentRun, + consumeAgentRunInputs, + heartbeatAgentRun, +} from "@/lib/ai/canvas-agent-runtime/store"; +import { readOwnedCanvas } from "@/lib/ai/canvas-agent-runtime/canvas"; +import { buildCanvasRuntimeTools } from "@/lib/ai/canvas-agent-runtime/tools"; +import { canvasRuntimeSystemPrompt } from "@/lib/ai/canvas-agent-runtime/prompt"; +import { verifyCanvasAgentResults } from "@/lib/ai/canvas-agent-runtime/review"; +import { inferCanvasTaskRequirements } from "@/lib/ai/canvas-agent-runtime/requirements"; +import { + resolveRuntimeConversationSources, + persistRuntimeConversationResults, +} from "@/lib/ai/canvas-agent-runtime/conversation"; +import { getCanvasAgentFeatureFlagsFromSystemSettings } from "@/lib/canvas/agent-feature-settings"; +import { canApplyCanvasAgentMutations } from "@/lib/canvas/agent-feature-flags"; + +import { + videoAgentSettingsSchema, + inferVideoTaskRequirements, + videoNodeUrl, + videoRuntimeSystemPrompt, +} from "@/lib/ai/canvas-agent-runtime/video-protocol"; +import { readOwnedVideoCanvas } from "@/lib/ai/canvas-agent-runtime/video-store"; +import { + buildVideoRuntimeTools, + createVideoRuntimePorts, +} from "@/lib/ai/canvas-agent-runtime/video-tools"; +import { probeVideoMetadata } from "@/lib/ai/canvas-agent-runtime/video-probe"; +import { verifyVideoAgentResults } from "@/lib/ai/canvas-agent-runtime/video-review"; + +export const runtime = "nodejs"; +export const maxDuration = 1200; +const requestSchema = postRequestBodySchema.extend({ + resumeRunId: z.string().uuid().optional(), + workspaceVariant: z.enum(["canvas", "video"]).default("canvas"), + videoAgentSettings: videoAgentSettingsSchema.optional(), +}); +export async function POST(request: Request) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); + try { + const body = requestSchema.parse(await request.json()); + if (!shouldUseCanvasAgentRuntime(body)) + throw new AgentRunError("此请求不属于画布 Agent 任务", 400); + const projectId = body.contextPayload!.projectId!; + const [ownedProject] = await db.select({ userId: canvasProject.userId, workspaceType: canvasProject.workspaceType }) + .from(canvasProject).where(eq(canvasProject.id, projectId)).limit(1); + if (!ownedProject || ownedProject.userId !== session.user.id) throw new AgentRunError("画布不存在", 404); + const isVideo = ownedProject.workspaceType === "video"; + const preset = body.agentPresetId ? await getAccessibleAgentPresetById({ id: body.agentPresetId, userId: session.user.id }) : null; + if (body.agentPresetId && (!preset || preset.status !== "active")) throw new AgentRunError("此 Agent 预设不可用", 403); + const objective = body.message.parts + .filter((part) => part.type === "text") + .map((part) => part.text) + .join("\n") + .trim(); + if (!objective) throw new AgentRunError("请描述需要完成的创作任务", 400); + const settings = await getSiteSettings(); + if ( + !settings.agentEnabled || + !canApplyCanvasAgentMutations(await getCanvasAgentFeatureFlagsFromSystemSettings()) + ) { + throw new AgentRunError("管理员尚未开启画布 Agent 操作", 403); + } + const [usageSettings] = await db.select().from(siteSettings).limit(1); + if (usageSettings?.usageTimeEnabled) { + const time = new Intl.DateTimeFormat("en-GB", { + timeZone: "Asia/Shanghai", + hour: "2-digit", + minute: "2-digit", + hourCycle: "h23", + }).format(new Date()); + if ( + time < (usageSettings.usageTimeStart || "08:00") || + time > (usageSettings.usageTimeEnd || "20:00") + ) + throw new AgentRunError(usageSettings.usageTimeMessage || "当前不在系统使用时间段内", 403); + } + const count = await getMessageCountByUserId({ id: session.user.id, differenceInHours: 24 }); + if (count > entitlementsByUserType[session.user.type].maxMessagesPerDay) + throw new AgentRunError("今日对话次数已达到限额", 429); + const imageCanvas = isVideo ? null : await readOwnedCanvas(session.user.id, projectId); + if (imageCanvas?.project.workspaceType === "video") + throw new AgentRunError("请使用视频画布 Agent 入口", 400); + const videoCanvas = isVideo ? await readOwnedVideoCanvas(session.user.id, projectId) : null; + const nodes = imageCanvas?.nodes || []; + const availableIds = isVideo + ? videoCanvas!.nodes.map((node) => node.id) + : nodes.map((node) => node.id); + const selectedNodeIds = body.contextPayload?.selectedNodeIds || []; + if (selectedNodeIds.some((id) => !availableIds.includes(id))) { + throw new AgentRunError("选中图片尚未保存到画布，请等待保存完成后重试。"); + } + const existingChat = await getChatById({ id: body.id }); + if (existingChat && existingChat.userId !== session.user.id) + throw new AgentRunError("对话不存在", 404); + if (!existingChat) + await saveChat({ + id: body.id, + userId: session.user.id, + title: objective.slice(0, 80), + visibility: "private", + }); + const conversation = isVideo + ? { + sourceNodeIds: selectedNodeIds, + targetReason: "video-selected-nodes", + previousContext: null, + } + : await resolveRuntimeConversationSources({ + userId: session.user.id, + chatId: body.id, + projectId, + text: objective, + parts: body.message.parts, + nodes, + selectedNodeIds, + }); + const configured = await getAgentAiModelWithProvider(); + if (!configured) throw new AgentRunError("请先在后台配置 Agent 对话模型", 503); + const model = await getLanguageModelById(configured.model.id); + if (!model) throw new AgentRunError("Agent 模型暂不可用", 503); + const visionEntry = configured.model.supportsVision + ? configured + : (await getMultimodalAiModelsWithProvider()).find((entry) => entry.model.supportsVision); + const visionModel = visionEntry ? await getLanguageModelById(visionEntry.model.id) : null; + const assistantMessageId = crypto.randomUUID(); + const previousVideoMessages = + existingChat + ? (await getMessagesByChatId({ id: body.id })).slice(-6).map((message) => ({ + role: message.role as "user" | "assistant", + content: (message.parts as Array<{ type: string; text?: string }>) + .filter((part) => part.type === "text") + .map((part) => part.text || "") + .join("\n") + .slice(0, 4000), + })) + : []; + const initialSnapshot = newAgentRunSnapshot(objective, [ + ...previousVideoMessages, + { role: "user", content: objective }, + ]); + initialSnapshot.workspace = isVideo ? "video" : "canvas"; + if (isVideo) { + initialSnapshot.videoRequirements = inferVideoTaskRequirements(objective); + initialSnapshot.videoSettings = videoAgentSettingsSchema.parse({ + ...body.videoAgentSettings, + ...(initialSnapshot.videoRequirements.duration + ? { duration: initialSnapshot.videoRequirements.duration } + : {}), + ...(initialSnapshot.videoRequirements.ratio + ? { ratio: initialSnapshot.videoRequirements.ratio } + : {}), + }); + } + initialSnapshot.sourceNodeIds = conversation.sourceNodeIds; + initialSnapshot.requirements = { + ...inferCanvasTaskRequirements(objective, initialSnapshot.sourceNodeIds.length), + sourceCount: initialSnapshot.sourceNodeIds.length, + }; + const run = await acquireAgentRun({ + id: body.message.id, + userId: session.user.id, + projectId, + chatId: body.id, + assistantMessageId, + resumeRunId: body.resumeRunId, + snapshot: initialSnapshot, + }); + const snapshot = run.snapshot; + const taskSourceNodeIds = snapshot.sourceNodeIds ?? selectedNodeIds; + const context = { + recentConversation: previousVideoMessages, + preset: preset ? { name: preset.name, type: preset.type, promptTemplate: preset.promptTemplate, defaultModelId: preset.defaultModelId, defaultParams: preset.defaultParams, slots: preset.slots.map(slot => ({ title: slot.title, promptTemplate: slot.promptTemplate, defaultParams: slot.defaultParams })) } : null, + videoSettings: snapshot.videoSettings, + selectedNodeIds: taskSourceNodeIds, + nodes: nodes + .filter((node) => taskSourceNodeIds.includes(node.id)) + .map((node) => ({ + id: node.id, + type: node.type, + position: node.position, + size: node.size, + title: node.data.title, + })), + skillInstructions: body.hiddenPrompt || undefined, + imageParameters: body.imageParams, + targetReason: conversation.targetReason, + previousConversationContext: conversation.previousContext, + }; + await saveMessages({ + messages: [ + { + id: body.message.id, + chatId: body.id, + role: "user", + parts: body.message.parts, + attachments: [], + createdAt: new Date(), + videoStorageStatus: null, + temporaryVideoUrl: null, + permanentVideoUrl: null, + archivedAt: null, + videoMetadata: null, + }, + ], + }); + let archiveVideoResults: (() => Promise) | undefined; + const stream = createUIMessageStream({ + execute: async ({ writer }) => { + const emit = (type: string, data: unknown) => writer.write({ type, data } as never); + let cancelled = false; + let leaseLost = false; + let activeModelController: AbortController | null = null; + const heartbeat = setInterval(() => { + void heartbeatAgentRun(run) + .then((current) => { + if (!current) leaseLost = true; + if (current?.cancelRequested) cancelled = true; + if (cancelled || leaseLost) activeModelController?.abort(); + }) + .catch(() => { + leaseLost = true; + activeModelController?.abort(); + }); + emit("data-keepalive", { label: "canvas-agent-runtime", ts: Date.now() }); + }, 10_000); + const textId = crypto.randomUUID(); + writer.write({ type: "start", messageId: assistantMessageId }); + writer.write({ type: "text-start", id: textId }); + writer.write({ + type: "text-delta", + id: textId, + delta: isVideo ? "正在按步骤处理你的视频任务。\n\n" : "正在按步骤处理你的图片任务。\n\n", + }); + try { + const videoPorts = isVideo + ? await createVideoRuntimePorts({ + userId: session.user.id, + chatId: body.id, + projectId, + runId: run.id, + assistantMessageId, + request, + snapshot, + emit, + }) + : null; + archiveVideoResults = videoPorts?.archiveResults; + const tools = isVideo + ? buildVideoRuntimeTools({ + runId: run.id, + projectId, + selectedNodeIds, + snapshot, + ports: videoPorts!, + }) + : await buildCanvasRuntimeTools({ + userId: session.user.id, + chatId: body.id, + projectId, + runId: run.id, + assistantMessageId, + selectedNodeIds: taskSourceNodeIds, + request, + model: visionModel || model, + supportsVision: Boolean(visionModel), + imageSize: body.imageParams?.imageSize, + preferredModelId: preset?.type === "image" ? preset.defaultModelId || undefined : undefined, + operationSelectedNodeIds: selectedNodeIds, + objective: [snapshot.objective, ...snapshot.inputs.map((input) => input.text)].join( + "\n", + ), + emit, + }); + const gateway = await createZcodeModelGateway(configured); + activeModelController = new AbortController(); + const status = await runZcodeCanvasTask({ + runId: run.id, snapshot, definitions: tools.definitions, + instructions: (isVideo ? videoRuntimeSystemPrompt : canvasRuntimeSystemPrompt)(snapshot.objective, snapshot.summary, snapshot.inputs.map(input => input.text), context), + context, gateway, signal: activeModelController.signal, + tools: tools.executable, + checkpoint: async (state, stateStatus) => { + if (leaseLost) throw new AgentRunError("任务执行已中断"); + const saved = await checkpointAgentRun(run, state, stateStatus); + cancelled ||= saved.cancelRequested; + emit("data-canvasAgentRun", agentRunView(saved)); + }, + consumeInputs: async (state) => { + if (leaseLost) throw new AgentRunError("任务执行已中断"); + const control = await consumeAgentRunInputs(run, state); + cancelled ||= control.cancelled; + return { cancelled }; + }, + isCancelled: async () => { + const current = await heartbeatAgentRun(run); + if (!current) throw new AgentRunError("任务执行已中断"); + cancelled ||= current.cancelRequested; + return cancelled; + }, + verify: async (state) => + isVideo + ? verifyVideoAgentResults({ + snapshot: state, + nodes: (await readOwnedVideoCanvas(session.user.id, projectId)).nodes, + inspect: async (node, requirements) => { + const metadata = await probeVideoMetadata(videoNodeUrl(node)); + const report = await videoPorts!.analyze({ + mode: "video-analysis", + prompt: `请基于真实抽帧画面检查主体、场景、镜头连贯性，说明不确定内容。用户要求：${requirements}`, + imageUrls: [], + videoUrls: [videoNodeUrl(node)], + }); + const verdict = await generateText({ + model, + system: + '你是视频验收员。依据实际抽帧分析判断是否满足用户要求；不能从文件名或提示词判断。若报告无法读取真实帧，必须不通过。抽帧只能检查画面，不能证明音频、精确时长或完整运动，涉及这些要求无法确认时说明需要人工检查。仅返回 JSON {"passed":boolean,"issues":string[]}。', + prompt: JSON.stringify({ requirements, report }), + maxOutputTokens: 1200, + maxRetries: 0, + abortSignal: AbortSignal.timeout(60_000), + }); + state.inputTokens += verdict.usage.inputTokens || 0; + state.outputTokens += verdict.usage.outputTokens || 0; + const match = verdict.text.match(/\{[\s\S]*\}/); + if (!match) throw new Error("视频检查未返回有效结果"); + return { + ...z + .object({ passed: z.boolean(), issues: z.array(z.string()).max(12) }) + .parse(JSON.parse(match[0])), + ...metadata, + }; + }, + }) + : verifyCanvasAgentResults({ + snapshot: state, + userId: session.user.id, + projectId, + model: visionModel || model, + supportsVision: Boolean(visionModel), + }), + }); + const finalText = snapshot.message; + if (!isVideo) + await persistRuntimeConversationResults({ + userId: session.user.id, + chatId: body.id, + projectId, + assistantMessageId, + snapshot, + }).catch((error) => { + console.warn("[Canvas Agent] Conversation memory update failed", { + runId: run.id, + error: publicAgentError(error), + }); + }); + await saveMessages({ + messages: [ + { + id: assistantMessageId, + chatId: body.id, + role: "assistant", + parts: [{ type: "text", text: finalText }], + attachments: [], + createdAt: new Date(), + videoStorageStatus: null, + temporaryVideoUrl: null, + permanentVideoUrl: null, + archivedAt: null, + videoMetadata: null, + }, + ], + }); + writer.write({ type: "text-delta", id: textId, delta: finalText }); + } catch (error) { + const text = publicAgentError(error); + await checkpointAgentRun(run, { ...snapshot, message: text }, "needs_review").catch( + () => undefined, + ); + writer.write({ type: "text-delta", id: textId, delta: text }); + } finally { + clearInterval(heartbeat); + writer.write({ type: "text-end", id: textId }); + writer.write({ type: "finish" }); + } + }, + }); + // Browser disconnection does not cancel paid image work. Keep consuming on the server. + const [client, background] = stream.tee(); + after(async () => { + const reader = background.getReader(); + try { + while (!(await reader.read()).done) { + /* Drain the server branch. */ + } + } finally { + reader.releaseLock(); + await archiveVideoResults?.(); + } + }); + return createUIMessageStreamResponse({ + stream: client, + headers: { "Cache-Control": "no-store", "X-Accel-Buffering": "no" }, + }); + } catch (error) { + return NextResponse.json( + { error: publicAgentError(error), message: publicAgentError(error) }, + { + status: + error instanceof AgentRunError ? error.status : error instanceof z.ZodError ? 400 : 503, + }, + ); + } +} diff --git a/app/api/canvas/agent-runs/route.ts b/app/api/canvas/agent-runs/route.ts new file mode 100644 index 000000000..5bd4b98ed --- /dev/null +++ b/app/api/canvas/agent-runs/route.ts @@ -0,0 +1,65 @@ +import { NextResponse } from "next/server"; +import { z } from "zod"; +import { auth } from "@/app/(auth)/auth"; +import { + AgentRunError, + agentRunView, + assertAgentRunOwnership, + controlAgentRun, + getAgentRun, +} from "@/lib/ai/canvas-agent-runtime/store"; +import { publicAgentError } from "@/lib/ai/canvas-agent-runtime/protocol"; + +export const dynamic = "force-dynamic"; +const scope = z.object({ + chatId: z.string().uuid(), + projectId: z.string().uuid(), + runId: z.string().uuid().optional(), +}); +const control = scope + .extend({ + runId: z.string().uuid(), + action: z.enum(["steer", "cancel"]), + inputId: z.string().uuid().optional(), + text: z.string().trim().min(1).max(2000).optional(), + }) + .refine( + (input) => input.action !== "steer" || Boolean(input.inputId && input.text), + "补充要求缺少内容或请求号", + ); + +export async function GET(request: Request) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); + try { + const input = scope.parse(Object.fromEntries(new URL(request.url).searchParams)); + await assertAgentRunOwnership(session.user.id, input.projectId, input.chatId); + const run = await getAgentRun(session.user.id, input.chatId, input.projectId, input.runId); + return NextResponse.json( + { run: run ? agentRunView(run) : null }, + { headers: { "Cache-Control": "private, no-store" } }, + ); + } catch (error) { + return failure(error); + } +} +export async function POST(request: Request) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); + try { + const input = control.parse(await request.json()); + await assertAgentRunOwnership(session.user.id, input.projectId, input.chatId); + return NextResponse.json({ run: await controlAgentRun({ ...input, userId: session.user.id }) }); + } catch (error) { + return failure(error); + } +} +function failure(error: unknown) { + return NextResponse.json( + { error: publicAgentError(error) }, + { + status: + error instanceof AgentRunError ? error.status : error instanceof z.ZodError ? 400 : 503, + }, + ); +} diff --git a/app/api/canvas/video-agent/route.ts b/app/api/canvas/video-agent/route.ts new file mode 100644 index 000000000..95cc9fa22 --- /dev/null +++ b/app/api/canvas/video-agent/route.ts @@ -0,0 +1,78 @@ +import { NextResponse } from "next/server"; +import { z } from "zod"; +import { auth } from "@/app/(auth)/auth"; +import { getChatById, getMessagesByChatId } from "@/lib/db/queries"; +import { + readOwnedVideoCanvas, + saveVideoAgentCanvas, +} from "@/lib/ai/canvas-agent-runtime/video-store"; +import { AgentRunError } from "@/lib/ai/canvas-agent-runtime/store"; +import { publicAgentError } from "@/lib/ai/canvas-agent-runtime/protocol"; + +export const dynamic = "force-dynamic"; +export async function GET(request: Request) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); + try { + const projectId = z.string().uuid().parse(new URL(request.url).searchParams.get("projectId")); + await readOwnedVideoCanvas(session.user.id, projectId); + // One stable conversation per video workspace survives browser reloads. + const chat = await getChatById({ id: projectId }); + if (chat && chat.userId !== session.user.id) throw new AgentRunError("对话不存在", 404); + const messages = chat ? (await getMessagesByChatId({ id: projectId })).slice(-40) : []; + return NextResponse.json( + { messages: messages.map(({ id, role, parts }) => ({ id, role, parts })) }, + { headers: { "Cache-Control": "private, no-store" } }, + ); + } catch (error) { + return NextResponse.json( + { error: publicAgentError(error) }, + { + status: + error instanceof AgentRunError ? error.status : error instanceof z.ZodError ? 400 : 503, + }, + ); + } +} + +const saveSchema = z.object({ + projectId: z.string().uuid(), + baseRevision: z.string().datetime(), + canvasData: z.object({ + nodes: z.array(z.record(z.unknown())).max(2000), + edges: z + .array( + z.object({ + id: z.string(), + source: z.string(), + target: z.string(), + type: z.string().optional(), + }), + ) + .max(4000), + selectedNodeIds: z.array(z.string()).max(20), + viewport: z.object({ + x: z.number().finite(), + y: z.number().finite(), + zoom: z.number().positive(), + }), + tool: z.literal("select"), + }), +}); +export async function POST(request: Request) { + const session = await auth(); + if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); + try { + const body = saveSchema.parse(await request.json()); + await saveVideoAgentCanvas(session.user.id, body.projectId, body.baseRevision, body.canvasData); + return NextResponse.json({ success: true }); + } catch (error) { + return NextResponse.json( + { error: publicAgentError(error) }, + { + status: + error instanceof AgentRunError ? error.status : error instanceof z.ZodError ? 400 : 503, + }, + ); + } +} diff --git a/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx b/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx index eecb33401..907016c7e 100644 --- a/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx +++ b/app/canvas/[id]/CanvasEditorLegacyPageClient.tsx @@ -87,6 +87,7 @@ import { CanvasAddImageDialog, type CanvasAddImageDialogImage, } from "@/components/canvas/CanvasAddImageDialog"; +import { CanvasAgentUiListener } from "@/components/canvas/chat-panel/canvas-agent-ui-listener"; import { CanvasToolbarIcon } from "@/components/canvas/CanvasToolbarIcon"; import { CanvasSurfaceHeader } from "@/components/canvas/CanvasSurfaceHeader"; import { Badge } from "@/components/ui/badge"; @@ -10268,6 +10269,7 @@ export default function CanvasEditorLegacyPage({ } as React.CSSProperties } > + {/* 1. Canvas Layer (Full Screen) - 使用 key 强制在 id 变化时重新挂载 */}
(null); const [attachments, setAttachments] = useState([]); const [selectedModelId, setSelectedModelId] = useState( effectiveAgentEnabled ? "auto" : "" @@ -7936,8 +7939,9 @@ export function CanvasChatPanel({ autoContextRef.current = null; contextPayloadRef.current = null; pendingCanvasAgentAnnotationEditTargetRef.current = null; - return { - body: { + const runtimeResumeRunId = canvasAgentResumeRunIdRef.current; + canvasAgentResumeRunIdRef.current = null; + const preparedBody = { id: rawId, message: request.messages.at(-1), agentPresetId: @@ -7948,6 +7952,7 @@ export function CanvasChatPanel({ : undefined, selectedChatModel: effectiveSelectedChatModel, composerMode: effectiveComposerMode, + workspaceVariant: isVideoWorkspace ? "video" : "canvas", selectedVisibilityType: "private", deepThinkingEnabled: effectiveComposerMode === "agent" @@ -7965,8 +7970,12 @@ export function CanvasChatPanel({ hiddenPrompt, autoContext, contextPayload, + ...(runtimeResumeRunId ? { resumeRunId: runtimeResumeRunId } : {}), ...request.body, - }, + }; + return { + api: shouldUseCanvasAgentRuntime(preparedBody) ? "/api/canvas/agent-chat" : "/api/chat", + body: preparedBody, }; }, }), @@ -7974,6 +7983,11 @@ export function CanvasChatPanel({ const eventChatId = inFlightChatIdRef.current || activeChatId; const isEventForActiveChat = eventChatId === activeChatIdRef.current; + if (dataPart.type === "data-canvasAgentRun") { + window.dispatchEvent(new CustomEvent("canvas-agent-run:update", { detail: dataPart.data })); + return; + } + if (dataPart.type === "data-imageGenerationStatus") { try { const payload: ImageGenerationStatusStreamPayload = @@ -8023,6 +8037,10 @@ export function CanvasChatPanel({ return; } + if (dataPart.type === "data-canvasAgentUiCommand") { + window.dispatchEvent(new CustomEvent("canvas-agent-ui-command", { detail: dataPart.data })); + return; + } if (dataPart.type === "data-canvasCapabilityCommand") { try { const rawPayload: CanvasCapabilityCommandStreamPayload = @@ -17872,6 +17890,18 @@ export function CanvasChatPanel({ )}
)} + { + if (isSubmitting) return; + canvasAgentResumeRunIdRef.current = runId; + void handleSend(input.trim() || "请继续这个任务的未完成步骤，保留已生成的结果，优先处理未通过检查的图片。"); + }} + onResults={(projectId, nodeIds) => { void refreshCanvasAfterAgentOperation({ projectId, createdNodeIds: nodeIds }); }} + /> {/* 混合输入区域：使用 contenteditable 实现真正的混合编辑 */}
([]); + const [isVideoAgentOpen, setIsVideoAgentOpen] = useState(false); + const receivedAgentNodeIds = useRef(new Set()); const [edges, setEdges] = useState([]); const [selectedNodeId, setSelectedNodeId] = useState(null); const [selectedEdgeId, setSelectedEdgeId] = useState(null); @@ -5043,7 +5049,9 @@ export function LiblibVideoCanvasReplica({ const title = normalizeCanvasTitle(project.title); setProjectTitle(title); setDraftTitle(title); - setNodes(restoreNodes(project.canvasData)); + const restored = restoreNodes(project.canvasData); + receivedAgentNodeIds.current = new Set(restored.map((node) => node.id)); + setNodes(restored); setEdges(restoreEdges(project.canvasData)); }; @@ -5111,6 +5119,66 @@ export function LiblibVideoCanvasReplica({ setIsSaving(false); }, [canvasId, edges, nodes, selectedNodeId, viewport]); + const refreshVideoAgentResults = useCallback(async (projectId: string, resultNodeIds: string[]) => { + if (projectId !== canvasId) return; + const result = await canvasService.getProject(canvasId); + if (!result.success || !result.data) return; + const incoming = restoreNodes(result.data.canvasData).filter((node) => resultNodeIds.includes(node.id)); + const unseen = incoming.filter((node) => !receivedAgentNodeIds.current.has(node.id)); + for (const node of incoming) receivedAgentNodeIds.current.add(node.id); + setNodes((current) => mergeVideoAgentResults(current.map((node) => { + const remote = incoming.find((item) => item.id === node.id); + return remote && remote.draftValues?.temporaryVideoUrl === node.draftValues?.mediaUrl && remote.draftValues?.mediaUrl !== node.draftValues?.mediaUrl ? { ...node, draftValues: { ...node.draftValues, mediaUrl: remote.draftValues?.mediaUrl, url: remote.draftValues?.url } } : node; + }), unseen)); + setEdges((current) => mergeVideoAgentResults(current, restoreEdges(result.data!.canvasData).filter((edge) => unseen.some((node) => node.id === edge.target)))); + }, [canvasId]); + + useEffect(() => { + const handler = async (event: Event) => { + const change = (event as CustomEvent<{ projectId: string; createdNodeIds?: string[]; updatedNodeIds?: string[]; deletedNodeIds?: string[]; action?: string; nodeIds?: string[]; viewportChanged?: boolean }>).detail; + if (change?.projectId !== canvasId) return; + const response = await canvasService.getProject(canvasId); + if (!response.success || !response.data) return; + const remote = restoreNodes(response.data.canvasData); + const changed = new Set([...(change.createdNodeIds || []), ...(change.updatedNodeIds || [])]); + const deleted = new Set(change.deletedNodeIds || []); + setNodes((current) => mergeVideoAgentResults(current.filter((node) => !deleted.has(node.id)).map((node) => changed.has(node.id) ? remote.find((item) => item.id === node.id) || node : node), remote.filter((node) => changed.has(node.id)))); + setEdges((current) => { + const kept = current.filter((edge) => !deleted.has(edge.source) && !deleted.has(edge.target) && !(change.action === "disconnect" && edge.source === change.nodeIds?.[0] && edge.target === change.nodeIds?.[1])); + return change.action === "connect" ? mergeVideoAgentResults(kept, restoreEdges(response.data!.canvasData).filter((edge) => edge.source === change.nodeIds?.[0] && edge.target === change.nodeIds?.[1])) : kept; + }); + if (change.viewportChanged && response.data.canvasData?.viewport) setViewport(response.data.canvasData.viewport); + }; + window.addEventListener("video-canvas-agent-mutation", handler); + return () => window.removeEventListener("video-canvas-agent-mutation", handler); + }, [canvasId]); + + const saveBeforeVideoAgent = useCallback(async () => { + const statusResponse = await fetch(`/api/canvas/agent-runs?${new URLSearchParams({ chatId: canvasId, projectId: canvasId })}`, { cache: "no-store" }); + if (statusResponse.ok && (await statusResponse.json()).run?.status === "running") throw new Error("当前视频任务还在执行，请使用补充要求。"); + const remote = await canvasService.getProject(canvasId); + if (!remote.success || !remote.data) throw new Error("无法读取视频画布，请稍后重试"); + const remoteNodes = restoreNodes(remote.data.canvasData); + const unseen = remoteNodes.filter((node) => node.draftValues?.canvasAgentRunId && !receivedAgentNodeIds.current.has(node.id)); + const syncedNodes = nodes.map((node) => { + const archived = remoteNodes.find((item) => item.id === node.id && item.draftValues?.temporaryVideoUrl === node.draftValues?.mediaUrl); + return archived ? { ...node, draftValues: { ...node.draftValues, mediaUrl: archived.draftValues?.mediaUrl, url: archived.draftValues?.url, temporaryVideoUrl: archived.draftValues?.temporaryVideoUrl } } : node; + }); + const mergedNodes = mergeVideoAgentResults(syncedNodes, unseen); + const mergedEdges = mergeVideoAgentResults(edges, restoreEdges(remote.data.canvasData).filter((edge) => unseen.some((node) => node.id === edge.target))); + const response = await fetch("/api/canvas/video-agent", { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ projectId: canvasId, baseRevision: remote.data.updatedAt, canvasData: { + nodes: serializeNodes(mergedNodes), edges: serializeEdges(mergedEdges), + selectedNodeIds: selectedNodeId ? [selectedNodeId] : [], viewport, tool: "select", + } }), + }); + const result = await response.json(); + if (!response.ok) throw new Error(result.error || "画布保存失败，请稍后重试"); + if (result.success) { setNodes(mergedNodes); setEdges(mergedEdges); for (const node of unseen) receivedAgentNodeIds.current.add(node.id); } + if (!result.success) throw new Error(result.error || "画布保存失败，请稍后重试"); + }, [canvasId, nodes, edges, selectedNodeId, viewport]); + const commitTitle = useCallback(async () => { const nextTitle = draftTitle.trim() || "未命名"; setProjectTitle(nextTitle); @@ -10857,6 +10925,50 @@ export function LiblibVideoCanvasReplica({ }; }, [lineConnectState, nodes]); + const handleAgentVideoUiCommand = async (command: CanvasAgentUiCommand) => { + const target = nodes.find((node) => node.id === (command.nodeIds[0] || selectedNodeId)); + if (command.nodeIds.some((id) => !nodes.some((node) => node.id === id))) throw new Error("目标节点不属于当前视频画布"); + if (target) { setSelectedNodeId(target.id); setSelectedEdgeId(null); } + if (command.featureId.startsWith("video.function.")) { + const key = command.featureId.slice("video.function.".length); + const tool = configuredTools.find((item) => item.toolKey === key); + if (!tool) throw new Error("此视频功能已关闭或尚未配置"); + if ((key === "grid-split" || key === "image-crop") && target) { await runSelectedImageToolAction(target.id, "crop"); return; } + if (key === "angle-edit" && target) { setAngleEditorNodeId(target.id); return; } + const created = createToolNode(tool, target ? target.x + target.width + 72 : 360, target?.y || 300); + addNodes([created], command.nodeIds.map((id) => ({ id: makeId("edge"), source: id, target: created.id }))); + setInspectorNodeId(created.id); + return; + } + const action = command.featureId.replace("video.ui.", ""); + const panels: Record> = { assets: "assets", roles: "roleLibrary", styles: "styleLibrary", effects: "effectLibrary", history: "history", workflows: "workflow", share: "share", publish: "publish", skills: "skills", help: "help" }; + if (panels[action]) { setActivePanel(panels[action]); return; } + if (action.startsWith("upload-")) { addTargetModule(action === "upload-video" ? "video" : action === "upload-audio" ? "audio" : "image"); toast.message("已添加上传模块，请选择本机文件"); return; } + if (action === "save") { await saveCanvas(); return; } + if (action === "export") { + const url = URL.createObjectURL(new Blob([JSON.stringify({ title: projectTitle, workspaceType: "video", canvasData: { nodes: serializeNodes(nodes), edges: serializeEdges(edges), viewport } }, null, 2)], { type: "application/json" })); + const link = document.createElement("a"); link.href = url; link.download = `${projectTitle}.json`; link.click(); setTimeout(() => URL.revokeObjectURL(url), 1000); return; + } + if (action === "import") { + const picker = document.createElement("input"); picker.type = "file"; picker.accept = ".json,application/json"; + picker.onchange = async () => { try { const file = picker.files?.[0]; if (!file) return; const data = JSON.parse(await file.text()); const imported = restoreNodes(data.canvasData || data); if (!imported.length) throw new Error("文件中没有可用的视频画布节点"); if (!window.confirm("确认将文件里的节点添加到当前画布？")) return; const ids = new Map(imported.map((node) => [node.id, makeId("import")])); addNodes(imported.map((node) => ({ ...node, id: ids.get(node.id)! })), restoreEdges(data.canvasData || data).filter((edge) => ids.has(edge.source) && ids.has(edge.target)).map((edge) => ({ id: makeId("edge"), source: ids.get(edge.source)!, target: ids.get(edge.target)! }))); } catch (error) { toast.error(error instanceof Error ? error.message : "导入失败"); } }; picker.click(); return; + } + if (!target) throw new Error("请先选中具体素材节点"); + if (action === "download") { downloadMediaNode(target); return; } + if (action === "preview") { setPreviewNode(target); return; } + if (action === "local-edit") { await runSelectedImageToolAction(target.id, "localEdit"); return; } + if (action === "annotate") { await runSelectedImageToolAction(target.id, "annotate"); return; } + if (action === "director") { setDirectorConsoleNodeId(target.id); return; } + if (action === "storyboard") { openStoryScriptWorkflow(target.id); return; } + if (action === "batch-storyboard" || action === "batch-video") { openStoryScriptWorkflow(target.id); toast.message("请检查分镜及费用后在工作台提交批量任务"); return; } + if (action === "video-hd") { setInspectorNodeId(target.id); toast.message("当前视频 AI 高清尚缺真实后端，不能将原视频冒充高清结果"); return; } + if (action === "video-clip") { setActiveVideoToolbarMode("clip"); return; } + if (action === "video-crop") { setActiveVideoToolbarMode("crop"); toast.message("当前裁剪导出后端尚未接入，不能当作已完成的视频裁剪"); return; } + if (action === "subtitle-erase") { setActiveVideoToolbarMode("subtitleErase"); toast.message("当前去字幕尚未接入真实处理后端"); return; } + if (["audio-split", "voice-split", "background-split", "frame-first", "frame-last", "frame-current"].includes(action)) { setPreviewNode(target); toast.message("已授权入口；当前页面此功能仍为模拟流程，Agent 不会生成占位素材或伪报完成"); return; } + throw new Error("此入口需要在相应的视频工作区打开"); + }; + return (
+
{lineConnectState ? ( @@ -11443,6 +11556,29 @@ export function LiblibVideoCanvasReplica({ /> ) : null} + {isVideoAgentOpen ? ( + isVideoModelDescriptor(model))} + imageModels={canvasModelOptions.filter((model) => isImageGenerationModelDescriptor(model))} + beforeSend={saveBeforeVideoAgent} + onResults={(projectId, ids) => void refreshVideoAgentResults(projectId, ids)} + onClose={() => setIsVideoAgentOpen(false)} + /> + ) : ( + + )} + void; + onResults: (projectId: string, nodeIds: string[]) => void; + onRunChange?: (run: AgentRunView | null) => void; +}) { + const [text, setText] = useState(""); + const [sending, setSending] = useState(false); + const [stopping, setStopping] = useState(false); + const activeScope = useRef(""); + activeScope.current = `${props.chatId}:${props.projectId}`; + const resultSignature = useRef(""); + const key = props.enabled + ? `/api/canvas/agent-runs?${new URLSearchParams({ chatId: props.chatId, projectId: props.projectId })}` + : null; + const { data, mutate } = useSWR<{ run: AgentRunView | null }>( + key, + async (url: string) => { + const response = await fetch(url, { cache: "no-store" }); + if (!response.ok) return { run: null }; + return response.json(); + }, + { + refreshInterval: (latest) => (latest?.run?.status === "running" || props.busy ? 2000 : 0), + revalidateOnFocus: true, + shouldRetryOnError: false, + }, + ); + const run = data?.run; + useEffect(() => { + if (data) props.onRunChange?.(data.run); + }, [data, props.onRunChange]); + useEffect(() => { + setText(""); + setStopping(false); + resultSignature.current = ""; + }, [props.chatId, props.projectId]); + useEffect(() => { + if (!props.enabled) return; + const handler = (event: Event) => { + const update = (event as CustomEvent).detail; + if (update?.chatId === props.chatId && update?.projectId === props.projectId) + void mutate({ run: update }, false); + }; + window.addEventListener("canvas-agent-run:update", handler); + return () => window.removeEventListener("canvas-agent-run:update", handler); + }, [props.enabled, props.chatId, props.projectId, mutate]); + useEffect(() => { + if (!run?.resultNodeIds.length) return; + const signature = `${run.id}:${run.resultNodeIds.join(",")}:${run.status}`; + if (signature === resultSignature.current) return; + resultSignature.current = signature; + props.onResults(run.projectId, run.resultNodeIds); + }, [run, props.onResults]); + const control = useCallback( + async (action: "steer" | "cancel") => { + if (!run || sending) return; + const scope = activeScope.current; + setSending(true); + try { + const response = await fetch("/api/canvas/agent-runs", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + action, + chatId: props.chatId, + projectId: props.projectId, + runId: run.id, + ...(action === "steer" ? { inputId: crypto.randomUUID(), text: text.trim() } : {}), + }), + }); + const result = await response.json(); + if (!response.ok) throw new Error(result.error || "操作失败，请重试"); + if (scope !== activeScope.current) return; + await mutate(result, false); + if (action === "steer") { + setText(""); + toast.message("补充要求已收到，将在下一步应用"); + } else { + setStopping(true); + toast.message("将停止后续步骤，已发出的生成请求会先收好结果"); + } + } catch (error) { + toast.error(error instanceof Error ? error.message : "操作失败"); + } finally { + if (scope === activeScope.current) setSending(false); + } + }, + [run, sending, props.chatId, props.projectId, text, mutate], + ); + if (!props.enabled || !run || run.status === "completed") return null; + const running = run.status === "running"; + const stopPending = stopping || run.cancelRequested; + return ( +
+
+ {running ? : null} + + {stopPending && running ? "正在停止后续步骤，等待当前生成结果" : run.message} + + {running ? ( + + ) : run.resumable ? ( + + ) : null} +
+
+ {run.runtime?.engine === "zcode" ? ZCode · {run.runtime.state === "starting" ? "正在启动独立沙箱" : "独立沙箱"} : null} + 已处理 {run.steps} 轮 · {run.toolCount} 个工具步骤 +
+ {run.inputs.slice(-2).map((input) => ( +
+ {input.applied ? "已应用" : "待应用"}：{input.text} +
+ ))} + {running && !stopPending ? ( +
{ + event.preventDefault(); + if (text.trim()) void control("steer"); + }} + > + setText(event.target.value)} + maxLength={2000} + disabled={sending} + placeholder="补充要求，将从下一步生效" + aria-label="补充任务要求" + className="min-w-0 flex-1 rounded-xl border border-zinc-200 bg-white px-3 py-2 outline-none focus:border-zinc-400 dark:border-zinc-700 dark:bg-zinc-950" + /> + +
+ ) : null} +
+ ); +} diff --git a/components/canvas/chat-panel/canvas-agent-ui-listener.tsx b/components/canvas/chat-panel/canvas-agent-ui-listener.tsx new file mode 100644 index 000000000..eaac63ef2 --- /dev/null +++ b/components/canvas/chat-panel/canvas-agent-ui-listener.tsx @@ -0,0 +1,78 @@ +"use client"; +import { useEffect, useRef } from "react"; +import { toast } from "sonner"; +import { getCanvasAgentAccessManifest } from "@/lib/ai/canvas-agent-runtime/capability-access"; +import { + canvasAgentUiCommandSchema, + CANVAS_AGENT_UI_COMMAND_EVENT, + type CanvasAgentUiCommand, +} from "@/lib/ai/canvas-agent-runtime/ui-command"; +import { useCanvasStore } from "@/store/canvas"; + +export function CanvasAgentUiListener(props: { + projectId: string; + workspace: "canvas" | "video"; + onVideoAction?: (command: CanvasAgentUiCommand) => void | Promise; +}) { + const latest = useRef(props); + latest.current = props; + const seen = useRef(new Set()); + useEffect(() => { + const handler = async (event: Event) => { + const parsed = canvasAgentUiCommandSchema.safeParse((event as CustomEvent).detail); + if (!parsed.success) return; + const command = parsed.data, + current = latest.current; + if ( + command.projectId !== current.projectId || + command.workspace !== current.workspace || + seen.current.has(command.requestId) + ) + return; + seen.current.add(command.requestId); + try { + if (current.workspace === "video") { + await current.onVideoAction?.(command); + return; + } + const feature = getCanvasAgentAccessManifest().find( + (item) => item.id === command.featureId && item.workspace === "canvas", + ); + if (!feature || !feature.id.startsWith("canvas.ui.")) throw new Error("这个界面入口不存在"); + const key = feature.id.slice("canvas.ui.".length); + if (/chatSend|chatStartGeneration/.test(key)) { + toast.message("已定位对话提交入口，请由你确认并发送；Agent 不会伪造下一条用户消息"); + return; + } + const state = useCanvasStore.getState(); + if (state.projectId !== current.projectId) return; + if (command.nodeIds.length) + state.selectNodes( + command.nodeIds.filter((id) => state.nodes.some((node) => node.id === id)), + ); + await new Promise((resolve) => + requestAnimationFrame(() => requestAnimationFrame(() => resolve())), + ); + const element = Array.from( + document.querySelectorAll("[data-canvas-feature-key]"), + ).find((item) => item.dataset.canvasFeatureKey === key && item.getClientRects().length); + const button = element?.closest("button,[role=button],a"); + if (!button || button.hasAttribute("disabled")) + throw new Error( + "此入口当前未显示或不可用；请切换相应工作区。工具授权不会恢复被隐藏的按钮。", + ); + if ( + /Delete|Generate|Confirm|Cancel|SaveText|realtimeSend/.test(key) && + !window.confirm(`Agent 请求操作「${feature.title}」。确认在当前画布执行？`) + ) + return; + button.click(); + } catch (error) { + toast.error(error instanceof Error ? error.message : "无法打开画布入口"); + } + }; + window.addEventListener(CANVAS_AGENT_UI_COMMAND_EVENT, handler); + return () => window.removeEventListener(CANVAS_AGENT_UI_COMMAND_EVENT, handler); + }, []); + return null; +} diff --git a/components/canvas/chat-panel/video-canvas-agent-panel.tsx b/components/canvas/chat-panel/video-canvas-agent-panel.tsx new file mode 100644 index 000000000..36e3049b9 --- /dev/null +++ b/components/canvas/chat-panel/video-canvas-agent-panel.tsx @@ -0,0 +1,324 @@ +"use client"; + +import { useChat } from "@ai-sdk/react"; +import { DefaultChatTransport, type UIMessage } from "ai"; +import { ArrowUp, MessageSquare, X } from "lucide-react"; +import { useCallback, useEffect, useRef, useState } from "react"; +import { toast } from "sonner"; +import { CanvasAgentRunPanel } from "./canvas-agent-run-panel"; +import { + videoAgentSettingsSchema, + type VideoAgentSettings, +} from "@/lib/ai/canvas-agent-runtime/video-protocol"; + +type ModelOption = { id: string; displayName: string }; +export function VideoCanvasAgentPanel(props: { + projectId: string; + selectedNodeIds: string[]; + videoModels: ModelOption[]; + imageModels: ModelOption[]; + beforeSend: () => Promise; + onResults: (projectId: string, nodeIds: string[]) => void; + onClose: () => void; +}) { + const [input, setInput] = useState(""); + const [settings, setSettings] = useState(() => + videoAgentSettingsSchema.parse({}), + ); + const [preparing, setPreparing] = useState(false); + const [loading, setLoading] = useState(true); + const [backgroundRunning, setBackgroundRunning] = useState(false); + const latest = useRef({ props, settings }); + latest.current = { props, settings }; + const resumeRef = useRef(undefined); + const mounted = useRef(true); + const resultIds = useRef([]); + const restoredRunId = useRef(""); + const { messages, setMessages, sendMessage, status } = useChat({ + id: `video-agent-${props.projectId}`, + generateId: () => crypto.randomUUID(), + transport: new DefaultChatTransport({ + api: "/api/canvas/agent-chat", + prepareSendMessagesRequest: ({ messages }) => { + const current = latest.current; + const resumeRunId = resumeRef.current; + resumeRef.current = undefined; + return { + body: { + id: current.props.projectId, + message: messages[messages.length - 1], + selectedChatModel: "agent", + selectedVisibilityType: "private", + composerMode: "agent", + workspaceVariant: "video", + videoAgentSettings: current.settings, + resumeRunId, + contextPayload: { + source: "canvas", + projectId: current.props.projectId, + selectedNodeIds: current.props.selectedNodeIds, + }, + }, + }; + }, + }), + onData: (part) => { + if (part.type === "data-videoCanvasMutation") window.dispatchEvent(new CustomEvent("video-canvas-agent-mutation", { detail: part.data })); + if (part.type === "data-canvasAgentUiCommand") window.dispatchEvent(new CustomEvent("canvas-agent-ui-command", { detail: part.data })); + if (part.type === "data-canvasAgentRun") + window.dispatchEvent(new CustomEvent("canvas-agent-run:update", { detail: part.data })); + }, + onError: (error) => toast.error(error.message || "视频任务启动失败，请重试"), + }); + useEffect(() => { + mounted.current = true; + const controller = new AbortController(); + void fetch(`/api/canvas/video-agent?projectId=${props.projectId}`, { + cache: "no-store", + signal: controller.signal, + }) + .then(async (response) => { + const data = await response.json(); + if (!response.ok) throw new Error(data.error || "读取视频对话失败"); + if (!controller.signal.aborted) setMessages(data.messages as UIMessage[]); + }) + .catch((error) => { + if (!controller.signal.aborted) toast.error(error.message); + }) + .finally(() => { + if (!controller.signal.aborted) setLoading(false); + }); + return () => { + mounted.current = false; + controller.abort(); + }; + }, [props.projectId, setMessages]); + const busy = backgroundRunning || preparing || status === "submitted" || status === "streaming"; + const submit = useCallback( + async (resumeRunId?: string) => { + if (busy || loading) return; + const text = + input.trim() || + (resumeRunId ? "请继续尚未完成的视频创作步骤，保留已完成的脚本、关键帧和视频。" : ""); + if (!text) return; + setPreparing(true); + try { + await latest.current.props.beforeSend(); + if (!mounted.current) return; + resumeRef.current = resumeRunId; + setInput(""); + await sendMessage({ text }); + } catch (error) { + toast.error(error instanceof Error ? error.message : "视频任务启动失败"); + } finally { + resumeRef.current = undefined; + if (mounted.current) setPreparing(false); + } + }, + [busy, loading, input, sendMessage], + ); + const onRunChange = useCallback( + (run: import("@/lib/ai/canvas-agent-runtime/protocol").AgentRunView | null) => { + resultIds.current = run?.resultNodeIds || []; + if (run?.videoSettings && restoredRunId.current !== run.id) { + restoredRunId.current = run.id; + setSettings(run.videoSettings); + } + setBackgroundRunning(run?.status === "running"); + }, + [], + ); + const onResults = useCallback( + (projectId: string, ids: string[]) => latest.current.props.onResults(projectId, ids), + [], + ); + useEffect(() => { + const timer = setInterval(() => { + if (resultIds.current.length) + latest.current.props.onResults(latest.current.props.projectId, resultIds.current); + }, 10_000); + return () => clearInterval(timer); + }, []); + return ( +