import { agentModelCandidates } from "@/lib/ai/runtime/agent-model-failover"; import { agentModelCircuitStore } from "@/lib/ai/runtime/agent-model-circuit-store"; import { createAgentVisionExecutor } from "@/lib/ai/runtime/agent-vision-failover"; import { canvasAgentProgress } from "@/lib/ai/canvas-agent-runtime/progress"; import { getStyleProImageModelEntries } from "@/lib/style-pro/model-options"; import { registerProjectAgentTools } from "@/lib/ai/canvas-agent-runtime/project-tools"; import { isZcodeWorkspaceAuthorized } from "@/lib/ai/canvas-agent-runtime/authorization"; import { createUIMessageStream, createUIMessageStreamResponse, generateText, } from "ai"; import { after, NextResponse } from "next/server"; import { z } from "zod"; import { db } from "@/lib/db"; import { siteSettings, canvasProject } from "@/lib/db/schema"; import { eq } from "drizzle-orm"; import { auth } from "@/app/(auth)/auth"; import { postRequestBodySchema } from "@/app/(chat)/api/chat/schema"; import { entitlementsByUserType } from "@/lib/ai/entitlements"; import { getLanguageModelById } from "@/lib/ai/dynamic-provider"; import { getAgentAiModelWithProvider, getAccessibleAgentPresetById, getEnabledAiModelsWithProvider, getChatById, getMessagesByChatId, getMessageCountByUserId, getSiteSettings, saveChat, saveMessages, } from "@/lib/db/queries"; import { runZcodeCanvasTask } from "@/lib/ai/zcode-agent/task"; import { createZcodeModelGateway } from "@/lib/ai/zcode-agent/model-gateway"; import { newAgentRunSnapshot, publicAgentError } from "@/lib/ai/canvas-agent-runtime/protocol"; import { shouldUseCanvasAgentRuntime } from "@/lib/ai/canvas-agent-runtime/routing"; import { acquireAgentRun, agentRunView, AgentRunError, checkpointAgentRun, consumeAgentRunInputs, heartbeatAgentRun, } from "@/lib/ai/canvas-agent-runtime/store"; import { readOwnedCanvas } from "@/lib/ai/canvas-agent-runtime/canvas"; import { buildCanvasRuntimeTools } from "@/lib/ai/canvas-agent-runtime/tools"; import { canvasRuntimeSystemPrompt } from "@/lib/ai/canvas-agent-runtime/prompt"; import { verifyCanvasAgentResults } from "@/lib/ai/canvas-agent-runtime/review"; import { inferCanvasTaskRequirements } from "@/lib/ai/canvas-agent-runtime/requirements"; import { resolveRuntimeConversationSources, persistRuntimeConversationResults, } from "@/lib/ai/canvas-agent-runtime/conversation"; import { getCanvasAgentFeatureFlagsFromSystemSettings } from "@/lib/canvas/agent-feature-settings"; import { canApplyCanvasAgentMutations } from "@/lib/canvas/agent-feature-flags"; import { videoAgentSettingsSchema, inferVideoTaskRequirements, videoNodeUrl, videoRuntimeSystemPrompt, } from "@/lib/ai/canvas-agent-runtime/video-protocol"; import { readOwnedVideoCanvas } from "@/lib/ai/canvas-agent-runtime/video-store"; import { buildVideoRuntimeTools, createVideoRuntimePorts, } from "@/lib/ai/canvas-agent-runtime/video-tools"; import { probeVideoMetadata } from "@/lib/ai/canvas-agent-runtime/video-probe"; import { verifyVideoAgentResults } from "@/lib/ai/canvas-agent-runtime/video-review"; export const runtime = "nodejs"; export const maxDuration = 1200; const requestSchema = postRequestBodySchema.extend({ resumeRunId: z.string().uuid().optional(), selectedImageModel: z.string().min(1).max(180).optional(), workspaceVariant: z.enum(["canvas", "video"]).default("canvas"), videoAgentSettings: videoAgentSettingsSchema.optional(), }); export async function POST(request: Request) { const session = await auth(); if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 }); try { const body = requestSchema.parse(await request.json()); if (!shouldUseCanvasAgentRuntime(body)) throw new AgentRunError("此请求不属于画布 Agent 任务", 400); const projectId = body.contextPayload!.projectId!; const [ownedProject] = await db.select({ userId: canvasProject.userId, workspaceType: canvasProject.workspaceType }) .from(canvasProject).where(eq(canvasProject.id, projectId)).limit(1); if (!ownedProject || ownedProject.userId !== session.user.id) throw new AgentRunError("画布不存在", 404); if (!isZcodeWorkspaceAuthorized(ownedProject.workspaceType)) throw new AgentRunError("视频画布暂未授权给 ZCode Agent,请使用手工功能。", 403); const isVideo = ownedProject.workspaceType === "video"; const preset = body.agentPresetId ? await getAccessibleAgentPresetById({ id: body.agentPresetId, userId: session.user.id }) : null; if (preset && String(preset.type) === "video") throw new AgentRunError("视频技能暂未授权给 ZCode Agent", 403); if (body.agentPresetId && (!preset || preset.status !== "active")) throw new AgentRunError("此 Agent 预设不可用", 403); const objective = body.message.parts .filter((part) => part.type === "text") .map((part) => part.text) .join("\n") .trim(); if (!objective) throw new AgentRunError("请描述需要完成的创作任务", 400); const settings = await getSiteSettings(); if ( !settings.agentEnabled || !canApplyCanvasAgentMutations(await getCanvasAgentFeatureFlagsFromSystemSettings()) ) { throw new AgentRunError("管理员尚未开启画布 Agent 操作", 403); } const [usageSettings] = await db.select().from(siteSettings).limit(1); if (usageSettings?.usageTimeEnabled) { const time = new Intl.DateTimeFormat("en-GB", { timeZone: "Asia/Shanghai", hour: "2-digit", minute: "2-digit", hourCycle: "h23", }).format(new Date()); if ( time < (usageSettings.usageTimeStart || "08:00") || time > (usageSettings.usageTimeEnd || "20:00") ) throw new AgentRunError(usageSettings.usageTimeMessage || "当前不在系统使用时间段内", 403); } const count = await getMessageCountByUserId({ id: session.user.id, differenceInHours: 24 }); if (count > entitlementsByUserType[session.user.type].maxMessagesPerDay) throw new AgentRunError("今日对话次数已达到限额", 429); const imageCanvas = isVideo ? null : await readOwnedCanvas(session.user.id, projectId); if (imageCanvas?.project.workspaceType === "video") throw new AgentRunError("请使用视频画布 Agent 入口", 400); const videoCanvas = isVideo ? await readOwnedVideoCanvas(session.user.id, projectId) : null; const nodes = imageCanvas?.nodes || []; const availableIds = isVideo ? videoCanvas!.nodes.map((node) => node.id) : nodes.map((node) => node.id); const selectedNodeIds = body.contextPayload?.selectedNodeIds || []; if (selectedNodeIds.some((id) => !availableIds.includes(id))) { throw new AgentRunError("选中图片尚未保存到画布,请等待保存完成后重试。"); } const existingChat = await getChatById({ id: body.id }); if (existingChat && existingChat.userId !== session.user.id) throw new AgentRunError("对话不存在", 404); if (!existingChat) await saveChat({ id: body.id, userId: session.user.id, title: objective.slice(0, 80), visibility: "private", }); const conversation = isVideo ? { sourceNodeIds: selectedNodeIds, targetReason: "video-selected-nodes", previousContext: null, } : await resolveRuntimeConversationSources({ userId: session.user.id, chatId: body.id, projectId, text: objective, parts: body.message.parts, nodes, selectedNodeIds, }); const candidates = agentModelCandidates(await getAgentAiModelWithProvider(), await getEnabledAiModelsWithProvider()); const configured = candidates[0]; if (!configured) throw new AgentRunError("请先在后台配置可用的多模态 Agent 对话模型", 503); let model = await getLanguageModelById(configured.model.id); for (const candidate of candidates.slice(1)) { if (model) break; model = await getLanguageModelById(candidate.model.id); } if (!model) throw new AgentRunError("Agent 模型暂不可用", 503); const visionModel = model; const executeVisionModel = createAgentVisionExecutor(candidates, agentModelCircuitStore, getLanguageModelById); // Validate the user's image choice independently of the Agent planning model. if (body.selectedImageModel) { const images = await getStyleProImageModelEntries(); if (!images.some(({ model }) => model.id === body.selectedImageModel && model.healthStatus !== "auto_disabled")) throw new AgentRunError("你选择的生图模型当前不可用,请重新选择;不会自动换模型生成。", 400); } const assistantMessageId = crypto.randomUUID(); const previousVideoMessages = existingChat ? (await getMessagesByChatId({ id: body.id })).slice(-6).map((message) => ({ role: message.role as "user" | "assistant", content: (message.parts as Array<{ type: string; text?: string }>) .filter((part) => part.type === "text") .map((part) => part.text || "") .join("\n") .slice(0, 4000), })) : []; const initialSnapshot = newAgentRunSnapshot(objective, [ ...previousVideoMessages, { role: "user", content: objective }, ]); initialSnapshot.workspace = isVideo ? "video" : "canvas"; if (isVideo) { initialSnapshot.videoRequirements = inferVideoTaskRequirements(objective); initialSnapshot.videoSettings = videoAgentSettingsSchema.parse({ ...body.videoAgentSettings, ...(initialSnapshot.videoRequirements.duration ? { duration: initialSnapshot.videoRequirements.duration } : {}), ...(initialSnapshot.videoRequirements.ratio ? { ratio: initialSnapshot.videoRequirements.ratio } : {}), }); } initialSnapshot.sourceNodeIds = conversation.sourceNodeIds; initialSnapshot.requirements = { ...inferCanvasTaskRequirements(objective, initialSnapshot.sourceNodeIds.length), sourceCount: initialSnapshot.sourceNodeIds.length, }; const run = await acquireAgentRun({ id: body.message.id, userId: session.user.id, projectId, chatId: body.id, assistantMessageId, resumeRunId: body.resumeRunId, snapshot: initialSnapshot, }); const snapshot = run.snapshot; snapshot.imagePreferences = { ...snapshot.imagePreferences, // Each submitted turn captures the current controls, including switching back to Auto. modelId: body.selectedImageModel, parameters: body.imageParams || snapshot.imagePreferences?.parameters, }; const taskSourceNodeIds = snapshot.sourceNodeIds ?? selectedNodeIds; const context = { recentConversation: previousVideoMessages, preset: preset ? { name: preset.name, type: preset.type, promptTemplate: preset.promptTemplate, defaultModelId: preset.defaultModelId, defaultParams: preset.defaultParams, slots: preset.slots.map(slot => ({ title: slot.title, promptTemplate: slot.promptTemplate, defaultParams: slot.defaultParams })) } : null, videoSettings: snapshot.videoSettings, selectedNodeIds: taskSourceNodeIds, nodes: nodes .filter((node) => taskSourceNodeIds.includes(node.id)) .map((node) => ({ id: node.id, type: node.type, position: node.position, size: node.size, title: node.data.title, })), skillInstructions: body.hiddenPrompt || undefined, imagePreferences: snapshot.imagePreferences, targetReason: conversation.targetReason, previousConversationContext: conversation.previousContext, }; await saveMessages({ messages: [ { id: body.message.id, chatId: body.id, role: "user", parts: body.message.parts, attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); let archiveVideoResults: (() => Promise) | undefined; const stream = createUIMessageStream({ execute: async ({ writer }) => { const emit = (type: string, data: unknown) => writer.write({ type, data, ...(type === "data-canvasAgentRun" ? { transient: true } : {}) } as never); let cancelled = false; let leaseLost = false; const activeModelController = new AbortController(); const withModel: typeof executeVisionModel = (execute, signal) => executeVisionModel(execute, signal ? AbortSignal.any([signal, activeModelController.signal]) : activeModelController.signal); const heartbeat = setInterval(() => { void heartbeatAgentRun(run) .then((current) => { if (!current) leaseLost = true; if (current?.cancelRequested) cancelled = true; if (cancelled || leaseLost) activeModelController?.abort(); }) .catch(() => { leaseLost = true; activeModelController?.abort(); }); emit("data-keepalive", { label: "canvas-agent-runtime", ts: Date.now() }); }, 10_000); const textId = crypto.randomUUID(); writer.write({ type: "start", messageId: assistantMessageId }); writer.write({ type: "text-start", id: textId }); writer.write({ type: "text-delta", id: textId, delta: isVideo ? "正在按步骤处理你的视频任务。\n\n" : "正在按步骤处理你的图片任务。\n\n", }); try { const videoPorts = isVideo ? await createVideoRuntimePorts({ userId: session.user.id, chatId: body.id, projectId, runId: run.id, assistantMessageId, request, snapshot, emit, }) : null; archiveVideoResults = videoPorts?.archiveResults; const tools = isVideo ? buildVideoRuntimeTools({ runId: run.id, projectId, selectedNodeIds, snapshot, ports: videoPorts!, }) : await buildCanvasRuntimeTools({ userId: session.user.id, chatId: body.id, projectId, runId: run.id, assistantMessageId, selectedNodeIds: taskSourceNodeIds, request, model: visionModel || model, supportsVision: Boolean(visionModel), withModel, imageSize: snapshot.imagePreferences?.parameters?.imageSize, imagePreferences: snapshot.imagePreferences, preferredModelId: snapshot.imagePreferences?.modelId || (preset?.type === "image" ? preset.defaultModelId || undefined : undefined), operationSelectedNodeIds: selectedNodeIds, objective: [snapshot.objective, ...snapshot.inputs.map((input) => input.text)].join( "\n", ), emit, }); await registerProjectAgentTools({ session, projectId, request, objective: snapshot.objective, definitions: tools.definitions, executable: tools.executable, emit, imagePreferences: snapshot.imagePreferences }); const gateway = await createZcodeModelGateway(configured, undefined, candidates); const status = await runZcodeCanvasTask({ runId: run.id, snapshot, definitions: tools.definitions, instructions: (isVideo ? videoRuntimeSystemPrompt : canvasRuntimeSystemPrompt)(snapshot.objective, snapshot.summary, snapshot.inputs.map(input => input.text), context), context, gateway, signal: activeModelController.signal, tools: tools.executable, checkpoint: async (state, stateStatus) => { if (leaseLost) throw new AgentRunError("任务执行已中断"); const saved = await checkpointAgentRun(run, state, stateStatus); cancelled ||= saved.cancelRequested; emit("data-canvasAgentRun", agentRunView(saved)); }, consumeInputs: async (state) => { if (leaseLost) throw new AgentRunError("任务执行已中断"); const control = await consumeAgentRunInputs(run, state); cancelled ||= control.cancelled; return { cancelled }; }, isCancelled: async () => { const current = await heartbeatAgentRun(run); if (!current) throw new AgentRunError("任务执行已中断"); cancelled ||= current.cancelRequested; return cancelled; }, verify: async (state) => isVideo ? verifyVideoAgentResults({ snapshot: state, nodes: (await readOwnedVideoCanvas(session.user.id, projectId)).nodes, inspect: async (node, requirements) => { const metadata = await probeVideoMetadata(videoNodeUrl(node)); const report = await videoPorts!.analyze({ mode: "video-analysis", prompt: `请基于真实抽帧画面检查主体、场景、镜头连贯性,说明不确定内容。用户要求:${requirements}`, imageUrls: [], videoUrls: [videoNodeUrl(node)], }); const verdict = await generateText({ model, system: '你是视频验收员。依据实际抽帧分析判断是否满足用户要求;不能从文件名或提示词判断。若报告无法读取真实帧,必须不通过。抽帧只能检查画面,不能证明音频、精确时长或完整运动,涉及这些要求无法确认时说明需要人工检查。仅返回 JSON {"passed":boolean,"issues":string[]}。', prompt: JSON.stringify({ requirements, report }), maxOutputTokens: 1200, maxRetries: 0, abortSignal: AbortSignal.timeout(60_000), }); state.inputTokens += verdict.usage.inputTokens || 0; state.outputTokens += verdict.usage.outputTokens || 0; const match = verdict.text.match(/\{[\s\S]*\}/); if (!match) throw new Error("视频检查未返回有效结果"); return { ...z .object({ passed: z.boolean(), issues: z.array(z.string()).max(12) }) .parse(JSON.parse(match[0])), ...metadata, }; }, }) : verifyCanvasAgentResults({ snapshot: state, userId: session.user.id, projectId, model: visionModel || model, supportsVision: Boolean(visionModel), withModel, signal: activeModelController.signal, }), }); const finalText = snapshot.message; const progress = canvasAgentProgress(agentRunView({ ...run, snapshot, status })); writer.write({ type: "data-canvasAgentProgress", id: run.id, data: progress }); if (!isVideo) await persistRuntimeConversationResults({ userId: session.user.id, chatId: body.id, projectId, assistantMessageId, snapshot, }).catch((error) => { console.warn("[Canvas Agent] Conversation memory update failed", { runId: run.id, error: publicAgentError(error), }); }); await saveMessages({ messages: [ { id: assistantMessageId, chatId: body.id, role: "assistant", parts: [{ type: "text", text: finalText }, { type: "data-canvasAgentProgress", data: progress }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null, }, ], }); writer.write({ type: "text-delta", id: textId, delta: finalText }); } catch (error) { const text = publicAgentError(error); await checkpointAgentRun(run, { ...snapshot, message: text }, "needs_review").catch( () => undefined, ); const progress = canvasAgentProgress(agentRunView({ ...run, snapshot: { ...snapshot, message: text }, status: "needs_review" })); writer.write({ type: "data-canvasAgentProgress", id: run.id, data: progress }); await saveMessages({ messages: [{ id: assistantMessageId, chatId: body.id, role: "assistant", parts: [{ type: "text", text }, { type: "data-canvasAgentProgress", data: progress }], attachments: [], createdAt: new Date(), videoStorageStatus: null, temporaryVideoUrl: null, permanentVideoUrl: null, archivedAt: null, videoMetadata: null }] }).catch(() => undefined); writer.write({ type: "text-delta", id: textId, delta: text }); } finally { clearInterval(heartbeat); writer.write({ type: "text-end", id: textId }); writer.write({ type: "finish" }); } }, }); // Browser disconnection does not cancel paid image work. Keep consuming on the server. const [client, background] = stream.tee(); after(async () => { const reader = background.getReader(); try { while (!(await reader.read()).done) { /* Drain the server branch. */ } } finally { reader.releaseLock(); await archiveVideoResults?.(); } }); return createUIMessageStreamResponse({ stream: client, headers: { "Cache-Control": "no-store", "X-Accel-Buffering": "no" }, }); } catch (error) { return NextResponse.json( { error: publicAgentError(error), message: publicAgentError(error) }, { status: error instanceof AgentRunError ? error.status : error instanceof z.ZodError ? 400 : 503, }, ); } }