454 lines
20 KiB
TypeScript
454 lines
20 KiB
TypeScript
import { registerProjectAgentTools } from "@/lib/ai/canvas-agent-runtime/project-tools";
|
||
import { isZcodeWorkspaceAuthorized } from "@/lib/ai/canvas-agent-runtime/authorization";
|
||
import {
|
||
createUIMessageStream,
|
||
createUIMessageStreamResponse,
|
||
generateText,
|
||
} from "ai";
|
||
import { after, NextResponse } from "next/server";
|
||
import { z } from "zod";
|
||
import { db } from "@/lib/db";
|
||
import { siteSettings, canvasProject } from "@/lib/db/schema";
|
||
import { eq } from "drizzle-orm";
|
||
import { auth } from "@/app/(auth)/auth";
|
||
import { postRequestBodySchema } from "@/app/(chat)/api/chat/schema";
|
||
import { entitlementsByUserType } from "@/lib/ai/entitlements";
|
||
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
||
import {
|
||
getAgentAiModelWithProvider,
|
||
getAccessibleAgentPresetById,
|
||
getMultimodalAiModelsWithProvider,
|
||
getChatById,
|
||
getMessagesByChatId,
|
||
getMessageCountByUserId,
|
||
getSiteSettings,
|
||
saveChat,
|
||
saveMessages,
|
||
} from "@/lib/db/queries";
|
||
import { runZcodeCanvasTask } from "@/lib/ai/zcode-agent/task";
|
||
import { createZcodeModelGateway } from "@/lib/ai/zcode-agent/model-gateway";
|
||
import { newAgentRunSnapshot, publicAgentError } from "@/lib/ai/canvas-agent-runtime/protocol";
|
||
import { shouldUseCanvasAgentRuntime } from "@/lib/ai/canvas-agent-runtime/routing";
|
||
import {
|
||
acquireAgentRun,
|
||
agentRunView,
|
||
AgentRunError,
|
||
checkpointAgentRun,
|
||
consumeAgentRunInputs,
|
||
heartbeatAgentRun,
|
||
} from "@/lib/ai/canvas-agent-runtime/store";
|
||
import { readOwnedCanvas } from "@/lib/ai/canvas-agent-runtime/canvas";
|
||
import { buildCanvasRuntimeTools } from "@/lib/ai/canvas-agent-runtime/tools";
|
||
import { canvasRuntimeSystemPrompt } from "@/lib/ai/canvas-agent-runtime/prompt";
|
||
import { verifyCanvasAgentResults } from "@/lib/ai/canvas-agent-runtime/review";
|
||
import { inferCanvasTaskRequirements } from "@/lib/ai/canvas-agent-runtime/requirements";
|
||
import {
|
||
resolveRuntimeConversationSources,
|
||
persistRuntimeConversationResults,
|
||
} from "@/lib/ai/canvas-agent-runtime/conversation";
|
||
import { getCanvasAgentFeatureFlagsFromSystemSettings } from "@/lib/canvas/agent-feature-settings";
|
||
import { canApplyCanvasAgentMutations } from "@/lib/canvas/agent-feature-flags";
|
||
|
||
import {
|
||
videoAgentSettingsSchema,
|
||
inferVideoTaskRequirements,
|
||
videoNodeUrl,
|
||
videoRuntimeSystemPrompt,
|
||
} from "@/lib/ai/canvas-agent-runtime/video-protocol";
|
||
import { readOwnedVideoCanvas } from "@/lib/ai/canvas-agent-runtime/video-store";
|
||
import {
|
||
buildVideoRuntimeTools,
|
||
createVideoRuntimePorts,
|
||
} from "@/lib/ai/canvas-agent-runtime/video-tools";
|
||
import { probeVideoMetadata } from "@/lib/ai/canvas-agent-runtime/video-probe";
|
||
import { verifyVideoAgentResults } from "@/lib/ai/canvas-agent-runtime/video-review";
|
||
|
||
export const runtime = "nodejs";
|
||
export const maxDuration = 1200;
|
||
const requestSchema = postRequestBodySchema.extend({
|
||
resumeRunId: z.string().uuid().optional(),
|
||
workspaceVariant: z.enum(["canvas", "video"]).default("canvas"),
|
||
videoAgentSettings: videoAgentSettingsSchema.optional(),
|
||
});
|
||
export async function POST(request: Request) {
|
||
const session = await auth();
|
||
if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 });
|
||
try {
|
||
const body = requestSchema.parse(await request.json());
|
||
if (!shouldUseCanvasAgentRuntime(body))
|
||
throw new AgentRunError("此请求不属于画布 Agent 任务", 400);
|
||
const projectId = body.contextPayload!.projectId!;
|
||
const [ownedProject] = await db.select({ userId: canvasProject.userId, workspaceType: canvasProject.workspaceType })
|
||
.from(canvasProject).where(eq(canvasProject.id, projectId)).limit(1);
|
||
if (!ownedProject || ownedProject.userId !== session.user.id) throw new AgentRunError("画布不存在", 404);
|
||
if (!isZcodeWorkspaceAuthorized(ownedProject.workspaceType)) throw new AgentRunError("视频画布暂未授权给 ZCode Agent，请使用手工功能。", 403);
|
||
const isVideo = ownedProject.workspaceType === "video";
|
||
const preset = body.agentPresetId ? await getAccessibleAgentPresetById({ id: body.agentPresetId, userId: session.user.id }) : null;
|
||
if (preset && String(preset.type) === "video") throw new AgentRunError("视频技能暂未授权给 ZCode Agent", 403);
|
||
if (body.agentPresetId && (!preset || preset.status !== "active")) throw new AgentRunError("此 Agent 预设不可用", 403);
|
||
const objective = body.message.parts
|
||
.filter((part) => part.type === "text")
|
||
.map((part) => part.text)
|
||
.join("\n")
|
||
.trim();
|
||
if (!objective) throw new AgentRunError("请描述需要完成的创作任务", 400);
|
||
const settings = await getSiteSettings();
|
||
if (
|
||
!settings.agentEnabled ||
|
||
!canApplyCanvasAgentMutations(await getCanvasAgentFeatureFlagsFromSystemSettings())
|
||
) {
|
||
throw new AgentRunError("管理员尚未开启画布 Agent 操作", 403);
|
||
}
|
||
const [usageSettings] = await db.select().from(siteSettings).limit(1);
|
||
if (usageSettings?.usageTimeEnabled) {
|
||
const time = new Intl.DateTimeFormat("en-GB", {
|
||
timeZone: "Asia/Shanghai",
|
||
hour: "2-digit",
|
||
minute: "2-digit",
|
||
hourCycle: "h23",
|
||
}).format(new Date());
|
||
if (
|
||
time < (usageSettings.usageTimeStart || "08:00") ||
|
||
time > (usageSettings.usageTimeEnd || "20:00")
|
||
)
|
||
throw new AgentRunError(usageSettings.usageTimeMessage || "当前不在系统使用时间段内", 403);
|
||
}
|
||
const count = await getMessageCountByUserId({ id: session.user.id, differenceInHours: 24 });
|
||
if (count > entitlementsByUserType[session.user.type].maxMessagesPerDay)
|
||
throw new AgentRunError("今日对话次数已达到限额", 429);
|
||
const imageCanvas = isVideo ? null : await readOwnedCanvas(session.user.id, projectId);
|
||
if (imageCanvas?.project.workspaceType === "video")
|
||
throw new AgentRunError("请使用视频画布 Agent 入口", 400);
|
||
const videoCanvas = isVideo ? await readOwnedVideoCanvas(session.user.id, projectId) : null;
|
||
const nodes = imageCanvas?.nodes || [];
|
||
const availableIds = isVideo
|
||
? videoCanvas!.nodes.map((node) => node.id)
|
||
: nodes.map((node) => node.id);
|
||
const selectedNodeIds = body.contextPayload?.selectedNodeIds || [];
|
||
if (selectedNodeIds.some((id) => !availableIds.includes(id))) {
|
||
throw new AgentRunError("选中图片尚未保存到画布，请等待保存完成后重试。");
|
||
}
|
||
const existingChat = await getChatById({ id: body.id });
|
||
if (existingChat && existingChat.userId !== session.user.id)
|
||
throw new AgentRunError("对话不存在", 404);
|
||
if (!existingChat)
|
||
await saveChat({
|
||
id: body.id,
|
||
userId: session.user.id,
|
||
title: objective.slice(0, 80),
|
||
visibility: "private",
|
||
});
|
||
const conversation = isVideo
|
||
? {
|
||
sourceNodeIds: selectedNodeIds,
|
||
targetReason: "video-selected-nodes",
|
||
previousContext: null,
|
||
}
|
||
: await resolveRuntimeConversationSources({
|
||
userId: session.user.id,
|
||
chatId: body.id,
|
||
projectId,
|
||
text: objective,
|
||
parts: body.message.parts,
|
||
nodes,
|
||
selectedNodeIds,
|
||
});
|
||
const configured = await getAgentAiModelWithProvider();
|
||
if (!configured) throw new AgentRunError("请先在后台配置 Agent 对话模型", 503);
|
||
const model = await getLanguageModelById(configured.model.id);
|
||
if (!model) throw new AgentRunError("Agent 模型暂不可用", 503);
|
||
const visionEntry = configured.model.supportsVision
|
||
? configured
|
||
: (await getMultimodalAiModelsWithProvider()).find((entry) => entry.model.supportsVision);
|
||
const visionModel = visionEntry ? await getLanguageModelById(visionEntry.model.id) : null;
|
||
const assistantMessageId = crypto.randomUUID();
|
||
const previousVideoMessages =
|
||
existingChat
|
||
? (await getMessagesByChatId({ id: body.id })).slice(-6).map((message) => ({
|
||
role: message.role as "user" | "assistant",
|
||
content: (message.parts as Array<{ type: string; text?: string }>)
|
||
.filter((part) => part.type === "text")
|
||
.map((part) => part.text || "")
|
||
.join("\n")
|
||
.slice(0, 4000),
|
||
}))
|
||
: [];
|
||
const initialSnapshot = newAgentRunSnapshot(objective, [
|
||
...previousVideoMessages,
|
||
{ role: "user", content: objective },
|
||
]);
|
||
initialSnapshot.workspace = isVideo ? "video" : "canvas";
|
||
if (isVideo) {
|
||
initialSnapshot.videoRequirements = inferVideoTaskRequirements(objective);
|
||
initialSnapshot.videoSettings = videoAgentSettingsSchema.parse({
|
||
...body.videoAgentSettings,
|
||
...(initialSnapshot.videoRequirements.duration
|
||
? { duration: initialSnapshot.videoRequirements.duration }
|
||
: {}),
|
||
...(initialSnapshot.videoRequirements.ratio
|
||
? { ratio: initialSnapshot.videoRequirements.ratio }
|
||
: {}),
|
||
});
|
||
}
|
||
initialSnapshot.sourceNodeIds = conversation.sourceNodeIds;
|
||
initialSnapshot.requirements = {
|
||
...inferCanvasTaskRequirements(objective, initialSnapshot.sourceNodeIds.length),
|
||
sourceCount: initialSnapshot.sourceNodeIds.length,
|
||
};
|
||
const run = await acquireAgentRun({
|
||
id: body.message.id,
|
||
userId: session.user.id,
|
||
projectId,
|
||
chatId: body.id,
|
||
assistantMessageId,
|
||
resumeRunId: body.resumeRunId,
|
||
snapshot: initialSnapshot,
|
||
});
|
||
const snapshot = run.snapshot;
|
||
const taskSourceNodeIds = snapshot.sourceNodeIds ?? selectedNodeIds;
|
||
const context = {
|
||
recentConversation: previousVideoMessages,
|
||
preset: preset ? { name: preset.name, type: preset.type, promptTemplate: preset.promptTemplate, defaultModelId: preset.defaultModelId, defaultParams: preset.defaultParams, slots: preset.slots.map(slot => ({ title: slot.title, promptTemplate: slot.promptTemplate, defaultParams: slot.defaultParams })) } : null,
|
||
videoSettings: snapshot.videoSettings,
|
||
selectedNodeIds: taskSourceNodeIds,
|
||
nodes: nodes
|
||
.filter((node) => taskSourceNodeIds.includes(node.id))
|
||
.map((node) => ({
|
||
id: node.id,
|
||
type: node.type,
|
||
position: node.position,
|
||
size: node.size,
|
||
title: node.data.title,
|
||
})),
|
||
skillInstructions: body.hiddenPrompt || undefined,
|
||
imageParameters: body.imageParams,
|
||
targetReason: conversation.targetReason,
|
||
previousConversationContext: conversation.previousContext,
|
||
};
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: body.message.id,
|
||
chatId: body.id,
|
||
role: "user",
|
||
parts: body.message.parts,
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
let archiveVideoResults: (() => Promise<void>) | undefined;
|
||
const stream = createUIMessageStream({
|
||
execute: async ({ writer }) => {
|
||
const emit = (type: string, data: unknown) => writer.write({ type, data } as never);
|
||
let cancelled = false;
|
||
let leaseLost = false;
|
||
let activeModelController: AbortController | null = null;
|
||
const heartbeat = setInterval(() => {
|
||
void heartbeatAgentRun(run)
|
||
.then((current) => {
|
||
if (!current) leaseLost = true;
|
||
if (current?.cancelRequested) cancelled = true;
|
||
if (cancelled || leaseLost) activeModelController?.abort();
|
||
})
|
||
.catch(() => {
|
||
leaseLost = true;
|
||
activeModelController?.abort();
|
||
});
|
||
emit("data-keepalive", { label: "canvas-agent-runtime", ts: Date.now() });
|
||
}, 10_000);
|
||
const textId = crypto.randomUUID();
|
||
writer.write({ type: "start", messageId: assistantMessageId });
|
||
writer.write({ type: "text-start", id: textId });
|
||
writer.write({
|
||
type: "text-delta",
|
||
id: textId,
|
||
delta: isVideo ? "正在按步骤处理你的视频任务。\n\n" : "正在按步骤处理你的图片任务。\n\n",
|
||
});
|
||
try {
|
||
const videoPorts = isVideo
|
||
? await createVideoRuntimePorts({
|
||
userId: session.user.id,
|
||
chatId: body.id,
|
||
projectId,
|
||
runId: run.id,
|
||
assistantMessageId,
|
||
request,
|
||
snapshot,
|
||
emit,
|
||
})
|
||
: null;
|
||
archiveVideoResults = videoPorts?.archiveResults;
|
||
const tools = isVideo
|
||
? buildVideoRuntimeTools({
|
||
runId: run.id,
|
||
projectId,
|
||
selectedNodeIds,
|
||
snapshot,
|
||
ports: videoPorts!,
|
||
})
|
||
: await buildCanvasRuntimeTools({
|
||
userId: session.user.id,
|
||
chatId: body.id,
|
||
projectId,
|
||
runId: run.id,
|
||
assistantMessageId,
|
||
selectedNodeIds: taskSourceNodeIds,
|
||
request,
|
||
model: visionModel || model,
|
||
supportsVision: Boolean(visionModel),
|
||
imageSize: body.imageParams?.imageSize,
|
||
preferredModelId: preset?.type === "image" ? preset.defaultModelId || undefined : undefined,
|
||
operationSelectedNodeIds: selectedNodeIds,
|
||
objective: [snapshot.objective, ...snapshot.inputs.map((input) => input.text)].join(
|
||
"\n",
|
||
),
|
||
emit,
|
||
});
|
||
await registerProjectAgentTools({ session, projectId, request, objective: snapshot.objective,
|
||
definitions: tools.definitions, executable: tools.executable, emit });
|
||
const gateway = await createZcodeModelGateway(configured);
|
||
activeModelController = new AbortController();
|
||
const status = await runZcodeCanvasTask({
|
||
runId: run.id, snapshot, definitions: tools.definitions,
|
||
instructions: (isVideo ? videoRuntimeSystemPrompt : canvasRuntimeSystemPrompt)(snapshot.objective, snapshot.summary, snapshot.inputs.map(input => input.text), context),
|
||
context, gateway, signal: activeModelController.signal,
|
||
tools: tools.executable,
|
||
checkpoint: async (state, stateStatus) => {
|
||
if (leaseLost) throw new AgentRunError("任务执行已中断");
|
||
const saved = await checkpointAgentRun(run, state, stateStatus);
|
||
cancelled ||= saved.cancelRequested;
|
||
emit("data-canvasAgentRun", agentRunView(saved));
|
||
},
|
||
consumeInputs: async (state) => {
|
||
if (leaseLost) throw new AgentRunError("任务执行已中断");
|
||
const control = await consumeAgentRunInputs(run, state);
|
||
cancelled ||= control.cancelled;
|
||
return { cancelled };
|
||
},
|
||
isCancelled: async () => {
|
||
const current = await heartbeatAgentRun(run);
|
||
if (!current) throw new AgentRunError("任务执行已中断");
|
||
cancelled ||= current.cancelRequested;
|
||
return cancelled;
|
||
},
|
||
verify: async (state) =>
|
||
isVideo
|
||
? verifyVideoAgentResults({
|
||
snapshot: state,
|
||
nodes: (await readOwnedVideoCanvas(session.user.id, projectId)).nodes,
|
||
inspect: async (node, requirements) => {
|
||
const metadata = await probeVideoMetadata(videoNodeUrl(node));
|
||
const report = await videoPorts!.analyze({
|
||
mode: "video-analysis",
|
||
prompt: `请基于真实抽帧画面检查主体、场景、镜头连贯性，说明不确定内容。用户要求：${requirements}`,
|
||
imageUrls: [],
|
||
videoUrls: [videoNodeUrl(node)],
|
||
});
|
||
const verdict = await generateText({
|
||
model,
|
||
system:
|
||
'你是视频验收员。依据实际抽帧分析判断是否满足用户要求；不能从文件名或提示词判断。若报告无法读取真实帧，必须不通过。抽帧只能检查画面，不能证明音频、精确时长或完整运动，涉及这些要求无法确认时说明需要人工检查。仅返回 JSON {"passed":boolean,"issues":string[]}。',
|
||
prompt: JSON.stringify({ requirements, report }),
|
||
maxOutputTokens: 1200,
|
||
maxRetries: 0,
|
||
abortSignal: AbortSignal.timeout(60_000),
|
||
});
|
||
state.inputTokens += verdict.usage.inputTokens || 0;
|
||
state.outputTokens += verdict.usage.outputTokens || 0;
|
||
const match = verdict.text.match(/\{[\s\S]*\}/);
|
||
if (!match) throw new Error("视频检查未返回有效结果");
|
||
return {
|
||
...z
|
||
.object({ passed: z.boolean(), issues: z.array(z.string()).max(12) })
|
||
.parse(JSON.parse(match[0])),
|
||
...metadata,
|
||
};
|
||
},
|
||
})
|
||
: verifyCanvasAgentResults({
|
||
snapshot: state,
|
||
userId: session.user.id,
|
||
projectId,
|
||
model: visionModel || model,
|
||
supportsVision: Boolean(visionModel),
|
||
}),
|
||
});
|
||
const finalText = snapshot.message;
|
||
if (!isVideo)
|
||
await persistRuntimeConversationResults({
|
||
userId: session.user.id,
|
||
chatId: body.id,
|
||
projectId,
|
||
assistantMessageId,
|
||
snapshot,
|
||
}).catch((error) => {
|
||
console.warn("[Canvas Agent] Conversation memory update failed", {
|
||
runId: run.id,
|
||
error: publicAgentError(error),
|
||
});
|
||
});
|
||
await saveMessages({
|
||
messages: [
|
||
{
|
||
id: assistantMessageId,
|
||
chatId: body.id,
|
||
role: "assistant",
|
||
parts: [{ type: "text", text: finalText }],
|
||
attachments: [],
|
||
createdAt: new Date(),
|
||
videoStorageStatus: null,
|
||
temporaryVideoUrl: null,
|
||
permanentVideoUrl: null,
|
||
archivedAt: null,
|
||
videoMetadata: null,
|
||
},
|
||
],
|
||
});
|
||
writer.write({ type: "text-delta", id: textId, delta: finalText });
|
||
} catch (error) {
|
||
const text = publicAgentError(error);
|
||
await checkpointAgentRun(run, { ...snapshot, message: text }, "needs_review").catch(
|
||
() => undefined,
|
||
);
|
||
writer.write({ type: "text-delta", id: textId, delta: text });
|
||
} finally {
|
||
clearInterval(heartbeat);
|
||
writer.write({ type: "text-end", id: textId });
|
||
writer.write({ type: "finish" });
|
||
}
|
||
},
|
||
});
|
||
// Browser disconnection does not cancel paid image work. Keep consuming on the server.
|
||
const [client, background] = stream.tee();
|
||
after(async () => {
|
||
const reader = background.getReader();
|
||
try {
|
||
while (!(await reader.read()).done) {
|
||
/* Drain the server branch. */
|
||
}
|
||
} finally {
|
||
reader.releaseLock();
|
||
await archiveVideoResults?.();
|
||
}
|
||
});
|
||
return createUIMessageStreamResponse({
|
||
stream: client,
|
||
headers: { "Cache-Control": "no-store", "X-Accel-Buffering": "no" },
|
||
});
|
||
} catch (error) {
|
||
return NextResponse.json(
|
||
{ error: publicAgentError(error), message: publicAgentError(error) },
|
||
{
|
||
status:
|
||
error instanceof AgentRunError ? error.status : error instanceof z.ZodError ? 400 : 503,
|
||
},
|
||
);
|
||
}
|
||
}
|