448 lines
19 KiB
TypeScript
448 lines
19 KiB
TypeScript
import {
|
|
createUIMessageStream,
|
|
createUIMessageStreamResponse,
|
|
generateText,
|
|
} from "ai";
|
|
import { after, NextResponse } from "next/server";
|
|
import { z } from "zod";
|
|
import { db } from "@/lib/db";
|
|
import { siteSettings, canvasProject } from "@/lib/db/schema";
|
|
import { eq } from "drizzle-orm";
|
|
import { auth } from "@/app/(auth)/auth";
|
|
import { postRequestBodySchema } from "@/app/(chat)/api/chat/schema";
|
|
import { entitlementsByUserType } from "@/lib/ai/entitlements";
|
|
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
|
import {
|
|
getAgentAiModelWithProvider,
|
|
getAccessibleAgentPresetById,
|
|
getMultimodalAiModelsWithProvider,
|
|
getChatById,
|
|
getMessagesByChatId,
|
|
getMessageCountByUserId,
|
|
getSiteSettings,
|
|
saveChat,
|
|
saveMessages,
|
|
} from "@/lib/db/queries";
|
|
import { runZcodeCanvasTask } from "@/lib/ai/zcode-agent/task";
|
|
import { createZcodeModelGateway } from "@/lib/ai/zcode-agent/model-gateway";
|
|
import { newAgentRunSnapshot, publicAgentError } from "@/lib/ai/canvas-agent-runtime/protocol";
|
|
import { shouldUseCanvasAgentRuntime } from "@/lib/ai/canvas-agent-runtime/routing";
|
|
import {
|
|
acquireAgentRun,
|
|
agentRunView,
|
|
AgentRunError,
|
|
checkpointAgentRun,
|
|
consumeAgentRunInputs,
|
|
heartbeatAgentRun,
|
|
} from "@/lib/ai/canvas-agent-runtime/store";
|
|
import { readOwnedCanvas } from "@/lib/ai/canvas-agent-runtime/canvas";
|
|
import { buildCanvasRuntimeTools } from "@/lib/ai/canvas-agent-runtime/tools";
|
|
import { canvasRuntimeSystemPrompt } from "@/lib/ai/canvas-agent-runtime/prompt";
|
|
import { verifyCanvasAgentResults } from "@/lib/ai/canvas-agent-runtime/review";
|
|
import { inferCanvasTaskRequirements } from "@/lib/ai/canvas-agent-runtime/requirements";
|
|
import {
|
|
resolveRuntimeConversationSources,
|
|
persistRuntimeConversationResults,
|
|
} from "@/lib/ai/canvas-agent-runtime/conversation";
|
|
import { getCanvasAgentFeatureFlagsFromSystemSettings } from "@/lib/canvas/agent-feature-settings";
|
|
import { canApplyCanvasAgentMutations } from "@/lib/canvas/agent-feature-flags";
|
|
|
|
import {
|
|
videoAgentSettingsSchema,
|
|
inferVideoTaskRequirements,
|
|
videoNodeUrl,
|
|
videoRuntimeSystemPrompt,
|
|
} from "@/lib/ai/canvas-agent-runtime/video-protocol";
|
|
import { readOwnedVideoCanvas } from "@/lib/ai/canvas-agent-runtime/video-store";
|
|
import {
|
|
buildVideoRuntimeTools,
|
|
createVideoRuntimePorts,
|
|
} from "@/lib/ai/canvas-agent-runtime/video-tools";
|
|
import { probeVideoMetadata } from "@/lib/ai/canvas-agent-runtime/video-probe";
|
|
import { verifyVideoAgentResults } from "@/lib/ai/canvas-agent-runtime/video-review";
|
|
|
|
export const runtime = "nodejs";
|
|
export const maxDuration = 1200;
|
|
const requestSchema = postRequestBodySchema.extend({
|
|
resumeRunId: z.string().uuid().optional(),
|
|
workspaceVariant: z.enum(["canvas", "video"]).default("canvas"),
|
|
videoAgentSettings: videoAgentSettingsSchema.optional(),
|
|
});
|
|
export async function POST(request: Request) {
|
|
const session = await auth();
|
|
if (!session?.user?.id) return NextResponse.json({ error: "未登录" }, { status: 401 });
|
|
try {
|
|
const body = requestSchema.parse(await request.json());
|
|
if (!shouldUseCanvasAgentRuntime(body))
|
|
throw new AgentRunError("此请求不属于画布 Agent 任务", 400);
|
|
const projectId = body.contextPayload!.projectId!;
|
|
const [ownedProject] = await db.select({ userId: canvasProject.userId, workspaceType: canvasProject.workspaceType })
|
|
.from(canvasProject).where(eq(canvasProject.id, projectId)).limit(1);
|
|
if (!ownedProject || ownedProject.userId !== session.user.id) throw new AgentRunError("画布不存在", 404);
|
|
const isVideo = ownedProject.workspaceType === "video";
|
|
const preset = body.agentPresetId ? await getAccessibleAgentPresetById({ id: body.agentPresetId, userId: session.user.id }) : null;
|
|
if (body.agentPresetId && (!preset || preset.status !== "active")) throw new AgentRunError("此 Agent 预设不可用", 403);
|
|
const objective = body.message.parts
|
|
.filter((part) => part.type === "text")
|
|
.map((part) => part.text)
|
|
.join("\n")
|
|
.trim();
|
|
if (!objective) throw new AgentRunError("请描述需要完成的创作任务", 400);
|
|
const settings = await getSiteSettings();
|
|
if (
|
|
!settings.agentEnabled ||
|
|
!canApplyCanvasAgentMutations(await getCanvasAgentFeatureFlagsFromSystemSettings())
|
|
) {
|
|
throw new AgentRunError("管理员尚未开启画布 Agent 操作", 403);
|
|
}
|
|
const [usageSettings] = await db.select().from(siteSettings).limit(1);
|
|
if (usageSettings?.usageTimeEnabled) {
|
|
const time = new Intl.DateTimeFormat("en-GB", {
|
|
timeZone: "Asia/Shanghai",
|
|
hour: "2-digit",
|
|
minute: "2-digit",
|
|
hourCycle: "h23",
|
|
}).format(new Date());
|
|
if (
|
|
time < (usageSettings.usageTimeStart || "08:00") ||
|
|
time > (usageSettings.usageTimeEnd || "20:00")
|
|
)
|
|
throw new AgentRunError(usageSettings.usageTimeMessage || "当前不在系统使用时间段内", 403);
|
|
}
|
|
const count = await getMessageCountByUserId({ id: session.user.id, differenceInHours: 24 });
|
|
if (count > entitlementsByUserType[session.user.type].maxMessagesPerDay)
|
|
throw new AgentRunError("今日对话次数已达到限额", 429);
|
|
const imageCanvas = isVideo ? null : await readOwnedCanvas(session.user.id, projectId);
|
|
if (imageCanvas?.project.workspaceType === "video")
|
|
throw new AgentRunError("请使用视频画布 Agent 入口", 400);
|
|
const videoCanvas = isVideo ? await readOwnedVideoCanvas(session.user.id, projectId) : null;
|
|
const nodes = imageCanvas?.nodes || [];
|
|
const availableIds = isVideo
|
|
? videoCanvas!.nodes.map((node) => node.id)
|
|
: nodes.map((node) => node.id);
|
|
const selectedNodeIds = body.contextPayload?.selectedNodeIds || [];
|
|
if (selectedNodeIds.some((id) => !availableIds.includes(id))) {
|
|
throw new AgentRunError("选中图片尚未保存到画布，请等待保存完成后重试。");
|
|
}
|
|
const existingChat = await getChatById({ id: body.id });
|
|
if (existingChat && existingChat.userId !== session.user.id)
|
|
throw new AgentRunError("对话不存在", 404);
|
|
if (!existingChat)
|
|
await saveChat({
|
|
id: body.id,
|
|
userId: session.user.id,
|
|
title: objective.slice(0, 80),
|
|
visibility: "private",
|
|
});
|
|
const conversation = isVideo
|
|
? {
|
|
sourceNodeIds: selectedNodeIds,
|
|
targetReason: "video-selected-nodes",
|
|
previousContext: null,
|
|
}
|
|
: await resolveRuntimeConversationSources({
|
|
userId: session.user.id,
|
|
chatId: body.id,
|
|
projectId,
|
|
text: objective,
|
|
parts: body.message.parts,
|
|
nodes,
|
|
selectedNodeIds,
|
|
});
|
|
const configured = await getAgentAiModelWithProvider();
|
|
if (!configured) throw new AgentRunError("请先在后台配置 Agent 对话模型", 503);
|
|
const model = await getLanguageModelById(configured.model.id);
|
|
if (!model) throw new AgentRunError("Agent 模型暂不可用", 503);
|
|
const visionEntry = configured.model.supportsVision
|
|
? configured
|
|
: (await getMultimodalAiModelsWithProvider()).find((entry) => entry.model.supportsVision);
|
|
const visionModel = visionEntry ? await getLanguageModelById(visionEntry.model.id) : null;
|
|
const assistantMessageId = crypto.randomUUID();
|
|
const previousVideoMessages =
|
|
existingChat
|
|
? (await getMessagesByChatId({ id: body.id })).slice(-6).map((message) => ({
|
|
role: message.role as "user" | "assistant",
|
|
content: (message.parts as Array<{ type: string; text?: string }>)
|
|
.filter((part) => part.type === "text")
|
|
.map((part) => part.text || "")
|
|
.join("\n")
|
|
.slice(0, 4000),
|
|
}))
|
|
: [];
|
|
const initialSnapshot = newAgentRunSnapshot(objective, [
|
|
...previousVideoMessages,
|
|
{ role: "user", content: objective },
|
|
]);
|
|
initialSnapshot.workspace = isVideo ? "video" : "canvas";
|
|
if (isVideo) {
|
|
initialSnapshot.videoRequirements = inferVideoTaskRequirements(objective);
|
|
initialSnapshot.videoSettings = videoAgentSettingsSchema.parse({
|
|
...body.videoAgentSettings,
|
|
...(initialSnapshot.videoRequirements.duration
|
|
? { duration: initialSnapshot.videoRequirements.duration }
|
|
: {}),
|
|
...(initialSnapshot.videoRequirements.ratio
|
|
? { ratio: initialSnapshot.videoRequirements.ratio }
|
|
: {}),
|
|
});
|
|
}
|
|
initialSnapshot.sourceNodeIds = conversation.sourceNodeIds;
|
|
initialSnapshot.requirements = {
|
|
...inferCanvasTaskRequirements(objective, initialSnapshot.sourceNodeIds.length),
|
|
sourceCount: initialSnapshot.sourceNodeIds.length,
|
|
};
|
|
const run = await acquireAgentRun({
|
|
id: body.message.id,
|
|
userId: session.user.id,
|
|
projectId,
|
|
chatId: body.id,
|
|
assistantMessageId,
|
|
resumeRunId: body.resumeRunId,
|
|
snapshot: initialSnapshot,
|
|
});
|
|
const snapshot = run.snapshot;
|
|
const taskSourceNodeIds = snapshot.sourceNodeIds ?? selectedNodeIds;
|
|
const context = {
|
|
recentConversation: previousVideoMessages,
|
|
preset: preset ? { name: preset.name, type: preset.type, promptTemplate: preset.promptTemplate, defaultModelId: preset.defaultModelId, defaultParams: preset.defaultParams, slots: preset.slots.map(slot => ({ title: slot.title, promptTemplate: slot.promptTemplate, defaultParams: slot.defaultParams })) } : null,
|
|
videoSettings: snapshot.videoSettings,
|
|
selectedNodeIds: taskSourceNodeIds,
|
|
nodes: nodes
|
|
.filter((node) => taskSourceNodeIds.includes(node.id))
|
|
.map((node) => ({
|
|
id: node.id,
|
|
type: node.type,
|
|
position: node.position,
|
|
size: node.size,
|
|
title: node.data.title,
|
|
})),
|
|
skillInstructions: body.hiddenPrompt || undefined,
|
|
imageParameters: body.imageParams,
|
|
targetReason: conversation.targetReason,
|
|
previousConversationContext: conversation.previousContext,
|
|
};
|
|
await saveMessages({
|
|
messages: [
|
|
{
|
|
id: body.message.id,
|
|
chatId: body.id,
|
|
role: "user",
|
|
parts: body.message.parts,
|
|
attachments: [],
|
|
createdAt: new Date(),
|
|
videoStorageStatus: null,
|
|
temporaryVideoUrl: null,
|
|
permanentVideoUrl: null,
|
|
archivedAt: null,
|
|
videoMetadata: null,
|
|
},
|
|
],
|
|
});
|
|
let archiveVideoResults: (() => Promise<void>) | undefined;
|
|
const stream = createUIMessageStream({
|
|
execute: async ({ writer }) => {
|
|
const emit = (type: string, data: unknown) => writer.write({ type, data } as never);
|
|
let cancelled = false;
|
|
let leaseLost = false;
|
|
let activeModelController: AbortController | null = null;
|
|
const heartbeat = setInterval(() => {
|
|
void heartbeatAgentRun(run)
|
|
.then((current) => {
|
|
if (!current) leaseLost = true;
|
|
if (current?.cancelRequested) cancelled = true;
|
|
if (cancelled || leaseLost) activeModelController?.abort();
|
|
})
|
|
.catch(() => {
|
|
leaseLost = true;
|
|
activeModelController?.abort();
|
|
});
|
|
emit("data-keepalive", { label: "canvas-agent-runtime", ts: Date.now() });
|
|
}, 10_000);
|
|
const textId = crypto.randomUUID();
|
|
writer.write({ type: "start", messageId: assistantMessageId });
|
|
writer.write({ type: "text-start", id: textId });
|
|
writer.write({
|
|
type: "text-delta",
|
|
id: textId,
|
|
delta: isVideo ? "正在按步骤处理你的视频任务。\n\n" : "正在按步骤处理你的图片任务。\n\n",
|
|
});
|
|
try {
|
|
const videoPorts = isVideo
|
|
? await createVideoRuntimePorts({
|
|
userId: session.user.id,
|
|
chatId: body.id,
|
|
projectId,
|
|
runId: run.id,
|
|
assistantMessageId,
|
|
request,
|
|
snapshot,
|
|
emit,
|
|
})
|
|
: null;
|
|
archiveVideoResults = videoPorts?.archiveResults;
|
|
const tools = isVideo
|
|
? buildVideoRuntimeTools({
|
|
runId: run.id,
|
|
projectId,
|
|
selectedNodeIds,
|
|
snapshot,
|
|
ports: videoPorts!,
|
|
})
|
|
: await buildCanvasRuntimeTools({
|
|
userId: session.user.id,
|
|
chatId: body.id,
|
|
projectId,
|
|
runId: run.id,
|
|
assistantMessageId,
|
|
selectedNodeIds: taskSourceNodeIds,
|
|
request,
|
|
model: visionModel || model,
|
|
supportsVision: Boolean(visionModel),
|
|
imageSize: body.imageParams?.imageSize,
|
|
preferredModelId: preset?.type === "image" ? preset.defaultModelId || undefined : undefined,
|
|
operationSelectedNodeIds: selectedNodeIds,
|
|
objective: [snapshot.objective, ...snapshot.inputs.map((input) => input.text)].join(
|
|
"\n",
|
|
),
|
|
emit,
|
|
});
|
|
const gateway = await createZcodeModelGateway(configured);
|
|
activeModelController = new AbortController();
|
|
const status = await runZcodeCanvasTask({
|
|
runId: run.id, snapshot, definitions: tools.definitions,
|
|
instructions: (isVideo ? videoRuntimeSystemPrompt : canvasRuntimeSystemPrompt)(snapshot.objective, snapshot.summary, snapshot.inputs.map(input => input.text), context),
|
|
context, gateway, signal: activeModelController.signal,
|
|
tools: tools.executable,
|
|
checkpoint: async (state, stateStatus) => {
|
|
if (leaseLost) throw new AgentRunError("任务执行已中断");
|
|
const saved = await checkpointAgentRun(run, state, stateStatus);
|
|
cancelled ||= saved.cancelRequested;
|
|
emit("data-canvasAgentRun", agentRunView(saved));
|
|
},
|
|
consumeInputs: async (state) => {
|
|
if (leaseLost) throw new AgentRunError("任务执行已中断");
|
|
const control = await consumeAgentRunInputs(run, state);
|
|
cancelled ||= control.cancelled;
|
|
return { cancelled };
|
|
},
|
|
isCancelled: async () => {
|
|
const current = await heartbeatAgentRun(run);
|
|
if (!current) throw new AgentRunError("任务执行已中断");
|
|
cancelled ||= current.cancelRequested;
|
|
return cancelled;
|
|
},
|
|
verify: async (state) =>
|
|
isVideo
|
|
? verifyVideoAgentResults({
|
|
snapshot: state,
|
|
nodes: (await readOwnedVideoCanvas(session.user.id, projectId)).nodes,
|
|
inspect: async (node, requirements) => {
|
|
const metadata = await probeVideoMetadata(videoNodeUrl(node));
|
|
const report = await videoPorts!.analyze({
|
|
mode: "video-analysis",
|
|
prompt: `请基于真实抽帧画面检查主体、场景、镜头连贯性，说明不确定内容。用户要求：${requirements}`,
|
|
imageUrls: [],
|
|
videoUrls: [videoNodeUrl(node)],
|
|
});
|
|
const verdict = await generateText({
|
|
model,
|
|
system:
|
|
'你是视频验收员。依据实际抽帧分析判断是否满足用户要求；不能从文件名或提示词判断。若报告无法读取真实帧，必须不通过。抽帧只能检查画面，不能证明音频、精确时长或完整运动，涉及这些要求无法确认时说明需要人工检查。仅返回 JSON {"passed":boolean,"issues":string[]}。',
|
|
prompt: JSON.stringify({ requirements, report }),
|
|
maxOutputTokens: 1200,
|
|
maxRetries: 0,
|
|
abortSignal: AbortSignal.timeout(60_000),
|
|
});
|
|
state.inputTokens += verdict.usage.inputTokens || 0;
|
|
state.outputTokens += verdict.usage.outputTokens || 0;
|
|
const match = verdict.text.match(/\{[\s\S]*\}/);
|
|
if (!match) throw new Error("视频检查未返回有效结果");
|
|
return {
|
|
...z
|
|
.object({ passed: z.boolean(), issues: z.array(z.string()).max(12) })
|
|
.parse(JSON.parse(match[0])),
|
|
...metadata,
|
|
};
|
|
},
|
|
})
|
|
: verifyCanvasAgentResults({
|
|
snapshot: state,
|
|
userId: session.user.id,
|
|
projectId,
|
|
model: visionModel || model,
|
|
supportsVision: Boolean(visionModel),
|
|
}),
|
|
});
|
|
const finalText = snapshot.message;
|
|
if (!isVideo)
|
|
await persistRuntimeConversationResults({
|
|
userId: session.user.id,
|
|
chatId: body.id,
|
|
projectId,
|
|
assistantMessageId,
|
|
snapshot,
|
|
}).catch((error) => {
|
|
console.warn("[Canvas Agent] Conversation memory update failed", {
|
|
runId: run.id,
|
|
error: publicAgentError(error),
|
|
});
|
|
});
|
|
await saveMessages({
|
|
messages: [
|
|
{
|
|
id: assistantMessageId,
|
|
chatId: body.id,
|
|
role: "assistant",
|
|
parts: [{ type: "text", text: finalText }],
|
|
attachments: [],
|
|
createdAt: new Date(),
|
|
videoStorageStatus: null,
|
|
temporaryVideoUrl: null,
|
|
permanentVideoUrl: null,
|
|
archivedAt: null,
|
|
videoMetadata: null,
|
|
},
|
|
],
|
|
});
|
|
writer.write({ type: "text-delta", id: textId, delta: finalText });
|
|
} catch (error) {
|
|
const text = publicAgentError(error);
|
|
await checkpointAgentRun(run, { ...snapshot, message: text }, "needs_review").catch(
|
|
() => undefined,
|
|
);
|
|
writer.write({ type: "text-delta", id: textId, delta: text });
|
|
} finally {
|
|
clearInterval(heartbeat);
|
|
writer.write({ type: "text-end", id: textId });
|
|
writer.write({ type: "finish" });
|
|
}
|
|
},
|
|
});
|
|
// Browser disconnection does not cancel paid image work. Keep consuming on the server.
|
|
const [client, background] = stream.tee();
|
|
after(async () => {
|
|
const reader = background.getReader();
|
|
try {
|
|
while (!(await reader.read()).done) {
|
|
/* Drain the server branch. */
|
|
}
|
|
} finally {
|
|
reader.releaseLock();
|
|
await archiveVideoResults?.();
|
|
}
|
|
});
|
|
return createUIMessageStreamResponse({
|
|
stream: client,
|
|
headers: { "Cache-Control": "no-store", "X-Accel-Buffering": "no" },
|
|
});
|
|
} catch (error) {
|
|
return NextResponse.json(
|
|
{ error: publicAgentError(error), message: publicAgentError(error) },
|
|
{
|
|
status:
|
|
error instanceof AgentRunError ? error.status : error instanceof z.ZodError ? 400 : 503,
|
|
},
|
|
);
|
|
}
|
|
}
|