416 lines
13 KiB
TypeScript
416 lines
13 KiB
TypeScript
import { generateText } from "ai";
|
||
import { NextResponse } from "next/server";
|
||
import { z } from "zod";
|
||
import { auth } from "@/app/(auth)/auth";
|
||
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
||
import { inlineVisionImages } from "@/lib/ai/vision-inline";
|
||
import { getAiModelAgentCodingModelId, isAiModelMultimodal } from "@/lib/db/ai-model-flags";
|
||
import {
|
||
getAgentAiModelWithProvider,
|
||
getEnabledAiModelsWithProvider,
|
||
} from "@/lib/db/queries";
|
||
|
||
export const runtime = "nodejs";
|
||
export const maxDuration = 180;
|
||
|
||
const requestSchema = z.object({
|
||
mode: z.enum(["video-analysis", "character-scene-extraction", "agent-plan"]),
|
||
prompt: z.string().trim().optional(),
|
||
imageUrls: z.array(z.string().trim()).optional(),
|
||
videoUrls: z.array(z.string().trim()).optional(),
|
||
nodeTitle: z.string().trim().optional(),
|
||
});
|
||
|
||
function sanitizeUrls(value: unknown, limit = 6) {
|
||
if (!Array.isArray(value)) return [];
|
||
return value
|
||
.map((item) => String(item || "").trim())
|
||
.filter((url) => /^https?:\/\//i.test(url))
|
||
.slice(0, limit);
|
||
}
|
||
|
||
function supportsVision(model: {
|
||
supportsVision?: boolean | null;
|
||
capabilities?: unknown;
|
||
defaultParams?: unknown;
|
||
}) {
|
||
return (
|
||
Boolean(model.supportsVision) ||
|
||
isAiModelMultimodal({
|
||
capabilities: model.capabilities,
|
||
defaultParams: model.defaultParams,
|
||
})
|
||
);
|
||
}
|
||
|
||
function extractJsonBlock<T = Record<string, unknown>>(text: string): T | null {
|
||
const matched = String(text || "").match(/\{[\s\S]*\}/);
|
||
if (!matched) return null;
|
||
try {
|
||
return JSON.parse(matched[0]) as T;
|
||
} catch {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
function normalizeText(value: unknown) {
|
||
return String(value || "").trim();
|
||
}
|
||
|
||
function normalizeStringList(value: unknown, limit = 8) {
|
||
if (!Array.isArray(value)) return [];
|
||
return Array.from(
|
||
new Set(
|
||
value
|
||
.map((item) => String(item || "").trim())
|
||
.filter(Boolean)
|
||
.slice(0, limit)
|
||
)
|
||
);
|
||
}
|
||
|
||
function normalizeShotIdeas(value: unknown) {
|
||
if (!Array.isArray(value)) return [];
|
||
return value
|
||
.map((item, index) => {
|
||
const raw =
|
||
item && typeof item === "object" && !Array.isArray(item)
|
||
? (item as Record<string, unknown>)
|
||
: {};
|
||
const title = normalizeText(raw.title || `镜头 ${index + 1}`);
|
||
const description = normalizeText(raw.description || raw.prompt || raw.summary);
|
||
if (!title && !description) return null;
|
||
return {
|
||
title: title || `镜头 ${index + 1}`,
|
||
description,
|
||
};
|
||
})
|
||
.filter(
|
||
(item): item is { title: string; description: string } => Boolean(item)
|
||
)
|
||
.slice(0, 6);
|
||
}
|
||
|
||
function normalizeCharacters(value: unknown) {
|
||
if (!Array.isArray(value)) return [];
|
||
return value
|
||
.map((item) => {
|
||
const raw =
|
||
item && typeof item === "object" && !Array.isArray(item)
|
||
? (item as Record<string, unknown>)
|
||
: {};
|
||
const name = normalizeText(raw.name || raw.title);
|
||
const description = normalizeText(raw.description || raw.summary);
|
||
const prompt = normalizeText(raw.prompt);
|
||
if (!name && !description && !prompt) return null;
|
||
return {
|
||
name: name || "未命名角色",
|
||
description,
|
||
prompt,
|
||
look: normalizeText(raw.look) || undefined,
|
||
wardrobe: normalizeText(raw.wardrobe) || undefined,
|
||
signatureAction: normalizeText(raw.signatureAction) || undefined,
|
||
traits: normalizeStringList(raw.traits, 6),
|
||
};
|
||
})
|
||
.filter((item): item is NonNullable<typeof item> => Boolean(item))
|
||
.slice(0, 8);
|
||
}
|
||
|
||
function normalizeScenes(value: unknown) {
|
||
if (!Array.isArray(value)) return [];
|
||
return value
|
||
.map((item) => {
|
||
const raw =
|
||
item && typeof item === "object" && !Array.isArray(item)
|
||
? (item as Record<string, unknown>)
|
||
: {};
|
||
const name = normalizeText(raw.name || raw.title);
|
||
const description = normalizeText(raw.description || raw.summary);
|
||
const prompt = normalizeText(raw.prompt);
|
||
if (!name && !description && !prompt) return null;
|
||
return {
|
||
name: name || "未命名场景",
|
||
description,
|
||
prompt,
|
||
mood: normalizeText(raw.mood) || undefined,
|
||
lighting: normalizeText(raw.lighting) || undefined,
|
||
props: normalizeStringList(raw.props, 6),
|
||
};
|
||
})
|
||
.filter((item): item is NonNullable<typeof item> => Boolean(item))
|
||
.slice(0, 8);
|
||
}
|
||
|
||
function buildVideoAnalysisPrompt(params: {
|
||
nodeTitle?: string;
|
||
prompt: string;
|
||
imageCount: number;
|
||
videoCount: number;
|
||
}) {
|
||
return [
|
||
"你是视频画布工作流里的视频分析节点，需要为后续分镜和生成节点提供结构化输出。",
|
||
params.nodeTitle ? `当前节点：${params.nodeTitle}` : "",
|
||
`上游参考图数量：${params.imageCount}`,
|
||
`上游视频数量：${params.videoCount}`,
|
||
params.prompt ? `上游脚本/说明：\n${params.prompt}` : "上游没有额外脚本，请重点依据参考画面分析。",
|
||
"",
|
||
"请只输出 JSON，不要输出 markdown，不要输出解释。",
|
||
"JSON 结构如下：",
|
||
`{
|
||
"summary": "一句话总结当前视频/镜头风格，不超过120字",
|
||
"highlights": ["3-5条镜头分析要点"],
|
||
"shotIdeas": [
|
||
{ "title": "镜头 1", "description": "一句话描述镜头目标、景别或动作" }
|
||
],
|
||
"visualMotifs": ["2-4个可复用的视觉母题"],
|
||
"recommendedNextSteps": ["2-4条建议的下游动作"]
|
||
}`,
|
||
"如果没有真实视频帧，只能依据参考图和脚本推断，也要正常输出，但措辞要稳健。",
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n");
|
||
}
|
||
|
||
function buildCharacterScenePrompt(params: {
|
||
nodeTitle?: string;
|
||
prompt: string;
|
||
imageCount: number;
|
||
}) {
|
||
return [
|
||
"你是视频画布工作流里的角色场景提取节点，需要从脚本、设定和参考图里提取角色与场景资产。",
|
||
params.nodeTitle ? `当前节点：${params.nodeTitle}` : "",
|
||
`参考图数量：${params.imageCount}`,
|
||
params.prompt ? `输入内容：\n${params.prompt}` : "没有额外脚本，请根据参考图提取角色和场景线索。",
|
||
"",
|
||
"请只输出 JSON，不要输出 markdown，不要输出解释。",
|
||
"JSON 结构如下：",
|
||
`{
|
||
"summary": "一句话总结当前故事世界与资产线索",
|
||
"characters": [
|
||
{
|
||
"name": "角色名",
|
||
"description": "角色外观与身份简介",
|
||
"look": "外貌特征",
|
||
"wardrobe": "服装/道具",
|
||
"traits": ["性格关键词"],
|
||
"signatureAction": "代表动作",
|
||
"prompt": "可直接用于生成角色图/角色视频的提示词"
|
||
}
|
||
],
|
||
"scenes": [
|
||
{
|
||
"name": "场景名",
|
||
"description": "空间、布景和氛围简介",
|
||
"mood": "情绪与气氛",
|
||
"lighting": "布光特征",
|
||
"props": ["核心道具"],
|
||
"prompt": "可直接用于生成场景图/场景视频的提示词"
|
||
}
|
||
],
|
||
"relationships": ["角色与场景或角色之间的关系"],
|
||
"recommendedNextSteps": ["建议连接的后续节点动作"]
|
||
}`,
|
||
"角色和场景都要尽量具体，避免空泛形容词。",
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n");
|
||
}
|
||
|
||
function buildAgentPlanPrompt(params: {
|
||
nodeTitle?: string;
|
||
prompt: string;
|
||
imageCount: number;
|
||
videoCount: number;
|
||
}) {
|
||
return [
|
||
"你是视频画布里的自定义代理节点，需要把当前创作目标拆成可执行的工作流计划。",
|
||
params.nodeTitle ? `当前节点：${params.nodeTitle}` : "",
|
||
`参考图数量：${params.imageCount}`,
|
||
`视频数量：${params.videoCount}`,
|
||
params.prompt ? `上游需求：\n${params.prompt}` : "没有额外脚本，请基于当前上下文给出稳妥计划。",
|
||
"",
|
||
"请只输出 JSON，不要输出 markdown，不要输出解释。",
|
||
"JSON 结构如下：",
|
||
`{
|
||
"summary": "一句话说明这个代理链的目标",
|
||
"agentPlan": ["3-6个执行步骤"],
|
||
"deliverables": ["2-4个预期交付物"],
|
||
"recommendedNodes": ["建议继续连接的节点类型或能力名"]
|
||
}`,
|
||
"步骤要偏工作流视角，而不是泛泛而谈。",
|
||
]
|
||
.filter(Boolean)
|
||
.join("\n");
|
||
}
|
||
|
||
export async function POST(request: Request) {
|
||
try {
|
||
const session = await auth();
|
||
if (!session?.user?.id) {
|
||
return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
|
||
}
|
||
|
||
const body = await request.json().catch(() => ({}));
|
||
const parsed = requestSchema.safeParse(body);
|
||
if (!parsed.success) {
|
||
return NextResponse.json(
|
||
{ success: false, error: "请求参数不合法" },
|
||
{ status: 400 }
|
||
);
|
||
}
|
||
|
||
const prompt = normalizeText(parsed.data.prompt);
|
||
const imageUrls = sanitizeUrls(parsed.data.imageUrls, 4);
|
||
const videoUrls = sanitizeUrls(parsed.data.videoUrls, 4);
|
||
if (!prompt && imageUrls.length === 0 && videoUrls.length === 0) {
|
||
return NextResponse.json(
|
||
{ success: false, error: "请至少提供脚本、图片或视频上下文" },
|
||
{ status: 400 }
|
||
);
|
||
}
|
||
|
||
const [allModels, agentModel] = await Promise.all([
|
||
getEnabledAiModelsWithProvider(false),
|
||
getAgentAiModelWithProvider(),
|
||
]);
|
||
const compatibleModels = allModels.filter(
|
||
(item) => !item.model.isSubModel && item.model.isEnabled
|
||
);
|
||
|
||
let selectedModel =
|
||
imageUrls.length > 0 &&
|
||
agentModel &&
|
||
supportsVision(agentModel.model)
|
||
? agentModel
|
||
: imageUrls.length > 0
|
||
? compatibleModels.find((item) => supportsVision(item.model))
|
||
: agentModel || compatibleModels[0];
|
||
|
||
if (!selectedModel) {
|
||
selectedModel = compatibleModels[0];
|
||
}
|
||
|
||
if (!selectedModel) {
|
||
return NextResponse.json(
|
||
{ success: false, error: "当前站点没有可用的分析模型" },
|
||
{ status: 500 }
|
||
);
|
||
}
|
||
|
||
const useVision = imageUrls.length > 0 && supportsVision(selectedModel.model);
|
||
const codingModelId = getAiModelAgentCodingModelId(selectedModel.model);
|
||
const model = await getLanguageModelById(selectedModel.model.id, codingModelId);
|
||
if (!model) {
|
||
return NextResponse.json(
|
||
{ success: false, error: "分析模型初始化失败" },
|
||
{ status: 500 }
|
||
);
|
||
}
|
||
|
||
const promptText =
|
||
parsed.data.mode === "video-analysis"
|
||
? buildVideoAnalysisPrompt({
|
||
nodeTitle: normalizeText(parsed.data.nodeTitle),
|
||
prompt,
|
||
imageCount: imageUrls.length,
|
||
videoCount: videoUrls.length,
|
||
})
|
||
: parsed.data.mode === "character-scene-extraction"
|
||
? buildCharacterScenePrompt({
|
||
nodeTitle: normalizeText(parsed.data.nodeTitle),
|
||
prompt,
|
||
imageCount: imageUrls.length,
|
||
})
|
||
: buildAgentPlanPrompt({
|
||
nodeTitle: normalizeText(parsed.data.nodeTitle),
|
||
prompt,
|
||
imageCount: imageUrls.length,
|
||
videoCount: videoUrls.length,
|
||
});
|
||
|
||
const referenceImages = useVision
|
||
? (
|
||
await inlineVisionImages(imageUrls, {
|
||
maxImages: 4,
|
||
maxSide: 960,
|
||
timeoutMs: 12000,
|
||
retries: 1,
|
||
loggerPrefix: "[Canvas Workflow Analyze]",
|
||
})
|
||
).inlineDataUrls
|
||
: [];
|
||
|
||
const { text } = await generateText({
|
||
model,
|
||
messages: [
|
||
{
|
||
role: "user",
|
||
content: [
|
||
{ type: "text", text: promptText },
|
||
...referenceImages.map((image) => ({ type: "image" as const, image })),
|
||
],
|
||
},
|
||
],
|
||
temperature: 0.2,
|
||
maxTokens: 1600,
|
||
} as any);
|
||
|
||
const parsedJson = extractJsonBlock<Record<string, unknown>>(text) || {};
|
||
const summary =
|
||
normalizeText(parsedJson.summary) ||
|
||
normalizeText(text).slice(0, 160);
|
||
|
||
if (parsed.data.mode === "video-analysis") {
|
||
return NextResponse.json({
|
||
success: true,
|
||
data: {
|
||
summary,
|
||
highlights: normalizeStringList(parsedJson.highlights, 6),
|
||
shotIdeas: normalizeShotIdeas(parsedJson.shotIdeas),
|
||
visualMotifs: normalizeStringList(parsedJson.visualMotifs, 6),
|
||
recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6),
|
||
model: selectedModel.model.displayName,
|
||
raw: text,
|
||
},
|
||
});
|
||
}
|
||
|
||
if (parsed.data.mode === "character-scene-extraction") {
|
||
return NextResponse.json({
|
||
success: true,
|
||
data: {
|
||
summary,
|
||
characters: normalizeCharacters(parsedJson.characters),
|
||
scenes: normalizeScenes(parsedJson.scenes),
|
||
relationships: normalizeStringList(parsedJson.relationships, 8),
|
||
recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6),
|
||
model: selectedModel.model.displayName,
|
||
raw: text,
|
||
},
|
||
});
|
||
}
|
||
|
||
return NextResponse.json({
|
||
success: true,
|
||
data: {
|
||
summary,
|
||
agentPlan: normalizeStringList(parsedJson.agentPlan, 8),
|
||
deliverables: normalizeStringList(parsedJson.deliverables, 6),
|
||
recommendedNodes: normalizeStringList(parsedJson.recommendedNodes, 6),
|
||
model: selectedModel.model.displayName,
|
||
raw: text,
|
||
},
|
||
});
|
||
} catch (error) {
|
||
console.error("[Canvas Workflow Analyze] Error:", error);
|
||
return NextResponse.json(
|
||
{
|
||
success: false,
|
||
error: error instanceof Error ? error.message : "工作流分析失败",
|
||
},
|
||
{ status: 500 }
|
||
);
|
||
}
|
||
}
|