feat: add video agent planning and canvas video workflow
This commit is contained in:
Your Name
parent
c281ef6e02
commit
5b915d9787
@ -196,6 +196,32 @@ type AgentRecommendedImageParams = {
|
||||
promptExtend?: boolean;
|
||||
};
|
||||
|
||||
type ChatVideoParams = NonNullable<PostRequestBody["videoParams"]>;
|
||||
type ChatVideoParamsDraft = Partial<ChatVideoParams>;
|
||||
type AgentRecommendedVideoParams = {
|
||||
mode?: "text-to-video" | "first-frame" | "first-last-frame" | "reference-images";
|
||||
resolution?: "480p" | "720p" | "1080p";
|
||||
ratio?: string;
|
||||
duration?: number;
|
||||
generateAudio?: boolean;
|
||||
cameraMove?: string;
|
||||
};
|
||||
|
||||
type AgentVideoPlan = {
|
||||
script: string;
|
||||
storyboard: Array<{
|
||||
shot: number;
|
||||
title: string;
|
||||
duration: number;
|
||||
description: string;
|
||||
prompt: string;
|
||||
camera?: string;
|
||||
}>;
|
||||
finalPrompt: string;
|
||||
negativePrompt?: string;
|
||||
recommendedVideoParams?: AgentRecommendedVideoParams;
|
||||
};
|
||||
|
||||
function hasUserCustomizedImageParams(params?: PostRequestBody["imageParams"]): boolean {
|
||||
if (!params) return false;
|
||||
const customizedAspectRatio =
|
||||
@ -250,6 +276,69 @@ function mergeImageParamsWithAgentRecommendation(params: {
|
||||
return Object.keys(merged).length > 0 ? merged : undefined;
|
||||
}
|
||||
|
||||
function hasUserCustomizedVideoParams(params?: PostRequestBody["videoParams"]): boolean {
|
||||
if (!params) return false;
|
||||
const defaults = {
|
||||
mode: "text-to-video",
|
||||
resolution: "720p",
|
||||
ratio: "16:9",
|
||||
duration: 5,
|
||||
generateAudio: true,
|
||||
cameraMove: "none",
|
||||
} as const;
|
||||
|
||||
if (params.mode && params.mode !== defaults.mode) return true;
|
||||
if (params.resolution && params.resolution !== defaults.resolution) return true;
|
||||
if (params.ratio && params.ratio !== defaults.ratio) return true;
|
||||
if (typeof params.duration === "number" && params.duration !== defaults.duration) return true;
|
||||
if (typeof params.generateAudio === "boolean" && params.generateAudio !== defaults.generateAudio)
|
||||
return true;
|
||||
if (typeof params.cameraMove === "string" && params.cameraMove !== defaults.cameraMove)
|
||||
return true;
|
||||
if (typeof params.seed === "number") return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
function mergeVideoParamsWithAgentRecommendation(params: {
|
||||
incoming?: PostRequestBody["videoParams"];
|
||||
recommended?: AgentRecommendedVideoParams;
|
||||
hasUserCustomSettings: boolean;
|
||||
}): PostRequestBody["videoParams"] {
|
||||
const { incoming, recommended, hasUserCustomSettings } = params;
|
||||
const base: ChatVideoParamsDraft = { ...(incoming || {}) };
|
||||
if (!recommended) {
|
||||
return Object.keys(base).length > 0 ? (base as ChatVideoParams) : undefined;
|
||||
}
|
||||
|
||||
const merged: ChatVideoParamsDraft = { ...base };
|
||||
if (!hasUserCustomSettings) {
|
||||
if (recommended.mode) merged.mode = recommended.mode;
|
||||
if (recommended.resolution) merged.resolution = recommended.resolution;
|
||||
if (recommended.ratio) merged.ratio = recommended.ratio;
|
||||
if (typeof recommended.duration === "number") merged.duration = recommended.duration;
|
||||
if (typeof recommended.generateAudio === "boolean") {
|
||||
merged.generateAudio = recommended.generateAudio;
|
||||
}
|
||||
if (recommended.cameraMove) merged.cameraMove = recommended.cameraMove;
|
||||
}
|
||||
|
||||
if (!merged.mode) {
|
||||
merged.mode = recommended.mode || (incoming?.mode as ChatVideoParams["mode"]) || "text-to-video";
|
||||
}
|
||||
if (!merged.resolution) {
|
||||
merged.resolution =
|
||||
recommended.resolution || (incoming?.resolution as ChatVideoParams["resolution"]) || "720p";
|
||||
}
|
||||
if (!merged.ratio) {
|
||||
merged.ratio = recommended.ratio || incoming?.ratio || "16:9";
|
||||
}
|
||||
if (typeof merged.duration !== "number") {
|
||||
merged.duration = recommended.duration || incoming?.duration || 5;
|
||||
}
|
||||
|
||||
return Object.keys(merged).length > 0 ? (merged as ChatVideoParams) : undefined;
|
||||
}
|
||||
|
||||
function sanitizeAgentOutputSettingsSection(params: {
|
||||
message?: string;
|
||||
userImageParams?: PostRequestBody["imageParams"];
|
||||
@ -942,6 +1031,8 @@ export async function POST(request: Request) {
|
||||
let smartAutoFriendlyMessage: string | undefined; // AI的自然对话回复
|
||||
let smartAutoOptimizedPrompt: string | undefined; // 优化后的提示词（不修改用户消息，只用于API调用）
|
||||
let smartAutoRecommendedImageParams: AgentRecommendedImageParams | undefined;
|
||||
let smartAutoVideoPlan: AgentVideoPlan | undefined;
|
||||
let smartAutoRecommendedVideoParams: AgentRecommendedVideoParams | undefined;
|
||||
let smartAutoDetectedIntent: "image_edit" | "vision" | "image_gen" | "video_gen" | "chat" | undefined;
|
||||
let smartAutoHistoryImages: string[] = []; // 从历史对话中提取的图片URL（支持多张）
|
||||
let smartAutoNeedsClarification = false; // 是否需要先追问用户
|
||||
@ -985,9 +1076,17 @@ export async function POST(request: Request) {
|
||||
"fal-image",
|
||||
"volcano-vision",
|
||||
]);
|
||||
const isFixedImageModel =
|
||||
const fixedVideoProviderTypes = new Set([
|
||||
"volcano-video",
|
||||
"qianwen-video",
|
||||
"fal-video",
|
||||
]);
|
||||
const isFixedVisualModel =
|
||||
!selectedModelConfig.isSubModel &&
|
||||
(selectedModelConfig.supportsImageGen ||
|
||||
(selectedModelProviderType
|
||||
? fixedVideoProviderTypes.has(selectedModelProviderType)
|
||||
: false) ||
|
||||
(selectedModelProviderType
|
||||
? fixedImageProviderTypes.has(selectedModelProviderType)
|
||||
: false));
|
||||
@ -998,11 +1097,11 @@ export async function POST(request: Request) {
|
||||
console.log(
|
||||
`[Smart Auto Mode] 🧠 Agent-only model detected, force auto routing: ${selectedModelConfig.displayName}`
|
||||
);
|
||||
} else if (isFixedImageModel) {
|
||||
} else if (isFixedVisualModel) {
|
||||
shouldRunAgentConversation = true;
|
||||
shouldKeepSelectedModel = true;
|
||||
console.log(
|
||||
`[Smart Auto Mode] 🎯 Agent conversation enabled for fixed image model: ${selectedModelConfig.displayName} (${selectedModelProviderType || "unknown"})`
|
||||
`[Smart Auto Mode] 🎯 Agent conversation enabled for fixed visual model: ${selectedModelConfig.displayName} (${selectedModelProviderType || "unknown"})`
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -1253,6 +1352,19 @@ export async function POST(request: Request) {
|
||||
optimized: autoGraphResult.optimizedPrompt.substring(0, 100),
|
||||
});
|
||||
}
|
||||
if (autoGraphResult.videoPlan) {
|
||||
smartAutoVideoPlan = autoGraphResult.videoPlan;
|
||||
smartAutoRecommendedVideoParams = autoGraphResult.videoPlan.recommendedVideoParams;
|
||||
if (!smartAutoOptimizedPrompt && autoGraphResult.videoPlan.finalPrompt?.trim()) {
|
||||
smartAutoOptimizedPrompt = autoGraphResult.videoPlan.finalPrompt.trim();
|
||||
}
|
||||
console.log("[Smart Auto Mode] 🎬 Agent video plan ready:", {
|
||||
hasScript: Boolean(autoGraphResult.videoPlan.script),
|
||||
storyboardCount: autoGraphResult.videoPlan.storyboard?.length || 0,
|
||||
hasFinalPrompt: Boolean(autoGraphResult.videoPlan.finalPrompt),
|
||||
recommendedVideoParams: smartAutoRecommendedVideoParams,
|
||||
});
|
||||
}
|
||||
if (autoGraphResult.recommendedImageParams) {
|
||||
smartAutoRecommendedImageParams = autoGraphResult.recommendedImageParams;
|
||||
console.log("[Smart Auto Mode] 🎛️ Agent recommended image params:", smartAutoRecommendedImageParams);
|
||||
@ -1348,10 +1460,15 @@ export async function POST(request: Request) {
|
||||
}
|
||||
|
||||
const hasUserCustomImageParams = hasUserCustomizedImageParams(requestBody.imageParams);
|
||||
const hasUserCustomVideoParams = hasUserCustomizedVideoParams(requestBody.videoParams);
|
||||
const shouldApplyAgentImageParams =
|
||||
shouldRunAgentConversation &&
|
||||
!smartAutoNeedsClarification &&
|
||||
(smartAutoDetectedIntent === "image_gen" || smartAutoDetectedIntent === "image_edit");
|
||||
const shouldApplyAgentVideoParams =
|
||||
shouldRunAgentConversation &&
|
||||
!smartAutoNeedsClarification &&
|
||||
smartAutoDetectedIntent === "video_gen";
|
||||
const effectiveImageParams = shouldApplyAgentImageParams
|
||||
? mergeImageParamsWithAgentRecommendation({
|
||||
incoming: requestBody.imageParams,
|
||||
@ -1360,6 +1477,13 @@ export async function POST(request: Request) {
|
||||
hasUserCustomSettings: hasUserCustomImageParams,
|
||||
})
|
||||
: requestBody.imageParams;
|
||||
const effectiveVideoParams = shouldApplyAgentVideoParams
|
||||
? mergeVideoParamsWithAgentRecommendation({
|
||||
incoming: requestBody.videoParams,
|
||||
recommended: smartAutoRecommendedVideoParams,
|
||||
hasUserCustomSettings: hasUserCustomVideoParams,
|
||||
})
|
||||
: requestBody.videoParams;
|
||||
const defaultImageQualityParams = {
|
||||
aspectRatio: "auto",
|
||||
imageSize: smartAutoDetectedIntent === "image_edit" ? "4K" : "2K",
|
||||
@ -1374,6 +1498,15 @@ export async function POST(request: Request) {
|
||||
hasUserCustomImageParams,
|
||||
});
|
||||
}
|
||||
if (shouldApplyAgentVideoParams) {
|
||||
console.log("[Smart Auto Mode] 🎯 Effective video params:", {
|
||||
incoming: requestBody.videoParams,
|
||||
recommended: smartAutoRecommendedVideoParams,
|
||||
merged: effectiveVideoParams,
|
||||
hasUserCustomVideoParams,
|
||||
hasVideoPlan: Boolean(smartAutoVideoPlan),
|
||||
});
|
||||
}
|
||||
|
||||
if (shouldRunAgentConversation && smartAutoFriendlyMessage) {
|
||||
smartAutoFriendlyMessage = sanitizeAgentOutputSettingsSection({
|
||||
@ -1591,7 +1724,7 @@ export async function POST(request: Request) {
|
||||
|
||||
if (subModels.length > 0) {
|
||||
// 这是一个主模型，需要根据 videoParams 选择子模型
|
||||
const videoParams = requestBody.videoParams;
|
||||
const videoParams = effectiveVideoParams;
|
||||
console.log("[Chat API] Video params:", videoParams);
|
||||
|
||||
// 如果没有 videoParams 或没有 mode，使用默认模式
|
||||
@ -2799,14 +2932,54 @@ export async function POST(request: Request) {
|
||||
const qianwenVideoStream = createUIMessageStream({
|
||||
execute: async ({ writer }) => {
|
||||
try {
|
||||
// 千问视频直接使用原始 URL，不转换为 base64
|
||||
const modelMessages = convertToModelMessages(uiMessagesForModel);
|
||||
const conversationMessage = smartAutoFriendlyMessage || null;
|
||||
if (conversationMessage) {
|
||||
const chatMessageId = generateUUID();
|
||||
writer.write({
|
||||
type: "data-appendMessage",
|
||||
data: JSON.stringify({
|
||||
id: chatMessageId,
|
||||
role: "assistant",
|
||||
parts: [{ type: "text", text: conversationMessage }],
|
||||
}),
|
||||
});
|
||||
|
||||
await saveMessages({
|
||||
messages: [
|
||||
{
|
||||
id: chatMessageId,
|
||||
chatId: id,
|
||||
role: "assistant",
|
||||
parts: [{ type: "text", text: conversationMessage }],
|
||||
attachments: [],
|
||||
createdAt: new Date(),
|
||||
videoStorageStatus: null,
|
||||
temporaryVideoUrl: null,
|
||||
permanentVideoUrl: null,
|
||||
archivedAt: null,
|
||||
videoMetadata: null,
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
const messagesForVideo = injectHistoryImagesToLastUserMessage(
|
||||
applyOptimizedPromptToLatestUserMessage(
|
||||
uiMessagesForModel,
|
||||
smartAutoOptimizedPrompt,
|
||||
hiddenPromptPrefix,
|
||||
),
|
||||
smartAutoHistoryImages,
|
||||
);
|
||||
|
||||
// 千问/FAL 视频直接使用原始 URL，不转换为 base64
|
||||
const modelMessages = convertToModelMessages(messagesForVideo);
|
||||
console.log(`[Chat API] ${currentVideoProvider} messages prepared, count:`, modelMessages.length);
|
||||
|
||||
// 调用 doGenerate 获取视频
|
||||
const result = await (languageModel as any).doGenerate({
|
||||
messages: modelMessages,
|
||||
videoParams: requestBody.videoParams,
|
||||
videoParams: effectiveVideoParams,
|
||||
});
|
||||
|
||||
const videoMarkdown = result.text || "视频生成失败";
|
||||
@ -2847,7 +3020,16 @@ export async function POST(request: Request) {
|
||||
temporaryVideoUrl: null,
|
||||
permanentVideoUrl: null,
|
||||
archivedAt: null,
|
||||
videoMetadata: null,
|
||||
videoMetadata: effectiveVideoParams
|
||||
? {
|
||||
provider: currentVideoProvider,
|
||||
mode: effectiveVideoParams.mode,
|
||||
resolution: effectiveVideoParams.resolution,
|
||||
ratio: effectiveVideoParams.ratio,
|
||||
duration: effectiveVideoParams.duration,
|
||||
createdAt: Date.now(),
|
||||
}
|
||||
: null,
|
||||
},
|
||||
],
|
||||
});
|
||||
@ -3061,9 +3243,50 @@ export async function POST(request: Request) {
|
||||
|
||||
// 导入视频生成函数
|
||||
const { doGenerateStream } = await import("@/lib/ai/providers/volcano-video");
|
||||
|
||||
const conversationMessage = smartAutoFriendlyMessage || null;
|
||||
if (conversationMessage) {
|
||||
const chatMessageId = generateUUID();
|
||||
dataStream.write({
|
||||
type: "data-appendMessage",
|
||||
data: JSON.stringify({
|
||||
id: chatMessageId,
|
||||
role: "assistant",
|
||||
parts: [{ type: "text", text: conversationMessage }],
|
||||
attachments: [],
|
||||
}),
|
||||
});
|
||||
|
||||
await saveMessages({
|
||||
messages: [
|
||||
{
|
||||
id: chatMessageId,
|
||||
chatId: id,
|
||||
role: "assistant",
|
||||
parts: [{ type: "text", text: conversationMessage }],
|
||||
attachments: [],
|
||||
createdAt: new Date(),
|
||||
videoStorageStatus: null,
|
||||
temporaryVideoUrl: null,
|
||||
permanentVideoUrl: null,
|
||||
archivedAt: null,
|
||||
videoMetadata: null,
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
const messagesForVideo = injectHistoryImagesToLastUserMessage(
|
||||
applyOptimizedPromptToLatestUserMessage(
|
||||
uiMessagesForModel,
|
||||
smartAutoOptimizedPrompt,
|
||||
hiddenPromptPrefix,
|
||||
),
|
||||
smartAutoHistoryImages,
|
||||
);
|
||||
|
||||
// 从最后一条用户消息中提取提示词和图片
|
||||
const lastUserMessage = uiMessages.filter(m => m.role === "user").pop();
|
||||
const lastUserMessage = messagesForVideo.filter(m => m.role === "user").pop();
|
||||
let prompt = "";
|
||||
const imageUrls: string[] = [];
|
||||
|
||||
@ -3120,7 +3343,7 @@ export async function POST(request: Request) {
|
||||
console.log("[Chat API] Video generation params:", {
|
||||
prompt,
|
||||
imageCount: imageUrls.length,
|
||||
videoParams: requestBody.videoParams,
|
||||
videoParams: effectiveVideoParams,
|
||||
});
|
||||
|
||||
// 🆕 重新执行子模型选择逻辑，确定要使用的模型 UUID
|
||||
@ -3134,7 +3357,7 @@ export async function POST(request: Request) {
|
||||
console.log(`[Chat API] Video: Found ${videoSubModels.length} sub-models for master model`);
|
||||
|
||||
if (videoSubModels.length > 0) {
|
||||
const videoMode = requestBody.videoParams?.mode || "text-to-video";
|
||||
const videoMode = effectiveVideoParams?.mode || "text-to-video";
|
||||
const modeToModelMap: Record<string, string> = {
|
||||
"text-to-video": "lite-t2v",
|
||||
"first-last-frame": "pro",
|
||||
@ -3161,15 +3384,15 @@ export async function POST(request: Request) {
|
||||
const videoOptions: any = {
|
||||
model: actualModelId,
|
||||
prompt: prompt || "生成视频",
|
||||
mode: requestBody.videoParams?.mode || "text-to-video",
|
||||
resolution: requestBody.videoParams?.resolution || "720p",
|
||||
ratio: requestBody.videoParams?.ratio || "16:9",
|
||||
duration: requestBody.videoParams?.duration || 5,
|
||||
fps: requestBody.videoParams?.fps || 24,
|
||||
cameraMove: requestBody.videoParams?.cameraMove || "none",
|
||||
cameraFixed: requestBody.videoParams?.cameraFixed || false,
|
||||
watermark: requestBody.videoParams?.watermark || false,
|
||||
seed: requestBody.videoParams?.seed,
|
||||
mode: effectiveVideoParams?.mode || "text-to-video",
|
||||
resolution: effectiveVideoParams?.resolution || "720p",
|
||||
ratio: effectiveVideoParams?.ratio || "16:9",
|
||||
duration: effectiveVideoParams?.duration || 5,
|
||||
fps: effectiveVideoParams?.fps || 24,
|
||||
cameraMove: effectiveVideoParams?.cameraMove || "none",
|
||||
cameraFixed: effectiveVideoParams?.cameraFixed || false,
|
||||
watermark: effectiveVideoParams?.watermark || false,
|
||||
seed: effectiveVideoParams?.seed,
|
||||
};
|
||||
|
||||
console.log("[Chat API] Video options constructed:", videoOptions);
|
||||
@ -3252,10 +3475,10 @@ export async function POST(request: Request) {
|
||||
// 构建视频元数据
|
||||
const videoMetadata = temporaryVideoUrl ? {
|
||||
provider: "volcano-video",
|
||||
mode: requestBody.videoParams?.mode,
|
||||
resolution: requestBody.videoParams?.resolution,
|
||||
ratio: requestBody.videoParams?.ratio,
|
||||
duration: requestBody.videoParams?.duration,
|
||||
mode: effectiveVideoParams?.mode,
|
||||
resolution: effectiveVideoParams?.resolution,
|
||||
ratio: effectiveVideoParams?.ratio,
|
||||
duration: effectiveVideoParams?.duration,
|
||||
createdAt: Date.now(),
|
||||
} : null;
|
||||
|
||||
|
||||
@ -24,6 +24,7 @@ import { extractImageUrlsFromText } from "@/lib/image-url";
|
||||
import type { Attachment, ChatMessage } from "@/lib/types";
|
||||
import { type CanvasLaunchPayload, getCanvasLaunchStorageKey } from "@/lib/canvas-launch";
|
||||
import { ImageParamsBar, type ImageGenerationParams } from "@/components/image-params-bar";
|
||||
import { VideoParamsBar, type VideoGenerationParams } from "@/components/video-params-bar";
|
||||
import { ImageGenerationLoading } from "@/components/image-generation-loading";
|
||||
|
||||
// 标签悬停预览信息
|
||||
@ -105,6 +106,98 @@ function extractImageUrls(text: string): string[] {
|
||||
return extractImageUrlsFromText(text);
|
||||
}
|
||||
|
||||
function extractVideoUrls(text: string): string[] {
|
||||
if (!text) return [];
|
||||
const urls = new Set<string>();
|
||||
|
||||
const videoTagPattern = /<video[^>]+src="([^"]+)"/gi;
|
||||
let match: RegExpExecArray | null = null;
|
||||
while ((match = videoTagPattern.exec(text)) !== null) {
|
||||
if (match[1]) urls.add(match[1]);
|
||||
}
|
||||
|
||||
const markdownVideoPattern = /\[[^\]]*]\((https?:\/\/[^)\s]+\.mp4(?:\?[^)\s]*)?)\)/gi;
|
||||
while ((match = markdownVideoPattern.exec(text)) !== null) {
|
||||
if (match[1]) urls.add(match[1]);
|
||||
}
|
||||
|
||||
const plainVideoPattern = /(https?:\/\/[^\s"'<>]+\.mp4(?:\?[^\s"'<>]+)?)/gi;
|
||||
while ((match = plainVideoPattern.exec(text)) !== null) {
|
||||
if (match[1]) urls.add(match[1]);
|
||||
}
|
||||
|
||||
return Array.from(urls);
|
||||
}
|
||||
|
||||
function normalizeVideoParamsForProvider(
|
||||
params: VideoGenerationParams | null | undefined,
|
||||
providerType?: string
|
||||
): VideoGenerationParams {
|
||||
const isFalVideo = providerType === "fal-video";
|
||||
const base: VideoGenerationParams = {
|
||||
mode: isFalVideo ? "first-frame" : "text-to-video",
|
||||
resolution: "720p",
|
||||
ratio: "16:9",
|
||||
duration: 5,
|
||||
generateAudio: true,
|
||||
watermark: false,
|
||||
cameraFixed: false,
|
||||
cameraMove: "none",
|
||||
fps: 24,
|
||||
};
|
||||
const merged = { ...base, ...(params || {}) };
|
||||
|
||||
if (isFalVideo) {
|
||||
if (merged.mode !== "first-frame" && merged.mode !== "first-last-frame") {
|
||||
merged.mode = "first-frame";
|
||||
}
|
||||
if (!["16:9", "9:16", "1:1"].includes(merged.ratio)) {
|
||||
merged.ratio = "16:9";
|
||||
}
|
||||
merged.duration = Math.max(3, Math.min(15, Math.round(Number(merged.duration) || 5)));
|
||||
merged.generateAudio = merged.generateAudio ?? true;
|
||||
return merged;
|
||||
}
|
||||
|
||||
if (!["text-to-video", "first-frame", "first-last-frame", "reference-images"].includes(merged.mode)) {
|
||||
merged.mode = "text-to-video";
|
||||
}
|
||||
if (!["21:9", "16:9", "4:3", "1:1", "3:4", "9:16"].includes(merged.ratio)) {
|
||||
merged.ratio = "16:9";
|
||||
}
|
||||
merged.duration = merged.duration >= 8 ? 10 : 5;
|
||||
return merged;
|
||||
}
|
||||
|
||||
function getVideoImageLimit(
|
||||
params: VideoGenerationParams | null | undefined,
|
||||
providerType?: string
|
||||
): { min: number; max: number } {
|
||||
if (!params) {
|
||||
return { min: 0, max: 0 };
|
||||
}
|
||||
const isFalVideo = providerType === "fal-video";
|
||||
if (isFalVideo) {
|
||||
if (params.mode === "first-last-frame") {
|
||||
return { min: 2, max: 2 };
|
||||
}
|
||||
return { min: 1, max: 1 };
|
||||
}
|
||||
|
||||
switch (params.mode) {
|
||||
case "text-to-video":
|
||||
return { min: 0, max: 0 };
|
||||
case "first-last-frame":
|
||||
return { min: 2, max: 2 };
|
||||
case "first-frame":
|
||||
return { min: 1, max: 1 };
|
||||
case "reference-images":
|
||||
return { min: 1, max: 4 };
|
||||
default:
|
||||
return { min: 0, max: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
function extractUserTextPrompt(message: ChatMessage): string {
|
||||
const parts = (message as any)?.parts || [];
|
||||
const textFromParts = parts
|
||||
@ -227,6 +320,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
imageSize: "2K",
|
||||
imageCount: 1,
|
||||
});
|
||||
const [videoParams, setVideoParams] = useState<VideoGenerationParams | null>(null);
|
||||
|
||||
// 🆕 保存图片参数到 localStorage
|
||||
const handleImageParamsChange = (params: ImageGenerationParams) => {
|
||||
@ -251,6 +345,26 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
}
|
||||
}, []);
|
||||
|
||||
const handleVideoParamsChange = (params: VideoGenerationParams) => {
|
||||
setVideoParams(params);
|
||||
try {
|
||||
localStorage.setItem("canvas-video-params", JSON.stringify(params));
|
||||
} catch (e) {
|
||||
console.error("Failed to save video params:", e);
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
try {
|
||||
const saved = localStorage.getItem("canvas-video-params");
|
||||
if (saved) {
|
||||
setVideoParams(JSON.parse(saved));
|
||||
}
|
||||
} catch (e) {
|
||||
console.error("Failed to load video params:", e);
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (!fullscreenPreview) return;
|
||||
const handleEsc = (event: KeyboardEvent) => {
|
||||
@ -711,6 +825,7 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
providerType: string;
|
||||
supportsVision?: boolean;
|
||||
supportsImageGen?: boolean;
|
||||
capabilities?: string[] | null;
|
||||
}>;
|
||||
}>("/api/models", fetcher);
|
||||
|
||||
@ -723,8 +838,11 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
canvasDefaultModelId?: string | null;
|
||||
}>("/api/site-settings", fetcher);
|
||||
|
||||
// 过滤支持图片生成的模型
|
||||
const imageModels = modelsData?.models?.filter(m => m.supportsImageGen) || [];
|
||||
const VIDEO_PROVIDER_TYPES = new Set(["volcano-video", "qianwen-video", "fal-video"]);
|
||||
const visualModels =
|
||||
modelsData?.models?.filter(
|
||||
(model) => model.supportsImageGen || VIDEO_PROVIDER_TYPES.has(model.providerType)
|
||||
) || [];
|
||||
|
||||
// Agent模式（自动路由）模型选项
|
||||
const SMART_AUTO_MODEL = {
|
||||
@ -736,8 +854,10 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
supportsImageGen: true,
|
||||
};
|
||||
|
||||
// 所有可选模型（Agent模式 + 生图模型）
|
||||
const allModels = [SMART_AUTO_MODEL, ...imageModels];
|
||||
// 所有可选模型（Agent模式 + 可视化模型）
|
||||
const allModels = [SMART_AUTO_MODEL, ...visualModels];
|
||||
const currentModel = allModels.find((model) => model.id === selectedModelId);
|
||||
const isVideoModel = Boolean(currentModel && VIDEO_PROVIDER_TYPES.has(currentModel.providerType));
|
||||
|
||||
// 默认选择模型（根据系统设置）
|
||||
useEffect(() => {
|
||||
@ -750,6 +870,8 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
|
||||
const currentModelIdRef = useRef(selectedModelId);
|
||||
const imageParamsRef = useRef(imageParams);
|
||||
const videoParamsRef = useRef(videoParams);
|
||||
const isVideoModelRef = useRef(isVideoModel);
|
||||
|
||||
useEffect(() => {
|
||||
currentModelIdRef.current = selectedModelId;
|
||||
@ -759,6 +881,22 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
imageParamsRef.current = imageParams;
|
||||
}, [imageParams]);
|
||||
|
||||
useEffect(() => {
|
||||
videoParamsRef.current = videoParams;
|
||||
}, [videoParams]);
|
||||
|
||||
useEffect(() => {
|
||||
isVideoModelRef.current = isVideoModel;
|
||||
}, [isVideoModel]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!isVideoModel) return;
|
||||
const normalized = normalizeVideoParamsForProvider(videoParams, currentModel?.providerType);
|
||||
if (!videoParams || JSON.stringify(videoParams) !== JSON.stringify(normalized)) {
|
||||
setVideoParams(normalized);
|
||||
}
|
||||
}, [isVideoModel, currentModel?.providerType, videoParams]);
|
||||
|
||||
// 合并 initialMessages 和历史消息
|
||||
const mergedInitialMessages = historyData?.messages ?? (activeChatId === chatId ? initialMessages : []);
|
||||
|
||||
@ -798,8 +936,8 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
message: request.messages.at(-1),
|
||||
selectedChatModel: currentModelIdRef.current,
|
||||
selectedVisibilityType: "private",
|
||||
videoParams: undefined, // 必须传递，即使是 undefined
|
||||
imageParams: imageParamsRef.current || undefined, // 传递图片参数
|
||||
videoParams: isVideoModelRef.current ? videoParamsRef.current || undefined : undefined,
|
||||
imageParams: isVideoModelRef.current ? undefined : imageParamsRef.current || undefined,
|
||||
hiddenPrompt,
|
||||
autoContext,
|
||||
...request.body,
|
||||
@ -974,11 +1112,16 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
if (message.role === 'assistant' && !processedMessageIds.current.has(message.id)) {
|
||||
const content = message.parts?.map((p: any) => p.text || '').join('') || '';
|
||||
const imageUrls = extractImageUrls(content);
|
||||
const videoUrls = extractVideoUrls(content);
|
||||
const generationPrompt = findNearestUserPrompt(messages, messageIndex);
|
||||
|
||||
if (imageUrls.length > 0) {
|
||||
|
||||
if (imageUrls.length > 0 || videoUrls.length > 0) {
|
||||
processedMessageIds.current.add(message.id);
|
||||
|
||||
|
||||
let addedImageCount = 0;
|
||||
let addedVideoCount = 0;
|
||||
|
||||
if (imageUrls.length > 0) {
|
||||
// 检查画布中是否已存在相同 URL 的图片
|
||||
const existingUrls = new Set(
|
||||
nodes.filter(n => n.type === 'image' && (n.data as any)?.src)
|
||||
@ -1017,10 +1160,45 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
},
|
||||
connections: [],
|
||||
});
|
||||
addedImageCount += 1;
|
||||
}
|
||||
|
||||
if (newUrls.length > 0) {
|
||||
toast.success(`已将 ${newUrls.length} 张图片添加到画布`);
|
||||
}
|
||||
|
||||
if (videoUrls.length > 0) {
|
||||
const existingVideoUrls = new Set(
|
||||
nodes.filter(n => n.type === 'video' && (n.data as any)?.src)
|
||||
.map(n => (n.data as any).src)
|
||||
);
|
||||
const newVideoUrls = videoUrls.filter(url => !existingVideoUrls.has(url));
|
||||
|
||||
for (let index = 0; index < newVideoUrls.length; index++) {
|
||||
const url = newVideoUrls[index];
|
||||
const existingNodes = nodes.length + index;
|
||||
const x = 120 + (existingNodes % 2) * 420;
|
||||
const y = 120 + Math.floor(existingNodes / 2) * 260;
|
||||
addNode({
|
||||
type: "video",
|
||||
position: { x, y },
|
||||
size: { width: 380, height: 220 },
|
||||
data: {
|
||||
src: url,
|
||||
status: "completed",
|
||||
prompt: generationPrompt || "对话生成",
|
||||
},
|
||||
connections: [],
|
||||
});
|
||||
addedVideoCount += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (addedImageCount > 0 || addedVideoCount > 0) {
|
||||
const notice = [
|
||||
addedImageCount > 0 ? `${addedImageCount} 张图片` : "",
|
||||
addedVideoCount > 0 ? `${addedVideoCount} 条视频` : "",
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("、");
|
||||
toast.success(`已将 ${notice} 添加到画布`);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1217,7 +1395,15 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
|
||||
// 🆕 如果没有附件，自动使用最近的图片（支持多张）
|
||||
let effectiveAttachments = [...attachments];
|
||||
if (effectiveAttachments.length === 0 && input.trim()) {
|
||||
const videoImageLimit = isVideoModel
|
||||
? getVideoImageLimit(videoParams, currentModel?.providerType)
|
||||
: null;
|
||||
|
||||
if (
|
||||
effectiveAttachments.length === 0 &&
|
||||
input.trim() &&
|
||||
(!videoImageLimit || videoImageLimit.max > 0)
|
||||
) {
|
||||
const recentAttachments = getRecentImageAttachments();
|
||||
if (recentAttachments.length > 0) {
|
||||
// 自动添加所有最近的图片作为附件
|
||||
@ -1231,6 +1417,23 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
console.log('[CanvasChat] 自动使用最近图片:', recentAttachments.length, '张');
|
||||
}
|
||||
}
|
||||
|
||||
if (videoImageLimit) {
|
||||
if (videoImageLimit.max === 0) {
|
||||
effectiveAttachments = [];
|
||||
} else if (effectiveAttachments.length > videoImageLimit.max) {
|
||||
effectiveAttachments = effectiveAttachments.slice(0, videoImageLimit.max);
|
||||
}
|
||||
|
||||
if (effectiveAttachments.length < videoImageLimit.min) {
|
||||
toast.error(
|
||||
videoImageLimit.min === 1
|
||||
? "当前视频模式需要至少 1 张参考图"
|
||||
: "当前视频模式需要 2 张首尾帧图片"
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// 🆕 检查是否有标注的图片
|
||||
const annotatedNodes = selectedImageNodes.filter(node => (node.data as any)?.hasAnnotations);
|
||||
@ -1327,7 +1530,19 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
editableDiv.innerHTML = '';
|
||||
console.log('[CanvasChat] 已清空输入框和选中状态');
|
||||
}
|
||||
}, [input, attachments, selectedModelId, sendMessage, getRecentImageAttachments, clearSelection, buildAutoContext]);
|
||||
}, [
|
||||
input,
|
||||
attachments,
|
||||
selectedModelId,
|
||||
sendMessage,
|
||||
getRecentImageAttachments,
|
||||
clearSelection,
|
||||
buildAutoContext,
|
||||
isVideoModel,
|
||||
videoParams,
|
||||
currentModel?.providerType,
|
||||
selectedImageNodes,
|
||||
]);
|
||||
|
||||
// 处理文件选择 - 上传后立即添加到画布
|
||||
const handleFileSelect = async (e: React.ChangeEvent<HTMLInputElement>) => {
|
||||
@ -1552,7 +1767,6 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
(part: any) => part.type === "text" && part.text?.includes("<!-- IMAGE_GENERATION_END -->")
|
||||
)
|
||||
);
|
||||
const currentModel = allModels.find(m => m.id === selectedModelId);
|
||||
const shouldShowImageLoading = hasGenerationStart && !hasGenerationEnd && !!currentModel;
|
||||
const loadingImageCount = imageParams.imageCount || 1;
|
||||
|
||||
@ -2118,13 +2332,23 @@ export function CanvasChatPanel({ chatId, initialMessages = [], onClose, onOpenF
|
||||
|
||||
{/* 中间：图片参数（可收缩） */}
|
||||
<div className="shrink-0">
|
||||
<ImageParamsBar
|
||||
modelName={allModels.find(m => m.id === selectedModelId)?.displayName}
|
||||
initialParams={imageParams}
|
||||
onParamsChange={handleImageParamsChange}
|
||||
inline={true}
|
||||
providerType={allModels.find(m => m.id === selectedModelId)?.providerType}
|
||||
/>
|
||||
{isVideoModel ? (
|
||||
<VideoParamsBar
|
||||
modelName={currentModel?.displayName}
|
||||
initialParams={videoParams || undefined}
|
||||
onParamsChange={handleVideoParamsChange}
|
||||
inline={true}
|
||||
providerType={currentModel?.providerType}
|
||||
/>
|
||||
) : (
|
||||
<ImageParamsBar
|
||||
modelName={currentModel?.displayName}
|
||||
initialParams={imageParams}
|
||||
onParamsChange={handleImageParamsChange}
|
||||
inline={true}
|
||||
providerType={currentModel?.providerType}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
@ -16,6 +16,32 @@ export interface AgentRecommendedImageParams {
|
||||
promptExtend?: boolean;
|
||||
}
|
||||
|
||||
export interface AgentRecommendedVideoParams {
|
||||
mode?: "text-to-video" | "first-frame" | "first-last-frame" | "reference-images";
|
||||
resolution?: "480p" | "720p" | "1080p";
|
||||
ratio?: "16:9" | "4:3" | "1:1" | "3:4" | "9:16" | "21:9" | "adaptive";
|
||||
duration?: number;
|
||||
generateAudio?: boolean;
|
||||
cameraMove?: string;
|
||||
}
|
||||
|
||||
export interface AgentVideoStoryboardShot {
|
||||
shot: number;
|
||||
title: string;
|
||||
duration: number;
|
||||
description: string;
|
||||
prompt: string;
|
||||
camera?: string;
|
||||
}
|
||||
|
||||
export interface AgentVideoPlan {
|
||||
script: string;
|
||||
storyboard: AgentVideoStoryboardShot[];
|
||||
finalPrompt: string;
|
||||
negativePrompt?: string;
|
||||
recommendedVideoParams?: AgentRecommendedVideoParams;
|
||||
}
|
||||
|
||||
export interface AutoAgentInput {
|
||||
text: string;
|
||||
hasImage: boolean;
|
||||
@ -30,6 +56,7 @@ export interface AutoAgentResult {
|
||||
friendlyMessage?: string;
|
||||
optimizedPrompt?: string;
|
||||
recommendedImageParams?: AgentRecommendedImageParams;
|
||||
videoPlan?: AgentVideoPlan;
|
||||
route?: RouteResult;
|
||||
}
|
||||
|
||||
@ -44,6 +71,7 @@ const AutoAgentState = Annotation.Root({
|
||||
friendlyMessage: Annotation<string | null>,
|
||||
optimizedPrompt: Annotation<string | null>,
|
||||
recommendedImageParams: Annotation<AgentRecommendedImageParams | null>,
|
||||
videoPlan: Annotation<AgentVideoPlan | null>,
|
||||
route: Annotation<RouteResult | null>,
|
||||
});
|
||||
|
||||
@ -84,6 +112,216 @@ function normalizeImageSize(value?: string): string | undefined {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function normalizeVideoMode(value?: string): AgentRecommendedVideoParams["mode"] | undefined {
|
||||
if (!value) return undefined;
|
||||
if (value === "text-to-video") return "text-to-video";
|
||||
if (value === "first-frame") return "first-frame";
|
||||
if (value === "first-last-frame") return "first-last-frame";
|
||||
if (value === "reference-images") return "reference-images";
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function normalizeVideoResolution(
|
||||
value?: string,
|
||||
): AgentRecommendedVideoParams["resolution"] | undefined {
|
||||
if (!value) return undefined;
|
||||
const normalized = value.toLowerCase();
|
||||
if (normalized === "480p") return "480p";
|
||||
if (normalized === "720p") return "720p";
|
||||
if (normalized === "1080p") return "1080p";
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function normalizeVideoRatio(value?: string): AgentRecommendedVideoParams["ratio"] | undefined {
|
||||
if (!value) return undefined;
|
||||
const normalized = value.trim();
|
||||
const allowed = new Set(["16:9", "4:3", "1:1", "3:4", "9:16", "21:9", "adaptive"]);
|
||||
return allowed.has(normalized) ? (normalized as AgentRecommendedVideoParams["ratio"]) : undefined;
|
||||
}
|
||||
|
||||
function normalizeVideoDuration(value?: number): number | undefined {
|
||||
if (typeof value !== "number" || !Number.isFinite(value)) return undefined;
|
||||
return Math.min(15, Math.max(3, Math.round(value)));
|
||||
}
|
||||
|
||||
function sanitizeRecommendedVideoParams(
|
||||
params: AgentRecommendedVideoParams | undefined,
|
||||
): AgentRecommendedVideoParams | undefined {
|
||||
if (!params) return undefined;
|
||||
const safe: AgentRecommendedVideoParams = {};
|
||||
const mode = normalizeVideoMode(params.mode);
|
||||
const resolution = normalizeVideoResolution(params.resolution);
|
||||
const ratio = normalizeVideoRatio(params.ratio);
|
||||
const duration = normalizeVideoDuration(params.duration);
|
||||
|
||||
if (mode) safe.mode = mode;
|
||||
if (resolution) safe.resolution = resolution;
|
||||
if (ratio) safe.ratio = ratio;
|
||||
if (duration) safe.duration = duration;
|
||||
if (typeof params.generateAudio === "boolean") safe.generateAudio = params.generateAudio;
|
||||
if (typeof params.cameraMove === "string" && params.cameraMove.trim()) {
|
||||
safe.cameraMove = params.cameraMove.trim().slice(0, 32);
|
||||
}
|
||||
|
||||
return Object.keys(safe).length > 0 ? safe : undefined;
|
||||
}
|
||||
|
||||
function inferVideoParamsByText(text: string, hasImage: boolean): AgentRecommendedVideoParams {
|
||||
const normalized = text.toLowerCase();
|
||||
const durationMatch = normalized.match(/(\d{1,2})\s*秒/);
|
||||
const duration = durationMatch ? normalizeVideoDuration(Number(durationMatch[1])) : undefined;
|
||||
|
||||
const mode = /首尾帧|首帧.*尾帧|结尾帧|尾帧/.test(normalized)
|
||||
? "first-last-frame"
|
||||
: hasImage
|
||||
? "first-frame"
|
||||
: "text-to-video";
|
||||
|
||||
const ratio = /竖屏|手机|9:16|竖版/.test(normalized)
|
||||
? "9:16"
|
||||
: /方形|方图|1:1/.test(normalized)
|
||||
? "1:1"
|
||||
: /超宽|21:9|电影感/.test(normalized)
|
||||
? "21:9"
|
||||
: "16:9";
|
||||
|
||||
const generateAudio = /配音|旁白|音效|有声|带声音/.test(normalized);
|
||||
return {
|
||||
mode,
|
||||
ratio,
|
||||
duration: duration || 5,
|
||||
resolution: "720p",
|
||||
generateAudio,
|
||||
};
|
||||
}
|
||||
|
||||
function buildVideoFinalPrompt(params: {
|
||||
script: string;
|
||||
storyboard: AgentVideoStoryboardShot[];
|
||||
originalText: string;
|
||||
}): string {
|
||||
const script = params.script.trim();
|
||||
const storyboard = params.storyboard
|
||||
.slice(0, 8)
|
||||
.map(
|
||||
(shot) =>
|
||||
`镜头${shot.shot}（${shot.duration}s）: ${shot.prompt || shot.description || shot.title}`.trim()
|
||||
)
|
||||
.join("\n");
|
||||
|
||||
return [
|
||||
`【用户目标】${params.originalText.trim() || "生成视频"}`,
|
||||
script ? `【脚本】${script}` : "",
|
||||
storyboard ? `【分镜】\n${storyboard}` : "",
|
||||
"【执行要求】镜头衔接自然，人物与主体保持一致，动作清晰，避免突兀跳切。",
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n\n")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function heuristicVideoPlan(state: AutoAgentStateType): AgentVideoPlan {
|
||||
const recommendedVideoParams = inferVideoParamsByText(state.text, state.hasImage);
|
||||
const duration = recommendedVideoParams.duration || 5;
|
||||
const perShot = Math.max(2, Math.round(duration / 3));
|
||||
const storyboard: AgentVideoStoryboardShot[] = [
|
||||
{
|
||||
shot: 1,
|
||||
title: "开场建立",
|
||||
duration: perShot,
|
||||
description: "建立主体与环境关系，明确视觉基调。",
|
||||
prompt: `开场镜头，展示主体与环境，围绕「${compactText(state.text, 50)}」建立氛围。`,
|
||||
camera: "稳定推进",
|
||||
},
|
||||
{
|
||||
shot: 2,
|
||||
title: "动作推进",
|
||||
duration: perShot,
|
||||
description: "主体动作或情绪推进，形成主要叙事段落。",
|
||||
prompt: `中段镜头，突出主体动作与情绪变化，强化故事推进。`,
|
||||
camera: "跟拍或环绕",
|
||||
},
|
||||
{
|
||||
shot: 3,
|
||||
title: "收束结尾",
|
||||
duration: Math.max(2, duration - perShot * 2),
|
||||
description: "情绪收束并输出最终视觉记忆点。",
|
||||
prompt: "结尾镜头，画面收束，保留主体识别与情绪余韵。",
|
||||
camera: "缓慢拉远",
|
||||
},
|
||||
];
|
||||
const script = `围绕“${compactText(state.text, 80)}”制作一条节奏清晰、主体连续的短视频，包含开场建立、过程推进与结尾收束三段。`;
|
||||
return {
|
||||
script,
|
||||
storyboard,
|
||||
finalPrompt: buildVideoFinalPrompt({
|
||||
script,
|
||||
storyboard,
|
||||
originalText: state.text,
|
||||
}),
|
||||
recommendedVideoParams,
|
||||
};
|
||||
}
|
||||
|
||||
function sanitizeVideoPlan(raw: unknown, state: AutoAgentStateType): AgentVideoPlan {
|
||||
if (!raw || typeof raw !== "object") return heuristicVideoPlan(state);
|
||||
const payload = raw as Record<string, any>;
|
||||
const recommendedVideoParams = sanitizeRecommendedVideoParams(payload.recommendedVideoParams);
|
||||
const script =
|
||||
typeof payload.script === "string" && payload.script.trim()
|
||||
? payload.script.trim().slice(0, 400)
|
||||
: `围绕“${compactText(state.text, 80)}”生成视频。`;
|
||||
|
||||
const storyboardInput = Array.isArray(payload.storyboard) ? payload.storyboard : [];
|
||||
const storyboard: AgentVideoStoryboardShot[] = storyboardInput
|
||||
.map((item: any, idx) => {
|
||||
const shot = Number.isFinite(item?.shot) ? Math.max(1, Math.round(item.shot)) : idx + 1;
|
||||
const duration = normalizeVideoDuration(Number(item?.duration)) || 3;
|
||||
const title =
|
||||
typeof item?.title === "string" && item.title.trim()
|
||||
? item.title.trim().slice(0, 24)
|
||||
: `镜头${shot}`;
|
||||
const description =
|
||||
typeof item?.description === "string" && item.description.trim()
|
||||
? item.description.trim().slice(0, 180)
|
||||
: title;
|
||||
const prompt =
|
||||
typeof item?.prompt === "string" && item.prompt.trim()
|
||||
? item.prompt.trim().slice(0, 260)
|
||||
: description;
|
||||
const camera =
|
||||
typeof item?.camera === "string" && item.camera.trim()
|
||||
? item.camera.trim().slice(0, 24)
|
||||
: undefined;
|
||||
return { shot, title, duration, description, prompt, camera };
|
||||
})
|
||||
.filter((item) => item.prompt);
|
||||
|
||||
const fallback = heuristicVideoPlan(state);
|
||||
const safeStoryboard = storyboard.length > 0 ? storyboard.slice(0, 8) : fallback.storyboard;
|
||||
const finalPrompt =
|
||||
typeof payload.finalPrompt === "string" && payload.finalPrompt.trim()
|
||||
? payload.finalPrompt.trim().slice(0, 1400)
|
||||
: buildVideoFinalPrompt({
|
||||
script,
|
||||
storyboard: safeStoryboard,
|
||||
originalText: state.text,
|
||||
});
|
||||
|
||||
const negativePrompt =
|
||||
typeof payload.negativePrompt === "string" && payload.negativePrompt.trim()
|
||||
? payload.negativePrompt.trim().slice(0, 360)
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
script,
|
||||
storyboard: safeStoryboard,
|
||||
finalPrompt,
|
||||
negativePrompt,
|
||||
recommendedVideoParams: recommendedVideoParams || fallback.recommendedVideoParams,
|
||||
};
|
||||
}
|
||||
|
||||
function sanitizeRecommendedImageParams(
|
||||
params: AgentRecommendedImageParams | undefined,
|
||||
intent?: AnalysisResult["intent"],
|
||||
@ -422,6 +660,87 @@ async function recommendImageParamsWithAgent(
|
||||
}
|
||||
}
|
||||
|
||||
async function recommendVideoPlanWithAgent(
|
||||
state: AutoAgentStateType,
|
||||
): Promise<AgentVideoPlan> {
|
||||
try {
|
||||
const {
|
||||
getEnabledAiModelsWithProvider,
|
||||
getAgentAiModelWithProvider,
|
||||
} = await import("@/lib/db/queries");
|
||||
const { getLanguageModelById } = await import("./dynamic-provider");
|
||||
const allModels = await getEnabledAiModelsWithProvider();
|
||||
const agentModel = await getAgentAiModelWithProvider();
|
||||
const plannerModelConfig = pickAgentReasoningModel(agentModel, allModels);
|
||||
if (!plannerModelConfig) {
|
||||
return heuristicVideoPlan(state);
|
||||
}
|
||||
const plannerModel = await getLanguageModelById(plannerModelConfig.model.id);
|
||||
if (!plannerModel) {
|
||||
return heuristicVideoPlan(state);
|
||||
}
|
||||
|
||||
const history = state.historyMessages
|
||||
.slice(-6)
|
||||
.map((msg) => `${msg.role === "user" ? "用户" : "助手"}: ${compactText(msg.content, 120)}`)
|
||||
.join("\n");
|
||||
|
||||
const prompt = [
|
||||
`用户请求：${state.text || "生成视频"}`,
|
||||
`是否有参考图：${state.hasImage ? "是" : "否"}（${state.imageUrls.length} 张）`,
|
||||
history ? `最近对话：\n${history}` : "",
|
||||
"请产出可直接用于视频生成的脚本与分镜。",
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n\n");
|
||||
|
||||
const { text } = await generateText({
|
||||
model: plannerModel,
|
||||
system: `你是视频生成 Agent 规划器，目标是最大化一次出片成功率。
|
||||
只输出 JSON，不要输出解释。格式：
|
||||
{
|
||||
"script": "120字内剧情脚本",
|
||||
"recommendedVideoParams": {
|
||||
"mode": "text-to-video|first-frame|first-last-frame|reference-images",
|
||||
"resolution": "480p|720p|1080p",
|
||||
"ratio": "16:9|4:3|1:1|3:4|9:16|21:9|adaptive",
|
||||
"duration": 5,
|
||||
"generateAudio": false,
|
||||
"cameraMove": "none|static|handheld|dynamic-orbit"
|
||||
},
|
||||
"storyboard": [
|
||||
{
|
||||
"shot": 1,
|
||||
"title": "镜头标题",
|
||||
"duration": 3,
|
||||
"description": "画面描述",
|
||||
"camera": "运镜方式",
|
||||
"prompt": "该镜头可直接喂给视频模型的提示词"
|
||||
}
|
||||
],
|
||||
"negativePrompt": "可选",
|
||||
"finalPrompt": "整合后的最终视频提示词"
|
||||
}
|
||||
要求：
|
||||
- 分镜 2~6 个，时长总计与 duration 接近。
|
||||
- 有参考图时优先 first-frame 或 first-last-frame。
|
||||
- 避免抽象空话，必须可执行。`,
|
||||
prompt,
|
||||
temperature: 0.2,
|
||||
maxTokens: 720,
|
||||
} as any);
|
||||
|
||||
const parsed = parseJsonBlock<Record<string, unknown>>(text);
|
||||
const plan = sanitizeVideoPlan(parsed, state);
|
||||
console.log(
|
||||
`[Auto Agent] Video planner model: ${plannerModelConfig.model.displayName} (${plannerModelConfig.provider.type})`
|
||||
);
|
||||
return plan;
|
||||
} catch {
|
||||
return heuristicVideoPlan(state);
|
||||
}
|
||||
}
|
||||
|
||||
async function refineRouteWithAgent(
|
||||
state: AutoAgentStateType,
|
||||
baseRoute: RouteResult,
|
||||
@ -525,6 +844,9 @@ const analyzeNode = async (state: AutoAgentStateType) => {
|
||||
const shouldOptimize = (state: AutoAgentStateType) => {
|
||||
if (!state.analysis) return "route";
|
||||
if (state.analysis.needsClarification) return "route";
|
||||
if (state.analysis.intent === "video_gen") {
|
||||
return "optimize";
|
||||
}
|
||||
if (state.analysis.intent === "image_gen") {
|
||||
return "optimize";
|
||||
}
|
||||
@ -538,6 +860,34 @@ const optimizeNode = async (state: AutoAgentStateType) => {
|
||||
if (!state.analysis) {
|
||||
return {};
|
||||
}
|
||||
if (state.analysis.intent === "video_gen") {
|
||||
const videoPlan = await recommendVideoPlanWithAgent(state);
|
||||
const storyboardPreview = videoPlan.storyboard
|
||||
.slice(0, 4)
|
||||
.map(
|
||||
(shot) =>
|
||||
`- 镜头${shot.shot}（${shot.duration}s）：${shot.title}｜${compactText(
|
||||
shot.description || shot.prompt,
|
||||
64,
|
||||
)}`
|
||||
)
|
||||
.join("\n");
|
||||
const friendlyMessage = [
|
||||
"### 需求理解",
|
||||
`- ${compactText(state.text || "生成视频", 110)}`,
|
||||
"### 脚本规划",
|
||||
`- ${videoPlan.script}`,
|
||||
"### 分镜规划",
|
||||
storyboardPreview || "- 已生成分镜，执行时自动拼接。",
|
||||
"### 执行说明",
|
||||
"- 已基于上述脚本与分镜优化提示词并提交视频模型执行。",
|
||||
].join("\n");
|
||||
return {
|
||||
friendlyMessage,
|
||||
optimizedPrompt: videoPlan.finalPrompt,
|
||||
videoPlan,
|
||||
};
|
||||
}
|
||||
if (state.analysis.intent === "image_edit" && !state.analysis.subIntent) {
|
||||
const recommendedImageParams = await recommendImageParamsWithAgent(state.analysis, state);
|
||||
return { recommendedImageParams };
|
||||
@ -618,6 +968,7 @@ export async function runAutoAgentGraph(params: AutoAgentInput): Promise<AutoAge
|
||||
friendlyMessage: null,
|
||||
optimizedPrompt: null,
|
||||
recommendedImageParams: null,
|
||||
videoPlan: null,
|
||||
route: null,
|
||||
});
|
||||
|
||||
@ -626,6 +977,7 @@ export async function runAutoAgentGraph(params: AutoAgentInput): Promise<AutoAge
|
||||
friendlyMessage: result.friendlyMessage ?? undefined,
|
||||
optimizedPrompt: result.optimizedPrompt ?? undefined,
|
||||
recommendedImageParams: result.recommendedImageParams ?? undefined,
|
||||
videoPlan: result.videoPlan ?? undefined,
|
||||
route: result.route ?? undefined,
|
||||
};
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user