305 lines
11 KiB
TypeScript
305 lines
11 KiB
TypeScript
import { and, eq, inArray } from "drizzle-orm";
|
|
import { NextResponse } from "next/server";
|
|
import { auth } from "@/app/(auth)/auth";
|
|
import { executeDirectVideoGeneration } from "@/lib/ai/tools/generate-video";
|
|
import {
|
|
resolveVideoReferenceSelectionRule,
|
|
selectVideoReferenceImages,
|
|
} from "@/lib/canvas/video-reference-selection";
|
|
import { getSystemSetting } from "@/lib/db/queries";
|
|
import { db } from "@/lib/db";
|
|
import { mediaAsset } from "@/lib/db/schema";
|
|
import { filterReachableRemoteUrls } from "@/lib/net/url-access";
|
|
import {
|
|
VIDEO_CANVAS_FUNCTION_CONFIG_KEY,
|
|
defaultVideoCanvasFunctionConfig,
|
|
normalizeVideoCanvasFunctionConfig,
|
|
} from "@/lib/video-canvas/config";
|
|
import { normalizeVideoCanvasGenerationRequestCount } from "@/lib/video-canvas/video-generation-parameters";
|
|
import {
|
|
buildVideoCanvasSubjectProviderReferences,
|
|
composeVideoCanvasSubjectPrompt,
|
|
normalizeVideoCanvasAttachedSubjects,
|
|
supportsVideoCanvasSubjectUsage,
|
|
toVideoCanvasAttachedSubject,
|
|
VIDEO_CANVAS_SUBJECT_SOURCE_TYPE,
|
|
type VideoCanvasAttachedSubject,
|
|
} from "@/lib/video-canvas/subjects";
|
|
import {
|
|
composeVideoCanvasSmartStoryboardPrompt,
|
|
normalizeVideoCanvasSmartStoryboard,
|
|
supportsVideoCanvasSmartStoryboard,
|
|
validateVideoCanvasSmartStoryboard,
|
|
type VideoCanvasSmartStoryboard,
|
|
} from "@/lib/video-canvas/smart-storyboard";
|
|
import type { VideoGenerationMode, VideoRatio, VideoResolution } from "@/types/ai";
|
|
|
|
export const maxDuration = 300;
|
|
|
|
interface GenerateVideoRequest {
|
|
autoValidateMaterial?: boolean;
|
|
prompt?: string;
|
|
mode?: VideoGenerationMode;
|
|
images?: string[];
|
|
audioUrls?: string[];
|
|
resolution?: VideoResolution;
|
|
ratio?: VideoRatio;
|
|
duration?: number;
|
|
generateAudio?: boolean;
|
|
count?: number;
|
|
watermark?: boolean;
|
|
cameraFixed?: boolean;
|
|
seed?: number;
|
|
returnLastFrame?: boolean;
|
|
modelLabel?: string;
|
|
preferredModelRef?: string;
|
|
searchEnabled?: boolean;
|
|
smartStoryboard?: VideoCanvasSmartStoryboard;
|
|
subjects?: VideoCanvasAttachedSubject[];
|
|
}
|
|
|
|
function inferMode(images: string[]): VideoGenerationMode {
|
|
if (images.length >= 3) return "reference-images";
|
|
if (images.length >= 2) return "first-last-frame";
|
|
if (images.length === 1) return "first-frame";
|
|
return "text-to-video";
|
|
}
|
|
|
|
export async function POST(request: Request) {
|
|
try {
|
|
const session = await auth();
|
|
if (!session?.user?.id) {
|
|
return NextResponse.json(
|
|
{ success: false, error: "未登录" },
|
|
{ status: 401 }
|
|
);
|
|
}
|
|
|
|
const body = (await request.json().catch(() => ({}))) as GenerateVideoRequest;
|
|
const prompt = String(body.prompt || "").trim();
|
|
const count = normalizeVideoCanvasGenerationRequestCount(body.count, 1);
|
|
if (count === null) {
|
|
return NextResponse.json(
|
|
{ success: false, error: "生成数量仅支持 1、2 或 4 个" },
|
|
{ status: 400 },
|
|
);
|
|
}
|
|
const generateAudio = body.generateAudio !== false;
|
|
const setting = await getSystemSetting({
|
|
key: VIDEO_CANVAS_FUNCTION_CONFIG_KEY,
|
|
}).catch(() => null);
|
|
const config = setting?.value
|
|
? normalizeVideoCanvasFunctionConfig(setting.value)
|
|
: defaultVideoCanvasFunctionConfig;
|
|
const autoValidateMaterial =
|
|
typeof body.autoValidateMaterial === "boolean"
|
|
? body.autoValidateMaterial
|
|
: config.videoAdvancedSettings.defaultAutoValidateMaterial;
|
|
const searchEnabled =
|
|
typeof body.searchEnabled === "boolean"
|
|
? body.searchEnabled
|
|
: config.videoAdvancedSettings.defaultSearchEnabled;
|
|
const subjectUsageEnabled = supportsVideoCanvasSubjectUsage(
|
|
config.subjectLibrary,
|
|
`${body.modelLabel || ""} ${body.preferredModelRef || ""}`,
|
|
);
|
|
const requestedSubjects = subjectUsageEnabled
|
|
? normalizeVideoCanvasAttachedSubjects(
|
|
body.subjects,
|
|
config.subjectLibrary.videoUsage.maxSubjects,
|
|
)
|
|
: [];
|
|
const requestedSubjectIds = requestedSubjects
|
|
.map((subject) => subject.id)
|
|
.filter((id) => /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(id));
|
|
const ownedSubjectRows = requestedSubjectIds.length > 0
|
|
? await db
|
|
.select({
|
|
description: mediaAsset.summary,
|
|
displayName: mediaAsset.displayName,
|
|
id: mediaAsset.id,
|
|
metadata: mediaAsset.metadata,
|
|
originalUrl: mediaAsset.originalUrl,
|
|
previewUrl: mediaAsset.previewUrl,
|
|
thumbnailUrl: mediaAsset.thumbnailUrl,
|
|
})
|
|
.from(mediaAsset)
|
|
.where(
|
|
and(
|
|
inArray(mediaAsset.id, requestedSubjectIds),
|
|
eq(mediaAsset.userId, session.user.id),
|
|
eq(mediaAsset.mediaType, "subject"),
|
|
eq(mediaAsset.sourceType, VIDEO_CANVAS_SUBJECT_SOURCE_TYPE),
|
|
eq(mediaAsset.status, "ready"),
|
|
),
|
|
)
|
|
: [];
|
|
const ownedSubjectsById = new Map(
|
|
ownedSubjectRows.map((subject) => [subject.id, subject]),
|
|
);
|
|
const subjects = requestedSubjects.flatMap((requestedSubject) => {
|
|
const ownedSubject = ownedSubjectsById.get(requestedSubject.id);
|
|
const normalized = ownedSubject
|
|
? toVideoCanvasAttachedSubject(ownedSubject)
|
|
: null;
|
|
return normalized ? [normalized] : [];
|
|
});
|
|
const subjectReferences = buildVideoCanvasSubjectProviderReferences(subjects);
|
|
const smartStoryboardSupported = supportsVideoCanvasSmartStoryboard(
|
|
config.smartStoryboard,
|
|
`${body.modelLabel || ""} ${body.preferredModelRef || ""}`,
|
|
);
|
|
const smartStoryboard = normalizeVideoCanvasSmartStoryboard(
|
|
body.smartStoryboard,
|
|
{
|
|
config: config.smartStoryboard,
|
|
fallbackPrompt: prompt,
|
|
fallbackTotalDurationSeconds: body.duration,
|
|
},
|
|
);
|
|
const storyboardValidation = validateVideoCanvasSmartStoryboard(
|
|
smartStoryboard,
|
|
);
|
|
if (
|
|
smartStoryboardSupported &&
|
|
smartStoryboard.enabled &&
|
|
!storyboardValidation.valid
|
|
) {
|
|
const message =
|
|
storyboardValidation.emptyShotIndexes.length > 0
|
|
? "智能分镜存在未填写的镜头描述"
|
|
: storyboardValidation.status === "over"
|
|
? "智能分镜总时长超出"
|
|
: "智能分镜总时长不足";
|
|
return NextResponse.json(
|
|
{ success: false, error: message },
|
|
{ status: 400 },
|
|
);
|
|
}
|
|
const promptWithStoryboard = smartStoryboardSupported
|
|
? composeVideoCanvasSmartStoryboardPrompt(
|
|
prompt,
|
|
smartStoryboard,
|
|
config.smartStoryboard,
|
|
)
|
|
: prompt;
|
|
const generationPrompt = composeVideoCanvasSubjectPrompt(
|
|
promptWithStoryboard,
|
|
subjects,
|
|
config.subjectLibrary.videoUsage.promptTemplate,
|
|
);
|
|
if (!generationPrompt) {
|
|
return NextResponse.json(
|
|
{ success: false, error: "请输入视频提示词" },
|
|
{ status: 400 }
|
|
);
|
|
}
|
|
const referenceRule = resolveVideoReferenceSelectionRule(
|
|
config.videoReferenceSelection,
|
|
`${body.modelLabel || ""} ${body.preferredModelRef || ""}`,
|
|
);
|
|
const rawImages = Array.isArray(body.images)
|
|
? body.images.map((item) => String(item || "").trim()).filter(Boolean).slice(0, 20)
|
|
: [];
|
|
const inputImages = body.mode
|
|
? body.mode === "text-to-video" || body.mode === "audio-to-video"
|
|
? rawImages.slice(0, 4)
|
|
: selectVideoReferenceImages(rawImages, body.mode, referenceRule.maxImages)
|
|
: rawImages.slice(0, referenceRule.maxImages);
|
|
const { kept: reachableImages, dropped: unreachableImages } =
|
|
await filterReachableRemoteUrls(inputImages, { timeoutMs: 3000 });
|
|
const inputAudioUrls = Array.isArray(body.audioUrls)
|
|
? body.audioUrls.map((item) => String(item || "").trim()).filter(Boolean).slice(0, 3)
|
|
: [];
|
|
const { kept: reachableAudioUrls, dropped: unreachableAudioUrls } =
|
|
await filterReachableRemoteUrls(inputAudioUrls, { timeoutMs: 3000 });
|
|
|
|
if (inputImages.length > 0 && reachableImages.length === 0) {
|
|
return NextResponse.json(
|
|
{ success: false, error: "参考图片链接不可访问，请重新上传后再试" },
|
|
{ status: 400 }
|
|
);
|
|
}
|
|
|
|
const mode = body.mode || (reachableAudioUrls.length > 0 ? "audio-to-video" : inferMode(reachableImages));
|
|
const selectedImages =
|
|
mode === "text-to-video" || mode === "audio-to-video"
|
|
? reachableImages
|
|
: selectVideoReferenceImages(
|
|
reachableImages,
|
|
mode,
|
|
referenceRule.maxImages,
|
|
);
|
|
if (mode === "audio-to-video" && reachableAudioUrls.length === 0) {
|
|
return NextResponse.json(
|
|
{ success: false, error: "音频生视频至少需要 1 个可访问的音频素材" },
|
|
{ status: 400 }
|
|
);
|
|
}
|
|
const result = await executeDirectVideoGeneration({
|
|
prompt: generationPrompt,
|
|
mode,
|
|
images: selectedImages,
|
|
audioUrls: reachableAudioUrls,
|
|
resolution: body.resolution,
|
|
ratio: body.ratio,
|
|
duration:
|
|
smartStoryboardSupported && smartStoryboard.enabled
|
|
? smartStoryboard.totalDurationSeconds
|
|
: typeof body.duration === "number" && Number.isFinite(body.duration)
|
|
? body.duration
|
|
: undefined,
|
|
generateAudio,
|
|
watermark: body.watermark,
|
|
cameraFixed: body.cameraFixed,
|
|
seed:
|
|
typeof body.seed === "number" && Number.isFinite(body.seed) ? body.seed : undefined,
|
|
returnLastFrame: body.returnLastFrame,
|
|
preferredModelRef:
|
|
typeof body.preferredModelRef === "string" ? body.preferredModelRef : undefined,
|
|
});
|
|
|
|
if (!result.success || !result.videoUrl) {
|
|
return NextResponse.json(
|
|
{ success: false, error: "视频生成失败，请稍后重试" },
|
|
{ status: 502 }
|
|
);
|
|
}
|
|
|
|
return NextResponse.json({
|
|
success: true,
|
|
data: {
|
|
url: result.videoUrl,
|
|
metadata: result.metadata || null,
|
|
modelDisplayName: result.modelDisplayName,
|
|
modelId: result.modelId,
|
|
modelRef: result.modelRef,
|
|
mode,
|
|
images: selectedImages,
|
|
audioUrls: reachableAudioUrls,
|
|
autoValidateMaterial,
|
|
count,
|
|
generateAudio,
|
|
droppedImages: unreachableImages,
|
|
droppedAudioUrls: unreachableAudioUrls,
|
|
searchEnabled,
|
|
subjects,
|
|
subjectReferences,
|
|
smartStoryboard:
|
|
smartStoryboardSupported && smartStoryboard.enabled
|
|
? smartStoryboard
|
|
: undefined,
|
|
},
|
|
});
|
|
} catch (error) {
|
|
console.error("[Canvas Generate Video] Error:", error);
|
|
return NextResponse.json(
|
|
{
|
|
success: false,
|
|
error: error instanceof Error ? error.message : "生成视频失败",
|
|
},
|
|
{ status: 500 }
|
|
);
|
|
}
|
|
}
|