import { and, eq, inArray } from "drizzle-orm"; import { NextResponse } from "next/server"; import { auth } from "@/app/(auth)/auth"; import { executeDirectVideoGeneration } from "@/lib/ai/tools/generate-video"; import { resolveVideoReferenceSelectionRule, selectVideoReferenceImages, } from "@/lib/canvas/video-reference-selection"; import { getSystemSetting } from "@/lib/db/queries"; import { db } from "@/lib/db"; import { mediaAsset } from "@/lib/db/schema"; import { filterReachableRemoteUrls } from "@/lib/net/url-access"; import { VIDEO_CANVAS_FUNCTION_CONFIG_KEY, defaultVideoCanvasFunctionConfig, normalizeVideoCanvasFunctionConfig, } from "@/lib/video-canvas/config"; import { normalizeVideoCanvasGenerationRequestCount } from "@/lib/video-canvas/video-generation-parameters"; import { buildVideoCanvasSubjectProviderReferences, composeVideoCanvasSubjectPrompt, normalizeVideoCanvasAttachedSubjects, supportsVideoCanvasSubjectUsage, toVideoCanvasAttachedSubject, VIDEO_CANVAS_SUBJECT_SOURCE_TYPE, type VideoCanvasAttachedSubject, } from "@/lib/video-canvas/subjects"; import { composeVideoCanvasSmartStoryboardPrompt, normalizeVideoCanvasSmartStoryboard, supportsVideoCanvasSmartStoryboard, validateVideoCanvasSmartStoryboard, type VideoCanvasSmartStoryboard, } from "@/lib/video-canvas/smart-storyboard"; import type { VideoGenerationMode, VideoRatio, VideoResolution } from "@/types/ai"; export const maxDuration = 300; interface GenerateVideoRequest { autoValidateMaterial?: boolean; prompt?: string; mode?: VideoGenerationMode; images?: string[]; audioUrls?: string[]; resolution?: VideoResolution; ratio?: VideoRatio; duration?: number; generateAudio?: boolean; count?: number; watermark?: boolean; cameraFixed?: boolean; seed?: number; returnLastFrame?: boolean; modelLabel?: string; preferredModelRef?: string; searchEnabled?: boolean; smartStoryboard?: VideoCanvasSmartStoryboard; subjects?: VideoCanvasAttachedSubject[]; } function inferMode(images: string[]): VideoGenerationMode { if (images.length >= 3) return "reference-images"; if (images.length >= 2) return "first-last-frame"; if (images.length === 1) return "first-frame"; return "text-to-video"; } export async function POST(request: Request) { try { const session = await auth(); if (!session?.user?.id) { return NextResponse.json( { success: false, error: "未登录" }, { status: 401 } ); } const body = (await request.json().catch(() => ({}))) as GenerateVideoRequest; const prompt = String(body.prompt || "").trim(); const count = normalizeVideoCanvasGenerationRequestCount(body.count, 1); if (count === null) { return NextResponse.json( { success: false, error: "生成数量仅支持 1、2 或 4 个" }, { status: 400 }, ); } const generateAudio = body.generateAudio !== false; const setting = await getSystemSetting({ key: VIDEO_CANVAS_FUNCTION_CONFIG_KEY, }).catch(() => null); const config = setting?.value ? normalizeVideoCanvasFunctionConfig(setting.value) : defaultVideoCanvasFunctionConfig; const autoValidateMaterial = typeof body.autoValidateMaterial === "boolean" ? body.autoValidateMaterial : config.videoAdvancedSettings.defaultAutoValidateMaterial; const searchEnabled = typeof body.searchEnabled === "boolean" ? body.searchEnabled : config.videoAdvancedSettings.defaultSearchEnabled; const subjectUsageEnabled = supportsVideoCanvasSubjectUsage( config.subjectLibrary, `${body.modelLabel || ""} ${body.preferredModelRef || ""}`, ); const requestedSubjects = subjectUsageEnabled ? normalizeVideoCanvasAttachedSubjects( body.subjects, config.subjectLibrary.videoUsage.maxSubjects, ) : []; const requestedSubjectIds = requestedSubjects .map((subject) => subject.id) .filter((id) => /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(id)); const ownedSubjectRows = requestedSubjectIds.length > 0 ? await db .select({ description: mediaAsset.summary, displayName: mediaAsset.displayName, id: mediaAsset.id, metadata: mediaAsset.metadata, originalUrl: mediaAsset.originalUrl, previewUrl: mediaAsset.previewUrl, thumbnailUrl: mediaAsset.thumbnailUrl, }) .from(mediaAsset) .where( and( inArray(mediaAsset.id, requestedSubjectIds), eq(mediaAsset.userId, session.user.id), eq(mediaAsset.mediaType, "subject"), eq(mediaAsset.sourceType, VIDEO_CANVAS_SUBJECT_SOURCE_TYPE), eq(mediaAsset.status, "ready"), ), ) : []; const ownedSubjectsById = new Map( ownedSubjectRows.map((subject) => [subject.id, subject]), ); const subjects = requestedSubjects.flatMap((requestedSubject) => { const ownedSubject = ownedSubjectsById.get(requestedSubject.id); const normalized = ownedSubject ? toVideoCanvasAttachedSubject(ownedSubject) : null; return normalized ? [normalized] : []; }); const subjectReferences = buildVideoCanvasSubjectProviderReferences(subjects); const smartStoryboardSupported = supportsVideoCanvasSmartStoryboard( config.smartStoryboard, `${body.modelLabel || ""} ${body.preferredModelRef || ""}`, ); const smartStoryboard = normalizeVideoCanvasSmartStoryboard( body.smartStoryboard, { config: config.smartStoryboard, fallbackPrompt: prompt, fallbackTotalDurationSeconds: body.duration, }, ); const storyboardValidation = validateVideoCanvasSmartStoryboard( smartStoryboard, ); if ( smartStoryboardSupported && smartStoryboard.enabled && !storyboardValidation.valid ) { const message = storyboardValidation.emptyShotIndexes.length > 0 ? "智能分镜存在未填写的镜头描述" : storyboardValidation.status === "over" ? "智能分镜总时长超出" : "智能分镜总时长不足"; return NextResponse.json( { success: false, error: message }, { status: 400 }, ); } const promptWithStoryboard = smartStoryboardSupported ? composeVideoCanvasSmartStoryboardPrompt( prompt, smartStoryboard, config.smartStoryboard, ) : prompt; const generationPrompt = composeVideoCanvasSubjectPrompt( promptWithStoryboard, subjects, config.subjectLibrary.videoUsage.promptTemplate, ); if (!generationPrompt) { return NextResponse.json( { success: false, error: "请输入视频提示词" }, { status: 400 } ); } const referenceRule = resolveVideoReferenceSelectionRule( config.videoReferenceSelection, `${body.modelLabel || ""} ${body.preferredModelRef || ""}`, ); const rawImages = Array.isArray(body.images) ? body.images.map((item) => String(item || "").trim()).filter(Boolean).slice(0, 20) : []; const inputImages = body.mode ? body.mode === "text-to-video" || body.mode === "audio-to-video" ? rawImages.slice(0, 4) : selectVideoReferenceImages(rawImages, body.mode, referenceRule.maxImages) : rawImages.slice(0, referenceRule.maxImages); const { kept: reachableImages, dropped: unreachableImages } = await filterReachableRemoteUrls(inputImages, { timeoutMs: 3000 }); const inputAudioUrls = Array.isArray(body.audioUrls) ? body.audioUrls.map((item) => String(item || "").trim()).filter(Boolean).slice(0, 3) : []; const { kept: reachableAudioUrls, dropped: unreachableAudioUrls } = await filterReachableRemoteUrls(inputAudioUrls, { timeoutMs: 3000 }); if (inputImages.length > 0 && reachableImages.length === 0) { return NextResponse.json( { success: false, error: "参考图片链接不可访问,请重新上传后再试" }, { status: 400 } ); } const mode = body.mode || (reachableAudioUrls.length > 0 ? "audio-to-video" : inferMode(reachableImages)); const selectedImages = mode === "text-to-video" || mode === "audio-to-video" ? reachableImages : selectVideoReferenceImages( reachableImages, mode, referenceRule.maxImages, ); if (mode === "audio-to-video" && reachableAudioUrls.length === 0) { return NextResponse.json( { success: false, error: "音频生视频至少需要 1 个可访问的音频素材" }, { status: 400 } ); } const result = await executeDirectVideoGeneration({ prompt: generationPrompt, mode, images: selectedImages, audioUrls: reachableAudioUrls, resolution: body.resolution, ratio: body.ratio, duration: smartStoryboardSupported && smartStoryboard.enabled ? smartStoryboard.totalDurationSeconds : typeof body.duration === "number" && Number.isFinite(body.duration) ? body.duration : undefined, generateAudio, watermark: body.watermark, cameraFixed: body.cameraFixed, seed: typeof body.seed === "number" && Number.isFinite(body.seed) ? body.seed : undefined, returnLastFrame: body.returnLastFrame, preferredModelRef: typeof body.preferredModelRef === "string" ? body.preferredModelRef : undefined, }); if (!result.success || !result.videoUrl) { return NextResponse.json( { success: false, error: "视频生成失败,请稍后重试" }, { status: 502 } ); } return NextResponse.json({ success: true, data: { url: result.videoUrl, metadata: result.metadata || null, modelDisplayName: result.modelDisplayName, modelId: result.modelId, modelRef: result.modelRef, mode, images: selectedImages, audioUrls: reachableAudioUrls, autoValidateMaterial, count, generateAudio, droppedImages: unreachableImages, droppedAudioUrls: unreachableAudioUrls, searchEnabled, subjects, subjectReferences, smartStoryboard: smartStoryboardSupported && smartStoryboard.enabled ? smartStoryboard : undefined, }, }); } catch (error) { console.error("[Canvas Generate Video] Error:", error); return NextResponse.json( { success: false, error: error instanceof Error ? error.message : "生成视频失败", }, { status: 500 } ); } }