diff --git a/app/api/canvas/video/process/route.ts b/app/api/canvas/video/process/route.ts index 459049544..9fd412245 100644 --- a/app/api/canvas/video/process/route.ts +++ b/app/api/canvas/video/process/route.ts @@ -1,42 +1,96 @@ -import {auth} from '@/app/(auth)/auth'; -import {NextResponse} from 'next/server'; -import {mkdtemp,readFile,rm} from 'node:fs/promises'; -import {tmpdir} from 'node:os'; -import {join} from 'node:path'; -import {randomUUID} from 'node:crypto'; -import {uploadFile} from '@/lib/storage/service'; -import {downloadCanvasMedia} from '@/lib/video-canvas/download-media'; -import {processVideoFile} from '@/lib/video-canvas/media-processing'; -import {videoProcessSchema} from '@/lib/video-canvas/media-contract'; -export const runtime='nodejs'; -export const maxDuration=300; -const running=new Set(); -export async function POST(request:Request) { - const session=await auth(); - if(!session?.user?.id) return NextResponse.json({success:false,error:'未登录'},{status:401}); - const parsed=videoProcessSchema.safeParse(await request.json().catch(()=>null)); - if(!parsed.success) return NextResponse.json({success:false,error:'视频处理参数无效'},{status:400}); - if(running.has(session.user.id)||running.size>=2) return NextResponse.json({success:false,error:'视频处理中,请稍后再提交'},{status:429}); +import { auth } from "@/app/(auth)/auth"; +import { NextResponse } from "next/server"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { randomUUID } from "node:crypto"; +import { uploadFile } from "@/lib/storage/service"; +import { downloadCanvasMedia } from "@/lib/video-canvas/download-media"; +import { processVideoFile } from "@/lib/video-canvas/media-processing"; +import { videoProcessSchema } from "@/lib/video-canvas/media-contract"; +export const runtime = "nodejs"; +export const maxDuration = 300; +const running = new Set(); +export async function POST(request: Request) { + const session = await auth(); + if (!session?.user?.id) + return NextResponse.json({ success: false, error: "未登录" }, { status: 401 }); + const parsed = videoProcessSchema.safeParse(await request.json().catch(() => null)); + if (!parsed.success) + return NextResponse.json({ success: false, error: "视频处理参数无效" }, { status: 400 }); + if (running.has(session.user.id) || running.size >= 2) + return NextResponse.json( + { success: false, error: "视频处理中,请稍后再提交" }, + { status: 429 }, + ); running.add(session.user.id); - let dir=''; + let dir = ""; try { - dir=await mkdtemp(join(tmpdir(),'canvas-video-')); - const input=join(dir,'input'); - await downloadCanvasMedia(parsed.data.url,input); - const save=async(name:string)=>{const buffer=await readFile(join(dir,name));return (await uploadFile(`canvas-video-${randomUUID()}-${name}`,new Uint8Array(buffer).buffer)).url;}; - if(parsed.data.operation==='trim') { - const metadata=await processVideoFile(input,join(dir,'clip.mp4'),'trim',undefined,parsed.data.range); - return NextResponse.json({success:true,data:{url:await save('clip.mp4'),...metadata}}); + dir = await mkdtemp(join(tmpdir(), "canvas-video-")); + const input = join(dir, "input"); + await downloadCanvasMedia(parsed.data.url, input); + const save = async (name: string) => { + const buffer = await readFile(join(dir, name)); + return ( + await uploadFile(`canvas-video-${randomUUID()}-${name}`, new Uint8Array(buffer).buffer) + ).url; + }; + if (parsed.data.operation === "frame" || parsed.data.operation === "slow") { + const name = parsed.data.operation === "frame" ? "frame.png" : "slow.mp4"; + const metadata = await processVideoFile( + input, + join(dir, name), + parsed.data.operation, + undefined, + undefined, + parsed.data, + ); + return NextResponse.json({ success: true, data: { url: await save(name), ...metadata } }); } - if(parsed.data.operation==='crop') { - const metadata=await processVideoFile(input,join(dir,'crop.mp4'),'crop',parsed.data.rect); - return NextResponse.json({success:true,data:{url:await save('crop.mp4'),...metadata}}); + if (parsed.data.operation === "trim") { + const metadata = await processVideoFile( + input, + join(dir, "clip.mp4"), + "trim", + undefined, + parsed.data.range, + ); + return NextResponse.json({ + success: true, + data: { url: await save("clip.mp4"), ...metadata }, + }); } - const metadata=await processVideoFile(input,join(dir,'audio.m4a'),'extract-audio'); - await processVideoFile(input,join(dir,'silent.mp4'),'silent-video'); - return NextResponse.json({success:true,data:{audioUrl:await save('audio.m4a'),url:await save('silent.mp4'),...metadata}}); - } catch(error) { - const message=error instanceof Error ? error.message : '视频处理失败'; - return NextResponse.json({success:false,error:message.startsWith('Command failed')?'视频处理失败,请确认文件完整且格式受支持':message},{status:400}); - } finally {running.delete(session.user.id);if(dir)await rm(dir,{recursive:true,force:true});} + if (parsed.data.operation === "crop") { + const metadata = await processVideoFile( + input, + join(dir, "crop.mp4"), + "crop", + parsed.data.rect, + ); + return NextResponse.json({ + success: true, + data: { url: await save("crop.mp4"), ...metadata }, + }); + } + const metadata = await processVideoFile(input, join(dir, "audio.m4a"), "extract-audio"); + await processVideoFile(input, join(dir, "silent.mp4"), "silent-video"); + return NextResponse.json({ + success: true, + data: { audioUrl: await save("audio.m4a"), url: await save("silent.mp4"), ...metadata }, + }); + } catch (error) { + const message = error instanceof Error ? error.message : "视频处理失败"; + return NextResponse.json( + { + success: false, + error: message.startsWith("Command failed") + ? "视频处理失败,请确认文件完整且格式受支持" + : message, + }, + { status: 400 }, + ); + } finally { + running.delete(session.user.id); + if (dir) await rm(dir, { recursive: true, force: true }); + } } diff --git a/components/canvas/LiblibVideoCanvasReplica.tsx b/components/canvas/LiblibVideoCanvasReplica.tsx index 167d8f435..af357ba82 100644 --- a/components/canvas/LiblibVideoCanvasReplica.tsx +++ b/components/canvas/LiblibVideoCanvasReplica.tsx @@ -99,6 +99,9 @@ import { videoGenerationService, workflowAnalysisService, } from "@/services/canvas"; +import { canvasGenerationCount, canvasImageRequest, canvasImageResultUrls } from "@/lib/video-canvas/generation-contract"; +import { anglePresets, angleRequestParameters } from "@/lib/video-canvas/angle-settings"; +import { downloadCanvasMedia } from "@/lib/video-canvas/download"; import { preserveGeneratedVideo } from "@/lib/video-canvas/archive-result"; import { VideoArchiveButton } from "@/components/canvas/VideoArchiveButton"; import { VideoPromptToolDialog } from "@/components/canvas/VideoPromptToolDialog"; @@ -638,6 +641,7 @@ type GridSplitState = { }; type AngleEditorSettings = { + preset?: string; horizontalAngle: number; promptEnabled: boolean; verticalAngle: number; @@ -2646,6 +2650,9 @@ function normalizeStoryScriptRow(row: unknown, index: number, nodeId: string): S if (!row || typeof row !== "object") return null; const record = row as Record; + if (["shot","duration","scene","sceneDescription","shotSize","lighting","dialogue","backgroundMusic","voiceAndSfx","cameraMovement","imagePrompt","motionPrompt","keyframeUrl"].every(key => typeof record[key] === "string")) { + return { ...record, id: typeof record.id === "string" ? record.id : `${nodeId}-story-row-${index}` } as StoryScriptRow; + } const shot = formatStoryShot(record.shot || record.id, index); const duration = String(record.duration || record.length || record.time || "5s").trim(); const scene = String(record.scene || record.title || record.sceneDescription || `镜头 ${shot}`).trim(); @@ -2705,11 +2712,7 @@ function getStoryScriptRows(node: CloneNode | null | undefined): StoryScriptRow[ const rows = rawRows .map((row, index) => normalizeStoryScriptRow(row, index, nodeId)) .filter((row): row is StoryScriptRow => Boolean(row)); - if (rows.length > 0) return rows; - return DEFAULT_STORY_SCRIPT_ROWS.map((row, index) => ({ - ...row, - id: `${nodeId}-story-fallback-${index}`, - })); + return rows; } type StoryWorkflowAsset = { @@ -2719,120 +2722,14 @@ type StoryWorkflowAsset = { type: "role" | "scene" | "prop"; }; -const STORY_WORKFLOW_ASSET_FALLBACKS: StoryWorkflowAsset[] = [ - { - description: "现代办公室加班女性,穿越前状态,疲惫压抑。", - imageUrl: SCRIPT_CHARACTER_ONE_URL, - name: "现代沈昭昭", - type: "role", - }, - { - description: "盛唐男装官服女主,朝堂危局中的核心角色。", - imageUrl: SCRIPT_CHARACTER_TWO_URL, - name: "沈昭昭", - type: "role", - }, - { - description: "龙椅之上威严注视的盛唐皇帝。", - imageUrl: CHARACTER_IMAGE_URL, - name: "皇帝", - type: "role", - }, - { - description: "恢弘金色朝堂,百官列立,压迫感强。", - imageUrl: FIRST_FRAME_IMAGE_URL, - name: "金銮殿", - type: "scene", - }, - { - description: "晴朗盛唐长安,街市与山河展开。", - imageUrl: VIDEO_FIRST_FRAME_SOURCE_URL, - name: "长安城白昼", - type: "scene", - }, - { - description: "夜色长安灯火连绵,结尾史诗气氛。", - imageUrl: AUDIO_REFERENCE_IMAGE_URL, - name: "长安城夜景", - type: "scene", - }, - { - description: "侍卫拔刀形成威胁,刀光映入角色瞳孔。", - imageUrl: FIRST_FRAME_IMAGE_URL, - name: "侍卫长刀", - type: "prop", - }, -]; - -function classifyStoryWorkflowAsset(name: string): StoryWorkflowAsset["type"] { - if (/殿|城|长安|办公室|街|山河|夜景|白昼|场景/.test(name)) return "scene"; - if (/刀|剑|笔|卷|书|道具/.test(name)) return "prop"; - return "role"; -} - -function getStoryWorkflowAssetImage(type: StoryWorkflowAsset["type"], index: number) { - if (type === "role") { - return [SCRIPT_CHARACTER_ONE_URL, SCRIPT_CHARACTER_TWO_URL, CHARACTER_IMAGE_URL][index % 3]; - } - if (type === "scene") { - return [FIRST_FRAME_IMAGE_URL, VIDEO_FIRST_FRAME_SOURCE_URL, AUDIO_REFERENCE_IMAGE_URL][index % 3]; - } - return [FIRST_FRAME_IMAGE_URL, CHARACTER_IMAGE_URL][index % 2]; -} - -function normalizeStoryWorkflowAssetName(rawName: string) { - const knownAsset = STORY_WORKFLOW_ASSET_FALLBACKS.find((asset) => rawName.includes(asset.name)); - if (knownAsset) return knownAsset.name; - return rawName - .replace(/(伏案|抬头|猛然|跪|目光|缓缓|站直|吟诵|转身|低声|满脸|发现|身着|看向|穿过|指着|厉声|内|外).*/, "") - .trim() - .slice(0, 8); -} - function getStoryWorkflowAssetGroups(rows: StoryScriptRow[]) { - const detectedNames = Array.from( - new Set( - rows.flatMap((row) => - [ - row.scene, - row.sceneDescription, - row.dialogue, - row.voiceAndSfx, - row.imagePrompt, - row.motionPrompt, - ] - .join(" ") - .match(/@([\u4e00-\u9fa5A-Za-z0-9_-]{2,18})/g) - ?.map((item) => normalizeStoryWorkflowAssetName(item.slice(1))) - .filter(Boolean) || [] - ) - ) - ); - const detectedAssets = detectedNames.map((name, index) => { - const type = classifyStoryWorkflowAsset(name); - return { - description: - type === "role" - ? "由脚本中 @ 引用自动抽取的角色资产。" - : type === "scene" - ? "由脚本镜头引用自动抽取的场景资产。" - : "由镜头动作和威胁物件抽取的道具资产。", - imageUrl: getStoryWorkflowAssetImage(type, index), - name, - type, - }; - }); - - const byName = new Map(); - [...detectedAssets, ...STORY_WORKFLOW_ASSET_FALLBACKS].forEach((asset) => { - if (!byName.has(asset.name)) byName.set(asset.name, asset); - }); - const assets = Array.from(byName.values()); - return { - prop: assets.filter((asset) => asset.type === "prop"), - role: assets.filter((asset) => asset.type === "role"), - scene: assets.filter((asset) => asset.type === "scene"), - }; + const names = [...new Set(rows.flatMap(row => + [row.scene, row.sceneDescription, row.imagePrompt, row.motionPrompt].join(" ").match(/@[\p{L}\p{N}_-]+/gu) || []))]; + const assets: StoryWorkflowAsset[] = names.map(name => ({ + name: name.slice(1), description: "脚本引用,请确认类型并准备图片", imageUrl: "", + type: /场景|街|城|室|厅|山|海/.test(name) ? "scene" : /刀|剑|杯|道具/.test(name) ? "prop" : "role", + })); + return { role: assets.filter(a => a.type === "role"), scene: assets.filter(a => a.type === "scene"), prop: assets.filter(a => a.type === "prop") }; } function buildStorySceneText(row: StoryScriptRow) { @@ -2858,7 +2755,7 @@ function buildStoryVideoPrompt(row: StoryScriptRow) { return [ row.imagePrompt, motionPrompt, - "[视觉风格:盛唐·国风二次元]", + ].filter(Boolean).join("\n\n"); } @@ -2888,334 +2785,424 @@ function StoryScriptWorkflowDialog({ node, onClose, onRun, + onPatch, onStepChange, step, + imageNodes, + imageModel, + analysisModel, }: { node: CloneNode; onClose: () => void; onRun: () => void; + onPatch: (patch: Record) => void; onStepChange: (step: StoryWorkflowStep) => void; step: StoryWorkflowStep; + imageNodes: CloneNode[]; + imageModel?: string; + analysisModel?: string; }) { const rows = getStoryScriptRows(node); - const assetGroups = getStoryWorkflowAssetGroups(rows); - const assetCount = assetGroups.role.length + assetGroups.scene.length + assetGroups.prop.length; - const [promptRow, setPromptRow] = useState(null); - const steps: Array<{ key: StoryWorkflowStep; label: string; subLabel: string }> = [ - { key: "shots", label: "确认镜头", subLabel: `${rows.length}个镜头已就绪` }, - { key: "assets", label: "准备资产", subLabel: `${assetCount}/${assetCount} 已生成` }, - { key: "prompts", label: "合成提示词", subLabel: `${rows.length}/${rows.length} 已合成` }, + const detected = getStoryWorkflowAssetGroups(rows); + const assets = ( + Array.isArray(node.draftValues?.storyAssets) + ? node.draftValues.storyAssets + : [...detected.role, ...detected.scene, ...detected.prop] + ) as StoryWorkflowAsset[]; + const style = String(node.draftValues?.storyStyle || ""); + const [busy, setBusy] = useState(null); + const [error, setError] = useState(""); + const uploadInput = useRef(null); + const uploadIndex = useRef(0); + const latest = useRef({ rows, assets }); + latest.current = { rows, assets }; + const updateAsset = (index: number, patch: Partial) => + onPatch({ + storyAssets: latest.current.assets.map((asset, i) => + i === index ? { ...asset, ...patch } : asset, + ), + }); + const updateRow = (index: number, patch: Partial) => + onPatch({ + rows: latest.current.rows.map((row, i) => (i === index ? { ...row, ...patch } : row)), + }); + const compose = async (index?: number) => { + if (busy) return; + const source = index === undefined ? rows : [rows[index]]; + if (!source.length) return; + setBusy("prompts"); + setError(""); + try { + const response = await workflowAnalysisService.analyze({ + mode: "story-script", + preferredModelRef: analysisModel, + prompt: `为以下镜头完善 imagePrompt 和 motionPrompt。保持镜头数量、顺序、故事内容和时长不变。返回 analysisRows。风格:${style || "遵循原脚本"}。资产描述:${JSON.stringify(assets.map(({ name, description, type }) => ({ name, description, type })))}。镜头:${JSON.stringify(source)}`, + imageUrls: assets + .map((asset) => asset.imageUrl) + .filter(Boolean) + .slice(0, 12), + nodeTitle: node.title, + }); + const result = response.data?.analysisRows; + if ( + !response.success || + !result || + result.length !== source.length || + result.some((row) => !row.imagePrompt || !row.motionPrompt) + ) + throw Error(response.error || "未返回完整镜头提示词,原内容已保留"); + const next = latest.current.rows.map((row, i) => { + const generated = index === undefined ? result[i] : i === index ? result[0] : null; + return generated + ? { ...row, imagePrompt: generated.imagePrompt!, motionPrompt: generated.motionPrompt! } + : row; + }); + onPatch({ rows: next }); + toast.success(`已合成 ${result.length} 个镜头提示词`); + } catch (e) { + setError(e instanceof Error ? e.message : "提示词合成失败"); + } finally { + setBusy(null); + } + }; + const generateAsset = async (index: number) => { + if (busy) return; + setBusy(`asset-${index}`); + setError(""); + try { + const asset = assets[index]; + const response = await imageGenerationService.generateImage({ + allowFailover: false, + model: imageModel, + imageCount: 1, + prompt: [asset.name, asset.description, style].filter(Boolean).join(","), + aspectRatio: "1:1", + }); + const url = response.success ? getFirstGeneratedImageUrl(response) : ""; + if (!url) throw Error(response.error || "资产图片生成失败"); + updateAsset(index, { imageUrl: url }); + } catch (e) { + setError(e instanceof Error ? e.message : "资产生成失败"); + } finally { + setBusy(null); + } + }; + const field = + "w-full rounded border border-white/10 bg-white/5 p-2 text-xs outline-none focus:border-white/40"; + const button = + "rounded-lg border border-white/10 px-3 py-2 text-xs hover:bg-white/10 disabled:opacity-40"; + const fields: Array<[keyof StoryScriptRow, string]> = [ + ["shot", "镜号"], + ["duration", "时长"], + ["sceneDescription", "画面描述"], + ["shotSize", "景别"], + ["lighting", "光影"], + ["dialogue", "对白"], + ["voiceAndSfx", "音效"], + ["cameraMovement", "运镜"], + ["imagePrompt", "图像提示词"], + ["motionPrompt", "运动提示词"], ]; - const tableColumns = [ - "镜号", - "时长", - "画面描述", - "景别", - "光影氛围", - "对白·旁白", - "音效", - "运镜", - "最终提示词", - "操作", - ]; - const gridTemplateColumns = - "56px 56px 310px 72px 150px 140px 130px 128px 260px 162px"; - - const renderShotTable = (highlightPrompt = false) => ( -
-
-
-
- {tableColumns.map((item) => ( -
- {item} -
- ))} -
- {rows.map((row) => ( -
- {[ - row.shot, - row.duration, - row.sceneDescription, - row.shotSize, - row.lighting, - row.dialogue, - row.voiceAndSfx, - row.cameraMovement, - row.imagePrompt, - ].map((cell, index) => ( -
-
{cell}
-
- ))} -
- - -
-
- ))} -
-
-
- ); - - const renderAssetGroup = ( - title: string, - assets: StoryWorkflowAsset[], - emptyLabel: string, - ) => ( -
-

{title}

-
- {assets.map((asset) => ( -
- -
- {asset.name} - {asset.description} -
-
- -
- -
- {["替换图片", "AI 生角色", "清除图片"].map((item) => ( -
{item}
- ))} -
删除
-
-
-
-
- ))} - -
-
- ); - + const readyAssets = assets.filter((asset) => asset.imageUrl).length; + const readyPrompts = rows.filter( + (row) => row.imagePrompt.trim() && row.motionPrompt.trim(), + ).length; return (
event.stopPropagation()} > -
-
- {node.title || "脚本生成器"} -
-
- {steps.map((item, index) => { - const active = item.key === step; - return ( - - ); - })} -
-
- 3/3 完成后可批量生视频 -
- + ))} + +
- -
- {step === "shots" ? renderShotTable(false) : null} + {error && ( +

+ {error} +

+ )} +
{step === "assets" ? ( -
-
- 全局风格 - 盛唐·国风二次元 -
-
- {renderAssetGroup("角色", assetGroups.role, "新增")} - {renderAssetGroup("场景", assetGroups.scene, "新增")} - {renderAssetGroup("道具", assetGroups.prop, "新增")} -
-
- ) : null} - {step === "prompts" ? renderShotTable(true) : null} - -
- {step === "assets" ? ( - 资产已生成,如再次生成将会覆盖之前的图片/场景/道具等资产 - ) : step === "prompts" ? ( - 最终提示词已合成,可继续查看单镜提示词或重新批量合成 - ) : ( - - )} - {step === "shots" ? ( - - ) : null} - {step === "assets" ? ( - - ) : null} - {step === "prompts" ? ( - - ) : null} -
-
- - {promptRow ? ( -
-
-
-

第 {promptRow.shot} 镜:最终提示词

- -
-
-
-
-

分镜提示词

- - 分镜图提示词·已生成 - + <> + + {( + [ + ["role", "角色"], + ["scene", "场景"], + ["prop", "道具"], + ] as const + ).map(([type, label]) => ( +
+

{label}

+
+ {assets.map((asset, index) => + asset.type !== type ? null : ( +
+ {asset.imageUrl ? ( + {asset.name} + ) : ( +
+ 尚未准备图片 +
+ )} + updateAsset(index, { name: e.target.value })} + /> +