feat(video-canvas): connect prompt tools and durable media processing
This commit is contained in:
parent
5b0d75a738
commit
2ba093b714
50
app/api/canvas/media/upload/route.ts
Normal file
50
app/api/canvas/media/upload/route.ts
Normal file
@ -0,0 +1,50 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { NextResponse } from "next/server";
|
||||
import { auth } from "@/app/(auth)/auth";
|
||||
import { uploadFile } from "@/lib/storage/service";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
export const maxDuration = 180;
|
||||
const probe = promisify(execFile);
|
||||
const running = new Set<string>();
|
||||
const formats: Record<string, string> = {
|
||||
"video/mp4": "mp4", "video/quicktime": "mov", "video/webm": "webm",
|
||||
"audio/mpeg": "mp3", "audio/mp3": "mp3", "audio/mp4": "m4a", "audio/x-m4a": "m4a",
|
||||
"audio/wav": "wav", "audio/x-wav": "wav", "audio/ogg": "ogg", "audio/flac": "flac", "audio/webm": "webm",
|
||||
};
|
||||
|
||||
export async function POST(request: Request) {
|
||||
const session = await auth();
|
||||
if (!session?.user?.id) return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
|
||||
if (running.has(session.user.id) || running.size >= 4) return NextResponse.json({ success: false, error: "正在上传,请稍后重试" }, { status: 429 });
|
||||
running.add(session.user.id);
|
||||
let dir = "";
|
||||
try {
|
||||
const form = await request.formData();
|
||||
const file = form.get("file");
|
||||
if (!(file instanceof File) || !formats[file.type]) return NextResponse.json({ success: false, error: "请上传 MP4、MOV、WebM 视频,或 MP3、M4A、WAV、OGG、FLAC 音频" }, { status: 400 });
|
||||
if (!file.size || file.size > 125 * 1024 * 1024) return NextResponse.json({ success: false, error: "素材必须非空且不超过 125 MiB" }, { status: 413 });
|
||||
dir = await mkdtemp(join(tmpdir(), "canvas-upload-"));
|
||||
const input = join(dir, "input");
|
||||
const buffer = await file.arrayBuffer();
|
||||
await writeFile(input, Buffer.from(buffer));
|
||||
const { stdout } = await probe("ffprobe", ["-v", "error", "-protocol_whitelist", "file", "-format_whitelist", "mov,matroska,webm,mp3,wav,ogg,flac", "-show_streams", "-show_format", "-of", "json", input], { timeout: 15000, maxBuffer: 1024 * 1024 });
|
||||
const info = JSON.parse(stdout);
|
||||
const kind = file.type.startsWith("video/") ? "video" : "audio";
|
||||
const stream = info.streams?.find((item: { codec_type: string }) => item.codec_type === kind);
|
||||
const duration = Number(info.format?.duration);
|
||||
if (!stream || !Number.isFinite(duration) || duration <= 0 || duration > 600) return NextResponse.json({ success: false, error: "请上传内容完整、10 分钟以内的音视频" }, { status: 400 });
|
||||
const uploaded = await uploadFile(`canvas-${randomUUID()}.${formats[file.type]}`, buffer);
|
||||
return NextResponse.json({ success: true, data: { url: uploaded.url, duration, width: stream.width, height: stream.height } });
|
||||
} catch {
|
||||
return NextResponse.json({ success: false, error: "素材上传失败,请确认文件完整或稍后重试" }, { status: 400 });
|
||||
} finally {
|
||||
running.delete(session.user.id);
|
||||
if (dir) await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
57
app/api/canvas/prompt-tools/route.ts
Normal file
57
app/api/canvas/prompt-tools/route.ts
Normal file
@ -0,0 +1,57 @@
|
||||
import { generateText } from "ai";
|
||||
import { NextResponse } from "next/server";
|
||||
import { auth } from "@/app/(auth)/auth";
|
||||
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
||||
import { getEnabledAiModelsWithProvider, getSystemSetting } from "@/lib/db/queries";
|
||||
import { VIDEO_CANVAS_FUNCTION_CONFIG_KEY, normalizeVideoCanvasFunctionConfig } from "@/lib/video-canvas/config";
|
||||
import { promptToolErrorMessage, promptToolFunctionKey, promptToolSchema, selectPromptToolModel } from "@/lib/video-canvas/prompt-tools";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
export const maxDuration = 120;
|
||||
const running = new Set<string>();
|
||||
|
||||
export async function POST(request: Request) {
|
||||
const session = await auth();
|
||||
if (!session?.user?.id) return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
|
||||
const parsed = promptToolSchema.safeParse(await request.json().catch(() => null));
|
||||
if (!parsed.success) return NextResponse.json({ success: false, error: parsed.error.issues[0]?.message || "参数无效" }, { status: 400 });
|
||||
if (running.has(session.user.id)) return NextResponse.json({ success: false, error: "提示词处理中,请稍候" }, { status: 429 });
|
||||
running.add(session.user.id);
|
||||
try {
|
||||
const [setting, available] = await Promise.all([
|
||||
getSystemSetting({ key: VIDEO_CANVAS_FUNCTION_CONFIG_KEY }),
|
||||
getEnabledAiModelsWithProvider(false),
|
||||
]);
|
||||
const config = normalizeVideoCanvasFunctionConfig(setting?.value);
|
||||
const definition = config.functions[promptToolFunctionKey(parsed.data.mode)];
|
||||
if (!config.enabled || !definition?.enabled) return NextResponse.json({ success: false, error: "此提示词功能已停用" }, { status: 403 });
|
||||
const requested = parsed.data.model && parsed.data.model !== "auto"
|
||||
? parsed.data.model : definition.model && definition.model !== "auto"
|
||||
? definition.model : config.defaultMultimodalModel;
|
||||
let selected;
|
||||
try {
|
||||
selected = selectPromptToolModel(available
|
||||
.filter(item => item.provider.isEnabled && item.model.isEnabled && !item.model.isSubModel)
|
||||
.map(item => ({ ...item.model, providerType: item.provider.type })), requested);
|
||||
} catch (error) {
|
||||
return NextResponse.json({ success: false, error: (error as Error).message }, { status: 400 });
|
||||
}
|
||||
const model = await getLanguageModelById(selected.id);
|
||||
if (!model) return NextResponse.json({ success: false, error: "提示词模型初始化失败" }, { status: 502 });
|
||||
const { text } = await generateText({
|
||||
model,
|
||||
system: definition.prompt,
|
||||
prompt: parsed.data.prompt,
|
||||
temperature: parsed.data.mode === "translate" ? 0.1 : 0.4,
|
||||
maxOutputTokens: 4000,
|
||||
maxRetries: 0,
|
||||
abortSignal: AbortSignal.any([request.signal, AbortSignal.timeout(90_000)]),
|
||||
});
|
||||
if (!text.trim()) throw new Error("empty model output");
|
||||
return NextResponse.json({ success: true, data: { text: text.trim(), model: selected.id, modelName: selected.displayName || selected.modelId } });
|
||||
} catch (error) {
|
||||
return NextResponse.json({ success: false, error: promptToolErrorMessage(error) }, { status: 502 });
|
||||
} finally {
|
||||
running.delete(session.user.id);
|
||||
}
|
||||
}
|
||||
@ -1,6 +1,5 @@
|
||||
import {auth} from '@/app/(auth)/auth';
|
||||
import {NextResponse} from 'next/server';
|
||||
import {z} from 'zod';
|
||||
import {mkdtemp,readFile,rm} from 'node:fs/promises';
|
||||
import {tmpdir} from 'node:os';
|
||||
import {join} from 'node:path';
|
||||
@ -8,14 +7,14 @@ import {randomUUID} from 'node:crypto';
|
||||
import {uploadFile} from '@/lib/storage/service';
|
||||
import {downloadCanvasMedia} from '@/lib/video-canvas/download-media';
|
||||
import {processVideoFile} from '@/lib/video-canvas/media-processing';
|
||||
import {videoProcessSchema} from '@/lib/video-canvas/media-contract';
|
||||
export const runtime='nodejs';
|
||||
export const maxDuration=300;
|
||||
const running=new Set<string>();
|
||||
const schema=z.object({operation:z.enum(['crop','audio-video']),url:z.string().url(),rect:z.object({x:z.number(),y:z.number(),width:z.number(),height:z.number()}).optional()});
|
||||
export async function POST(request:Request) {
|
||||
const session=await auth();
|
||||
if(!session?.user?.id) return NextResponse.json({success:false,error:'未登录'},{status:401});
|
||||
const parsed=schema.safeParse(await request.json().catch(()=>null));
|
||||
const parsed=videoProcessSchema.safeParse(await request.json().catch(()=>null));
|
||||
if(!parsed.success) return NextResponse.json({success:false,error:'视频处理参数无效'},{status:400});
|
||||
if(running.has(session.user.id)||running.size>=2) return NextResponse.json({success:false,error:'视频处理中,请稍后再提交'},{status:429});
|
||||
running.add(session.user.id);
|
||||
@ -25,6 +24,10 @@ export async function POST(request:Request) {
|
||||
const input=join(dir,'input');
|
||||
await downloadCanvasMedia(parsed.data.url,input);
|
||||
const save=async(name:string)=>{const buffer=await readFile(join(dir,name));return (await uploadFile(`canvas-video-${randomUUID()}-${name}`,new Uint8Array(buffer).buffer)).url;};
|
||||
if(parsed.data.operation==='trim') {
|
||||
const metadata=await processVideoFile(input,join(dir,'clip.mp4'),'trim',undefined,parsed.data.range);
|
||||
return NextResponse.json({success:true,data:{url:await save('clip.mp4'),...metadata}});
|
||||
}
|
||||
if(parsed.data.operation==='crop') {
|
||||
const metadata=await processVideoFile(input,join(dir,'crop.mp4'),'crop',parsed.data.rect);
|
||||
return NextResponse.json({success:true,data:{url:await save('crop.mp4'),...metadata}});
|
||||
|
||||
@ -883,7 +883,7 @@ export async function POST(request: Request) {
|
||||
? [...inlinedImageReferences, ...extractedVideoFrames].slice(0, 6)
|
||||
: [];
|
||||
|
||||
const { text } = await generateText({
|
||||
const { text, finishReason } = await generateText({
|
||||
model,
|
||||
messages: [
|
||||
{
|
||||
@ -895,10 +895,18 @@ export async function POST(request: Request) {
|
||||
},
|
||||
],
|
||||
temperature: 0.2,
|
||||
maxTokens: parsed.data.mode === "story-script" ? 4200 : parsed.data.mode === "video-analysis" ? 2200 : 1600,
|
||||
} as any);
|
||||
maxOutputTokens: parsed.data.mode === "story-script" ? 12000 : parsed.data.mode === "video-analysis" ? 2200 : 1600,
|
||||
maxRetries: 0,
|
||||
abortSignal: AbortSignal.any([request.signal, AbortSignal.timeout(150_000)]),
|
||||
});
|
||||
|
||||
const parsedJson = extractJsonBlock<Record<string, unknown>>(text) || {};
|
||||
if (finishReason === "length" || !text.trim()) {
|
||||
return NextResponse.json({ success: false, error: "分析结果不完整,请缩短输入或重新生成" }, { status: 502 });
|
||||
}
|
||||
const parsedJson = extractJsonBlock<Record<string, unknown>>(text);
|
||||
if (!parsedJson) {
|
||||
return NextResponse.json({ success: false, error: "分析模型未返回完整结构化结果,请重新生成" }, { status: 502 });
|
||||
}
|
||||
const summary =
|
||||
normalizeText(parsedJson.summary) ||
|
||||
normalizeText(text).slice(0, 160);
|
||||
@ -982,12 +990,13 @@ export async function POST(request: Request) {
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("[Canvas Workflow Analyze] Error:", error);
|
||||
const timedOut = error instanceof Error && /timeout|timed out|abort/i.test(`${error.name} ${error.message}`);
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : "工作流分析失败",
|
||||
error: timedOut ? "分析超时,请稍后重试或选择其他分析模型" : error instanceof Error ? error.message : "工作流分析失败",
|
||||
},
|
||||
{ status: 500 }
|
||||
{ status: timedOut ? 504 : 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@ -92,6 +92,8 @@ import {
|
||||
videoGenerationService,
|
||||
workflowAnalysisService,
|
||||
} from "@/services/canvas";
|
||||
import { VideoPromptToolDialog } from "@/components/canvas/VideoPromptToolDialog";
|
||||
import type { PromptToolMode } from "@/lib/video-canvas/prompt-tools";
|
||||
import { useVideoCanvasPersistence } from "@/hooks/use-video-canvas-persistence";
|
||||
import type { CanvasInitialProject } from "@/types/canvas";
|
||||
import {
|
||||
@ -3404,7 +3406,7 @@ function getConfiguredToolType(definition: VideoCanvasFunctionDefinition) {
|
||||
if (definition.kind === "audio") return "audio";
|
||||
if (definition.kind === "video" || definition.route === "canvas-generate-video") return "video";
|
||||
if (definition.kind === "workflow") return "compose";
|
||||
if (definition.kind === "multimodal" || definition.route === "canvas-workflow-analyze") return "text";
|
||||
if (definition.kind === "multimodal" || definition.route === "canvas-workflow-analyze" || definition.route === "canvas-prompt-tools") return "text";
|
||||
return "image";
|
||||
}
|
||||
|
||||
@ -3545,7 +3547,7 @@ function getFunctionModel(
|
||||
const fallback =
|
||||
definition.kind === "video"
|
||||
? config.defaultVideoModel
|
||||
: definition.kind === "multimodal"
|
||||
: definition.kind === "multimodal" || definition.route === "canvas-prompt-tools"
|
||||
? config.defaultMultimodalModel
|
||||
: config.defaultImageModel;
|
||||
return normalizeModelRef(definition.model || fallback) || undefined;
|
||||
@ -3558,7 +3560,7 @@ function getFunctionModelValue(
|
||||
const fallback =
|
||||
definition.kind === "video"
|
||||
? config.defaultVideoModel
|
||||
: definition.kind === "multimodal"
|
||||
: definition.kind === "multimodal" || definition.route === "canvas-prompt-tools"
|
||||
? config.defaultMultimodalModel
|
||||
: config.defaultImageModel;
|
||||
return String(definition.model || fallback || "auto").trim() || "auto";
|
||||
@ -4404,6 +4406,11 @@ function normalizeTargetModuleNode(node: CloneNode, kind: AddModuleKind, fallbac
|
||||
}
|
||||
|
||||
function normalizeRestoredNode(node: CloneNode, fallbackIndex: number): CloneNode {
|
||||
if (getDraftString(node, "uploadStatus") === "uploading" ||
|
||||
(node.kind === "media" && getNodeMediaUrl(node).startsWith("blob:"))) {
|
||||
return { ...node, draftValues: { ...node.draftValues, mediaUrl: "", url: [],
|
||||
uploadStatus: "failed", uploadError: "上次素材未完成上传,请重新上传文件" } };
|
||||
}
|
||||
const action = getDraftString(node, "action");
|
||||
const isCharacterThreeViewResultNode = isCharacterThreeViewNode(node);
|
||||
const isNineGridResultNode =
|
||||
@ -4781,6 +4788,8 @@ export function LiblibVideoCanvasReplica({
|
||||
const [historyListDownloadPendingId, setHistoryListDownloadPendingId] = useState<string | null>(null);
|
||||
const [historyWatermarkRulesOpen, setHistoryWatermarkRulesOpen] = useState(false);
|
||||
const [isProjectLoading, setIsProjectLoading] = useState(true);
|
||||
const [projectLoadError, setProjectLoadError] = useState("");
|
||||
const [projectLoadAttempt, setProjectLoadAttempt] = useState(0);
|
||||
const initialProjectRef = useRef(initialProject);
|
||||
initialProjectRef.current = initialProject;
|
||||
const [loadedProject, setLoadedProject] = useState<{ id: string; revision: string } | null>(null);
|
||||
@ -4809,6 +4818,8 @@ export function LiblibVideoCanvasReplica({
|
||||
const [subtitleEraseRegions, setSubtitleEraseRegions] = useState<Record<string, SubtitleEraseRegion[]>>({});
|
||||
const [subtitleEraseRegionHistory, setSubtitleEraseRegionHistory] =
|
||||
useState<Record<string, SubtitleEraseRegionHistory>>({});
|
||||
const clipRequestRunning = useRef(false);
|
||||
const [clipProcessingNodeId, setClipProcessingNodeId] = useState<string | null>(null);
|
||||
const [videoClipRanges, setVideoClipRanges] = useState<Record<string, VideoClipRange>>({});
|
||||
const [videoAnalysisStates, setVideoAnalysisStates] = useState<Record<string, VideoAnalysisState>>({});
|
||||
const [videoCropGenerateStates, setVideoCropGenerateStates] = useState<Record<string, VideoCropGenerateState>>({});
|
||||
@ -4898,10 +4909,11 @@ export function LiblibVideoCanvasReplica({
|
||||
|
||||
async function loadProject() {
|
||||
setIsProjectLoading(true);
|
||||
setProjectLoadError("");
|
||||
setLoadedProject(null);
|
||||
try {
|
||||
if (initialProjectRef.current?.id === canvasId) hydrateProject(initialProjectRef.current);
|
||||
const response = await fetch(`/api/canvas/${encodeURIComponent(canvasId)}`);
|
||||
const response = await fetch(`/api/canvas/${encodeURIComponent(canvasId)}`, { signal: AbortSignal.timeout(45_000) });
|
||||
let result = await response.json();
|
||||
if (!active) return;
|
||||
if (response.status === 404) {
|
||||
@ -4923,14 +4935,16 @@ export function LiblibVideoCanvasReplica({
|
||||
hydrateProject(result.data);
|
||||
setLoadedProject({ id: canvasId, revision: String(result.data.updatedAt) });
|
||||
} catch (error) {
|
||||
if (active) toast.error(error instanceof Error ? error.message : "加载画布失败,请刷新重试");
|
||||
if (active) setProjectLoadError(error instanceof Error && error.name === "TimeoutError"
|
||||
? "画布加载超时,请检查网络后重试"
|
||||
: error instanceof Error ? error.message : "加载画布失败,请重试");
|
||||
} finally {
|
||||
if (active) setIsProjectLoading(false);
|
||||
}
|
||||
}
|
||||
void loadProject();
|
||||
return () => { active = false; };
|
||||
}, [canvasId]);
|
||||
}, [canvasId, projectLoadAttempt]);
|
||||
|
||||
const canvasSnapshot = useMemo(() => JSON.stringify({
|
||||
title: projectTitle,
|
||||
@ -5654,19 +5668,48 @@ export function LiblibVideoCanvasReplica({
|
||||
const addUploadedFilesAtPoint = useCallback(
|
||||
(files: File[], center: { x: number; y: number }) => {
|
||||
if (files.length === 0) return;
|
||||
const newNodes = files.slice(0, 10).map((file, index) => {
|
||||
const created = createUploadedMediaNode(file, URL.createObjectURL(file), 0, 0, nodes.length + index + 1);
|
||||
return {
|
||||
...created,
|
||||
x: Math.round(center.x - created.width / 2 + index * 450),
|
||||
const batch = files.slice(0, 10);
|
||||
const newNodes = batch.map((file, index) => {
|
||||
const created = createUploadedMediaNode(file, "", 0, 0, nodes.length + index + 1);
|
||||
return { ...created, x: Math.round(center.x - created.width / 2 + index * 450),
|
||||
y: Math.round(center.y - created.height / 2),
|
||||
};
|
||||
draftValues: { ...created.draftValues, uploadStatus: "uploading" } };
|
||||
});
|
||||
addNodes(newNodes, []);
|
||||
setActivePanel(null);
|
||||
setCanvasCommand(null);
|
||||
setConnectionMenu(null);
|
||||
toast.success(files.length === 1 ? `已添加 ${newNodes[0]?.title || "素材"}` : `已添加 ${newNodes.length} 个素材`);
|
||||
void (async () => {
|
||||
for (const [index, file] of batch.entries()) {
|
||||
const id = newNodes[index].id;
|
||||
try {
|
||||
let data: { url: string; duration?: number; width?: number; height?: number };
|
||||
if (file.type.startsWith("image/")) {
|
||||
const result = await imageGenerationService.uploadImage(file);
|
||||
if (!result.success || !result.data?.url) throw new Error(result.error || "图片上传失败");
|
||||
data = result.data;
|
||||
} else {
|
||||
const form = new FormData();
|
||||
form.append("file", file);
|
||||
const response = await fetch("/api/canvas/media/upload", { method: "POST", body: form, signal: AbortSignal.timeout(180_000) });
|
||||
const result = await response.json();
|
||||
if (!response.ok || !result.success || !result.data?.url) throw new Error(result.error || "上传失败");
|
||||
data = result.data;
|
||||
}
|
||||
setNodes(current => current.map(node => node.id === id ? { ...node, draftValues: {
|
||||
...node.draftValues, mediaUrl: data.url, url: [data.url], uploadStatus: "complete",
|
||||
durationSeconds: data.duration, outputSizeLabel: data.width && data.height ? `${data.width} × ${data.height}` : undefined,
|
||||
} } : node));
|
||||
toast.success(`${newNodes[index].title} 上传完成`);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "上传失败,请重新上传";
|
||||
setNodes(current => current.map(node => node.id === id ? { ...node, draftValues: {
|
||||
...node.draftValues, uploadStatus: "failed", uploadError: message,
|
||||
} } : node));
|
||||
toast.error(`${newNodes[index].title}:${message}`);
|
||||
}
|
||||
}
|
||||
})();
|
||||
},
|
||||
[addNodes, nodes.length]
|
||||
);
|
||||
@ -8540,46 +8583,46 @@ export function LiblibVideoCanvasReplica({
|
||||
);
|
||||
|
||||
const updateVideoClipRange = useCallback((nodeId: string, nextRange: VideoClipRange) => {
|
||||
setVideoClipRanges((current) => ({
|
||||
...current,
|
||||
[nodeId]: {
|
||||
start: Math.max(0, Math.min(nextRange.start, 0.94)),
|
||||
end: Math.max(Math.min(nextRange.start + 0.04, 1), Math.min(nextRange.end, 1)),
|
||||
},
|
||||
}));
|
||||
const start = Math.max(0, Math.min(nextRange.start, 0.94));
|
||||
const range = { start, end: Math.max(start + 0.04, Math.min(nextRange.end, 1)) };
|
||||
setVideoClipRanges(current => ({ ...current, [nodeId]: range }));
|
||||
setNodes(current => current.map(node => node.id === nodeId
|
||||
? { ...node, draftValues: { ...node.draftValues, clipRange: range } } : node));
|
||||
}, []);
|
||||
|
||||
const applyVideoClipSelection = useCallback(
|
||||
(nodeId: string) => {
|
||||
const node = nodes.find((item) => item.id === nodeId);
|
||||
if (!node) return;
|
||||
const existingRange = normalizeVideoClipRange(
|
||||
videoClipRanges[nodeId] || nodes.find((item) => item.id === nodeId)?.draftValues?.clipRange
|
||||
);
|
||||
const normalizedRange =
|
||||
existingRange.end > existingRange.start
|
||||
? existingRange
|
||||
: { start: existingRange.start, end: Math.min(1, existingRange.start + 0.12) };
|
||||
const durationSeconds = getVideoNodeDurationSeconds(node) * Math.max(0.04, normalizedRange.end - normalizedRange.start);
|
||||
setNodes((current) =>
|
||||
current.map((item) =>
|
||||
item.id === nodeId
|
||||
? {
|
||||
...item,
|
||||
draftValues: {
|
||||
...(item.draftValues || {}),
|
||||
clipDurationLabel: formatClipDurationLabel(durationSeconds),
|
||||
clipRange: normalizedRange,
|
||||
},
|
||||
}
|
||||
: item
|
||||
)
|
||||
);
|
||||
const applyVideoClipSelection = useCallback(async (nodeId: string) => {
|
||||
const source = nodes.find(item => item.id === nodeId);
|
||||
if (!source || clipRequestRunning.current) return;
|
||||
const url = getNodeReferenceMediaUrl(source) || getNodeMediaUrl(source);
|
||||
if (!url) { toast.error("请先上传视频"); return; }
|
||||
const range = normalizeVideoClipRange(videoClipRanges[nodeId] || source.draftValues?.clipRange);
|
||||
clipRequestRunning.current = true;
|
||||
setClipProcessingNodeId(nodeId);
|
||||
try {
|
||||
const response = await fetch("/api/canvas/video/process", {
|
||||
method: "POST", headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ operation: "trim", url, range }),
|
||||
signal: AbortSignal.timeout(295_000),
|
||||
});
|
||||
const result = await response.json();
|
||||
if (!response.ok || !result.success || !result.data?.url) throw new Error(result.error || "视频剪辑失败");
|
||||
const created = createMediaResourceNode({ action: "VIDEO_TRIM", mediaType: "video", mediaUrl: result.data.url,
|
||||
title: `${source.title} · 剪辑`, x: source.x + source.width + 80, y: source.y });
|
||||
const clipNode = { ...created, subtitle: "剪辑结果", draftValues: { ...created.draftValues,
|
||||
generated: true, generationStatus: "complete", generationProgress: 100, uploadStatus: "complete",
|
||||
durationSeconds: result.data.duration, outputSizeLabel: `${result.data.width} × ${result.data.height}`,
|
||||
sourceVideoNodeId: source.id, sourceClipRange: range } };
|
||||
addNodes([clipNode], [{ id: makeId("edge"), source: nodeId, target: clipNode.id }]);
|
||||
setSelectedNodeId(clipNode.id);
|
||||
setActiveVideoToolbarMode(null);
|
||||
toast.success("已记录当前剪辑范围,视频裁剪导出后端待接入。");
|
||||
},
|
||||
[nodes, videoClipRanges]
|
||||
);
|
||||
toast.success("视频剪辑完成,已生成新视频");
|
||||
} catch (error) {
|
||||
toast.error(error instanceof Error ? error.message : "视频剪辑失败,原视频已保留");
|
||||
} finally {
|
||||
clipRequestRunning.current = false;
|
||||
setClipProcessingNodeId(null);
|
||||
}
|
||||
}, [addNodes, nodes, videoClipRanges]);
|
||||
|
||||
const downloadMediaNode = useCallback((node: CloneNode) => {
|
||||
const mediaUrl = getNodeMediaUrl(node) || getNodeReferenceMediaUrl(node);
|
||||
@ -10914,6 +10957,7 @@ export function LiblibVideoCanvasReplica({
|
||||
) : null}
|
||||
{activeVideoToolbarMode === "clip" ? (
|
||||
<VideoClipEditorBar
|
||||
busy={clipProcessingNodeId === selectedVideoMediaNode.id}
|
||||
anchor={selectedVideoAnchor}
|
||||
clipRange={
|
||||
normalizeVideoClipRange(
|
||||
@ -11508,9 +11552,17 @@ export function LiblibVideoCanvasReplica({
|
||||
/>
|
||||
) : null}
|
||||
|
||||
{isBusy ? (
|
||||
{projectLoadError ? (
|
||||
<div className="fixed inset-0 z-[1900] flex items-center justify-center bg-black/65" data-canvas-overlay="true" onPointerDown={event => event.stopPropagation()}>
|
||||
<div role="alert" className="max-w-sm rounded-2xl border border-white/15 bg-[#262626] p-6 text-center text-white">
|
||||
<p className="font-medium">画布加载失败</p><p className="mt-2 text-sm text-white/60">{projectLoadError}</p>
|
||||
<p className="mt-2 text-xs text-white/45">重新加载成功后即可继续编辑。</p>
|
||||
<button className="mt-4 rounded-lg bg-white px-4 py-2 text-sm text-black" onClick={() => setProjectLoadAttempt(value => value + 1)} type="button">重新加载画布</button>
|
||||
</div>
|
||||
</div>
|
||||
) : isBusy ? (
|
||||
<div
|
||||
className="pointer-events-none fixed left-1/2 top-20 z-[45] flex -translate-x-1/2 items-center gap-2 rounded-[8px] border border-white/10 bg-[#202020]/88 px-3 py-2 text-[12px] text-white/62 shadow-[0_14px_36px_rgba(0,0,0,0.34)] backdrop-blur-[10px]"
|
||||
className="fixed inset-0 z-[1900] flex items-center justify-center gap-2 bg-black/45 text-sm text-white/70"
|
||||
data-canvas-overlay="true"
|
||||
>
|
||||
<Loader2 className="h-4 w-4 animate-spin" />
|
||||
@ -13445,6 +13497,7 @@ function VideoStoryCanvasNode({ node, selected }: { node: CloneNode; selected: b
|
||||
|
||||
function VideoClipEditorBar({
|
||||
anchor,
|
||||
busy = false,
|
||||
clipRange,
|
||||
durationSeconds,
|
||||
onApply,
|
||||
@ -13454,6 +13507,7 @@ function VideoClipEditorBar({
|
||||
title,
|
||||
}: {
|
||||
anchor?: PresetControlAnchor;
|
||||
busy?: boolean;
|
||||
clipRange: VideoClipRange;
|
||||
durationSeconds: number;
|
||||
onApply: () => void;
|
||||
@ -13470,7 +13524,7 @@ function VideoClipEditorBar({
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (!dragHandle) return;
|
||||
if (!dragHandle || busy) return;
|
||||
const handlePointerMove = (event: PointerEvent) => {
|
||||
const rect = trackRef.current?.getBoundingClientRect();
|
||||
if (!rect || rect.width <= 0) return;
|
||||
@ -13494,7 +13548,7 @@ function VideoClipEditorBar({
|
||||
window.removeEventListener("pointermove", handlePointerMove);
|
||||
window.removeEventListener("pointerup", handlePointerUp);
|
||||
};
|
||||
}, [clipRange.end, clipRange.start, dragHandle, onChange]);
|
||||
}, [busy, clipRange.end, clipRange.start, dragHandle, onChange]);
|
||||
|
||||
const selectionWidth = Math.max(clipRange.end - clipRange.start, 0.04);
|
||||
const durationLabel = formatClipDurationLabel(durationSeconds * selectionWidth);
|
||||
@ -13540,6 +13594,7 @@ function VideoClipEditorBar({
|
||||
>
|
||||
<button
|
||||
className="absolute left-[-8px] top-1/2 h-7 w-4 -translate-y-1/2 rounded-full bg-white shadow-[0_6px_18px_rgba(0,0,0,0.34)]"
|
||||
disabled={busy}
|
||||
onPointerDown={(event) => {
|
||||
event.stopPropagation();
|
||||
setDragHandle("start");
|
||||
@ -13548,6 +13603,7 @@ function VideoClipEditorBar({
|
||||
/>
|
||||
<button
|
||||
className="absolute right-[-8px] top-1/2 h-7 w-4 -translate-y-1/2 rounded-full bg-white shadow-[0_6px_18px_rgba(0,0,0,0.34)]"
|
||||
disabled={busy}
|
||||
onPointerDown={(event) => {
|
||||
event.stopPropagation();
|
||||
setDragHandle("end");
|
||||
@ -13569,6 +13625,8 @@ function VideoClipEditorBar({
|
||||
</button>
|
||||
<button
|
||||
className="flex h-10 w-10 items-center justify-center rounded-[10px] text-white/74 transition hover:bg-white/8 hover:text-white"
|
||||
disabled={busy}
|
||||
aria-label="重置剪辑范围"
|
||||
onClick={onReset}
|
||||
type="button"
|
||||
>
|
||||
@ -13576,10 +13634,13 @@ function VideoClipEditorBar({
|
||||
</button>
|
||||
<button
|
||||
className="flex h-11 w-11 items-center justify-center rounded-full bg-white text-black shadow-[0_10px_24px_rgba(255,255,255,0.18)] transition hover:bg-white/90"
|
||||
disabled={busy}
|
||||
aria-label={busy ? "正在导出剪辑" : "导出所选片段"}
|
||||
title={busy ? "正在导出剪辑" : "导出所选片段"}
|
||||
onClick={onApply}
|
||||
type="button"
|
||||
>
|
||||
<Check className="h-[20px] w-[20px]" strokeWidth={2.5} />
|
||||
{busy ? <Loader2 className="h-5 w-5 animate-spin" /> : <Check className="h-[20px] w-[20px]" strokeWidth={2.5} />}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
@ -14620,6 +14681,56 @@ function VideoTargetModuleComposer({
|
||||
const [assistMenu, setAssistMenu] = useState<"reference" | "mark" | "effects" | "motion" | "role" | null>(null);
|
||||
const [advancedSettingsOpen, setAdvancedSettingsOpen] = useState(false);
|
||||
const [prompt, setPrompt] = useState(initialPrompt);
|
||||
const promptEditorRef = useRef<HTMLDivElement>(null);
|
||||
// Keep typing browser-owned; remount only when applying an external replacement.
|
||||
const [promptEditorSeed, setPromptEditorSeed] = useState({ revision: 0, text: initialPrompt });
|
||||
const [promptToolMode, setPromptToolMode] = useState<PromptToolMode | null>(null);
|
||||
const [promptToolModel, setPromptToolModel] = useState("auto");
|
||||
const [promptToolBusy, setPromptToolBusy] = useState(false);
|
||||
const [promptToolOutput, setPromptToolOutput] = useState("");
|
||||
const [promptToolSource, setPromptToolSource] = useState("");
|
||||
const [promptToolError, setPromptToolError] = useState("");
|
||||
const promptToolRequest = useRef<AbortController | null>(null);
|
||||
const closePromptTool = () => {
|
||||
promptToolRequest.current?.abort();
|
||||
promptToolRequest.current = null;
|
||||
setPromptToolMode(null);
|
||||
setPromptToolBusy(false);
|
||||
};
|
||||
useEffect(() => {
|
||||
setPromptToolMode(null);
|
||||
setPromptToolBusy(false);
|
||||
return () => { promptToolRequest.current?.abort(); promptToolRequest.current = null; };
|
||||
}, [node.id]);
|
||||
const runPromptTool = async (mode: PromptToolMode, modelRef = promptToolModel) => {
|
||||
if (!prompt.trim()) { toast.info("请先输入提示词"); return; }
|
||||
promptToolRequest.current?.abort();
|
||||
const controller = new AbortController();
|
||||
promptToolRequest.current = controller;
|
||||
setPromptToolMode(mode);
|
||||
setPromptToolSource(prompt);
|
||||
setPromptToolOutput("");
|
||||
setPromptToolError("");
|
||||
setPromptToolBusy(true);
|
||||
try {
|
||||
const response = await fetch("/api/canvas/prompt-tools", {
|
||||
method: "POST", headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ mode, prompt, model: modelRef }),
|
||||
signal: AbortSignal.any([controller.signal, AbortSignal.timeout(100_000)]),
|
||||
});
|
||||
const result = await response.json();
|
||||
if (!response.ok || !result.success || !result.data?.text) throw new Error(result.error || "提示词处理失败");
|
||||
if (promptToolRequest.current === controller) setPromptToolOutput(result.data.text);
|
||||
} catch (error) {
|
||||
if (promptToolRequest.current === controller && !controller.signal.aborted) {
|
||||
setPromptToolError(error instanceof Error && error.name === "TimeoutError"
|
||||
? "提示词处理超时,请重试。原提示词已保留。"
|
||||
: error instanceof Error ? error.message : "提示词处理失败,原文已保留");
|
||||
}
|
||||
} finally {
|
||||
if (promptToolRequest.current === controller) { promptToolRequest.current = null; setPromptToolBusy(false); }
|
||||
}
|
||||
};
|
||||
const [markMenuAnchor, setMarkMenuAnchor] = useState<{
|
||||
id: string;
|
||||
left: number;
|
||||
@ -14653,6 +14764,11 @@ function VideoTargetModuleComposer({
|
||||
|
||||
useEffect(() => {
|
||||
setActiveTab(initialTab);
|
||||
const editorClone = promptEditorRef.current?.cloneNode(true) as HTMLElement | undefined;
|
||||
editorClone?.querySelectorAll("[data-magic-id]").forEach(element => element.remove());
|
||||
if ((editorClone?.textContent || "").replace(/\u00a0/g, " ").trim() !== initialPrompt) {
|
||||
setPromptEditorSeed(current => ({ revision: current.revision + 1, text: initialPrompt }));
|
||||
}
|
||||
setPrompt(initialPrompt);
|
||||
setSettings(initialSettings);
|
||||
}, [initialPrompt, initialTab, node.id]);
|
||||
@ -14781,6 +14897,17 @@ function VideoTargetModuleComposer({
|
||||
width: composerWidth,
|
||||
}}
|
||||
>
|
||||
{promptToolMode ? <VideoPromptToolDialog
|
||||
mode={promptToolMode} busy={promptToolBusy} error={promptToolError}
|
||||
output={promptToolOutput} source={promptToolSource} currentPrompt={prompt}
|
||||
model={promptToolModel}
|
||||
models={filterCanvasModels(modelOptions, "text").map(item => ({ id: getCanvasModelRef(item), label: getCanvasModelDisplayName(getCanvasModelRef(item), modelOptions) }))}
|
||||
onModelChange={value => { setPromptToolModel(value); setPromptToolOutput(""); setPromptToolError(""); }}
|
||||
onOutputChange={setPromptToolOutput} onClose={closePromptTool}
|
||||
onRun={() => { void runPromptTool(promptToolMode); }}
|
||||
onApply={() => { setPromptEditorSeed(current => ({ revision: current.revision + 1, text: promptToolOutput })); setPrompt(promptToolOutput); onPromptChange(promptToolOutput); closePromptTool(); toast.success("已填入提示词"); }}
|
||||
/> : null}
|
||||
|
||||
<div
|
||||
className="relative overflow-visible rounded-[16px] border-[0.5px] border-[#363636] bg-[#262626] text-white shadow-[0_4px_10px_rgba(0,0,0,0.12)] transition-[height] duration-300"
|
||||
style={{ height: composerHeight, width: composerWidth }}
|
||||
@ -15009,6 +15136,8 @@ function VideoTargetModuleComposer({
|
||||
<div
|
||||
className="tiny-scrollbar h-full w-full overflow-y-auto bg-transparent px-3 py-2 text-[14px] leading-6 text-white/78 outline-none"
|
||||
contentEditable
|
||||
key={promptEditorSeed.revision}
|
||||
ref={promptEditorRef}
|
||||
onInput={(event) => {
|
||||
const clone = event.currentTarget.cloneNode(true) as HTMLElement;
|
||||
clone.querySelectorAll("[data-magic-id]").forEach((element) => element.remove());
|
||||
@ -15035,7 +15164,7 @@ function VideoTargetModuleComposer({
|
||||
suppressContentEditableWarning
|
||||
tabIndex={0}
|
||||
>
|
||||
{prompt}
|
||||
{promptEditorSeed.text}
|
||||
{magicMarks.length > 0 ? " " : null}
|
||||
{magicMarks.map((mark) => {
|
||||
const label = mark.labelOptions[mark.selectedIndex] || mark.labelOptions[0];
|
||||
@ -15171,7 +15300,7 @@ function VideoTargetModuleComposer({
|
||||
<div className="flex shrink-0 items-center gap-1">
|
||||
<button
|
||||
className="flex size-8 shrink-0 items-center justify-center rounded-lg text-white/74 transition-colors hover:bg-white/8 hover:text-white"
|
||||
onClick={() => toast.info(prompt.trim() ? "提示词优化后端待接入。" : "请先输入提示词或添加参考素材。")}
|
||||
onClick={() => { setPromptToolModel("auto"); void runPromptTool("optimize", "auto"); }}
|
||||
title="提示词优化"
|
||||
type="button"
|
||||
>
|
||||
@ -15179,7 +15308,7 @@ function VideoTargetModuleComposer({
|
||||
</button>
|
||||
<button
|
||||
className="flex size-8 shrink-0 items-center justify-center rounded-lg text-white/74 transition-colors hover:bg-white/8 hover:text-white"
|
||||
onClick={() => toast.info(prompt.trim() ? "翻译提示词后端待接入。" : "请先输入提示词。")}
|
||||
onClick={() => { setPromptToolModel("auto"); void runPromptTool("translate", "auto"); }}
|
||||
title="翻译提示词"
|
||||
type="button"
|
||||
>
|
||||
@ -21600,7 +21729,11 @@ function PresetMediaCanvasNode({
|
||||
<span className="shrink-0 text-[14px] font-medium text-white/34">{outputSizeLabel}</span>
|
||||
) : null}
|
||||
</div>
|
||||
{mediaType === "audio" ? (
|
||||
{getDraftString(node, "uploadStatus") === "uploading" || getDraftString(node, "uploadStatus") === "failed" ? (
|
||||
<div className="flex h-[calc(100%-32px)] items-center justify-center rounded-xl border border-white/10 bg-black/20 p-4 text-center text-sm text-white/65">
|
||||
{getDraftString(node, "uploadStatus") === "uploading" ? <><Loader2 className="mr-2 h-4 w-4 animate-spin" />正在上传素材…</> : <span className="text-red-200">{getDraftString(node, "uploadError") || "上传失败,请重新上传"}</span>}
|
||||
</div>
|
||||
) : mediaType === "audio" ? (
|
||||
<OfficialAudioCard durationSeconds={getDraftNumber(node, "durationSeconds", 3)} mediaUrl={mediaUrl} selected={selected} />
|
||||
) : mediaType === "video" ? (
|
||||
<div
|
||||
|
||||
49
components/canvas/VideoPromptToolDialog.tsx
Normal file
49
components/canvas/VideoPromptToolDialog.tsx
Normal file
@ -0,0 +1,49 @@
|
||||
"use client";
|
||||
|
||||
import { Dialog, DialogContent, DialogTitle } from "@/components/ui/dialog";
|
||||
import type { PromptToolMode } from "@/lib/video-canvas/prompt-tools";
|
||||
|
||||
export function VideoPromptToolDialog({ mode, busy, error, output, source, currentPrompt, model, models, onModelChange, onOutputChange, onRun, onApply, onClose }: {
|
||||
mode: PromptToolMode;
|
||||
busy: boolean;
|
||||
error: string;
|
||||
output: string;
|
||||
source: string;
|
||||
currentPrompt: string;
|
||||
model: string;
|
||||
models: { id: string; label: string }[];
|
||||
onModelChange: (value: string) => void;
|
||||
onOutputChange: (value: string) => void;
|
||||
onRun: () => void;
|
||||
onApply: () => void;
|
||||
onClose: () => void;
|
||||
}) {
|
||||
const title = mode === "optimize" ? "提示词优化" : "提示词翻译";
|
||||
return (
|
||||
<Dialog open onOpenChange={open => { if (!open) onClose(); }}>
|
||||
<DialogContent hideCloseButton aria-describedby={undefined} onPointerDown={event => event.stopPropagation()} className="flex max-h-[90dvh] w-[620px] max-w-full flex-col gap-4 overflow-y-auto rounded-2xl border border-white/10 bg-[#262626] p-5 text-white shadow-2xl">
|
||||
<div className="flex items-center justify-between">
|
||||
<DialogTitle className="text-base font-medium">{title}</DialogTitle>
|
||||
<button aria-label="关闭提示词工具" className="rounded px-2 py-1 text-white/60 hover:bg-white/10" onClick={onClose} type="button">✕</button>
|
||||
</div>
|
||||
<label className="flex items-center gap-3 text-sm text-white/65">
|
||||
处理模型
|
||||
<select aria-label="提示词处理模型" className="min-w-0 flex-1 rounded-lg border border-white/15 bg-[#202020] px-3 py-2" disabled={busy} onChange={event => onModelChange(event.target.value)} value={model}>
|
||||
<option value="auto">使用管理员默认模型</option>
|
||||
{model !== "auto" && !models.some(item => item.id === model) ? <option value={model}>当前配置模型(暂不可选)</option> : null}
|
||||
{models.map(item => <option key={item.id} value={item.id}>{item.label}</option>)}
|
||||
</select>
|
||||
</label>
|
||||
<details className="text-sm text-white/55"><summary className="cursor-pointer">查看原提示词</summary><p className="mt-2 whitespace-pre-wrap">{source}</p></details>
|
||||
{error ? <p role="alert" className="rounded-lg bg-red-400/10 p-3 text-sm text-red-200">{error}</p> : null}
|
||||
<textarea aria-label={`${title}结果`} className="min-h-[220px] w-full resize-y rounded-xl border border-white/10 bg-black/20 p-3 text-sm leading-6 outline-none focus:border-white/30" disabled={busy} onChange={event => onOutputChange(event.target.value)} placeholder={busy ? "正在处理,请稍候…" : "处理结果会显示在这里,确认后填入视频节点。"} value={output} />
|
||||
{source !== currentPrompt ? <p className="text-xs text-amber-200">原文已修改,应用结果将替换当前提示词。</p> : null}
|
||||
<div className="flex items-center justify-end gap-2">
|
||||
<button className="rounded-lg px-3 py-2 text-sm text-white/65 hover:bg-white/10" onClick={onClose} type="button">{busy ? "取消处理" : "取消"}</button>
|
||||
<button className="rounded-lg border border-white/15 px-3 py-2 text-sm disabled:opacity-40" disabled={busy} onClick={onRun} type="button">{busy ? "处理中…" : output ? "重新生成" : "重试"}</button>
|
||||
<button className="rounded-lg bg-white px-4 py-2 text-sm text-black disabled:opacity-40" disabled={busy || !output.trim() || Boolean(error)} onClick={onApply} type="button">应用到提示词</button>
|
||||
</div>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
62
docs/video-canvas-completion-2026-09-16.md
Normal file
62
docs/video-canvas-completion-2026-09-16.md
Normal file
@ -0,0 +1,62 @@
|
||||
# 视频画布继续完善与复测(2026-09-16)
|
||||
|
||||
本记录接续 [上一轮真实测试](video-canvas-real-test-2026-09-16.md)。独立工作树本地验证,沿用真实服务商和数据库配置,使用专用测试账号;没有启用模拟接口,没有修改真实用户画布或默认模型。
|
||||
|
||||
## 本轮完成的功能
|
||||
|
||||
| 功能 | 与 LibTV 对照的缺口 | 本轮实现与验证 |
|
||||
|---|---|---|
|
||||
| 视频提示词优化 | 原入口只提示后端待接入 | 新增真实文字模型接口;默认沿用管理员配置的官方千问 3.8,可选择已有文字模型,预览编辑后应用;真实优化 HTTP 200 |
|
||||
| 视频提示词翻译 | 原入口只提示后端待接入 | 中英文双向翻译;保留素材引用;真实翻译 HTTP 200,浏览器填回通过 |
|
||||
| 提示词结果编辑 | 应用后旧文字与新结果叠加 | 修复 contentEditable 与 React 更新冲突;浏览器实测应用后只剩新译文 |
|
||||
| 弹窗交互 | 缺少完整模态交互 | 使用现有 Dialog 组件,焦点约束、Escape 关闭、取消请求、失败保留原文、禁止空结果应用 |
|
||||
| 视频时段剪辑 | 只记录范围,不输出成片 | 按源视频实际时长定位,FFmpeg 导出独立 MP4,保留音轨,创建关联结果节点;真实 8 秒片源导出 4 秒新片通过 |
|
||||
| 视频画面裁剪 | 上轮仅处理函数通过 | 本轮完整下载、处理、上传 HTTP API 通过,输出 640×360、8 秒新片 |
|
||||
| 视频音轨拆分 | 上轮仅处理函数通过 | 完整 API 输出独立 AAC 音频和无音轨 MP4;不冒充人声/背景音分离 |
|
||||
| 本地音视频上传 | 原来只存 blob 地址,刷新失效 | 新增真实上传接口和媒体探测,保存对象存储地址;MP4/M4A 上传通过,伪装 MP4 被拒绝;最多 125 MiB、10 分钟 |
|
||||
| 上传状态恢复 | 中断后可能一直等待 | 显示上传中/失败,恢复旧 blob 或未完成上传时明确要求重新上传 |
|
||||
| 脚本/画面分析长时间等待 | 旧 SDK 参数被类型断言掩盖,缺少调用时限 | 改用 maxOutputTokens,脚本预算 12000、视频分析 2200、其他分析 1600;150 秒推理时限、175 秒页面请求时限、关闭 SDK 自动重试;空白/截断/非 JSON 结果不再冒充结构化成功 |
|
||||
| 项目加载失败 | 失败后还可能继续编辑未就绪画布 | 加载超时和失败显示重试,加载成功前阻止编辑,避免保存覆盖 |
|
||||
|
||||
## 真实调用结果
|
||||
|
||||
本地开发编译、数据库隧道和接口前置处理均计入 HTTP 耗时,不代表生产模型纯推理时间。
|
||||
|
||||
| 测试 | 结果 | 完整 HTTP 耗时 / 产物 |
|
||||
|---|---|---|
|
||||
| 官方千问 3.8 翻译 | 通过 | 39.325 秒;保留 @主体1,红杯/白桌/8 秒限制完整 |
|
||||
| 官方千问 3.8 优化 | 通过 | 68.437 秒;返回可直接使用的视频提示词 |
|
||||
| 官方千问 3.8 图片理解复测 | 通过 | 85.597 秒,HTTP 200;正确识别红杯、右侧杯柄、咖啡、灰白背景,无虚构人物 |
|
||||
| 官方千问 3.8 脚本分析复测 | 未通过 | 103.003 秒,HTTP 500;连接被对端关闭,未完成脚本生成 |
|
||||
| 选择 Agnes 1.5 进行翻译/优化 | 失败 | HTTP 502;不替换用户原文,不改变默认模型 |
|
||||
| 误选视频模型做提示词处理 | 正确拒绝 | HTTP 400,要求文字模型 |
|
||||
| Veo 3.1 Fast 文生视频 | 通过 | 151.007 秒;8 秒、1280×720、H264 + AAC,1,005,090 字节,已解码检查红杯画面 |
|
||||
| Veo 首帧视频复测 | 未通过 | 164.342 秒;本站到服务商连接失败 `fetch failed`,不据此判定模型不支持 |
|
||||
| Veo 首尾帧复测 | 未通过 | 15.942 秒;输入图片可达性检查失败,HTTP 400,未提交上游生成 |
|
||||
| Veo 多参考复测 | 未通过 | 83.545 秒;上游拒绝 HTTP 503,请求 ID `3b068421-5c82-423b-a05b-6a557ab5c689` |
|
||||
| 时段剪辑 | 通过 | 21.837 秒;4 秒、1280×720、H264 + AAC,748,275 字节 |
|
||||
| 画面裁剪 | 通过 | 17.115 秒;8 秒、640×360、H264 + AAC,897,501 字节 |
|
||||
| 音视频拆分 | 通过 | 27.781 秒;静音 MP4 1,526,056 字节,AAC 197,583 字节 |
|
||||
| 视频上传 | 通过 | 33.266 秒,返回真实对象存储地址及 8 秒时长 |
|
||||
| 音频上传 | 通过 | 22.951 秒,返回 M4A 地址及 8 秒时长 |
|
||||
| 无效 MP4 上传 | 正确拒绝 | HTTP 400,文件内容校验失败 |
|
||||
|
||||
文生视频请求走自动匹配,响应确认实际使用 `veo-3.1-fast`。此次成功更新了上一轮该模式失败的状态,但不证明其他模式或每次调用均成功。生成接口本次返回服务商地址,其长期留存尚未验收;与本轮修复的用户上传文件持久化是两个环节。
|
||||
|
||||
本机代理对媒体域名使用保留网段 DNS。完整媒体 API 测试仅在开发进程对两个已知媒体域名使用经服务器解析的公网地址;生产安全下载器的 IP 校验和 SSRF 防护没有放宽。
|
||||
|
||||
## 页面回归与边界
|
||||
|
||||
- 官方千问翻译弹窗成功显示结果,应用后只保留新译文;保存重试成功后刷新,重新选择节点仍为同一译文。证据 `prompt-after-refresh.txt/png`。
|
||||
- 本地冷编译和接口前置检查曾使保存超时,页面显示失败并允许重试;点击重试后已保存。未把本地耗时当作生产性能结论。
|
||||
- 浏览器上传自动化被文件选择器事件超时阻塞,无法完整验收“从页面选文件到刷新后播放”;真实上传 HTTP 接口已单独通过。不能将接口通过写成整个页面流程通过。
|
||||
- 58 项定向回归通过:6 项新工具契约、12 项模型/参数/媒体安全、4 项保存队列、36 项 VIP98K 相关回归。
|
||||
- 源码 TypeScript 检查通过;未宣称完成生产构建,临时 Next 生成目录不纳入源码检查。
|
||||
|
||||
## 仍未完成
|
||||
|
||||
音频驱动视频没有已确认可用模型;视频超清、去字幕、专业人声分离、AutoLink、智能剪辑及逐帧功能未完成全量对齐。布局与参考站相近,但本轮以真实功能闭环为主,不能称为全站像素级复刻。
|
||||
|
||||
服务器已确认存在 `/usr/bin/ffmpeg` 与 `/usr/bin/ffprobe`。本轮代码未部署生产。原始证据保留在本地 `.qa/video-completion/`,包含 API 响应、ffprobe 信息、实际产物和回归日志;带临时媒体签名的原始响应不提交 Git。
|
||||
|
||||
收尾:两个专用测试账号已停用并撤销活跃会话;临时环境凭据已删除;本地开发服务、数据库隧道和本轮浏览器标签已关闭。开发工具自动修改的 next-env.d.ts / tsconfig.json 已还原,主工作区未修改。
|
||||
@ -1,5 +1,7 @@
|
||||
# 视频画布模型匹配与逐项对照(2026-09-16)
|
||||
|
||||
> 后续进展见 [继续完善与复测记录](video-canvas-completion-2026-09-16.md):官方千问 3.8 提示词处理、Veo 文生视频、完整媒体处理 API 已出现真实成功结果。下文保留上一轮当时的测试状态。
|
||||
|
||||
参考:[LibTV 画布](https://www.liblib.tv/canvas)。以登录后的 1280×720 画布为界面对照基准。本记录对应此前“模型接通即可,不继续真实生成测试”的阶段;用户随后已要求恢复真实测试,最新结果见 [真实测试与参考站对照](video-canvas-real-test-2026-09-16.md)。因此下表区分接口与参数核验、界面核验、实际生成,不能把未生成的功能标为产物验收通过,也不把局部对齐称为整站像素级复刻完成。
|
||||
|
||||
## 模型结论
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
# 视频画布真实测试与 LibTV 对照(2026-09-16)
|
||||
|
||||
> 后续进展见 [继续完善与复测记录](video-canvas-completion-2026-09-16.md):官方千问 3.8 提示词处理、Veo 文生视频、完整媒体处理 API 已出现真实成功结果。下文保留上一轮当时的测试状态。
|
||||
|
||||
本轮按用户最新指令恢复真实调用。在独立本地工作树、127.0.0.1:3016、两个专用测试账号下运行真实接口,未启用 CANVAS_E2E_LOCAL 模拟模式。使用现有数据库配置及服务商线路;未改真实用户画布。参考站为 https://www.liblib.tv/canvas,浏览器统一为 1280×720。
|
||||
|
||||
**结论:有视频模型,但本轮尚不能验收视频生成成功。图片生成和本地视频处理通过;保存缺陷已本地修复;仍有未接入入口。不能把模型目录可见等同于可生成,也不能把当前页面称为完整像素级复刻。**
|
||||
|
||||
@ -13,6 +13,7 @@ export type VideoCanvasBackendRoute =
|
||||
| "canvas-generate-image"
|
||||
| "canvas-generate-video"
|
||||
| "canvas-workflow-analyze"
|
||||
| "canvas-prompt-tools"
|
||||
| "angle-edit"
|
||||
| "enhance"
|
||||
| "enhance-pro"
|
||||
@ -95,6 +96,18 @@ export type VideoCanvasFunctionConfig = {
|
||||
};
|
||||
|
||||
const defaultFunctions: VideoCanvasFunctionDefinition[] = [
|
||||
{
|
||||
key: "video-prompt-optimize", label: "视频提示词优化", kind: "multimodal",
|
||||
route: "canvas-prompt-tools", enabled: true, requiresBackend: true,
|
||||
prompt: "你是专业视频提示词编辑。忠实保留用户原有主体、动作、场景、数字、品牌、@素材引用和限制,补齐景别、机位、运镜、动作连续性、光线、材质和声音等缺失细节。不得擅自改变故事,不添加不存在的角色。只输出可直接交给视频模型的中文提示词正文,不要标题、解释、Markdown或引号。",
|
||||
description: "使用已配置的文字模型优化提示词,预览确认后填入视频节点。",
|
||||
},
|
||||
{
|
||||
key: "video-prompt-translate", label: "视频提示词翻译", kind: "multimodal",
|
||||
route: "canvas-prompt-tools", enabled: true, requiresBackend: true,
|
||||
prompt: "你是视频提示词翻译器。中文为主时译为英文,英文为主时译为中文。完整保留 @素材引用、专有名词、数字、镜头编号、参数和限制,不扩写、不删减。只输出译文,不要标题、解释、Markdown或引号。",
|
||||
description: "中英文自动识别双向翻译,预览确认后填入视频节点。",
|
||||
},
|
||||
...[
|
||||
["character-face-three-view", "角色脸部三视图", "以参考图片的角色为唯一身份依据,生成横向排布的正面、左侧面、右侧面脸部三视图。保持五官、肤色、发型、饰品和年龄一致,头肩构图,纯净背景,三个角度大小一致。"],
|
||||
["character-design-sheet", "角色设定图", "根据参考角色生成完整角色设定图,包含正面、侧面、背面全身及服饰配件细节。所有视图保持同一身份、体型和服装材质,以干净背景和清楚分区呈现。"],
|
||||
|
||||
22
lib/video-canvas/media-contract.ts
Normal file
22
lib/video-canvas/media-contract.ts
Normal file
@ -0,0 +1,22 @@
|
||||
import { z } from "zod";
|
||||
|
||||
const fraction = z.number().finite().min(0).max(1);
|
||||
export const videoTimeRangeSchema = z.object({ start: fraction, end: fraction })
|
||||
.refine(range => range.end > range.start, "结束位置必须晚于开始位置");
|
||||
export type VideoTimeRange = z.infer<typeof videoTimeRangeSchema>;
|
||||
|
||||
export const videoProcessSchema = z.discriminatedUnion("operation", [
|
||||
z.object({ operation: z.literal("crop"), url: z.string().url(), rect: z.object({
|
||||
x: fraction, y: fraction, width: fraction.positive(), height: fraction.positive(),
|
||||
}).refine(rect => rect.x + rect.width <= 1.001 && rect.y + rect.height <= 1.001, "裁剪区域超出画面") }),
|
||||
z.object({ operation: z.literal("audio-video"), url: z.string().url() }),
|
||||
z.object({ operation: z.literal("trim"), url: z.string().url(), range: videoTimeRangeSchema }),
|
||||
]);
|
||||
|
||||
export function toVideoTimeRange(range: VideoTimeRange, duration: number) {
|
||||
if (!videoTimeRangeSchema.safeParse(range).success || !Number.isFinite(duration) || duration <= 0) throw new Error("剪辑范围无效");
|
||||
const start = range.start * duration;
|
||||
const end = range.end * duration;
|
||||
if (end - start < 0.1) throw new Error("剪辑时长不能少于 0.1 秒");
|
||||
return { start, end, duration: end - start };
|
||||
}
|
||||
@ -1,5 +1,6 @@
|
||||
import {execFile} from 'node:child_process';
|
||||
import {promisify} from 'node:util';
|
||||
import { toVideoTimeRange, type VideoTimeRange } from './media-contract';
|
||||
|
||||
export type MediaRect = {x:number;y:number;width:number;height:number};
|
||||
export function toVideoPixelRect(rect:MediaRect,width:number,height:number) {
|
||||
@ -12,7 +13,7 @@ export function toVideoPixelRect(rect:MediaRect,width:number,height:number) {
|
||||
}
|
||||
const run=promisify(execFile);
|
||||
const inputSafety=['-protocol_whitelist','file','-format_whitelist','mov,matroska,webm,avi,mpeg,mpegts,flv,ogg'];
|
||||
export async function processVideoFile(input:string,output:string,operation:'crop'|'extract-audio'|'silent-video',rect?:MediaRect) {
|
||||
export async function processVideoFile(input:string,output:string,operation:'crop'|'extract-audio'|'silent-video'|'trim',rect?:MediaRect,range?:VideoTimeRange) {
|
||||
const {stdout}=await run('ffprobe',['-v','error',...inputSafety,'-show_streams','-show_format','-of','json',input],{timeout:15000,maxBuffer:1024*1024});
|
||||
const info=JSON.parse(stdout), video=info.streams?.find((s:any)=>s.codec_type==='video');
|
||||
if(!video) throw new Error('素材中没有视频画面');
|
||||
@ -21,11 +22,20 @@ export async function processVideoFile(input:string,output:string,operation:'cro
|
||||
if(!Number.isFinite(duration)||duration<=0||duration>600) throw new Error('本地视频处理支持 10 分钟以内的视频');
|
||||
let args:string[];
|
||||
let dimensions={width:Number(video.width),height:Number(video.height)};
|
||||
let outputDuration = duration;
|
||||
if(operation==='extract-audio') {
|
||||
if(!info.streams.some((s:any)=>s.codec_type==='audio')) throw new Error('原视频不包含音轨');
|
||||
args=['-map','0:a:0','-vn','-c:a','aac','-b:a','192k'];
|
||||
} else if(operation==='silent-video') {
|
||||
args=['-map','0:v:0','-an','-c:v','copy','-movflags','+faststart'];
|
||||
} else if(operation==='trim') {
|
||||
if (!range) throw new Error('请选择剪辑范围');
|
||||
const selected = toVideoTimeRange(range, duration);
|
||||
outputDuration = selected.duration;
|
||||
dimensions = {width:Math.floor(dimensions.width/2)*2,height:Math.floor(dimensions.height/2)*2};
|
||||
args=['-ss',String(selected.start),'-t',String(selected.duration),'-map','0:v:0','-map','0:a?',
|
||||
'-vf','scale=trunc(iw/2)*2:trunc(ih/2)*2','-c:v','libx264','-preset','veryfast','-crf','20',
|
||||
'-pix_fmt','yuv420p','-c:a','aac','-movflags','+faststart'];
|
||||
} else {
|
||||
if(!rect) throw new Error('请选择裁剪区域');
|
||||
const crop=toVideoPixelRect(rect,dimensions.width,dimensions.height);
|
||||
@ -33,5 +43,5 @@ export async function processVideoFile(input:string,output:string,operation:'cro
|
||||
args=['-map','0:v:0','-map','0:a?','-vf',`crop=${crop.width}:${crop.height}:${crop.x}:${crop.y}`,'-c:v','libx264','-preset','veryfast','-crf','20','-pix_fmt','yuv420p','-c:a','aac','-movflags','+faststart'];
|
||||
}
|
||||
await run('ffmpeg',['-nostdin','-y','-v','error','-threads','2','-filter_threads','1',...inputSafety,'-i',input,...args,'-threads','2',output],{timeout:180000,maxBuffer:1024*1024});
|
||||
return {duration,...dimensions};
|
||||
return {duration:outputDuration,...dimensions};
|
||||
}
|
||||
|
||||
32
lib/video-canvas/prompt-tools.ts
Normal file
32
lib/video-canvas/prompt-tools.ts
Normal file
@ -0,0 +1,32 @@
|
||||
import { z } from "zod";
|
||||
import { isCanvasLanguageModel, type CanvasModelDescriptor } from "./model-matching";
|
||||
|
||||
export const promptToolSchema = z.object({
|
||||
mode: z.enum(["optimize", "translate"]),
|
||||
prompt: z.string().trim().min(1, "请先输入提示词").max(12000, "提示词不能超过 12000 字"),
|
||||
model: z.string().trim().max(200).optional(),
|
||||
});
|
||||
|
||||
export type PromptToolMode = z.infer<typeof promptToolSchema>["mode"];
|
||||
export const promptToolFunctionKey = (mode: PromptToolMode) => `video-prompt-${mode}`;
|
||||
|
||||
export function selectPromptToolModel<T extends CanvasModelDescriptor>(models: T[], requested?: string) {
|
||||
const candidates = models.filter(model => model.isEnabled !== false && isCanvasLanguageModel(model));
|
||||
const ref = requested?.trim();
|
||||
const model = ref && ref !== "auto"
|
||||
? candidates.find(item => item.id === ref || item.modelId === ref)
|
||||
: candidates[0];
|
||||
if (!model) throw new Error(ref && ref !== "auto"
|
||||
? "所选提示词模型不可用或不是文字模型,请重新选择"
|
||||
: "当前没有可用的提示词处理模型");
|
||||
return model;
|
||||
}
|
||||
|
||||
export function promptToolErrorMessage(error: unknown) {
|
||||
const message = error instanceof Error ? error.message : "";
|
||||
if (/overdue|good standing|insufficient.*balance|欠费/i.test(message)) {
|
||||
return "模型服务商账户余额或状态异常,请管理员检查。原提示词已保留。";
|
||||
}
|
||||
if (/abort|timeout|timed out/i.test(message)) return "提示词处理超时,请重试。原提示词已保留。";
|
||||
return "提示词模型调用失败,请稍后重试或选择其他文字模型。原提示词已保留。";
|
||||
}
|
||||
@ -3,6 +3,7 @@ import { defineConfig } from "@playwright/test";
|
||||
export default defineConfig({
|
||||
testDir: "./tests",
|
||||
testMatch: [
|
||||
/routes\/video-canvas-tools\.test\.ts/,
|
||||
/routes\/video-canvas-save-queue\.test\.ts/,
|
||||
/routes\/video-canvas-model-matching\.test\.ts/,
|
||||
/routes\/canvas-chat-helpers\.test\.ts/,
|
||||
|
||||
@ -1236,11 +1236,13 @@ export const workflowAnalysisService = {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(params),
|
||||
signal: AbortSignal.timeout(175_000),
|
||||
});
|
||||
const data = await response.json();
|
||||
return data;
|
||||
} catch (error) {
|
||||
return { success: false, error: '工作流分析失败' };
|
||||
return { success: false, error: error instanceof Error && error.name === 'TimeoutError'
|
||||
? '分析超时,请稍后重试,原内容已保留' : '工作流分析失败' };
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
53
tests/routes/video-canvas-tools.test.ts
Normal file
53
tests/routes/video-canvas-tools.test.ts
Normal file
@ -0,0 +1,53 @@
|
||||
import { test, expect } from "@playwright/test";
|
||||
import { promptToolSchema, selectPromptToolModel, promptToolErrorMessage } from "@/lib/video-canvas/prompt-tools";
|
||||
import { toVideoTimeRange, videoProcessSchema } from "@/lib/video-canvas/media-contract";
|
||||
import { normalizeVideoCanvasFunctionConfig } from "@/lib/video-canvas/config";
|
||||
|
||||
test("prompt tools reject empty and oversized input before calling a model", () => {
|
||||
expect(promptToolSchema.safeParse({ mode: "translate", prompt: " " }).success).toBe(false);
|
||||
expect(promptToolSchema.safeParse({ mode: "optimize", prompt: "a".repeat(12001) }).success).toBe(false);
|
||||
expect(promptToolSchema.parse({ mode: "translate", prompt: " @角色1 保持 8 秒 " }).prompt).toBe("@角色1 保持 8 秒");
|
||||
});
|
||||
|
||||
test("prompt tools exclude media models and never silently replace an explicit model", () => {
|
||||
const image = { id: "image", modelId: "gpt-image-2", supportsImageGen: true };
|
||||
const video = { id: "video", modelId: "veo-3.1-fast", capabilities: { kind: "video" } };
|
||||
const text = { id: "text", modelId: "qwen3.8-max", isEnabled: true };
|
||||
expect(selectPromptToolModel([image, video, text], "auto")).toEqual(text);
|
||||
expect(selectPromptToolModel([text], "qwen3.8-max")).toEqual(text);
|
||||
expect(() => selectPromptToolModel([image, video, text], "image")).toThrow("文字模型");
|
||||
expect(() => selectPromptToolModel([{ ...text, isEnabled: false }], "text")).toThrow();
|
||||
expect(() => selectPromptToolModel([text], "missing")).toThrow();
|
||||
});
|
||||
|
||||
test("prompt tools preserve administrator model and prompt overrides", () => {
|
||||
const config = normalizeVideoCanvasFunctionConfig({ defaultMultimodalModel: "qwen38", functions: {
|
||||
"video-prompt-translate": { enabled: false, model: "explicit-model", prompt: "Translate exactly." },
|
||||
} });
|
||||
expect(config.functions["video-prompt-optimize"].route).toBe("canvas-prompt-tools");
|
||||
expect(config.functions["video-prompt-translate"]).toMatchObject({ enabled: false, model: "explicit-model", prompt: "Translate exactly." });
|
||||
expect(config.defaultMultimodalModel).toBe("qwen38");
|
||||
});
|
||||
|
||||
test("upstream errors give an actionable message without returning provider internals", () => {
|
||||
expect(promptToolErrorMessage(new Error("Access denied, account in good standing overdue-payment"))).toContain("账户");
|
||||
expect(promptToolErrorMessage(new Error("timed out"))).toContain("超时");
|
||||
expect(promptToolErrorMessage(new Error("private upstream URL"))).not.toContain("private");
|
||||
});
|
||||
|
||||
test("trim uses source duration rather than assuming eight seconds", () => {
|
||||
expect(toVideoTimeRange({ start: 0.25, end: 0.75 }, 8)).toEqual({ start: 2, end: 6, duration: 4 });
|
||||
expect(toVideoTimeRange({ start: 0.1, end: 0.5 }, 20)).toEqual({ start: 2, end: 10, duration: 8 });
|
||||
});
|
||||
|
||||
test("invalid trim and crop requests are rejected before downloading", () => {
|
||||
const url = "https://example.com/video.mp4";
|
||||
for (const range of [{ start: 0.5, end: 0.5 }, { start: -0.1, end: 1 }, { start: 0, end: 2 }]) {
|
||||
expect(videoProcessSchema.safeParse({ operation: "trim", url, range }).success).toBe(false);
|
||||
}
|
||||
expect(videoProcessSchema.safeParse({ operation: "trim", url }).success).toBe(false);
|
||||
expect(videoProcessSchema.safeParse({ operation: "crop", url, rect: { x: 0.8, y: 0, width: 0.8, height: 1 } }).success).toBe(false);
|
||||
expect(() => toVideoTimeRange({ start: 0, end: 0.001 }, 8)).toThrow("0.1 秒");
|
||||
expect(() => toVideoTimeRange({ start: 0, end: 1 }, NaN)).toThrow();
|
||||
expect(videoProcessSchema.safeParse({ operation: "audio-video", url }).success).toBe(true);
|
||||
});
|
||||
Loading…
x
Reference in New Issue
Block a user