From cd9f340a895b0a472352d56716ce51772ba9cd2a Mon Sep 17 00:00:00 2001 From: 134 Date: Fri, 18 Sep 2026 12:14:45 +0800 Subject: [PATCH] Complete durable canvas groups and add isolated P40 video upscaling --- .gitignore | 4 + app/api/canvas/generation-tasks/route.ts | 3 + app/api/canvas/group-runs/route.ts | 39 ++ app/api/canvas/group-templates/route.ts | 40 ++ .../video-upscale/capabilities/route.ts | 39 ++ components/canvas/CanvasGroupToolbar.tsx | 125 ++++++ components/canvas/GroupStoryboardPreview.tsx | 122 ++++++ .../canvas/LiblibVideoCanvasReplica.tsx | 197 ++++++--- .../video-canvas-groups-upscale-2026-09-18.md | 39 ++ hooks/use-canvas-group-runs.ts | 90 ++++ lib/video-canvas/connections.ts | 2 +- lib/video-canvas/group-execution.ts | 333 +++++++++++++++ lib/video-canvas/groups.ts | 398 ++++++++++++++++++ lib/video-canvas/task-contract.ts | 144 +++++-- lib/video-canvas/task-recovery.ts | 36 +- lib/video-canvas/task-store.ts | 305 ++++++++++++-- lib/video-canvas/task-worker.ts | 89 +++- lib/video-canvas/video-upscale-contract.ts | 33 ++ lib/video-canvas/video-upscale-remote.ts | 181 ++++++++ lib/video-canvas/video-upscale.ts | 62 +++ playwright.helpers.config.ts | 1 + scripts/qa/video-upscale-service.cjs | 103 +++++ services/video-upscale/README.md | 46 ++ services/video-upscale/build.cjs | 32 ++ services/video-upscale/job.ts | 85 ++++ services/video-upscale/server.ts | 328 +++++++++++++++ services/video-upscale/upscale.py | 79 ++++ services/video-upscale/vendor/LICENSE | 29 ++ services/video-upscale/vendor/SOURCE.md | 4 + services/video-upscale/vendor/srvgg_arch.py | 67 +++ .../video-upscale-tunnel.service | 17 + services/video-upscale/video-upscale.service | 33 ++ .../video-canvas-groups-upscale.test.ts | 310 ++++++++++++++ 33 files changed, 3260 insertions(+), 155 deletions(-) create mode 100644 app/api/canvas/group-runs/route.ts create mode 100644 app/api/canvas/group-templates/route.ts create mode 100644 app/api/canvas/video-upscale/capabilities/route.ts create mode 100644 components/canvas/CanvasGroupToolbar.tsx create mode 100644 components/canvas/GroupStoryboardPreview.tsx create mode 100644 docs/video-canvas-groups-upscale-2026-09-18.md create mode 100644 hooks/use-canvas-group-runs.ts create mode 100644 lib/video-canvas/group-execution.ts create mode 100644 lib/video-canvas/groups.ts create mode 100644 lib/video-canvas/video-upscale-contract.ts create mode 100644 lib/video-canvas/video-upscale-remote.ts create mode 100644 lib/video-canvas/video-upscale.ts create mode 100644 scripts/qa/video-upscale-service.cjs create mode 100644 services/video-upscale/README.md create mode 100644 services/video-upscale/build.cjs create mode 100644 services/video-upscale/job.ts create mode 100644 services/video-upscale/server.ts create mode 100644 services/video-upscale/upscale.py create mode 100644 services/video-upscale/vendor/LICENSE create mode 100644 services/video-upscale/vendor/SOURCE.md create mode 100644 services/video-upscale/vendor/srvgg_arch.py create mode 100644 services/video-upscale/video-upscale-tunnel.service create mode 100644 services/video-upscale/video-upscale.service create mode 100644 tests/routes/video-canvas-groups-upscale.test.ts diff --git a/.gitignore b/.gitignore index a0e8f5c0d..91e1bcc4c 100644 --- a/.gitignore +++ b/.gitignore @@ -120,3 +120,7 @@ services/smart-edit/__pycache__/ # Local verification artefacts (also excluded from Tailwind's automatic source scan). .qa/ .next-video-*/ + +services/video-upscale/dist/ +services/video-upscale/__pycache__/ +services/video-upscale/vendor/__pycache__/ diff --git a/app/api/canvas/generation-tasks/route.ts b/app/api/canvas/generation-tasks/route.ts index 056ac2819..588f6e8ee 100644 --- a/app/api/canvas/generation-tasks/route.ts +++ b/app/api/canvas/generation-tasks/route.ts @@ -1,3 +1,4 @@ +import { videoUpscaleSchema } from "@/lib/video-canvas/video-upscale-contract"; import { smartEditRequestSchema } from "@/lib/video-canvas/smart-edit/contract"; import { audioRequestSchema } from "@/lib/video-canvas/audio-contract"; import { canResumeCanvasTask } from "@/lib/video-canvas/task-recovery"; @@ -43,6 +44,7 @@ export async function POST(request: Request) { throw Object.assign(Error("任务数据过大，请使用已上传的素材链接"), { status: 413 }); const parsed = canvasTaskRequestSchema.safeParse(JSON.parse(text)); if (!parsed.success) throw Object.assign(Error("任务参数不合法"), { status: 400 }); + if(parsed.data.workflow)throw Object.assign(Error("编组任务必须使用整组提交接口"),{status:400}); if ( parsed.data.kind === "image-operation" && !imageOperationSchema.safeParse(parsed.data.params).success @@ -61,6 +63,7 @@ export async function POST(request: Request) { !smartEditRequestSchema.safeParse(parsed.data.params).success ) throw Object.assign(Error("剪辑参数无效"), { status: 400 }); + if (parsed.data.kind === "video-upscale" && !videoUpscaleSchema.safeParse(parsed.data.params).success) throw Object.assign(Error("视频高清参数无效"), {status:400}); const userId = await owner(parsed.data.projectId); const task = await createCanvasGenerationTask(userId, parsed.data); kick(userId, parsed.data.projectId); diff --git a/app/api/canvas/group-runs/route.ts b/app/api/canvas/group-runs/route.ts new file mode 100644 index 000000000..c32ba9f78 --- /dev/null +++ b/app/api/canvas/group-runs/route.ts @@ -0,0 +1,39 @@ +import { auth } from "@/app/(auth)/auth"; +import { after, NextResponse } from "next/server"; +import { canvasTaskRequestSchema } from "@/lib/video-canvas/task-contract"; +import { + createCanvasGroupTasks, + ownsTaskProject, +} from "@/lib/video-canvas/task-store"; +import { runCanvasGenerationQueue } from "@/lib/video-canvas/task-worker"; +export const runtime = "nodejs", + maxDuration = 900; +export async function POST(request: Request) { + try { + const session = await auth(); + if (!session?.user?.id) + return NextResponse.json({ error: "未登录" }, { status: 401 }); + const text = await request.text(); + if (Buffer.byteLength(text) > 1048576) + return NextResponse.json({ error: "编组数据过大" }, { status: 413 }); + const parsed = canvasTaskRequestSchema + .array() + .min(1) + .max(40) + .safeParse(JSON.parse(text)); + if (!parsed.success) + return NextResponse.json({ error: "编组参数无效" }, { status: 400 }); + const projectId = parsed.data[0].projectId, + userId = session.user.id; + if (!(await ownsTaskProject(userId, projectId))) + return NextResponse.json({ error: "画布不存在" }, { status: 404 }); + const tasks = await createCanvasGroupTasks(userId, parsed.data); + after(() => runCanvasGenerationQueue(userId, projectId)); + return NextResponse.json({ success: true, tasks }, { status: 202 }); + } catch (e) { + return NextResponse.json( + { error: e instanceof Error ? e.message : "编组执行失败" }, + { status: Number((e as any)?.status) || 400 } + ); + } +} diff --git a/app/api/canvas/group-templates/route.ts b/app/api/canvas/group-templates/route.ts new file mode 100644 index 000000000..aa7dec7c0 --- /dev/null +++ b/app/api/canvas/group-templates/route.ts @@ -0,0 +1,40 @@ +import { auth } from "@/app/(auth)/auth"; +import { db } from "@/lib/db"; +import { canvasProject } from "@/lib/db/schema"; +import { eq, and, sql, desc } from "drizzle-orm"; +import { NextResponse } from "next/server"; +export async function GET() { + const session = await auth(); + if (!session?.user?.id) + return NextResponse.json({ error: "未登录" }, { status: 401 }); + const rows = await db + .select({ + projectId: canvasProject.id, + title: canvasProject.title, + templates: sql`${canvasProject.canvasData}->'groupTemplates'`, + }) + .from(canvasProject) + .where( + and( + eq(canvasProject.userId, session.user.id), + eq(canvasProject.workspaceType, "video"), + sql`${canvasProject.canvasData} ? 'groupTemplates'` + ) + ) + .orderBy(desc(canvasProject.updatedAt)) + .limit(100); + return NextResponse.json( + { + templates: rows.flatMap((row) => + Array.isArray(row.templates) + ? row.templates.map((template) => ({ + ...template, + projectId: row.projectId, + projectTitle: row.title, + })) + : [] + ), + }, + { headers: { "Cache-Control": "no-store" } } + ); +} diff --git a/app/api/canvas/video-upscale/capabilities/route.ts b/app/api/canvas/video-upscale/capabilities/route.ts new file mode 100644 index 000000000..9bdc36754 --- /dev/null +++ b/app/api/canvas/video-upscale/capabilities/route.ts @@ -0,0 +1,39 @@ +import { auth } from "@/app/(auth)/auth"; +import { upscaleConfig } from "@/lib/video-canvas/video-upscale-remote"; +import { NextResponse } from "next/server"; +export async function GET() { + if (!(await auth())?.user?.id) + return NextResponse.json({ error: "未登录" }, { status: 401 }); + try { + const config = upscaleConfig(); + if (!config) + return NextResponse.json({ + available: false, + message: "视频高清服务尚未接通", + }); + const response = await fetch(config.base + "/health", { + headers: { authorization: "Bearer " + config.token }, + signal: AbortSignal.timeout(5000), + redirect: "error", + cache: "no-store", + }); + if (!response.ok) throw Error("unavailable"); + const data = await response.json(); + return NextResponse.json( + { + available: data.status === "ok", + engine: "Real-ESRGAN", + resolutions: ["1080P", "2K", "4K"], + maxScale: 4, + maxSeconds: 120, + queued: data.queued, + }, + { headers: { "Cache-Control": "no-store" } } + ); + } catch { + return NextResponse.json( + { available: false, message: "视频高清服务暂时不可用，请稍后重试" }, + { headers: { "Cache-Control": "no-store" } } + ); + } +} diff --git a/components/canvas/CanvasGroupToolbar.tsx b/components/canvas/CanvasGroupToolbar.tsx new file mode 100644 index 000000000..da2889165 --- /dev/null +++ b/components/canvas/CanvasGroupToolbar.tsx @@ -0,0 +1,125 @@ +"use client"; +import { + Download, + Play, + Ungroup, + Share2, + Grid3X3, + ScanLine, +} from "lucide-react"; +export function CanvasGroupToolbar({ + title, + color, + status, + busy, + storyboard, + canStoryboard, + onRename, + onColor, + onLayout, + onRun, + onUngroup, + onSave, + onDownload, + onStoryboard, + onCancel, +}: { + title: string; + color: string; + status: string; + busy: boolean; + storyboard: boolean; + canStoryboard: boolean; + onRename: (s: string) => void; + onColor: (s: string) => void; + onLayout: (s: "grid" | "horizontal" | "vertical") => void; + onRun: () => void; + onUngroup: () => void; + onSave: () => void; + onDownload: () => void; + onStoryboard: () => void; + onCancel: () => void; +}) { + return ( +
e.stopPropagation()} + > +
+ onColor(e.target.value)} + className="h-7 w-7 cursor-pointer rounded-full border-0 bg-transparent" + /> + + + + + + + + +
+
+ onRename(e.target.value)} + maxLength={120} + className="w-48 border-b border-white/10 bg-transparent py-1" + /> + {status || "Shift 点击多选 · ⌘/Ctrl+G 编组"} + {busy && } +
+
+ ); +} diff --git a/components/canvas/GroupStoryboardPreview.tsx b/components/canvas/GroupStoryboardPreview.tsx new file mode 100644 index 000000000..cdcea4b0a --- /dev/null +++ b/components/canvas/GroupStoryboardPreview.tsx @@ -0,0 +1,122 @@ +"use client"; +import { useEffect, useState } from "react"; +import { ArrowLeft, ArrowRight, X } from "lucide-react"; +export type GroupShot = { + id: string; + title: string; + url: string; + type: "image" | "video"; +}; +export function GroupStoryboardPreview({ + shots, + onOrder, + onClose, +}: { + shots: GroupShot[]; + onOrder: (ids: string[]) => void; + onClose: () => void; +}) { + const [index, setIndex] = useState(0), + shot = shots[Math.min(index, shots.length - 1)]; + useEffect(() => { + const key = (e: KeyboardEvent) => { + if (e.key === "Escape") onClose(); + }; + window.addEventListener("keydown", key); + return () => window.removeEventListener("keydown", key); + }, [onClose]); + if (!shot) return null; + const move = (delta: number) => { + const next = index + delta; + if (next < 0 || next >= shots.length) return; + const ids = shots.map((s) => s.id); + [ids[index], ids[next]] = [ids[next], ids[index]]; + onOrder(ids); + setIndex(next); + }; + return ( +
e.stopPropagation()} + > +
+

+ 分镜 {index + 1}/{shots.length} · {shot.title} +

+ +
+
+ {shot.type === "video" ? ( +
+
+ + + {index + 1} / {shots.length} + + + + +
+
+ {shots.map((s, i) => ( + + ))} +
+
+ ); +} diff --git a/components/canvas/LiblibVideoCanvasReplica.tsx b/components/canvas/LiblibVideoCanvasReplica.tsx index 5ea99d442..5276a5ab5 100644 --- a/components/canvas/LiblibVideoCanvasReplica.tsx +++ b/components/canvas/LiblibVideoCanvasReplica.tsx @@ -1,4 +1,11 @@ "use client"; +import { useCanvasGroupRuns } from "@/hooks/use-canvas-group-runs"; +import { hasRecoverableCanvasTask } from "@/lib/video-canvas/task-recovery"; +import { GroupStoryboardPreview } from "./GroupStoryboardPreview"; +import { CanvasGroupToolbar } from "./CanvasGroupToolbar"; +import { isCanvasGroup, groupMembers, makeCanvasGroup, ungroupCanvasNodes, moveCanvasSelection, layoutCanvasGroup, migrateLegacyGroups, exportCanvasGroup, instantiateCanvasGroup, groupBounds, assignGroupMembers } from "@/lib/video-canvas/groups"; +import { compileGroupRun } from "@/lib/video-canvas/group-execution"; + import { requireResolvedReferenceImages, connectionError, connectionTypeError, videoInputLimitError, connectedText, connectedInputNodes, assertConnectedInputsReady, withConnectedText, connectedImageUrls } from "@/lib/video-canvas/connections"; import { resolveSmartEditIntent, assertBeatMusic } from "@/lib/video-canvas/smart-edit/intent"; import { SmartEditDialog } from "./smart-edit/SmartEditDialog"; @@ -354,6 +361,7 @@ type DragState = startY: number; nodeX: number; nodeY: number; + ids?: string[]; } | { kind: "canvas"; @@ -2387,7 +2395,7 @@ function createVideoHdUpscaleNode({ action: "VIDEO_HD_UPSCALE", frameInterpolation: option === "不补帧" ? "不补帧" : existingNode?.draftValues?.frameInterpolation || "不补帧", generated: existingNode?.draftValues?.generated || false, - hdEngine: option === "Topazlabs" ? "Topazlabs" : existingNode?.draftValues?.hdEngine || "Topazlabs", + hdEngine: option === "Real-ESRGAN" ? "Real-ESRGAN" : existingNode?.draftValues?.hdEngine || "Real-ESRGAN", hdResolution: resolution, mediaType: "video", nodeType: "video-hd", @@ -4618,6 +4626,15 @@ export function LiblibVideoCanvasReplica({ const activeGenerationSubmissions = useRef(new Set()); const [edges, setEdges] = useState([]); const [selectedNodeId, setSelectedNodeId] = useState(null); + const [multiSelection, setMultiSelection] = useState([]); + const [groupTemplates,setGroupTemplates]=useState([]); + const {data:savedGroupTemplates}=useSWR<{templates:any[]}>("/api/canvas/group-templates",fetcher); + const [groupSubmitting,setGroupSubmitting]=useState(false); + const [storyboardGroupId,setStoryboardGroupId]=useState(null); + const [selectionRect,setSelectionRect]=useState<{x:number;y:number;width:number;height:number}|null>(null); + const marqueeStart=useRef<{x:number;y:number}|null>(null); + const groupResize=useRef<{id:string;x:number;y:number;width:number;height:number;startX:number;startY:number;corner:string}|null>(null); + const [selectedEdgeId, setSelectedEdgeId] = useState(null); const [lineConnectState, setLineConnectState] = useState(null); const [referenceSelectionTargetId, setReferenceSelectionTargetId] = useState(null); @@ -4645,6 +4662,7 @@ export function LiblibVideoCanvasReplica({ initialProjectRef.current = initialProject; const [loadedProject, setLoadedProject] = useState<{ id: string; revision: string } | null>(null); const generationTasks = useVideoCanvasTasks(canvasId, loadedProject, setNodes, setEdges, edges); + const groupRuns=useCanvasGroupRuns(canvasId,generationTasks.refresh); const [, setShareEnabled] = useState(false); const [publishPublic, setPublishPublic] = useState(false); const [canvasCommand, setCanvasCommand] = useState(null); @@ -4752,7 +4770,9 @@ export function LiblibVideoCanvasReplica({ const data = project.canvasData as { viewport?: typeof DEFAULT_VIEWPORT; selectedNodeIds?: string[] } | undefined; setProjectTitle(title); setDraftTitle(title); - setNodes(restored); + setNodes(migrateLegacyGroups(restored)); + setMultiSelection([]); + setGroupTemplates((project.canvasData as any)?.groupTemplates||[]); setEdges(restoreEdges(project.canvasData)); const storedViewport = data?.viewport; setViewport(storedViewport && [storedViewport.x, storedViewport.y, storedViewport.zoom].every(Number.isFinite) @@ -4836,12 +4856,13 @@ export function LiblibVideoCanvasReplica({ canvasData: { nodes: serializeNodes(nodes), edges: serializeEdges(edges), - selectedNodeIds: selectedNodeId ? [selectedNodeId] : [], + selectedNodeIds: multiSelection.length ? multiSelection : selectedNodeId ? [selectedNodeId] : [], + groupTemplates, viewport, tool: "select", }, }), - [nodes, edges, selectedNodeId, viewport, projectTitle], + [nodes, edges, selectedNodeId, multiSelection, groupTemplates, viewport, projectTitle], ); const applyMergedSnapshot = useCallback((value: string) => { const merged = JSON.parse(value); @@ -4915,7 +4936,7 @@ export function LiblibVideoCanvasReplica({ const removeGenerationNode = useCallback(async (nodeId: string) => { const node=nodes.find(item=>item.id===nodeId); try {await generationTasks.dismiss(nodeId,getDraftString(node,"generationTaskId"));}catch(error){toast.error(error instanceof Error?error.message:"移除失败");return;} - setNodes((current) => current.filter((node) => node.id !== nodeId)); + setNodes((current) => current.some(n=>n.id===nodeId&&isCanvasGroup(n))?ungroupCanvasNodes(current,nodeId):current.filter((node) => node.id !== nodeId)); setEdges((current) => current.filter((edge) => edge.source !== nodeId && edge.target !== nodeId)); setSelectedNodeId((current) => (current === nodeId ? null : current)); setSelectedEdgeId(null); @@ -4997,7 +5018,7 @@ export function LiblibVideoCanvasReplica({ }, }; addNodes( - [group, reference, prompt, presetNode], + [{...group,draftValues:{...group.draftValues,groupVersion:1}}, ...[reference,prompt,presetNode].map(n=>({...n,draftValues:{...n.draftValues,parentGroupId:group.id}}))], [ { id: makeId("edge"), source: reference.id, target: prompt.id }, { id: makeId("edge"), source: prompt.id, target: presetNode.id }, @@ -8184,11 +8205,16 @@ export function LiblibVideoCanvasReplica({ [addNodes, nodes] ); - const generateVideoHdUpscaleNode = useCallback((nodeId: string) => { - const error = "当前没有已接通的视频超清模型，请先配置视频超分服务。"; - setNodes(current => current.map(node => node.id === nodeId ? {...node, draftValues:{...node.draftValues,generated:false,generationStatus:"failed",generationError:error}} : node)); - toast.error(error); - }, []); + const generateVideoHdUpscaleNode = useCallback(async (nodeId: string) => { + const node=nodes.find(n=>n.id===nodeId); if(!node)return; + try { + const incoming=connectedInputNodes(nodes,edges,nodeId); assertConnectedInputsReady(incoming); + const videos=incoming.filter(n=>n.inputType==="video"); + if(videos.length!==1)throw Error("请连接一个已就绪的视频作为高清处理输入"); + const url=getNodeMediaUrl(videos[0]);if(!url)throw Error("上游视频尚未生成"); + await generationTasks.submit(node,"video-upscale",{url,resolution:getDraftString(node,"hdResolution")||"1080P"},edges.filter(e=>e.target===nodeId)); + }catch(error){toast.error(error instanceof Error?error.message:"视频高清提交失败");} + }, [nodes,edges,generationTasks.submit]); const applyVideoCropSelection = useCallback(async (nodeId: string) => { const source = nodes.find(node => node.id === nodeId); @@ -8778,6 +8804,7 @@ export function LiblibVideoCanvasReplica({ setNodeContextMenu(null); setConnectionMenu(null); + if (getDraftString(node,"action")==="VIDEO_HD_UPSCALE") {await generateVideoHdUpscaleNode(node.id);return;} if (getTargetModuleKind(node) === "smart-edit") { setSmartEditNodeId(node.id); return; } if (node.inputType === "audio" && node.draftValues?.audioParams) { const incoming = readyCanvasInputs(nodes, edges, nodeId); @@ -9014,6 +9041,7 @@ export function LiblibVideoCanvasReplica({ edges, nodes, submitVideoTargetComposer, + generateVideoHdUpscaleNode, videoCanvasFunctionConfig, generationTasks.submit, requestVideoAnalysis, @@ -9024,7 +9052,7 @@ export function LiblibVideoCanvasReplica({ const node=nodes.find(item=>item.id===nodeId); if(getDraftString(node,"generationStatus")==="pending") {cancelGenerationNode(nodeId);return;} try {await generationTasks.dismiss(nodeId,getDraftString(node,"generationTaskId"));}catch(error){toast.error(error instanceof Error?error.message:"移除失败");return;} - setNodes((current) => current.filter((node) => node.id !== nodeId)); + setNodes((current) => current.some(n=>n.id===nodeId&&isCanvasGroup(n))?ungroupCanvasNodes(current,nodeId):current.filter((node) => node.id !== nodeId)); setEdges((current) => current.filter((edge) => edge.source !== nodeId && edge.target !== nodeId)); setSelectedNodeId((current) => (current === nodeId ? null : current)); setSelectedEdgeId(null); @@ -9374,6 +9402,55 @@ export function LiblibVideoCanvasReplica({ [] ); + const selectedGroup=multiSelection.length<=1 && selectedNode && isCanvasGroup(selectedNode)?selectedNode:null; + const storyboardMembers=selectedGroup?groupMembers(nodes,selectedGroup.id,false):[]; + const canGroupStoryboard=storyboardMembers.length>1&&storyboardMembers.every(n=>["image","video"].includes(n.inputType||"")&&!getTargetModuleKind(n)&&!!getNodeMediaUrl(n)&&!["pending","failed","interrupted"].includes(getDraftString(n,"generationStatus"))); + const storyboardGroup=nodes.find(n=>n.id===storyboardGroupId),shotOrder=storyboardGroup?.draftValues?.shotNodeIds as string[]|undefined; + const previewShots=storyboardGroup?groupMembers(nodes,storyboardGroup.id,false).filter(n=>["image","video"].includes(n.inputType||"")&&getNodeMediaUrl(n)).sort((a,b)=>(shotOrder?.indexOf(a.id)??0)-(shotOrder?.indexOf(b.id)??0)).map(n=>({id:n.id,title:n.title,url:getNodeMediaUrl(n),type:n.inputType as "image"|"video"})):[]; + const selectedGroupTasks=generationTasks.tasks.filter(t=>t.request.workflow?.groupId===selectedGroup?.id); + const latestGroupRun=selectedGroupTasks.at(-1)?.request.workflow?.runId; + const latestGroupTasks=selectedGroupTasks.filter(t=>t.request.workflow?.runId===latestGroupRun); + const groupBusy=groupSubmitting||groupRuns.submitting||latestGroupTasks.some(t=>["pending","running"].includes(t.status)||t.status==="interrupted"&&hasRecoverableCanvasTask(t)); + const createSelectedGroup=useCallback(()=>{ + try{const id=makeId("group");setNodes(makeCanvasGroup(nodes,multiSelection.length?multiSelection:selectedNodeId?[selectedNodeId]:[],id));setMultiSelection([]);setSelectedNodeId(id);}catch(e){toast.error((e as Error).message);} + },[nodes,multiSelection,selectedNodeId]); + const releaseGroup=useCallback(()=>{if(!selectedNodeId)return;setNodes(current=>ungroupCanvasNodes(current,selectedNodeId));setSelectedNodeId(null);setMultiSelection([]);},[selectedNodeId]); + const executeGroup=async()=>{ + if(!selectedGroup||groupSubmitting)return; + setGroupSubmitting(true); + try { + const requests=compileGroupRun(nodes,edges,selectedGroup.id,canvasId,n=>{ + const node=n as CloneNode,d=node.draftValues||{},p=getNodeParams(node),settings=(p.settings||{}) as Record; + if(d.action==="VIDEO_HD_UPSCALE")return {kind:"video-upscale",params:{resolution:d.hdResolution||"1080P"}}; + if(d.audioParams)return {kind:"audio",params:d.audioParams as Record}; + if(getTargetModuleKind(node)==="smart-edit"||d.nodeType==="smart-edit"){ + const draft=(d.smartEditDraft||{}) as any,plan=draft.plans?.[draft.selectedPlanIndex||0]||d.editPlan; + const changes=edges.some(e=>e.target===node.id&&groupMembers(nodes,selectedGroup.id).some(m=>m.id===e.source&&(/GENERATE|UPSCALE/.test(String(m.draftValues?.action))||m.draftValues?.generationTaskId))); + if(plan&&!changes)return {kind:"smart-edit",params:{operation:"render",plan}}; + return {kind:"smart-edit",params:{operation:"plan",autoRender:true,mode:draft.mode||"sequence",targetDuration:draft.targetDuration||300,ratio:draft.ratio||"16:9",resolution:draft.resolution||"720p",semantic:draft.semantic??false,prompt:draft.prompt||"",removeSilence:draft.removeSilence||false,transcribe:draft.transcribe||false,music:draft.music,beatSync:draft.beatSync||false,variants:1}}; + } + const key=String(p.functionKey||d.functionKey||(node.inputType==="video"?"text-to-video":node.inputType==="image"?"":"story-script")); + const definition=key?getConfiguredFunction(videoCanvasFunctionConfig,key):{key:"image-generate",kind:"image-edit" as const,route:"canvas-generate-image" as const,enabled:true,label:"图片生成",prompt:"",description:""}; + if(!definition?.enabled||!videoCanvasFunctionConfig.enabled)throw Error(`${node.title}：功能未启用或未配置`); + const model=getFunctionModel(videoCanvasFunctionConfig,definition,normalizeModelRef(settings.model||p.model)); + const prompt=composeFunctionPrompt(definition,String(p.prompt||d.prompt||"")); + if(node.inputType==="video")return {kind:"video",params:{prompt,preferredModelRef:model,mode:p.mode||settings.mode||(edges.some(e=>e.target===node.id&&nodes.find(n=>n.id===e.source)?.inputType==="image")?"first-frame":"text-to-video"),duration:Number(String(settings.duration||p.duration||5).replace(/s$/,"")),ratio:settings.ratio||p.ratio||"16:9",resolution:settings.resolution||p.resolution||"720p",generateAudio:settings.audio!=="关闭"}}; + if(node.inputType==="image"){ + if(["enhance","extend","remove-bg"].includes(definition.route||""))return {kind:"image-operation",params:{operation:definition.route,prompt}}; + return {kind:"image",params:{prompt,model,allowFailover:false,imageCount:1,imageSize:settings.resolution||p.imageSize||"1K",aspectRatio:settings.ratio||p.ratio||"auto",billingFeature:getImageGenerationBillingFeature(key),promptExtend:true}}; + } + if(node.inputType==="text"||node.kind==="script"||node.kind==="text")return {kind:"analysis",params:{prompt,preferredModelRef:model,mode:key==="story-script"?"story-script":key==="video-analysis"?"video-analysis":"agent-plan",nodeTitle:node.title}}; + throw Error(`${node.title}：尚未配置可执行动作`); + },()=>crypto.randomUUID()); + await groupRuns.submit(requests); + generationTasks.refresh();toast.success(`已提交 ${requests.length} 个节点，按连线依赖执行`); + }catch(e){toast.error((e as Error).message);}finally{setGroupSubmitting(false);} + }; + const patchGroup=(patch:Record)=>setNodes(current=>current.map(n=>n.id===selectedGroup?.id?{...n,draftValues:{...n.draftValues,...patch}}:n)); + const saveGroupTemplate=()=>{if(!selectedGroup)return;const template=exportCanvasGroup(nodes,edges,selectedGroup.id);setGroupTemplates(current=>[...current.filter(t=>t.sourceId!==selectedGroup.id),{...template,sourceId:selectedGroup.id}].slice(-30));toast.success("已保存到我的工具箱，画布保存后可跨画布复用");}; + const downloadGroup=()=>{if(!selectedGroup)return;const url=URL.createObjectURL(new Blob([JSON.stringify(exportCanvasGroup(nodes,edges,selectedGroup.id),null,2)],{type:"application/json"}));const a=document.createElement("a");a.href=url;a.download=`${selectedGroup.title.replace(/[\\/:*?"<>|]/g,"-")}.canvas-group.json`;a.click();setTimeout(()=>URL.revokeObjectURL(url),1000);}; + const importGroup=(template:any)=>{try{const value=instantiateCanvasGroup(template,()=>makeId("group-node"),Math.max(0,(200-viewport.x)/viewport.zoom),Math.max(0,(200-viewport.y)/viewport.zoom));setNodes(current=>[...current,...value.nodes as unknown as CloneNode[]]);setEdges(current=>[...current,...value.edges]);setSelectedNodeId(value.groupId);setMultiSelection([]);}catch(e){toast.error((e as Error).message);}}; + const startNodeDrag = useCallback( (event: ReactPointerEvent, node: CloneNode) => { event.stopPropagation(); @@ -9482,6 +9559,8 @@ export function LiblibVideoCanvasReplica({ return; } + if(event.shiftKey){setMultiSelection(current=>{const ids=current.length?current:selectedNodeId?[selectedNodeId]:[];return ids.includes(node.id)?ids.filter(id=>id!==node.id):[...ids,node.id];});setSelectedNodeId(node.id);return;} + if(!multiSelection.includes(node.id))setMultiSelection([]); event.currentTarget.setPointerCapture(event.pointerId); setSelectedNodeId(node.id); setSelectedEdgeId(null); @@ -9491,6 +9570,7 @@ export function LiblibVideoCanvasReplica({ dragRef.current = { kind: "node", id: node.id, + ids:multiSelection.includes(node.id)?multiSelection:[node.id], startX: event.clientX, startY: event.clientY, nodeX: node.x, @@ -9499,6 +9579,7 @@ export function LiblibVideoCanvasReplica({ }, [ analyzeVideoCanvasMark, + multiSelection,selectedNodeId, panMode, viewport.x, viewport.y, connectExistingNodes, getCanvasPoint, @@ -9512,6 +9593,8 @@ export function LiblibVideoCanvasReplica({ const startCanvasDrag = useCallback((event: ReactPointerEvent) => { if (event.button !== 0) return; if ((event.target as HTMLElement).closest("[data-canvas-overlay]")) return; + if(event.shiftKey){const point=getCanvasPoint(event.clientX,event.clientY);marqueeStart.current=point;setSelectionRect({...point,width:0,height:0});event.currentTarget.setPointerCapture(event.pointerId);return;} + setMultiSelection([]); setSelectedNodeId(null); setSelectedEdgeId(null); setInspectorNodeId(null); @@ -9526,13 +9609,14 @@ export function LiblibVideoCanvasReplica({ viewportX: viewport.x, viewportY: viewport.y, }; - }, [viewport.x, viewport.y]); + }, [viewport.x, viewport.y,getCanvasPoint]); useEffect(() => { const onKeyDown = (event: KeyboardEvent) => { if (smartEditNodeId) return; const target = event.target as HTMLElement | null; if (target?.closest("input, textarea, select, [contenteditable='true']")) return; + if((event.metaKey||event.ctrlKey)&&event.key.toLowerCase()==="g"){event.preventDefault();if(event.shiftKey&&selectedNodeId&&nodes.some(n=>n.id===selectedNodeId&&isCanvasGroup(n)))releaseGroup();else createSelectedGroup();return;} if (event.code === "Space") { event.preventDefault(); spacePanRef.current = true; return; } if (!event.metaKey && !event.ctrlKey && !event.altKey && event.key.toLowerCase() === "l") { event.preventDefault(); @@ -9583,10 +9667,12 @@ export function LiblibVideoCanvasReplica({ window.addEventListener("keyup", onKeyUp); window.addEventListener("blur", releasePan); return () => { window.removeEventListener("keydown", onKeyDown); window.removeEventListener("keyup", onKeyUp); window.removeEventListener("blur", releasePan); }; - }, [createLineConnectState, deleteNode, deleteSelectedEdge, nodes, selectedEdgeId, selectedNodeId, smartEditNodeId]); + }, [createLineConnectState, createSelectedGroup, releaseGroup, deleteNode, deleteSelectedEdge, nodes, selectedEdgeId, selectedNodeId, smartEditNodeId]); const onPointerMove = useCallback( (event: ReactPointerEvent) => { + if(marqueeStart.current){const start=marqueeStart.current,point=getCanvasPoint(event.clientX,event.clientY),rect={x:Math.min(start.x,point.x),y:Math.min(start.y,point.y),width:Math.abs(point.x-start.x),height:Math.abs(point.y-start.y)};setSelectionRect(rect);setMultiSelection(nodes.filter(n=>!isCanvasGroup(n)&&n.x>=rect.x&&n.y>=rect.y&&n.x+n.width<=rect.x+rect.width&&n.y+n.height<=rect.y+rect.height).map(n=>n.id));return;} + if(groupResize.current){const r=groupResize.current,dx=(event.clientX-r.startX)/viewport.zoom,dy=(event.clientY-r.startY)/viewport.zoom;setNodes(current=>current.map(n=>{if(n.id!==r.id)return n;const children=groupMembers(current,r.id,false),bound=groupBounds(children);const left=r.corner.includes("w")?Math.min(r.x+dx,children.length?bound.x:r.x+r.width-160):r.x,top=r.corner.includes("n")?Math.min(r.y+dy,children.length?bound.y:r.y+r.height-100):r.y;const right=r.corner.includes("e")?Math.max(r.x+r.width+dx,children.length?bound.x+bound.width:r.x+160):r.x+r.width,bottom=r.corner.includes("s")?Math.max(r.y+r.height+dy,children.length?bound.y+bound.height:r.y+100):r.y+r.height;return {...n,x:left,y:top,width:right-left,height:bottom-top};}));return;} const endpointDrag = connectionDragRef.current; if (endpointDrag) { updateEndpointConnectionDrag(event.clientX, event.clientY); @@ -9613,13 +9699,7 @@ export function LiblibVideoCanvasReplica({ const gridSize = snapEnabled ? 8 : 1; const nextX = Math.round((drag.nodeX + dx) / gridSize) * gridSize; const nextY = Math.round((drag.nodeY + dy) / gridSize) * gridSize; - setNodes((current) => - current.map((node) => - node.id === drag.id - ? { ...node, x: nextX, y: nextY } - : node - ) - ); + setNodes(current=>{const anchor=current.find(n=>n.id===drag.id);return anchor?moveCanvasSelection(current,drag.ids||[drag.id],nextX-anchor.x,nextY-anchor.y):current;}); return; } setViewport((current) => ({ @@ -9628,10 +9708,11 @@ export function LiblibVideoCanvasReplica({ y: Math.round(drag.viewportY + event.clientY - drag.startY), })); }, - [getCanvasPoint, getConnectionHoverNodeId, lineConnectState?.sourceId, lineConnectState?.targetId, snapEnabled, updateEndpointConnectionDrag, viewport.zoom] + [nodes,getCanvasPoint, getConnectionHoverNodeId, lineConnectState?.sourceId, lineConnectState?.targetId, snapEnabled, updateEndpointConnectionDrag, viewport.zoom] ); const onPointerUp = useCallback((event: ReactPointerEvent) => { + if(marqueeStart.current||groupResize.current){marqueeStart.current=null;groupResize.current=null;setSelectionRect(null);return;} const endpointDrag = connectionDragRef.current; if (endpointDrag) { if (finishEndpointConnectionDrag(event.clientX, event.clientY)) { @@ -10031,6 +10112,19 @@ export function LiblibVideoCanvasReplica({ /> ) : null} + {storyboardGroupId&&setStoryboardGroupId(null)} onOrder={ids=>setNodes(current=>{const changed=current.map(n=>n.id===storyboardGroupId?{...n,draftValues:{...n.draftValues,shotNodeIds:ids}}:n);const ordered=[...changed.filter(n=>!ids.includes(n.id)),...ids.map(id=>changed.find(n=>n.id===id)!).filter(Boolean)];return layoutCanvasGroup(ordered,storyboardGroupId,"horizontal");})}/>} + {selectedGroup&&t.status==="succeeded").length}/${latestGroupTasks.length} · 失败 ${latestGroupTasks.filter(t=>t.status==="failed").length}`:""} + onRename={title=>setNodes(current=>current.map(n=>n.id===selectedGroup.id?{...n,title}:n))} onColor={color=>patchGroup({groupColor:color})} + onLayout={layout=>setNodes(current=>layoutCanvasGroup(current,selectedGroup.id,layout))} onRun={()=>void executeGroup()} onUngroup={releaseGroup} onSave={saveGroupTemplate} onDownload={downloadGroup} + onStoryboard={()=>{if(!canGroupStoryboard)return;if(selectedGroup.draftValues?.groupMode!=="storyboard"){patchGroup({groupMode:"storyboard",shotNodeIds:storyboardMembers.sort((a,b)=>a.y-b.y||a.x-b.x).map(n=>n.id)});setNodes(current=>layoutCanvasGroup(current,selectedGroup.id,"horizontal"));}setStoryboardGroupId(selectedGroup.id);}} + onCancel={async()=>{for(const task of latestGroupTasks.filter(t=>t.status==="pending"))await generationTasks.cancel(task.id);}}/>} + {multiSelection.length>1&&!selectedGroup&&} + {!selectedGroup&&(multiSelection.length>0||selectedNodeId)&&
} +
我的编组工具箱 + {groupTemplates.map((template,index)=>
)} + {(savedGroupTemplates?.templates||[]).filter(t=>t.projectId!==canvasId).map((template,index)=>)} + +
{ const file=event.target.files?.[0],nodeId=replaceFileNodeId.current;event.target.value=""; if(!file||!nodeId)return; @@ -10068,14 +10162,15 @@ export function LiblibVideoCanvasReplica({ preview={lineConnectPreview} selectedEdgeId={selectedEdgeId} /> - {nodes.map((node) => ( + {selectionRect&&
} + {[...nodes.filter(isCanvasGroup).sort((a,b)=>b.width*b.height-a.width*a.height),...nodes.filter(n=>!isCanvasGroup(n))].map((node) => ( edges.some(edge => edge.target === node.id && edge.source === source.id) && source.inputType === "video") || node) : undefined} - selected={node.id === selectedNodeId} + selected={multiSelection.length?multiSelection.includes(node.id):node.id === selectedNodeId} onContextMenu={(event) => openNodeContext(event, node)} onCancelGeneration={() => cancelGenerationNode(node.id)} onChooseFile={()=>{replaceFileNodeId.current=node.id;if(replaceFileInputRef.current){replaceFileInputRef.current.accept=node.inputType==="audio"?"audio/*":node.inputType==="video"?"video/*":"image/*";replaceFileInputRef.current.click();}}} @@ -10087,7 +10182,11 @@ export function LiblibVideoCanvasReplica({ onModuleAction={(actionLabel) => runTargetModuleAction(node.id, actionLabel)} onOpenConnectionMenu={(event, side) => openConnectionMenu(event, node, side)} onOpenScriptWorkflow={() => openStoryScriptWorkflow(node.id)} - onPointerDown={(event) => startNodeDrag(event, node)} + onPointerDown={(event) => { + const corner=(event.target as HTMLElement).dataset.groupResize; + if(corner){event.stopPropagation();event.currentTarget.setPointerCapture(event.pointerId);groupResize.current={id:node.id,x:node.x,y:node.y,width:node.width,height:node.height,startX:event.clientX,startY:event.clientY,corner};return;} + startNodeDrag(event,node); + }} onStartConnectionDrag={(event, side) => startConnectionEndpointDrag(event, node, side)} onStartConnectionMouseDrag={(event, side) => startConnectionEndpointMouseDrag(event, node, side)} onRunNode={() => { @@ -12355,10 +12454,10 @@ function VideoHdSettingsPopover({ onSelect }: { onSelect: (option: string) => vo
@@ -12915,13 +13014,17 @@ function VideoHdUpscaleCanvasNode({ onGenerate: () => void; selected: boolean; }) { + const [previewError,setPreviewError]=useState(false); + const {data:upscaleCapability}=useSWR<{available:boolean;message?:string}>("/api/canvas/video-upscale/capabilities",fetcher,{refreshInterval:30000}); const generationStatus = getDraftString(node, "generationStatus"); const isGenerating = generationStatus === "pending"; - const generated = Boolean(node.draftValues?.generated); + const generated = Boolean(node.draftValues?.generated) && !isGenerating; const mediaUrl = getNodeMediaUrl(node); + useEffect(()=>setPreviewError(false),[mediaUrl]); const progress = getDraftNumber(node, "generationProgress", 0); const resolution = getDraftString(node, "hdResolution") || "1080P"; - const outputSizeLabel = getDraftString(node, "outputSizeLabel") || "1920 × 1080"; + const meta=node.draftValues?.videoMeta as any; + const outputSizeLabel = meta?.width&&meta?.height?`${meta.width} × ${meta.height}`:getDraftString(node, "outputSizeLabel") || resolution; return (
@@ -12943,12 +13046,15 @@ function VideoHdUpscaleCanvasNode({
- 配置参数 + {getDraftString(node,"generationError")||getDraftString(node,"pendingLabel")||upscaleCapability?.message||`${resolution} · AI 超分 · 保留音频与帧率`}
{isGenerating ? ( @@ -18383,7 +18489,7 @@ function CloneCanvasNode({ selected: boolean; }) { if (getDraftString(node, "action") === "TOOLBOX_TEMPLATE_GROUP") { - return ; + return ; } if ( @@ -18551,23 +18657,12 @@ function CloneCanvasNode({ ); } -function ToolboxTemplateGroupCanvasNode({ node, selected }: { node: CloneNode; selected: boolean }) { - return ( -
-
- {node.title} -
-
- ); +function ToolboxTemplateGroupCanvasNode({ node, selected, onPointerDown, onContextMenu }: { node: CloneNode; selected: boolean; onPointerDown:(event:ReactPointerEvent)=>void;onContextMenu:(event:ReactMouseEvent)=>void }) { + const color=String(node.draftValues?.groupColor||"#aaaaaa"); + return
+
{node.draftValues?.groupMode==="storyboard"?"分镜组 · ":""}{node.title}
+ {selected&&["nw","ne","sw","se"].map(corner=>
)} +
; } function MaterialStyleCanvasNode({ diff --git a/docs/video-canvas-groups-upscale-2026-09-18.md b/docs/video-canvas-groups-upscale-2026-09-18.md new file mode 100644 index 000000000..3721e3a76 --- /dev/null +++ b/docs/video-canvas-groups-upscale-2026-09-18.md @@ -0,0 +1,39 @@ +# 编组和独立视频高清验收 — 2026-09-18 + +工作分支 `codex/video-canvas-parity-0916`。本次没有更新主站前端/主项目版本。 + +## 编组 + +- Shift 点击/框选、Ctrl/⌘+G、新建组、嵌套成员、整体移动、四角调整边框、颜色/名称、网格/横排/竖排、移动到组/移出、解组保留节点和连线。旧预设背景框按几何包含关系迁移一次，后续按明确 parentGroupId 工作。 +- 全部就绪图片/视频可转分镜组，预览、前后镜头、调整顺序；混合工具或未就绪素材时禁用转换。顺序保存在画布中，导出重建 ID。 +- 我的工具箱保存到所属画布，按用户权限跨画布查询；模板 JSON 导入/导出。内部节点和连线重建 ID，运行编号清除；外部媒体依赖转换为必填输入节点，缺素材时整批拒绝。角色/特效外部依赖要求先纳入组内。 +- 整组执行一次性原子入库，客户端持久 outbox 使用同一 runId/requestId 重试；关闭/刷新浏览器后服务端继续执行。按真实连线获取本次上游结果，拒绝循环、错误类型、缺输入、跨批次依赖。上游失败/取消/无可恢复编号的中断会阻断下游，不取旧输出。 +- 支持现有图片/视频/文字分析/音频/图片处理/视频高清任务。智能剪辑遇到本次新视频时拆成隐藏选片阶段 + 成片阶段，依赖新生成素材；保留已确认且输入未变的现有剪辑方案。默认新组合使用顺序剪辑，已有智能参数按保存配置。取消只取消尚未提交的任务。 +- 选中宫格切片需先落成独立图片节点再整组执行，有明确拦截，未用整张原图冒充切片。 + +## 视频高清 + +Real-ESRGAN 独立部署到算力服务器物理 GPU 3（P40）。保留千问 GPU 0/1，GPU 2 不使用。服务与 48→算力私有 API 隧道均启用；48 的 `/etc/video-upscale-client.env` 已准备，尚未注入/发布主项目版本。部署和约束见 `services/video-upscale/README.md`。 + +真实样例均由色彩图案/正弦音合成，无付费模型： + +| 输入 | 输出 | 验证 | +| --- | --- | --- | +| 960×540、2fps、1秒、音轨 | 3840×2160 | 2帧，2fps，音轨保留，1.002秒 | +| 270×480、2fps、1秒、无音轨 | 1080×1920 | 2帧，2fps，1秒 | +| 960×540 + 90度旋转信息、含音轨 | 1080×1920 | 归一化方向，音轨/帧率/帧数保留 | +| 主项目编组提交 960×540 | 2560×1440 | 真实 GPU → 持久任务成功 → 本地对象存储归档 → 页面恢复 src | + +第一轮横屏/竖屏短片含传输与轮询分别约 8.4/6.9 秒。仅是 1 秒 2 帧链路测试，不能推算常见 24/30fps 长视频吞吐。 + +重启测试发现并修复两处问题：128核机器 FFmpeg 默认滤镜线程超过服务限额；SIGTERM 杀子进程时曾将正在处理的任务记成 failed。修复后同一任务重启续跑，attempts=2，返回真实 4K 结果。未鉴权 401、重复请求复用记录、参数冲突 409 均通过。 + +## 验证证据与边界 + +- 75 项定向回归通过，包含连线、编组、任务恢复、智能剪辑；新增组内自动选片/成片依赖测试。 +- 浏览器实测：多选编组、布局、整体拖动、调整边框、模板保存导入/必填输入拦截、分镜转换/预览/排序、真实提交参数依赖、刷新恢复排队、解组保留成员。 +- 队列读取仅查询当前待处理任务及直接依赖，避免每次 claim 读取整个项目历史。 +- 真实 PostgreSQL：原子批量提交、幂等、冲突拒绝、项目所有权；上游失败时下游 attempts=0。 +- 真实主项目 API：2K 成片归档成功，FFprobe 验证 H.264/AAC、2560×1440、帧数/时长；刷新后保留结果链接。 +- 当前验收 Chrome 对 `127.0.0.1:19440` 本地对象存储的媒体加载返回 `Media load rejected by URL safety check`。没有绕过浏览器策略；网页实际播放不能记为通过。界面增加明确“结果已保存、预览加载失败”的提示与打开链接，避免黑屏被理解为仍在处理。尚需在获准访问的正式/测试对象存储域名上完成播放验收后发布主项目。 +- 本地证据保存在忽略目录 `.qa/group-upscale-0918/`，可复用 GPU 验证脚本 `scripts/qa/video-upscale-service.cjs`。正式运行代码/文档不包含密钥或 QA 账号。 diff --git a/hooks/use-canvas-group-runs.ts b/hooks/use-canvas-group-runs.ts new file mode 100644 index 000000000..8e446e6d0 --- /dev/null +++ b/hooks/use-canvas-group-runs.ts @@ -0,0 +1,90 @@ +"use client"; +import { useCallback, useEffect, useRef, useState } from "react"; +import { + canvasTaskRequestSchema, + type CanvasTaskRequest, +} from "@/lib/video-canvas/task-contract"; +import { toast } from "sonner"; +export function useCanvasGroupRuns(projectId: string, refresh: () => void) { + const [submitting, setSubmitting] = useState(false), + active = useRef(projectId), + refreshRef = useRef(refresh), + sending = useRef(false); + active.current = projectId; + refreshRef.current = refresh; + const key = `video-canvas-group-outbox:${projectId}`; + const read = useCallback(() => { + try { + const parsed = canvasTaskRequestSchema + .array() + .min(1) + .max(40) + .safeParse(JSON.parse(localStorage.getItem(key) || "null")); + return parsed.success && + parsed.data.every((r) => r.projectId === projectId && r.workflow) + ? parsed.data + : null; + } catch { + return null; + } + }, [key, projectId]); + const send = useCallback( + async (requests: CanvasTaskRequest[]) => { + if (sending.current) return; + sending.current = true; + setSubmitting(true); + try { + const response = await fetch("/api/canvas/group-runs", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(requests), + signal: AbortSignal.timeout(20000), + }); + if (!response.ok) { + if (response.status < 500 && response.status !== 429) { + localStorage.removeItem(key); + const body = await response.json(); + throw Error(body.error || "编组请求被拒绝"); + } + throw Error("编组提交尚未确认，已保存，将查询同一批任务"); + } + localStorage.removeItem(key); + if (active.current === projectId) refreshRef.current(); + } finally { + sending.current = false; + if (active.current === projectId) setSubmitting(false); + } + }, + [key, projectId] + ); + const submit = useCallback( + async (requests: CanvasTaskRequest[]) => { + const pending = read(); + if (pending && pending[0].workflow?.runId !== requests[0].workflow?.runId) + throw Error("上一批编组提交仍在确认，请稍后重试"); + localStorage.setItem(key, JSON.stringify(requests)); + await send(requests); + }, + [key, read, send] + ); + useEffect(() => { + setSubmitting(false); + const retry = () => { + const requests = read(); + if (requests && !sending.current) + void send(requests).catch((error) => { + if (!read()) toast.error(error.message); + }); + }; + retry(); + const timer = setInterval(retry, 10000); + window.addEventListener("online", retry); + window.addEventListener("focus", retry); + return () => { + clearInterval(timer); + window.removeEventListener("online", retry); + window.removeEventListener("focus", retry); + }; + }, [read, send]); + return { submit, submitting }; +} diff --git a/lib/video-canvas/connections.ts b/lib/video-canvas/connections.ts index e7821bdad..f6c8552cd 100644 --- a/lib/video-canvas/connections.ts +++ b/lib/video-canvas/connections.ts @@ -89,7 +89,7 @@ export function connectionTypeError(source: ConnectionNode, target: ConnectionNo const nodeType = draft.nodeType; if (source.kind === "material-effect" || source.kind === "material-role") return target.inputType === "video" && nodeType !== "smart-edit" ? null : "角色和视频特效素材只能连接到视频生成节点"; - const allowed = nodeType === "smart-edit" || nodeType === "video-clip" ? ["video"] + const allowed = nodeType === "smart-edit" || nodeType === "video-clip" || draft.action === "VIDEO_HD_UPSCALE" ? ["video"] : target.inputType === "image" ? ["image", "text"] : target.inputType === "video" ? (key === "audio-to-video" ? ["image", "text", "audio"] : ["image", "text"]) : target.inputType === "audio" ? ["text"] diff --git a/lib/video-canvas/group-execution.ts b/lib/video-canvas/group-execution.ts new file mode 100644 index 000000000..d204c336c --- /dev/null +++ b/lib/video-canvas/group-execution.ts @@ -0,0 +1,333 @@ +import { + assertConnectedInputsReady, + connectedInputNodes, + connectedText, + connectionTypeError, + videoInputLimitError, + withConnectedText, + type ConnectionNode, + type CanvasConnection, +} from "./connections"; +import { groupMembers, isCanvasGroup, type GroupNode } from "./groups"; +import { hasRecoverableCanvasTask } from "./task-recovery"; +import type { CanvasTaskRequest, CanvasGenerationTask } from "./task-contract"; +export function executableGroupNode(node: GroupNode) { + const d = node.draftValues || {}, + action = String(d.action || ""); + if (isCanvasGroup(node) || node.kind?.startsWith("material-")) return false; + if ( + d.generationTaskId || + d.audioParams || + /GENERATE|UPSCALE|STORY_ANALYSIS|SMART_EDIT/.test(action) + ) + return true; + if ( + /_RESOURCE$/.test(action) || + (node.kind === "text" && + connectedText(node) && + !d.params?.prompt && + !d.prompt) + ) + return false; + return ( + ["model", "workflow", "script", "preset"].includes(node.kind || "") || + !!d.params?.prompt + ); +} +export function compileGroupRun( + nodes: GroupNode[], + edges: CanvasConnection[], + groupId: string, + projectId: string, + prepare: (node: GroupNode) => Pick, + uuid: () => string +) { + const members = groupMembers(nodes, groupId).filter(executableGroupNode); + if (!members.length) throw Error("编组没有可执行节点"); + if (members.length > 40) throw Error("整组最多执行 40 个节点"); + const runId = uuid(), + ids = new Map(members.map((n) => [n.id, uuid()])); + const requests: CanvasTaskRequest[] = members.map((node) => { + const inputs = connectedInputNodes(nodes, edges, node.id).map((source) => { + const error = connectionTypeError(source, node); + if (error) throw Error(`${node.title}：${error}`); + if (!ids.has(source.id)) assertConnectedInputsReady([source]); + if (source.draftValues?.gridSplit?.selectedIndices?.length) + throw Error("编组执行前请将选中的宫格保存为独立图片节点"); + return { node: source, requestId: ids.get(source.id) }; + }); + return { + ...prepare(node), + requestId: ids.get(node.id)!, + projectId, + node: node as CanvasTaskRequest["node"], + edges: edges.filter((e) => e.target === node.id), + workflow: { runId, groupId, inputs }, + }; + }); + const expanded = requests.flatMap((request) => { + if (request.kind !== "smart-edit" || request.params.autoRender !== true) + return [request]; + const planNodeId = `group-plan-${uuid()}`, + planRequestId = uuid(), + planNode = { + ...request.node, + id: planNodeId, + title: `${request.node.title} · 选片方案`, + }; + const { autoRender, ...params } = request.params; + const plan = { + ...request, + node: planNode, + requestId: planRequestId, + params, + hiddenNodeIds: [planNodeId], + edges: request.edges.map((e) => ({ + ...e, + id: uuid(), + target: planNodeId, + })), + }; + const render = { + ...request, + params: { operation: "render" }, + hiddenNodeIds: [planNodeId], + workflow: { + ...request.workflow!, + inputs: [ + { + node: planNode, + requestId: planRequestId, + output: "edit-plan" as const, + }, + ], + }, + edges: [{ id: uuid(), source: planNodeId, target: request.node.id }], + }; + return [plan, render]; + }); + if (expanded.length > 40) + throw Error("包含选片阶段后整组超过 40 个任务，请拆分编组"); + validateGroupBatch(expanded); + return expanded; +} +export function validateGroupBatch(requests: CanvasTaskRequest[]) { + if (!requests.length || requests.length > 40) throw Error("整组任务数量无效"); + const first = requests[0], + map = new Map(requests.map((r) => [r.requestId, r])); + if ( + map.size !== requests.length || + new Set(requests.map((r) => r.node.id)).size !== requests.length + ) + throw Error("重复的整组任务"); + for (const r of requests) { + if ( + !r.workflow || + r.projectId !== first.projectId || + r.workflow.runId !== first.workflow?.runId || + r.workflow.groupId !== first.workflow?.groupId + ) + throw Error("整组任务不属于同一画布和编组"); + for (const input of r.workflow.inputs) { + if ( + !r.edges.some( + (e) => e.source === input.node.id && e.target === r.node.id + ) + ) + throw Error("依赖缺少对应连线"); + if (input.output === "edit-plan") { + const source = input.requestId ? map.get(input.requestId) : null; + if ( + r.kind !== "smart-edit" || + r.params.operation !== "render" || + source?.kind !== "smart-edit" || + source.params.operation !== "plan" + ) + throw Error("剪辑方案依赖无效"); + } else { + const error = connectionTypeError(input.node, r.node); + if (error) throw Error(error); + } + if ( + input.requestId && + map.get(input.requestId)?.node.id !== input.node.id + ) + throw Error("依赖不属于本次编组执行"); + if (!input.requestId) assertConnectedInputsReady([input.node]); + } + if ( + r.edges.some( + (e) => + e.target !== r.node.id || + !r.workflow!.inputs.some((i) => i.node.id === e.source) + ) + ) + throw Error("连线缺少对应输入"); + } + const visited = new Set(), + visiting = new Set(); + const visit = (id: string) => { + if (visiting.has(id)) throw Error("编组存在循环依赖"); + if (visited.has(id)) return; + visiting.add(id); + for (const input of map.get(id)!.workflow!.inputs) + if (input.requestId) visit(input.requestId); + visiting.delete(id); + visited.add(id); + }; + for (const id of map.keys()) visit(id); +} +export function groupDependencyState( + request: CanvasTaskRequest, + tasks: CanvasGenerationTask[] +) { + const dependencies = + request.workflow?.inputs + .filter((i) => i.requestId) + .map((i) => + tasks.find( + (t) => + t.requestId === i.requestId && + t.request.workflow?.runId === request.workflow?.runId + ) + ) || []; + if ( + dependencies.some( + (t) => + !t || + ["failed", "cancelled"].includes(t.status) || + (t.status === "interrupted" && !hasRecoverableCanvasTask(t)) + ) + ) + return "failed"; + return dependencies.every((t) => t?.status === "succeeded") + ? "ready" + : "waiting"; +} +export function resolveGroupInputs( + request: CanvasTaskRequest, + tasks: CanvasGenerationTask[] +): Record { + if (!request.workflow) return request.params; + if (groupDependencyState(request, tasks) !== "ready") + throw Error("编组上游任务尚未成功完成"); + const planInput = request.workflow.inputs.find( + (i) => i.output === "edit-plan" + ); + if (planInput) { + const plan = tasks.find((t) => t.requestId === planInput.requestId)?.result + ?.plans?.[0]; + if (!plan) throw Error("上游未返回剪辑方案"); + return { ...request.params, plan }; + } + const nodes = request.workflow.inputs.map((input) => { + if (!input.requestId) return input.node; + const task = tasks.find((t) => t.requestId === input.requestId)!; + const data = task.result || {}; + const urls = + task.request.kind === "image" || task.request.kind === "image-operation" + ? (data.images || []) + .map((i: any) => i.originalUrl || i.thumbnailUrl) + .filter(Boolean) + : data.url + ? [data.url] + : []; + const text = data.raw || data.summary || (data.agentPlan || []).join("\n"); + return { + ...input.node, + draftValues: { + ...input.node.draftValues, + generationStatus: "complete", + mediaUrl: urls[0], + url: urls, + content: text, + rows: data.analysisRows || data.shotIdeas, + }, + }; + }); + assertConnectedInputsReady(nodes); + const urls = (type: string) => + nodes + .filter((n) => n.inputType === type) + .flatMap((n) => + n.draftValues?.url?.length + ? n.draftValues.url + : [ + n.draftValues?.mediaUrl || + n.draftValues?.resultImage || + n.draftValues?.thumbnailUrl, + ] + ) + .filter(Boolean); + const roleImages = nodes + .filter((n) => n.kind === "material-role" && n.inputType !== "image") + .map((n) => n.draftValues?.thumbnailUrl || n.draftValues?.mediaUrl) + .filter(Boolean); + const images = [...urls("image"), ...roleImages], + videos = urls("video"), + audio = urls("audio"); + const p: Record = { + ...request.params, + prompt: withConnectedText(String(request.params.prompt || ""), nodes), + }; + if (request.kind === "image") + return { + ...p, + referenceImage: images[0], + referenceImages: images.slice(1), + }; + if (request.kind === "video") { + const error = videoInputLimitError(String(p.mode), images.length); + if (error) throw Error(error); + return { + ...p, + prompt: [ + p.prompt, + ...nodes.map( + (n) => n.draftValues?.rolePrompt || n.draftValues?.effectPrompt + ), + ] + .filter(Boolean) + .join("\n\n"), + images, + audioUrls: audio, + }; + } + if (request.kind === "analysis") + return { ...p, imageUrls: images, videoUrls: videos }; + if (request.kind === "audio") + return { + ...p, + text: [request.params.text, ...nodes.map(connectedText)] + .filter(Boolean) + .join("\n\n"), + }; + if (request.kind === "video-upscale") { + if (videos.length !== 1) throw Error("视频高清需要一个上游视频"); + return { ...p, url: videos[0] }; + } + if (request.kind === "image-operation") { + if (images.length !== 1) throw Error("图片处理需要一个上游图片"); + return { ...p, imageUrl: images[0] }; + } + if (request.kind === "smart-edit") { + if (request.params.operation === "plan") { + if (!videos.length) throw Error("智能剪辑没有已就绪的视频输入"); + return { + ...p, + assets: videos.map((url: string, i: number) => ({ + id: `group-asset-${i + 1}`, + name: + nodes.filter((n) => n.inputType === "video")[i]?.title || + `素材 ${i + 1}`, + url, + })), + }; + } + const plan = request.params.plan as any; + if (videos.some((url) => !plan?.assets?.some((a: any) => a.url === url))) + throw Error("剪辑方案与当前连线素材不一致，请重新确认方案"); + return p; + } + throw Error("编组节点暂不支持执行"); +} diff --git a/lib/video-canvas/groups.ts b/lib/video-canvas/groups.ts new file mode 100644 index 000000000..ed5aa1a15 --- /dev/null +++ b/lib/video-canvas/groups.ts @@ -0,0 +1,398 @@ +import { connectionError } from "./connections"; +import type { CanvasConnection, ConnectionNode } from "./connections"; +export type GroupNode = ConnectionNode & { + x: number; + y: number; + width: number; + height: number; + title: string; +}; +export const isCanvasGroup = (node: ConnectionNode) => + node.draftValues?.action === "TOOLBOX_TEMPLATE_GROUP"; +export const parentGroup = (node: ConnectionNode) => + String(node.draftValues?.parentGroupId || ""); +export function groupMembers( + nodes: T[], + groupId: string, + deep = true +): T[] { + const result: T[] = [], + seen = new Set([groupId]); + const visit = (id: string) => { + for (const node of nodes.filter((n) => parentGroup(n) === id)) { + if (seen.has(node.id)) throw Error("编组包含循环成员关系"); + seen.add(node.id); + result.push(node); + if (deep && isCanvasGroup(node)) visit(node.id); + } + }; + visit(groupId); + return result; +} +export function groupBounds(nodes: GroupNode[]) { + if (!nodes.length) return { x: 0, y: 0, width: 400, height: 260 }; + const x = Math.min(...nodes.map((n) => n.x)) - 32, + y = Math.min(...nodes.map((n) => n.y)) - 48; + return { + x, + y, + width: Math.max(...nodes.map((n) => n.x + n.width)) - x + 32, + height: Math.max(...nodes.map((n) => n.y + n.height)) - y + 32, + }; +} +export function selectedRoots(nodes: T[], ids: string[]) { + const selected = new Set(ids); + return nodes.filter( + (node) => + selected.has(node.id) && + !nodes.some( + (group) => + selected.has(group.id) && + isCanvasGroup(group) && + groupMembers(nodes, group.id).some((member) => member.id === node.id) + ) + ); +} +export function makeCanvasGroup( + nodes: T[], + ids: string[], + id: string, + title = "未命名编组" +) { + const members = selectedRoots(nodes, ids); + if (members.length < 2) throw Error("请至少选择两个节点进行编组"); + const parents = new Set(members.map(parentGroup)), + parent = parents.size === 1 ? parentGroup(members[0]) : ""; + const group = { + id, + kind: "model", + title, + ...groupBounds(members), + accent: "#a0a0a0", + draftValues: { + action: "TOOLBOX_TEMPLATE_GROUP", + nodeType: "group", + parentGroupId: parent, + groupColor: "#a0a0a0", + groupVersion: 1, + }, + }; + const selected = new Set(members.map((n) => n.id)); + return [ + ...nodes.map((n) => + selected.has(n.id) + ? { ...n, draftValues: { ...n.draftValues, parentGroupId: id } } + : n + ), + group as unknown as T, + ]; +} +export function ungroupCanvasNodes( + nodes: T[], + groupId: string +) { + const group = nodes.find((n) => n.id === groupId); + if (!group || !isCanvasGroup(group)) throw Error("未找到编组"); + return nodes + .filter((n) => n.id !== groupId) + .map((n) => + parentGroup(n) === groupId + ? { + ...n, + draftValues: { + ...n.draftValues, + parentGroupId: parentGroup(group), + }, + } + : n + ); +} +export function moveCanvasSelection( + nodes: T[], + ids: string[], + dx: number, + dy: number +) { + const moving = new Set(ids); + for (const id of ids) + if (nodes.some((n) => n.id === id && isCanvasGroup(n))) + for (const child of groupMembers(nodes, id)) moving.add(child.id); + let result = nodes.map((n) => + moving.has(n.id) ? { ...n, x: n.x + dx, y: n.y + dy } : n + ); + // Expand stationary ancestor frames when a member is dragged outside them. + for (let pass = 0; pass < nodes.length; pass++) { + let changed = false; + result = result.map((group) => { + if (!isCanvasGroup(group) || moving.has(group.id)) return group; + const children = groupMembers(result, group.id, false); + if (!children.length) return group; + const bound = groupBounds(children), + x = Math.min(group.x, bound.x), + y = Math.min(group.y, bound.y); + const width = Math.max(group.x + group.width, bound.x + bound.width) - x, + height = Math.max(group.y + group.height, bound.y + bound.height) - y; + if ( + x === group.x && + y === group.y && + width === group.width && + height === group.height + ) + return group; + changed = true; + return { ...group, x, y, width, height }; + }); + if (!changed) break; + } + return result; +} +export function layoutCanvasGroup( + nodes: T[], + groupId: string, + layout: "grid" | "horizontal" | "vertical" +) { + const group = nodes.find((n) => n.id === groupId); + if (!group) throw Error("编组不存在"); + const members = groupMembers(nodes, groupId, false); + const cols = + layout === "horizontal" + ? members.length + : layout === "vertical" + ? 1 + : Math.max(1, Math.ceil(Math.sqrt(members.length))); + const cellWidth = Math.max(180, ...members.map((n) => n.width)) + 48, + cellHeight = Math.max(120, ...members.map((n) => n.height)) + 64; + let result = nodes; + for (const [i, member] of members.entries()) + result = moveCanvasSelection( + result, + [member.id], + group.x + 32 + (i % cols) * cellWidth - member.x, + group.y + 48 + Math.floor(i / cols) * cellHeight - member.y + ); + const bounds = groupBounds(groupMembers(result, groupId, false)); + return result.map((n) => (n.id === groupId ? { ...n, ...bounds } : n)); +} +export function migrateLegacyGroups(nodes: T[]) { + // Old preset frames stored geometry but no membership. Infer once on load. + const groups = nodes.filter( + (n) => isCanvasGroup(n) && !n.draftValues?.groupVersion + ); + return nodes.map((n) => { + if (isCanvasGroup(n)) + return { ...n, draftValues: { ...n.draftValues, groupVersion: 1 } }; + if (parentGroup(n)) return n; + const enclosing = groups + .filter( + (g) => + n.x >= g.x && + n.y >= g.y && + n.x + n.width <= g.x + g.width && + n.y + n.height <= g.y + g.height + ) + .sort((a, b) => a.width * a.height - b.width * b.height)[0]; + return enclosing + ? { ...n, draftValues: { ...n.draftValues, parentGroupId: enclosing.id } } + : n; + }); +} +export function exportCanvasGroup( + nodes: T[], + edges: CanvasConnection[], + groupId: string +) { + const group = nodes.find((n) => n.id === groupId); + if (!group) throw Error("编组不存在"); + let all = [group, ...groupMembers(nodes, groupId)]; + const ids = new Set(all.map((n) => n.id)); + const externalIds = [ + ...new Set( + edges + .filter((e) => ids.has(e.target) && !ids.has(e.source)) + .map((e) => e.source) + ), + ]; + const inputs = externalIds.map((id, index) => { + const source = nodes.find((n) => n.id === id); + if ( + !source || + !["image", "video", "audio", "text"].includes(source.inputType || "") + ) + throw Error("请先将组外角色、特效等上游节点加入编组，再保存模板"); + return { + id, + title: `输入：${source.title}`.slice(0, 120), + kind: "input", + inputType: source.inputType, + x: group.x + 32, + y: group.y + 48 + index * 210, + width: 240, + height: 160, + accent: "#aaaaaa", + draftValues: { + action: `${source.inputType!.toUpperCase()}_RESOURCE`, + parentGroupId: group.id, + mediaType: source.inputType, + url: [], + content: [], + generated: false, + generationStatus: "idle", + templateInput: true, + }, + } as unknown as T; + }); + if (inputs.length) { + all = all.map((n) => + n.id === group.id + ? { + ...n, + width: n.width + 280, + height: Math.max(n.height, inputs.length * 210 + 80), + } + : { ...n, x: n.x + 280 } + ); + all.push(...inputs); + for (const input of inputs) ids.add(input.id); + } + return { + version: 1, + name: group.title, + nodes: all.map((n) => ({ + ...n, + draftValues: { + ...n.draftValues, + params: n.draftValues?.params + ? { + ...n.draftValues.params, + taskId: undefined, + sourceNodeId: undefined, + } + : undefined, + ...(n.id === groupId ? { parentGroupId: "" } : {}), + generationTaskId: undefined, + taskRequestId: undefined, + appliedTaskStatus: undefined, + generationStatus: + n.draftValues?.mediaUrl || n.draftValues?.content + ? "complete" + : "idle", + generationError: undefined, + pendingLabel: undefined, + }, + })), + edges: edges.filter((e) => ids.has(e.source) && ids.has(e.target)), + }; +} +export function instantiateCanvasGroup( + template: { nodes: T[]; edges: CanvasConnection[] }, + id: () => string, + x: number, + y: number +) { + if ( + !template || + !Array.isArray(template.nodes) || + !template.nodes.length || + template.nodes.length > 100 + ) + throw Error("编组模板节点数量无效"); + if ( + !Array.isArray(template.edges) || + template.edges.length > 300 || + new Set(template.nodes.map((n) => n.id)).size !== template.nodes.length || + template.nodes.some( + (n) => + !n.id || + ![n.x, n.y, n.width, n.height].every(Number.isFinite) || + n.width <= 0 || + n.height <= 0 || + n.width > 10000 || + n.height > 10000 + ) + ) + throw Error("编组模板数据无效"); + for (const node of template.nodes) { + if ( + parentGroup(node) && + !template.nodes.some( + (n) => n.id === parentGroup(node) && isCanvasGroup(n) + ) + ) + throw Error("编组模板成员关系无效"); + if (isCanvasGroup(node)) groupMembers(template.nodes, node.id); + } + for (const edge of template.edges) { + const error = connectionError( + template.nodes, + template.edges.filter((e) => e !== edge), + edge.source, + edge.target + ); + if (error) throw Error(error); + } + const map = new Map(template.nodes.map((n) => [n.id, id()])); + const root = template.nodes.find((n) => isCanvasGroup(n) && !parentGroup(n)); + if (!root) throw Error("编组模板缺少根组"); + if ( + groupMembers(template.nodes, root.id).length !== + template.nodes.length - 1 + ) + throw Error("模板必须只有一个完整根组"); + const nodes = template.nodes.map((n) => ({ + ...n, + id: map.get(n.id)!, + x: n.x + x - root.x, + y: n.y + y - root.y, + draftValues: { + ...n.draftValues, + parentGroupId: map.get(parentGroup(n)) || "", + shotNodeIds: n.draftValues?.shotNodeIds + ?.map((key: string) => map.get(key)) + .filter(Boolean), + sourceNodeId: map.get(n.draftValues?.sourceNodeId) || undefined, + sourceVideoNodeId: map.get(n.draftValues?.sourceVideoNodeId) || undefined, + }, + })); + return { + nodes, + edges: template.edges + .map((e) => ({ + ...e, + id: id(), + source: map.get(e.source)!, + target: map.get(e.target)!, + })) + .filter((e) => e.source && e.target), + groupId: map.get(root.id)!, + }; +} + +export function assignGroupMembers( + nodes: T[], + ids: string[], + groupId: string +) { + if (groupId && !nodes.some((n) => n.id === groupId && isCanvasGroup(n))) + throw Error("编组不存在"); + const roots = selectedRoots(nodes, ids); + if ( + roots.some( + (n) => + n.id === groupId || + (isCanvasGroup(n) && + groupMembers(nodes, n.id).some((child) => child.id === groupId)) + ) + ) + throw Error("不能将编组加入自身或自己的子组"); + const selected = new Set(roots.map((n) => n.id)); + return moveCanvasSelection( + nodes.map((n) => + selected.has(n.id) + ? { ...n, draftValues: { ...n.draftValues, parentGroupId: groupId } } + : n + ), + [], + 0, + 0 + ); +} diff --git a/lib/video-canvas/task-contract.ts b/lib/video-canvas/task-contract.ts index 872eecc48..e7938ac47 100644 --- a/lib/video-canvas/task-contract.ts +++ b/lib/video-canvas/task-contract.ts @@ -1,12 +1,23 @@ import { hasRecoverableCanvasTask } from "./task-recovery"; import { normalizeVideoAnalysisRows } from "./video-story"; -import { storyTaskAttachmentSchema, mergeStoryTaskResult } from "./story-task-attachment"; +import { + storyTaskAttachmentSchema, + mergeStoryTaskResult, +} from "./story-task-attachment"; import { z } from "zod"; export const canvasTaskRequestSchema = z.object({ requestId: z.string().uuid(), projectId: z.string().uuid(), - kind: z.enum(["video", "image", "analysis", "image-operation", "audio", "smart-edit"]), + kind: z.enum([ + "video", + "image", + "analysis", + "image-operation", + "audio", + "smart-edit", + "video-upscale", + ]), node: z .object({ id: z.string().min(1).max(180), @@ -21,6 +32,27 @@ export const canvasTaskRequestSchema = z.object({ }) .passthrough(), params: z.record(z.unknown()), + workflow: z + .object({ + runId: z.string().uuid(), + groupId: z.string().min(1).max(180), + inputs: z + .array( + z.object({ + output: z.literal("edit-plan").optional(), + requestId: z.string().uuid().optional(), + node: z.object({ + id: z.string().min(1).max(180), + title: z.string().optional(), + kind: z.string().optional(), + inputType: z.string().optional(), + draftValues: z.record(z.any()).optional(), + }), + }) + ) + .max(30), + }) + .optional(), attachment: storyTaskAttachmentSchema.optional(), hiddenNodeIds: z.array(z.string()).max(20).optional(), edges: z @@ -47,9 +79,15 @@ export function canCancelCanvasTask(status: string) { return status === "pending"; } -export function requireAnalysisResult(data: Record | undefined, mode: string) { +export function requireAnalysisResult( + data: Record | undefined, + mode: string +) { if (!data) throw Error("分析没有返回结果，原内容已保留"); - if (mode === "video-analysis" && !normalizeVideoAnalysisRows(data.analysisRows).length) + if ( + mode === "video-analysis" && + !normalizeVideoAnalysisRows(data.analysisRows).length + ) throw Error("视频解析没有返回有效镜头，原内容已保留"); if ( mode === "director-reconstruct" && @@ -66,8 +104,8 @@ export function requireAnalysisResult(data: Record | undefined, mod !rows.length || !rows.some((row) => [row.sceneDescription, row.description, row.imagePrompt].some( - (value) => typeof value === "string" && value.trim(), - ), + (value) => typeof value === "string" && value.trim() + ) )) ) { throw Error("脚本没有返回有效镜头，原内容已保留"); @@ -76,7 +114,7 @@ export function requireAnalysisResult(data: Record | undefined, mod mode !== "story-script" && mode !== "video-analysis" && ![data.raw, data.summary, ...(data.agentPlan || [])].some( - (value) => typeof value === "string" && value.trim(), + (value) => typeof value === "string" && value.trim() ) ) { throw Error("分析返回了空内容，原内容已保留"); @@ -87,13 +125,20 @@ export function requireAnalysisResult(data: Record | undefined, mod // Stable result IDs and request IDs prevent repeat polling from duplicating images // or an old task from overwriting a later generation on the same node. export function reconcileCanvasTasks< - T extends { id: string; draftValues?: Record; [key: string]: any }, + T extends { + id: string; + draftValues?: Record; + [key: string]: any; + }, >(nodes: T[], tasks: CanvasGenerationTask[], projectRevision: string) { let result = nodes; const latestByNode = new Map(); for (const task of tasks) { const previous = latestByNode.get(task.request.node.id); - if (!previous || Date.parse(task.createdAt) >= Date.parse(previous.createdAt)) + if ( + !previous || + Date.parse(task.createdAt) >= Date.parse(previous.createdAt) + ) latestByNode.set(task.request.node.id, task); } for (const task of latestByNode.values()) { @@ -109,7 +154,9 @@ export function reconcileCanvasTasks< const node = result[index]; const draft = node.draftValues || {}; if (draft.taskRequestId && draft.taskRequestId !== task.requestId) { - const savedTask = tasks.find((item) => item.requestId === draft.taskRequestId); + const savedTask = tasks.find( + (item) => item.requestId === draft.taskRequestId + ); if ( !savedTask || Date.parse(task.createdAt) <= Date.parse(savedTask.createdAt) || @@ -117,7 +164,7 @@ export function reconcileCanvasTasks< ) continue; } - const appliedStatus = `${task.id}:${task.status}${task.request.kind === "smart-edit" ? `:${task.result?.smartEdit?.stage || ""}` : ""}`; + const appliedStatus = `${task.id}:${task.status}${["smart-edit", "video-upscale"].includes(task.request.kind) ? `:${task.result?.upscale?.stage || task.result?.smartEdit?.stage || ""}` : ""}`; if (draft.appliedTaskStatus === appliedStatus) continue; const nextDraft = { ...draft, @@ -128,7 +175,11 @@ export function reconcileCanvasTasks< generationProgress: 0, generationStatus: "pending", pendingLabel: - task.status === "pending" ? "排队中" : task.result?.smartEdit?.stage || "生成中", + task.status === "pending" + ? "排队中" + : task.result?.upscale?.stage || + task.result?.smartEdit?.stage || + "生成中", }; let subtitle = nextDraft.pendingLabel; if (task.status === "succeeded") { @@ -144,15 +195,20 @@ export function reconcileCanvasTasks< if (task.request.params.operation !== "render") { const saved = draft.smartEditDraft; const processing = ["separate", "transcribe"].includes( - String(task.request.params.operation), + String(task.request.params.operation) ); const plans = processing && saved?.plans?.length ? saved.plans.map((plan: unknown, i: number) => - i === (saved.selectedPlanIndex || 0) ? data.plans?.[0] || plan : plan, + i === (saved.selectedPlanIndex || 0) + ? data.plans?.[0] || plan + : plan ) : data.plans; - Object.assign(nextDraft, { smartEditPlans: plans, smartEditPlanTaskId: task.id }); + Object.assign(nextDraft, { + smartEditPlans: plans, + smartEditPlanTaskId: task.id, + }); if (processing && saved) Object.assign(nextDraft, { smartEditDraft: { ...saved, plans, planTaskId: task.id }, @@ -176,22 +232,29 @@ export function reconcileCanvasTasks< action: "AUDIO_RESOURCE", mediaType: "audio", audioMeta: data.metadata, - audioParams: task.request.params, + audioParams: + task.request.node.draftValues?.audioParams || task.request.params, }); - if (task.request.kind === "video") + if (["video", "video-upscale"].includes(task.request.kind)) Object.assign(nextDraft, { mediaUrl: data.url, url: [data.url], videoMeta: data.metadata, }); - if (task.request.kind === "image" || task.request.kind === "image-operation") { + if ( + task.request.kind === "image" || + task.request.kind === "image-operation" + ) { const urls = (data.images || []) .map((image: any) => image.originalUrl || image.thumbnailUrl) .filter(Boolean); Object.assign(nextDraft, { mediaUrl: urls[0], url: urls, - action: draft.action === "IMAGE_GENERATE_NINE_GRID" ? draft.action : "IMAGE_RESOURCE", + action: + draft.action === "IMAGE_GENERATE_NINE_GRID" + ? draft.action + : "IMAGE_RESOURCE", }); for (let i = 1; i < urls.length; i++) { const id = `${source.id}-result-${task.id}-${i}`; @@ -203,7 +266,11 @@ export function reconcileCanvasTasks< id, title: `${node.title} ${i + 1}`, y: node.y + i * (node.height + 40), - draftValues: { ...nextDraft, mediaUrl: urls[i], url: [urls[i]] }, + draftValues: { + ...nextDraft, + mediaUrl: urls[i], + url: [urls[i]], + }, } as T, ]; } @@ -218,7 +285,9 @@ export function reconcileCanvasTasks< }); if (task.request.kind === "analysis") Object.assign(nextDraft, { - content: [data.raw || data.summary || data.agentPlan?.join("\n") || ""], + content: [ + data.raw || data.summary || data.agentPlan?.join("\n") || "", + ], ...(Array.isArray(data.analysisRows || data.shotIdeas) ? { rows: data.analysisRows || data.shotIdeas } : {}), @@ -235,7 +304,8 @@ export function reconcileCanvasTasks< : "任务失败，请查看任务状态"), }); if (task.status === "interrupted" && hasRecoverableCanvasTask(task)) - nextDraft.generationError = "任务查询中断，稍后会自动查询原任务，请勿重复生成。"; + nextDraft.generationError = + "任务查询中断，稍后会自动查询原任务，请勿重复生成。"; subtitle = task.status === "cancelled" ? "已取消" @@ -245,7 +315,10 @@ export function reconcileCanvasTasks< : "待核实" : "生成失败"; } - if (task.request.kind === "analysis" && task.request.params.mode === "video-analysis") { + if ( + task.request.kind === "analysis" && + task.request.params.mode === "video-analysis" + ) { Object.assign(nextDraft, { status: task.status === "succeeded" @@ -260,7 +333,7 @@ export function reconcileCanvasTasks< rows: normalizeVideoAnalysisRows( task.result?.analysisRows, task.result?.framePreviews || [], - source.id, + source.id ), framePreviews: task.result?.framePreviews || [], summary: task.result?.summary || "", @@ -273,11 +346,15 @@ export function reconcileCanvasTasks< ...item, ...(task.status === "succeeded" && task.request.kind !== "analysis" && - !(task.request.kind === "smart-edit" && task.request.params.operation !== "render") + !( + task.request.kind === "smart-edit" && + task.request.params.operation !== "render" + ) ? { kind: "media", inputType: - task.request.kind === "video" || task.request.kind === "smart-edit" + ["video", "video-upscale"].includes(task.request.kind) || + task.request.kind === "smart-edit" ? "video" : task.request.kind === "audio" ? "audio" @@ -287,7 +364,7 @@ export function reconcileCanvasTasks< subtitle, draftValues: nextDraft, } - : item, + : item ); } for (const task of tasks) { @@ -299,14 +376,21 @@ export function reconcileCanvasTasks< !result.some((node) => node.id === task.request.node.id) ) continue; - const targetIndex = result.findIndex((node) => node.id === attachment.nodeId); + const targetIndex = result.findIndex( + (node) => node.id === attachment.nodeId + ); if (targetIndex < 0) continue; const node = result[targetIndex], draft = node.draftValues || {}; - const next = mergeStoryTaskResult(draft, attachment, task.result || {}, task.id); + const next = mergeStoryTaskResult( + draft, + attachment, + task.result || {}, + task.id + ); if (next !== draft) result = result.map((item, index) => - index === targetIndex ? { ...item, draftValues: next } : item, + index === targetIndex ? { ...item, draftValues: next } : item ); } return result; diff --git a/lib/video-canvas/task-recovery.ts b/lib/video-canvas/task-recovery.ts index 01d0f60ff..40b3bf374 100644 --- a/lib/video-canvas/task-recovery.ts +++ b/lib/video-canvas/task-recovery.ts @@ -10,7 +10,9 @@ export type RecoveryTask = { request: { kind: string; params: Record }; result?: Record | null; }; -export function getEnhanceCheckpoint(task: Pick) { +export function getEnhanceCheckpoint( + task: Pick +) { const upstream = task.result?.upstreamTask; if ( task.request.kind !== "image-operation" || @@ -27,7 +29,9 @@ export function getEnhanceCheckpoint(task: Pick) { +export function getVideoCheckpoint( + task: Pick +) { if (task.request.kind !== "video") return null; const parsed = videoCheckpointSchema.safeParse(task.result?.upstreamTask); if (!parsed.success) return null; @@ -41,7 +45,9 @@ export function getVideoCheckpoint(task: Pick) { +export function getAudioCheckpoint( + task: Pick +) { if (task.request.kind !== "audio") return null; const parsed = audioCheckpointSchema.safeParse(task.result?.upstreamAudio); return parsed.success && parsed.data.modelRef === task.request.params.modelRef @@ -49,7 +55,7 @@ export function getAudioCheckpoint(task: Pick, + task: Pick ): SmartEditCheckpoint | null { const value = task.result?.smartEdit; if ( @@ -75,7 +81,8 @@ export function hasRecoverableCanvasTask(task: RecoveryTask, now = Date.now()) { getEnhanceCheckpoint(task) || getVideoCheckpoint(task) || getAudioCheckpoint(task) || - getSmartEditCheckpoint(task), + getSmartEditCheckpoint(task) || + getUpscaleCheckpoint(task) ) && Number.isFinite(created) && now >= created && @@ -84,8 +91,23 @@ export function hasRecoverableCanvasTask(task: RecoveryTask, now = Date.now()) { ); } export function canResumeCanvasTask(task: RecoveryTask, now = Date.now()) { - if (task.status !== "interrupted" || !hasRecoverableCanvasTask(task, now)) return false; + if (task.status !== "interrupted" || !hasRecoverableCanvasTask(task, now)) + return false; const updated = Date.parse(task.updatedAt || ""); - const delay = Math.min(300000, 30000 * 2 ** Math.min(4, Math.max(0, (task.attempts || 1) - 1))); + const delay = Math.min( + 300000, + 30000 * 2 ** Math.min(4, Math.max(0, (task.attempts || 1) - 1)) + ); return Number.isFinite(updated) && now - updated >= delay; } + +export function getUpscaleCheckpoint( + task: Pick +) { + const value = task.result?.upscale?.remote; + return task.request.kind === "video-upscale" && + /^[a-f0-9]{64}$/.test(value?.id || "") && + /^[a-f0-9]{64}$/.test(value?.endpointHash || "") + ? (value as { id: string; endpointHash: string }) + : null; +} diff --git a/lib/video-canvas/task-store.ts b/lib/video-canvas/task-store.ts index e9abacfd5..3805a5613 100644 --- a/lib/video-canvas/task-store.ts +++ b/lib/video-canvas/task-store.ts @@ -1,16 +1,20 @@ +import { validateGroupBatch, groupDependencyState } from "./group-execution"; import { canResumeCanvasTask, hasRecoverableCanvasTask } from "./task-recovery"; import "server-only"; -import { and, asc, desc, eq, gte, lt, sql } from "drizzle-orm"; +import { and, asc, desc, eq, gte, lt, sql, inArray } from "drizzle-orm"; import { db } from "@/lib/db"; import { canvasCapabilityTask as table, canvasProject } from "@/lib/db/schema"; -import { type CanvasTaskRequest, type CanvasGenerationTask } from "./task-contract"; +import { + type CanvasTaskRequest, + type CanvasGenerationTask, +} from "./task-contract"; const capabilityId = "video-canvas.generation"; const scope = (userId: string, projectId: string) => and( eq(table.userId, userId), eq(table.projectId, projectId), - eq(table.capabilityId, capabilityId), + eq(table.capabilityId, capabilityId) ); const serialize = (row: typeof table.$inferSelect): CanvasGenerationTask => ({ id: row.id, @@ -29,26 +33,37 @@ export async function ownsTaskProject(userId: string, projectId: string) { await db .select({ id: canvasProject.id }) .from(canvasProject) - .where(and(eq(canvasProject.id, projectId), eq(canvasProject.userId, userId))) + .where( + and(eq(canvasProject.id, projectId), eq(canvasProject.userId, userId)) + ) .limit(1) - )[0], + )[0] ); } -export async function createCanvasGenerationTask(userId: string, request: CanvasTaskRequest) { +export async function createCanvasGenerationTask( + userId: string, + request: CanvasTaskRequest +) { return db.transaction(async (tx) => { await tx.execute( - sql`select pg_advisory_xact_lock(hashtext(${capabilityId}), hashtext(${request.projectId}))`, + sql`select pg_advisory_xact_lock(hashtext(${capabilityId}), hashtext(${request.projectId}))` ); const existing = ( await tx .select() .from(table) - .where(and(scope(userId, request.projectId), eq(table.requestId, request.requestId))) + .where( + and( + scope(userId, request.projectId), + eq(table.requestId, request.requestId) + ) + ) .limit(1) )[0]; if (existing) { if ( - JSON.stringify(existing.request) !== JSON.stringify(JSON.parse(JSON.stringify(request))) + JSON.stringify(existing.request) !== + JSON.stringify(JSON.parse(JSON.stringify(request))) ) { // JSONB property order is not stable. Compare the immutable logical request. const old = existing.request as CanvasTaskRequest; @@ -57,33 +72,46 @@ export async function createCanvasGenerationTask(userId: string, request: Canvas old.kind !== request.kind || JSON.stringify(sortObject(old.attachment)) !== JSON.stringify(sortObject(request.attachment)) || - JSON.stringify(sortObject(old.params)) !== JSON.stringify(sortObject(request.params)) + JSON.stringify(sortObject(old.params)) !== + JSON.stringify(sortObject(request.params)) ) - throw Object.assign(Error("同一任务 ID 不能用于不同请求"), { status: 409 }); + throw Object.assign(Error("同一任务 ID 不能用于不同请求"), { + status: 409, + }); } return serialize(existing); } const active = await tx .select() .from(table) - .where(and(scope(userId, request.projectId), sql`${table.status} in ('pending','running')`)); + .where( + and( + scope(userId, request.projectId), + sql`${table.status} in ('pending','running')` + ) + ); const duplicate = active.find( - (row) => (row.request as CanvasTaskRequest).node.id === request.node.id, + (row) => (row.request as CanvasTaskRequest).node.id === request.node.id ); if (duplicate) return serialize(duplicate); const interrupted = await tx .select() .from(table) - .where(and(scope(userId, request.projectId), eq(table.status, "interrupted"))); + .where( + and(scope(userId, request.projectId), eq(table.status, "interrupted")) + ); const recovering = interrupted.find( (row) => (row.request as CanvasTaskRequest).node.id === request.node.id && - hasRecoverableCanvasTask(serialize(row)), + hasRecoverableCanvasTask(serialize(row)) ); if (recovering) return serialize(recovering); if (active.length >= 40) - throw Object.assign(Error("当前画布排队任务已达 40 个，请等待完成后继续"), { status: 429 }); + throw Object.assign( + Error("当前画布排队任务已达 40 个，请等待完成后继续"), + { status: 429 } + ); const [created] = await tx .insert(table) .values({ @@ -107,11 +135,14 @@ function sortObject(value: any): any { ? Object.fromEntries( Object.keys(value) .sort() - .map((key) => [key, sortObject(value[key])]), + .map((key) => [key, sortObject(value[key])]) ) : value; } -export async function listCanvasGenerationTasks(userId: string, projectId: string) { +export async function listCanvasGenerationTasks( + userId: string, + projectId: string +) { // A worker lost to process shutdown must never silently re-submit a paid request. await db .update(table) @@ -125,8 +156,8 @@ export async function listCanvasGenerationTasks(userId: string, projectId: strin and( scope(userId, projectId), eq(table.status, "running"), - lt(table.updatedAt, new Date(Date.now() - 120000)), - ), + lt(table.updatedAt, new Date(Date.now() - 120000)) + ) ); const rows = await db .select() @@ -136,10 +167,13 @@ export async function listCanvasGenerationTasks(userId: string, projectId: strin .limit(240); return rows.map(serialize).reverse(); } -export async function claimCanvasGenerationTask(userId: string, projectId: string) { +export async function claimCanvasGenerationTask( + userId: string, + projectId: string +) { return db.transaction(async (tx) => { await tx.execute( - sql`select pg_advisory_xact_lock(hashtext(${capabilityId}), hashtext(${projectId}))`, + sql`select pg_advisory_xact_lock(hashtext(${capabilityId}), hashtext(${projectId}))` ); if ( ( @@ -151,12 +185,57 @@ export async function claimCanvasGenerationTask(userId: string, projectId: strin ).length ) return null; - let [next] = await tx + const pending = await tx .select() .from(table) .where(and(scope(userId, projectId), eq(table.status, "pending"))) .orderBy(asc(table.createdAt)) - .limit(1); + .limit(40); + const dependencyIds = [ + ...new Set( + pending.flatMap( + (row) => + (row.request as CanvasTaskRequest).workflow?.inputs + .map((input) => input.requestId) + .filter((id): id is string => Boolean(id)) || [] + ) + ), + ]; + const predecessors = dependencyIds.length + ? await tx + .select() + .from(table) + .where( + and( + scope(userId, projectId), + inArray(table.requestId, dependencyIds) + ) + ) + : []; + const tasks = [ + ...new Map( + [...pending, ...predecessors].map((row) => [row.id, serialize(row)]) + ).values(), + ]; + let next: typeof table.$inferSelect | undefined; + for (const candidate of pending) { + const state = groupDependencyState( + candidate.request as CanvasTaskRequest, + tasks + ); + if (state === "failed") { + await tx + .update(table) + .set({ + status: "failed", + error: "上游任务失败或取消，未提交此节点", + updatedAt: new Date(), + completedAt: new Date(), + }) + .where(eq(table.id, candidate.id)); + tasks.find((t) => t.id === candidate.id)!.status = "failed"; + } else if (state === "ready" && !next) next = candidate; + } if (!next) { const interrupted = await tx .select() @@ -166,8 +245,8 @@ export async function claimCanvasGenerationTask(userId: string, projectId: strin scope(userId, projectId), eq(table.status, "interrupted"), gte(table.createdAt, new Date(Date.now() - 86400000)), - lt(table.attempts, 20), - ), + lt(table.attempts, 20) + ) ) .orderBy(asc(table.updatedAt)); next = interrupted.find((row) => canResumeCanvasTask(serialize(row)))!; @@ -187,7 +266,10 @@ export async function claimCanvasGenerationTask(userId: string, projectId: strin return claimed ? serialize(claimed) : null; }); } -export async function heartbeatCanvasGenerationTask(id: string, attempt?: number) { +export async function heartbeatCanvasGenerationTask( + id: string, + attempt?: number +) { await db .update(table) .set({ updatedAt: new Date() }) @@ -195,15 +277,15 @@ export async function heartbeatCanvasGenerationTask(id: string, attempt?: number and( eq(table.id, id), eq(table.status, "running"), - attempt === undefined ? undefined : eq(table.attempts, attempt), - ), + attempt === undefined ? undefined : eq(table.attempts, attempt) + ) ); } export async function finishCanvasGenerationTask( id: string, result?: Record, error?: string, - attempt?: number, + attempt?: number ) { // A late real result can recover an interrupted task, but cannot replace a newer request ID. await db @@ -219,11 +301,15 @@ export async function finishCanvasGenerationTask( and( eq(table.id, id), sql`${table.status} in ('running','interrupted')`, - attempt === undefined ? undefined : eq(table.attempts, attempt), - ), + attempt === undefined ? undefined : eq(table.attempts, attempt) + ) ); } -export async function cancelCanvasGenerationTask(userId: string, projectId: string, id: string) { +export async function cancelCanvasGenerationTask( + userId: string, + projectId: string, + id: string +) { const [cancelled] = await db .update(table) .set({ @@ -232,7 +318,13 @@ export async function cancelCanvasGenerationTask(userId: string, projectId: stri updatedAt: new Date(), completedAt: new Date(), }) - .where(and(scope(userId, projectId), eq(table.id, id), eq(table.status, "pending"))) + .where( + and( + scope(userId, projectId), + eq(table.id, id), + eq(table.status, "pending") + ) + ) .returning(); if (cancelled) return { ok: true, task: serialize(cancelled) }; const [current] = await db @@ -240,13 +332,16 @@ export async function cancelCanvasGenerationTask(userId: string, projectId: stri .from(table) .where(and(scope(userId, projectId), eq(table.id, id))) .limit(1); - return { ok: current?.status === "cancelled", task: current ? serialize(current) : null }; + return { + ok: current?.status === "cancelled", + task: current ? serialize(current) : null, + }; } export async function hideCanvasGenerationNode( userId: string, projectId: string, id: string, - nodeId: string, + nodeId: string ) { return db.transaction(async (tx) => { const [task] = await tx @@ -255,15 +350,22 @@ export async function hideCanvasGenerationNode( .where(and(scope(userId, projectId), eq(table.id, id))) .for("update"); if (!task) return false; - const request = task.request as CanvasTaskRequest & { hiddenNodeIds?: string[] }; - if (nodeId !== request.node.id && !nodeId.startsWith(`${request.node.id}-result-`)) + const request = task.request as CanvasTaskRequest & { + hiddenNodeIds?: string[]; + }; + if ( + nodeId !== request.node.id && + !nodeId.startsWith(`${request.node.id}-result-`) + ) return false; await tx .update(table) .set({ request: { ...request, - hiddenNodeIds: [...new Set([...(request.hiddenNodeIds || []), nodeId])], + hiddenNodeIds: [ + ...new Set([...(request.hiddenNodeIds || []), nodeId]), + ], }, }) .where(eq(table.id, id)); @@ -274,7 +376,7 @@ export async function hideCanvasGenerationNode( export async function checkpointCanvasGenerationTask( id: string, result: Record, - attempt?: number, + attempt?: number ) { await db .update(table) @@ -283,11 +385,15 @@ export async function checkpointCanvasGenerationTask( and( eq(table.id, id), eq(table.status, "running"), - attempt === undefined ? undefined : eq(table.attempts, attempt), - ), + attempt === undefined ? undefined : eq(table.attempts, attempt) + ) ); } -export async function interruptCanvasGenerationTask(id: string, error: string, attempt?: number) { +export async function interruptCanvasGenerationTask( + id: string, + error: string, + attempt?: number +) { await db .update(table) .set({ status: "interrupted", error, updatedAt: new Date() }) @@ -295,8 +401,8 @@ export async function interruptCanvasGenerationTask(id: string, error: string, a and( eq(table.id, id), eq(table.status, "running"), - attempt === undefined ? undefined : eq(table.attempts, attempt), - ), + attempt === undefined ? undefined : eq(table.attempts, attempt) + ) ); } @@ -304,7 +410,114 @@ export async function ownsCanvasTaskLease(id: string, attempt: number) { const [row] = await db .select({ id: table.id }) .from(table) - .where(and(eq(table.id, id), eq(table.attempts, attempt), eq(table.status, "running"))) + .where( + and( + eq(table.id, id), + eq(table.attempts, attempt), + eq(table.status, "running") + ) + ) .limit(1); return Boolean(row); } + +export async function createCanvasGroupTasks( + userId: string, + requests: CanvasTaskRequest[] +) { + validateGroupBatch(requests); + return db.transaction(async (tx) => { + const projectId = requests[0].projectId; + await tx.execute( + sql`select pg_advisory_xact_lock(hashtext(${capabilityId}), hashtext(${projectId}))` + ); + const same = await tx + .select() + .from(table) + .where( + and( + scope(userId, projectId), + sql`${table.request}->'workflow'->>'runId' = ${requests[0].workflow!.runId}` + ) + ); + if (same.length) { + if ( + same.length !== requests.length || + requests.some((r) => { + const old = same.find((t) => t.requestId === r.requestId); + return ( + !old || + JSON.stringify(sortObject(old.request)) !== + JSON.stringify(sortObject(r)) + ); + }) + ) + throw Object.assign(Error("同一编组执行编号不能用于不同参数"), { + status: 409, + }); + return requests.map((request) => + serialize(same.find((row) => row.requestId === request.requestId)!) + ); + } + const candidates = await tx + .select() + .from(table) + .where( + and( + scope(userId, projectId), + sql`(${table.status} in ('pending','running') or (${table.status}='interrupted' and ${table.createdAt} > now() - interval '24 hours'))` + ) + ); + const active = candidates.filter( + (t) => + ["pending", "running"].includes(t.status) || + (t.status === "interrupted" && hasRecoverableCanvasTask(serialize(t))) + ); + if (active.length + requests.length > 40) + throw Object.assign(Error("画布任务队列已满"), { status: 429 }); + if ( + active.some((t) => + requests.some( + (r) => r.node.id === (t.request as CanvasTaskRequest).node.id + ) + ) + ) + throw Object.assign(Error("编组中已有节点正在执行，请等待完成"), { + status: 409, + }); + const now = Date.now(); + const rows = await tx + .insert(table) + .values( + requests.map((request, i) => ({ + requestId: request.requestId, + userId, + projectId, + capabilityId, + source: "video-canvas", + request, + createdAt: new Date(now + i), + updatedAt: new Date(now), + })) + ) + .returning(); + return rows.map(serialize); + }); +} +export async function groupTaskInputs( + userId: string, + request: CanvasTaskRequest +) { + const rows = await db + .select() + .from(table) + .where( + and( + scope(userId, request.projectId), + sql`${table.request}->'workflow'->>'runId' = ${request.workflow?.runId || ""}` + ) + ); + return rows + .map(serialize) + .filter((t) => t.request.workflow?.runId === request.workflow?.runId); +} diff --git a/lib/video-canvas/task-worker.ts b/lib/video-canvas/task-worker.ts index b1f6b596e..508e216a7 100644 --- a/lib/video-canvas/task-worker.ts +++ b/lib/video-canvas/task-worker.ts @@ -1,3 +1,7 @@ +import { resolveGroupInputs } from "./group-execution"; +import { groupTaskInputs } from "./task-store"; +import { executeVideoUpscale } from "./video-upscale"; +import { getUpscaleCheckpoint } from "./task-recovery"; import { executeSmartEdit } from "./smart-edit/execute"; import { executeCanvasAudio } from "./audio-generation"; import { ProviderTaskUncertainError } from "./video-task-lifecycle"; @@ -13,7 +17,10 @@ import { ImageOperationUncertainError, CanvasTaskLeaseLostError, } from "./image-operation"; -import { executeEnhancePOST, executeEnhanceGET } from "@/lib/canvas/operations/enhance"; +import { + executeEnhancePOST, + executeEnhanceGET, +} from "@/lib/canvas/operations/enhance"; import { executeExtendPOST } from "@/lib/canvas/operations/extend"; import { executeRemoveBackgroundPOST } from "@/lib/canvas/operations/rmbg"; import { executeAngleEditPOST } from "@/lib/canvas/operations/angle-edit"; @@ -26,7 +33,10 @@ import "server-only"; import { executeDirectVideoGeneration } from "@/lib/ai/tools/generate-video"; import { executeCanvasImageResponse } from "@/lib/canvas/generate-image-response"; import { executeWorkflowAnalysisResponse } from "./workflow-analysis-response"; -import { requireAnalysisResult, type CanvasGenerationTask } from "./task-contract"; +import { + requireAnalysisResult, + type CanvasGenerationTask, +} from "./task-contract"; import { validateVideoReferences } from "./reference-input"; import { probeRemoteUrl } from "@/lib/net/url-access"; import { @@ -46,21 +56,40 @@ async function execute(task: CanvasGenerationTask, userId: string) { const videoResume = getVideoCheckpoint(task); const audioResume = getAudioCheckpoint(task); const editResume = getSmartEditCheckpoint(task); - if ((task.attempts || 1) > 1 && !resume && !videoResume && !audioResume && !editResume) + const upscaleResume = getUpscaleCheckpoint(task); + if ( + (task.attempts || 1) > 1 && + !resume && + !videoResume && + !audioResume && + !editResume && + !upscaleResume + ) throw Error("缺少上游任务编号，禁止重新提交"); - const params = request.params; + const params = request.workflow + ? resolveGroupInputs(request, await groupTaskInputs(userId, request)) + : request.params; + if (request.kind === "video-upscale") + return executeVideoUpscale(params, userId, task.id, { + resume: upscaleResume || undefined, + active: assertLease, + save: (value) => + checkpointCanvasGenerationTask(task.id, value, task.attempts), + }); if (request.kind === "smart-edit") return executeSmartEdit(params, userId, { taskId: task.id, resume: editResume, assertActive: assertLease, - checkpoint: (value) => checkpointCanvasGenerationTask(task.id, value, task.attempts), + checkpoint: (value) => + checkpointCanvasGenerationTask(task.id, value, task.attempts), }); if (request.kind === "audio") return executeCanvasAudio(params, { resume: audioResume, assertActive: assertLease, - checkpoint: (value) => checkpointCanvasGenerationTask(task.id, value, task.attempts), + checkpoint: (value) => + checkpointCanvasGenerationTask(task.id, value, task.attempts), }); if (request.kind === "image-operation") return executeImageOperation( @@ -76,14 +105,15 @@ async function execute(task: CanvasGenerationTask, userId: string) { { resume, assertLease, - checkpoint: (result) => checkpointCanvasGenerationTask(task.id, result, task.attempts), - }, + checkpoint: (result) => + checkpointCanvasGenerationTask(task.id, result, task.attempts), + } ); if (request.kind === "video") { const checked = videoResume ? { images: params.images as string[] | undefined, error: undefined } : await validateVideoReferences(params.images, (url) => - probeRemoteUrl(url, { timeoutMs: 8000 }), + probeRemoteUrl(url, { timeoutMs: 8000 }) ); if (checked.error) throw Error(checked.error); const result = await executeDirectVideoGeneration({ @@ -95,7 +125,11 @@ async function execute(task: CanvasGenerationTask, userId: string) { resume: videoResume, assertActive: assertLease, save: (checkpoint) => - checkpointCanvasGenerationTask(task.id, { upstreamTask: checkpoint }, task.attempts), + checkpointCanvasGenerationTask( + task.id, + { upstreamTask: checkpoint }, + task.attempts + ), }, }); if (!result.success || !result.videoUrl) throw Error("视频生成未返回结果"); @@ -105,7 +139,9 @@ async function execute(task: CanvasGenerationTask, userId: string) { ...result.metadata, temporaryVideoUrl: result.videoUrl, storageStatus: - result.metadata?.permanentVideoUrl === result.videoUrl ? "archived" : "temporary", + result.metadata?.permanentVideoUrl === result.videoUrl + ? "archived" + : "temporary", }, modelId: result.modelId, modelRef: result.modelRef, @@ -115,35 +151,44 @@ async function execute(task: CanvasGenerationTask, userId: string) { request.kind === "image" ? await executeCanvasImageResponse( { ...params, prompt: String(params.prompt || ""), async: false }, - userId, + userId ) : await executeWorkflowAnalysisResponse(params); const body = await response.json(); if (!response.ok || !body.success) throw Error(body.error || "任务执行失败"); if ( request.kind === "image" && - !body.data?.images?.some((image: any) => image.originalUrl || image.thumbnailUrl) + !body.data?.images?.some( + (image: any) => image.originalUrl || image.thumbnailUrl + ) ) throw Error("图片生成没有返回结果"); - if (request.kind === "analysis") requireAnalysisResult(body.data, String(params.mode)); + if (request.kind === "analysis") + requireAnalysisResult(body.data, String(params.mode)); requireStoryPromptResult(body.data, request.attachment); return body.data; } -export async function runCanvasGenerationQueue(userId: string, projectId: string) { +export async function runCanvasGenerationQueue( + userId: string, + projectId: string +) { // Work is independent of the browser request. Atomic DB claiming also covers multiple processes. for (let i = 0; i < 40; i++) { const task = await claimCanvasGenerationTask(userId, projectId); if (!task) return; const timer = setInterval( - () => void heartbeatCanvasGenerationTask(task.id, task.attempts).catch(() => {}), - 20000, + () => + void heartbeatCanvasGenerationTask(task.id, task.attempts).catch( + () => {} + ), + 20000 ); try { await finishCanvasGenerationTask( task.id, await execute(task, userId), undefined, - task.attempts, + task.attempts ); } catch (error) { if (error instanceof CanvasTaskLeaseLostError) return; @@ -151,14 +196,18 @@ export async function runCanvasGenerationQueue(userId: string, projectId: string error instanceof ImageOperationUncertainError || error instanceof ProviderTaskUncertainError ) { - await interruptCanvasGenerationTask(task.id, error.message, task.attempts); + await interruptCanvasGenerationTask( + task.id, + error.message, + task.attempts + ); continue; } await finishCanvasGenerationTask( task.id, undefined, error instanceof Error ? error.message : "任务执行失败", - task.attempts, + task.attempts ); } finally { clearInterval(timer); diff --git a/lib/video-canvas/video-upscale-contract.ts b/lib/video-canvas/video-upscale-contract.ts new file mode 100644 index 000000000..02b38b645 --- /dev/null +++ b/lib/video-canvas/video-upscale-contract.ts @@ -0,0 +1,33 @@ +import { z } from "zod"; +export const videoUpscaleSchema = z.object({ + url: z + .string() + .url() + .refine((v) => /^https?:\/\//.test(v)), + resolution: z.enum(["1080P", "2K", "4K"]), +}); +export type VideoUpscaleInput = z.infer; +export function upscaleDimensions( + width: number, + height: number, + resolution: VideoUpscaleInput["resolution"] +) { + if (![width, height].every((v) => Number.isInteger(v) && v > 0)) + throw Error("视频尺寸无效"); + const [long, short] = + resolution === "4K" + ? [3840, 2160] + : resolution === "2K" + ? [2560, 1440] + : [1920, 1080]; + const ratio = Math.min( + (width >= height ? long : short) / width, + (width >= height ? short : long) / height + ); + if (ratio <= 1) throw Error("源视频已达到所选分辨率，请选择更高分辨率"); + if (ratio > 4) throw Error("单次 AI 超分最大 4 倍，请先选择较低目标分辨率"); + return { + width: Math.floor((width * ratio) / 2) * 2, + height: Math.floor((height * ratio) / 2) * 2, + }; +} diff --git a/lib/video-canvas/video-upscale-remote.ts b/lib/video-canvas/video-upscale-remote.ts new file mode 100644 index 000000000..12d9433f0 --- /dev/null +++ b/lib/video-canvas/video-upscale-remote.ts @@ -0,0 +1,181 @@ +import { createHash } from "node:crypto"; +import { createWriteStream } from "node:fs"; +import { Readable, Transform } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import path from "node:path"; +import { z } from "zod"; +import type { VideoUpscaleInput } from "./video-upscale-contract"; +import { ProviderTaskUncertainError } from "./video-task-lifecycle"; + +export type UpscaleCheckpoint = { id: string; endpointHash: string }; +const sha = (value: string) => createHash("sha256").update(value).digest("hex"); +export function upscaleConfig() { + const value = process.env.VIDEO_UPSCALE_URL; + if (!value) return null; + const url = new URL(value); + if ( + !["http:", "https:"].includes(url.protocol) || + url.username || + url.password || + url.search || + url.hash + ) + throw Error("视频超分服务地址配置无效"); + const base = value.replace(/\/$/, ""), + token = process.env.VIDEO_UPSCALE_TOKEN; + if (!token || token.length < 32) throw Error("视频超分服务认证未配置"); + return { base, token }; +} +const statusSchema = z.object({ + id: z.string().regex(/^[a-f0-9]{64}$/), + status: z.enum(["queued", "running", "succeeded", "failed"]), + stage: z.string().max(300), + error: z.string().max(500).optional(), + result: z + .object({ + width: z.number().int().positive().max(3840), + height: z.number().int().positive().max(3840), + durationSeconds: z.number().positive().max(120.2), + fps: z.number().positive().max(60), + frames: z.number().int().positive().max(7200), + audio: z.boolean(), + }) + .optional(), +}); +export async function remoteUpscale( + plan: VideoUpscaleInput, + userId: string, + taskId: string, + dir: string, + lifecycle: { + resume?: UpscaleCheckpoint; + active: () => Promise; + save: (checkpoint: UpscaleCheckpoint, stage: string) => Promise; + } +) { + const config = upscaleConfig(); + if (!config) + throw Error( + lifecycle.resume + ? "远程超分任务的服务配置已移除，不能回退本机重复超分" + : "视频高清服务尚未接通" + ); + const checkpoint = lifecycle.resume || { + id: sha("video-upscale-v1:" + userId + ":" + taskId), + endpointHash: sha(config.base), + }; + if ( + !/^[a-f0-9]{64}$/.test(checkpoint.id) || + checkpoint.endpointHash !== sha(config.base) + ) + throw Error("超分任务所属服务器已变化，请恢复原服务配置后继续"); + await lifecycle.save(checkpoint, "提交算力服务器"); + const headers = { + authorization: "Bearer " + config.token, + "content-type": "application/json", + }; + async function request(route: string, body?: unknown) { + await lifecycle.active(); + let response: Response; + try { + response = await fetch(config!.base + route, { + method: body ? "POST" : "GET", + headers, + body: body ? JSON.stringify(body) : undefined, + signal: AbortSignal.timeout(20000), + redirect: "error", + cache: "no-store", + }); + } catch { + throw new ProviderTaskUncertainError( + "算力服务器连接中断，任务已保存，可继续查询" + ); + } + if ([429, 500, 502, 503, 504].includes(response.status)) + throw new ProviderTaskUncertainError( + "算力服务器繁忙，任务已保存等待继续" + ); + if (!response.ok) + throw Error( + `超分服务请求失败（${response.status}），请检查服务状态或任务参数` + ); + let data: unknown; + try { + data = await response.json(); + } catch { + throw new ProviderTaskUncertainError("超分服务响应中断，将查询同一任务"); + } + const result = statusSchema.parse(data); + if (result.id !== checkpoint.id) + throw Error("超分服务返回了不匹配的任务编号"); + return result; + } + let status = await request("/v1/upscales", { + id: checkpoint.id, + tenant: sha(userId), + plan, + }); + const deadline = Date.now() + 8 * 60000; + while (status.status === "queued" || status.status === "running") { + await lifecycle.save(checkpoint, status.stage); + if (Date.now() > deadline) + throw new ProviderTaskUncertainError("远程超分仍在运行，已保存任务进度"); + await new Promise((resolve) => setTimeout(resolve, 3000)); + status = await request("/v1/upscales/" + checkpoint.id); + } + if (status.status === "failed") + throw Error(status.error || "算力服务器超分失败"); + if (!status.result) throw Error("超分服务未返回成片参数"); + await lifecycle.save(checkpoint, "读取远程成片"); + async function download(kind: string, destination: string, maxBytes: number) { + await lifecycle.active(); + try { + const response = await fetch( + config!.base + "/v1/upscales/" + checkpoint.id + "/" + kind, + { + headers, + signal: AbortSignal.timeout(180000), + redirect: "error", + cache: "no-store", + } + ); + if ( + !response.ok || + !response.body || + Number(response.headers.get("content-length")) > maxBytes + ) + throw Error("远程成片不可读取或超过处理上限"); + let size = 0; + const limit = new Transform({ + transform(chunk, _encoding, callback) { + size += chunk.length; + callback( + size > maxBytes ? Error("远程成片超过处理上限") : null, + chunk + ); + }, + }); + await pipeline( + Readable.fromWeb(response.body as any), + limit, + createWriteStream(destination, { mode: 0o600 }) + ); + } catch { + throw new ProviderTaskUncertainError( + "成片已生成但读取中断，将继续读取同一任务" + ); + } + } + const file = path.join(dir, "result.mp4"); + await download("video", file, 300 * 1024 * 1024); + + return { + file, + width: status.result.width, + height: status.result.height, + duration: status.result.durationSeconds, + fps: status.result.fps, + frames: status.result.frames, + audio: status.result.audio, + }; +} diff --git a/lib/video-canvas/video-upscale.ts b/lib/video-canvas/video-upscale.ts new file mode 100644 index 000000000..edf230dfb --- /dev/null +++ b/lib/video-canvas/video-upscale.ts @@ -0,0 +1,62 @@ +import { ProviderTaskUncertainError } from "./video-task-lifecycle"; +import "server-only"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { uploadFile } from "@/lib/storage/service"; +import { videoUpscaleSchema } from "./video-upscale-contract"; +import { remoteUpscale, type UpscaleCheckpoint } from "./video-upscale-remote"; +export async function executeVideoUpscale( + params: Record, + userId: string, + taskId: string, + lifecycle: { + resume?: UpscaleCheckpoint; + active: () => Promise; + save: (value: Record) => Promise; + } +) { + const input = videoUpscaleSchema.parse(params), + dir = await mkdtemp(path.join(tmpdir(), "upscale-")); + try { + const output = await remoteUpscale(input, userId, taskId, dir, { + resume: lifecycle.resume, + active: lifecycle.active, + save: (remote, stage) => lifecycle.save({ upscale: { remote, stage } }), + }); + await lifecycle.active(); + const file = await readFile(output.file); + let stored; + try { + stored = await uploadFile( + `video-upscale-${taskId}.mp4`, + (file.byteOffset === 0 && file.byteLength === file.buffer.byteLength + ? file.buffer + : file.buffer.slice( + file.byteOffset, + file.byteOffset + file.byteLength + )) as ArrayBuffer + ); + } catch { + throw new ProviderTaskUncertainError( + "高清结果已生成，归档暂时失败，将继续归档同一结果" + ); + } + return { + url: stored.url, + metadata: { + width: output.width, + height: output.height, + durationSeconds: output.duration, + fps: output.fps, + frames: output.frames, + hasAudio: output.audio, + engine: "Real-ESRGAN", + model: "realesr-general-x4v3", + resolution: input.resolution, + }, + }; + } finally { + await rm(dir, { recursive: true, force: true }); + } +} diff --git a/playwright.helpers.config.ts b/playwright.helpers.config.ts index f68ff2135..7cb34ef3d 100644 --- a/playwright.helpers.config.ts +++ b/playwright.helpers.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from "@playwright/test"; export default defineConfig({ testDir: "./tests", testMatch: [ + /routes\/video-canvas-groups-upscale\.test\.ts/, /routes\/video-canvas-connections\.test\.ts/, /routes\/video-canvas-smart-edit\.test\.ts/, /routes\/video-canvas-tasks\.test\.ts/, diff --git a/scripts/qa/video-upscale-service.cjs b/scripts/qa/video-upscale-service.cjs new file mode 100644 index 000000000..ad5069128 --- /dev/null +++ b/scripts/qa/video-upscale-service.cjs @@ -0,0 +1,103 @@ +// Real authenticated GPU API smoke test. Inputs must be the documented synthetic one-second clips. +const fs = require("node:fs"), + crypto = require("node:crypto"), + assert = require("node:assert/strict"); +const { execFileSync } = require("node:child_process"); +const base = process.env.VIDEO_UPSCALE_QA_URL, + token = process.env.VIDEO_UPSCALE_QA_TOKEN, + source = process.env.VIDEO_UPSCALE_QA_SOURCE; +if (!base || !token || !source) + throw Error( + "Set VIDEO_UPSCALE_QA_URL, VIDEO_UPSCALE_QA_TOKEN and VIDEO_UPSCALE_QA_SOURCE" + ); +const sha = (s) => crypto.createHash("sha256").update(s).digest("hex"), + headers = { + authorization: "Bearer " + token, + "content-type": "application/json", + }; +const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +(async () => { + const request = async (route, body) => { + const r = await fetch(base + route, { + method: body ? "POST" : "GET", + headers, + body: body && JSON.stringify(body), + signal: AbortSignal.timeout(15000), + }); + return { status: r.status, body: await r.json() }; + }; + assert.equal((await fetch(base + "/health")).status, 401); + const input = { + id: sha("upscale-qa-" + crypto.randomUUID()), + tenant: sha("isolated-qa"), + plan: { url: source, resolution: "4K" }, + }; + assert.equal((await request("/v1/upscales", input)).status, 202); + assert.equal((await request("/v1/upscales", input)).status, 200); + assert.equal( + ( + await request("/v1/upscales", { + ...input, + plan: { ...input.plan, resolution: "2K" }, + }) + ).status, + 409 + ); + const host = process.env.VIDEO_UPSCALE_QA_SSH_HOST, + control = process.env.VIDEO_UPSCALE_QA_SSH_CONTROL; + if (host) { + for (let i = 0; i < 30; i++) { + if ((await request("/v1/upscales/" + input.id)).body.status === "running") + break; + await wait(100); + } + execFileSync( + "ssh", + [ + ...(control ? ["-S", control] : []), + "-o", + "BatchMode=yes", + host, + "systemctl restart video-upscale", + ], + { timeout: 30000, stdio: "pipe" } + ); + await wait(1500); + assert.equal((await request("/v1/upscales", input)).status, 200); + } + let result; + for (let i = 0; i < 240; i++) { + result = (await request("/v1/upscales/" + input.id)).body; + if (["succeeded", "failed"].includes(result.status)) break; + await wait(2000); + } + assert.equal(result.status, "succeeded", JSON.stringify(result)); + assert.equal(result.result.width, 3840); + assert.equal(result.result.height, 2160); + assert.equal(result.result.frames, 2); + assert.equal(result.result.audio, true); + if (host) assert.equal(result.attempts, 2); + const response = await fetch(base + "/v1/upscales/" + input.id + "/video", { + headers, + }); + assert(response.ok); + fs.mkdirSync(".qa/video-upscale-service", { recursive: true }); + fs.writeFileSync( + ".qa/video-upscale-service/result.mp4", + Buffer.from(await response.arrayBuffer()) + ); + assert.equal( + (await fetch(base + "/v1/upscales/" + input.id + "/video")).status, + 401 + ); + fs.writeFileSync( + ".qa/video-upscale-service/report.json", + JSON.stringify(result, null, 2) + ); + console.log( + "PASS: GPU upscale, authenticated download, idempotency, conflicting parameters, restart recovery, dimensions/frame count/audio" + ); +})().catch((e) => { + console.error(e); + process.exitCode = 1; +}); diff --git a/services/video-upscale/README.md b/services/video-upscale/README.md new file mode 100644 index 000000000..5298f546e --- /dev/null +++ b/services/video-upscale/README.md @@ -0,0 +1,46 @@ +# 独立视频高清服务 + +主项目负责用户/项目鉴权、持久任务、归档；此服务只下载视频并在独立 GPU 上逐帧超分。没有 CPU 回退，不使用千问显卡。当前部署固定 `CUDA_VISIBLE_DEVICES=3`，GPU 2 有硬件故障历史，不分配给本服务。 + +## 模型和环境 + +使用 [Real-ESRGAN](https://github.com/xinntao/Real-ESRGAN) 的 `realesr-general-x4v3`，官方 SRVGGNetCompact 架构（64 通道、32 卷积层、PReLU、4 倍输出）；源码与 BSD-3-Clause 许可位于 `vendor/`，来源 commit 见 `vendor/SOURCE.md`。不是 Topaz 模型。模型文件不提交仓库。 + +权重：[官方 v0.2.5.0 发布](https://github.com/xinntao/Real-ESRGAN/releases/download/v0.2.5.0/realesr-general-x4v3.pth)，SHA-256 `8dc7edb9ac80ccdc30c3a5dca6616509367f05fbc184ad95b731f05bece96292`，推理前强校验。 + +P40 环境实测：Python 3.12.3、PyTorch 2.5.1+cu121、NumPy 1.26.4、Pillow 10.4.0、系统 FFmpeg/ffprobe（libx264/AAC）。PyTorch 使用 CUDA 12.1 官方 wheel 源；必须固定此版本，不能未经 P40 验证直接升级 CUDA/PyTorch。FP32 推理，192px tile + 40px halo，整帧输出在 CPU 内存，避免全帧网络推理耗尽显存。每个服务同时处理一个任务。 + +## 部署与 API + +构建：`node services/video-upscale/build.cjs`，将 `dist/` 内容放到 `/opt/video-upscale/app/`。环境位于 `/opt/video-upscale/venv/`，权重 `/opt/video-upscale/models/realesr-general-x4v3.pth`。参考 `video-upscale.service` 创建专用 `video-upscale` 用户和其拥有的 `/var/lib/video-upscale`。 + +`/etc/video-upscale.env` 权限 600，包含 `UPSCALE_API_TOKEN`（至少 32 字符）、`UPSCALE_HOST=127.0.0.1`、`UPSCALE_PORT=18341`、`UPSCALE_DATA_DIR=/var/lib/video-upscale`。禁止将凭据放进 Git 或浏览器。 + +所有 API 要求 `Authorization: Bearer `： + +- `GET /health`：服务、队列和 GPU 配置。 +- `POST /v1/upscales`：`{id,tenant,plan:{url,resolution}}`，id/tenant 均为 SHA-256 标识，resolution 为 `1080P`/`2K`/`4K`。 +- `GET /v1/upscales/:id`：状态、阶段、帧数/帧率/音轨、尺寸与时长。 +- `GET /v1/upscales/:id/video`：鉴权读取结果。 + +48 服务器使用 `video-upscale-tunnel.service`，复用已有专用隧道账号和固定 known_hosts，只扩展该公钥的 `permitlisten` 到 `127.0.0.1:18341`；原 18340 合成隧道不变。主项目接入变量为 `VIDEO_UPSCALE_URL=http://127.0.0.1:18341` 和 `VIDEO_UPSCALE_TOKEN`，部署时从 root-only `/etc/video-upscale-client.env` 注入，不能依赖本机临时 SSH 转发。 + +任务落盘 fsync + rename；相同 ID/参数幂等，相同 ID/不同参数 409。服务重启保留 running 记录，最多恢复两次；`KillMode=mixed` 先通知 Node 保存关闭状态，再终止子进程，避免把正常重启误记为推理失败。主项目保存远程 ID/服务指纹，轮询超时、读取/归档失败继续查询同一任务。 + +## 处理边界 + +输入 ≤125 MiB、≤120 秒、≤60fps、≤4K 像素数；支持 MP4/MOV、Matroska/WebM、AVI、FLV、MPEG-TS。仅恒定帧率；VFR 明确拒绝，不能静默改变音画同步。手机 90/180/270 度旋转信息会归一化，保留帧数、帧率、音轨。音频重编码 AAC，视频 H.264/yuv420p + faststart。不做补帧。 + +目标以 1920×1080 / 2560×1440 / 3840×2160 为横屏边界，竖屏交换宽高，等比例缩放、偶数尺寸。目标必须大于原图且至多 4 倍；目标已满足或要求超过 4 倍时拒绝，不拿插值假称模型支持更高倍率。网络输出可能产生细节变化，不能保证真实细节还原。 + +单任务超时 2 小时；≤30 个在途任务，≤200 个保留任务，完成/失败保留 7 天；可用磁盘 <10 GiB 拒绝开始。输入禁止内网/DNS 重绑定/不安全重定向，FFmpeg 网络协议关闭；输出下载 ≤300 MiB。systemd 内存 8 GiB、CPU 4 核、任务/线程 256，滤镜线程固定 1，避免 128 核宿主机默认线程数导致编码失败。 + +## 验收 + +`scripts/qa/video-upscale-service.cjs` 验证真实 API、GPU、音轨/帧数/尺寸、重复提交、参数冲突、未鉴权下载及重启恢复。需设置 `VIDEO_UPSCALE_QA_URL`、`VIDEO_UPSCALE_QA_TOKEN`、`VIDEO_UPSCALE_QA_SOURCE`（960×540、2fps、1 秒、含音轨的合成测试片）；可选 `VIDEO_UPSCALE_QA_SSH_HOST` / `VIDEO_UPSCALE_QA_SSH_CONTROL` 执行独立服务重启。只重启视频高清服务，不重启千问或主项目。 + +测试素材生成示例（应上传到获准访问的测试存储域名）： + +```sh +ffmpeg -f lavfi -i 'testsrc2=size=960x540:rate=2:duration=1' -f lavfi -i 'sine=frequency=440:sample_rate=48000:duration=1' -c:v libx264 -pix_fmt yuv420p -c:a aac -shortest landscape.mp4 +``` diff --git a/services/video-upscale/build.cjs b/services/video-upscale/build.cjs new file mode 100644 index 000000000..69226d5f6 --- /dev/null +++ b/services/video-upscale/build.cjs @@ -0,0 +1,32 @@ +const fs = require("node:fs"), + path = require("node:path"); +const esbuild = require( + path.resolve( + "node_modules/.pnpm", + fs + .readdirSync("node_modules/.pnpm") + .find((n) => n.startsWith("esbuild@0.25.")), + "node_modules/esbuild" + ) +); +esbuild + .build({ + entryPoints: ["services/video-upscale/server.ts"], + outfile: "services/video-upscale/dist/server.cjs", + bundle: true, + platform: "node", + target: "node22", + format: "cjs", + }) + .then(() => { + fs.copyFileSync( + "services/video-upscale/upscale.py", + "services/video-upscale/dist/upscale.py" + ); + fs.cpSync( + "services/video-upscale/vendor", + "services/video-upscale/dist/vendor", + { recursive: true } + ); + }) + .catch(() => process.exit(1)); diff --git a/services/video-upscale/job.ts b/services/video-upscale/job.ts new file mode 100644 index 000000000..ed409b1f5 --- /dev/null +++ b/services/video-upscale/job.ts @@ -0,0 +1,85 @@ +import { spawn } from "node:child_process"; +import path from "node:path"; +import { downloadCanvasMedia } from "../../lib/video-canvas/download-media"; +import { + videoUpscaleSchema, + type VideoUpscaleInput, +} from "../../lib/video-canvas/video-upscale-contract"; +export async function upscaleJob( + value: VideoUpscaleInput, + _tenant: string, + dir: string, + progress: (stage: string) => Promise +) { + const input = videoUpscaleSchema.parse(value); + const file = path.join(dir, "input.bin"); + await downloadCanvasMedia(input.url, file); + await progress("GPU 正在进行 AI 超分"); + return new Promise((resolve, reject) => { + const child = spawn( + process.env.UPSCALE_PYTHON || "/opt/video-upscale/venv/bin/python", + [ + process.env.UPSCALE_SCRIPT || path.join(__dirname, "upscale.py"), + "--input", + file, + "--output", + path.join(dir, "result.mp4"), + "--resolution", + input.resolution, + "--model", + process.env.UPSCALE_MODEL || + "/opt/video-upscale/models/realesr-general-x4v3.pth", + ], + { stdio: ["ignore", "pipe", "pipe"], detached: true } + ); + let buffer = "", + result: any, + error = "", + stages = Promise.resolve(); + const timer = setTimeout(() => { + try { + process.kill(-child.pid!, "SIGKILL"); + } catch {} + }, 2 * 3600000); + child.stdout.on("data", (chunk) => { + buffer += chunk; + const lines = buffer.split("\n"); + buffer = lines.pop()!; + for (const line of lines) + try { + const value = JSON.parse(line); + if (value.result) result = value.result; + if (value.stage) + stages = stages + .then(() => progress(value.stage)) + .catch(() => { + try { + process.kill(-child.pid!, "SIGKILL"); + } catch {} + }); + } catch {} + }); + child.stderr.on("data", (chunk) => { + error = (error + chunk).slice(-4000); + }); + child.on("error", (e) => { + clearTimeout(timer); + reject(e); + }); + child.on("close", async (code) => { + clearTimeout(timer); + await stages.catch(() => {}); + if (code !== 0 || !result) { + console.error( + "Upscale inference failed", + error.replace(/https?:\/\/\S+/g, "[url]") + ); + reject( + Error( + "AI 超分失败，请检查视频是否为恒定帧率、120 秒以内，且放大倍率为 1–4 倍" + ) + ); + } else resolve({ ...result, duration: result.durationSeconds }); + }); + }); +} diff --git a/services/video-upscale/server.ts b/services/video-upscale/server.ts new file mode 100644 index 000000000..9bd80d40a --- /dev/null +++ b/services/video-upscale/server.ts @@ -0,0 +1,328 @@ +import http from "node:http"; +import { createReadStream } from "node:fs"; +import { + mkdir, + readFile, + rename, + readdir, + rm, + stat, + statfs, + open, +} from "node:fs/promises"; +import path from "node:path"; +import { createHash, timingSafeEqual } from "node:crypto"; +import { pipeline } from "node:stream/promises"; +import { z } from "zod"; +import { + videoUpscaleSchema, + type VideoUpscaleInput, +} from "../../lib/video-canvas/video-upscale-contract"; +import { upscaleJob } from "./job"; + +const root = path.resolve( + process.env.UPSCALE_DATA_DIR || "/var/lib/video-upscale" +); +const token = process.env.UPSCALE_API_TOKEN || ""; +if (token.length < 32) + throw Error("UPSCALE_API_TOKEN must have at least 32 characters"); +const jobs = new Map(); +type Job = { + id: string; + digest: string; + tenant: string; + plan: VideoUpscaleInput; + status: "queued" | "running" | "succeeded" | "failed"; + stage: string; + createdAt: string; + updatedAt: string; + attempts: number; + error?: string; + result?: Record; +}; +const inputSchema = z.object({ + id: z.string().regex(/^[a-f0-9]{64}$/), + tenant: z.string().regex(/^[a-f0-9]{64}$/), + plan: videoUpscaleSchema, +}); +let busy = false, + stopping = false, + accepting = false; +const file = (id: string) => path.join(root, id + ".json"); +async function persist(job: Job) { + job.updatedAt = new Date().toISOString(); + const temp = file(job.id) + ".tmp"; + const handle = await open(temp, "w", 0o600); + try { + await handle.writeFile(JSON.stringify(job)); + await handle.sync(); + } finally { + await handle.close(); + } + await rename(temp, file(job.id)); + const directory = await open(root, "r"); + try { + await directory.sync(); + } finally { + await directory.close(); + } + jobs.set(job.id, job); +} +function publicJob(job: Job) { + return { + id: job.id, + status: job.status, + stage: job.stage, + attempts: job.attempts, + createdAt: job.createdAt, + updatedAt: job.updatedAt, + error: job.error, + result: job.result, + }; +} +async function prune() { + for (const job of jobs.values()) + if ( + ["succeeded", "failed"].includes(job.status) && + Date.now() - Date.parse(job.updatedAt) > 7 * 86400000 + ) { + await rm(path.join(root, job.id), { recursive: true, force: true }); + await rm(file(job.id), { force: true }); + jobs.delete(job.id); + } +} +async function work() { + if (busy || stopping) return; + const job = [...jobs.values()].find((j) => j.status === "queued"); + if (!job) return; + busy = true; + const dir = path.join(root, job.id); + try { + await prune(); + const disk = await statfs(root); + if (disk.bavail * disk.bsize < 10 * 1024 ** 3) + throw Error("超分磁盘剩余空间不足 10 GiB"); + job.status = "running"; + job.attempts++; + job.stage = "准备素材"; + await persist(job); + await rm(dir, { recursive: true, force: true }); + await mkdir(dir, { mode: 0o700 }); + const result = await upscaleJob( + job.plan, + job.tenant, + dir, + async (stage) => { + job.stage = stage; + await persist(job); + } + ); + job.result = result; + // Keep only the deliverables. Intermediate files must not accumulate per job. + for (const name of await readdir(dir)) + if (name !== "result.mp4") + await rm(path.join(dir, name), { recursive: true, force: true }).catch( + () => { + console.warn( + JSON.stringify({ event: "upscale-cleanup-deferred", id: job.id }) + ); + } + ); + job.status = "succeeded"; + job.stage = "超分完成"; + await persist(job); + } catch (error) { + if (stopping) return; // Preserve running journal; restart will resume this same job. + job.status = "failed"; + job.stage = "超分失败"; + // Do not expose ffmpeg arguments, private cache paths or signed source URLs. + job.error = + error instanceof Error && + !/Command failed|https?:|\/var\/|\/tmp\//.test(error.message) + ? error.message.slice(0, 300) + : "素材处理失败，请检查格式、参数或服务日志"; + await rm(dir, { recursive: true, force: true }).catch(() => {}); + await persist(job); + console.error( + JSON.stringify({ event: "upscale-failed", id: job.id, error: job.error }) + ); + } finally { + busy = false; + } + setImmediate( + () => + void work().catch((e) => { + console.error("Render queue persistence failure"); + process.exit(1); + }) + ); +} +function json(res: http.ServerResponse, status: number, value: unknown) { + res.writeHead(status, { + "content-type": "application/json; charset=utf-8", + "cache-control": "no-store", + }); + res.end(JSON.stringify(value)); +} +async function body(req: http.IncomingMessage) { + let size = 0; + const chunks: Buffer[] = []; + for await (const chunk of req) { + size += chunk.length; + if (size > 1024 * 1024) throw Error("请求超过 1 MiB"); + chunks.push(chunk); + } + return JSON.parse(Buffer.concat(chunks).toString("utf8")); +} +function authenticated(req: http.IncomingMessage) { + const expected = Buffer.from("Bearer " + token), + actual = Buffer.from(req.headers.authorization || ""); + return expected.length === actual.length && timingSafeEqual(expected, actual); +} +async function main() { + await mkdir(root, { recursive: true, mode: 0o700 }); + // This service runs as a single systemd process. A restart requeues local work at most twice. + for (const name of await readdir(root)) + if (/^[a-f0-9]{64}\.json$/.test(name)) { + const job: Job = JSON.parse( + await readFile(path.join(root, name), "utf8") + ); + if (job.id + ".json" !== name) throw Error("Invalid upscale record"); + if (job.status === "running") { + job.status = job.attempts < 3 ? "queued" : "failed"; + job.stage = + job.status === "queued" + ? "服务重启，等待恢复超分" + : "多次中断，已停止"; + if (job.status === "failed") job.error = job.stage; + await persist(job); + } + jobs.set(job.id, job); + } + await prune(); + const server = http.createServer(async (req, res) => { + if (!authenticated(req)) return json(res, 401, { error: "Unauthorized" }); + const url = new URL(req.url || "/", "http://localhost"); + try { + if (req.method === "GET" && url.pathname === "/health") + return json(res, 200, { + status: "ok", + version: 1, + active: busy ? 1 : 0, + queued: [...jobs.values()].filter((j) => j.status === "queued") + .length, + render: "realesr-general-x4v3", + gpu: process.env.CUDA_VISIBLE_DEVICES, + }); + if (req.method === "POST" && url.pathname === "/v1/upscales") { + const input = inputSchema.parse(await body(req)); + // Serialize admission across the async filesystem write to preserve idempotency. + if (accepting) return json(res, 503, { error: "请重试提交" }); + accepting = true; + try { + const digest = createHash("sha256") + .update(JSON.stringify({ tenant: input.tenant, plan: input.plan })) + .digest("hex"); + const existing = jobs.get(input.id); + if (existing) + return json( + res, + existing.digest === digest ? 200 : 409, + existing.digest === digest + ? publicJob(existing) + : { error: "任务 ID 已用于不同参数" } + ); + if ( + jobs.size >= 200 || + [...jobs.values()].filter((j) => + ["queued", "running"].includes(j.status) + ).length >= 30 + ) + return json(res, 429, { error: "超分队列已满，请稍后重试" }); + const job: Job = { + ...input, + digest, + status: "queued", + stage: "排队中", + attempts: 0, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }; + await persist(job); + json(res, 202, publicJob(job)); + void work().catch(() => { + console.error("Render queue failed"); + process.exit(1); + }); + return; + } finally { + accepting = false; + } + } + const match = url.pathname.match( + /^\/v1\/upscales\/([a-f0-9]{64})(?:\/(video))?$/ + ); + if (req.method === "GET" && match) { + const job = jobs.get(match[1]); + if (!job) return json(res, 404, { error: "任务不存在或已过期" }); + if (!match[2]) return json(res, 200, publicJob(job)); + if (job.status !== "succeeded") + return json(res, 409, { error: "成片尚未就绪" }); + const output = path.join(root, job.id, "result.mp4"); + const info = await stat(output); + res.writeHead(200, { + "content-type": "video/mp4", + "content-length": info.size, + "cache-control": "no-store", + }); + await pipeline(createReadStream(output), res); + return; + } + json(res, 404, { error: "Not found" }); + } catch (error) { + if (res.headersSent) { + res.destroy(); + return; + } + json(res, 400, { + error: + error instanceof z.ZodError + ? "超分参数无效" + : error instanceof Error && !/https?:|\/var\//.test(error.message) + ? error.message.slice(0, 300) + : "请求处理失败", + }); + } + }); + server.requestTimeout = 120000; + server.headersTimeout = 10000; + server.listen( + Number(process.env.UPSCALE_PORT || 18341), + process.env.UPSCALE_HOST || "127.0.0.1" + ); + const fatal = () => { + console.error("Render queue persistence failure"); + process.exit(1); + }; + const timer = setInterval(() => { + void work().catch(fatal); + }, 3000); + const cleanupTimer = setInterval(() => { + if (!busy && !accepting) void prune().catch(fatal); + }, 3600000); + cleanupTimer.unref(); + for (const signal of ["SIGTERM", "SIGINT"]) + process.on(signal, () => { + stopping = true; + clearInterval(timer); + clearInterval(cleanupTimer); + server.close(); + // systemd KillMode=mixed also terminates FFmpeg; persisted running jobs resume on restart. + setTimeout(() => process.exit(0), 1000).unref(); + }); + console.log("Video upscale service ready"); +} +void main().catch(() => { + console.error("Video upscale service startup failed"); + process.exitCode = 1; +}); diff --git a/services/video-upscale/upscale.py b/services/video-upscale/upscale.py new file mode 100644 index 000000000..ec6dbb78f --- /dev/null +++ b/services/video-upscale/upscale.py @@ -0,0 +1,79 @@ +"""Bounded, tiled Real-ESRGAN video inference. No CPU fallback or frame interpolation.""" +import argparse, json, subprocess, sys, os, hashlib +from fractions import Fraction +from pathlib import Path +import numpy as np +import torch +from PIL import Image +sys.path.insert(0, str(Path(__file__).parent / 'vendor')) +from srvgg_arch import SRVGGNetCompact + +def probe(file): + return json.loads(subprocess.check_output(['ffprobe','-v','error','-protocol_whitelist','file,pipe','-show_streams','-show_format','-of','json',file], timeout=30)) + +def main(): + p=argparse.ArgumentParser() + for name in ['input','output','model','resolution']: p.add_argument('--'+name, required=True) + p.add_argument('--tile', type=int, default=192) + a=p.parse_args() + if not torch.cuda.is_available(): raise RuntimeError('CUDA unavailable; refusing CPU fallback') + torch.set_num_threads(4) + info=probe(a.input); v=next(s for s in info['streams'] if s['codec_type']=='video') + if not any(name in info['format']['format_name'].split(',') for name in ['mov','mp4','matroska','webm','avi','flv','mpegts']): raise ValueError('Unsupported video container') + rotation=next((int(side['rotation'])%360 for side in v.get('side_data_list',[]) if 'rotation' in side),0) + if rotation%90: raise ValueError('Unsupported rotation angle') + w,h=int(v['width']),int(v['height']) + if rotation in [90,270]: w,h=h,w + fps=Fraction(v['avg_frame_rate']) + duration=float(info['format']['duration']) + if not 03840*2160: raise ValueError('Input exceeds 120 s / 60 fps / 4K limits') + # Reject variable frame rate rather than silently altering timing/audio sync. + frame_times=subprocess.check_output(['ffprobe','-v','error','-protocol_whitelist','file,pipe','-select_streams','v:0','-show_entries','frame=best_effort_timestamp_time','-of','csv=p=0',a.input],timeout=60).decode().splitlines() + timestamps=[float(s.split(',')[0]) for s in frame_times if s.split(',')[0].strip()] + if any(abs((b-c)-float(1/fps))>max(.002,float(1/fps)*.05) for c,b in zip(timestamps,timestamps[1:])): raise ValueError('Variable frame rate video must be converted to constant frame rate first') + long,short={'1080P':(1920,1080),'2K':(2560,1440),'4K':(3840,2160)}[a.resolution] + scale=min((long if w>=h else short)/w,(short if w>=h else long)/h) + if not 1.2: raise RuntimeError('Output timing mismatch') + if any(s['codec_type']=='audio' for s in actual['streams']) != any(s['codec_type']=='audio' for s in info['streams']): raise RuntimeError('Output audio stream missing') + os.unlink(temp) + print(json.dumps({'result':{'width':ow,'height':oh,'durationSeconds':float(actual['format']['duration']),'fps':float(fps),'frames':count,'audio':any(s['codec_type']=='audio' for s in actual['streams']),'engine':'Real-ESRGAN','model':'realesr-general-x4v3'}}),flush=True) +if __name__=='__main__': main() diff --git a/services/video-upscale/vendor/LICENSE b/services/video-upscale/vendor/LICENSE new file mode 100644 index 000000000..552a1eeaf --- /dev/null +++ b/services/video-upscale/vendor/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2021, Xintao Wang +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/services/video-upscale/vendor/SOURCE.md b/services/video-upscale/vendor/SOURCE.md new file mode 100644 index 000000000..1e1e7643c --- /dev/null +++ b/services/video-upscale/vendor/SOURCE.md @@ -0,0 +1,4 @@ +Source: https://github.com/xinntao/Real-ESRGAN +Commit: a4abfb2979a7bbff3f69f58f58ae324608821e27 +BSD-3-Clause license, retained alongside source. +Only the BasicSR registry import/decorator is removed; inference architecture is unchanged. diff --git a/services/video-upscale/vendor/srvgg_arch.py b/services/video-upscale/vendor/srvgg_arch.py new file mode 100644 index 000000000..7882a1b47 --- /dev/null +++ b/services/video-upscale/vendor/srvgg_arch.py @@ -0,0 +1,67 @@ +from torch import nn as nn +from torch.nn import functional as F + + +class SRVGGNetCompact(nn.Module): + """A compact VGG-style network structure for super-resolution. + + It is a compact network structure, which performs upsampling in the last layer and no convolution is + conducted on the HR feature space. + + Args: + num_in_ch (int): Channel number of inputs. Default: 3. + num_out_ch (int): Channel number of outputs. Default: 3. + num_feat (int): Channel number of intermediate features. Default: 64. + num_conv (int): Number of convolution layers in the body network. Default: 16. + upscale (int): Upsampling factor. Default: 4. + act_type (str): Activation type, options: 'relu', 'prelu', 'leakyrelu'. Default: prelu. + """ + + def __init__(self, num_in_ch=3, num_out_ch=3, num_feat=64, num_conv=16, upscale=4, act_type='prelu'): + super(SRVGGNetCompact, self).__init__() + self.num_in_ch = num_in_ch + self.num_out_ch = num_out_ch + self.num_feat = num_feat + self.num_conv = num_conv + self.upscale = upscale + self.act_type = act_type + + self.body = nn.ModuleList() + # the first conv + self.body.append(nn.Conv2d(num_in_ch, num_feat, 3, 1, 1)) + # the first activation + if act_type == 'relu': + activation = nn.ReLU(inplace=True) + elif act_type == 'prelu': + activation = nn.PReLU(num_parameters=num_feat) + elif act_type == 'leakyrelu': + activation = nn.LeakyReLU(negative_slope=0.1, inplace=True) + self.body.append(activation) + + # the body structure + for _ in range(num_conv): + self.body.append(nn.Conv2d(num_feat, num_feat, 3, 1, 1)) + # activation + if act_type == 'relu': + activation = nn.ReLU(inplace=True) + elif act_type == 'prelu': + activation = nn.PReLU(num_parameters=num_feat) + elif act_type == 'leakyrelu': + activation = nn.LeakyReLU(negative_slope=0.1, inplace=True) + self.body.append(activation) + + # the last conv + self.body.append(nn.Conv2d(num_feat, num_out_ch * upscale * upscale, 3, 1, 1)) + # upsample + self.upsampler = nn.PixelShuffle(upscale) + + def forward(self, x): + out = x + for i in range(0, len(self.body)): + out = self.body[i](out) + + out = self.upsampler(out) + # add the nearest upsampled image, so that the network learns the residual + base = F.interpolate(x, scale_factor=self.upscale, mode='nearest') + out += base + return out diff --git a/services/video-upscale/video-upscale-tunnel.service b/services/video-upscale/video-upscale-tunnel.service new file mode 100644 index 000000000..142d4c2d8 --- /dev/null +++ b/services/video-upscale/video-upscale-tunnel.service @@ -0,0 +1,17 @@ +[Unit] +Description=Private upscale API tunnel to main application +After=network-online.target video-upscale.service +Wants=network-online.target + +[Service] +Type=simple +ExecStart=/usr/bin/ssh -NT -o BatchMode=yes -o ExitOnForwardFailure=yes -o ServerAliveInterval=20 -o ServerAliveCountMax=3 -o StrictHostKeyChecking=yes -o UserKnownHostsFile=/etc/video-render-tunnel/known_hosts -i /etc/video-render-tunnel/id_ed25519 -p 64192 -R 127.0.0.1:18341:127.0.0.1:18341 video-render-tunnel@45.192.99.48 +Restart=always +RestartSec=5 +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true + +[Install] +WantedBy=multi-user.target diff --git a/services/video-upscale/video-upscale.service b/services/video-upscale/video-upscale.service new file mode 100644 index 000000000..ef4b6ade5 --- /dev/null +++ b/services/video-upscale/video-upscale.service @@ -0,0 +1,33 @@ +[Unit] +Description=Isolated Real-ESRGAN video upscale API +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=video-upscale +Group=video-upscale +WorkingDirectory=/opt/video-upscale/app +EnvironmentFile=/etc/video-upscale.env +Environment=CUDA_VISIBLE_DEVICES=3 +Environment=OMP_NUM_THREADS=4 +Environment=OPENBLAS_NUM_THREADS=4 +Environment=MKL_NUM_THREADS=4 +Environment=UPSCALE_PYTHON=/opt/video-upscale/venv/bin/python +ExecStart=/usr/bin/node /opt/video-upscale/app/server.cjs +Restart=on-failure +RestartSec=5 +KillMode=mixed +TimeoutStopSec=15 +UMask=0077 +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/var/lib/video-upscale +MemoryMax=8G +CPUQuota=400% +TasksMax=256 + +[Install] +WantedBy=multi-user.target diff --git a/tests/routes/video-canvas-groups-upscale.test.ts b/tests/routes/video-canvas-groups-upscale.test.ts new file mode 100644 index 000000000..88b169e14 --- /dev/null +++ b/tests/routes/video-canvas-groups-upscale.test.ts @@ -0,0 +1,310 @@ +import { test, expect } from "@playwright/test"; +import { randomUUID } from "node:crypto"; +import { + makeCanvasGroup, + groupMembers, + moveCanvasSelection, + ungroupCanvasNodes, + layoutCanvasGroup, + exportCanvasGroup, + instantiateCanvasGroup, + migrateLegacyGroups, + assignGroupMembers, + type GroupNode, +} from "@/lib/video-canvas/groups"; +import { + compileGroupRun, + validateGroupBatch, + groupDependencyState, + resolveGroupInputs, +} from "@/lib/video-canvas/group-execution"; +import { + upscaleDimensions, + videoUpscaleSchema, +} from "@/lib/video-canvas/video-upscale-contract"; +import { + reconcileCanvasTasks, + type CanvasGenerationTask, + type CanvasTaskRequest, +} from "@/lib/video-canvas/task-contract"; +import { hasRecoverableCanvasTask } from "@/lib/video-canvas/task-recovery"; +const n = (id: string, x = 100, y = 100): GroupNode => ({ + id, + title: id, + kind: "media", + inputType: "image", + x, + y, + width: 200, + height: 100, + draftValues: { + action: "IMAGE_RESOURCE", + mediaUrl: `https://media.example/${id}.png`, + }, +}); +const edge = (source: string, target: string) => ({ + id: `${source}-${target}`, + source, + target, +}); +const task = ( + request: CanvasTaskRequest, + status = "pending", + result: any = null +): CanvasGenerationTask => ({ + id: randomUUID(), + requestId: request.requestId, + request, + status, + result, + createdAt: new Date().toISOString(), +}); +function batch() { + const source = n("source"), + image = { + ...n("image", 400), + draftValues: { + action: "IMAGE_GENERATE", + params: { prompt: "产品" }, + mediaUrl: "https://media.example/OLD.png", + }, + }, + video = { + ...n("video", 700), + inputType: "video", + draftValues: { action: "VIDEO_GENERATE" }, + }, + hd = { + ...n("hd", 1000), + inputType: "video", + draftValues: { action: "VIDEO_HD_UPSCALE" }, + }; + const nodes = makeCanvasGroup( + [source, image, video, hd], + ["source", "image", "video", "hd"], + "group" + ); + return compileGroupRun( + nodes, + [edge("source", "image"), edge("image", "video"), edge("video", "hd")], + "group", + randomUUID(), + (node) => + node.id === "image" + ? { kind: "image", params: { prompt: "产品" } } + : node.id === "video" + ? { kind: "video", params: { prompt: "运镜", mode: "first-frame" } } + : { kind: "video-upscale", params: { resolution: "4K" } }, + randomUUID + ); +} +test("nested groups move descendants exactly once and ungroup preserves edges/member coordinates", () => { + let nodes = makeCanvasGroup( + [n("a"), n("b", 400), n("c", 800)], + ["a", "b"], + "inner" + ); + nodes = makeCanvasGroup(nodes, ["inner", "c"], "outer"); + const before = nodes; + nodes = moveCanvasSelection(nodes, ["outer", "inner", "a"], 50, 75); + for (const node of nodes) { + const old = before.find((n) => n.id === node.id)!; + expect([node.x, node.y]).toEqual([old.x + 50, old.y + 75]); + } + const ungrouped = ungroupCanvasNodes(nodes, "inner"); + expect( + groupMembers(ungrouped, "outer") + .map((n) => n.id) + .sort() + ).toEqual(["a", "b", "c"]); + expect(ungrouped.find((n) => n.id === "a")?.x).toBe( + nodes.find((n) => n.id === "a")?.x + ); + expect(() => assignGroupMembers(before, ["outer"], "inner")).toThrow("子组"); + expect( + groupMembers(assignGroupMembers(before, ["a"], ""), "inner") + ).toHaveLength(1); +}); +test("layout fits frame, legacy preset migrates once, export/import remaps internal IDs only", () => { + const nodes = makeCanvasGroup([n("a"), n("b", 400)], ["a", "b"], "g"); + const laid = layoutCanvasGroup(nodes, "g", "vertical"); + expect(laid.find((n) => n.id === "a")?.x).toBe( + laid.find((n) => n.id === "b")?.x + ); + const template = exportCanvasGroup( + [...laid, n("external", 900)], + [edge("a", "b"), edge("external", "a")], + "g" + ); + const imported = instantiateCanvasGroup(template, randomUUID, 900, 300); + expect(imported.edges).toHaveLength(2); + expect( + imported.nodes.every((n) => !["a", "b", "g"].includes(n.id)) + ).toBeTruthy(); + expect(groupMembers(imported.nodes, imported.groupId)).toHaveLength(3); + expect( + imported.nodes.some( + (n) => n.draftValues?.templateInput && !n.draftValues?.mediaUrl + ) + ).toBeTruthy(); + const legacy = nodes.map((n) => ({ + ...n, + draftValues: { + ...n.draftValues, + parentGroupId: undefined, + groupVersion: undefined, + }, + })); + expect(groupMembers(migrateLegacyGroups(legacy), "g")).toHaveLength(2); +}); +test("invalid imported member cycles and missing endpoints are rejected", () => { + const nodes = makeCanvasGroup([n("a"), n("b", 400)], ["a", "b"], "g"), + template = exportCanvasGroup(nodes, [], "g"); + expect(() => + instantiateCanvasGroup( + { ...template, edges: [edge("missing", "a")] }, + randomUUID, + 0, + 0 + ) + ).toThrow("不存在"); + const cyclic = template.nodes.map((n) => + n.id === "g" + ? { ...n, draftValues: { ...n.draftValues, parentGroupId: "g" } } + : n + ); + expect(() => + instantiateCanvasGroup({ ...template, nodes: cyclic }, randomUUID, 0, 0) + ).toThrow("循环"); +}); +test("whole-group run resolves this run's fresh output through image > video > HD", () => { + const requests = batch(); + validateGroupBatch(requests); + expect(requests).toHaveLength(3); + const tasks = requests.map((r) => task(r)); + expect(groupDependencyState(requests[0], tasks)).toBe("ready"); + expect(groupDependencyState(requests[1], tasks)).toBe("waiting"); + tasks[0].status = "succeeded"; + tasks[0].result = { + images: [{ originalUrl: "https://media.example/FRESH.png" }], + }; + expect(resolveGroupInputs(requests[1], tasks).images).toEqual([ + "https://media.example/FRESH.png", + ]); + tasks[1].status = "succeeded"; + tasks[1].result = { url: "https://media.example/NEW.mp4" }; + expect(resolveGroupInputs(requests[2], tasks).url).toBe( + "https://media.example/NEW.mp4" + ); +}); +test("failed/cancelled/uncertain upstream blocks downstream without using old output", () => { + const requests = batch(); + for (const status of ["failed", "cancelled", "interrupted"]) { + const tasks = requests.map((r) => task(r)); + tasks[0].status = status; + expect(groupDependencyState(requests[1], tasks)).toBe("failed"); + expect(() => resolveGroupInputs(requests[1], tasks)).toThrow("尚未成功"); + } +}); +test("batch rejects foreign dependency, wrong edge, cycle and mixed projects", () => { + let r = batch(); + r[1].workflow!.inputs[0].requestId = randomUUID(); + expect(() => validateGroupBatch(r)).toThrow("本次编组"); + r = batch(); + r[1].edges = []; + expect(() => validateGroupBatch(r)).toThrow("连线"); + r = batch(); + r[1].projectId = randomUUID(); + expect(() => validateGroupBatch(r)).toThrow("同一画布"); + r = batch().slice(0, 2); + r[0].workflow!.inputs = [{ node: r[1].node, requestId: r[1].requestId }]; + r[0].edges = [edge(r[1].node.id, r[0].node.id)]; + r[0].node.inputType = "text"; + r[1].node.inputType = "text"; + r[0].workflow!.inputs[0].node.inputType = "text"; + r[1].workflow!.inputs[0].node.inputType = "text"; + expect(() => validateGroupBatch(r)).toThrow("循环"); +}); +test("video HD dimensions preserve aspect and enforce meaningful 1–4x AI scale", () => { + expect(upscaleDimensions(960, 540, "4K")).toEqual({ + width: 3840, + height: 2160, + }); + expect(upscaleDimensions(540, 960, "4K")).toEqual({ + width: 2160, + height: 3840, + }); + expect(upscaleDimensions(640, 480, "2K")).toEqual({ + width: 1920, + height: 1440, + }); + expect(() => upscaleDimensions(1920, 1080, "1080P")).toThrow("已达到"); + expect(() => upscaleDimensions(320, 180, "4K")).toThrow("4 倍"); + expect( + videoUpscaleSchema.safeParse({ + url: "file:///etc/passwd", + resolution: "4K", + }).success + ).toBeFalsy(); +}); +test("HD task survives refresh with remote ID and becomes playable video", () => { + const request = batch()[2], + t = task(request, "interrupted", { + upscale: { + remote: { id: "a".repeat(64), endpointHash: "b".repeat(64) }, + stage: "第 5 帧", + }, + }); + expect(hasRecoverableCanvasTask(t)).toBeTruthy(); + t.status = "succeeded"; + t.result = { + url: "https://media.example/hd.mp4", + metadata: { width: 3840, height: 2160 }, + }; + const result = reconcileCanvasTasks([request.node], [t], "2020-01-01")[0]; + expect(result.inputType).toBe("video"); + expect(result.draftValues?.mediaUrl).toBe(t.result.url); + expect(result.draftValues?.generationStatus).toBe("complete"); +}); + +test("generated video feeds fresh smart-edit plan and automatic render as durable stages", () => { + const video = { + ...n("v"), + inputType: "video", + draftValues: { action: "VIDEO_GENERATE" }, + }, + edit = { + ...n("edit", 400), + inputType: "video", + draftValues: { action: "SMART_EDIT_WORKFLOW", nodeType: "smart-edit" }, + }; + const nodes = makeCanvasGroup([video, edit], ["v", "edit"], "g"); + const requests = compileGroupRun( + nodes, + [edge("v", "edit")], + "g", + randomUUID(), + (node) => + node.id === "v" + ? { kind: "video", params: { mode: "text-to-video", prompt: "走动" } } + : { + kind: "smart-edit", + params: { operation: "plan", autoRender: true, mode: "sequence" }, + }, + randomUUID + ); + expect(requests).toHaveLength(3); + validateGroupBatch(requests); + const tasks = requests.map((r) => task(r)); + tasks[0].status = "succeeded"; + tasks[0].result = { url: "https://media.example/fresh.mp4" }; + expect(resolveGroupInputs(requests[1], tasks).assets).toEqual([ + { id: "group-asset-1", name: "v", url: "https://media.example/fresh.mp4" }, + ]); + tasks[1].status = "succeeded"; + tasks[1].result = { plans: [{ title: "fresh plan" }] }; + expect(resolveGroupInputs(requests[2], tasks).plan).toEqual({ + title: "fresh plan", + }); + expect(requests[1].hiddenNodeIds).toContain(requests[1].node.id); +});