lunanting/ai-chatbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(video-canvas): persist generation tasks and repair workflow controls

Browse Source
This commit is contained in:
134 2026-09-17 12:32:47 +08:00
parent 23f86dfce1
commit 0d17638a8d
17 changed files with 2572 additions and 1843 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

164
app/api/canvas/generate/image/route.ts
View File

@ -1,63 +1,10 @@
import { executeCanvasImageResponse } from "@/lib/canvas/generate-image-response";
import { NextResponse } from "next/server";
import { auth } from "@/app/(auth)/auth";
import {
createCanvasImageTask,
enqueueCanvasImageTask,
getCanvasImageTaskForUser,
} from "@/lib/canvas/image-task-queue";
import {
generateCanvasImage,
type CanvasGenerateImageRequest,
} from "@/lib/canvas/generate-image";
import { getPointsSystemConfig } from "@/lib/db/queries";
import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge";
import {
getImageToolCreditCost,
type ImageToolCreditFeature,
} from "@/lib/points/config";
import { getCanvasImageTaskForUser } from "@/lib/canvas/image-task-queue";
export const maxDuration = 1200;
interface GenerateImageRequest extends CanvasGenerateImageRequest {
async?: boolean;
}
function normalizeBillingFeature(
value: unknown
): Extract<ImageToolCreditFeature, "multi-view" | "repaint"> | null {
if (value === "multi-view" || value === "repaint") return value;
return null;
}
function toGenerateImagePayload(body: GenerateImageRequest): CanvasGenerateImageRequest {
return {
allowFailover: typeof body.allowFailover === "boolean" ? body.allowFailover : undefined,
prompt: String(body.prompt || "").trim(),
negativePrompt:
typeof body.negativePrompt === "string" ? body.negativePrompt : undefined,
model: typeof body.model === "string" ? body.model : undefined,
aspectRatio:
typeof body.aspectRatio === "string" && body.aspectRatio.trim()
? body.aspectRatio.trim()
: "auto",
imageSize:
typeof body.imageSize === "string" && body.imageSize.trim()
? body.imageSize.trim()
: undefined,
imageCount:
typeof body.imageCount === "number" && Number.isFinite(body.imageCount)
? body.imageCount
: 1,
promptExtend:
typeof body.promptExtend === "boolean" ? body.promptExtend : undefined,
referenceImage:
typeof body.referenceImage === "string" ? body.referenceImage : undefined,
referenceImages: Array.isArray(body.referenceImages)
? body.referenceImages.map((item) => String(item || "").trim()).filter(Boolean)
: [],
};
}
export async function GET(request: Request) {
try {
const session = await auth();
@ -68,10 +15,7 @@ export async function GET(request: Request) {
const { searchParams } = new URL(request.url);
const taskId = String(searchParams.get("taskId") || "").trim();
if (!taskId) {
return NextResponse.json(
{ success: false, error: "缺少 taskId 参数" },
{ status: 400 }
);
return NextResponse.json({ success: false, error: "缺少 taskId 参数" }, { status: 400 });
}
const task = await getCanvasImageTaskForUser({ taskId, userId: session.user.id });
@ -100,105 +44,15 @@ export async function GET(request: Request) {
success: false,
error: error instanceof Error ? error.message : "查询任务失败",
},
{ status: 500 }
{ status: 500 },
);
}
}
export async function POST(request: Request) {
try {
const session = await auth();
if (!session?.user?.id) {
return NextResponse.json(
{ success: false, error: "未登录" },
{ status: 401 }
);
}
const body: GenerateImageRequest = await request.json();
const payload: CanvasGenerateImageRequest = {
...toGenerateImagePayload(body),
ownerKey: session.user.id,
};
if (!payload.prompt) {
return NextResponse.json(
{ success: false, error: "请输入提示词" },
{ status: 400 }
);
}
const billingFeature = normalizeBillingFeature(body.billingFeature);
const pointsCharge = billingFeature
? await (async () => {
const pointsConfig = await getPointsSystemConfig();
return prepareFeaturePointsCharge({
userId: session.user.id,
unitCost: getImageToolCreditCost(pointsConfig, billingFeature),
reason:
billingFeature === "multi-view"
? "image_multi_view"
: "image_repaint",
modelId: "canvas-generate-image",
note:
billingFeature === "multi-view"
? "图片多视角生成消耗积分"
: "图片重绘消耗积分",
pointsConfig,
});
})()
: null;
if (pointsCharge && !pointsCharge.ok) {
return NextResponse.json(pointsCharge.payload, { status: pointsCharge.status });
}
const asyncMode = body.async === true;
if (asyncMode) {
const task = await createCanvasImageTask({
userId: session.user.id,
request: payload as unknown as Record<string, unknown>,
});
await enqueueCanvasImageTask(task.id);
const currentTask = await getCanvasImageTaskForUser({
taskId: task.id,
userId: session.user.id,
});
if (pointsCharge?.chargeable) {
try {
await pointsCharge.consume({
note:
billingFeature === "multi-view"
? `图片多视角生成消耗积分（${task.id}）`
: `图片重绘消耗积分（${task.id}）`,
});
} catch (pointsError) {
console.error("[Points] Failed to consume canvas image task points:", pointsError);
}
}
return NextResponse.json({
success: true,
taskId: task.id,
status: currentTask?.status || task.status,
queueStatus: currentTask?.queueStatus,
});
}
const result = await generateCanvasImage(payload);
if (pointsCharge?.chargeable) {
try {
await pointsCharge.consume();
} catch (pointsError) {
console.error("[Points] Failed to consume canvas image points:", pointsError);
}
}
return NextResponse.json({
success: true,
data: result,
});
} catch (error) {
console.error("[Canvas Generate Image] Error:", error);
return NextResponse.json(
{ success: false, error: error instanceof Error ? error.message : "生成失败" },
{ status: 500 }
);
}
const session = await auth();
if (!session?.user?.id)
return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
const body = await request.json().catch(() => ({}));
return executeCanvasImageResponse(body, session.user.id);
}

102
app/api/canvas/generation-tasks/route.ts Normal file
View File

@ -0,0 +1,102 @@
import { after, NextResponse } from "next/server";
import { auth } from "@/app/(auth)/auth";
import { z } from "zod";
import { canvasTaskRequestSchema } from "@/lib/video-canvas/task-contract";
import {
cancelCanvasGenerationTask,
createCanvasGenerationTask,
listCanvasGenerationTasks,
ownsTaskProject,
} from "@/lib/video-canvas/task-store";
import { runCanvasGenerationQueue } from "@/lib/video-canvas/task-worker";
export const runtime = "nodejs";
export const maxDuration = 900;
async function owner(projectId: unknown) {
const session = await auth();
if (!session?.user?.id) throw Object.assign(Error("未登录"), { status: 401 });
if (!z.string().uuid().safeParse(projectId).success)
throw Object.assign(Error("画布 ID 无效"), { status: 400 });
if (!(await ownsTaskProject(session.user.id, projectId as string)))
throw Object.assign(Error("画布不存在"), { status: 404 });
return session.user.id;
}
const fail = (error: unknown) =>
NextResponse.json(
{ success: false, error: error instanceof Error ? error.message : "任务操作失败" },
{ status: Number((error as any)?.status) || (error instanceof SyntaxError ? 400 : 500) },
);
const kick = (userId: string, projectId: string) =>
after(() =>
runCanvasGenerationQueue(userId, projectId).catch((error) =>
console.error("[Canvas tasks]", error),
),
);
export async function POST(request: Request) {
try {
const text = await request.text();
if (new TextEncoder().encode(text).byteLength > 262144)
throw Object.assign(Error("任务数据过大，请使用已上传的素材链接"), { status: 413 });
const parsed = canvasTaskRequestSchema.safeParse(JSON.parse(text));
if (!parsed.success) throw Object.assign(Error("任务参数不合法"), { status: 400 });
const userId = await owner(parsed.data.projectId);
const task = await createCanvasGenerationTask(userId, parsed.data);
kick(userId, parsed.data.projectId);
return NextResponse.json({ success: true, task }, { status: 202 });
} catch (error) {
return fail(error);
}
}
export async function GET(request: Request) {
try {
const projectId = new URL(request.url).searchParams.get("projectId") || "";
const userId = await owner(projectId);
const tasks = await listCanvasGenerationTasks(userId, projectId);
if (tasks.some((task) => task.status === "pending")) kick(userId, projectId);
return NextResponse.json(
{ success: true, tasks },
{ headers: { "Cache-Control": "no-store" } },
);
} catch (error) {
return fail(error);
}
}
export async function DELETE(request: Request) {
try {
const { projectId, taskId } = await request.json();
const userId = await owner(projectId);
if (!z.string().uuid().safeParse(taskId).success)
throw Object.assign(Error("任务 ID 无效"), { status: 400 });
const result = await cancelCanvasGenerationTask(userId, projectId, taskId);
if (!result.task)
return NextResponse.json({ success: false, error: "任务不存在" }, { status: 404 });
if (!result.ok)
return NextResponse.json(
{
success: false,
error:
result.task.status === "running"
? "任务已提交供应商，当前线路不支持中止。结果会继续保存，请勿重复生成。"
: "任务已结束或状态待核实，请先刷新任务状态。",
task: result.task,
},
{ status: 409 },
);
return NextResponse.json({ success: true, task: result.task });
} catch (error) {
return fail(error);
}
}
export async function PATCH(request: Request) {
try {
const { projectId, taskId, nodeId } = await request.json();
const userId = await owner(projectId);
if (!z.string().uuid().safeParse(taskId).success || typeof nodeId !== "string")
return NextResponse.json({ success: false }, { status: 400 });
const { hideCanvasGenerationNode } = await import("@/lib/video-canvas/task-store");
const success = await hideCanvasGenerationNode(userId, projectId, taskId, nodeId);
return NextResponse.json({ success }, { status: success ? 200 : 404 });
} catch (error) {
return fail(error);
}
}

786
app/api/canvas/workflow/analyze/route.ts
View File

@ -1,786 +1,12 @@
import { normalizeStoryScriptRows } from "@/lib/video-canvas/story-script";
import { isCanvasLanguageModel, isCanvasVisionModel } from "@/lib/video-canvas/model-matching";
import { generateText } from "ai";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import { mkdtemp, readFile, rm } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { executeWorkflowAnalysisResponse } from "@/lib/video-canvas/workflow-analysis-response";
import { NextResponse } from "next/server";
import sharp from "sharp";
import { z } from "zod";
import { auth } from "@/app/(auth)/auth";
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
import { inlineVisionImages } from "@/lib/ai/vision-inline";
import { getAiModelAgentCodingModelId, isAiModelMultimodal } from "@/lib/db/ai-model-flags";
import {
getAgentAiModelWithProvider,
getEnabledAiModelsWithProvider,
} from "@/lib/db/queries";
export const runtime = "nodejs";
export const maxDuration = 180;
const execFileAsync = promisify(execFile);
const VIDEO_ANALYSIS_FRAME_COUNT = 4;
const VIDEO_ANALYSIS_MAX_SIDE = 960;
const VIDEO_ANALYSIS_TIMEOUT_MS = 15000;
const requestSchema = z.object({
mode: z.enum(["video-analysis", "character-scene-extraction", "story-script", "agent-plan"]),
prompt: z.string().trim().optional(),
imageUrls: z.array(z.string().trim()).optional(),
videoUrls: z.array(z.string().trim()).optional(),
nodeTitle: z.string().trim().optional(),
preferredModelRef: z.string().trim().optional(),
model: z.string().trim().optional(),
});
function sanitizeUrls(value: unknown, limit = 6) {
if (!Array.isArray(value)) return [];
return value
.map((item) => String(item || "").trim())
.filter((url) => /^https?:\/\//i.test(url))
.slice(0, limit);
}
function supportsVision(model: {
supportsVision?: boolean | null;
capabilities?: unknown;
defaultParams?: unknown;
}) {
return (
Boolean(model.supportsVision) ||
isAiModelMultimodal({
capabilities: model.capabilities,
defaultParams: model.defaultParams,
})
);
}
function extractJsonBlock<T = Record<string, unknown>>(text: string): T | null {
const matched = String(text || "").match(/\{[\s\S]*\}/);
if (!matched) return null;
try {
return JSON.parse(matched[0]) as T;
} catch {
return null;
}
}
function normalizeText(value: unknown) {
return String(value || "").trim();
}
function shouldUseLocalWorkflowE2E(request: Request) {
const hostname = new URL(request.url).hostname;
return (
process.env.CANVAS_E2E_LOCAL === "true" &&
(hostname === "127.0.0.1" || hostname === "localhost" || hostname === "::1")
);
}
function normalizeStringList(value: unknown, limit = 8) {
if (!Array.isArray(value)) return [];
return Array.from(
new Set(
value
.map((item) => String(item || "").trim())
.filter(Boolean)
.slice(0, limit)
)
);
}
function normalizeShotIdeas(value: unknown) {
if (!Array.isArray(value)) return [];
return value
.map((item, index) => {
const raw =
item && typeof item === "object" && !Array.isArray(item)
? (item as Record<string, unknown>)
: {};
const title = normalizeText(raw.title || `镜头 ${index + 1}`);
const description = normalizeText(raw.description || raw.prompt || raw.summary);
if (!title && !description) return null;
return {
title: title || `镜头 ${index + 1}`,
description,
};
})
.filter(
(item): item is { title: string; description: string } => Boolean(item)
)
.slice(0, 6);
}
function normalizeVideoAnalysisRows(
value: unknown,
framePreviews: string[],
) {
if (!Array.isArray(value)) return [];
return value
.map((item, index) => {
const raw =
item && typeof item === "object" && !Array.isArray(item)
? (item as Record<string, unknown>)
: {};
const sceneDescription = normalizeText(
raw.sceneDescription || raw.scene || raw.title || raw.description,
);
const narrative = normalizeText(
raw.narrative || raw.story || raw.storyContent || raw.narration || raw.summary,
);
const shot = normalizeText(raw.shot || raw.shotNumber || raw.index || index + 1);
const startTime = normalizeText(raw.startTime || raw.start || raw.start_time);
const endTime = normalizeText(raw.endTime || raw.end || raw.end_time);
const duration = normalizeText(raw.duration || raw.durationSeconds || raw.duration_seconds);
const shotSize = normalizeText(raw.shotSize || raw.frameSize || raw.shot_type || raw.sceneScale);
const cameraAngle = normalizeText(raw.cameraAngle || raw.angle || raw.camera_angle);
const cameraMovement = normalizeText(
raw.cameraMovement || raw.cameraMove || raw.camera_motion || raw.motionType,
);
const focusDepth = normalizeText(
raw.focusDepth || raw.depthOfField || raw.focalDepth || raw.focus_depth,
);
const lighting = normalizeText(raw.lighting || raw.light || raw.lightDesign);
const backgroundMusic = normalizeText(
raw.backgroundMusic || raw.music || raw.bgm,
);
const voiceAndSfx = normalizeText(
raw.voiceAndSfx || raw.sound || raw.audio || raw.sfx,
);
const imagePrompt = normalizeText(
raw.imagePrompt || raw.image_generation_prompt || raw.image,
);
const motionPrompt = normalizeText(
raw.motionPrompt || raw.motion || raw.videoPrompt || raw.video_motion_prompt,
);
const keyframeIndex = Number(
raw.keyframeIndex || raw.frameIndex || raw.key_frame_index || index + 1,
);
if (
!sceneDescription &&
!narrative &&
!backgroundMusic &&
!voiceAndSfx &&
!imagePrompt &&
!motionPrompt &&
!shotSize &&
!cameraAngle &&
!cameraMovement &&
!focusDepth &&
!lighting
) {
return null;
}
const normalizedFrameIndex =
Number.isFinite(keyframeIndex) && keyframeIndex > 0
? Math.max(0, Math.min(framePreviews.length - 1, Math.round(keyframeIndex) - 1))
: index % Math.max(framePreviews.length, 1);
return {
shot: shot || String(index + 1),
startTime: startTime || (index === 0 ? "0" : ""),
endTime,
duration,
sceneDescription: sceneDescription || `镜头 ${index + 1}`,
narrative: narrative || sceneDescription || `镜头 ${index + 1}`,
shotSize: shotSize || "中景至近景",
cameraAngle: cameraAngle || "平视",
cameraMovement: cameraMovement || "缓慢推镜头 (Slow Zoom In)",
focusDepth: focusDepth || "浅景深",
lighting: lighting || "冷调低沉照明，主体光线柔和。",
backgroundMusic: backgroundMusic || "无",
voiceAndSfx: voiceAndSfx || "无",
imagePrompt: imagePrompt || sceneDescription || "延续当前视频气质生成关键帧",
motionPrompt: motionPrompt || "保持镜头运动自然、连贯、符合前后节奏。",
keyframeUrl: framePreviews[normalizedFrameIndex] || "",
};
})
.filter(
(
item,
): item is {
shot: string;
startTime: string;
endTime: string;
duration: string;
sceneDescription: string;
narrative: string;
shotSize: string;
cameraAngle: string;
cameraMovement: string;
focusDepth: string;
lighting: string;
backgroundMusic: string;
voiceAndSfx: string;
imagePrompt: string;
motionPrompt: string;
keyframeUrl: string;
} => Boolean(item),
)
.slice(0, 8);
}
function normalizeCharacters(value: unknown) {
if (!Array.isArray(value)) return [];
return value
.map((item) => {
const raw =
item && typeof item === "object" && !Array.isArray(item)
? (item as Record<string, unknown>)
: {};
const name = normalizeText(raw.name || raw.title);
const description = normalizeText(raw.description || raw.summary);
const prompt = normalizeText(raw.prompt);
if (!name && !description && !prompt) return null;
return {
name: name || "未命名角色",
description,
prompt,
look: normalizeText(raw.look) || undefined,
wardrobe: normalizeText(raw.wardrobe) || undefined,
signatureAction: normalizeText(raw.signatureAction) || undefined,
traits: normalizeStringList(raw.traits, 6),
};
})
.filter((item): item is NonNullable<typeof item> => Boolean(item))
.slice(0, 8);
}
function normalizeScenes(value: unknown) {
if (!Array.isArray(value)) return [];
return value
.map((item) => {
const raw =
item && typeof item === "object" && !Array.isArray(item)
? (item as Record<string, unknown>)
: {};
const name = normalizeText(raw.name || raw.title);
const description = normalizeText(raw.description || raw.summary);
const prompt = normalizeText(raw.prompt);
if (!name && !description && !prompt) return null;
return {
name: name || "未命名场景",
description,
prompt,
mood: normalizeText(raw.mood) || undefined,
lighting: normalizeText(raw.lighting) || undefined,
props: normalizeStringList(raw.props, 6),
};
})
.filter((item): item is NonNullable<typeof item> => Boolean(item))
.slice(0, 8);
}
function buildVideoAnalysisPrompt(params: {
nodeTitle?: string;
prompt: string;
frameCount: number;
imageCount: number;
videoCount: number;
}) {
return [
"你是视频画布工作流里的视频分析节点，需要为后续分镜和生成节点提供结构化输出。",
params.nodeTitle ? `当前节点：${params.nodeTitle}` : "",
`上游参考图数量：${params.imageCount}`,
`上游视频数量：${params.videoCount}`,
`已提供按时间顺序抽取的视频关键帧数量：${params.frameCount}`,
params.prompt ? `上游脚本/说明：\n${params.prompt}` : "上游没有额外脚本，请重点依据参考画面分析。",
"",
"请只输出 JSON，不要输出 markdown，不要输出解释。",
"JSON 结构如下：",
`{
"summary": "一句话总结当前视频/镜头风格，不超过120字",
"highlights": ["3-5条镜头分析要点"],
"analysisRows": [
{
"shot": "1",
"startTime": "0",
"endTime": "5",
"duration": "5",
"sceneDescription": "画面描述，说明主体、环境、动作、构图和色调",
"narrative": "叙事内容，说明这一镜头承担的剧情/情绪/信息作用",
"shotSize": "景别，例如中景至近景",
"cameraAngle": "摄影机角度，例如平视",
"cameraMovement": "摄影机运动，例如缓慢推镜头 (Slow Zoom In)",
"focusDepth": "焦距与景深，例如浅景深",
"lighting": "光线，例如冷调低沉照明，屏幕光作为侧逆光",
"backgroundMusic": "这一段适合的背景音乐氛围",
"voiceAndSfx": "人声/环境声/音效建议，没有就写无",
"imagePrompt": "后续生成关键帧时可直接使用的英文图像提示词",
"motionPrompt": "后续生成视频时可直接使用的英文运动提示词",
"keyframeIndex": 1
}
],
"shotIdeas": [
{ "title": "镜头 1", "description": "一句话描述镜头目标、景别或动作" }
],
"visualMotifs": ["2-4个可复用的视觉母题"],
"recommendedNextSteps": ["2-4条建议的下游动作"]
}`,
"analysisRows 请按镜头时间顺序输出；短视频可以输出 1 行，复杂视频输出 2-6 行。",
"每行必须覆盖镜号、开始时间、结束时间、时长、画面描述、叙事内容、景别、摄影机角度、摄影机运动、焦距与景深、光线、背景音乐、人声/音效、图像生成提示词、视频运动提示词和关键帧。",
"imagePrompt 和 motionPrompt 优先使用英文；其他字段使用中文；keyframeIndex 使用 1 开始的序号，对应你看到的关键帧顺序。",
"如果没有真实视频帧，只能依据参考图和脚本推断，也要正常输出，但措辞要稳健。",
]
.filter(Boolean)
.join("\n");
}
async function getVideoDurationSeconds(videoUrl: string) {
try {
const { stdout } = await execFileAsync(
"ffprobe",
[
"-v",
"error",
"-show_entries",
"format=duration",
"-of",
"default=noprint_wrappers=1:nokey=1",
videoUrl,
],
{
timeout: VIDEO_ANALYSIS_TIMEOUT_MS,
maxBuffer: 1024 * 1024,
},
);
const duration = Number.parseFloat(String(stdout || "").trim());
return Number.isFinite(duration) && duration > 0 ? duration : null;
} catch {
return null;
}
}
function buildVideoFrameTimestamps(durationSeconds: number | null, frameCount: number) {
const safeCount = Math.max(1, frameCount);
if (!durationSeconds || !Number.isFinite(durationSeconds) || durationSeconds <= 0.5) {
return Array.from({ length: safeCount }).map((_, index) =>
Number((0.3 + index * 0.45).toFixed(2)),
);
}
const start = Math.min(0.35, durationSeconds * 0.12);
const end = Math.max(start + 0.6, durationSeconds * 0.88);
if (safeCount === 1) return [Number(((start + end) / 2).toFixed(2))];
const step = (end - start) / (safeCount - 1);
return Array.from({ length: safeCount }).map((_, index) =>
Number((start + step * index).toFixed(2)),
);
}
async function downscaleVideoFrameForVision(inputBuffer: Buffer) {
return sharp(inputBuffer, { failOn: "none" })
.rotate()
.resize({
width: VIDEO_ANALYSIS_MAX_SIDE,
height: VIDEO_ANALYSIS_MAX_SIDE,
fit: "inside",
withoutEnlargement: true,
})
.jpeg({ quality: 82, mozjpeg: true })
.toBuffer();
}
async function extractVideoFrameDataUrls(videoUrl: string, frameCount = VIDEO_ANALYSIS_FRAME_COUNT) {
const tempDir = await mkdtemp(path.join(os.tmpdir(), "canvas-video-analysis-"));
try {
const durationSeconds = await getVideoDurationSeconds(videoUrl);
const timestamps = buildVideoFrameTimestamps(durationSeconds, frameCount);
const frames = await Promise.all(
timestamps.map(async (timestamp, index) => {
const outputPath = path.join(tempDir, `frame-${index + 1}.jpg`);
await execFileAsync(
"ffmpeg",
[
"-hide_banner",
"-loglevel",
"error",
"-ss",
String(timestamp),
"-i",
videoUrl,
"-frames:v",
"1",
"-q:v",
"4",
"-y",
outputPath,
],
{
timeout: VIDEO_ANALYSIS_TIMEOUT_MS,
maxBuffer: 2 * 1024 * 1024,
},
);
const buffer = await readFile(outputPath);
const processed = await downscaleVideoFrameForVision(buffer);
return `data:image/jpeg;base64,${processed.toString("base64")}`;
}),
);
return frames.filter(Boolean);
} catch (error) {
console.warn("[Canvas Workflow Analyze] extract video frames failed:", {
message: error instanceof Error ? error.message : String(error),
videoUrl,
});
return [];
} finally {
await rm(tempDir, { force: true, recursive: true }).catch(() => undefined);
}
}
function buildCharacterScenePrompt(params: {
nodeTitle?: string;
prompt: string;
imageCount: number;
}) {
return [
"你是视频画布工作流里的角色场景提取节点，需要从脚本、设定和参考图里提取角色与场景资产。",
params.nodeTitle ? `当前节点：${params.nodeTitle}` : "",
`参考图数量：${params.imageCount}`,
params.prompt ? `输入内容：\n${params.prompt}` : "没有额外脚本，请根据参考图提取角色和场景线索。",
"",
"请只输出 JSON，不要输出 markdown，不要输出解释。",
"JSON 结构如下：",
`{
"summary": "一句话总结当前故事世界与资产线索",
"characters": [
{
"name": "角色名",
"description": "角色外观与身份简介",
"look": "外貌特征",
"wardrobe": "服装/道具",
"traits": ["性格关键词"],
"signatureAction": "代表动作",
"prompt": "可直接用于生成角色图/角色视频的提示词"
}
],
"scenes": [
{
"name": "场景名",
"description": "空间、布景和氛围简介",
"mood": "情绪与气氛",
"lighting": "布光特征",
"props": ["核心道具"],
"prompt": "可直接用于生成场景图/场景视频的提示词"
}
],
"relationships": ["角色与场景或角色之间的关系"],
"recommendedNextSteps": ["建议连接的后续节点动作"]
}`,
"角色和场景都要尽量具体，避免空泛形容词。",
]
.filter(Boolean)
.join("\n");
}
function buildAgentPlanPrompt(params: {
nodeTitle?: string;
prompt: string;
imageCount: number;
videoCount: number;
}) {
return [
"你是视频画布里的自定义代理节点，需要把当前创作目标拆成可执行的工作流计划。",
params.nodeTitle ? `当前节点：${params.nodeTitle}` : "",
`参考图数量：${params.imageCount}`,
`视频数量：${params.videoCount}`,
params.prompt ? `上游需求：\n${params.prompt}` : "没有额外脚本，请基于当前上下文给出稳妥计划。",
"",
"请只输出 JSON，不要输出 markdown，不要输出解释。",
"JSON 结构如下：",
`{
"summary": "一句话说明这个代理链的目标",
"agentPlan": ["3-6个执行步骤"],
"deliverables": ["2-4个预期交付物"],
"recommendedNodes": ["建议继续连接的节点类型或能力名"]
}`,
"步骤要偏工作流视角，而不是泛泛而谈。",
]
.filter(Boolean)
.join("\n");
}
function buildStoryScriptPrompt(params: {
nodeTitle?: string;
prompt: string;
imageCount: number;
videoCount: number;
}) {
return [
"你是视频画布里的故事脚本生成节点，需要把用户输入、参考图和参考视频整理成可直接用于后续分镜/生图/生视频的结构化脚本表。",
params.nodeTitle ? `当前节点：${params.nodeTitle}` : "",
`参考图数量：${params.imageCount}`,
`视频数量：${params.videoCount}`,
params.prompt ? `输入内容：\n${params.prompt}` : "没有额外脚本，请生成一个节奏清晰的短视频脚本。",
"",
"请只输出 JSON，不要输出 markdown，不要输出解释。",
"JSON 结构如下：",
`{
"summary": "一句话概括故事主题、情绪和视频方向，不超过120字",
"analysisRows": [
{
"shot": "01",
"duration": "5s",
"scene": "场景名或地点，如 盛唐·金銮殿·日",
"sceneDescription": "画面内容、人物动作和剧情推进，不要重复 scene 字段",
"shotSize": "景别，如中景、特写、全景、大远景",
"lighting": "光影氛围，如暖金主光带阴影，气氛紧张压抑",
"dialogue": "对白或旁白，没有就写 +",
"backgroundMusic": "背景音乐或情绪氛围，没有就写无",
"voiceAndSfx": "环境声、动作声或音效，没有就写无",
"cameraMovement": "运镜，如缓慢推镜头、固定、横移、上摇、环绕轨道跟拍",
"imagePrompt": "后续生成该镜头关键帧时可直接使用的中文图像提示词",
"motionPrompt": "后续生成视频时可直接使用的中文视频运动提示词"
}
],
"assetCandidates": {
"style": "全局视觉风格，如 盛唐·国风二次元",
"characters": ["需要生成的角色资产名"],
"scenes": ["需要生成的场景资产名"],
"props": ["需要生成的道具资产名"]
},
"shotIdeas": [
{ "title": "镜头 1", "description": "一句话描述镜头目标、景别或动作" }
],
"recommendedNextSteps": ["2-4条建议的下游动作"]
}`,
"优先严格遵守用户指定的镜头数量和总时长；未指定时生成 6 个镜头，最多 18 个。镜头时长之和必须符合用户要求，8 秒短片不得扩写成几十秒。只生成实际需要的镜头，不重复凑数。",
"每行要覆盖竞品脚本表同等信息：镜号、时长、画面描述、景别、光影氛围、对白·旁白、音效、运镜、最终图像提示词和视频运动提示词。",
"画面描述中用 @角色名 或 @场景/道具名 标出可复用资产，例如 @沈昭昭、@金銮殿、@侍卫长刀。",
"imagePrompt 要像可直接生成分镜图的中文提示词，包含主体、环境、构图、光影、风格与一致性要求；motionPrompt 要按起始状态、动作过程、结束状态、音效组织。",
"assetCandidates 只提取脚本中确实出现的角色、场景和道具；产品视频没有人物时 characters 必须为空，禁止为凑数虚构角色。",
"每行要像专业视频画布脚本表：场景具体、画面可生成、声音可执行、镜头节奏明确，不要只写剧情摘要。",
]
.filter(Boolean)
.join("\n");
}
export async function POST(request: Request) {
try {
const session = await auth();
if (!session?.user?.id) {
return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
}
const body = await request.json().catch(() => ({}));
const parsed = requestSchema.safeParse(body);
if (!parsed.success) {
return NextResponse.json(
{ success: false, error: "请求参数不合法" },
{ status: 400 }
);
}
const prompt = normalizeText(parsed.data.prompt);
const imageUrls = sanitizeUrls(parsed.data.imageUrls, 4);
const videoUrls = sanitizeUrls(parsed.data.videoUrls, 4);
const extractedVideoFrames =
parsed.data.mode === "video-analysis" && videoUrls.length > 0
? await extractVideoFrameDataUrls(videoUrls[0], VIDEO_ANALYSIS_FRAME_COUNT)
: [];
const preferredModelRef = normalizeText(parsed.data.preferredModelRef || parsed.data.model);
if (!prompt && imageUrls.length === 0 && videoUrls.length === 0) {
return NextResponse.json(
{ success: false, error: "请至少提供脚本、图片或视频上下文" },
{ status: 400 }
);
}
const [allModels, agentModel] = await Promise.all([
getEnabledAiModelsWithProvider(false),
getAgentAiModelWithProvider(),
]);
const hasVisionInputs = imageUrls.length > 0 || extractedVideoFrames.length > 0;
const compatibleModels = allModels.filter((item) => {
const descriptor = {...item.model, providerType: item.provider.type};
return !item.model.isSubModel && item.model.isEnabled &&
(hasVisionInputs ? isCanvasVisionModel(descriptor) : isCanvasLanguageModel(descriptor));
});
const requestedRef = preferredModelRef === "auto" ? "" : preferredModelRef;
const preferredModel = requestedRef ? compatibleModels.find(
item => item.model.id === requestedRef || item.model.modelId === requestedRef
) : undefined;
if (requestedRef && !preferredModel) {
return NextResponse.json({ success: false, error: "所选模型不可用或不支持当前分析输入，请重新选择分析模型" }, { status: 400 });
}
const selectedModel = preferredModel || compatibleModels.find(item => item.model.id === agentModel?.model.id) || compatibleModels[0];
if (!selectedModel) {
return NextResponse.json({ success: false, error: hasVisionInputs ? "当前没有可用的视觉理解模型" : "当前没有可用的文字分析模型" }, { status: 400 });
}
const useVision = hasVisionInputs && supportsVision(selectedModel.model);
const codingModelId = getAiModelAgentCodingModelId(selectedModel.model);
const model = await getLanguageModelById(selectedModel.model.id, codingModelId);
if (!model) {
return NextResponse.json(
{ success: false, error: "分析模型初始化失败" },
{ status: 500 }
);
}
const promptText =
parsed.data.mode === "video-analysis"
? buildVideoAnalysisPrompt({
frameCount: extractedVideoFrames.length,
nodeTitle: normalizeText(parsed.data.nodeTitle),
prompt,
imageCount: imageUrls.length,
videoCount: videoUrls.length,
})
: parsed.data.mode === "character-scene-extraction"
? buildCharacterScenePrompt({
nodeTitle: normalizeText(parsed.data.nodeTitle),
prompt,
imageCount: imageUrls.length,
})
: parsed.data.mode === "story-script"
? buildStoryScriptPrompt({
nodeTitle: normalizeText(parsed.data.nodeTitle),
prompt,
imageCount: imageUrls.length,
videoCount: videoUrls.length,
})
: buildAgentPlanPrompt({
nodeTitle: normalizeText(parsed.data.nodeTitle),
prompt,
imageCount: imageUrls.length,
videoCount: videoUrls.length,
});
const inlinedImageReferences = useVision
? (
await inlineVisionImages(imageUrls, {
maxImages: 4,
maxSide: 960,
timeoutMs: 12000,
retries: 1,
loggerPrefix: "[Canvas Workflow Analyze]",
})
).inlineDataUrls
: [];
const referenceImages = useVision
? [...inlinedImageReferences, ...extractedVideoFrames].slice(0, 6)
: [];
const { text, finishReason } = await generateText({
model,
messages: [
{
role: "user",
content: [
{ type: "text", text: promptText },
...referenceImages.map((image) => ({ type: "image" as const, image })),
],
},
],
temperature: 0.2,
maxOutputTokens: parsed.data.mode === "story-script" ? 12000 : parsed.data.mode === "video-analysis" ? 2200 : 1600,
maxRetries: 0,
abortSignal: AbortSignal.any([request.signal, AbortSignal.timeout(150_000)]),
});
if (finishReason === "length" || !text.trim()) {
return NextResponse.json({ success: false, error: "分析结果不完整，请缩短输入或重新生成" }, { status: 502 });
}
const parsedJson = extractJsonBlock<Record<string, unknown>>(text);
if (!parsedJson) {
return NextResponse.json({ success: false, error: "分析模型未返回完整结构化结果，请重新生成" }, { status: 502 });
}
const summary =
normalizeText(parsedJson.summary) ||
normalizeText(text).slice(0, 160);
if (parsed.data.mode === "video-analysis") {
const analysisRows = normalizeVideoAnalysisRows(
parsedJson.analysisRows,
extractedVideoFrames,
);
const shotIdeas =
normalizeShotIdeas(parsedJson.shotIdeas).length > 0
? normalizeShotIdeas(parsedJson.shotIdeas)
: analysisRows.map((row, index) => ({
title: `镜头 ${index + 1}`,
description: row.sceneDescription,
}));
return NextResponse.json({
success: true,
data: {
summary,
analysisRows,
framePreviews: extractedVideoFrames,
highlights: normalizeStringList(parsedJson.highlights, 6),
shotIdeas,
visualMotifs: normalizeStringList(parsedJson.visualMotifs, 6),
recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6),
model: selectedModel.model.displayName,
raw: text,
},
});
}
if (parsed.data.mode === "character-scene-extraction") {
return NextResponse.json({
success: true,
data: {
summary,
characters: normalizeCharacters(parsedJson.characters),
scenes: normalizeScenes(parsedJson.scenes),
relationships: normalizeStringList(parsedJson.relationships, 8),
recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6),
model: selectedModel.model.displayName,
raw: text,
},
});
}
if (parsed.data.mode === "story-script") {
const analysisRows = normalizeStoryScriptRows(
parsedJson.analysisRows || parsedJson.storyRows || parsedJson.rows || parsedJson.shots,
);
if (analysisRows.length === 0) {
return NextResponse.json({ success: false, error: "分析模型没有返回有效镜头，请重新生成" }, { status: 502 });
}
return NextResponse.json({
success: true,
data: {
summary,
analysisRows,
shotIdeas:
normalizeShotIdeas(parsedJson.shotIdeas).length > 0
? normalizeShotIdeas(parsedJson.shotIdeas)
: analysisRows.map((row, index) => ({
title: `镜头 ${index + 1}`,
description: row.sceneDescription,
})),
recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6),
model: selectedModel.model.displayName,
raw: text,
},
});
}
return NextResponse.json({
success: true,
data: {
summary,
agentPlan: normalizeStringList(parsedJson.agentPlan, 8),
deliverables: normalizeStringList(parsedJson.deliverables, 6),
recommendedNodes: normalizeStringList(parsedJson.recommendedNodes, 6),
model: selectedModel.model.displayName,
raw: text,
},
});
} catch (error) {
console.error("[Canvas Workflow Analyze] Error:", error);
const timedOut = error instanceof Error && /timeout|timed out|abort/i.test(`${error.name} ${error.message}`);
return NextResponse.json(
{
success: false,
error: timedOut ? "分析超时，请稍后重试或选择其他分析模型" : error instanceof Error ? error.message : "工作流分析失败",
},
{ status: timedOut ? 504 : 500 }
);
}
const session = await auth();
if (!session?.user?.id)
return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
const body = await request.json().catch(() => ({}));
return executeWorkflowAnalysisResponse(body, request.signal);
}

1159
components/canvas/LiblibVideoCanvasReplica.tsx
View File

File diff suppressed because it is too large Load Diff

78
components/canvas/PromptTranslateButton.tsx Normal file
View File

@ -0,0 +1,78 @@
"use client";
import { useEffect, useRef, useState } from "react";
import { toast } from "sonner";
export function PromptTranslateButton({
prompt,
model,
onApply,
}: {
prompt: string;
model?: string;
onApply: (text: string) => void;
}) {
const [busy, setBusy] = useState(false),
[result, setResult] = useState("");
const controller = useRef<AbortController | null>(null);
useEffect(() => () => controller.current?.abort(), []);
return (
<>
<button
type="button"
disabled={busy || !prompt.trim()}
className="rounded px-2 py-1 text-sm disabled:opacity-40"
onClick={async () => {
const request = new AbortController();
controller.current = request;
setBusy(true);
try {
const response = await fetch("/api/canvas/prompt-tools", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ mode: "translate", prompt, model }),
signal: AbortSignal.any([request.signal, AbortSignal.timeout(100000)]),
});
const body = await response.json();
if (!response.ok || !body.success || !body.data?.text)
throw Error(body.error || "翻译没有返回结果");
if (!request.signal.aborted) setResult(body.data.text);
} catch (error) {
if (!request.signal.aborted)
toast.error(error instanceof Error ? error.message : "翻译失败");
} finally {
setBusy(false);
}
}}
>
{busy ? "翻译中…" : "翻译"}
</button>
{result ? (
<div
role="dialog"
aria-label="翻译结果"
className="fixed inset-4 z-[700] flex flex-col gap-3 rounded-xl border border-white/20 bg-[#222] p-5 text-white"
>
<textarea
aria-label="翻译内容"
className="min-h-0 flex-1 resize-none bg-transparent"
value={result}
onChange={(event) => setResult(event.target.value)}
/>
<div className="flex justify-end gap-3">
<button type="button" onClick={() => setResult("")}>
放弃
</button>
<button
type="button"
onClick={() => {
onApply(result);
setResult("");
}}
>
应用翻译
</button>
</div>
</div>
) : null}
</>
);
}

51
docs/video-canvas-task-repairs-2026-09-17.md Normal file
View File

@ -0,0 +1,51 @@
# 视频画布任务与交互修复 · 2026-09-17
承接 `23f86dfc` 的全面扫描。用户补充“浏览器刷新一下就好了”，本轮重点修复任务结果与页面状态脱节，同时处理 F02–F07 中已确认的入口问题。本记录是分支修复与本地验收，未部署 48 服务器，也没有把原 231 项清单改成全部通过。
## 修复范围
| 扫描项 | 本轮改动 | 验证及边界 |
|---|---|---|
| F01 任务恢复 | 新增统一生成任务接口、数据库受理记录、请求 ID 去重、原子领取、心跳与结果保存；客户端提交前保存待核对请求，刷新/重开/online/focus/visibilitychange 时查询；去掉虚假进度；旧版无任务 ID 的 pending 明确提示无法恢复 | 主图像/视频生成、配置工具的图像/视频/分析、脚本再生成、批量分镜/视频接入。局部编辑按已有 taskId 恢复查询。真实数据库与浏览器验收见下文。专用图像后期、角度编辑、脚本弹窗资产生成/逐镜提示词仍有同步请求，未宣称全部入口具备同等恢复能力 |
| F02 取消 | 排队任务通过数据库 CAS 取消；已提交供应商的任务返回明确说明，继续保存结果；删除已完成结果写入任务隐藏记录 | 数据库并发及浏览器排队取消通过；没有冒充支持上游中止 |
| F03 模板 | 待执行模板设为 idle；参考样例标记示例；空脚本不再显示硬编码成稿 | 代码检查；未逐个模板做生产生成 |
| F04 脚本 | 脚本入口统一 story-script；空响应/无有效镜头返回失败并保留旧稿 | 结果契约回归通过；脚本资产生成与单镜提示词处理不是此次持久队列接入范围 |
| F05 批量 | 批量分镜和视频实际入队；按有无首帧选择兼容模式，采用模型声明的离散时长/比例/分辨率；完成任务不重复提交，失败/空节点可继续执行 | 模型参数和任务契约通过；没有新增真实付费批量请求 |
| F06 素材弹窗 | 去掉假分页和无效图标，增加搜索、纵向滚动、自适应列、素材名称与明确数量单位 | 实际组件注入 100 个素材，第 100 项能选中、确认回传，搜索返回 1 项；使用当前 Tailwind 编译产物验收 |
| F07 次级控件 | 文本及音频生视频提示词翻译接真实接口并预览后应用；旧脚本编辑聚焦；资源替换接上传；通知显示真实任务；模板筛选使用配置数据；退出接现有登出流程；未实现按钮明确禁用 | 代码与类型检查。未实际退出生产账户，未声称所有停用高级能力已实现 |
任务表复用既有 `0091_add_canvas_capability_tasks.sql`，没有新增数据库迁移。上线前须核对该既有迁移已应用。
## 任务一致性
- 同一请求 ID 重试不会重复执行；同一节点已有活动任务时复用现有任务。
- 同画布任务串行领取，多执行者通过 PostgreSQL 事务锁与状态更新竞争。
- 刷新前画布尚未保存的已受理节点，可从任务快照补回；不会从旧任务复活已保存删除的节点。
- 多图结果使用任务 ID 与序号组成稳定结果节点 ID，轮询不重复追加，重新生成不误用上次附图。
- 旧请求不能覆盖新请求；画布保存落后于新任务时，根据任务时间和保存版本恢复最新结果。
- 连线仅首次恢复时补齐，后续轮询不反复恢复用户已经删除的连线。
- 执行进程心跳超过 120 秒未更新，任务标记待核实；不自动重新提交付费请求。若原执行稍后确实返回，仍允许保存真实结果。
- 浏览器离开与服务端执行进程重启是不同情况：本轮解决前者的主要入口；进程中断后尚未实现供应商任务 ID 的自动续查。
## 验证记录
1. **真实本地 PostgreSQL**：`scripts/qa/video-canvas-task-db.cjs` 建立隔离测试库，加载实际任务表迁移和实际 store。10 次并发相同提交只创建 1 条记录；10 次并发领取仅 1 次成功；跨用户隔离、排队取消、执行中不可取消、串行队列、结果保存、结果隐藏、过期不重投、晚到结果保存全部通过。这是并发数据库调用测试，不是 10 个操作系统进程实测。
2. **真实浏览器任务恢复**：验收页使用实际 React Hook、实际任务 route/worker/store 与本地 PostgreSQL；鉴权使用固定测试用户，NextResponse/after 使用本地适配器，模型返回使用 12 秒测试桩。提交图片后立即刷新，最终出现 2 个结果且只 1 条任务；视频执行时关闭页面，再打开能恢复结果；后续分析排队任务成功取消；移除附图后刷新不复活。画布快照保存使用本地验收服务器，不是生产画布接口。
3. **实际素材组件**：100 项列表可定位选择最后一项，确认回传 `测试素材 100`；搜索同名只剩 1 项且保留选中状态；已有第 100 项再连续点选第 1–10 项，最终只选中 10 项，第 10 项（第 11 次选择）被正确拦截。验收期间先发现旧测试样式缓存，重新编译当前源码样式后验证通过。
4. **自动化**：最终 helpers/route 回归 437 项通过，无跳过，包含本次新增 13 项任务/批量参数测试和既有真实 FFmpeg 检查。不是 437 个生产端到端测试。
5. **静态检查**：TypeScript 无错误；`git diff --check` 通过。新增服务端模块与旧接口共享执行逻辑，图像计费仍走原有计费模块。
没有进行新的付费模型测试，没有重新宣称参考站像素级一致，没有覆盖操作系统休眠、真实断网、多标签冲突、生产 Next.js 进程重启或生产部署后的回归。原扫描中 79 项缺失/停用能力依然需要单独实现与验证，详见原验证矩阵。
## 复现
```sh
# FFmpeg / ffprobe 必须在 PATH 中，避免媒体测试跳过
pnpm exec playwright test --config=playwright.helpers.config.ts
pnpm exec tsc --noEmit --incremental false
# 隔离数据库测试依赖仅装入忽略的 .qa 目录，不改项目依赖
npm install --prefix .qa/video-task-db --no-save embedded-postgres
node scripts/qa/video-canvas-task-db.cjs
```
浏览器验收脚手架位于忽略目录 `.qa/video-durable` 和 `.qa/video-repair`；上述记录保留测试方法和模拟边界，不能代替后续部署验收。

2
docs/video-canvas-validation-scan-2026-09-17.md
View File

@ -1,5 +1,7 @@
# 视频画布全面验证扫描 · 2026-09-17
后续修复记录：[任务与交互修复](video-canvas-task-repairs-2026-09-17.md)。以下内容保留扫描时的基线，不代表修复后的现状；也未将未验收项批量改为通过。
结论：**仍有未验证功能，也有已确认但尚未修复的功能缺口，不能判定全面验收通过。** 本轮以 `f10e6b7c` 为代码起点，核对原 10 组、231 项对照清单，追踪活动页面的调用、请求和保存流程，补跑相关回归与实际组件浏览器验证。没有重新提交付费模型任务，没有重新实测参考网站，也没有部署 48 服务器。
## 证据范围

272
hooks/use-video-canvas-tasks.ts Normal file
View File

@ -0,0 +1,272 @@
"use client";
import { useCallback, useEffect, useRef, useState } from "react";
import { toast } from "sonner";
import {
reconcileCanvasTasks,
type CanvasGenerationTask,
type CanvasTaskRequest,
} from "@/lib/video-canvas/task-contract";
type Node = CanvasTaskRequest["node"];
type Edge = CanvasTaskRequest["edges"][number];
const storageKey = (projectId: string) => `video-canvas-outbox:${projectId}`;
function outbox(projectId: string): CanvasTaskRequest[] {
try {
const entries = JSON.parse(localStorage.getItem(storageKey(projectId)) || "[]");
return Array.isArray(entries)
? entries.filter(
(item) =>
item?.projectId === projectId && typeof item?.requestId === "string" && item?.node?.id,
)
: [];
} catch {
return [];
}
}
function remember(request: CanvasTaskRequest) {
const entries = outbox(request.projectId).filter((item) => item.requestId !== request.requestId);
localStorage.setItem(storageKey(request.projectId), JSON.stringify([...entries, request]));
}
function forget(projectId: string, requestId: string) {
localStorage.setItem(
storageKey(projectId),
JSON.stringify(outbox(projectId).filter((item) => item.requestId !== requestId)),
);
}
export function useVideoCanvasTasks<T extends Node>(
projectId: string,
loaded: { id: string; revision: string } | null,
setNodes: React.Dispatch<React.SetStateAction<T[]>>,
setEdges: React.Dispatch<React.SetStateAction<Edge[]>>,
) {
const [tasks, setTasks] = useState<CanvasGenerationTask[]>([]);
const hiddenNodes = useRef(new Set<string>());
const recoveredEdges = useRef(new Set<string>());
const refreshRef = useRef<() => void>(() => {});
const activeProject = useRef(projectId);
activeProject.current = projectId;
const apply = useCallback(
(tasks: CanvasGenerationTask[]) => {
if (activeProject.current !== projectId || !loaded) return;
setTasks((current) => {
const next = new Map(current.map((task) => [task.id, task]));
for (const task of tasks) next.set(task.id, task);
return [...next.values()];
});
const visibleTasks = tasks.map((task) => ({
...task,
request: {
...task.request,
hiddenNodeIds: [...(task.request.hiddenNodeIds || []), ...hiddenNodes.current],
},
}));
setNodes((nodes) => reconcileCanvasTasks(nodes as any, visibleTasks, loaded.revision) as T[]);
const recoverable = visibleTasks.filter((task) => {
if (recoveredEdges.current.has(task.id)) return false;
recoveredEdges.current.add(task.id);
return Date.parse(task.createdAt) > Date.parse(loaded.revision);
});
setEdges((edges) => {
const existing = new Set(edges.map((edge) => `${edge.source}>${edge.target}`));
const extras = recoverable
.flatMap((task) =>
task.request.edges.filter(
(edge) =>
!task.request.hiddenNodeIds.includes(edge.source) &&
!task.request.hiddenNodeIds.includes(edge.target),
),
)
.filter((edge) => {
const key = `${edge.source}>${edge.target}`;
if (existing.has(key)) return false;
existing.add(key);
return true;
});
return extras.length ? [...edges, ...extras] : edges;
});
},
[projectId, loaded, setNodes, setEdges],
);
const send = useCallback(
async (request: CanvasTaskRequest) => {
const response = await fetch("/api/canvas/generation-tasks", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(request),
signal: AbortSignal.timeout(20000),
});
const body = await response.json();
if (!response.ok || !body.success) {
if (response.status >= 400 && response.status < 500) {
forget(projectId, request.requestId);
if (activeProject.current === projectId)
setNodes((nodes) =>
nodes.map((node) =>
node.id === request.node.id && node.draftValues?.taskRequestId === request.requestId
? {
...node,
draftValues: {
...node.draftValues,
generationStatus: "failed",
generationError: body.error,
},
}
: node,
),
);
}
throw Error(body.error || "提交暂时失败，恢复连接后自动核对");
}
forget(projectId, request.requestId);
if (activeProject.current !== projectId) return body.task as CanvasGenerationTask;
// The server may return an existing task for the same active node.
setNodes((nodes) =>
nodes.map((node) =>
node.id === request.node.id
? {
...node,
draftValues: {
...node.draftValues,
taskRequestId: body.task.requestId,
generationTaskId: body.task.id,
},
}
: node,
),
);
apply([body.task]);
return body.task as CanvasGenerationTask;
},
[projectId, apply, setNodes],
);
const submit = useCallback(
async (
node: T,
kind: CanvasTaskRequest["kind"],
params: Record<string, unknown>,
edges: Edge[] = [],
) => {
if (!loaded || loaded.id !== projectId) throw Error("画布尚未加载完成，请稍后提交");
const requestId = crypto.randomUUID();
const snapshot = {
...node,
draftValues: {
...node.draftValues,
taskRequestId: requestId,
generationTaskId: undefined,
appliedTaskStatus: undefined,
generationStatus: "pending",
generationProgress: 0,
pendingLabel: "提交中",
generationError: undefined,
},
};
const request: CanvasTaskRequest = {
projectId,
requestId,
kind,
node: snapshot,
params,
edges,
};
// Persist before POST: a refresh during an uncertain network response retries the same ID.
remember(request);
setNodes((nodes) => nodes.map((item) => (item.id === node.id ? snapshot : item)));
try {
await send(request);
} catch (error) {
toast.error(
error instanceof Error ? error.message : "连接中断，恢复后将核对任务，请勿重复提交",
);
}
refreshRef.current();
},
[loaded, projectId, send, setNodes],
);
useEffect(() => {
if (!loaded || loaded.id !== projectId) return;
hiddenNodes.current.clear();
recoveredEdges.current.clear();
setTasks([]);
let stopped = false,
running = false;
const refresh = async () => {
if (stopped || running) return;
running = true;
try {
const response = await fetch(
`/api/canvas/generation-tasks?projectId=${encodeURIComponent(projectId)}`,
{ cache: "no-store", signal: AbortSignal.timeout(15000) },
);
if (!response.ok) return;
const body = await response.json();
if (stopped || !body.success) return;
apply(body.tasks);
const accepted = new Map<string, CanvasGenerationTask>(
body.tasks.map((task: CanvasGenerationTask) => [task.requestId, task]),
);
for (const request of outbox(projectId)) {
if (stopped) break;
if (accepted.has(request.requestId)) {
forget(projectId, request.requestId);
continue;
}
await send(request);
}
} catch {
/* Network loss is not a generation failure. The next online/focus event queries again. */
} finally {
running = false;
}
};
refreshRef.current = () => void refresh();
const wake = () => {
if (document.visibilityState === "visible") void refresh();
};
const timer = setInterval(() => void refresh(), 5000);
window.addEventListener("online", wake);
window.addEventListener("focus", wake);
document.addEventListener("visibilitychange", wake);
void refresh();
return () => {
stopped = true;
clearInterval(timer);
window.removeEventListener("online", wake);
window.removeEventListener("focus", wake);
document.removeEventListener("visibilitychange", wake);
};
}, [projectId, loaded, apply, send]);
const cancel = useCallback(
async (taskId: string) => {
const response = await fetch("/api/canvas/generation-tasks", {
method: "DELETE",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ projectId, taskId }),
signal: AbortSignal.timeout(15000),
});
const body = await response.json();
if (body.task) apply([body.task]);
if (!response.ok || !body.success) throw Error(body.error || "取消失败，请刷新任务状态");
refreshRef.current();
},
[projectId, apply],
);
const dismiss = useCallback(
async (nodeId: string, taskId?: string) => {
hiddenNodes.current.add(nodeId);
if (!taskId) return;
const response = await fetch("/api/canvas/generation-tasks", {
method: "PATCH",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ projectId, taskId, nodeId }),
signal: AbortSignal.timeout(15000),
});
if (!response.ok) {
hiddenNodes.current.delete(nodeId);
throw Error("移除结果失败，请重试");
}
},
[projectId],
);
return { submit, cancel, dismiss, tasks, refresh: () => refreshRef.current() };
}

125
lib/canvas/generate-image-response.ts Normal file
View File

@ -0,0 +1,125 @@
import "server-only";
import { NextResponse } from "next/server";
import {
createCanvasImageTask,
enqueueCanvasImageTask,
} from "@/lib/canvas/image-task-queue";
import { generateCanvasImage, type CanvasGenerateImageRequest } from "@/lib/canvas/generate-image";
import { getPointsSystemConfig } from "@/lib/db/queries";
import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge";
import { getImageToolCreditCost, type ImageToolCreditFeature } from "@/lib/points/config";
interface GenerateImageRequest extends CanvasGenerateImageRequest {
async?: boolean;
}
function normalizeBillingFeature(
value: unknown,
): Extract<ImageToolCreditFeature, "multi-view" | "repaint"> | null {
if (value === "multi-view" || value === "repaint") return value;
return null;
}
function toGenerateImagePayload(body: GenerateImageRequest): CanvasGenerateImageRequest {
return {
allowFailover: typeof body.allowFailover === "boolean" ? body.allowFailover : undefined,
prompt: String(body.prompt || "").trim(),
negativePrompt: typeof body.negativePrompt === "string" ? body.negativePrompt : undefined,
model: typeof body.model === "string" ? body.model : undefined,
aspectRatio:
typeof body.aspectRatio === "string" && body.aspectRatio.trim()
? body.aspectRatio.trim()
: "auto",
imageSize:
typeof body.imageSize === "string" && body.imageSize.trim()
? body.imageSize.trim()
: undefined,
imageCount:
typeof body.imageCount === "number" && Number.isFinite(body.imageCount) ? body.imageCount : 1,
promptExtend: typeof body.promptExtend === "boolean" ? body.promptExtend : undefined,
referenceImage: typeof body.referenceImage === "string" ? body.referenceImage : undefined,
referenceImages: Array.isArray(body.referenceImages)
? body.referenceImages.map((item) => String(item || "").trim()).filter(Boolean)
: [],
};
}
export async function executeCanvasImageResponse(body: GenerateImageRequest, userId: string) {
try {
const payload: CanvasGenerateImageRequest = {
...toGenerateImagePayload(body),
ownerKey: userId,
};
if (!payload.prompt) {
return NextResponse.json({ success: false, error: "请输入提示词" }, { status: 400 });
}
const billingFeature = normalizeBillingFeature(body.billingFeature);
const pointsCharge = billingFeature
? await (async () => {
const pointsConfig = await getPointsSystemConfig();
return prepareFeaturePointsCharge({
userId: userId,
unitCost: getImageToolCreditCost(pointsConfig, billingFeature),
reason: billingFeature === "multi-view" ? "image_multi_view" : "image_repaint",
modelId: "canvas-generate-image",
note: billingFeature === "multi-view" ? "图片多视角生成消耗积分" : "图片重绘消耗积分",
pointsConfig,
});
})()
: null;
if (pointsCharge && !pointsCharge.ok) {
return NextResponse.json(pointsCharge.payload, { status: pointsCharge.status });
}
const asyncMode = body.async === true;
if (asyncMode) {
const task = await createCanvasImageTask({
userId: userId,
request: payload as unknown as Record<string, unknown>,
});
await enqueueCanvasImageTask(task.id);
const currentTask = await getCanvasImageTaskForUser({
taskId: task.id,
userId: userId,
});
if (pointsCharge?.chargeable) {
try {
await pointsCharge.consume({
note:
billingFeature === "multi-view"
? `图片多视角生成消耗积分（${task.id}）`
: `图片重绘消耗积分（${task.id}）`,
});
} catch (pointsError) {
console.error("[Points] Failed to consume canvas image task points:", pointsError);
}
}
return NextResponse.json({
success: true,
taskId: task.id,
status: currentTask?.status || task.status,
queueStatus: currentTask?.queueStatus,
});
}
const result = await generateCanvasImage(payload);
if (pointsCharge?.chargeable) {
try {
await pointsCharge.consume();
} catch (pointsError) {
console.error("[Points] Failed to consume canvas image points:", pointsError);
}
}
return NextResponse.json({
success: true,
data: result,
});
} catch (error) {
console.error("[Canvas Generate Image] Error:", error);
return NextResponse.json(
{ success: false, error: error instanceof Error ? error.message : "生成失败" },
{ status: 500 },
);
}
}

13
lib/video-canvas/model-matching.ts
View File

@ -108,3 +108,16 @@ export function validateCanvasVideoInputs(mode: VideoGenerationMode, images: str
if (mode === "first-last-frame" && images.length !== 2) throw new Error("首尾帧生视频需要恰好两张参考图片");
if (mode === "reference-images" && images.length === 0) throw new Error("参考生视频需要至少一张参考图片");
}
export function chooseStoryVideoSettings(models: CanvasModelDescriptor[], hasImage: boolean, seconds: number, preferredRef?: string) {
const mode:VideoGenerationMode = hasImage ? "first-frame" : "text-to-video";
const choices = filterCanvasModels(models,"video",mode);
const model = choices.find(item=>item.id===preferredRef||item.modelId===preferredRef) || choices[0];
if (!model) throw Error(hasImage ? "没有可用的首帧视频模型" : "镜头缺少首帧，且没有可用的文生视频模型");
const options = getCanvasVideoParameters(model,mode);
if (!options.durations.length || !options.ratios.length || !options.resolutions.length) throw Error("视频模型缺少已确认的时长、比例或清晰度规格，请先完善模型配置");
const duration = options.durations.reduce((best,value)=>Math.abs(value-seconds)<Math.abs(best-seconds)?value:best,options.durations[0]);
return {mode,model:model.id||model.modelId||"",duration,
ratio:options.ratios.includes("16:9")?"16:9":options.ratios[0],
resolution:options.resolutions.includes("720p")?"720p":options.resolutions[0],generateAudio:options.supportsAudio};
}

182
lib/video-canvas/task-contract.ts Normal file
View File

@ -0,0 +1,182 @@
import { z } from "zod";
export const canvasTaskRequestSchema = z.object({
requestId: z.string().uuid(),
projectId: z.string().uuid(),
kind: z.enum(["video", "image", "analysis"]),
node: z
.object({
id: z.string().min(1).max(180),
title: z.string().max(300),
x: z.number().finite(),
y: z.number().finite(),
width: z.number().positive().max(10000),
height: z.number().positive().max(10000),
kind: z.string().max(30),
inputType: z.string().max(30).optional(),
draftValues: z.record(z.unknown()).optional(),
})
.passthrough(),
params: z.record(z.unknown()),
hiddenNodeIds: z.array(z.string()).max(20).optional(),
edges: z
.array(z.object({ id: z.string(), source: z.string(), target: z.string() }))
.max(30)
.default([]),
});
export type CanvasTaskRequest = z.infer<typeof canvasTaskRequestSchema>;
export type CanvasGenerationTask = {
id: string;
requestId: string;
status: string;
createdAt: string;
request: CanvasTaskRequest;
result?: Record<string, any> | null;
error?: string | null;
};
export function isCanvasTaskActive(status: string) {
return status === "pending" || status === "running";
}
export function canCancelCanvasTask(status: string) {
return status === "pending";
}
export function requireAnalysisResult(data: Record<string, any> | undefined, mode: string) {
if (!data) throw Error("分析没有返回结果，原内容已保留");
const rows = data.analysisRows || data.shotIdeas;
if (
mode === "story-script" &&
(!Array.isArray(rows) ||
!rows.length ||
!rows.some((row) =>
[row.sceneDescription, row.description, row.imagePrompt].some(
(value) => typeof value === "string" && value.trim(),
),
))
) {
throw Error("脚本没有返回有效镜头，原内容已保留");
}
if (
mode !== "story-script" &&
![data.raw, data.summary, ...(data.agentPlan || [])].some(
(value) => typeof value === "string" && value.trim(),
)
) {
throw Error("分析返回了空内容，原内容已保留");
}
return data;
}
// Stable result IDs and request IDs prevent repeat polling from duplicating images
// or an old task from overwriting a later generation on the same node.
export function reconcileCanvasTasks<
T extends { id: string; draftValues?: Record<string, any>; [key: string]: any },
>(nodes: T[], tasks: CanvasGenerationTask[], projectRevision: string) {
let result = nodes;
const latestByNode = new Map<string, CanvasGenerationTask>();
for (const task of tasks) {
const previous = latestByNode.get(task.request.node.id);
if (!previous || Date.parse(task.createdAt) >= Date.parse(previous.createdAt))
latestByNode.set(task.request.node.id, task);
}
for (const task of latestByNode.values()) {
const source = task.request.node;
const hidden = new Set(task.request.hiddenNodeIds || []);
if (hidden.has(source.id)) continue;
let index = result.findIndex((node) => node.id === source.id);
if (index < 0) {
if (Date.parse(task.createdAt) <= Date.parse(projectRevision)) continue;
result = [...result, source as unknown as T];
index = result.length - 1;
}
const node = result[index];
const draft = node.draftValues || {};
if (draft.taskRequestId && draft.taskRequestId !== task.requestId) {
const savedTask = tasks.find((item) => item.requestId === draft.taskRequestId);
if (
!savedTask ||
Date.parse(task.createdAt) <= Date.parse(savedTask.createdAt) ||
Date.parse(task.createdAt) <= Date.parse(projectRevision)
)
continue;
}
if (draft.appliedTaskStatus === `${task.id}:${task.status}`) continue;
const nextDraft = {
...draft,
taskRequestId: task.requestId,
generationTaskId: task.id,
appliedTaskStatus: `${task.id}:${task.status}`,
generationError: undefined as string | undefined,
generationProgress: 0,
generationStatus: "pending",
pendingLabel: task.status === "pending" ? "排队中" : "生成中",
};
let subtitle = nextDraft.pendingLabel;
if (task.status === "succeeded") {
const data = task.result || {};
Object.assign(nextDraft, {
generated: true,
generationStatus: "complete",
generationProgress: 100,
pendingLabel: "",
});
subtitle = "生成结果";
if (task.request.kind === "video")
Object.assign(nextDraft, { mediaUrl: data.url, url: [data.url], videoMeta: data.metadata });
if (task.request.kind === "image") {
const urls = (data.images || [])
.map((image: any) => image.originalUrl || image.thumbnailUrl)
.filter(Boolean);
Object.assign(nextDraft, {
mediaUrl: urls[0],
url: urls,
action: draft.action === "IMAGE_GENERATE_NINE_GRID" ? draft.action : "IMAGE_RESOURCE",
});
for (let i = 1; i < urls.length; i++) {
const id = `${source.id}-result-${task.id}-${i}`;
if (!hidden.has(id) && !result.some((item) => item.id === id))
result = [
...result,
{
...node,
id,
title: `${node.title} ${i + 1}`,
y: node.y + i * (node.height + 40),
draftValues: { ...nextDraft, mediaUrl: urls[i], url: [urls[i]] },
} as T,
];
}
}
if (task.request.kind === "analysis")
Object.assign(nextDraft, {
content: [data.raw || data.summary || data.agentPlan?.join("\n") || ""],
...(Array.isArray(data.analysisRows || data.shotIdeas)
? { rows: data.analysisRows || data.shotIdeas }
: {}),
});
} else if (["failed", "cancelled", "interrupted"].includes(task.status)) {
Object.assign(nextDraft, {
generated: false,
generationStatus: "failed",
pendingLabel: "",
generationError:
task.error ||
(task.status === "cancelled" ? "已取消排队，未提交供应商" : "任务失败，请查看任务状态"),
});
subtitle = task.status === "cancelled" ? "已取消" : "生成失败";
}
result = result.map((item, i) =>
i === index
? {
...item,
...(task.status === "succeeded" && task.request.kind !== "analysis"
? { kind: "media", inputType: task.request.kind }
: {}),
subtitle,
draftValues: nextDraft,
}
: item,
);
}
return result;
}

227
lib/video-canvas/task-store.ts Normal file
View File

@ -0,0 +1,227 @@
import "server-only";
import { and, asc, desc, eq, lt, sql } from "drizzle-orm";
import { db } from "@/lib/db";
import { canvasCapabilityTask as table, canvasProject } from "@/lib/db/schema";
import { type CanvasTaskRequest, type CanvasGenerationTask } from "./task-contract";
const capabilityId = "video-canvas.generation";
const scope = (userId: string, projectId: string) =>
and(
eq(table.userId, userId),
eq(table.projectId, projectId),
eq(table.capabilityId, capabilityId),
);
const serialize = (row: typeof table.$inferSelect): CanvasGenerationTask => ({
id: row.id,
requestId: row.requestId,
status: row.status,
createdAt: row.createdAt.toISOString(),
request: row.request as CanvasTaskRequest,
result: row.result as Record<string, any> | null,
error: row.error,
});
export async function ownsTaskProject(userId: string, projectId: string) {
return Boolean(
(
await db
.select({ id: canvasProject.id })
.from(canvasProject)
.where(and(eq(canvasProject.id, projectId), eq(canvasProject.userId, userId)))
.limit(1)
)[0],
);
}
export async function createCanvasGenerationTask(userId: string, request: CanvasTaskRequest) {
return db.transaction(async (tx) => {
await tx.execute(
sql`select pg_advisory_xact_lock(hashtext(${capabilityId}), hashtext(${request.projectId}))`,
);
const existing = (
await tx
.select()
.from(table)
.where(and(scope(userId, request.projectId), eq(table.requestId, request.requestId)))
.limit(1)
)[0];
if (existing) {
if (
JSON.stringify(existing.request) !== JSON.stringify(JSON.parse(JSON.stringify(request)))
) {
// JSONB property order is not stable. Compare the immutable logical request.
const old = existing.request as CanvasTaskRequest;
if (
old.node.id !== request.node.id ||
old.kind !== request.kind ||
JSON.stringify(sortObject(old.params)) !== JSON.stringify(sortObject(request.params))
)
throw Object.assign(Error("同一任务 ID 不能用于不同请求"), { status: 409 });
}
return serialize(existing);
}
const active = await tx
.select()
.from(table)
.where(and(scope(userId, request.projectId), sql`${table.status} in ('pending','running')`));
const duplicate = active.find(
(row) => (row.request as CanvasTaskRequest).node.id === request.node.id,
);
if (duplicate) return serialize(duplicate);
if (active.length >= 40)
throw Object.assign(Error("当前画布排队任务已达 40 个，请等待完成后继续"), { status: 429 });
const [created] = await tx
.insert(table)
.values({
requestId: request.requestId,
userId,
projectId: request.projectId,
capabilityId,
source: "video-canvas",
request,
createdAt: new Date(),
updatedAt: new Date(),
})
.returning();
return serialize(created);
});
}
function sortObject(value: any): any {
return Array.isArray(value)
? value.map(sortObject)
: value && typeof value === "object"
? Object.fromEntries(
Object.keys(value)
.sort()
.map((key) => [key, sortObject(value[key])]),
)
: value;
}
export async function listCanvasGenerationTasks(userId: string, projectId: string) {
// A worker lost to process shutdown must never silently re-submit a paid request.
await db
.update(table)
.set({
status: "interrupted",
error:
"服务端执行连接中断，供应商任务可能仍在运行。请核实结果后再决定是否重新生成，系统不会自动重复提交。",
updatedAt: new Date(),
})
.where(
and(
scope(userId, projectId),
eq(table.status, "running"),
lt(table.updatedAt, new Date(Date.now() - 120000)),
),
);
const rows = await db
.select()
.from(table)
.where(scope(userId, projectId))
.orderBy(desc(table.createdAt))
.limit(240);
return rows.map(serialize).reverse();
}
export async function claimCanvasGenerationTask(userId: string, projectId: string) {
return db.transaction(async (tx) => {
await tx.execute(
sql`select pg_advisory_xact_lock(hashtext(${capabilityId}), hashtext(${projectId}))`,
);
if (
(
await tx
.select({ id: table.id })
.from(table)
.where(and(scope(userId, projectId), eq(table.status, "running")))
.limit(1)
).length
)
return null;
const [next] = await tx
.select()
.from(table)
.where(and(scope(userId, projectId), eq(table.status, "pending")))
.orderBy(asc(table.createdAt))
.limit(1);
if (!next) return null;
const [claimed] = await tx
.update(table)
.set({
status: "running",
startedAt: new Date(),
updatedAt: new Date(),
attempts: sql`${table.attempts}+1`,
})
.where(and(eq(table.id, next.id), eq(table.status, "pending")))
.returning();
return claimed ? serialize(claimed) : null;
});
}
export async function heartbeatCanvasGenerationTask(id: string) {
await db
.update(table)
.set({ updatedAt: new Date() })
.where(and(eq(table.id, id), eq(table.status, "running")));
}
export async function finishCanvasGenerationTask(
id: string,
result?: Record<string, any>,
error?: string,
) {
// A late real result can recover an interrupted task, but cannot replace a newer request ID.
await db
.update(table)
.set({
status: result ? "succeeded" : "failed",
result: result || null,
error: error || null,
updatedAt: new Date(),
completedAt: new Date(),
})
.where(and(eq(table.id, id), sql`${table.status} in ('running','interrupted')`));
}
export async function cancelCanvasGenerationTask(userId: string, projectId: string, id: string) {
const [cancelled] = await db
.update(table)
.set({
status: "cancelled",
error: "已取消排队，未提交供应商",
updatedAt: new Date(),
completedAt: new Date(),
})
.where(and(scope(userId, projectId), eq(table.id, id), eq(table.status, "pending")))
.returning();
if (cancelled) return { ok: true, task: serialize(cancelled) };
const [current] = await db
.select()
.from(table)
.where(and(scope(userId, projectId), eq(table.id, id)))
.limit(1);
return { ok: current?.status === "cancelled", task: current ? serialize(current) : null };
}
export async function hideCanvasGenerationNode(
userId: string,
projectId: string,
id: string,
nodeId: string,
) {
return db.transaction(async (tx) => {
const [task] = await tx
.select()
.from(table)
.where(and(scope(userId, projectId), eq(table.id, id)))
.for("update");
if (!task) return false;
const request = task.request as CanvasTaskRequest & { hiddenNodeIds?: string[] };
if (nodeId !== request.node.id && !nodeId.startsWith(`${request.node.id}-result-`))
return false;
await tx
.update(table)
.set({
request: {
...request,
hiddenNodeIds: [...new Set([...(request.hiddenNodeIds || []), nodeId])],
},
})
.where(eq(table.id, id));
return true;
});
}

78
lib/video-canvas/task-worker.ts Normal file
View File

@ -0,0 +1,78 @@
import "server-only";
import { executeDirectVideoGeneration } from "@/lib/ai/tools/generate-video";
import { executeCanvasImageResponse } from "@/lib/canvas/generate-image-response";
import { executeWorkflowAnalysisResponse } from "./workflow-analysis-response";
import { requireAnalysisResult, type CanvasTaskRequest } from "./task-contract";
import { validateVideoReferences } from "./reference-input";
import { probeRemoteUrl } from "@/lib/net/url-access";
import {
claimCanvasGenerationTask,
finishCanvasGenerationTask,
heartbeatCanvasGenerationTask,
} from "./task-store";
async function execute(request: CanvasTaskRequest, userId: string) {
const params = request.params;
if (request.kind === "video") {
const checked = await validateVideoReferences(params.images, (url) =>
probeRemoteUrl(url, { timeoutMs: 8000 }),
);
if (checked.error) throw Error(checked.error);
const result = await executeDirectVideoGeneration({
...params,
prompt: String(params.prompt || ""),
mode: params.mode as any,
images: checked.images,
});
if (!result.success || !result.videoUrl) throw Error("视频生成未返回结果");
return {
url: result.videoUrl,
metadata: {
...result.metadata,
temporaryVideoUrl: result.videoUrl,
storageStatus:
result.metadata?.permanentVideoUrl === result.videoUrl ? "archived" : "temporary",
},
modelId: result.modelId,
modelRef: result.modelRef,
};
}
const response =
request.kind === "image"
? await executeCanvasImageResponse(
{ ...params, prompt: String(params.prompt || ""), async: false },
userId,
)
: await executeWorkflowAnalysisResponse(params);
const body = await response.json();
if (!response.ok || !body.success) throw Error(body.error || "任务执行失败");
if (
request.kind === "image" &&
!body.data?.images?.some((image: any) => image.originalUrl || image.thumbnailUrl)
)
throw Error("图片生成没有返回结果");
if (request.kind === "analysis") requireAnalysisResult(body.data, String(params.mode));
return body.data;
}
export async function runCanvasGenerationQueue(userId: string, projectId: string) {
// Work is independent of the browser request. Atomic DB claiming also covers multiple processes.
for (let i = 0; i < 40; i++) {
const task = await claimCanvasGenerationTask(userId, projectId);
if (!task) return;
const timer = setInterval(
() => void heartbeatCanvasGenerationTask(task.id).catch(() => {}),
20000,
);
try {
await finishCanvasGenerationTask(task.id, await execute(task.request, userId));
} catch (error) {
await finishCanvasGenerationTask(
task.id,
undefined,
error instanceof Error ? error.message : "任务执行失败",
);
} finally {
clearInterval(timer);
}
}
}

797
lib/video-canvas/workflow-analysis-response.ts Normal file
View File

@ -0,0 +1,797 @@
import "server-only";
import { normalizeStoryScriptRows } from "@/lib/video-canvas/story-script";
import { isCanvasLanguageModel, isCanvasVisionModel } from "@/lib/video-canvas/model-matching";
import { generateText } from "ai";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import { mkdtemp, readFile, rm } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { NextResponse } from "next/server";
import sharp from "sharp";
import { z } from "zod";
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
import { inlineVisionImages } from "@/lib/ai/vision-inline";
import { getAiModelAgentCodingModelId, isAiModelMultimodal } from "@/lib/db/ai-model-flags";
import { getAgentAiModelWithProvider, getEnabledAiModelsWithProvider } from "@/lib/db/queries";
const execFileAsync = promisify(execFile);
const VIDEO_ANALYSIS_FRAME_COUNT = 4;
const VIDEO_ANALYSIS_MAX_SIDE = 960;
const VIDEO_ANALYSIS_TIMEOUT_MS = 15000;
const requestSchema = z.object({
mode: z.enum(["video-analysis", "character-scene-extraction", "story-script", "agent-plan"]),
prompt: z.string().trim().optional(),
imageUrls: z.array(z.string().trim()).optional(),
videoUrls: z.array(z.string().trim()).optional(),
nodeTitle: z.string().trim().optional(),
preferredModelRef: z.string().trim().optional(),
model: z.string().trim().optional(),
});
function sanitizeUrls(value: unknown, limit = 6) {
if (!Array.isArray(value)) return [];
return value
.map((item) => String(item || "").trim())
.filter((url) => /^https?:\/\//i.test(url))
.slice(0, limit);
}
function supportsVision(model: {
supportsVision?: boolean | null;
capabilities?: unknown;
defaultParams?: unknown;
}) {
return (
Boolean(model.supportsVision) ||
isAiModelMultimodal({
capabilities: model.capabilities,
defaultParams: model.defaultParams,
})
);
}
function extractJsonBlock<T = Record<string, unknown>>(text: string): T | null {
const matched = String(text || "").match(/\{[\s\S]*\}/);
if (!matched) return null;
try {
return JSON.parse(matched[0]) as T;
} catch {
return null;
}
}
function normalizeText(value: unknown) {
return String(value || "").trim();
}
function normalizeStringList(value: unknown, limit = 8) {
if (!Array.isArray(value)) return [];
return Array.from(
new Set(
value
.map((item) => String(item || "").trim())
.filter(Boolean)
.slice(0, limit),
),
);
}
function normalizeShotIdeas(value: unknown) {
if (!Array.isArray(value)) return [];
return value
.map((item, index) => {
const raw =
item && typeof item === "object" && !Array.isArray(item)
? (item as Record<string, unknown>)
: {};
const title = normalizeText(raw.title || `镜头 ${index + 1}`);
const description = normalizeText(raw.description || raw.prompt || raw.summary);
if (!title && !description) return null;
return {
title: title || `镜头 ${index + 1}`,
description,
};
})
.filter((item): item is { title: string; description: string } => Boolean(item))
.slice(0, 6);
}
function normalizeVideoAnalysisRows(value: unknown, framePreviews: string[]) {
if (!Array.isArray(value)) return [];
return value
.map((item, index) => {
const raw =
item && typeof item === "object" && !Array.isArray(item)
? (item as Record<string, unknown>)
: {};
const sceneDescription = normalizeText(
raw.sceneDescription || raw.scene || raw.title || raw.description,
);
const narrative = normalizeText(
raw.narrative || raw.story || raw.storyContent || raw.narration || raw.summary,
);
const shot = normalizeText(raw.shot || raw.shotNumber || raw.index || index + 1);
const startTime = normalizeText(raw.startTime || raw.start || raw.start_time);
const endTime = normalizeText(raw.endTime || raw.end || raw.end_time);
const duration = normalizeText(raw.duration || raw.durationSeconds || raw.duration_seconds);
const shotSize = normalizeText(
raw.shotSize || raw.frameSize || raw.shot_type || raw.sceneScale,
);
const cameraAngle = normalizeText(raw.cameraAngle || raw.angle || raw.camera_angle);
const cameraMovement = normalizeText(
raw.cameraMovement || raw.cameraMove || raw.camera_motion || raw.motionType,
);
const focusDepth = normalizeText(
raw.focusDepth || raw.depthOfField || raw.focalDepth || raw.focus_depth,
);
const lighting = normalizeText(raw.lighting || raw.light || raw.lightDesign);
const backgroundMusic = normalizeText(raw.backgroundMusic || raw.music || raw.bgm);
const voiceAndSfx = normalizeText(raw.voiceAndSfx || raw.sound || raw.audio || raw.sfx);
const imagePrompt = normalizeText(
raw.imagePrompt || raw.image_generation_prompt || raw.image,
);
const motionPrompt = normalizeText(
raw.motionPrompt || raw.motion || raw.videoPrompt || raw.video_motion_prompt,
);
const keyframeIndex = Number(
raw.keyframeIndex || raw.frameIndex || raw.key_frame_index || index + 1,
);
if (
!sceneDescription &&
!narrative &&
!backgroundMusic &&
!voiceAndSfx &&
!imagePrompt &&
!motionPrompt &&
!shotSize &&
!cameraAngle &&
!cameraMovement &&
!focusDepth &&
!lighting
) {
return null;
}
const normalizedFrameIndex =
Number.isFinite(keyframeIndex) && keyframeIndex > 0
? Math.max(0, Math.min(framePreviews.length - 1, Math.round(keyframeIndex) - 1))
: index % Math.max(framePreviews.length, 1);
return {
shot: shot || String(index + 1),
startTime: startTime || (index === 0 ? "0" : ""),
endTime,
duration,
sceneDescription: sceneDescription || `镜头 ${index + 1}`,
narrative: narrative || sceneDescription || `镜头 ${index + 1}`,
shotSize: shotSize || "中景至近景",
cameraAngle: cameraAngle || "平视",
cameraMovement: cameraMovement || "缓慢推镜头 (Slow Zoom In)",
focusDepth: focusDepth || "浅景深",
lighting: lighting || "冷调低沉照明，主体光线柔和。",
backgroundMusic: backgroundMusic || "无",
voiceAndSfx: voiceAndSfx || "无",
imagePrompt: imagePrompt || sceneDescription || "延续当前视频气质生成关键帧",
motionPrompt: motionPrompt || "保持镜头运动自然、连贯、符合前后节奏。",
keyframeUrl: framePreviews[normalizedFrameIndex] || "",
};
})
.filter(
(
item,
): item is {
shot: string;
startTime: string;
endTime: string;
duration: string;
sceneDescription: string;
narrative: string;
shotSize: string;
cameraAngle: string;
cameraMovement: string;
focusDepth: string;
lighting: string;
backgroundMusic: string;
voiceAndSfx: string;
imagePrompt: string;
motionPrompt: string;
keyframeUrl: string;
} => Boolean(item),
)
.slice(0, 8);
}
function normalizeCharacters(value: unknown) {
if (!Array.isArray(value)) return [];
return value
.map((item) => {
const raw =
item && typeof item === "object" && !Array.isArray(item)
? (item as Record<string, unknown>)
: {};
const name = normalizeText(raw.name || raw.title);
const description = normalizeText(raw.description || raw.summary);
const prompt = normalizeText(raw.prompt);
if (!name && !description && !prompt) return null;
return {
name: name || "未命名角色",
description,
prompt,
look: normalizeText(raw.look) || undefined,
wardrobe: normalizeText(raw.wardrobe) || undefined,
signatureAction: normalizeText(raw.signatureAction) || undefined,
traits: normalizeStringList(raw.traits, 6),
};
})
.filter((item): item is NonNullable<typeof item> => Boolean(item))
.slice(0, 8);
}
function normalizeScenes(value: unknown) {
if (!Array.isArray(value)) return [];
return value
.map((item) => {
const raw =
item && typeof item === "object" && !Array.isArray(item)
? (item as Record<string, unknown>)
: {};
const name = normalizeText(raw.name || raw.title);
const description = normalizeText(raw.description || raw.summary);
const prompt = normalizeText(raw.prompt);
if (!name && !description && !prompt) return null;
return {
name: name || "未命名场景",
description,
prompt,
mood: normalizeText(raw.mood) || undefined,
lighting: normalizeText(raw.lighting) || undefined,
props: normalizeStringList(raw.props, 6),
};
})
.filter((item): item is NonNullable<typeof item> => Boolean(item))
.slice(0, 8);
}
function buildVideoAnalysisPrompt(params: {
nodeTitle?: string;
prompt: string;
frameCount: number;
imageCount: number;
videoCount: number;
}) {
return [
"你是视频画布工作流里的视频分析节点，需要为后续分镜和生成节点提供结构化输出。",
params.nodeTitle ? `当前节点：${params.nodeTitle}` : "",
`上游参考图数量：${params.imageCount}`,
`上游视频数量：${params.videoCount}`,
`已提供按时间顺序抽取的视频关键帧数量：${params.frameCount}`,
params.prompt
? `上游脚本/说明：\n${params.prompt}`
: "上游没有额外脚本，请重点依据参考画面分析。",
"",
"请只输出 JSON，不要输出 markdown，不要输出解释。",
"JSON 结构如下：",
`{
"summary": "一句话总结当前视频/镜头风格，不超过120字",
"highlights": ["3-5条镜头分析要点"],
"analysisRows": [
{
"shot": "1",
"startTime": "0",
"endTime": "5",
"duration": "5",
"sceneDescription": "画面描述，说明主体、环境、动作、构图和色调",
"narrative": "叙事内容，说明这一镜头承担的剧情/情绪/信息作用",
"shotSize": "景别，例如中景至近景",
"cameraAngle": "摄影机角度，例如平视",
"cameraMovement": "摄影机运动，例如缓慢推镜头 (Slow Zoom In)",
"focusDepth": "焦距与景深，例如浅景深",
"lighting": "光线，例如冷调低沉照明，屏幕光作为侧逆光",
"backgroundMusic": "这一段适合的背景音乐氛围",
"voiceAndSfx": "人声/环境声/音效建议，没有就写无",
"imagePrompt": "后续生成关键帧时可直接使用的英文图像提示词",
"motionPrompt": "后续生成视频时可直接使用的英文运动提示词",
"keyframeIndex": 1
}
],
"shotIdeas": [
{ "title": "镜头 1", "description": "一句话描述镜头目标、景别或动作" }
],
"visualMotifs": ["2-4个可复用的视觉母题"],
"recommendedNextSteps": ["2-4条建议的下游动作"]
}`,
"analysisRows 请按镜头时间顺序输出；短视频可以输出 1 行，复杂视频输出 2-6 行。",
"每行必须覆盖镜号、开始时间、结束时间、时长、画面描述、叙事内容、景别、摄影机角度、摄影机运动、焦距与景深、光线、背景音乐、人声/音效、图像生成提示词、视频运动提示词和关键帧。",
"imagePrompt 和 motionPrompt 优先使用英文；其他字段使用中文；keyframeIndex 使用 1 开始的序号，对应你看到的关键帧顺序。",
"如果没有真实视频帧，只能依据参考图和脚本推断，也要正常输出，但措辞要稳健。",
]
.filter(Boolean)
.join("\n");
}
async function getVideoDurationSeconds(videoUrl: string) {
try {
const { stdout } = await execFileAsync(
"ffprobe",
[
"-v",
"error",
"-show_entries",
"format=duration",
"-of",
"default=noprint_wrappers=1:nokey=1",
videoUrl,
],
{
timeout: VIDEO_ANALYSIS_TIMEOUT_MS,
maxBuffer: 1024 * 1024,
},
);
const duration = Number.parseFloat(String(stdout || "").trim());
return Number.isFinite(duration) && duration > 0 ? duration : null;
} catch {
return null;
}
}
function buildVideoFrameTimestamps(durationSeconds: number | null, frameCount: number) {
const safeCount = Math.max(1, frameCount);
if (!durationSeconds || !Number.isFinite(durationSeconds) || durationSeconds <= 0.5) {
return Array.from({ length: safeCount }).map((_, index) =>
Number((0.3 + index * 0.45).toFixed(2)),
);
}
const start = Math.min(0.35, durationSeconds * 0.12);
const end = Math.max(start + 0.6, durationSeconds * 0.88);
if (safeCount === 1) return [Number(((start + end) / 2).toFixed(2))];
const step = (end - start) / (safeCount - 1);
return Array.from({ length: safeCount }).map((_, index) =>
Number((start + step * index).toFixed(2)),
);
}
async function downscaleVideoFrameForVision(inputBuffer: Buffer) {
return sharp(inputBuffer, { failOn: "none" })
.rotate()
.resize({
width: VIDEO_ANALYSIS_MAX_SIDE,
height: VIDEO_ANALYSIS_MAX_SIDE,
fit: "inside",
withoutEnlargement: true,
})
.jpeg({ quality: 82, mozjpeg: true })
.toBuffer();
}
async function extractVideoFrameDataUrls(
videoUrl: string,
frameCount = VIDEO_ANALYSIS_FRAME_COUNT,
) {
const tempDir = await mkdtemp(path.join(os.tmpdir(), "canvas-video-analysis-"));
try {
const durationSeconds = await getVideoDurationSeconds(videoUrl);
const timestamps = buildVideoFrameTimestamps(durationSeconds, frameCount);
const frames = await Promise.all(
timestamps.map(async (timestamp, index) => {
const outputPath = path.join(tempDir, `frame-${index + 1}.jpg`);
await execFileAsync(
"ffmpeg",
[
"-hide_banner",
"-loglevel",
"error",
"-ss",
String(timestamp),
"-i",
videoUrl,
"-frames:v",
"1",
"-q:v",
"4",
"-y",
outputPath,
],
{
timeout: VIDEO_ANALYSIS_TIMEOUT_MS,
maxBuffer: 2 * 1024 * 1024,
},
);
const buffer = await readFile(outputPath);
const processed = await downscaleVideoFrameForVision(buffer);
return `data:image/jpeg;base64,${processed.toString("base64")}`;
}),
);
return frames.filter(Boolean);
} catch (error) {
console.warn("[Canvas Workflow Analyze] extract video frames failed:", {
message: error instanceof Error ? error.message : String(error),
videoUrl,
});
return [];
} finally {
await rm(tempDir, { force: true, recursive: true }).catch(() => undefined);
}
}
function buildCharacterScenePrompt(params: {
nodeTitle?: string;
prompt: string;
imageCount: number;
}) {
return [
"你是视频画布工作流里的角色场景提取节点，需要从脚本、设定和参考图里提取角色与场景资产。",
params.nodeTitle ? `当前节点：${params.nodeTitle}` : "",
`参考图数量：${params.imageCount}`,
params.prompt
? `输入内容：\n${params.prompt}`
: "没有额外脚本，请根据参考图提取角色和场景线索。",
"",
"请只输出 JSON，不要输出 markdown，不要输出解释。",
"JSON 结构如下：",
`{
"summary": "一句话总结当前故事世界与资产线索",
"characters": [
{
"name": "角色名",
"description": "角色外观与身份简介",
"look": "外貌特征",
"wardrobe": "服装/道具",
"traits": ["性格关键词"],
"signatureAction": "代表动作",
"prompt": "可直接用于生成角色图/角色视频的提示词"
}
],
"scenes": [
{
"name": "场景名",
"description": "空间、布景和氛围简介",
"mood": "情绪与气氛",
"lighting": "布光特征",
"props": ["核心道具"],
"prompt": "可直接用于生成场景图/场景视频的提示词"
}
],
"relationships": ["角色与场景或角色之间的关系"],
"recommendedNextSteps": ["建议连接的后续节点动作"]
}`,
"角色和场景都要尽量具体，避免空泛形容词。",
]
.filter(Boolean)
.join("\n");
}
function buildAgentPlanPrompt(params: {
nodeTitle?: string;
prompt: string;
imageCount: number;
videoCount: number;
}) {
return [
"你是视频画布里的自定义代理节点，需要把当前创作目标拆成可执行的工作流计划。",
params.nodeTitle ? `当前节点：${params.nodeTitle}` : "",
`参考图数量：${params.imageCount}`,
`视频数量：${params.videoCount}`,
params.prompt ? `上游需求：\n${params.prompt}` : "没有额外脚本，请基于当前上下文给出稳妥计划。",
"",
"请只输出 JSON，不要输出 markdown，不要输出解释。",
"JSON 结构如下：",
`{
"summary": "一句话说明这个代理链的目标",
"agentPlan": ["3-6个执行步骤"],
"deliverables": ["2-4个预期交付物"],
"recommendedNodes": ["建议继续连接的节点类型或能力名"]
}`,
"步骤要偏工作流视角，而不是泛泛而谈。",
]
.filter(Boolean)
.join("\n");
}
function buildStoryScriptPrompt(params: {
nodeTitle?: string;
prompt: string;
imageCount: number;
videoCount: number;
}) {
return [
"你是视频画布里的故事脚本生成节点，需要把用户输入、参考图和参考视频整理成可直接用于后续分镜/生图/生视频的结构化脚本表。",
params.nodeTitle ? `当前节点：${params.nodeTitle}` : "",
`参考图数量：${params.imageCount}`,
`视频数量：${params.videoCount}`,
params.prompt
? `输入内容：\n${params.prompt}`
: "没有额外脚本，请生成一个节奏清晰的短视频脚本。",
"",
"请只输出 JSON，不要输出 markdown，不要输出解释。",
"JSON 结构如下：",
`{
"summary": "一句话概括故事主题、情绪和视频方向，不超过120字",
"analysisRows": [
{
"shot": "01",
"duration": "5s",
"scene": "场景名或地点，如 盛唐·金銮殿·日",
"sceneDescription": "画面内容、人物动作和剧情推进，不要重复 scene 字段",
"shotSize": "景别，如中景、特写、全景、大远景",
"lighting": "光影氛围，如暖金主光带阴影，气氛紧张压抑",
"dialogue": "对白或旁白，没有就写 +",
"backgroundMusic": "背景音乐或情绪氛围，没有就写无",
"voiceAndSfx": "环境声、动作声或音效，没有就写无",
"cameraMovement": "运镜，如缓慢推镜头、固定、横移、上摇、环绕轨道跟拍",
"imagePrompt": "后续生成该镜头关键帧时可直接使用的中文图像提示词",
"motionPrompt": "后续生成视频时可直接使用的中文视频运动提示词"
}
],
"assetCandidates": {
"style": "全局视觉风格，如 盛唐·国风二次元",
"characters": ["需要生成的角色资产名"],
"scenes": ["需要生成的场景资产名"],
"props": ["需要生成的道具资产名"]
},
"shotIdeas": [
{ "title": "镜头 1", "description": "一句话描述镜头目标、景别或动作" }
],
"recommendedNextSteps": ["2-4条建议的下游动作"]
}`,
"优先严格遵守用户指定的镜头数量和总时长；未指定时生成 6 个镜头，最多 18 个。镜头时长之和必须符合用户要求，8 秒短片不得扩写成几十秒。只生成实际需要的镜头，不重复凑数。",
"每行要覆盖竞品脚本表同等信息：镜号、时长、画面描述、景别、光影氛围、对白·旁白、音效、运镜、最终图像提示词和视频运动提示词。",
"画面描述中用 @角色名 或 @场景/道具名 标出可复用资产，例如 @沈昭昭、@金銮殿、@侍卫长刀。",
"imagePrompt 要像可直接生成分镜图的中文提示词，包含主体、环境、构图、光影、风格与一致性要求；motionPrompt 要按起始状态、动作过程、结束状态、音效组织。",
"assetCandidates 只提取脚本中确实出现的角色、场景和道具；产品视频没有人物时 characters 必须为空，禁止为凑数虚构角色。",
"每行要像专业视频画布脚本表：场景具体、画面可生成、声音可执行、镜头节奏明确，不要只写剧情摘要。",
]
.filter(Boolean)
.join("\n");
}
export async function executeWorkflowAnalysisResponse(body: unknown, signal?: AbortSignal) {
try {
const parsed = requestSchema.safeParse(body);
if (!parsed.success) {
return NextResponse.json({ success: false, error: "请求参数不合法" }, { status: 400 });
}
const prompt = normalizeText(parsed.data.prompt);
const imageUrls = sanitizeUrls(parsed.data.imageUrls, 4);
const videoUrls = sanitizeUrls(parsed.data.videoUrls, 4);
const extractedVideoFrames =
parsed.data.mode === "video-analysis" && videoUrls.length > 0
? await extractVideoFrameDataUrls(videoUrls[0], VIDEO_ANALYSIS_FRAME_COUNT)
: [];
const preferredModelRef = normalizeText(parsed.data.preferredModelRef || parsed.data.model);
if (!prompt && imageUrls.length === 0 && videoUrls.length === 0) {
return NextResponse.json(
{ success: false, error: "请至少提供脚本、图片或视频上下文" },
{ status: 400 },
);
}
const [allModels, agentModel] = await Promise.all([
getEnabledAiModelsWithProvider(false),
getAgentAiModelWithProvider(),
]);
const hasVisionInputs = imageUrls.length > 0 || extractedVideoFrames.length > 0;
const compatibleModels = allModels.filter((item) => {
const descriptor = { ...item.model, providerType: item.provider.type };
return (
!item.model.isSubModel &&
item.model.isEnabled &&
(hasVisionInputs ? isCanvasVisionModel(descriptor) : isCanvasLanguageModel(descriptor))
);
});
const requestedRef = preferredModelRef === "auto" ? "" : preferredModelRef;
const preferredModel = requestedRef
? compatibleModels.find(
(item) => item.model.id === requestedRef || item.model.modelId === requestedRef,
)
: undefined;
if (requestedRef && !preferredModel) {
return NextResponse.json(
{ success: false, error: "所选模型不可用或不支持当前分析输入，请重新选择分析模型" },
{ status: 400 },
);
}
const selectedModel =
preferredModel ||
compatibleModels.find((item) => item.model.id === agentModel?.model.id) ||
compatibleModels[0];
if (!selectedModel) {
return NextResponse.json(
{
success: false,
error: hasVisionInputs ? "当前没有可用的视觉理解模型" : "当前没有可用的文字分析模型",
},
{ status: 400 },
);
}
const useVision = hasVisionInputs && supportsVision(selectedModel.model);
const codingModelId = getAiModelAgentCodingModelId(selectedModel.model);
const model = await getLanguageModelById(selectedModel.model.id, codingModelId);
if (!model) {
return NextResponse.json({ success: false, error: "分析模型初始化失败" }, { status: 500 });
}
const promptText =
parsed.data.mode === "video-analysis"
? buildVideoAnalysisPrompt({
frameCount: extractedVideoFrames.length,
nodeTitle: normalizeText(parsed.data.nodeTitle),
prompt,
imageCount: imageUrls.length,
videoCount: videoUrls.length,
})
: parsed.data.mode === "character-scene-extraction"
? buildCharacterScenePrompt({
nodeTitle: normalizeText(parsed.data.nodeTitle),
prompt,
imageCount: imageUrls.length,
})
: parsed.data.mode === "story-script"
? buildStoryScriptPrompt({
nodeTitle: normalizeText(parsed.data.nodeTitle),
prompt,
imageCount: imageUrls.length,
videoCount: videoUrls.length,
})
: buildAgentPlanPrompt({
nodeTitle: normalizeText(parsed.data.nodeTitle),
prompt,
imageCount: imageUrls.length,
videoCount: videoUrls.length,
});
const inlinedImageReferences = useVision
? (
await inlineVisionImages(imageUrls, {
maxImages: 4,
maxSide: 960,
timeoutMs: 12000,
retries: 1,
loggerPrefix: "[Canvas Workflow Analyze]",
})
).inlineDataUrls
: [];
const referenceImages = useVision
? [...inlinedImageReferences, ...extractedVideoFrames].slice(0, 6)
: [];
const { text, finishReason } = await generateText({
model,
messages: [
{
role: "user",
content: [
{ type: "text", text: promptText },
...referenceImages.map((image) => ({ type: "image" as const, image })),
],
},
],
temperature: 0.2,
maxOutputTokens:
parsed.data.mode === "story-script"
? 12000
: parsed.data.mode === "video-analysis"
? 2200
: 1600,
maxRetries: 0,
abortSignal: signal
? AbortSignal.any([signal, AbortSignal.timeout(150_000)])
: AbortSignal.timeout(150_000),
});
if (finishReason === "length" || !text.trim()) {
return NextResponse.json(
{ success: false, error: "分析结果不完整，请缩短输入或重新生成" },
{ status: 502 },
);
}
const parsedJson = extractJsonBlock<Record<string, unknown>>(text);
if (!parsedJson) {
return NextResponse.json(
{ success: false, error: "分析模型未返回完整结构化结果，请重新生成" },
{ status: 502 },
);
}
const summary = normalizeText(parsedJson.summary) || normalizeText(text).slice(0, 160);
if (parsed.data.mode === "video-analysis") {
const analysisRows = normalizeVideoAnalysisRows(
parsedJson.analysisRows,
extractedVideoFrames,
);
const shotIdeas =
normalizeShotIdeas(parsedJson.shotIdeas).length > 0
? normalizeShotIdeas(parsedJson.shotIdeas)
: analysisRows.map((row, index) => ({
title: `镜头 ${index + 1}`,
description: row.sceneDescription,
}));
return NextResponse.json({
success: true,
data: {
summary,
analysisRows,
framePreviews: extractedVideoFrames,
highlights: normalizeStringList(parsedJson.highlights, 6),
shotIdeas,
visualMotifs: normalizeStringList(parsedJson.visualMotifs, 6),
recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6),
model: selectedModel.model.displayName,
raw: text,
},
});
}
if (parsed.data.mode === "character-scene-extraction") {
return NextResponse.json({
success: true,
data: {
summary,
characters: normalizeCharacters(parsedJson.characters),
scenes: normalizeScenes(parsedJson.scenes),
relationships: normalizeStringList(parsedJson.relationships, 8),
recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6),
model: selectedModel.model.displayName,
raw: text,
},
});
}
if (parsed.data.mode === "story-script") {
const analysisRows = normalizeStoryScriptRows(
parsedJson.analysisRows || parsedJson.storyRows || parsedJson.rows || parsedJson.shots,
);
if (analysisRows.length === 0) {
return NextResponse.json(
{ success: false, error: "分析模型没有返回有效镜头，请重新生成" },
{ status: 502 },
);
}
return NextResponse.json({
success: true,
data: {
summary,
analysisRows,
shotIdeas:
normalizeShotIdeas(parsedJson.shotIdeas).length > 0
? normalizeShotIdeas(parsedJson.shotIdeas)
: analysisRows.map((row, index) => ({
title: `镜头 ${index + 1}`,
description: row.sceneDescription,
})),
recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6),
model: selectedModel.model.displayName,
raw: text,
},
});
}
return NextResponse.json({
success: true,
data: {
summary,
agentPlan: normalizeStringList(parsedJson.agentPlan, 8),
deliverables: normalizeStringList(parsedJson.deliverables, 6),
recommendedNodes: normalizeStringList(parsedJson.recommendedNodes, 6),
model: selectedModel.model.displayName,
raw: text,
},
});
} catch (error) {
console.error("[Canvas Workflow Analyze] Error:", error);
const timedOut =
error instanceof Error && /timeout|timed out|abort/i.test(`${error.name} ${error.message}`);
return NextResponse.json(
{
success: false,
error: timedOut
? "分析超时，请稍后重试或选择其他分析模型"
: error instanceof Error
? error.message
: "工作流分析失败",
},
{ status: timedOut ? 504 : 500 },
);
}
}

1
playwright.helpers.config.ts
View File

@ -3,6 +3,7 @@ import { defineConfig } from "@playwright/test";
export default defineConfig({
testDir: "./tests",
testMatch: [
/routes\/video-canvas-tasks\.test\.ts/,
/routes\/video-canvas-director\.test\.ts/,
/routes\/aliyun-canvas-video\.test\.ts/,
/routes\/video-canvas-followup\.test\.ts/,

155
scripts/qa/video-canvas-task-db.cjs Normal file
View File

@ -0,0 +1,155 @@
// Local disposable PostgreSQL only. Install embedded-postgres into .qa/video-task-db first.
const fs = require("fs"),
path = require("path"),
assert = require("node:assert/strict");
(async () => {
const { default: EmbeddedPostgres } = await import(
path.resolve(".qa/video-task-db/node_modules/embedded-postgres/dist/index.js")
);
const pg = new EmbeddedPostgres({
databaseDir: path.resolve(".qa/video-task-db/data"),
port: 55439,
user: "canvas_qa",
password: "local-disposable-only",
persistent: false,
onLog: (message) => console.log(message),
onError: (message) => console.error(message),
});
let client;
try {
await pg.initialise();
await pg.start();
client = pg.getPgClient();
await client.connect();
process.env.POSTGRES_URL =
"postgres://canvas_qa:local-disposable-only@127.0.0.1:55439/postgres";
await client.query(
'CREATE TABLE "User" (id uuid PRIMARY KEY); CREATE TABLE "CanvasProject" (id uuid PRIMARY KEY, "userId" uuid);',
);
await client.query(
fs.readFileSync("lib/db/migrations/0091_add_canvas_capability_tasks.sql", "utf8"),
);
const esbuild = require(
path.resolve(
"node_modules/.pnpm",
fs.readdirSync("node_modules/.pnpm").find((name) => name.startsWith("esbuild@0.25.")),
"node_modules/esbuild",
),
);
await esbuild.build({
entryPoints: ["lib/video-canvas/task-store.ts"],
outfile: ".qa/video-task-db/store.cjs",
bundle: true,
platform: "node",
format: "cjs",
packages: "external",
alias: { "@": process.cwd() },
plugins: [
{
name: "server-only",
setup(build) {
build.onResolve({ filter: /^server-only$/ }, () => ({
path: "empty",
namespace: "qa",
}));
build.onLoad({ filter: /.*/, namespace: "qa" }, () => ({ contents: "" }));
},
},
],
});
const store = require(path.resolve(".qa/video-task-db/store.cjs"));
const user = crypto.randomUUID(),
other = crypto.randomUUID(),
project = crypto.randomUUID();
await client.query('INSERT INTO "User" VALUES ($1),($2)', [user, other]);
await client.query('INSERT INTO "CanvasProject" VALUES ($1,$2)', [project, user]);
assert.equal(await store.ownsTaskProject(user, project), true);
assert.equal(await store.ownsTaskProject(other, project), false);
const request = {
requestId: crypto.randomUUID(),
projectId: project,
kind: "video",
params: { mode: "text-to-video", prompt: "QA" },
node: {
id: "node-a",
title: "QA",
kind: "media",
inputType: "video",
x: 0,
y: 0,
width: 300,
height: 200,
},
edges: [],
};
const copies = await Promise.all(
Array.from({ length: 10 }, () => store.createCanvasGenerationTask(user, request)),
);
assert.equal(new Set(copies.map((task) => task.id)).size, 1);
const task = copies[0];
await assert.rejects(
store.createCanvasGenerationTask(user, {
...request,
params: { ...request.params, prompt: "different" },
}),
);
const claims = await Promise.all(
Array.from({ length: 10 }, () => store.claimCanvasGenerationTask(user, project)),
);
assert.equal(claims.filter(Boolean).length, 1);
assert.equal((await store.cancelCanvasGenerationTask(user, project, task.id)).ok, false);
assert.equal((await store.cancelCanvasGenerationTask(other, project, task.id)).task, null);
const pending = await store.createCanvasGenerationTask(user, {
...request,
requestId: crypto.randomUUID(),
node: { ...request.node, id: "node-b" },
});
assert.equal(await store.claimCanvasGenerationTask(user, project), null);
assert.equal((await store.cancelCanvasGenerationTask(user, project, pending.id)).ok, true);
await store.finishCanvasGenerationTask(task.id, { url: "https://example.com/qa.mp4" });
assert.equal(
(await store.listCanvasGenerationTasks(user, project)).find((t) => t.id === task.id).result
.url,
"https://example.com/qa.mp4",
);
assert.equal((await store.listCanvasGenerationTasks(other, project)).length, 0);
assert.equal(await store.hideCanvasGenerationNode(user, project, task.id, "node-a"), true);
assert.deepEqual(
(await store.listCanvasGenerationTasks(user, project)).find((t) => t.id === task.id).request
.hiddenNodeIds,
["node-a"],
);
const stale = await store.createCanvasGenerationTask(user, {
...request,
requestId: crypto.randomUUID(),
node: { ...request.node, id: "node-c" },
});
await store.claimCanvasGenerationTask(user, project);
await client.query(
"UPDATE \"CanvasCapabilityTask\" SET \"updatedAt\"=(now() at time zone 'UTC')-interval '3 minutes' WHERE id=$1",
[stale.id],
);
assert.equal(
(await store.listCanvasGenerationTasks(user, project)).find((t) => t.id === stale.id).status,
"interrupted",
);
assert.equal(await store.claimCanvasGenerationTask(user, project), null);
await store.finishCanvasGenerationTask(stale.id, { url: "https://example.com/late.mp4" });
assert.equal(
(await store.listCanvasGenerationTasks(user, project)).find((t) => t.id === stale.id).status,
"succeeded",
);
console.log(
"PASS: real PostgreSQL ownership, idempotency (10 submits), exclusive claiming (10 workers), serial queue, cancellation CAS, durable results, hidden nodes, interrupted task without resubmission, late-result recovery.",
);
} finally {
if (client) await client.end();
await pg.stop();
}
})().then(
() => process.exit(0),
(error) => {
console.error(error);
process.exit(1);
},
);

223
tests/routes/video-canvas-tasks.test.ts Normal file
View File

@ -0,0 +1,223 @@
import { test, expect } from "@playwright/test";
import {
canvasTaskRequestSchema,
reconcileCanvasTasks,
requireAnalysisResult,
canCancelCanvasTask,
type CanvasGenerationTask,
} from "@/lib/video-canvas/task-contract";
import { chooseStoryVideoSettings } from "@/lib/video-canvas/model-matching";
type TaskNode = { id: string; draftValues?: Record<string, any>; [key: string]: any };
const node: TaskNode = {
id: "n1",
title: "镜头",
kind: "media",
inputType: "image",
x: 0,
y: 0,
width: 300,
height: 200,
draftValues: { rows: [{ sceneDescription: "旧内容" }] },
};
const request = {
requestId: "11111111-1111-4111-8111-111111111111",
projectId: "22222222-2222-4222-8222-222222222222",
kind: "image" as const,
node,
params: { prompt: "cup" },
edges: [],
};
const task: CanvasGenerationTask = {
id: "task1",
requestId: request.requestId,
status: "succeeded",
createdAt: "2026-09-17T01:01:00Z",
request: request as any,
result: {
images: [
{ originalUrl: "https://example.com/1.png" },
{ originalUrl: "https://example.com/2.png" },
],
},
};
test("restores accepted tasks missing from a pre-submit canvas snapshot, including every image", () => {
const result = reconcileCanvasTasks<TaskNode>([], [task], "2026-09-17T01:00:00Z");
expect(result.map((n) => n.id)).toEqual(["n1", "n1-result-task1-1"]);
expect(result[0].draftValues?.url).toHaveLength(2);
expect(reconcileCanvasTasks<TaskNode>(result, [task], "2026-09-17T01:00:00Z")).toBe(result);
});
test("does not resurrect nodes removed after task creation or hidden on the server", () => {
expect(reconcileCanvasTasks<TaskNode>([], [task], "2026-09-17T01:02:00Z")).toEqual([]);
expect(
reconcileCanvasTasks<TaskNode>(
[],
[{ ...task, request: { ...request, node: node as any, hiddenNodeIds: ["n1"] } }],
"2026-09-17T01:00:00Z",
),
).toEqual([]);
});
test("late result from an older request cannot overwrite a new generation on the same node", () => {
const current = {
...node,
draftValues: { taskRequestId: "new-request", mediaUrl: "new-result" },
};
expect(reconcileCanvasTasks<TaskNode>([current], [task], "2026-09-17T01:00:00Z")[0]).toBe(
current,
);
});
test("task failure retains the previous script instead of marking an empty result successful", () => {
const result = reconcileCanvasTasks<TaskNode>(
[node],
[{ ...task, status: "failed", error: "模型错误" }],
"2026-09-17T01:00:00Z",
)[0];
expect(result.draftValues).toMatchObject({
rows: [{ sceneDescription: "旧内容" }],
generationStatus: "failed",
generationError: "模型错误",
});
});
test("queue and running states show actual state without invented progress", () => {
const pending = reconcileCanvasTasks<TaskNode>(
[node],
[{ ...task, status: "pending" }],
"2026-09-17T01:00:00Z",
);
expect(pending[0].draftValues).toMatchObject({ generationProgress: 0, pendingLabel: "排队中" });
expect(
reconcileCanvasTasks<TaskNode>(
pending,
[{ ...task, status: "running" }],
"2026-09-17T01:00:00Z",
)[0].draftValues?.pendingLabel,
).toBe("生成中");
});
test("interrupted tasks accept a late real result while cancelled queued tasks stay visibly cancelled", () => {
const pending = reconcileCanvasTasks<TaskNode>(
[node],
[{ ...task, status: "interrupted", error: "待核实" }],
"2026-09-17T01:00:00Z",
);
expect(
reconcileCanvasTasks<TaskNode>(pending, [task], "2026-09-17T01:00:00Z")[0].draftValues
?.generationStatus,
).toBe("complete");
expect(
reconcileCanvasTasks<TaskNode>(
[node],
[{ ...task, status: "cancelled" }],
"2026-09-17T01:00:00Z",
)[0].subtitle,
).toBe("已取消");
expect(["pending", "running", "succeeded", "interrupted"].map(canCancelCanvasTask)).toEqual([
true,
false,
false,
false,
]);
});
test("analysis rejects success envelopes without meaningful output or script rows", () => {
for (const data of [
undefined,
{},
{ analysisRows: [] },
{ analysisRows: [{}], raw: "some summary" },
])
expect(() => requireAnalysisResult(data, "story-script")).toThrow();
expect(() => requireAnalysisResult({ raw: " " }, "agent-plan")).toThrow();
expect(
requireAnalysisResult({ analysisRows: [{ sceneDescription: "杯柄特写" }] }, "story-script"),
).toBeTruthy();
});
test("task snapshots require stable project/request identities and bounded geometry", () => {
expect(canvasTaskRequestSchema.safeParse(request).success).toBe(true);
for (const patch of [
{ requestId: "bad" },
{ projectId: "bad" },
{ node: { ...node, width: Infinity } },
{ node: { ...node, id: "" } },
])
expect(canvasTaskRequestSchema.safeParse({ ...request, ...patch }).success).toBe(false);
});
const model = {
id: "video-model",
modelId: "wan2.6-i2v-flash",
providerType: "qianwen-video",
capabilities: {
kind: "video",
supportedModes: ["first-frame"],
durationOptions: [5, 10],
supportedRatios: ["adaptive"],
supportedResolutions: ["720p"],
supportGenerateAudio: true,
},
};
test("story batches match the actual input mode and discrete provider parameters", () => {
expect(chooseStoryVideoSettings([model], true, 7)).toMatchObject({
mode: "first-frame",
duration: 5,
ratio: "adaptive",
resolution: "720p",
generateAudio: true,
});
expect(() => chooseStoryVideoSettings([model], false, 5)).toThrow("文生视频");
const text = {
...model,
id: "text-model",
providerType: "vip98k",
modelId: "veo",
capabilities: { ...model.capabilities, supportedModes: ["text-to-video"] },
};
expect(chooseStoryVideoSettings([model, text], false, 9, model.id)).toMatchObject({
mode: "text-to-video",
model: "text-model",
duration: 10,
});
});
test("unknown model specifications cannot silently invent 720p or five seconds", () => {
expect(() =>
chooseStoryVideoSettings(
[{ ...model, capabilities: { kind: "video", supportedModes: ["first-frame"] } }],
true,
5,
),
).toThrow("规格");
});
test("refresh recovers the newest accepted request when the saved node still points to an older result", () => {
const newer = {
...task,
id: "task2",
requestId: "new-request",
createdAt: "2026-09-17T01:03:00Z",
result: { images: [{ originalUrl: "https://example.com/new.png" }] },
};
const old = { ...node, draftValues: { taskRequestId: task.requestId, mediaUrl: "old" } };
const result = reconcileCanvasTasks<TaskNode>([old], [task, newer], "2026-09-17T01:02:00Z");
expect(result[0].draftValues?.mediaUrl).toBe("https://example.com/new.png");
expect(result[0].draftValues?.pendingLabel).toBe("");
expect(
reconcileCanvasTasks<TaskNode>([], [task, newer], "2026-09-17T01:00:00Z")[0].draftValues
?.taskRequestId,
).toBe("new-request");
});
test("grid results preserve the grid action contract", () => {
const grid = { ...node, draftValues: { action: "IMAGE_GENERATE_NINE_GRID" } };
expect(
reconcileCanvasTasks<TaskNode>([grid], [task], "2026-09-17T01:00:00Z")[0].draftValues?.action,
).toBe("IMAGE_GENERATE_NINE_GRID");
});
test("a rerun retains distinct extra results without reusing the previous request's image", () => {
const first = reconcileCanvasTasks<TaskNode>([node], [task], "2026-09-17T01:00:00Z");
const newer = {
...task,
id: "task2",
requestId: "next",
createdAt: "2026-09-17T01:03:00Z",
result: { images: [{ originalUrl: "new1" }, { originalUrl: "new2" }] },
};
const next = reconcileCanvasTasks<TaskNode>(first, [task, newer], "2026-09-17T01:02:00Z");
expect(next.find((n) => n.id === "n1-result-task2-1")?.draftValues?.mediaUrl).toBe("new2");
expect(reconcileCanvasTasks<TaskNode>(next, [task, newer], "2026-09-17T01:02:00Z")).toBe(next);
});