fix(video-canvas): persist generation tasks and repair workflow controls
This commit is contained in:
134
parent
23f86dfce1
commit
0d17638a8d
@ -1,63 +1,10 @@
|
|||||||
|
import { executeCanvasImageResponse } from "@/lib/canvas/generate-image-response";
|
||||||
import { NextResponse } from "next/server";
|
import { NextResponse } from "next/server";
|
||||||
import { auth } from "@/app/(auth)/auth";
|
import { auth } from "@/app/(auth)/auth";
|
||||||
import {
|
import { getCanvasImageTaskForUser } from "@/lib/canvas/image-task-queue";
|
||||||
createCanvasImageTask,
|
|
||||||
enqueueCanvasImageTask,
|
|
||||||
getCanvasImageTaskForUser,
|
|
||||||
} from "@/lib/canvas/image-task-queue";
|
|
||||||
import {
|
|
||||||
generateCanvasImage,
|
|
||||||
type CanvasGenerateImageRequest,
|
|
||||||
} from "@/lib/canvas/generate-image";
|
|
||||||
import { getPointsSystemConfig } from "@/lib/db/queries";
|
|
||||||
import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge";
|
|
||||||
import {
|
|
||||||
getImageToolCreditCost,
|
|
||||||
type ImageToolCreditFeature,
|
|
||||||
} from "@/lib/points/config";
|
|
||||||
|
|
||||||
export const maxDuration = 1200;
|
export const maxDuration = 1200;
|
||||||
|
|
||||||
interface GenerateImageRequest extends CanvasGenerateImageRequest {
|
|
||||||
async?: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
function normalizeBillingFeature(
|
|
||||||
value: unknown
|
|
||||||
): Extract<ImageToolCreditFeature, "multi-view" | "repaint"> | null {
|
|
||||||
if (value === "multi-view" || value === "repaint") return value;
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
function toGenerateImagePayload(body: GenerateImageRequest): CanvasGenerateImageRequest {
|
|
||||||
return {
|
|
||||||
allowFailover: typeof body.allowFailover === "boolean" ? body.allowFailover : undefined,
|
|
||||||
prompt: String(body.prompt || "").trim(),
|
|
||||||
negativePrompt:
|
|
||||||
typeof body.negativePrompt === "string" ? body.negativePrompt : undefined,
|
|
||||||
model: typeof body.model === "string" ? body.model : undefined,
|
|
||||||
aspectRatio:
|
|
||||||
typeof body.aspectRatio === "string" && body.aspectRatio.trim()
|
|
||||||
? body.aspectRatio.trim()
|
|
||||||
: "auto",
|
|
||||||
imageSize:
|
|
||||||
typeof body.imageSize === "string" && body.imageSize.trim()
|
|
||||||
? body.imageSize.trim()
|
|
||||||
: undefined,
|
|
||||||
imageCount:
|
|
||||||
typeof body.imageCount === "number" && Number.isFinite(body.imageCount)
|
|
||||||
? body.imageCount
|
|
||||||
: 1,
|
|
||||||
promptExtend:
|
|
||||||
typeof body.promptExtend === "boolean" ? body.promptExtend : undefined,
|
|
||||||
referenceImage:
|
|
||||||
typeof body.referenceImage === "string" ? body.referenceImage : undefined,
|
|
||||||
referenceImages: Array.isArray(body.referenceImages)
|
|
||||||
? body.referenceImages.map((item) => String(item || "").trim()).filter(Boolean)
|
|
||||||
: [],
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function GET(request: Request) {
|
export async function GET(request: Request) {
|
||||||
try {
|
try {
|
||||||
const session = await auth();
|
const session = await auth();
|
||||||
@ -68,10 +15,7 @@ export async function GET(request: Request) {
|
|||||||
const { searchParams } = new URL(request.url);
|
const { searchParams } = new URL(request.url);
|
||||||
const taskId = String(searchParams.get("taskId") || "").trim();
|
const taskId = String(searchParams.get("taskId") || "").trim();
|
||||||
if (!taskId) {
|
if (!taskId) {
|
||||||
return NextResponse.json(
|
return NextResponse.json({ success: false, error: "缺少 taskId 参数" }, { status: 400 });
|
||||||
{ success: false, error: "缺少 taskId 参数" },
|
|
||||||
{ status: 400 }
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const task = await getCanvasImageTaskForUser({ taskId, userId: session.user.id });
|
const task = await getCanvasImageTaskForUser({ taskId, userId: session.user.id });
|
||||||
@ -100,105 +44,15 @@ export async function GET(request: Request) {
|
|||||||
success: false,
|
success: false,
|
||||||
error: error instanceof Error ? error.message : "查询任务失败",
|
error: error instanceof Error ? error.message : "查询任务失败",
|
||||||
},
|
},
|
||||||
{ status: 500 }
|
{ status: 500 },
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function POST(request: Request) {
|
export async function POST(request: Request) {
|
||||||
try {
|
const session = await auth();
|
||||||
const session = await auth();
|
if (!session?.user?.id)
|
||||||
if (!session?.user?.id) {
|
return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
|
||||||
return NextResponse.json(
|
const body = await request.json().catch(() => ({}));
|
||||||
{ success: false, error: "未登录" },
|
return executeCanvasImageResponse(body, session.user.id);
|
||||||
{ status: 401 }
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const body: GenerateImageRequest = await request.json();
|
|
||||||
const payload: CanvasGenerateImageRequest = {
|
|
||||||
...toGenerateImagePayload(body),
|
|
||||||
ownerKey: session.user.id,
|
|
||||||
};
|
|
||||||
if (!payload.prompt) {
|
|
||||||
return NextResponse.json(
|
|
||||||
{ success: false, error: "请输入提示词" },
|
|
||||||
{ status: 400 }
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const billingFeature = normalizeBillingFeature(body.billingFeature);
|
|
||||||
const pointsCharge = billingFeature
|
|
||||||
? await (async () => {
|
|
||||||
const pointsConfig = await getPointsSystemConfig();
|
|
||||||
return prepareFeaturePointsCharge({
|
|
||||||
userId: session.user.id,
|
|
||||||
unitCost: getImageToolCreditCost(pointsConfig, billingFeature),
|
|
||||||
reason:
|
|
||||||
billingFeature === "multi-view"
|
|
||||||
? "image_multi_view"
|
|
||||||
: "image_repaint",
|
|
||||||
modelId: "canvas-generate-image",
|
|
||||||
note:
|
|
||||||
billingFeature === "multi-view"
|
|
||||||
? "图片多视角生成消耗积分"
|
|
||||||
: "图片重绘消耗积分",
|
|
||||||
pointsConfig,
|
|
||||||
});
|
|
||||||
})()
|
|
||||||
: null;
|
|
||||||
if (pointsCharge && !pointsCharge.ok) {
|
|
||||||
return NextResponse.json(pointsCharge.payload, { status: pointsCharge.status });
|
|
||||||
}
|
|
||||||
|
|
||||||
const asyncMode = body.async === true;
|
|
||||||
if (asyncMode) {
|
|
||||||
const task = await createCanvasImageTask({
|
|
||||||
userId: session.user.id,
|
|
||||||
request: payload as unknown as Record<string, unknown>,
|
|
||||||
});
|
|
||||||
await enqueueCanvasImageTask(task.id);
|
|
||||||
const currentTask = await getCanvasImageTaskForUser({
|
|
||||||
taskId: task.id,
|
|
||||||
userId: session.user.id,
|
|
||||||
});
|
|
||||||
if (pointsCharge?.chargeable) {
|
|
||||||
try {
|
|
||||||
await pointsCharge.consume({
|
|
||||||
note:
|
|
||||||
billingFeature === "multi-view"
|
|
||||||
? `图片多视角生成消耗积分（${task.id}）`
|
|
||||||
: `图片重绘消耗积分（${task.id}）`,
|
|
||||||
});
|
|
||||||
} catch (pointsError) {
|
|
||||||
console.error("[Points] Failed to consume canvas image task points:", pointsError);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return NextResponse.json({
|
|
||||||
success: true,
|
|
||||||
taskId: task.id,
|
|
||||||
status: currentTask?.status || task.status,
|
|
||||||
queueStatus: currentTask?.queueStatus,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
const result = await generateCanvasImage(payload);
|
|
||||||
if (pointsCharge?.chargeable) {
|
|
||||||
try {
|
|
||||||
await pointsCharge.consume();
|
|
||||||
} catch (pointsError) {
|
|
||||||
console.error("[Points] Failed to consume canvas image points:", pointsError);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return NextResponse.json({
|
|
||||||
success: true,
|
|
||||||
data: result,
|
|
||||||
});
|
|
||||||
} catch (error) {
|
|
||||||
console.error("[Canvas Generate Image] Error:", error);
|
|
||||||
return NextResponse.json(
|
|
||||||
{ success: false, error: error instanceof Error ? error.message : "生成失败" },
|
|
||||||
{ status: 500 }
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
102
app/api/canvas/generation-tasks/route.ts
Normal file
102
app/api/canvas/generation-tasks/route.ts
Normal file
@ -0,0 +1,102 @@
|
|||||||
|
import { after, NextResponse } from "next/server";
|
||||||
|
import { auth } from "@/app/(auth)/auth";
|
||||||
|
import { z } from "zod";
|
||||||
|
import { canvasTaskRequestSchema } from "@/lib/video-canvas/task-contract";
|
||||||
|
import {
|
||||||
|
cancelCanvasGenerationTask,
|
||||||
|
createCanvasGenerationTask,
|
||||||
|
listCanvasGenerationTasks,
|
||||||
|
ownsTaskProject,
|
||||||
|
} from "@/lib/video-canvas/task-store";
|
||||||
|
import { runCanvasGenerationQueue } from "@/lib/video-canvas/task-worker";
|
||||||
|
export const runtime = "nodejs";
|
||||||
|
export const maxDuration = 900;
|
||||||
|
|
||||||
|
async function owner(projectId: unknown) {
|
||||||
|
const session = await auth();
|
||||||
|
if (!session?.user?.id) throw Object.assign(Error("未登录"), { status: 401 });
|
||||||
|
if (!z.string().uuid().safeParse(projectId).success)
|
||||||
|
throw Object.assign(Error("画布 ID 无效"), { status: 400 });
|
||||||
|
if (!(await ownsTaskProject(session.user.id, projectId as string)))
|
||||||
|
throw Object.assign(Error("画布不存在"), { status: 404 });
|
||||||
|
return session.user.id;
|
||||||
|
}
|
||||||
|
const fail = (error: unknown) =>
|
||||||
|
NextResponse.json(
|
||||||
|
{ success: false, error: error instanceof Error ? error.message : "任务操作失败" },
|
||||||
|
{ status: Number((error as any)?.status) || (error instanceof SyntaxError ? 400 : 500) },
|
||||||
|
);
|
||||||
|
const kick = (userId: string, projectId: string) =>
|
||||||
|
after(() =>
|
||||||
|
runCanvasGenerationQueue(userId, projectId).catch((error) =>
|
||||||
|
console.error("[Canvas tasks]", error),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
export async function POST(request: Request) {
|
||||||
|
try {
|
||||||
|
const text = await request.text();
|
||||||
|
if (new TextEncoder().encode(text).byteLength > 262144)
|
||||||
|
throw Object.assign(Error("任务数据过大，请使用已上传的素材链接"), { status: 413 });
|
||||||
|
const parsed = canvasTaskRequestSchema.safeParse(JSON.parse(text));
|
||||||
|
if (!parsed.success) throw Object.assign(Error("任务参数不合法"), { status: 400 });
|
||||||
|
const userId = await owner(parsed.data.projectId);
|
||||||
|
const task = await createCanvasGenerationTask(userId, parsed.data);
|
||||||
|
kick(userId, parsed.data.projectId);
|
||||||
|
return NextResponse.json({ success: true, task }, { status: 202 });
|
||||||
|
} catch (error) {
|
||||||
|
return fail(error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
export async function GET(request: Request) {
|
||||||
|
try {
|
||||||
|
const projectId = new URL(request.url).searchParams.get("projectId") || "";
|
||||||
|
const userId = await owner(projectId);
|
||||||
|
const tasks = await listCanvasGenerationTasks(userId, projectId);
|
||||||
|
if (tasks.some((task) => task.status === "pending")) kick(userId, projectId);
|
||||||
|
return NextResponse.json(
|
||||||
|
{ success: true, tasks },
|
||||||
|
{ headers: { "Cache-Control": "no-store" } },
|
||||||
|
);
|
||||||
|
} catch (error) {
|
||||||
|
return fail(error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
export async function DELETE(request: Request) {
|
||||||
|
try {
|
||||||
|
const { projectId, taskId } = await request.json();
|
||||||
|
const userId = await owner(projectId);
|
||||||
|
if (!z.string().uuid().safeParse(taskId).success)
|
||||||
|
throw Object.assign(Error("任务 ID 无效"), { status: 400 });
|
||||||
|
const result = await cancelCanvasGenerationTask(userId, projectId, taskId);
|
||||||
|
if (!result.task)
|
||||||
|
return NextResponse.json({ success: false, error: "任务不存在" }, { status: 404 });
|
||||||
|
if (!result.ok)
|
||||||
|
return NextResponse.json(
|
||||||
|
{
|
||||||
|
success: false,
|
||||||
|
error:
|
||||||
|
result.task.status === "running"
|
||||||
|
? "任务已提交供应商，当前线路不支持中止。结果会继续保存，请勿重复生成。"
|
||||||
|
: "任务已结束或状态待核实，请先刷新任务状态。",
|
||||||
|
task: result.task,
|
||||||
|
},
|
||||||
|
{ status: 409 },
|
||||||
|
);
|
||||||
|
return NextResponse.json({ success: true, task: result.task });
|
||||||
|
} catch (error) {
|
||||||
|
return fail(error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
export async function PATCH(request: Request) {
|
||||||
|
try {
|
||||||
|
const { projectId, taskId, nodeId } = await request.json();
|
||||||
|
const userId = await owner(projectId);
|
||||||
|
if (!z.string().uuid().safeParse(taskId).success || typeof nodeId !== "string")
|
||||||
|
return NextResponse.json({ success: false }, { status: 400 });
|
||||||
|
const { hideCanvasGenerationNode } = await import("@/lib/video-canvas/task-store");
|
||||||
|
const success = await hideCanvasGenerationNode(userId, projectId, taskId, nodeId);
|
||||||
|
return NextResponse.json({ success }, { status: success ? 200 : 404 });
|
||||||
|
} catch (error) {
|
||||||
|
return fail(error);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1,786 +1,12 @@
|
|||||||
import { normalizeStoryScriptRows } from "@/lib/video-canvas/story-script";
|
import { executeWorkflowAnalysisResponse } from "@/lib/video-canvas/workflow-analysis-response";
|
||||||
import { isCanvasLanguageModel, isCanvasVisionModel } from "@/lib/video-canvas/model-matching";
|
|
||||||
import { generateText } from "ai";
|
|
||||||
import { execFile } from "node:child_process";
|
|
||||||
import { promisify } from "node:util";
|
|
||||||
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
|
||||||
import os from "node:os";
|
|
||||||
import path from "node:path";
|
|
||||||
import { NextResponse } from "next/server";
|
import { NextResponse } from "next/server";
|
||||||
import sharp from "sharp";
|
|
||||||
import { z } from "zod";
|
|
||||||
import { auth } from "@/app/(auth)/auth";
|
import { auth } from "@/app/(auth)/auth";
|
||||||
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
|
||||||
import { inlineVisionImages } from "@/lib/ai/vision-inline";
|
|
||||||
import { getAiModelAgentCodingModelId, isAiModelMultimodal } from "@/lib/db/ai-model-flags";
|
|
||||||
import {
|
|
||||||
getAgentAiModelWithProvider,
|
|
||||||
getEnabledAiModelsWithProvider,
|
|
||||||
} from "@/lib/db/queries";
|
|
||||||
|
|
||||||
export const runtime = "nodejs";
|
export const runtime = "nodejs";
|
||||||
export const maxDuration = 180;
|
export const maxDuration = 180;
|
||||||
|
|
||||||
const execFileAsync = promisify(execFile);
|
|
||||||
const VIDEO_ANALYSIS_FRAME_COUNT = 4;
|
|
||||||
const VIDEO_ANALYSIS_MAX_SIDE = 960;
|
|
||||||
const VIDEO_ANALYSIS_TIMEOUT_MS = 15000;
|
|
||||||
|
|
||||||
const requestSchema = z.object({
|
|
||||||
mode: z.enum(["video-analysis", "character-scene-extraction", "story-script", "agent-plan"]),
|
|
||||||
prompt: z.string().trim().optional(),
|
|
||||||
imageUrls: z.array(z.string().trim()).optional(),
|
|
||||||
videoUrls: z.array(z.string().trim()).optional(),
|
|
||||||
nodeTitle: z.string().trim().optional(),
|
|
||||||
preferredModelRef: z.string().trim().optional(),
|
|
||||||
model: z.string().trim().optional(),
|
|
||||||
});
|
|
||||||
|
|
||||||
function sanitizeUrls(value: unknown, limit = 6) {
|
|
||||||
if (!Array.isArray(value)) return [];
|
|
||||||
return value
|
|
||||||
.map((item) => String(item || "").trim())
|
|
||||||
.filter((url) => /^https?:\/\//i.test(url))
|
|
||||||
.slice(0, limit);
|
|
||||||
}
|
|
||||||
|
|
||||||
function supportsVision(model: {
|
|
||||||
supportsVision?: boolean | null;
|
|
||||||
capabilities?: unknown;
|
|
||||||
defaultParams?: unknown;
|
|
||||||
}) {
|
|
||||||
return (
|
|
||||||
Boolean(model.supportsVision) ||
|
|
||||||
isAiModelMultimodal({
|
|
||||||
capabilities: model.capabilities,
|
|
||||||
defaultParams: model.defaultParams,
|
|
||||||
})
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function extractJsonBlock<T = Record<string, unknown>>(text: string): T | null {
|
|
||||||
const matched = String(text || "").match(/\{[\s\S]*\}/);
|
|
||||||
if (!matched) return null;
|
|
||||||
try {
|
|
||||||
return JSON.parse(matched[0]) as T;
|
|
||||||
} catch {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function normalizeText(value: unknown) {
|
|
||||||
return String(value || "").trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
function shouldUseLocalWorkflowE2E(request: Request) {
|
|
||||||
const hostname = new URL(request.url).hostname;
|
|
||||||
return (
|
|
||||||
process.env.CANVAS_E2E_LOCAL === "true" &&
|
|
||||||
(hostname === "127.0.0.1" || hostname === "localhost" || hostname === "::1")
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function normalizeStringList(value: unknown, limit = 8) {
|
|
||||||
if (!Array.isArray(value)) return [];
|
|
||||||
return Array.from(
|
|
||||||
new Set(
|
|
||||||
value
|
|
||||||
.map((item) => String(item || "").trim())
|
|
||||||
.filter(Boolean)
|
|
||||||
.slice(0, limit)
|
|
||||||
)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function normalizeShotIdeas(value: unknown) {
|
|
||||||
if (!Array.isArray(value)) return [];
|
|
||||||
return value
|
|
||||||
.map((item, index) => {
|
|
||||||
const raw =
|
|
||||||
item && typeof item === "object" && !Array.isArray(item)
|
|
||||||
? (item as Record<string, unknown>)
|
|
||||||
: {};
|
|
||||||
const title = normalizeText(raw.title || `镜头 ${index + 1}`);
|
|
||||||
const description = normalizeText(raw.description || raw.prompt || raw.summary);
|
|
||||||
if (!title && !description) return null;
|
|
||||||
return {
|
|
||||||
title: title || `镜头 ${index + 1}`,
|
|
||||||
description,
|
|
||||||
};
|
|
||||||
})
|
|
||||||
.filter(
|
|
||||||
(item): item is { title: string; description: string } => Boolean(item)
|
|
||||||
)
|
|
||||||
.slice(0, 6);
|
|
||||||
}
|
|
||||||
|
|
||||||
function normalizeVideoAnalysisRows(
|
|
||||||
value: unknown,
|
|
||||||
framePreviews: string[],
|
|
||||||
) {
|
|
||||||
if (!Array.isArray(value)) return [];
|
|
||||||
return value
|
|
||||||
.map((item, index) => {
|
|
||||||
const raw =
|
|
||||||
item && typeof item === "object" && !Array.isArray(item)
|
|
||||||
? (item as Record<string, unknown>)
|
|
||||||
: {};
|
|
||||||
const sceneDescription = normalizeText(
|
|
||||||
raw.sceneDescription || raw.scene || raw.title || raw.description,
|
|
||||||
);
|
|
||||||
const narrative = normalizeText(
|
|
||||||
raw.narrative || raw.story || raw.storyContent || raw.narration || raw.summary,
|
|
||||||
);
|
|
||||||
const shot = normalizeText(raw.shot || raw.shotNumber || raw.index || index + 1);
|
|
||||||
const startTime = normalizeText(raw.startTime || raw.start || raw.start_time);
|
|
||||||
const endTime = normalizeText(raw.endTime || raw.end || raw.end_time);
|
|
||||||
const duration = normalizeText(raw.duration || raw.durationSeconds || raw.duration_seconds);
|
|
||||||
const shotSize = normalizeText(raw.shotSize || raw.frameSize || raw.shot_type || raw.sceneScale);
|
|
||||||
const cameraAngle = normalizeText(raw.cameraAngle || raw.angle || raw.camera_angle);
|
|
||||||
const cameraMovement = normalizeText(
|
|
||||||
raw.cameraMovement || raw.cameraMove || raw.camera_motion || raw.motionType,
|
|
||||||
);
|
|
||||||
const focusDepth = normalizeText(
|
|
||||||
raw.focusDepth || raw.depthOfField || raw.focalDepth || raw.focus_depth,
|
|
||||||
);
|
|
||||||
const lighting = normalizeText(raw.lighting || raw.light || raw.lightDesign);
|
|
||||||
const backgroundMusic = normalizeText(
|
|
||||||
raw.backgroundMusic || raw.music || raw.bgm,
|
|
||||||
);
|
|
||||||
const voiceAndSfx = normalizeText(
|
|
||||||
raw.voiceAndSfx || raw.sound || raw.audio || raw.sfx,
|
|
||||||
);
|
|
||||||
const imagePrompt = normalizeText(
|
|
||||||
raw.imagePrompt || raw.image_generation_prompt || raw.image,
|
|
||||||
);
|
|
||||||
const motionPrompt = normalizeText(
|
|
||||||
raw.motionPrompt || raw.motion || raw.videoPrompt || raw.video_motion_prompt,
|
|
||||||
);
|
|
||||||
const keyframeIndex = Number(
|
|
||||||
raw.keyframeIndex || raw.frameIndex || raw.key_frame_index || index + 1,
|
|
||||||
);
|
|
||||||
if (
|
|
||||||
!sceneDescription &&
|
|
||||||
!narrative &&
|
|
||||||
!backgroundMusic &&
|
|
||||||
!voiceAndSfx &&
|
|
||||||
!imagePrompt &&
|
|
||||||
!motionPrompt &&
|
|
||||||
!shotSize &&
|
|
||||||
!cameraAngle &&
|
|
||||||
!cameraMovement &&
|
|
||||||
!focusDepth &&
|
|
||||||
!lighting
|
|
||||||
) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
const normalizedFrameIndex =
|
|
||||||
Number.isFinite(keyframeIndex) && keyframeIndex > 0
|
|
||||||
? Math.max(0, Math.min(framePreviews.length - 1, Math.round(keyframeIndex) - 1))
|
|
||||||
: index % Math.max(framePreviews.length, 1);
|
|
||||||
return {
|
|
||||||
shot: shot || String(index + 1),
|
|
||||||
startTime: startTime || (index === 0 ? "0" : ""),
|
|
||||||
endTime,
|
|
||||||
duration,
|
|
||||||
sceneDescription: sceneDescription || `镜头 ${index + 1}`,
|
|
||||||
narrative: narrative || sceneDescription || `镜头 ${index + 1}`,
|
|
||||||
shotSize: shotSize || "中景至近景",
|
|
||||||
cameraAngle: cameraAngle || "平视",
|
|
||||||
cameraMovement: cameraMovement || "缓慢推镜头 (Slow Zoom In)",
|
|
||||||
focusDepth: focusDepth || "浅景深",
|
|
||||||
lighting: lighting || "冷调低沉照明，主体光线柔和。",
|
|
||||||
backgroundMusic: backgroundMusic || "无",
|
|
||||||
voiceAndSfx: voiceAndSfx || "无",
|
|
||||||
imagePrompt: imagePrompt || sceneDescription || "延续当前视频气质生成关键帧",
|
|
||||||
motionPrompt: motionPrompt || "保持镜头运动自然、连贯、符合前后节奏。",
|
|
||||||
keyframeUrl: framePreviews[normalizedFrameIndex] || "",
|
|
||||||
};
|
|
||||||
})
|
|
||||||
.filter(
|
|
||||||
(
|
|
||||||
item,
|
|
||||||
): item is {
|
|
||||||
shot: string;
|
|
||||||
startTime: string;
|
|
||||||
endTime: string;
|
|
||||||
duration: string;
|
|
||||||
sceneDescription: string;
|
|
||||||
narrative: string;
|
|
||||||
shotSize: string;
|
|
||||||
cameraAngle: string;
|
|
||||||
cameraMovement: string;
|
|
||||||
focusDepth: string;
|
|
||||||
lighting: string;
|
|
||||||
backgroundMusic: string;
|
|
||||||
voiceAndSfx: string;
|
|
||||||
imagePrompt: string;
|
|
||||||
motionPrompt: string;
|
|
||||||
keyframeUrl: string;
|
|
||||||
} => Boolean(item),
|
|
||||||
)
|
|
||||||
.slice(0, 8);
|
|
||||||
}
|
|
||||||
|
|
||||||
function normalizeCharacters(value: unknown) {
|
|
||||||
if (!Array.isArray(value)) return [];
|
|
||||||
return value
|
|
||||||
.map((item) => {
|
|
||||||
const raw =
|
|
||||||
item && typeof item === "object" && !Array.isArray(item)
|
|
||||||
? (item as Record<string, unknown>)
|
|
||||||
: {};
|
|
||||||
const name = normalizeText(raw.name || raw.title);
|
|
||||||
const description = normalizeText(raw.description || raw.summary);
|
|
||||||
const prompt = normalizeText(raw.prompt);
|
|
||||||
if (!name && !description && !prompt) return null;
|
|
||||||
return {
|
|
||||||
name: name || "未命名角色",
|
|
||||||
description,
|
|
||||||
prompt,
|
|
||||||
look: normalizeText(raw.look) || undefined,
|
|
||||||
wardrobe: normalizeText(raw.wardrobe) || undefined,
|
|
||||||
signatureAction: normalizeText(raw.signatureAction) || undefined,
|
|
||||||
traits: normalizeStringList(raw.traits, 6),
|
|
||||||
};
|
|
||||||
})
|
|
||||||
.filter((item): item is NonNullable<typeof item> => Boolean(item))
|
|
||||||
.slice(0, 8);
|
|
||||||
}
|
|
||||||
|
|
||||||
function normalizeScenes(value: unknown) {
|
|
||||||
if (!Array.isArray(value)) return [];
|
|
||||||
return value
|
|
||||||
.map((item) => {
|
|
||||||
const raw =
|
|
||||||
item && typeof item === "object" && !Array.isArray(item)
|
|
||||||
? (item as Record<string, unknown>)
|
|
||||||
: {};
|
|
||||||
const name = normalizeText(raw.name || raw.title);
|
|
||||||
const description = normalizeText(raw.description || raw.summary);
|
|
||||||
const prompt = normalizeText(raw.prompt);
|
|
||||||
if (!name && !description && !prompt) return null;
|
|
||||||
return {
|
|
||||||
name: name || "未命名场景",
|
|
||||||
description,
|
|
||||||
prompt,
|
|
||||||
mood: normalizeText(raw.mood) || undefined,
|
|
||||||
lighting: normalizeText(raw.lighting) || undefined,
|
|
||||||
props: normalizeStringList(raw.props, 6),
|
|
||||||
};
|
|
||||||
})
|
|
||||||
.filter((item): item is NonNullable<typeof item> => Boolean(item))
|
|
||||||
.slice(0, 8);
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildVideoAnalysisPrompt(params: {
|
|
||||||
nodeTitle?: string;
|
|
||||||
prompt: string;
|
|
||||||
frameCount: number;
|
|
||||||
imageCount: number;
|
|
||||||
videoCount: number;
|
|
||||||
}) {
|
|
||||||
return [
|
|
||||||
"你是视频画布工作流里的视频分析节点，需要为后续分镜和生成节点提供结构化输出。",
|
|
||||||
params.nodeTitle ? `当前节点：${params.nodeTitle}` : "",
|
|
||||||
`上游参考图数量：${params.imageCount}`,
|
|
||||||
`上游视频数量：${params.videoCount}`,
|
|
||||||
`已提供按时间顺序抽取的视频关键帧数量：${params.frameCount}`,
|
|
||||||
params.prompt ? `上游脚本/说明：\n${params.prompt}` : "上游没有额外脚本，请重点依据参考画面分析。",
|
|
||||||
"",
|
|
||||||
"请只输出 JSON，不要输出 markdown，不要输出解释。",
|
|
||||||
"JSON 结构如下：",
|
|
||||||
`{
|
|
||||||
"summary": "一句话总结当前视频/镜头风格，不超过120字",
|
|
||||||
"highlights": ["3-5条镜头分析要点"],
|
|
||||||
"analysisRows": [
|
|
||||||
{
|
|
||||||
"shot": "1",
|
|
||||||
"startTime": "0",
|
|
||||||
"endTime": "5",
|
|
||||||
"duration": "5",
|
|
||||||
"sceneDescription": "画面描述，说明主体、环境、动作、构图和色调",
|
|
||||||
"narrative": "叙事内容，说明这一镜头承担的剧情/情绪/信息作用",
|
|
||||||
"shotSize": "景别，例如中景至近景",
|
|
||||||
"cameraAngle": "摄影机角度，例如平视",
|
|
||||||
"cameraMovement": "摄影机运动，例如缓慢推镜头 (Slow Zoom In)",
|
|
||||||
"focusDepth": "焦距与景深，例如浅景深",
|
|
||||||
"lighting": "光线，例如冷调低沉照明，屏幕光作为侧逆光",
|
|
||||||
"backgroundMusic": "这一段适合的背景音乐氛围",
|
|
||||||
"voiceAndSfx": "人声/环境声/音效建议，没有就写无",
|
|
||||||
"imagePrompt": "后续生成关键帧时可直接使用的英文图像提示词",
|
|
||||||
"motionPrompt": "后续生成视频时可直接使用的英文运动提示词",
|
|
||||||
"keyframeIndex": 1
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"shotIdeas": [
|
|
||||||
{ "title": "镜头 1", "description": "一句话描述镜头目标、景别或动作" }
|
|
||||||
],
|
|
||||||
"visualMotifs": ["2-4个可复用的视觉母题"],
|
|
||||||
"recommendedNextSteps": ["2-4条建议的下游动作"]
|
|
||||||
}`,
|
|
||||||
"analysisRows 请按镜头时间顺序输出；短视频可以输出 1 行，复杂视频输出 2-6 行。",
|
|
||||||
"每行必须覆盖镜号、开始时间、结束时间、时长、画面描述、叙事内容、景别、摄影机角度、摄影机运动、焦距与景深、光线、背景音乐、人声/音效、图像生成提示词、视频运动提示词和关键帧。",
|
|
||||||
"imagePrompt 和 motionPrompt 优先使用英文；其他字段使用中文；keyframeIndex 使用 1 开始的序号，对应你看到的关键帧顺序。",
|
|
||||||
"如果没有真实视频帧，只能依据参考图和脚本推断，也要正常输出，但措辞要稳健。",
|
|
||||||
]
|
|
||||||
.filter(Boolean)
|
|
||||||
.join("\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
async function getVideoDurationSeconds(videoUrl: string) {
|
|
||||||
try {
|
|
||||||
const { stdout } = await execFileAsync(
|
|
||||||
"ffprobe",
|
|
||||||
[
|
|
||||||
"-v",
|
|
||||||
"error",
|
|
||||||
"-show_entries",
|
|
||||||
"format=duration",
|
|
||||||
"-of",
|
|
||||||
"default=noprint_wrappers=1:nokey=1",
|
|
||||||
videoUrl,
|
|
||||||
],
|
|
||||||
{
|
|
||||||
timeout: VIDEO_ANALYSIS_TIMEOUT_MS,
|
|
||||||
maxBuffer: 1024 * 1024,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
const duration = Number.parseFloat(String(stdout || "").trim());
|
|
||||||
return Number.isFinite(duration) && duration > 0 ? duration : null;
|
|
||||||
} catch {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildVideoFrameTimestamps(durationSeconds: number | null, frameCount: number) {
|
|
||||||
const safeCount = Math.max(1, frameCount);
|
|
||||||
if (!durationSeconds || !Number.isFinite(durationSeconds) || durationSeconds <= 0.5) {
|
|
||||||
return Array.from({ length: safeCount }).map((_, index) =>
|
|
||||||
Number((0.3 + index * 0.45).toFixed(2)),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
const start = Math.min(0.35, durationSeconds * 0.12);
|
|
||||||
const end = Math.max(start + 0.6, durationSeconds * 0.88);
|
|
||||||
if (safeCount === 1) return [Number(((start + end) / 2).toFixed(2))];
|
|
||||||
const step = (end - start) / (safeCount - 1);
|
|
||||||
return Array.from({ length: safeCount }).map((_, index) =>
|
|
||||||
Number((start + step * index).toFixed(2)),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function downscaleVideoFrameForVision(inputBuffer: Buffer) {
|
|
||||||
return sharp(inputBuffer, { failOn: "none" })
|
|
||||||
.rotate()
|
|
||||||
.resize({
|
|
||||||
width: VIDEO_ANALYSIS_MAX_SIDE,
|
|
||||||
height: VIDEO_ANALYSIS_MAX_SIDE,
|
|
||||||
fit: "inside",
|
|
||||||
withoutEnlargement: true,
|
|
||||||
})
|
|
||||||
.jpeg({ quality: 82, mozjpeg: true })
|
|
||||||
.toBuffer();
|
|
||||||
}
|
|
||||||
|
|
||||||
async function extractVideoFrameDataUrls(videoUrl: string, frameCount = VIDEO_ANALYSIS_FRAME_COUNT) {
|
|
||||||
const tempDir = await mkdtemp(path.join(os.tmpdir(), "canvas-video-analysis-"));
|
|
||||||
try {
|
|
||||||
const durationSeconds = await getVideoDurationSeconds(videoUrl);
|
|
||||||
const timestamps = buildVideoFrameTimestamps(durationSeconds, frameCount);
|
|
||||||
const frames = await Promise.all(
|
|
||||||
timestamps.map(async (timestamp, index) => {
|
|
||||||
const outputPath = path.join(tempDir, `frame-${index + 1}.jpg`);
|
|
||||||
await execFileAsync(
|
|
||||||
"ffmpeg",
|
|
||||||
[
|
|
||||||
"-hide_banner",
|
|
||||||
"-loglevel",
|
|
||||||
"error",
|
|
||||||
"-ss",
|
|
||||||
String(timestamp),
|
|
||||||
"-i",
|
|
||||||
videoUrl,
|
|
||||||
"-frames:v",
|
|
||||||
"1",
|
|
||||||
"-q:v",
|
|
||||||
"4",
|
|
||||||
"-y",
|
|
||||||
outputPath,
|
|
||||||
],
|
|
||||||
{
|
|
||||||
timeout: VIDEO_ANALYSIS_TIMEOUT_MS,
|
|
||||||
maxBuffer: 2 * 1024 * 1024,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
const buffer = await readFile(outputPath);
|
|
||||||
const processed = await downscaleVideoFrameForVision(buffer);
|
|
||||||
return `data:image/jpeg;base64,${processed.toString("base64")}`;
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
return frames.filter(Boolean);
|
|
||||||
} catch (error) {
|
|
||||||
console.warn("[Canvas Workflow Analyze] extract video frames failed:", {
|
|
||||||
message: error instanceof Error ? error.message : String(error),
|
|
||||||
videoUrl,
|
|
||||||
});
|
|
||||||
return [];
|
|
||||||
} finally {
|
|
||||||
await rm(tempDir, { force: true, recursive: true }).catch(() => undefined);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildCharacterScenePrompt(params: {
|
|
||||||
nodeTitle?: string;
|
|
||||||
prompt: string;
|
|
||||||
imageCount: number;
|
|
||||||
}) {
|
|
||||||
return [
|
|
||||||
"你是视频画布工作流里的角色场景提取节点，需要从脚本、设定和参考图里提取角色与场景资产。",
|
|
||||||
params.nodeTitle ? `当前节点：${params.nodeTitle}` : "",
|
|
||||||
`参考图数量：${params.imageCount}`,
|
|
||||||
params.prompt ? `输入内容：\n${params.prompt}` : "没有额外脚本，请根据参考图提取角色和场景线索。",
|
|
||||||
"",
|
|
||||||
"请只输出 JSON，不要输出 markdown，不要输出解释。",
|
|
||||||
"JSON 结构如下：",
|
|
||||||
`{
|
|
||||||
"summary": "一句话总结当前故事世界与资产线索",
|
|
||||||
"characters": [
|
|
||||||
{
|
|
||||||
"name": "角色名",
|
|
||||||
"description": "角色外观与身份简介",
|
|
||||||
"look": "外貌特征",
|
|
||||||
"wardrobe": "服装/道具",
|
|
||||||
"traits": ["性格关键词"],
|
|
||||||
"signatureAction": "代表动作",
|
|
||||||
"prompt": "可直接用于生成角色图/角色视频的提示词"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"scenes": [
|
|
||||||
{
|
|
||||||
"name": "场景名",
|
|
||||||
"description": "空间、布景和氛围简介",
|
|
||||||
"mood": "情绪与气氛",
|
|
||||||
"lighting": "布光特征",
|
|
||||||
"props": ["核心道具"],
|
|
||||||
"prompt": "可直接用于生成场景图/场景视频的提示词"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"relationships": ["角色与场景或角色之间的关系"],
|
|
||||||
"recommendedNextSteps": ["建议连接的后续节点动作"]
|
|
||||||
}`,
|
|
||||||
"角色和场景都要尽量具体，避免空泛形容词。",
|
|
||||||
]
|
|
||||||
.filter(Boolean)
|
|
||||||
.join("\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildAgentPlanPrompt(params: {
|
|
||||||
nodeTitle?: string;
|
|
||||||
prompt: string;
|
|
||||||
imageCount: number;
|
|
||||||
videoCount: number;
|
|
||||||
}) {
|
|
||||||
return [
|
|
||||||
"你是视频画布里的自定义代理节点，需要把当前创作目标拆成可执行的工作流计划。",
|
|
||||||
params.nodeTitle ? `当前节点：${params.nodeTitle}` : "",
|
|
||||||
`参考图数量：${params.imageCount}`,
|
|
||||||
`视频数量：${params.videoCount}`,
|
|
||||||
params.prompt ? `上游需求：\n${params.prompt}` : "没有额外脚本，请基于当前上下文给出稳妥计划。",
|
|
||||||
"",
|
|
||||||
"请只输出 JSON，不要输出 markdown，不要输出解释。",
|
|
||||||
"JSON 结构如下：",
|
|
||||||
`{
|
|
||||||
"summary": "一句话说明这个代理链的目标",
|
|
||||||
"agentPlan": ["3-6个执行步骤"],
|
|
||||||
"deliverables": ["2-4个预期交付物"],
|
|
||||||
"recommendedNodes": ["建议继续连接的节点类型或能力名"]
|
|
||||||
}`,
|
|
||||||
"步骤要偏工作流视角，而不是泛泛而谈。",
|
|
||||||
]
|
|
||||||
.filter(Boolean)
|
|
||||||
.join("\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildStoryScriptPrompt(params: {
|
|
||||||
nodeTitle?: string;
|
|
||||||
prompt: string;
|
|
||||||
imageCount: number;
|
|
||||||
videoCount: number;
|
|
||||||
}) {
|
|
||||||
return [
|
|
||||||
"你是视频画布里的故事脚本生成节点，需要把用户输入、参考图和参考视频整理成可直接用于后续分镜/生图/生视频的结构化脚本表。",
|
|
||||||
params.nodeTitle ? `当前节点：${params.nodeTitle}` : "",
|
|
||||||
`参考图数量：${params.imageCount}`,
|
|
||||||
`视频数量：${params.videoCount}`,
|
|
||||||
params.prompt ? `输入内容：\n${params.prompt}` : "没有额外脚本，请生成一个节奏清晰的短视频脚本。",
|
|
||||||
"",
|
|
||||||
"请只输出 JSON，不要输出 markdown，不要输出解释。",
|
|
||||||
"JSON 结构如下：",
|
|
||||||
`{
|
|
||||||
"summary": "一句话概括故事主题、情绪和视频方向，不超过120字",
|
|
||||||
"analysisRows": [
|
|
||||||
{
|
|
||||||
"shot": "01",
|
|
||||||
"duration": "5s",
|
|
||||||
"scene": "场景名或地点，如 盛唐·金銮殿·日",
|
|
||||||
"sceneDescription": "画面内容、人物动作和剧情推进，不要重复 scene 字段",
|
|
||||||
"shotSize": "景别，如中景、特写、全景、大远景",
|
|
||||||
"lighting": "光影氛围，如暖金主光带阴影，气氛紧张压抑",
|
|
||||||
"dialogue": "对白或旁白，没有就写 +",
|
|
||||||
"backgroundMusic": "背景音乐或情绪氛围，没有就写无",
|
|
||||||
"voiceAndSfx": "环境声、动作声或音效，没有就写无",
|
|
||||||
"cameraMovement": "运镜，如缓慢推镜头、固定、横移、上摇、环绕轨道跟拍",
|
|
||||||
"imagePrompt": "后续生成该镜头关键帧时可直接使用的中文图像提示词",
|
|
||||||
"motionPrompt": "后续生成视频时可直接使用的中文视频运动提示词"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"assetCandidates": {
|
|
||||||
"style": "全局视觉风格，如 盛唐·国风二次元",
|
|
||||||
"characters": ["需要生成的角色资产名"],
|
|
||||||
"scenes": ["需要生成的场景资产名"],
|
|
||||||
"props": ["需要生成的道具资产名"]
|
|
||||||
},
|
|
||||||
"shotIdeas": [
|
|
||||||
{ "title": "镜头 1", "description": "一句话描述镜头目标、景别或动作" }
|
|
||||||
],
|
|
||||||
"recommendedNextSteps": ["2-4条建议的下游动作"]
|
|
||||||
}`,
|
|
||||||
"优先严格遵守用户指定的镜头数量和总时长；未指定时生成 6 个镜头，最多 18 个。镜头时长之和必须符合用户要求，8 秒短片不得扩写成几十秒。只生成实际需要的镜头，不重复凑数。",
|
|
||||||
"每行要覆盖竞品脚本表同等信息：镜号、时长、画面描述、景别、光影氛围、对白·旁白、音效、运镜、最终图像提示词和视频运动提示词。",
|
|
||||||
"画面描述中用 @角色名 或 @场景/道具名 标出可复用资产，例如 @沈昭昭、@金銮殿、@侍卫长刀。",
|
|
||||||
"imagePrompt 要像可直接生成分镜图的中文提示词，包含主体、环境、构图、光影、风格与一致性要求；motionPrompt 要按起始状态、动作过程、结束状态、音效组织。",
|
|
||||||
"assetCandidates 只提取脚本中确实出现的角色、场景和道具；产品视频没有人物时 characters 必须为空，禁止为凑数虚构角色。",
|
|
||||||
"每行要像专业视频画布脚本表：场景具体、画面可生成、声音可执行、镜头节奏明确，不要只写剧情摘要。",
|
|
||||||
]
|
|
||||||
.filter(Boolean)
|
|
||||||
.join("\n");
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function POST(request: Request) {
|
export async function POST(request: Request) {
|
||||||
try {
|
const session = await auth();
|
||||||
const session = await auth();
|
if (!session?.user?.id)
|
||||||
if (!session?.user?.id) {
|
return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
|
||||||
return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
|
const body = await request.json().catch(() => ({}));
|
||||||
}
|
return executeWorkflowAnalysisResponse(body, request.signal);
|
||||||
|
|
||||||
const body = await request.json().catch(() => ({}));
|
|
||||||
const parsed = requestSchema.safeParse(body);
|
|
||||||
if (!parsed.success) {
|
|
||||||
return NextResponse.json(
|
|
||||||
{ success: false, error: "请求参数不合法" },
|
|
||||||
{ status: 400 }
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const prompt = normalizeText(parsed.data.prompt);
|
|
||||||
const imageUrls = sanitizeUrls(parsed.data.imageUrls, 4);
|
|
||||||
const videoUrls = sanitizeUrls(parsed.data.videoUrls, 4);
|
|
||||||
const extractedVideoFrames =
|
|
||||||
parsed.data.mode === "video-analysis" && videoUrls.length > 0
|
|
||||||
? await extractVideoFrameDataUrls(videoUrls[0], VIDEO_ANALYSIS_FRAME_COUNT)
|
|
||||||
: [];
|
|
||||||
const preferredModelRef = normalizeText(parsed.data.preferredModelRef || parsed.data.model);
|
|
||||||
if (!prompt && imageUrls.length === 0 && videoUrls.length === 0) {
|
|
||||||
return NextResponse.json(
|
|
||||||
{ success: false, error: "请至少提供脚本、图片或视频上下文" },
|
|
||||||
{ status: 400 }
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
const [allModels, agentModel] = await Promise.all([
|
|
||||||
getEnabledAiModelsWithProvider(false),
|
|
||||||
getAgentAiModelWithProvider(),
|
|
||||||
]);
|
|
||||||
const hasVisionInputs = imageUrls.length > 0 || extractedVideoFrames.length > 0;
|
|
||||||
const compatibleModels = allModels.filter((item) => {
|
|
||||||
const descriptor = {...item.model, providerType: item.provider.type};
|
|
||||||
return !item.model.isSubModel && item.model.isEnabled &&
|
|
||||||
(hasVisionInputs ? isCanvasVisionModel(descriptor) : isCanvasLanguageModel(descriptor));
|
|
||||||
});
|
|
||||||
const requestedRef = preferredModelRef === "auto" ? "" : preferredModelRef;
|
|
||||||
const preferredModel = requestedRef ? compatibleModels.find(
|
|
||||||
item => item.model.id === requestedRef || item.model.modelId === requestedRef
|
|
||||||
) : undefined;
|
|
||||||
if (requestedRef && !preferredModel) {
|
|
||||||
return NextResponse.json({ success: false, error: "所选模型不可用或不支持当前分析输入，请重新选择分析模型" }, { status: 400 });
|
|
||||||
}
|
|
||||||
const selectedModel = preferredModel || compatibleModels.find(item => item.model.id === agentModel?.model.id) || compatibleModels[0];
|
|
||||||
if (!selectedModel) {
|
|
||||||
return NextResponse.json({ success: false, error: hasVisionInputs ? "当前没有可用的视觉理解模型" : "当前没有可用的文字分析模型" }, { status: 400 });
|
|
||||||
}
|
|
||||||
|
|
||||||
const useVision = hasVisionInputs && supportsVision(selectedModel.model);
|
|
||||||
const codingModelId = getAiModelAgentCodingModelId(selectedModel.model);
|
|
||||||
const model = await getLanguageModelById(selectedModel.model.id, codingModelId);
|
|
||||||
if (!model) {
|
|
||||||
return NextResponse.json(
|
|
||||||
{ success: false, error: "分析模型初始化失败" },
|
|
||||||
{ status: 500 }
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const promptText =
|
|
||||||
parsed.data.mode === "video-analysis"
|
|
||||||
? buildVideoAnalysisPrompt({
|
|
||||||
frameCount: extractedVideoFrames.length,
|
|
||||||
nodeTitle: normalizeText(parsed.data.nodeTitle),
|
|
||||||
prompt,
|
|
||||||
imageCount: imageUrls.length,
|
|
||||||
videoCount: videoUrls.length,
|
|
||||||
})
|
|
||||||
: parsed.data.mode === "character-scene-extraction"
|
|
||||||
? buildCharacterScenePrompt({
|
|
||||||
nodeTitle: normalizeText(parsed.data.nodeTitle),
|
|
||||||
prompt,
|
|
||||||
imageCount: imageUrls.length,
|
|
||||||
})
|
|
||||||
: parsed.data.mode === "story-script"
|
|
||||||
? buildStoryScriptPrompt({
|
|
||||||
nodeTitle: normalizeText(parsed.data.nodeTitle),
|
|
||||||
prompt,
|
|
||||||
imageCount: imageUrls.length,
|
|
||||||
videoCount: videoUrls.length,
|
|
||||||
})
|
|
||||||
: buildAgentPlanPrompt({
|
|
||||||
nodeTitle: normalizeText(parsed.data.nodeTitle),
|
|
||||||
prompt,
|
|
||||||
imageCount: imageUrls.length,
|
|
||||||
videoCount: videoUrls.length,
|
|
||||||
});
|
|
||||||
|
|
||||||
const inlinedImageReferences = useVision
|
|
||||||
? (
|
|
||||||
await inlineVisionImages(imageUrls, {
|
|
||||||
maxImages: 4,
|
|
||||||
maxSide: 960,
|
|
||||||
timeoutMs: 12000,
|
|
||||||
retries: 1,
|
|
||||||
loggerPrefix: "[Canvas Workflow Analyze]",
|
|
||||||
})
|
|
||||||
).inlineDataUrls
|
|
||||||
: [];
|
|
||||||
const referenceImages = useVision
|
|
||||||
? [...inlinedImageReferences, ...extractedVideoFrames].slice(0, 6)
|
|
||||||
: [];
|
|
||||||
|
|
||||||
const { text, finishReason } = await generateText({
|
|
||||||
model,
|
|
||||||
messages: [
|
|
||||||
{
|
|
||||||
role: "user",
|
|
||||||
content: [
|
|
||||||
{ type: "text", text: promptText },
|
|
||||||
...referenceImages.map((image) => ({ type: "image" as const, image })),
|
|
||||||
],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
temperature: 0.2,
|
|
||||||
maxOutputTokens: parsed.data.mode === "story-script" ? 12000 : parsed.data.mode === "video-analysis" ? 2200 : 1600,
|
|
||||||
maxRetries: 0,
|
|
||||||
abortSignal: AbortSignal.any([request.signal, AbortSignal.timeout(150_000)]),
|
|
||||||
});
|
|
||||||
|
|
||||||
if (finishReason === "length" || !text.trim()) {
|
|
||||||
return NextResponse.json({ success: false, error: "分析结果不完整，请缩短输入或重新生成" }, { status: 502 });
|
|
||||||
}
|
|
||||||
const parsedJson = extractJsonBlock<Record<string, unknown>>(text);
|
|
||||||
if (!parsedJson) {
|
|
||||||
return NextResponse.json({ success: false, error: "分析模型未返回完整结构化结果，请重新生成" }, { status: 502 });
|
|
||||||
}
|
|
||||||
const summary =
|
|
||||||
normalizeText(parsedJson.summary) ||
|
|
||||||
normalizeText(text).slice(0, 160);
|
|
||||||
|
|
||||||
if (parsed.data.mode === "video-analysis") {
|
|
||||||
const analysisRows = normalizeVideoAnalysisRows(
|
|
||||||
parsedJson.analysisRows,
|
|
||||||
extractedVideoFrames,
|
|
||||||
);
|
|
||||||
const shotIdeas =
|
|
||||||
normalizeShotIdeas(parsedJson.shotIdeas).length > 0
|
|
||||||
? normalizeShotIdeas(parsedJson.shotIdeas)
|
|
||||||
: analysisRows.map((row, index) => ({
|
|
||||||
title: `镜头 ${index + 1}`,
|
|
||||||
description: row.sceneDescription,
|
|
||||||
}));
|
|
||||||
return NextResponse.json({
|
|
||||||
success: true,
|
|
||||||
data: {
|
|
||||||
summary,
|
|
||||||
analysisRows,
|
|
||||||
framePreviews: extractedVideoFrames,
|
|
||||||
highlights: normalizeStringList(parsedJson.highlights, 6),
|
|
||||||
shotIdeas,
|
|
||||||
visualMotifs: normalizeStringList(parsedJson.visualMotifs, 6),
|
|
||||||
recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6),
|
|
||||||
model: selectedModel.model.displayName,
|
|
||||||
raw: text,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
if (parsed.data.mode === "character-scene-extraction") {
|
|
||||||
return NextResponse.json({
|
|
||||||
success: true,
|
|
||||||
data: {
|
|
||||||
summary,
|
|
||||||
characters: normalizeCharacters(parsedJson.characters),
|
|
||||||
scenes: normalizeScenes(parsedJson.scenes),
|
|
||||||
relationships: normalizeStringList(parsedJson.relationships, 8),
|
|
||||||
recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6),
|
|
||||||
model: selectedModel.model.displayName,
|
|
||||||
raw: text,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
if (parsed.data.mode === "story-script") {
|
|
||||||
const analysisRows = normalizeStoryScriptRows(
|
|
||||||
parsedJson.analysisRows || parsedJson.storyRows || parsedJson.rows || parsedJson.shots,
|
|
||||||
);
|
|
||||||
if (analysisRows.length === 0) {
|
|
||||||
return NextResponse.json({ success: false, error: "分析模型没有返回有效镜头，请重新生成" }, { status: 502 });
|
|
||||||
}
|
|
||||||
return NextResponse.json({
|
|
||||||
success: true,
|
|
||||||
data: {
|
|
||||||
summary,
|
|
||||||
analysisRows,
|
|
||||||
shotIdeas:
|
|
||||||
normalizeShotIdeas(parsedJson.shotIdeas).length > 0
|
|
||||||
? normalizeShotIdeas(parsedJson.shotIdeas)
|
|
||||||
: analysisRows.map((row, index) => ({
|
|
||||||
title: `镜头 ${index + 1}`,
|
|
||||||
description: row.sceneDescription,
|
|
||||||
})),
|
|
||||||
recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6),
|
|
||||||
model: selectedModel.model.displayName,
|
|
||||||
raw: text,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
return NextResponse.json({
|
|
||||||
success: true,
|
|
||||||
data: {
|
|
||||||
summary,
|
|
||||||
agentPlan: normalizeStringList(parsedJson.agentPlan, 8),
|
|
||||||
deliverables: normalizeStringList(parsedJson.deliverables, 6),
|
|
||||||
recommendedNodes: normalizeStringList(parsedJson.recommendedNodes, 6),
|
|
||||||
model: selectedModel.model.displayName,
|
|
||||||
raw: text,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
} catch (error) {
|
|
||||||
console.error("[Canvas Workflow Analyze] Error:", error);
|
|
||||||
const timedOut = error instanceof Error && /timeout|timed out|abort/i.test(`${error.name} ${error.message}`);
|
|
||||||
return NextResponse.json(
|
|
||||||
{
|
|
||||||
success: false,
|
|
||||||
error: timedOut ? "分析超时，请稍后重试或选择其他分析模型" : error instanceof Error ? error.message : "工作流分析失败",
|
|
||||||
},
|
|
||||||
{ status: timedOut ? 504 : 500 }
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
78
components/canvas/PromptTranslateButton.tsx
Normal file
78
components/canvas/PromptTranslateButton.tsx
Normal file
@ -0,0 +1,78 @@
|
|||||||
|
"use client";
|
||||||
|
import { useEffect, useRef, useState } from "react";
|
||||||
|
import { toast } from "sonner";
|
||||||
|
export function PromptTranslateButton({
|
||||||
|
prompt,
|
||||||
|
model,
|
||||||
|
onApply,
|
||||||
|
}: {
|
||||||
|
prompt: string;
|
||||||
|
model?: string;
|
||||||
|
onApply: (text: string) => void;
|
||||||
|
}) {
|
||||||
|
const [busy, setBusy] = useState(false),
|
||||||
|
[result, setResult] = useState("");
|
||||||
|
const controller = useRef<AbortController | null>(null);
|
||||||
|
useEffect(() => () => controller.current?.abort(), []);
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
disabled={busy || !prompt.trim()}
|
||||||
|
className="rounded px-2 py-1 text-sm disabled:opacity-40"
|
||||||
|
onClick={async () => {
|
||||||
|
const request = new AbortController();
|
||||||
|
controller.current = request;
|
||||||
|
setBusy(true);
|
||||||
|
try {
|
||||||
|
const response = await fetch("/api/canvas/prompt-tools", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify({ mode: "translate", prompt, model }),
|
||||||
|
signal: AbortSignal.any([request.signal, AbortSignal.timeout(100000)]),
|
||||||
|
});
|
||||||
|
const body = await response.json();
|
||||||
|
if (!response.ok || !body.success || !body.data?.text)
|
||||||
|
throw Error(body.error || "翻译没有返回结果");
|
||||||
|
if (!request.signal.aborted) setResult(body.data.text);
|
||||||
|
} catch (error) {
|
||||||
|
if (!request.signal.aborted)
|
||||||
|
toast.error(error instanceof Error ? error.message : "翻译失败");
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{busy ? "翻译中…" : "翻译"}
|
||||||
|
</button>
|
||||||
|
{result ? (
|
||||||
|
<div
|
||||||
|
role="dialog"
|
||||||
|
aria-label="翻译结果"
|
||||||
|
className="fixed inset-4 z-[700] flex flex-col gap-3 rounded-xl border border-white/20 bg-[#222] p-5 text-white"
|
||||||
|
>
|
||||||
|
<textarea
|
||||||
|
aria-label="翻译内容"
|
||||||
|
className="min-h-0 flex-1 resize-none bg-transparent"
|
||||||
|
value={result}
|
||||||
|
onChange={(event) => setResult(event.target.value)}
|
||||||
|
/>
|
||||||
|
<div className="flex justify-end gap-3">
|
||||||
|
<button type="button" onClick={() => setResult("")}>
|
||||||
|
放弃
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={() => {
|
||||||
|
onApply(result);
|
||||||
|
setResult("");
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
应用翻译
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
</>
|
||||||
|
);
|
||||||
|
}
|
||||||
51
docs/video-canvas-task-repairs-2026-09-17.md
Normal file
51
docs/video-canvas-task-repairs-2026-09-17.md
Normal file
@ -0,0 +1,51 @@
|
|||||||
|
# 视频画布任务与交互修复 · 2026-09-17
|
||||||
|
|
||||||
|
承接 `23f86dfc` 的全面扫描。用户补充“浏览器刷新一下就好了”，本轮重点修复任务结果与页面状态脱节，同时处理 F02–F07 中已确认的入口问题。本记录是分支修复与本地验收，未部署 48 服务器，也没有把原 231 项清单改成全部通过。
|
||||||
|
|
||||||
|
## 修复范围
|
||||||
|
|
||||||
|
| 扫描项 | 本轮改动 | 验证及边界 |
|
||||||
|
|---|---|---|
|
||||||
|
| F01 任务恢复 | 新增统一生成任务接口、数据库受理记录、请求 ID 去重、原子领取、心跳与结果保存；客户端提交前保存待核对请求，刷新/重开/online/focus/visibilitychange 时查询；去掉虚假进度；旧版无任务 ID 的 pending 明确提示无法恢复 | 主图像/视频生成、配置工具的图像/视频/分析、脚本再生成、批量分镜/视频接入。局部编辑按已有 taskId 恢复查询。真实数据库与浏览器验收见下文。专用图像后期、角度编辑、脚本弹窗资产生成/逐镜提示词仍有同步请求，未宣称全部入口具备同等恢复能力 |
|
||||||
|
| F02 取消 | 排队任务通过数据库 CAS 取消；已提交供应商的任务返回明确说明，继续保存结果；删除已完成结果写入任务隐藏记录 | 数据库并发及浏览器排队取消通过；没有冒充支持上游中止 |
|
||||||
|
| F03 模板 | 待执行模板设为 idle；参考样例标记示例；空脚本不再显示硬编码成稿 | 代码检查；未逐个模板做生产生成 |
|
||||||
|
| F04 脚本 | 脚本入口统一 story-script；空响应/无有效镜头返回失败并保留旧稿 | 结果契约回归通过；脚本资产生成与单镜提示词处理不是此次持久队列接入范围 |
|
||||||
|
| F05 批量 | 批量分镜和视频实际入队；按有无首帧选择兼容模式，采用模型声明的离散时长/比例/分辨率；完成任务不重复提交，失败/空节点可继续执行 | 模型参数和任务契约通过；没有新增真实付费批量请求 |
|
||||||
|
| F06 素材弹窗 | 去掉假分页和无效图标，增加搜索、纵向滚动、自适应列、素材名称与明确数量单位 | 实际组件注入 100 个素材，第 100 项能选中、确认回传，搜索返回 1 项；使用当前 Tailwind 编译产物验收 |
|
||||||
|
| F07 次级控件 | 文本及音频生视频提示词翻译接真实接口并预览后应用；旧脚本编辑聚焦；资源替换接上传；通知显示真实任务；模板筛选使用配置数据；退出接现有登出流程；未实现按钮明确禁用 | 代码与类型检查。未实际退出生产账户，未声称所有停用高级能力已实现 |
|
||||||
|
|
||||||
|
任务表复用既有 `0091_add_canvas_capability_tasks.sql`，没有新增数据库迁移。上线前须核对该既有迁移已应用。
|
||||||
|
|
||||||
|
## 任务一致性
|
||||||
|
|
||||||
|
- 同一请求 ID 重试不会重复执行；同一节点已有活动任务时复用现有任务。
|
||||||
|
- 同画布任务串行领取，多执行者通过 PostgreSQL 事务锁与状态更新竞争。
|
||||||
|
- 刷新前画布尚未保存的已受理节点，可从任务快照补回；不会从旧任务复活已保存删除的节点。
|
||||||
|
- 多图结果使用任务 ID 与序号组成稳定结果节点 ID，轮询不重复追加，重新生成不误用上次附图。
|
||||||
|
- 旧请求不能覆盖新请求；画布保存落后于新任务时，根据任务时间和保存版本恢复最新结果。
|
||||||
|
- 连线仅首次恢复时补齐，后续轮询不反复恢复用户已经删除的连线。
|
||||||
|
- 执行进程心跳超过 120 秒未更新，任务标记待核实；不自动重新提交付费请求。若原执行稍后确实返回，仍允许保存真实结果。
|
||||||
|
- 浏览器离开与服务端执行进程重启是不同情况：本轮解决前者的主要入口；进程中断后尚未实现供应商任务 ID 的自动续查。
|
||||||
|
|
||||||
|
## 验证记录
|
||||||
|
|
||||||
|
1. **真实本地 PostgreSQL**：`scripts/qa/video-canvas-task-db.cjs` 建立隔离测试库，加载实际任务表迁移和实际 store。10 次并发相同提交只创建 1 条记录；10 次并发领取仅 1 次成功；跨用户隔离、排队取消、执行中不可取消、串行队列、结果保存、结果隐藏、过期不重投、晚到结果保存全部通过。这是并发数据库调用测试，不是 10 个操作系统进程实测。
|
||||||
|
2. **真实浏览器任务恢复**：验收页使用实际 React Hook、实际任务 route/worker/store 与本地 PostgreSQL；鉴权使用固定测试用户，NextResponse/after 使用本地适配器，模型返回使用 12 秒测试桩。提交图片后立即刷新，最终出现 2 个结果且只 1 条任务；视频执行时关闭页面，再打开能恢复结果；后续分析排队任务成功取消；移除附图后刷新不复活。画布快照保存使用本地验收服务器，不是生产画布接口。
|
||||||
|
3. **实际素材组件**：100 项列表可定位选择最后一项，确认回传 `测试素材 100`；搜索同名只剩 1 项且保留选中状态；已有第 100 项再连续点选第 1–10 项，最终只选中 10 项，第 10 项（第 11 次选择）被正确拦截。验收期间先发现旧测试样式缓存，重新编译当前源码样式后验证通过。
|
||||||
|
4. **自动化**：最终 helpers/route 回归 437 项通过，无跳过，包含本次新增 13 项任务/批量参数测试和既有真实 FFmpeg 检查。不是 437 个生产端到端测试。
|
||||||
|
5. **静态检查**：TypeScript 无错误；`git diff --check` 通过。新增服务端模块与旧接口共享执行逻辑，图像计费仍走原有计费模块。
|
||||||
|
|
||||||
|
没有进行新的付费模型测试，没有重新宣称参考站像素级一致，没有覆盖操作系统休眠、真实断网、多标签冲突、生产 Next.js 进程重启或生产部署后的回归。原扫描中 79 项缺失/停用能力依然需要单独实现与验证，详见原验证矩阵。
|
||||||
|
|
||||||
|
## 复现
|
||||||
|
|
||||||
|
```sh
|
||||||
|
# FFmpeg / ffprobe 必须在 PATH 中，避免媒体测试跳过
|
||||||
|
pnpm exec playwright test --config=playwright.helpers.config.ts
|
||||||
|
pnpm exec tsc --noEmit --incremental false
|
||||||
|
# 隔离数据库测试依赖仅装入忽略的 .qa 目录，不改项目依赖
|
||||||
|
npm install --prefix .qa/video-task-db --no-save embedded-postgres
|
||||||
|
node scripts/qa/video-canvas-task-db.cjs
|
||||||
|
```
|
||||||
|
|
||||||
|
浏览器验收脚手架位于忽略目录 `.qa/video-durable` 和 `.qa/video-repair`；上述记录保留测试方法和模拟边界，不能代替后续部署验收。
|
||||||
@ -1,5 +1,7 @@
|
|||||||
# 视频画布全面验证扫描 · 2026-09-17
|
# 视频画布全面验证扫描 · 2026-09-17
|
||||||
|
|
||||||
|
后续修复记录：[任务与交互修复](video-canvas-task-repairs-2026-09-17.md)。以下内容保留扫描时的基线，不代表修复后的现状；也未将未验收项批量改为通过。
|
||||||
|
|
||||||
结论：**仍有未验证功能，也有已确认但尚未修复的功能缺口，不能判定全面验收通过。** 本轮以 `f10e6b7c` 为代码起点，核对原 10 组、231 项对照清单，追踪活动页面的调用、请求和保存流程，补跑相关回归与实际组件浏览器验证。没有重新提交付费模型任务，没有重新实测参考网站，也没有部署 48 服务器。
|
结论：**仍有未验证功能，也有已确认但尚未修复的功能缺口，不能判定全面验收通过。** 本轮以 `f10e6b7c` 为代码起点，核对原 10 组、231 项对照清单，追踪活动页面的调用、请求和保存流程，补跑相关回归与实际组件浏览器验证。没有重新提交付费模型任务，没有重新实测参考网站，也没有部署 48 服务器。
|
||||||
|
|
||||||
## 证据范围
|
## 证据范围
|
||||||
|
|||||||
272
hooks/use-video-canvas-tasks.ts
Normal file
272
hooks/use-video-canvas-tasks.ts
Normal file
@ -0,0 +1,272 @@
|
|||||||
|
"use client";
|
||||||
|
import { useCallback, useEffect, useRef, useState } from "react";
|
||||||
|
import { toast } from "sonner";
|
||||||
|
import {
|
||||||
|
reconcileCanvasTasks,
|
||||||
|
type CanvasGenerationTask,
|
||||||
|
type CanvasTaskRequest,
|
||||||
|
} from "@/lib/video-canvas/task-contract";
|
||||||
|
|
||||||
|
type Node = CanvasTaskRequest["node"];
|
||||||
|
type Edge = CanvasTaskRequest["edges"][number];
|
||||||
|
const storageKey = (projectId: string) => `video-canvas-outbox:${projectId}`;
|
||||||
|
function outbox(projectId: string): CanvasTaskRequest[] {
|
||||||
|
try {
|
||||||
|
const entries = JSON.parse(localStorage.getItem(storageKey(projectId)) || "[]");
|
||||||
|
return Array.isArray(entries)
|
||||||
|
? entries.filter(
|
||||||
|
(item) =>
|
||||||
|
item?.projectId === projectId && typeof item?.requestId === "string" && item?.node?.id,
|
||||||
|
)
|
||||||
|
: [];
|
||||||
|
} catch {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
function remember(request: CanvasTaskRequest) {
|
||||||
|
const entries = outbox(request.projectId).filter((item) => item.requestId !== request.requestId);
|
||||||
|
localStorage.setItem(storageKey(request.projectId), JSON.stringify([...entries, request]));
|
||||||
|
}
|
||||||
|
function forget(projectId: string, requestId: string) {
|
||||||
|
localStorage.setItem(
|
||||||
|
storageKey(projectId),
|
||||||
|
JSON.stringify(outbox(projectId).filter((item) => item.requestId !== requestId)),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
export function useVideoCanvasTasks<T extends Node>(
|
||||||
|
projectId: string,
|
||||||
|
loaded: { id: string; revision: string } | null,
|
||||||
|
setNodes: React.Dispatch<React.SetStateAction<T[]>>,
|
||||||
|
setEdges: React.Dispatch<React.SetStateAction<Edge[]>>,
|
||||||
|
) {
|
||||||
|
const [tasks, setTasks] = useState<CanvasGenerationTask[]>([]);
|
||||||
|
const hiddenNodes = useRef(new Set<string>());
|
||||||
|
const recoveredEdges = useRef(new Set<string>());
|
||||||
|
const refreshRef = useRef<() => void>(() => {});
|
||||||
|
const activeProject = useRef(projectId);
|
||||||
|
activeProject.current = projectId;
|
||||||
|
const apply = useCallback(
|
||||||
|
(tasks: CanvasGenerationTask[]) => {
|
||||||
|
if (activeProject.current !== projectId || !loaded) return;
|
||||||
|
setTasks((current) => {
|
||||||
|
const next = new Map(current.map((task) => [task.id, task]));
|
||||||
|
for (const task of tasks) next.set(task.id, task);
|
||||||
|
return [...next.values()];
|
||||||
|
});
|
||||||
|
const visibleTasks = tasks.map((task) => ({
|
||||||
|
...task,
|
||||||
|
request: {
|
||||||
|
...task.request,
|
||||||
|
hiddenNodeIds: [...(task.request.hiddenNodeIds || []), ...hiddenNodes.current],
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
setNodes((nodes) => reconcileCanvasTasks(nodes as any, visibleTasks, loaded.revision) as T[]);
|
||||||
|
const recoverable = visibleTasks.filter((task) => {
|
||||||
|
if (recoveredEdges.current.has(task.id)) return false;
|
||||||
|
recoveredEdges.current.add(task.id);
|
||||||
|
return Date.parse(task.createdAt) > Date.parse(loaded.revision);
|
||||||
|
});
|
||||||
|
setEdges((edges) => {
|
||||||
|
const existing = new Set(edges.map((edge) => `${edge.source}>${edge.target}`));
|
||||||
|
const extras = recoverable
|
||||||
|
.flatMap((task) =>
|
||||||
|
task.request.edges.filter(
|
||||||
|
(edge) =>
|
||||||
|
!task.request.hiddenNodeIds.includes(edge.source) &&
|
||||||
|
!task.request.hiddenNodeIds.includes(edge.target),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.filter((edge) => {
|
||||||
|
const key = `${edge.source}>${edge.target}`;
|
||||||
|
if (existing.has(key)) return false;
|
||||||
|
existing.add(key);
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
return extras.length ? [...edges, ...extras] : edges;
|
||||||
|
});
|
||||||
|
},
|
||||||
|
[projectId, loaded, setNodes, setEdges],
|
||||||
|
);
|
||||||
|
const send = useCallback(
|
||||||
|
async (request: CanvasTaskRequest) => {
|
||||||
|
const response = await fetch("/api/canvas/generation-tasks", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify(request),
|
||||||
|
signal: AbortSignal.timeout(20000),
|
||||||
|
});
|
||||||
|
const body = await response.json();
|
||||||
|
if (!response.ok || !body.success) {
|
||||||
|
if (response.status >= 400 && response.status < 500) {
|
||||||
|
forget(projectId, request.requestId);
|
||||||
|
if (activeProject.current === projectId)
|
||||||
|
setNodes((nodes) =>
|
||||||
|
nodes.map((node) =>
|
||||||
|
node.id === request.node.id && node.draftValues?.taskRequestId === request.requestId
|
||||||
|
? {
|
||||||
|
...node,
|
||||||
|
draftValues: {
|
||||||
|
...node.draftValues,
|
||||||
|
generationStatus: "failed",
|
||||||
|
generationError: body.error,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
: node,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
throw Error(body.error || "提交暂时失败，恢复连接后自动核对");
|
||||||
|
}
|
||||||
|
forget(projectId, request.requestId);
|
||||||
|
if (activeProject.current !== projectId) return body.task as CanvasGenerationTask;
|
||||||
|
// The server may return an existing task for the same active node.
|
||||||
|
setNodes((nodes) =>
|
||||||
|
nodes.map((node) =>
|
||||||
|
node.id === request.node.id
|
||||||
|
? {
|
||||||
|
...node,
|
||||||
|
draftValues: {
|
||||||
|
...node.draftValues,
|
||||||
|
taskRequestId: body.task.requestId,
|
||||||
|
generationTaskId: body.task.id,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
: node,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
apply([body.task]);
|
||||||
|
return body.task as CanvasGenerationTask;
|
||||||
|
},
|
||||||
|
[projectId, apply, setNodes],
|
||||||
|
);
|
||||||
|
const submit = useCallback(
|
||||||
|
async (
|
||||||
|
node: T,
|
||||||
|
kind: CanvasTaskRequest["kind"],
|
||||||
|
params: Record<string, unknown>,
|
||||||
|
edges: Edge[] = [],
|
||||||
|
) => {
|
||||||
|
if (!loaded || loaded.id !== projectId) throw Error("画布尚未加载完成，请稍后提交");
|
||||||
|
const requestId = crypto.randomUUID();
|
||||||
|
const snapshot = {
|
||||||
|
...node,
|
||||||
|
draftValues: {
|
||||||
|
...node.draftValues,
|
||||||
|
taskRequestId: requestId,
|
||||||
|
generationTaskId: undefined,
|
||||||
|
appliedTaskStatus: undefined,
|
||||||
|
generationStatus: "pending",
|
||||||
|
generationProgress: 0,
|
||||||
|
pendingLabel: "提交中",
|
||||||
|
generationError: undefined,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const request: CanvasTaskRequest = {
|
||||||
|
projectId,
|
||||||
|
requestId,
|
||||||
|
kind,
|
||||||
|
node: snapshot,
|
||||||
|
params,
|
||||||
|
edges,
|
||||||
|
};
|
||||||
|
// Persist before POST: a refresh during an uncertain network response retries the same ID.
|
||||||
|
remember(request);
|
||||||
|
setNodes((nodes) => nodes.map((item) => (item.id === node.id ? snapshot : item)));
|
||||||
|
try {
|
||||||
|
await send(request);
|
||||||
|
} catch (error) {
|
||||||
|
toast.error(
|
||||||
|
error instanceof Error ? error.message : "连接中断，恢复后将核对任务，请勿重复提交",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
refreshRef.current();
|
||||||
|
},
|
||||||
|
[loaded, projectId, send, setNodes],
|
||||||
|
);
|
||||||
|
useEffect(() => {
|
||||||
|
if (!loaded || loaded.id !== projectId) return;
|
||||||
|
hiddenNodes.current.clear();
|
||||||
|
recoveredEdges.current.clear();
|
||||||
|
setTasks([]);
|
||||||
|
let stopped = false,
|
||||||
|
running = false;
|
||||||
|
const refresh = async () => {
|
||||||
|
if (stopped || running) return;
|
||||||
|
running = true;
|
||||||
|
try {
|
||||||
|
const response = await fetch(
|
||||||
|
`/api/canvas/generation-tasks?projectId=${encodeURIComponent(projectId)}`,
|
||||||
|
{ cache: "no-store", signal: AbortSignal.timeout(15000) },
|
||||||
|
);
|
||||||
|
if (!response.ok) return;
|
||||||
|
const body = await response.json();
|
||||||
|
if (stopped || !body.success) return;
|
||||||
|
apply(body.tasks);
|
||||||
|
const accepted = new Map<string, CanvasGenerationTask>(
|
||||||
|
body.tasks.map((task: CanvasGenerationTask) => [task.requestId, task]),
|
||||||
|
);
|
||||||
|
for (const request of outbox(projectId)) {
|
||||||
|
if (stopped) break;
|
||||||
|
if (accepted.has(request.requestId)) {
|
||||||
|
forget(projectId, request.requestId);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
await send(request);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
/* Network loss is not a generation failure. The next online/focus event queries again. */
|
||||||
|
} finally {
|
||||||
|
running = false;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
refreshRef.current = () => void refresh();
|
||||||
|
const wake = () => {
|
||||||
|
if (document.visibilityState === "visible") void refresh();
|
||||||
|
};
|
||||||
|
const timer = setInterval(() => void refresh(), 5000);
|
||||||
|
window.addEventListener("online", wake);
|
||||||
|
window.addEventListener("focus", wake);
|
||||||
|
document.addEventListener("visibilitychange", wake);
|
||||||
|
void refresh();
|
||||||
|
return () => {
|
||||||
|
stopped = true;
|
||||||
|
clearInterval(timer);
|
||||||
|
window.removeEventListener("online", wake);
|
||||||
|
window.removeEventListener("focus", wake);
|
||||||
|
document.removeEventListener("visibilitychange", wake);
|
||||||
|
};
|
||||||
|
}, [projectId, loaded, apply, send]);
|
||||||
|
const cancel = useCallback(
|
||||||
|
async (taskId: string) => {
|
||||||
|
const response = await fetch("/api/canvas/generation-tasks", {
|
||||||
|
method: "DELETE",
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify({ projectId, taskId }),
|
||||||
|
signal: AbortSignal.timeout(15000),
|
||||||
|
});
|
||||||
|
const body = await response.json();
|
||||||
|
if (body.task) apply([body.task]);
|
||||||
|
if (!response.ok || !body.success) throw Error(body.error || "取消失败，请刷新任务状态");
|
||||||
|
refreshRef.current();
|
||||||
|
},
|
||||||
|
[projectId, apply],
|
||||||
|
);
|
||||||
|
const dismiss = useCallback(
|
||||||
|
async (nodeId: string, taskId?: string) => {
|
||||||
|
hiddenNodes.current.add(nodeId);
|
||||||
|
if (!taskId) return;
|
||||||
|
const response = await fetch("/api/canvas/generation-tasks", {
|
||||||
|
method: "PATCH",
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify({ projectId, taskId, nodeId }),
|
||||||
|
signal: AbortSignal.timeout(15000),
|
||||||
|
});
|
||||||
|
if (!response.ok) {
|
||||||
|
hiddenNodes.current.delete(nodeId);
|
||||||
|
throw Error("移除结果失败，请重试");
|
||||||
|
}
|
||||||
|
},
|
||||||
|
[projectId],
|
||||||
|
);
|
||||||
|
return { submit, cancel, dismiss, tasks, refresh: () => refreshRef.current() };
|
||||||
|
}
|
||||||
125
lib/canvas/generate-image-response.ts
Normal file
125
lib/canvas/generate-image-response.ts
Normal file
@ -0,0 +1,125 @@
|
|||||||
|
import "server-only";
|
||||||
|
import { NextResponse } from "next/server";
|
||||||
|
import {
|
||||||
|
createCanvasImageTask,
|
||||||
|
enqueueCanvasImageTask,
|
||||||
|
} from "@/lib/canvas/image-task-queue";
|
||||||
|
import { generateCanvasImage, type CanvasGenerateImageRequest } from "@/lib/canvas/generate-image";
|
||||||
|
import { getPointsSystemConfig } from "@/lib/db/queries";
|
||||||
|
import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge";
|
||||||
|
import { getImageToolCreditCost, type ImageToolCreditFeature } from "@/lib/points/config";
|
||||||
|
|
||||||
|
interface GenerateImageRequest extends CanvasGenerateImageRequest {
|
||||||
|
async?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeBillingFeature(
|
||||||
|
value: unknown,
|
||||||
|
): Extract<ImageToolCreditFeature, "multi-view" | "repaint"> | null {
|
||||||
|
if (value === "multi-view" || value === "repaint") return value;
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function toGenerateImagePayload(body: GenerateImageRequest): CanvasGenerateImageRequest {
|
||||||
|
return {
|
||||||
|
allowFailover: typeof body.allowFailover === "boolean" ? body.allowFailover : undefined,
|
||||||
|
prompt: String(body.prompt || "").trim(),
|
||||||
|
negativePrompt: typeof body.negativePrompt === "string" ? body.negativePrompt : undefined,
|
||||||
|
model: typeof body.model === "string" ? body.model : undefined,
|
||||||
|
aspectRatio:
|
||||||
|
typeof body.aspectRatio === "string" && body.aspectRatio.trim()
|
||||||
|
? body.aspectRatio.trim()
|
||||||
|
: "auto",
|
||||||
|
imageSize:
|
||||||
|
typeof body.imageSize === "string" && body.imageSize.trim()
|
||||||
|
? body.imageSize.trim()
|
||||||
|
: undefined,
|
||||||
|
imageCount:
|
||||||
|
typeof body.imageCount === "number" && Number.isFinite(body.imageCount) ? body.imageCount : 1,
|
||||||
|
promptExtend: typeof body.promptExtend === "boolean" ? body.promptExtend : undefined,
|
||||||
|
referenceImage: typeof body.referenceImage === "string" ? body.referenceImage : undefined,
|
||||||
|
referenceImages: Array.isArray(body.referenceImages)
|
||||||
|
? body.referenceImages.map((item) => String(item || "").trim()).filter(Boolean)
|
||||||
|
: [],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function executeCanvasImageResponse(body: GenerateImageRequest, userId: string) {
|
||||||
|
try {
|
||||||
|
const payload: CanvasGenerateImageRequest = {
|
||||||
|
...toGenerateImagePayload(body),
|
||||||
|
ownerKey: userId,
|
||||||
|
};
|
||||||
|
if (!payload.prompt) {
|
||||||
|
return NextResponse.json({ success: false, error: "请输入提示词" }, { status: 400 });
|
||||||
|
}
|
||||||
|
|
||||||
|
const billingFeature = normalizeBillingFeature(body.billingFeature);
|
||||||
|
const pointsCharge = billingFeature
|
||||||
|
? await (async () => {
|
||||||
|
const pointsConfig = await getPointsSystemConfig();
|
||||||
|
return prepareFeaturePointsCharge({
|
||||||
|
userId: userId,
|
||||||
|
unitCost: getImageToolCreditCost(pointsConfig, billingFeature),
|
||||||
|
reason: billingFeature === "multi-view" ? "image_multi_view" : "image_repaint",
|
||||||
|
modelId: "canvas-generate-image",
|
||||||
|
note: billingFeature === "multi-view" ? "图片多视角生成消耗积分" : "图片重绘消耗积分",
|
||||||
|
pointsConfig,
|
||||||
|
});
|
||||||
|
})()
|
||||||
|
: null;
|
||||||
|
if (pointsCharge && !pointsCharge.ok) {
|
||||||
|
return NextResponse.json(pointsCharge.payload, { status: pointsCharge.status });
|
||||||
|
}
|
||||||
|
|
||||||
|
const asyncMode = body.async === true;
|
||||||
|
if (asyncMode) {
|
||||||
|
const task = await createCanvasImageTask({
|
||||||
|
userId: userId,
|
||||||
|
request: payload as unknown as Record<string, unknown>,
|
||||||
|
});
|
||||||
|
await enqueueCanvasImageTask(task.id);
|
||||||
|
const currentTask = await getCanvasImageTaskForUser({
|
||||||
|
taskId: task.id,
|
||||||
|
userId: userId,
|
||||||
|
});
|
||||||
|
if (pointsCharge?.chargeable) {
|
||||||
|
try {
|
||||||
|
await pointsCharge.consume({
|
||||||
|
note:
|
||||||
|
billingFeature === "multi-view"
|
||||||
|
? `图片多视角生成消耗积分（${task.id}）`
|
||||||
|
: `图片重绘消耗积分（${task.id}）`,
|
||||||
|
});
|
||||||
|
} catch (pointsError) {
|
||||||
|
console.error("[Points] Failed to consume canvas image task points:", pointsError);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return NextResponse.json({
|
||||||
|
success: true,
|
||||||
|
taskId: task.id,
|
||||||
|
status: currentTask?.status || task.status,
|
||||||
|
queueStatus: currentTask?.queueStatus,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await generateCanvasImage(payload);
|
||||||
|
if (pointsCharge?.chargeable) {
|
||||||
|
try {
|
||||||
|
await pointsCharge.consume();
|
||||||
|
} catch (pointsError) {
|
||||||
|
console.error("[Points] Failed to consume canvas image points:", pointsError);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return NextResponse.json({
|
||||||
|
success: true,
|
||||||
|
data: result,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
console.error("[Canvas Generate Image] Error:", error);
|
||||||
|
return NextResponse.json(
|
||||||
|
{ success: false, error: error instanceof Error ? error.message : "生成失败" },
|
||||||
|
{ status: 500 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -108,3 +108,16 @@ export function validateCanvasVideoInputs(mode: VideoGenerationMode, images: str
|
|||||||
if (mode === "first-last-frame" && images.length !== 2) throw new Error("首尾帧生视频需要恰好两张参考图片");
|
if (mode === "first-last-frame" && images.length !== 2) throw new Error("首尾帧生视频需要恰好两张参考图片");
|
||||||
if (mode === "reference-images" && images.length === 0) throw new Error("参考生视频需要至少一张参考图片");
|
if (mode === "reference-images" && images.length === 0) throw new Error("参考生视频需要至少一张参考图片");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function chooseStoryVideoSettings(models: CanvasModelDescriptor[], hasImage: boolean, seconds: number, preferredRef?: string) {
|
||||||
|
const mode:VideoGenerationMode = hasImage ? "first-frame" : "text-to-video";
|
||||||
|
const choices = filterCanvasModels(models,"video",mode);
|
||||||
|
const model = choices.find(item=>item.id===preferredRef||item.modelId===preferredRef) || choices[0];
|
||||||
|
if (!model) throw Error(hasImage ? "没有可用的首帧视频模型" : "镜头缺少首帧，且没有可用的文生视频模型");
|
||||||
|
const options = getCanvasVideoParameters(model,mode);
|
||||||
|
if (!options.durations.length || !options.ratios.length || !options.resolutions.length) throw Error("视频模型缺少已确认的时长、比例或清晰度规格，请先完善模型配置");
|
||||||
|
const duration = options.durations.reduce((best,value)=>Math.abs(value-seconds)<Math.abs(best-seconds)?value:best,options.durations[0]);
|
||||||
|
return {mode,model:model.id||model.modelId||"",duration,
|
||||||
|
ratio:options.ratios.includes("16:9")?"16:9":options.ratios[0],
|
||||||
|
resolution:options.resolutions.includes("720p")?"720p":options.resolutions[0],generateAudio:options.supportsAudio};
|
||||||
|
}
|
||||||
|
|||||||
182
lib/video-canvas/task-contract.ts
Normal file
182
lib/video-canvas/task-contract.ts
Normal file
@ -0,0 +1,182 @@
|
|||||||
|
import { z } from "zod";
|
||||||
|
|
||||||
|
export const canvasTaskRequestSchema = z.object({
|
||||||
|
requestId: z.string().uuid(),
|
||||||
|
projectId: z.string().uuid(),
|
||||||
|
kind: z.enum(["video", "image", "analysis"]),
|
||||||
|
node: z
|
||||||
|
.object({
|
||||||
|
id: z.string().min(1).max(180),
|
||||||
|
title: z.string().max(300),
|
||||||
|
x: z.number().finite(),
|
||||||
|
y: z.number().finite(),
|
||||||
|
width: z.number().positive().max(10000),
|
||||||
|
height: z.number().positive().max(10000),
|
||||||
|
kind: z.string().max(30),
|
||||||
|
inputType: z.string().max(30).optional(),
|
||||||
|
draftValues: z.record(z.unknown()).optional(),
|
||||||
|
})
|
||||||
|
.passthrough(),
|
||||||
|
params: z.record(z.unknown()),
|
||||||
|
hiddenNodeIds: z.array(z.string()).max(20).optional(),
|
||||||
|
edges: z
|
||||||
|
.array(z.object({ id: z.string(), source: z.string(), target: z.string() }))
|
||||||
|
.max(30)
|
||||||
|
.default([]),
|
||||||
|
});
|
||||||
|
export type CanvasTaskRequest = z.infer<typeof canvasTaskRequestSchema>;
|
||||||
|
export type CanvasGenerationTask = {
|
||||||
|
id: string;
|
||||||
|
requestId: string;
|
||||||
|
status: string;
|
||||||
|
createdAt: string;
|
||||||
|
request: CanvasTaskRequest;
|
||||||
|
result?: Record<string, any> | null;
|
||||||
|
error?: string | null;
|
||||||
|
};
|
||||||
|
export function isCanvasTaskActive(status: string) {
|
||||||
|
return status === "pending" || status === "running";
|
||||||
|
}
|
||||||
|
export function canCancelCanvasTask(status: string) {
|
||||||
|
return status === "pending";
|
||||||
|
}
|
||||||
|
|
||||||
|
export function requireAnalysisResult(data: Record<string, any> | undefined, mode: string) {
|
||||||
|
if (!data) throw Error("分析没有返回结果，原内容已保留");
|
||||||
|
const rows = data.analysisRows || data.shotIdeas;
|
||||||
|
if (
|
||||||
|
mode === "story-script" &&
|
||||||
|
(!Array.isArray(rows) ||
|
||||||
|
!rows.length ||
|
||||||
|
!rows.some((row) =>
|
||||||
|
[row.sceneDescription, row.description, row.imagePrompt].some(
|
||||||
|
(value) => typeof value === "string" && value.trim(),
|
||||||
|
),
|
||||||
|
))
|
||||||
|
) {
|
||||||
|
throw Error("脚本没有返回有效镜头，原内容已保留");
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
mode !== "story-script" &&
|
||||||
|
![data.raw, data.summary, ...(data.agentPlan || [])].some(
|
||||||
|
(value) => typeof value === "string" && value.trim(),
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
throw Error("分析返回了空内容，原内容已保留");
|
||||||
|
}
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stable result IDs and request IDs prevent repeat polling from duplicating images
|
||||||
|
// or an old task from overwriting a later generation on the same node.
|
||||||
|
export function reconcileCanvasTasks<
|
||||||
|
T extends { id: string; draftValues?: Record<string, any>; [key: string]: any },
|
||||||
|
>(nodes: T[], tasks: CanvasGenerationTask[], projectRevision: string) {
|
||||||
|
let result = nodes;
|
||||||
|
const latestByNode = new Map<string, CanvasGenerationTask>();
|
||||||
|
for (const task of tasks) {
|
||||||
|
const previous = latestByNode.get(task.request.node.id);
|
||||||
|
if (!previous || Date.parse(task.createdAt) >= Date.parse(previous.createdAt))
|
||||||
|
latestByNode.set(task.request.node.id, task);
|
||||||
|
}
|
||||||
|
for (const task of latestByNode.values()) {
|
||||||
|
const source = task.request.node;
|
||||||
|
const hidden = new Set(task.request.hiddenNodeIds || []);
|
||||||
|
if (hidden.has(source.id)) continue;
|
||||||
|
let index = result.findIndex((node) => node.id === source.id);
|
||||||
|
if (index < 0) {
|
||||||
|
if (Date.parse(task.createdAt) <= Date.parse(projectRevision)) continue;
|
||||||
|
result = [...result, source as unknown as T];
|
||||||
|
index = result.length - 1;
|
||||||
|
}
|
||||||
|
const node = result[index];
|
||||||
|
const draft = node.draftValues || {};
|
||||||
|
if (draft.taskRequestId && draft.taskRequestId !== task.requestId) {
|
||||||
|
const savedTask = tasks.find((item) => item.requestId === draft.taskRequestId);
|
||||||
|
if (
|
||||||
|
!savedTask ||
|
||||||
|
Date.parse(task.createdAt) <= Date.parse(savedTask.createdAt) ||
|
||||||
|
Date.parse(task.createdAt) <= Date.parse(projectRevision)
|
||||||
|
)
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (draft.appliedTaskStatus === `${task.id}:${task.status}`) continue;
|
||||||
|
const nextDraft = {
|
||||||
|
...draft,
|
||||||
|
taskRequestId: task.requestId,
|
||||||
|
generationTaskId: task.id,
|
||||||
|
appliedTaskStatus: `${task.id}:${task.status}`,
|
||||||
|
generationError: undefined as string | undefined,
|
||||||
|
generationProgress: 0,
|
||||||
|
generationStatus: "pending",
|
||||||
|
pendingLabel: task.status === "pending" ? "排队中" : "生成中",
|
||||||
|
};
|
||||||
|
let subtitle = nextDraft.pendingLabel;
|
||||||
|
if (task.status === "succeeded") {
|
||||||
|
const data = task.result || {};
|
||||||
|
Object.assign(nextDraft, {
|
||||||
|
generated: true,
|
||||||
|
generationStatus: "complete",
|
||||||
|
generationProgress: 100,
|
||||||
|
pendingLabel: "",
|
||||||
|
});
|
||||||
|
subtitle = "生成结果";
|
||||||
|
if (task.request.kind === "video")
|
||||||
|
Object.assign(nextDraft, { mediaUrl: data.url, url: [data.url], videoMeta: data.metadata });
|
||||||
|
if (task.request.kind === "image") {
|
||||||
|
const urls = (data.images || [])
|
||||||
|
.map((image: any) => image.originalUrl || image.thumbnailUrl)
|
||||||
|
.filter(Boolean);
|
||||||
|
Object.assign(nextDraft, {
|
||||||
|
mediaUrl: urls[0],
|
||||||
|
url: urls,
|
||||||
|
action: draft.action === "IMAGE_GENERATE_NINE_GRID" ? draft.action : "IMAGE_RESOURCE",
|
||||||
|
});
|
||||||
|
for (let i = 1; i < urls.length; i++) {
|
||||||
|
const id = `${source.id}-result-${task.id}-${i}`;
|
||||||
|
if (!hidden.has(id) && !result.some((item) => item.id === id))
|
||||||
|
result = [
|
||||||
|
...result,
|
||||||
|
{
|
||||||
|
...node,
|
||||||
|
id,
|
||||||
|
title: `${node.title} ${i + 1}`,
|
||||||
|
y: node.y + i * (node.height + 40),
|
||||||
|
draftValues: { ...nextDraft, mediaUrl: urls[i], url: [urls[i]] },
|
||||||
|
} as T,
|
||||||
|
];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (task.request.kind === "analysis")
|
||||||
|
Object.assign(nextDraft, {
|
||||||
|
content: [data.raw || data.summary || data.agentPlan?.join("\n") || ""],
|
||||||
|
...(Array.isArray(data.analysisRows || data.shotIdeas)
|
||||||
|
? { rows: data.analysisRows || data.shotIdeas }
|
||||||
|
: {}),
|
||||||
|
});
|
||||||
|
} else if (["failed", "cancelled", "interrupted"].includes(task.status)) {
|
||||||
|
Object.assign(nextDraft, {
|
||||||
|
generated: false,
|
||||||
|
generationStatus: "failed",
|
||||||
|
pendingLabel: "",
|
||||||
|
generationError:
|
||||||
|
task.error ||
|
||||||
|
(task.status === "cancelled" ? "已取消排队，未提交供应商" : "任务失败，请查看任务状态"),
|
||||||
|
});
|
||||||
|
subtitle = task.status === "cancelled" ? "已取消" : "生成失败";
|
||||||
|
}
|
||||||
|
result = result.map((item, i) =>
|
||||||
|
i === index
|
||||||
|
? {
|
||||||
|
...item,
|
||||||
|
...(task.status === "succeeded" && task.request.kind !== "analysis"
|
||||||
|
? { kind: "media", inputType: task.request.kind }
|
||||||
|
: {}),
|
||||||
|
subtitle,
|
||||||
|
draftValues: nextDraft,
|
||||||
|
}
|
||||||
|
: item,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
227
lib/video-canvas/task-store.ts
Normal file
227
lib/video-canvas/task-store.ts
Normal file
@ -0,0 +1,227 @@
|
|||||||
|
import "server-only";
|
||||||
|
import { and, asc, desc, eq, lt, sql } from "drizzle-orm";
|
||||||
|
import { db } from "@/lib/db";
|
||||||
|
import { canvasCapabilityTask as table, canvasProject } from "@/lib/db/schema";
|
||||||
|
import { type CanvasTaskRequest, type CanvasGenerationTask } from "./task-contract";
|
||||||
|
|
||||||
|
const capabilityId = "video-canvas.generation";
|
||||||
|
const scope = (userId: string, projectId: string) =>
|
||||||
|
and(
|
||||||
|
eq(table.userId, userId),
|
||||||
|
eq(table.projectId, projectId),
|
||||||
|
eq(table.capabilityId, capabilityId),
|
||||||
|
);
|
||||||
|
const serialize = (row: typeof table.$inferSelect): CanvasGenerationTask => ({
|
||||||
|
id: row.id,
|
||||||
|
requestId: row.requestId,
|
||||||
|
status: row.status,
|
||||||
|
createdAt: row.createdAt.toISOString(),
|
||||||
|
request: row.request as CanvasTaskRequest,
|
||||||
|
result: row.result as Record<string, any> | null,
|
||||||
|
error: row.error,
|
||||||
|
});
|
||||||
|
export async function ownsTaskProject(userId: string, projectId: string) {
|
||||||
|
return Boolean(
|
||||||
|
(
|
||||||
|
await db
|
||||||
|
.select({ id: canvasProject.id })
|
||||||
|
.from(canvasProject)
|
||||||
|
.where(and(eq(canvasProject.id, projectId), eq(canvasProject.userId, userId)))
|
||||||
|
.limit(1)
|
||||||
|
)[0],
|
||||||
|
);
|
||||||
|
}
|
||||||
|
export async function createCanvasGenerationTask(userId: string, request: CanvasTaskRequest) {
|
||||||
|
return db.transaction(async (tx) => {
|
||||||
|
await tx.execute(
|
||||||
|
sql`select pg_advisory_xact_lock(hashtext(${capabilityId}), hashtext(${request.projectId}))`,
|
||||||
|
);
|
||||||
|
const existing = (
|
||||||
|
await tx
|
||||||
|
.select()
|
||||||
|
.from(table)
|
||||||
|
.where(and(scope(userId, request.projectId), eq(table.requestId, request.requestId)))
|
||||||
|
.limit(1)
|
||||||
|
)[0];
|
||||||
|
if (existing) {
|
||||||
|
if (
|
||||||
|
JSON.stringify(existing.request) !== JSON.stringify(JSON.parse(JSON.stringify(request)))
|
||||||
|
) {
|
||||||
|
// JSONB property order is not stable. Compare the immutable logical request.
|
||||||
|
const old = existing.request as CanvasTaskRequest;
|
||||||
|
if (
|
||||||
|
old.node.id !== request.node.id ||
|
||||||
|
old.kind !== request.kind ||
|
||||||
|
JSON.stringify(sortObject(old.params)) !== JSON.stringify(sortObject(request.params))
|
||||||
|
)
|
||||||
|
throw Object.assign(Error("同一任务 ID 不能用于不同请求"), { status: 409 });
|
||||||
|
}
|
||||||
|
return serialize(existing);
|
||||||
|
}
|
||||||
|
const active = await tx
|
||||||
|
.select()
|
||||||
|
.from(table)
|
||||||
|
.where(and(scope(userId, request.projectId), sql`${table.status} in ('pending','running')`));
|
||||||
|
const duplicate = active.find(
|
||||||
|
(row) => (row.request as CanvasTaskRequest).node.id === request.node.id,
|
||||||
|
);
|
||||||
|
if (duplicate) return serialize(duplicate);
|
||||||
|
if (active.length >= 40)
|
||||||
|
throw Object.assign(Error("当前画布排队任务已达 40 个，请等待完成后继续"), { status: 429 });
|
||||||
|
const [created] = await tx
|
||||||
|
.insert(table)
|
||||||
|
.values({
|
||||||
|
requestId: request.requestId,
|
||||||
|
userId,
|
||||||
|
projectId: request.projectId,
|
||||||
|
capabilityId,
|
||||||
|
source: "video-canvas",
|
||||||
|
request,
|
||||||
|
createdAt: new Date(),
|
||||||
|
updatedAt: new Date(),
|
||||||
|
})
|
||||||
|
.returning();
|
||||||
|
return serialize(created);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
function sortObject(value: any): any {
|
||||||
|
return Array.isArray(value)
|
||||||
|
? value.map(sortObject)
|
||||||
|
: value && typeof value === "object"
|
||||||
|
? Object.fromEntries(
|
||||||
|
Object.keys(value)
|
||||||
|
.sort()
|
||||||
|
.map((key) => [key, sortObject(value[key])]),
|
||||||
|
)
|
||||||
|
: value;
|
||||||
|
}
|
||||||
|
export async function listCanvasGenerationTasks(userId: string, projectId: string) {
|
||||||
|
// A worker lost to process shutdown must never silently re-submit a paid request.
|
||||||
|
await db
|
||||||
|
.update(table)
|
||||||
|
.set({
|
||||||
|
status: "interrupted",
|
||||||
|
error:
|
||||||
|
"服务端执行连接中断，供应商任务可能仍在运行。请核实结果后再决定是否重新生成，系统不会自动重复提交。",
|
||||||
|
updatedAt: new Date(),
|
||||||
|
})
|
||||||
|
.where(
|
||||||
|
and(
|
||||||
|
scope(userId, projectId),
|
||||||
|
eq(table.status, "running"),
|
||||||
|
lt(table.updatedAt, new Date(Date.now() - 120000)),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
const rows = await db
|
||||||
|
.select()
|
||||||
|
.from(table)
|
||||||
|
.where(scope(userId, projectId))
|
||||||
|
.orderBy(desc(table.createdAt))
|
||||||
|
.limit(240);
|
||||||
|
return rows.map(serialize).reverse();
|
||||||
|
}
|
||||||
|
export async function claimCanvasGenerationTask(userId: string, projectId: string) {
|
||||||
|
return db.transaction(async (tx) => {
|
||||||
|
await tx.execute(
|
||||||
|
sql`select pg_advisory_xact_lock(hashtext(${capabilityId}), hashtext(${projectId}))`,
|
||||||
|
);
|
||||||
|
if (
|
||||||
|
(
|
||||||
|
await tx
|
||||||
|
.select({ id: table.id })
|
||||||
|
.from(table)
|
||||||
|
.where(and(scope(userId, projectId), eq(table.status, "running")))
|
||||||
|
.limit(1)
|
||||||
|
).length
|
||||||
|
)
|
||||||
|
return null;
|
||||||
|
const [next] = await tx
|
||||||
|
.select()
|
||||||
|
.from(table)
|
||||||
|
.where(and(scope(userId, projectId), eq(table.status, "pending")))
|
||||||
|
.orderBy(asc(table.createdAt))
|
||||||
|
.limit(1);
|
||||||
|
if (!next) return null;
|
||||||
|
const [claimed] = await tx
|
||||||
|
.update(table)
|
||||||
|
.set({
|
||||||
|
status: "running",
|
||||||
|
startedAt: new Date(),
|
||||||
|
updatedAt: new Date(),
|
||||||
|
attempts: sql`${table.attempts}+1`,
|
||||||
|
})
|
||||||
|
.where(and(eq(table.id, next.id), eq(table.status, "pending")))
|
||||||
|
.returning();
|
||||||
|
return claimed ? serialize(claimed) : null;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
export async function heartbeatCanvasGenerationTask(id: string) {
|
||||||
|
await db
|
||||||
|
.update(table)
|
||||||
|
.set({ updatedAt: new Date() })
|
||||||
|
.where(and(eq(table.id, id), eq(table.status, "running")));
|
||||||
|
}
|
||||||
|
export async function finishCanvasGenerationTask(
|
||||||
|
id: string,
|
||||||
|
result?: Record<string, any>,
|
||||||
|
error?: string,
|
||||||
|
) {
|
||||||
|
// A late real result can recover an interrupted task, but cannot replace a newer request ID.
|
||||||
|
await db
|
||||||
|
.update(table)
|
||||||
|
.set({
|
||||||
|
status: result ? "succeeded" : "failed",
|
||||||
|
result: result || null,
|
||||||
|
error: error || null,
|
||||||
|
updatedAt: new Date(),
|
||||||
|
completedAt: new Date(),
|
||||||
|
})
|
||||||
|
.where(and(eq(table.id, id), sql`${table.status} in ('running','interrupted')`));
|
||||||
|
}
|
||||||
|
export async function cancelCanvasGenerationTask(userId: string, projectId: string, id: string) {
|
||||||
|
const [cancelled] = await db
|
||||||
|
.update(table)
|
||||||
|
.set({
|
||||||
|
status: "cancelled",
|
||||||
|
error: "已取消排队，未提交供应商",
|
||||||
|
updatedAt: new Date(),
|
||||||
|
completedAt: new Date(),
|
||||||
|
})
|
||||||
|
.where(and(scope(userId, projectId), eq(table.id, id), eq(table.status, "pending")))
|
||||||
|
.returning();
|
||||||
|
if (cancelled) return { ok: true, task: serialize(cancelled) };
|
||||||
|
const [current] = await db
|
||||||
|
.select()
|
||||||
|
.from(table)
|
||||||
|
.where(and(scope(userId, projectId), eq(table.id, id)))
|
||||||
|
.limit(1);
|
||||||
|
return { ok: current?.status === "cancelled", task: current ? serialize(current) : null };
|
||||||
|
}
|
||||||
|
export async function hideCanvasGenerationNode(
|
||||||
|
userId: string,
|
||||||
|
projectId: string,
|
||||||
|
id: string,
|
||||||
|
nodeId: string,
|
||||||
|
) {
|
||||||
|
return db.transaction(async (tx) => {
|
||||||
|
const [task] = await tx
|
||||||
|
.select()
|
||||||
|
.from(table)
|
||||||
|
.where(and(scope(userId, projectId), eq(table.id, id)))
|
||||||
|
.for("update");
|
||||||
|
if (!task) return false;
|
||||||
|
const request = task.request as CanvasTaskRequest & { hiddenNodeIds?: string[] };
|
||||||
|
if (nodeId !== request.node.id && !nodeId.startsWith(`${request.node.id}-result-`))
|
||||||
|
return false;
|
||||||
|
await tx
|
||||||
|
.update(table)
|
||||||
|
.set({
|
||||||
|
request: {
|
||||||
|
...request,
|
||||||
|
hiddenNodeIds: [...new Set([...(request.hiddenNodeIds || []), nodeId])],
|
||||||
|
},
|
||||||
|
})
|
||||||
|
.where(eq(table.id, id));
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
}
|
||||||
78
lib/video-canvas/task-worker.ts
Normal file
78
lib/video-canvas/task-worker.ts
Normal file
@ -0,0 +1,78 @@
|
|||||||
|
import "server-only";
|
||||||
|
import { executeDirectVideoGeneration } from "@/lib/ai/tools/generate-video";
|
||||||
|
import { executeCanvasImageResponse } from "@/lib/canvas/generate-image-response";
|
||||||
|
import { executeWorkflowAnalysisResponse } from "./workflow-analysis-response";
|
||||||
|
import { requireAnalysisResult, type CanvasTaskRequest } from "./task-contract";
|
||||||
|
import { validateVideoReferences } from "./reference-input";
|
||||||
|
import { probeRemoteUrl } from "@/lib/net/url-access";
|
||||||
|
import {
|
||||||
|
claimCanvasGenerationTask,
|
||||||
|
finishCanvasGenerationTask,
|
||||||
|
heartbeatCanvasGenerationTask,
|
||||||
|
} from "./task-store";
|
||||||
|
|
||||||
|
async function execute(request: CanvasTaskRequest, userId: string) {
|
||||||
|
const params = request.params;
|
||||||
|
if (request.kind === "video") {
|
||||||
|
const checked = await validateVideoReferences(params.images, (url) =>
|
||||||
|
probeRemoteUrl(url, { timeoutMs: 8000 }),
|
||||||
|
);
|
||||||
|
if (checked.error) throw Error(checked.error);
|
||||||
|
const result = await executeDirectVideoGeneration({
|
||||||
|
...params,
|
||||||
|
prompt: String(params.prompt || ""),
|
||||||
|
mode: params.mode as any,
|
||||||
|
images: checked.images,
|
||||||
|
});
|
||||||
|
if (!result.success || !result.videoUrl) throw Error("视频生成未返回结果");
|
||||||
|
return {
|
||||||
|
url: result.videoUrl,
|
||||||
|
metadata: {
|
||||||
|
...result.metadata,
|
||||||
|
temporaryVideoUrl: result.videoUrl,
|
||||||
|
storageStatus:
|
||||||
|
result.metadata?.permanentVideoUrl === result.videoUrl ? "archived" : "temporary",
|
||||||
|
},
|
||||||
|
modelId: result.modelId,
|
||||||
|
modelRef: result.modelRef,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const response =
|
||||||
|
request.kind === "image"
|
||||||
|
? await executeCanvasImageResponse(
|
||||||
|
{ ...params, prompt: String(params.prompt || ""), async: false },
|
||||||
|
userId,
|
||||||
|
)
|
||||||
|
: await executeWorkflowAnalysisResponse(params);
|
||||||
|
const body = await response.json();
|
||||||
|
if (!response.ok || !body.success) throw Error(body.error || "任务执行失败");
|
||||||
|
if (
|
||||||
|
request.kind === "image" &&
|
||||||
|
!body.data?.images?.some((image: any) => image.originalUrl || image.thumbnailUrl)
|
||||||
|
)
|
||||||
|
throw Error("图片生成没有返回结果");
|
||||||
|
if (request.kind === "analysis") requireAnalysisResult(body.data, String(params.mode));
|
||||||
|
return body.data;
|
||||||
|
}
|
||||||
|
export async function runCanvasGenerationQueue(userId: string, projectId: string) {
|
||||||
|
// Work is independent of the browser request. Atomic DB claiming also covers multiple processes.
|
||||||
|
for (let i = 0; i < 40; i++) {
|
||||||
|
const task = await claimCanvasGenerationTask(userId, projectId);
|
||||||
|
if (!task) return;
|
||||||
|
const timer = setInterval(
|
||||||
|
() => void heartbeatCanvasGenerationTask(task.id).catch(() => {}),
|
||||||
|
20000,
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
await finishCanvasGenerationTask(task.id, await execute(task.request, userId));
|
||||||
|
} catch (error) {
|
||||||
|
await finishCanvasGenerationTask(
|
||||||
|
task.id,
|
||||||
|
undefined,
|
||||||
|
error instanceof Error ? error.message : "任务执行失败",
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
clearInterval(timer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
797
lib/video-canvas/workflow-analysis-response.ts
Normal file
797
lib/video-canvas/workflow-analysis-response.ts
Normal file
@ -0,0 +1,797 @@
|
|||||||
|
import "server-only";
|
||||||
|
import { normalizeStoryScriptRows } from "@/lib/video-canvas/story-script";
|
||||||
|
import { isCanvasLanguageModel, isCanvasVisionModel } from "@/lib/video-canvas/model-matching";
|
||||||
|
import { generateText } from "ai";
|
||||||
|
import { execFile } from "node:child_process";
|
||||||
|
import { promisify } from "node:util";
|
||||||
|
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||||||
|
import os from "node:os";
|
||||||
|
import path from "node:path";
|
||||||
|
import { NextResponse } from "next/server";
|
||||||
|
import sharp from "sharp";
|
||||||
|
import { z } from "zod";
|
||||||
|
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
|
||||||
|
import { inlineVisionImages } from "@/lib/ai/vision-inline";
|
||||||
|
import { getAiModelAgentCodingModelId, isAiModelMultimodal } from "@/lib/db/ai-model-flags";
|
||||||
|
import { getAgentAiModelWithProvider, getEnabledAiModelsWithProvider } from "@/lib/db/queries";
|
||||||
|
|
||||||
|
const execFileAsync = promisify(execFile);
|
||||||
|
const VIDEO_ANALYSIS_FRAME_COUNT = 4;
|
||||||
|
const VIDEO_ANALYSIS_MAX_SIDE = 960;
|
||||||
|
const VIDEO_ANALYSIS_TIMEOUT_MS = 15000;
|
||||||
|
|
||||||
|
const requestSchema = z.object({
|
||||||
|
mode: z.enum(["video-analysis", "character-scene-extraction", "story-script", "agent-plan"]),
|
||||||
|
prompt: z.string().trim().optional(),
|
||||||
|
imageUrls: z.array(z.string().trim()).optional(),
|
||||||
|
videoUrls: z.array(z.string().trim()).optional(),
|
||||||
|
nodeTitle: z.string().trim().optional(),
|
||||||
|
preferredModelRef: z.string().trim().optional(),
|
||||||
|
model: z.string().trim().optional(),
|
||||||
|
});
|
||||||
|
|
||||||
|
function sanitizeUrls(value: unknown, limit = 6) {
|
||||||
|
if (!Array.isArray(value)) return [];
|
||||||
|
return value
|
||||||
|
.map((item) => String(item || "").trim())
|
||||||
|
.filter((url) => /^https?:\/\//i.test(url))
|
||||||
|
.slice(0, limit);
|
||||||
|
}
|
||||||
|
|
||||||
|
function supportsVision(model: {
|
||||||
|
supportsVision?: boolean | null;
|
||||||
|
capabilities?: unknown;
|
||||||
|
defaultParams?: unknown;
|
||||||
|
}) {
|
||||||
|
return (
|
||||||
|
Boolean(model.supportsVision) ||
|
||||||
|
isAiModelMultimodal({
|
||||||
|
capabilities: model.capabilities,
|
||||||
|
defaultParams: model.defaultParams,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function extractJsonBlock<T = Record<string, unknown>>(text: string): T | null {
|
||||||
|
const matched = String(text || "").match(/\{[\s\S]*\}/);
|
||||||
|
if (!matched) return null;
|
||||||
|
try {
|
||||||
|
return JSON.parse(matched[0]) as T;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeText(value: unknown) {
|
||||||
|
return String(value || "").trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeStringList(value: unknown, limit = 8) {
|
||||||
|
if (!Array.isArray(value)) return [];
|
||||||
|
return Array.from(
|
||||||
|
new Set(
|
||||||
|
value
|
||||||
|
.map((item) => String(item || "").trim())
|
||||||
|
.filter(Boolean)
|
||||||
|
.slice(0, limit),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeShotIdeas(value: unknown) {
|
||||||
|
if (!Array.isArray(value)) return [];
|
||||||
|
return value
|
||||||
|
.map((item, index) => {
|
||||||
|
const raw =
|
||||||
|
item && typeof item === "object" && !Array.isArray(item)
|
||||||
|
? (item as Record<string, unknown>)
|
||||||
|
: {};
|
||||||
|
const title = normalizeText(raw.title || `镜头 ${index + 1}`);
|
||||||
|
const description = normalizeText(raw.description || raw.prompt || raw.summary);
|
||||||
|
if (!title && !description) return null;
|
||||||
|
return {
|
||||||
|
title: title || `镜头 ${index + 1}`,
|
||||||
|
description,
|
||||||
|
};
|
||||||
|
})
|
||||||
|
.filter((item): item is { title: string; description: string } => Boolean(item))
|
||||||
|
.slice(0, 6);
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeVideoAnalysisRows(value: unknown, framePreviews: string[]) {
|
||||||
|
if (!Array.isArray(value)) return [];
|
||||||
|
return value
|
||||||
|
.map((item, index) => {
|
||||||
|
const raw =
|
||||||
|
item && typeof item === "object" && !Array.isArray(item)
|
||||||
|
? (item as Record<string, unknown>)
|
||||||
|
: {};
|
||||||
|
const sceneDescription = normalizeText(
|
||||||
|
raw.sceneDescription || raw.scene || raw.title || raw.description,
|
||||||
|
);
|
||||||
|
const narrative = normalizeText(
|
||||||
|
raw.narrative || raw.story || raw.storyContent || raw.narration || raw.summary,
|
||||||
|
);
|
||||||
|
const shot = normalizeText(raw.shot || raw.shotNumber || raw.index || index + 1);
|
||||||
|
const startTime = normalizeText(raw.startTime || raw.start || raw.start_time);
|
||||||
|
const endTime = normalizeText(raw.endTime || raw.end || raw.end_time);
|
||||||
|
const duration = normalizeText(raw.duration || raw.durationSeconds || raw.duration_seconds);
|
||||||
|
const shotSize = normalizeText(
|
||||||
|
raw.shotSize || raw.frameSize || raw.shot_type || raw.sceneScale,
|
||||||
|
);
|
||||||
|
const cameraAngle = normalizeText(raw.cameraAngle || raw.angle || raw.camera_angle);
|
||||||
|
const cameraMovement = normalizeText(
|
||||||
|
raw.cameraMovement || raw.cameraMove || raw.camera_motion || raw.motionType,
|
||||||
|
);
|
||||||
|
const focusDepth = normalizeText(
|
||||||
|
raw.focusDepth || raw.depthOfField || raw.focalDepth || raw.focus_depth,
|
||||||
|
);
|
||||||
|
const lighting = normalizeText(raw.lighting || raw.light || raw.lightDesign);
|
||||||
|
const backgroundMusic = normalizeText(raw.backgroundMusic || raw.music || raw.bgm);
|
||||||
|
const voiceAndSfx = normalizeText(raw.voiceAndSfx || raw.sound || raw.audio || raw.sfx);
|
||||||
|
const imagePrompt = normalizeText(
|
||||||
|
raw.imagePrompt || raw.image_generation_prompt || raw.image,
|
||||||
|
);
|
||||||
|
const motionPrompt = normalizeText(
|
||||||
|
raw.motionPrompt || raw.motion || raw.videoPrompt || raw.video_motion_prompt,
|
||||||
|
);
|
||||||
|
const keyframeIndex = Number(
|
||||||
|
raw.keyframeIndex || raw.frameIndex || raw.key_frame_index || index + 1,
|
||||||
|
);
|
||||||
|
if (
|
||||||
|
!sceneDescription &&
|
||||||
|
!narrative &&
|
||||||
|
!backgroundMusic &&
|
||||||
|
!voiceAndSfx &&
|
||||||
|
!imagePrompt &&
|
||||||
|
!motionPrompt &&
|
||||||
|
!shotSize &&
|
||||||
|
!cameraAngle &&
|
||||||
|
!cameraMovement &&
|
||||||
|
!focusDepth &&
|
||||||
|
!lighting
|
||||||
|
) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const normalizedFrameIndex =
|
||||||
|
Number.isFinite(keyframeIndex) && keyframeIndex > 0
|
||||||
|
? Math.max(0, Math.min(framePreviews.length - 1, Math.round(keyframeIndex) - 1))
|
||||||
|
: index % Math.max(framePreviews.length, 1);
|
||||||
|
return {
|
||||||
|
shot: shot || String(index + 1),
|
||||||
|
startTime: startTime || (index === 0 ? "0" : ""),
|
||||||
|
endTime,
|
||||||
|
duration,
|
||||||
|
sceneDescription: sceneDescription || `镜头 ${index + 1}`,
|
||||||
|
narrative: narrative || sceneDescription || `镜头 ${index + 1}`,
|
||||||
|
shotSize: shotSize || "中景至近景",
|
||||||
|
cameraAngle: cameraAngle || "平视",
|
||||||
|
cameraMovement: cameraMovement || "缓慢推镜头 (Slow Zoom In)",
|
||||||
|
focusDepth: focusDepth || "浅景深",
|
||||||
|
lighting: lighting || "冷调低沉照明，主体光线柔和。",
|
||||||
|
backgroundMusic: backgroundMusic || "无",
|
||||||
|
voiceAndSfx: voiceAndSfx || "无",
|
||||||
|
imagePrompt: imagePrompt || sceneDescription || "延续当前视频气质生成关键帧",
|
||||||
|
motionPrompt: motionPrompt || "保持镜头运动自然、连贯、符合前后节奏。",
|
||||||
|
keyframeUrl: framePreviews[normalizedFrameIndex] || "",
|
||||||
|
};
|
||||||
|
})
|
||||||
|
.filter(
|
||||||
|
(
|
||||||
|
item,
|
||||||
|
): item is {
|
||||||
|
shot: string;
|
||||||
|
startTime: string;
|
||||||
|
endTime: string;
|
||||||
|
duration: string;
|
||||||
|
sceneDescription: string;
|
||||||
|
narrative: string;
|
||||||
|
shotSize: string;
|
||||||
|
cameraAngle: string;
|
||||||
|
cameraMovement: string;
|
||||||
|
focusDepth: string;
|
||||||
|
lighting: string;
|
||||||
|
backgroundMusic: string;
|
||||||
|
voiceAndSfx: string;
|
||||||
|
imagePrompt: string;
|
||||||
|
motionPrompt: string;
|
||||||
|
keyframeUrl: string;
|
||||||
|
} => Boolean(item),
|
||||||
|
)
|
||||||
|
.slice(0, 8);
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeCharacters(value: unknown) {
|
||||||
|
if (!Array.isArray(value)) return [];
|
||||||
|
return value
|
||||||
|
.map((item) => {
|
||||||
|
const raw =
|
||||||
|
item && typeof item === "object" && !Array.isArray(item)
|
||||||
|
? (item as Record<string, unknown>)
|
||||||
|
: {};
|
||||||
|
const name = normalizeText(raw.name || raw.title);
|
||||||
|
const description = normalizeText(raw.description || raw.summary);
|
||||||
|
const prompt = normalizeText(raw.prompt);
|
||||||
|
if (!name && !description && !prompt) return null;
|
||||||
|
return {
|
||||||
|
name: name || "未命名角色",
|
||||||
|
description,
|
||||||
|
prompt,
|
||||||
|
look: normalizeText(raw.look) || undefined,
|
||||||
|
wardrobe: normalizeText(raw.wardrobe) || undefined,
|
||||||
|
signatureAction: normalizeText(raw.signatureAction) || undefined,
|
||||||
|
traits: normalizeStringList(raw.traits, 6),
|
||||||
|
};
|
||||||
|
})
|
||||||
|
.filter((item): item is NonNullable<typeof item> => Boolean(item))
|
||||||
|
.slice(0, 8);
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeScenes(value: unknown) {
|
||||||
|
if (!Array.isArray(value)) return [];
|
||||||
|
return value
|
||||||
|
.map((item) => {
|
||||||
|
const raw =
|
||||||
|
item && typeof item === "object" && !Array.isArray(item)
|
||||||
|
? (item as Record<string, unknown>)
|
||||||
|
: {};
|
||||||
|
const name = normalizeText(raw.name || raw.title);
|
||||||
|
const description = normalizeText(raw.description || raw.summary);
|
||||||
|
const prompt = normalizeText(raw.prompt);
|
||||||
|
if (!name && !description && !prompt) return null;
|
||||||
|
return {
|
||||||
|
name: name || "未命名场景",
|
||||||
|
description,
|
||||||
|
prompt,
|
||||||
|
mood: normalizeText(raw.mood) || undefined,
|
||||||
|
lighting: normalizeText(raw.lighting) || undefined,
|
||||||
|
props: normalizeStringList(raw.props, 6),
|
||||||
|
};
|
||||||
|
})
|
||||||
|
.filter((item): item is NonNullable<typeof item> => Boolean(item))
|
||||||
|
.slice(0, 8);
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildVideoAnalysisPrompt(params: {
|
||||||
|
nodeTitle?: string;
|
||||||
|
prompt: string;
|
||||||
|
frameCount: number;
|
||||||
|
imageCount: number;
|
||||||
|
videoCount: number;
|
||||||
|
}) {
|
||||||
|
return [
|
||||||
|
"你是视频画布工作流里的视频分析节点，需要为后续分镜和生成节点提供结构化输出。",
|
||||||
|
params.nodeTitle ? `当前节点：${params.nodeTitle}` : "",
|
||||||
|
`上游参考图数量：${params.imageCount}`,
|
||||||
|
`上游视频数量：${params.videoCount}`,
|
||||||
|
`已提供按时间顺序抽取的视频关键帧数量：${params.frameCount}`,
|
||||||
|
params.prompt
|
||||||
|
? `上游脚本/说明：\n${params.prompt}`
|
||||||
|
: "上游没有额外脚本，请重点依据参考画面分析。",
|
||||||
|
"",
|
||||||
|
"请只输出 JSON，不要输出 markdown，不要输出解释。",
|
||||||
|
"JSON 结构如下：",
|
||||||
|
`{
|
||||||
|
"summary": "一句话总结当前视频/镜头风格，不超过120字",
|
||||||
|
"highlights": ["3-5条镜头分析要点"],
|
||||||
|
"analysisRows": [
|
||||||
|
{
|
||||||
|
"shot": "1",
|
||||||
|
"startTime": "0",
|
||||||
|
"endTime": "5",
|
||||||
|
"duration": "5",
|
||||||
|
"sceneDescription": "画面描述，说明主体、环境、动作、构图和色调",
|
||||||
|
"narrative": "叙事内容，说明这一镜头承担的剧情/情绪/信息作用",
|
||||||
|
"shotSize": "景别，例如中景至近景",
|
||||||
|
"cameraAngle": "摄影机角度，例如平视",
|
||||||
|
"cameraMovement": "摄影机运动，例如缓慢推镜头 (Slow Zoom In)",
|
||||||
|
"focusDepth": "焦距与景深，例如浅景深",
|
||||||
|
"lighting": "光线，例如冷调低沉照明，屏幕光作为侧逆光",
|
||||||
|
"backgroundMusic": "这一段适合的背景音乐氛围",
|
||||||
|
"voiceAndSfx": "人声/环境声/音效建议，没有就写无",
|
||||||
|
"imagePrompt": "后续生成关键帧时可直接使用的英文图像提示词",
|
||||||
|
"motionPrompt": "后续生成视频时可直接使用的英文运动提示词",
|
||||||
|
"keyframeIndex": 1
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"shotIdeas": [
|
||||||
|
{ "title": "镜头 1", "description": "一句话描述镜头目标、景别或动作" }
|
||||||
|
],
|
||||||
|
"visualMotifs": ["2-4个可复用的视觉母题"],
|
||||||
|
"recommendedNextSteps": ["2-4条建议的下游动作"]
|
||||||
|
}`,
|
||||||
|
"analysisRows 请按镜头时间顺序输出；短视频可以输出 1 行，复杂视频输出 2-6 行。",
|
||||||
|
"每行必须覆盖镜号、开始时间、结束时间、时长、画面描述、叙事内容、景别、摄影机角度、摄影机运动、焦距与景深、光线、背景音乐、人声/音效、图像生成提示词、视频运动提示词和关键帧。",
|
||||||
|
"imagePrompt 和 motionPrompt 优先使用英文；其他字段使用中文；keyframeIndex 使用 1 开始的序号，对应你看到的关键帧顺序。",
|
||||||
|
"如果没有真实视频帧，只能依据参考图和脚本推断，也要正常输出，但措辞要稳健。",
|
||||||
|
]
|
||||||
|
.filter(Boolean)
|
||||||
|
.join("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getVideoDurationSeconds(videoUrl: string) {
|
||||||
|
try {
|
||||||
|
const { stdout } = await execFileAsync(
|
||||||
|
"ffprobe",
|
||||||
|
[
|
||||||
|
"-v",
|
||||||
|
"error",
|
||||||
|
"-show_entries",
|
||||||
|
"format=duration",
|
||||||
|
"-of",
|
||||||
|
"default=noprint_wrappers=1:nokey=1",
|
||||||
|
videoUrl,
|
||||||
|
],
|
||||||
|
{
|
||||||
|
timeout: VIDEO_ANALYSIS_TIMEOUT_MS,
|
||||||
|
maxBuffer: 1024 * 1024,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const duration = Number.parseFloat(String(stdout || "").trim());
|
||||||
|
return Number.isFinite(duration) && duration > 0 ? duration : null;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildVideoFrameTimestamps(durationSeconds: number | null, frameCount: number) {
|
||||||
|
const safeCount = Math.max(1, frameCount);
|
||||||
|
if (!durationSeconds || !Number.isFinite(durationSeconds) || durationSeconds <= 0.5) {
|
||||||
|
return Array.from({ length: safeCount }).map((_, index) =>
|
||||||
|
Number((0.3 + index * 0.45).toFixed(2)),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const start = Math.min(0.35, durationSeconds * 0.12);
|
||||||
|
const end = Math.max(start + 0.6, durationSeconds * 0.88);
|
||||||
|
if (safeCount === 1) return [Number(((start + end) / 2).toFixed(2))];
|
||||||
|
const step = (end - start) / (safeCount - 1);
|
||||||
|
return Array.from({ length: safeCount }).map((_, index) =>
|
||||||
|
Number((start + step * index).toFixed(2)),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function downscaleVideoFrameForVision(inputBuffer: Buffer) {
|
||||||
|
return sharp(inputBuffer, { failOn: "none" })
|
||||||
|
.rotate()
|
||||||
|
.resize({
|
||||||
|
width: VIDEO_ANALYSIS_MAX_SIDE,
|
||||||
|
height: VIDEO_ANALYSIS_MAX_SIDE,
|
||||||
|
fit: "inside",
|
||||||
|
withoutEnlargement: true,
|
||||||
|
})
|
||||||
|
.jpeg({ quality: 82, mozjpeg: true })
|
||||||
|
.toBuffer();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function extractVideoFrameDataUrls(
|
||||||
|
videoUrl: string,
|
||||||
|
frameCount = VIDEO_ANALYSIS_FRAME_COUNT,
|
||||||
|
) {
|
||||||
|
const tempDir = await mkdtemp(path.join(os.tmpdir(), "canvas-video-analysis-"));
|
||||||
|
try {
|
||||||
|
const durationSeconds = await getVideoDurationSeconds(videoUrl);
|
||||||
|
const timestamps = buildVideoFrameTimestamps(durationSeconds, frameCount);
|
||||||
|
const frames = await Promise.all(
|
||||||
|
timestamps.map(async (timestamp, index) => {
|
||||||
|
const outputPath = path.join(tempDir, `frame-${index + 1}.jpg`);
|
||||||
|
await execFileAsync(
|
||||||
|
"ffmpeg",
|
||||||
|
[
|
||||||
|
"-hide_banner",
|
||||||
|
"-loglevel",
|
||||||
|
"error",
|
||||||
|
"-ss",
|
||||||
|
String(timestamp),
|
||||||
|
"-i",
|
||||||
|
videoUrl,
|
||||||
|
"-frames:v",
|
||||||
|
"1",
|
||||||
|
"-q:v",
|
||||||
|
"4",
|
||||||
|
"-y",
|
||||||
|
outputPath,
|
||||||
|
],
|
||||||
|
{
|
||||||
|
timeout: VIDEO_ANALYSIS_TIMEOUT_MS,
|
||||||
|
maxBuffer: 2 * 1024 * 1024,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const buffer = await readFile(outputPath);
|
||||||
|
const processed = await downscaleVideoFrameForVision(buffer);
|
||||||
|
return `data:image/jpeg;base64,${processed.toString("base64")}`;
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
return frames.filter(Boolean);
|
||||||
|
} catch (error) {
|
||||||
|
console.warn("[Canvas Workflow Analyze] extract video frames failed:", {
|
||||||
|
message: error instanceof Error ? error.message : String(error),
|
||||||
|
videoUrl,
|
||||||
|
});
|
||||||
|
return [];
|
||||||
|
} finally {
|
||||||
|
await rm(tempDir, { force: true, recursive: true }).catch(() => undefined);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildCharacterScenePrompt(params: {
|
||||||
|
nodeTitle?: string;
|
||||||
|
prompt: string;
|
||||||
|
imageCount: number;
|
||||||
|
}) {
|
||||||
|
return [
|
||||||
|
"你是视频画布工作流里的角色场景提取节点，需要从脚本、设定和参考图里提取角色与场景资产。",
|
||||||
|
params.nodeTitle ? `当前节点：${params.nodeTitle}` : "",
|
||||||
|
`参考图数量：${params.imageCount}`,
|
||||||
|
params.prompt
|
||||||
|
? `输入内容：\n${params.prompt}`
|
||||||
|
: "没有额外脚本，请根据参考图提取角色和场景线索。",
|
||||||
|
"",
|
||||||
|
"请只输出 JSON，不要输出 markdown，不要输出解释。",
|
||||||
|
"JSON 结构如下：",
|
||||||
|
`{
|
||||||
|
"summary": "一句话总结当前故事世界与资产线索",
|
||||||
|
"characters": [
|
||||||
|
{
|
||||||
|
"name": "角色名",
|
||||||
|
"description": "角色外观与身份简介",
|
||||||
|
"look": "外貌特征",
|
||||||
|
"wardrobe": "服装/道具",
|
||||||
|
"traits": ["性格关键词"],
|
||||||
|
"signatureAction": "代表动作",
|
||||||
|
"prompt": "可直接用于生成角色图/角色视频的提示词"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"scenes": [
|
||||||
|
{
|
||||||
|
"name": "场景名",
|
||||||
|
"description": "空间、布景和氛围简介",
|
||||||
|
"mood": "情绪与气氛",
|
||||||
|
"lighting": "布光特征",
|
||||||
|
"props": ["核心道具"],
|
||||||
|
"prompt": "可直接用于生成场景图/场景视频的提示词"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"relationships": ["角色与场景或角色之间的关系"],
|
||||||
|
"recommendedNextSteps": ["建议连接的后续节点动作"]
|
||||||
|
}`,
|
||||||
|
"角色和场景都要尽量具体，避免空泛形容词。",
|
||||||
|
]
|
||||||
|
.filter(Boolean)
|
||||||
|
.join("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildAgentPlanPrompt(params: {
|
||||||
|
nodeTitle?: string;
|
||||||
|
prompt: string;
|
||||||
|
imageCount: number;
|
||||||
|
videoCount: number;
|
||||||
|
}) {
|
||||||
|
return [
|
||||||
|
"你是视频画布里的自定义代理节点，需要把当前创作目标拆成可执行的工作流计划。",
|
||||||
|
params.nodeTitle ? `当前节点：${params.nodeTitle}` : "",
|
||||||
|
`参考图数量：${params.imageCount}`,
|
||||||
|
`视频数量：${params.videoCount}`,
|
||||||
|
params.prompt ? `上游需求：\n${params.prompt}` : "没有额外脚本，请基于当前上下文给出稳妥计划。",
|
||||||
|
"",
|
||||||
|
"请只输出 JSON，不要输出 markdown，不要输出解释。",
|
||||||
|
"JSON 结构如下：",
|
||||||
|
`{
|
||||||
|
"summary": "一句话说明这个代理链的目标",
|
||||||
|
"agentPlan": ["3-6个执行步骤"],
|
||||||
|
"deliverables": ["2-4个预期交付物"],
|
||||||
|
"recommendedNodes": ["建议继续连接的节点类型或能力名"]
|
||||||
|
}`,
|
||||||
|
"步骤要偏工作流视角，而不是泛泛而谈。",
|
||||||
|
]
|
||||||
|
.filter(Boolean)
|
||||||
|
.join("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildStoryScriptPrompt(params: {
|
||||||
|
nodeTitle?: string;
|
||||||
|
prompt: string;
|
||||||
|
imageCount: number;
|
||||||
|
videoCount: number;
|
||||||
|
}) {
|
||||||
|
return [
|
||||||
|
"你是视频画布里的故事脚本生成节点，需要把用户输入、参考图和参考视频整理成可直接用于后续分镜/生图/生视频的结构化脚本表。",
|
||||||
|
params.nodeTitle ? `当前节点：${params.nodeTitle}` : "",
|
||||||
|
`参考图数量：${params.imageCount}`,
|
||||||
|
`视频数量：${params.videoCount}`,
|
||||||
|
params.prompt
|
||||||
|
? `输入内容：\n${params.prompt}`
|
||||||
|
: "没有额外脚本，请生成一个节奏清晰的短视频脚本。",
|
||||||
|
"",
|
||||||
|
"请只输出 JSON，不要输出 markdown，不要输出解释。",
|
||||||
|
"JSON 结构如下：",
|
||||||
|
`{
|
||||||
|
"summary": "一句话概括故事主题、情绪和视频方向，不超过120字",
|
||||||
|
"analysisRows": [
|
||||||
|
{
|
||||||
|
"shot": "01",
|
||||||
|
"duration": "5s",
|
||||||
|
"scene": "场景名或地点，如 盛唐·金銮殿·日",
|
||||||
|
"sceneDescription": "画面内容、人物动作和剧情推进，不要重复 scene 字段",
|
||||||
|
"shotSize": "景别，如中景、特写、全景、大远景",
|
||||||
|
"lighting": "光影氛围，如暖金主光带阴影，气氛紧张压抑",
|
||||||
|
"dialogue": "对白或旁白，没有就写 +",
|
||||||
|
"backgroundMusic": "背景音乐或情绪氛围，没有就写无",
|
||||||
|
"voiceAndSfx": "环境声、动作声或音效，没有就写无",
|
||||||
|
"cameraMovement": "运镜，如缓慢推镜头、固定、横移、上摇、环绕轨道跟拍",
|
||||||
|
"imagePrompt": "后续生成该镜头关键帧时可直接使用的中文图像提示词",
|
||||||
|
"motionPrompt": "后续生成视频时可直接使用的中文视频运动提示词"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"assetCandidates": {
|
||||||
|
"style": "全局视觉风格，如 盛唐·国风二次元",
|
||||||
|
"characters": ["需要生成的角色资产名"],
|
||||||
|
"scenes": ["需要生成的场景资产名"],
|
||||||
|
"props": ["需要生成的道具资产名"]
|
||||||
|
},
|
||||||
|
"shotIdeas": [
|
||||||
|
{ "title": "镜头 1", "description": "一句话描述镜头目标、景别或动作" }
|
||||||
|
],
|
||||||
|
"recommendedNextSteps": ["2-4条建议的下游动作"]
|
||||||
|
}`,
|
||||||
|
"优先严格遵守用户指定的镜头数量和总时长；未指定时生成 6 个镜头，最多 18 个。镜头时长之和必须符合用户要求，8 秒短片不得扩写成几十秒。只生成实际需要的镜头，不重复凑数。",
|
||||||
|
"每行要覆盖竞品脚本表同等信息：镜号、时长、画面描述、景别、光影氛围、对白·旁白、音效、运镜、最终图像提示词和视频运动提示词。",
|
||||||
|
"画面描述中用 @角色名 或 @场景/道具名 标出可复用资产，例如 @沈昭昭、@金銮殿、@侍卫长刀。",
|
||||||
|
"imagePrompt 要像可直接生成分镜图的中文提示词，包含主体、环境、构图、光影、风格与一致性要求；motionPrompt 要按起始状态、动作过程、结束状态、音效组织。",
|
||||||
|
"assetCandidates 只提取脚本中确实出现的角色、场景和道具；产品视频没有人物时 characters 必须为空，禁止为凑数虚构角色。",
|
||||||
|
"每行要像专业视频画布脚本表：场景具体、画面可生成、声音可执行、镜头节奏明确，不要只写剧情摘要。",
|
||||||
|
]
|
||||||
|
.filter(Boolean)
|
||||||
|
.join("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function executeWorkflowAnalysisResponse(body: unknown, signal?: AbortSignal) {
|
||||||
|
try {
|
||||||
|
const parsed = requestSchema.safeParse(body);
|
||||||
|
if (!parsed.success) {
|
||||||
|
return NextResponse.json({ success: false, error: "请求参数不合法" }, { status: 400 });
|
||||||
|
}
|
||||||
|
|
||||||
|
const prompt = normalizeText(parsed.data.prompt);
|
||||||
|
const imageUrls = sanitizeUrls(parsed.data.imageUrls, 4);
|
||||||
|
const videoUrls = sanitizeUrls(parsed.data.videoUrls, 4);
|
||||||
|
const extractedVideoFrames =
|
||||||
|
parsed.data.mode === "video-analysis" && videoUrls.length > 0
|
||||||
|
? await extractVideoFrameDataUrls(videoUrls[0], VIDEO_ANALYSIS_FRAME_COUNT)
|
||||||
|
: [];
|
||||||
|
const preferredModelRef = normalizeText(parsed.data.preferredModelRef || parsed.data.model);
|
||||||
|
if (!prompt && imageUrls.length === 0 && videoUrls.length === 0) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ success: false, error: "请至少提供脚本、图片或视频上下文" },
|
||||||
|
{ status: 400 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const [allModels, agentModel] = await Promise.all([
|
||||||
|
getEnabledAiModelsWithProvider(false),
|
||||||
|
getAgentAiModelWithProvider(),
|
||||||
|
]);
|
||||||
|
const hasVisionInputs = imageUrls.length > 0 || extractedVideoFrames.length > 0;
|
||||||
|
const compatibleModels = allModels.filter((item) => {
|
||||||
|
const descriptor = { ...item.model, providerType: item.provider.type };
|
||||||
|
return (
|
||||||
|
!item.model.isSubModel &&
|
||||||
|
item.model.isEnabled &&
|
||||||
|
(hasVisionInputs ? isCanvasVisionModel(descriptor) : isCanvasLanguageModel(descriptor))
|
||||||
|
);
|
||||||
|
});
|
||||||
|
const requestedRef = preferredModelRef === "auto" ? "" : preferredModelRef;
|
||||||
|
const preferredModel = requestedRef
|
||||||
|
? compatibleModels.find(
|
||||||
|
(item) => item.model.id === requestedRef || item.model.modelId === requestedRef,
|
||||||
|
)
|
||||||
|
: undefined;
|
||||||
|
if (requestedRef && !preferredModel) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ success: false, error: "所选模型不可用或不支持当前分析输入，请重新选择分析模型" },
|
||||||
|
{ status: 400 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const selectedModel =
|
||||||
|
preferredModel ||
|
||||||
|
compatibleModels.find((item) => item.model.id === agentModel?.model.id) ||
|
||||||
|
compatibleModels[0];
|
||||||
|
if (!selectedModel) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{
|
||||||
|
success: false,
|
||||||
|
error: hasVisionInputs ? "当前没有可用的视觉理解模型" : "当前没有可用的文字分析模型",
|
||||||
|
},
|
||||||
|
{ status: 400 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const useVision = hasVisionInputs && supportsVision(selectedModel.model);
|
||||||
|
const codingModelId = getAiModelAgentCodingModelId(selectedModel.model);
|
||||||
|
const model = await getLanguageModelById(selectedModel.model.id, codingModelId);
|
||||||
|
if (!model) {
|
||||||
|
return NextResponse.json({ success: false, error: "分析模型初始化失败" }, { status: 500 });
|
||||||
|
}
|
||||||
|
|
||||||
|
const promptText =
|
||||||
|
parsed.data.mode === "video-analysis"
|
||||||
|
? buildVideoAnalysisPrompt({
|
||||||
|
frameCount: extractedVideoFrames.length,
|
||||||
|
nodeTitle: normalizeText(parsed.data.nodeTitle),
|
||||||
|
prompt,
|
||||||
|
imageCount: imageUrls.length,
|
||||||
|
videoCount: videoUrls.length,
|
||||||
|
})
|
||||||
|
: parsed.data.mode === "character-scene-extraction"
|
||||||
|
? buildCharacterScenePrompt({
|
||||||
|
nodeTitle: normalizeText(parsed.data.nodeTitle),
|
||||||
|
prompt,
|
||||||
|
imageCount: imageUrls.length,
|
||||||
|
})
|
||||||
|
: parsed.data.mode === "story-script"
|
||||||
|
? buildStoryScriptPrompt({
|
||||||
|
nodeTitle: normalizeText(parsed.data.nodeTitle),
|
||||||
|
prompt,
|
||||||
|
imageCount: imageUrls.length,
|
||||||
|
videoCount: videoUrls.length,
|
||||||
|
})
|
||||||
|
: buildAgentPlanPrompt({
|
||||||
|
nodeTitle: normalizeText(parsed.data.nodeTitle),
|
||||||
|
prompt,
|
||||||
|
imageCount: imageUrls.length,
|
||||||
|
videoCount: videoUrls.length,
|
||||||
|
});
|
||||||
|
|
||||||
|
const inlinedImageReferences = useVision
|
||||||
|
? (
|
||||||
|
await inlineVisionImages(imageUrls, {
|
||||||
|
maxImages: 4,
|
||||||
|
maxSide: 960,
|
||||||
|
timeoutMs: 12000,
|
||||||
|
retries: 1,
|
||||||
|
loggerPrefix: "[Canvas Workflow Analyze]",
|
||||||
|
})
|
||||||
|
).inlineDataUrls
|
||||||
|
: [];
|
||||||
|
const referenceImages = useVision
|
||||||
|
? [...inlinedImageReferences, ...extractedVideoFrames].slice(0, 6)
|
||||||
|
: [];
|
||||||
|
|
||||||
|
const { text, finishReason } = await generateText({
|
||||||
|
model,
|
||||||
|
messages: [
|
||||||
|
{
|
||||||
|
role: "user",
|
||||||
|
content: [
|
||||||
|
{ type: "text", text: promptText },
|
||||||
|
...referenceImages.map((image) => ({ type: "image" as const, image })),
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
temperature: 0.2,
|
||||||
|
maxOutputTokens:
|
||||||
|
parsed.data.mode === "story-script"
|
||||||
|
? 12000
|
||||||
|
: parsed.data.mode === "video-analysis"
|
||||||
|
? 2200
|
||||||
|
: 1600,
|
||||||
|
maxRetries: 0,
|
||||||
|
abortSignal: signal
|
||||||
|
? AbortSignal.any([signal, AbortSignal.timeout(150_000)])
|
||||||
|
: AbortSignal.timeout(150_000),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (finishReason === "length" || !text.trim()) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ success: false, error: "分析结果不完整，请缩短输入或重新生成" },
|
||||||
|
{ status: 502 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const parsedJson = extractJsonBlock<Record<string, unknown>>(text);
|
||||||
|
if (!parsedJson) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ success: false, error: "分析模型未返回完整结构化结果，请重新生成" },
|
||||||
|
{ status: 502 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const summary = normalizeText(parsedJson.summary) || normalizeText(text).slice(0, 160);
|
||||||
|
|
||||||
|
if (parsed.data.mode === "video-analysis") {
|
||||||
|
const analysisRows = normalizeVideoAnalysisRows(
|
||||||
|
parsedJson.analysisRows,
|
||||||
|
extractedVideoFrames,
|
||||||
|
);
|
||||||
|
const shotIdeas =
|
||||||
|
normalizeShotIdeas(parsedJson.shotIdeas).length > 0
|
||||||
|
? normalizeShotIdeas(parsedJson.shotIdeas)
|
||||||
|
: analysisRows.map((row, index) => ({
|
||||||
|
title: `镜头 ${index + 1}`,
|
||||||
|
description: row.sceneDescription,
|
||||||
|
}));
|
||||||
|
return NextResponse.json({
|
||||||
|
success: true,
|
||||||
|
data: {
|
||||||
|
summary,
|
||||||
|
analysisRows,
|
||||||
|
framePreviews: extractedVideoFrames,
|
||||||
|
highlights: normalizeStringList(parsedJson.highlights, 6),
|
||||||
|
shotIdeas,
|
||||||
|
visualMotifs: normalizeStringList(parsedJson.visualMotifs, 6),
|
||||||
|
recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6),
|
||||||
|
model: selectedModel.model.displayName,
|
||||||
|
raw: text,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (parsed.data.mode === "character-scene-extraction") {
|
||||||
|
return NextResponse.json({
|
||||||
|
success: true,
|
||||||
|
data: {
|
||||||
|
summary,
|
||||||
|
characters: normalizeCharacters(parsedJson.characters),
|
||||||
|
scenes: normalizeScenes(parsedJson.scenes),
|
||||||
|
relationships: normalizeStringList(parsedJson.relationships, 8),
|
||||||
|
recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6),
|
||||||
|
model: selectedModel.model.displayName,
|
||||||
|
raw: text,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (parsed.data.mode === "story-script") {
|
||||||
|
const analysisRows = normalizeStoryScriptRows(
|
||||||
|
parsedJson.analysisRows || parsedJson.storyRows || parsedJson.rows || parsedJson.shots,
|
||||||
|
);
|
||||||
|
if (analysisRows.length === 0) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ success: false, error: "分析模型没有返回有效镜头，请重新生成" },
|
||||||
|
{ status: 502 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return NextResponse.json({
|
||||||
|
success: true,
|
||||||
|
data: {
|
||||||
|
summary,
|
||||||
|
analysisRows,
|
||||||
|
shotIdeas:
|
||||||
|
normalizeShotIdeas(parsedJson.shotIdeas).length > 0
|
||||||
|
? normalizeShotIdeas(parsedJson.shotIdeas)
|
||||||
|
: analysisRows.map((row, index) => ({
|
||||||
|
title: `镜头 ${index + 1}`,
|
||||||
|
description: row.sceneDescription,
|
||||||
|
})),
|
||||||
|
recommendedNextSteps: normalizeStringList(parsedJson.recommendedNextSteps, 6),
|
||||||
|
model: selectedModel.model.displayName,
|
||||||
|
raw: text,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return NextResponse.json({
|
||||||
|
success: true,
|
||||||
|
data: {
|
||||||
|
summary,
|
||||||
|
agentPlan: normalizeStringList(parsedJson.agentPlan, 8),
|
||||||
|
deliverables: normalizeStringList(parsedJson.deliverables, 6),
|
||||||
|
recommendedNodes: normalizeStringList(parsedJson.recommendedNodes, 6),
|
||||||
|
model: selectedModel.model.displayName,
|
||||||
|
raw: text,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
console.error("[Canvas Workflow Analyze] Error:", error);
|
||||||
|
const timedOut =
|
||||||
|
error instanceof Error && /timeout|timed out|abort/i.test(`${error.name} ${error.message}`);
|
||||||
|
return NextResponse.json(
|
||||||
|
{
|
||||||
|
success: false,
|
||||||
|
error: timedOut
|
||||||
|
? "分析超时，请稍后重试或选择其他分析模型"
|
||||||
|
: error instanceof Error
|
||||||
|
? error.message
|
||||||
|
: "工作流分析失败",
|
||||||
|
},
|
||||||
|
{ status: timedOut ? 504 : 500 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -3,6 +3,7 @@ import { defineConfig } from "@playwright/test";
|
|||||||
export default defineConfig({
|
export default defineConfig({
|
||||||
testDir: "./tests",
|
testDir: "./tests",
|
||||||
testMatch: [
|
testMatch: [
|
||||||
|
/routes\/video-canvas-tasks\.test\.ts/,
|
||||||
/routes\/video-canvas-director\.test\.ts/,
|
/routes\/video-canvas-director\.test\.ts/,
|
||||||
/routes\/aliyun-canvas-video\.test\.ts/,
|
/routes\/aliyun-canvas-video\.test\.ts/,
|
||||||
/routes\/video-canvas-followup\.test\.ts/,
|
/routes\/video-canvas-followup\.test\.ts/,
|
||||||
|
|||||||
155
scripts/qa/video-canvas-task-db.cjs
Normal file
155
scripts/qa/video-canvas-task-db.cjs
Normal file
@ -0,0 +1,155 @@
|
|||||||
|
// Local disposable PostgreSQL only. Install embedded-postgres into .qa/video-task-db first.
|
||||||
|
const fs = require("fs"),
|
||||||
|
path = require("path"),
|
||||||
|
assert = require("node:assert/strict");
|
||||||
|
(async () => {
|
||||||
|
const { default: EmbeddedPostgres } = await import(
|
||||||
|
path.resolve(".qa/video-task-db/node_modules/embedded-postgres/dist/index.js")
|
||||||
|
);
|
||||||
|
const pg = new EmbeddedPostgres({
|
||||||
|
databaseDir: path.resolve(".qa/video-task-db/data"),
|
||||||
|
port: 55439,
|
||||||
|
user: "canvas_qa",
|
||||||
|
password: "local-disposable-only",
|
||||||
|
persistent: false,
|
||||||
|
onLog: (message) => console.log(message),
|
||||||
|
onError: (message) => console.error(message),
|
||||||
|
});
|
||||||
|
let client;
|
||||||
|
try {
|
||||||
|
await pg.initialise();
|
||||||
|
await pg.start();
|
||||||
|
client = pg.getPgClient();
|
||||||
|
await client.connect();
|
||||||
|
process.env.POSTGRES_URL =
|
||||||
|
"postgres://canvas_qa:local-disposable-only@127.0.0.1:55439/postgres";
|
||||||
|
await client.query(
|
||||||
|
'CREATE TABLE "User" (id uuid PRIMARY KEY); CREATE TABLE "CanvasProject" (id uuid PRIMARY KEY, "userId" uuid);',
|
||||||
|
);
|
||||||
|
await client.query(
|
||||||
|
fs.readFileSync("lib/db/migrations/0091_add_canvas_capability_tasks.sql", "utf8"),
|
||||||
|
);
|
||||||
|
const esbuild = require(
|
||||||
|
path.resolve(
|
||||||
|
"node_modules/.pnpm",
|
||||||
|
fs.readdirSync("node_modules/.pnpm").find((name) => name.startsWith("esbuild@0.25.")),
|
||||||
|
"node_modules/esbuild",
|
||||||
|
),
|
||||||
|
);
|
||||||
|
await esbuild.build({
|
||||||
|
entryPoints: ["lib/video-canvas/task-store.ts"],
|
||||||
|
outfile: ".qa/video-task-db/store.cjs",
|
||||||
|
bundle: true,
|
||||||
|
platform: "node",
|
||||||
|
format: "cjs",
|
||||||
|
packages: "external",
|
||||||
|
alias: { "@": process.cwd() },
|
||||||
|
plugins: [
|
||||||
|
{
|
||||||
|
name: "server-only",
|
||||||
|
setup(build) {
|
||||||
|
build.onResolve({ filter: /^server-only$/ }, () => ({
|
||||||
|
path: "empty",
|
||||||
|
namespace: "qa",
|
||||||
|
}));
|
||||||
|
build.onLoad({ filter: /.*/, namespace: "qa" }, () => ({ contents: "" }));
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
const store = require(path.resolve(".qa/video-task-db/store.cjs"));
|
||||||
|
const user = crypto.randomUUID(),
|
||||||
|
other = crypto.randomUUID(),
|
||||||
|
project = crypto.randomUUID();
|
||||||
|
await client.query('INSERT INTO "User" VALUES ($1),($2)', [user, other]);
|
||||||
|
await client.query('INSERT INTO "CanvasProject" VALUES ($1,$2)', [project, user]);
|
||||||
|
assert.equal(await store.ownsTaskProject(user, project), true);
|
||||||
|
assert.equal(await store.ownsTaskProject(other, project), false);
|
||||||
|
const request = {
|
||||||
|
requestId: crypto.randomUUID(),
|
||||||
|
projectId: project,
|
||||||
|
kind: "video",
|
||||||
|
params: { mode: "text-to-video", prompt: "QA" },
|
||||||
|
node: {
|
||||||
|
id: "node-a",
|
||||||
|
title: "QA",
|
||||||
|
kind: "media",
|
||||||
|
inputType: "video",
|
||||||
|
x: 0,
|
||||||
|
y: 0,
|
||||||
|
width: 300,
|
||||||
|
height: 200,
|
||||||
|
},
|
||||||
|
edges: [],
|
||||||
|
};
|
||||||
|
const copies = await Promise.all(
|
||||||
|
Array.from({ length: 10 }, () => store.createCanvasGenerationTask(user, request)),
|
||||||
|
);
|
||||||
|
assert.equal(new Set(copies.map((task) => task.id)).size, 1);
|
||||||
|
const task = copies[0];
|
||||||
|
await assert.rejects(
|
||||||
|
store.createCanvasGenerationTask(user, {
|
||||||
|
...request,
|
||||||
|
params: { ...request.params, prompt: "different" },
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
const claims = await Promise.all(
|
||||||
|
Array.from({ length: 10 }, () => store.claimCanvasGenerationTask(user, project)),
|
||||||
|
);
|
||||||
|
assert.equal(claims.filter(Boolean).length, 1);
|
||||||
|
assert.equal((await store.cancelCanvasGenerationTask(user, project, task.id)).ok, false);
|
||||||
|
assert.equal((await store.cancelCanvasGenerationTask(other, project, task.id)).task, null);
|
||||||
|
const pending = await store.createCanvasGenerationTask(user, {
|
||||||
|
...request,
|
||||||
|
requestId: crypto.randomUUID(),
|
||||||
|
node: { ...request.node, id: "node-b" },
|
||||||
|
});
|
||||||
|
assert.equal(await store.claimCanvasGenerationTask(user, project), null);
|
||||||
|
assert.equal((await store.cancelCanvasGenerationTask(user, project, pending.id)).ok, true);
|
||||||
|
await store.finishCanvasGenerationTask(task.id, { url: "https://example.com/qa.mp4" });
|
||||||
|
assert.equal(
|
||||||
|
(await store.listCanvasGenerationTasks(user, project)).find((t) => t.id === task.id).result
|
||||||
|
.url,
|
||||||
|
"https://example.com/qa.mp4",
|
||||||
|
);
|
||||||
|
assert.equal((await store.listCanvasGenerationTasks(other, project)).length, 0);
|
||||||
|
assert.equal(await store.hideCanvasGenerationNode(user, project, task.id, "node-a"), true);
|
||||||
|
assert.deepEqual(
|
||||||
|
(await store.listCanvasGenerationTasks(user, project)).find((t) => t.id === task.id).request
|
||||||
|
.hiddenNodeIds,
|
||||||
|
["node-a"],
|
||||||
|
);
|
||||||
|
const stale = await store.createCanvasGenerationTask(user, {
|
||||||
|
...request,
|
||||||
|
requestId: crypto.randomUUID(),
|
||||||
|
node: { ...request.node, id: "node-c" },
|
||||||
|
});
|
||||||
|
await store.claimCanvasGenerationTask(user, project);
|
||||||
|
await client.query(
|
||||||
|
"UPDATE \"CanvasCapabilityTask\" SET \"updatedAt\"=(now() at time zone 'UTC')-interval '3 minutes' WHERE id=$1",
|
||||||
|
[stale.id],
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
(await store.listCanvasGenerationTasks(user, project)).find((t) => t.id === stale.id).status,
|
||||||
|
"interrupted",
|
||||||
|
);
|
||||||
|
assert.equal(await store.claimCanvasGenerationTask(user, project), null);
|
||||||
|
await store.finishCanvasGenerationTask(stale.id, { url: "https://example.com/late.mp4" });
|
||||||
|
assert.equal(
|
||||||
|
(await store.listCanvasGenerationTasks(user, project)).find((t) => t.id === stale.id).status,
|
||||||
|
"succeeded",
|
||||||
|
);
|
||||||
|
console.log(
|
||||||
|
"PASS: real PostgreSQL ownership, idempotency (10 submits), exclusive claiming (10 workers), serial queue, cancellation CAS, durable results, hidden nodes, interrupted task without resubmission, late-result recovery.",
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
if (client) await client.end();
|
||||||
|
await pg.stop();
|
||||||
|
}
|
||||||
|
})().then(
|
||||||
|
() => process.exit(0),
|
||||||
|
(error) => {
|
||||||
|
console.error(error);
|
||||||
|
process.exit(1);
|
||||||
|
},
|
||||||
|
);
|
||||||
223
tests/routes/video-canvas-tasks.test.ts
Normal file
223
tests/routes/video-canvas-tasks.test.ts
Normal file
@ -0,0 +1,223 @@
|
|||||||
|
import { test, expect } from "@playwright/test";
|
||||||
|
import {
|
||||||
|
canvasTaskRequestSchema,
|
||||||
|
reconcileCanvasTasks,
|
||||||
|
requireAnalysisResult,
|
||||||
|
canCancelCanvasTask,
|
||||||
|
type CanvasGenerationTask,
|
||||||
|
} from "@/lib/video-canvas/task-contract";
|
||||||
|
import { chooseStoryVideoSettings } from "@/lib/video-canvas/model-matching";
|
||||||
|
type TaskNode = { id: string; draftValues?: Record<string, any>; [key: string]: any };
|
||||||
|
const node: TaskNode = {
|
||||||
|
id: "n1",
|
||||||
|
title: "镜头",
|
||||||
|
kind: "media",
|
||||||
|
inputType: "image",
|
||||||
|
x: 0,
|
||||||
|
y: 0,
|
||||||
|
width: 300,
|
||||||
|
height: 200,
|
||||||
|
draftValues: { rows: [{ sceneDescription: "旧内容" }] },
|
||||||
|
};
|
||||||
|
const request = {
|
||||||
|
requestId: "11111111-1111-4111-8111-111111111111",
|
||||||
|
projectId: "22222222-2222-4222-8222-222222222222",
|
||||||
|
kind: "image" as const,
|
||||||
|
node,
|
||||||
|
params: { prompt: "cup" },
|
||||||
|
edges: [],
|
||||||
|
};
|
||||||
|
const task: CanvasGenerationTask = {
|
||||||
|
id: "task1",
|
||||||
|
requestId: request.requestId,
|
||||||
|
status: "succeeded",
|
||||||
|
createdAt: "2026-09-17T01:01:00Z",
|
||||||
|
request: request as any,
|
||||||
|
result: {
|
||||||
|
images: [
|
||||||
|
{ originalUrl: "https://example.com/1.png" },
|
||||||
|
{ originalUrl: "https://example.com/2.png" },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
test("restores accepted tasks missing from a pre-submit canvas snapshot, including every image", () => {
|
||||||
|
const result = reconcileCanvasTasks<TaskNode>([], [task], "2026-09-17T01:00:00Z");
|
||||||
|
expect(result.map((n) => n.id)).toEqual(["n1", "n1-result-task1-1"]);
|
||||||
|
expect(result[0].draftValues?.url).toHaveLength(2);
|
||||||
|
expect(reconcileCanvasTasks<TaskNode>(result, [task], "2026-09-17T01:00:00Z")).toBe(result);
|
||||||
|
});
|
||||||
|
test("does not resurrect nodes removed after task creation or hidden on the server", () => {
|
||||||
|
expect(reconcileCanvasTasks<TaskNode>([], [task], "2026-09-17T01:02:00Z")).toEqual([]);
|
||||||
|
expect(
|
||||||
|
reconcileCanvasTasks<TaskNode>(
|
||||||
|
[],
|
||||||
|
[{ ...task, request: { ...request, node: node as any, hiddenNodeIds: ["n1"] } }],
|
||||||
|
"2026-09-17T01:00:00Z",
|
||||||
|
),
|
||||||
|
).toEqual([]);
|
||||||
|
});
|
||||||
|
test("late result from an older request cannot overwrite a new generation on the same node", () => {
|
||||||
|
const current = {
|
||||||
|
...node,
|
||||||
|
draftValues: { taskRequestId: "new-request", mediaUrl: "new-result" },
|
||||||
|
};
|
||||||
|
expect(reconcileCanvasTasks<TaskNode>([current], [task], "2026-09-17T01:00:00Z")[0]).toBe(
|
||||||
|
current,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
test("task failure retains the previous script instead of marking an empty result successful", () => {
|
||||||
|
const result = reconcileCanvasTasks<TaskNode>(
|
||||||
|
[node],
|
||||||
|
[{ ...task, status: "failed", error: "模型错误" }],
|
||||||
|
"2026-09-17T01:00:00Z",
|
||||||
|
)[0];
|
||||||
|
expect(result.draftValues).toMatchObject({
|
||||||
|
rows: [{ sceneDescription: "旧内容" }],
|
||||||
|
generationStatus: "failed",
|
||||||
|
generationError: "模型错误",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
test("queue and running states show actual state without invented progress", () => {
|
||||||
|
const pending = reconcileCanvasTasks<TaskNode>(
|
||||||
|
[node],
|
||||||
|
[{ ...task, status: "pending" }],
|
||||||
|
"2026-09-17T01:00:00Z",
|
||||||
|
);
|
||||||
|
expect(pending[0].draftValues).toMatchObject({ generationProgress: 0, pendingLabel: "排队中" });
|
||||||
|
expect(
|
||||||
|
reconcileCanvasTasks<TaskNode>(
|
||||||
|
pending,
|
||||||
|
[{ ...task, status: "running" }],
|
||||||
|
"2026-09-17T01:00:00Z",
|
||||||
|
)[0].draftValues?.pendingLabel,
|
||||||
|
).toBe("生成中");
|
||||||
|
});
|
||||||
|
test("interrupted tasks accept a late real result while cancelled queued tasks stay visibly cancelled", () => {
|
||||||
|
const pending = reconcileCanvasTasks<TaskNode>(
|
||||||
|
[node],
|
||||||
|
[{ ...task, status: "interrupted", error: "待核实" }],
|
||||||
|
"2026-09-17T01:00:00Z",
|
||||||
|
);
|
||||||
|
expect(
|
||||||
|
reconcileCanvasTasks<TaskNode>(pending, [task], "2026-09-17T01:00:00Z")[0].draftValues
|
||||||
|
?.generationStatus,
|
||||||
|
).toBe("complete");
|
||||||
|
expect(
|
||||||
|
reconcileCanvasTasks<TaskNode>(
|
||||||
|
[node],
|
||||||
|
[{ ...task, status: "cancelled" }],
|
||||||
|
"2026-09-17T01:00:00Z",
|
||||||
|
)[0].subtitle,
|
||||||
|
).toBe("已取消");
|
||||||
|
expect(["pending", "running", "succeeded", "interrupted"].map(canCancelCanvasTask)).toEqual([
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
test("analysis rejects success envelopes without meaningful output or script rows", () => {
|
||||||
|
for (const data of [
|
||||||
|
undefined,
|
||||||
|
{},
|
||||||
|
{ analysisRows: [] },
|
||||||
|
{ analysisRows: [{}], raw: "some summary" },
|
||||||
|
])
|
||||||
|
expect(() => requireAnalysisResult(data, "story-script")).toThrow();
|
||||||
|
expect(() => requireAnalysisResult({ raw: " " }, "agent-plan")).toThrow();
|
||||||
|
expect(
|
||||||
|
requireAnalysisResult({ analysisRows: [{ sceneDescription: "杯柄特写" }] }, "story-script"),
|
||||||
|
).toBeTruthy();
|
||||||
|
});
|
||||||
|
test("task snapshots require stable project/request identities and bounded geometry", () => {
|
||||||
|
expect(canvasTaskRequestSchema.safeParse(request).success).toBe(true);
|
||||||
|
for (const patch of [
|
||||||
|
{ requestId: "bad" },
|
||||||
|
{ projectId: "bad" },
|
||||||
|
{ node: { ...node, width: Infinity } },
|
||||||
|
{ node: { ...node, id: "" } },
|
||||||
|
])
|
||||||
|
expect(canvasTaskRequestSchema.safeParse({ ...request, ...patch }).success).toBe(false);
|
||||||
|
});
|
||||||
|
const model = {
|
||||||
|
id: "video-model",
|
||||||
|
modelId: "wan2.6-i2v-flash",
|
||||||
|
providerType: "qianwen-video",
|
||||||
|
capabilities: {
|
||||||
|
kind: "video",
|
||||||
|
supportedModes: ["first-frame"],
|
||||||
|
durationOptions: [5, 10],
|
||||||
|
supportedRatios: ["adaptive"],
|
||||||
|
supportedResolutions: ["720p"],
|
||||||
|
supportGenerateAudio: true,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
test("story batches match the actual input mode and discrete provider parameters", () => {
|
||||||
|
expect(chooseStoryVideoSettings([model], true, 7)).toMatchObject({
|
||||||
|
mode: "first-frame",
|
||||||
|
duration: 5,
|
||||||
|
ratio: "adaptive",
|
||||||
|
resolution: "720p",
|
||||||
|
generateAudio: true,
|
||||||
|
});
|
||||||
|
expect(() => chooseStoryVideoSettings([model], false, 5)).toThrow("文生视频");
|
||||||
|
const text = {
|
||||||
|
...model,
|
||||||
|
id: "text-model",
|
||||||
|
providerType: "vip98k",
|
||||||
|
modelId: "veo",
|
||||||
|
capabilities: { ...model.capabilities, supportedModes: ["text-to-video"] },
|
||||||
|
};
|
||||||
|
expect(chooseStoryVideoSettings([model, text], false, 9, model.id)).toMatchObject({
|
||||||
|
mode: "text-to-video",
|
||||||
|
model: "text-model",
|
||||||
|
duration: 10,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
test("unknown model specifications cannot silently invent 720p or five seconds", () => {
|
||||||
|
expect(() =>
|
||||||
|
chooseStoryVideoSettings(
|
||||||
|
[{ ...model, capabilities: { kind: "video", supportedModes: ["first-frame"] } }],
|
||||||
|
true,
|
||||||
|
5,
|
||||||
|
),
|
||||||
|
).toThrow("规格");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("refresh recovers the newest accepted request when the saved node still points to an older result", () => {
|
||||||
|
const newer = {
|
||||||
|
...task,
|
||||||
|
id: "task2",
|
||||||
|
requestId: "new-request",
|
||||||
|
createdAt: "2026-09-17T01:03:00Z",
|
||||||
|
result: { images: [{ originalUrl: "https://example.com/new.png" }] },
|
||||||
|
};
|
||||||
|
const old = { ...node, draftValues: { taskRequestId: task.requestId, mediaUrl: "old" } };
|
||||||
|
const result = reconcileCanvasTasks<TaskNode>([old], [task, newer], "2026-09-17T01:02:00Z");
|
||||||
|
expect(result[0].draftValues?.mediaUrl).toBe("https://example.com/new.png");
|
||||||
|
expect(result[0].draftValues?.pendingLabel).toBe("");
|
||||||
|
expect(
|
||||||
|
reconcileCanvasTasks<TaskNode>([], [task, newer], "2026-09-17T01:00:00Z")[0].draftValues
|
||||||
|
?.taskRequestId,
|
||||||
|
).toBe("new-request");
|
||||||
|
});
|
||||||
|
test("grid results preserve the grid action contract", () => {
|
||||||
|
const grid = { ...node, draftValues: { action: "IMAGE_GENERATE_NINE_GRID" } };
|
||||||
|
expect(
|
||||||
|
reconcileCanvasTasks<TaskNode>([grid], [task], "2026-09-17T01:00:00Z")[0].draftValues?.action,
|
||||||
|
).toBe("IMAGE_GENERATE_NINE_GRID");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a rerun retains distinct extra results without reusing the previous request's image", () => {
|
||||||
|
const first = reconcileCanvasTasks<TaskNode>([node], [task], "2026-09-17T01:00:00Z");
|
||||||
|
const newer = {
|
||||||
|
...task,
|
||||||
|
id: "task2",
|
||||||
|
requestId: "next",
|
||||||
|
createdAt: "2026-09-17T01:03:00Z",
|
||||||
|
result: { images: [{ originalUrl: "new1" }, { originalUrl: "new2" }] },
|
||||||
|
};
|
||||||
|
const next = reconcileCanvasTasks<TaskNode>(first, [task, newer], "2026-09-17T01:02:00Z");
|
||||||
|
expect(next.find((n) => n.id === "n1-result-task2-1")?.draftValues?.mediaUrl).toBe("new2");
|
||||||
|
expect(reconcileCanvasTasks<TaskNode>(next, [task, newer], "2026-09-17T01:02:00Z")).toBe(next);
|
||||||
|
});
|
||||||
Loading…
x
Reference in New Issue
Block a user