积分来源:{getVideoCanvasFunctionBillingCopy(item).shortLabel}
@@ -877,6 +4474,26 @@ export function VideoCanvasSettingsClient() {
onChange={(event) => updateFunction(item.key, { prompt: event.target.value })}
value={item.prompt}
/>
+ {item.generationPrompt !== undefined ? (
+
+ ) : null}
))}
diff --git a/app/(chat)/api/chat/route.ts b/app/(chat)/api/chat/route.ts
index d110c81d7..f2314c3e5 100644
--- a/app/(chat)/api/chat/route.ts
+++ b/app/(chat)/api/chat/route.ts
@@ -5739,44 +5739,53 @@ export async function POST(request: Request) {
);
const assistantMessageId = generateUUID();
- await saveMessages({
- messages: [
- {
- id: assistantMessageId,
- chatId: id,
- role: "assistant",
- parts: [{ type: "text", text: assistantText }],
- attachments: [],
- createdAt: new Date(),
- videoStorageStatus: null,
- temporaryVideoUrl: null,
- permanentVideoUrl: null,
- archivedAt: null,
- videoMetadata: null,
- },
- ],
- });
+ try {
+ await saveMessages({
+ messages: [
+ {
+ id: assistantMessageId,
+ chatId: id,
+ role: "assistant",
+ parts: [{ type: "text", text: assistantText }],
+ attachments: [],
+ createdAt: new Date(),
+ videoStorageStatus: null,
+ temporaryVideoUrl: null,
+ permanentVideoUrl: null,
+ archivedAt: null,
+ videoMetadata: null,
+ },
+ ],
+ });
- await saveGeneratedImages(
- assistantText,
- session.user.id,
- id,
- assistantMessageId,
- params.promptForPersistence || baseReplacementPrompt,
- resolvedModelRef,
- {
- sourceAssetIds,
- sourceImageUrls: params.sourceImageUrls,
- relationType: "edited_from",
- sourceType: "detail_replacement",
- projectId: contextPayload?.projectId || null,
- userPrompt: requestTextContent,
- optimizedPrompt: params.promptForPersistence || baseReplacementPrompt,
- requestedImageCount: 1,
- aspectRatio: resolvedAspectRatio,
- imageSize: persistedImageSize,
- },
- );
+ await saveGeneratedImages(
+ assistantText,
+ session.user.id,
+ id,
+ assistantMessageId,
+ params.promptForPersistence || baseReplacementPrompt,
+ resolvedModelRef,
+ {
+ sourceAssetIds,
+ sourceImageUrls: params.sourceImageUrls,
+ relationType: "edited_from",
+ sourceType: "detail_replacement",
+ projectId: contextPayload?.projectId || null,
+ userPrompt: requestTextContent,
+ optimizedPrompt: params.promptForPersistence || baseReplacementPrompt,
+ requestedImageCount: 1,
+ aspectRatio: resolvedAspectRatio,
+ imageSize: persistedImageSize,
+ },
+ );
+ } catch (persistError) {
+ chatApiLogger.warn("[Chat API] Failed to persist direct detail replacement result; streaming response anyway:", {
+ chatId: id,
+ referenceId: params.reference.id,
+ referenceLabel: params.reference.label,
+ error: persistError,
+ });
+ }
writeAssistantTextChunk(
writer,
@@ -5827,6 +5836,19 @@ export async function POST(request: Request) {
return;
}
+ const fallbackSourceImageUrls = Array.from(
+ new Set([
+ ...productInputImageUrls,
+ ...(modelInputImageUrl ? [modelInputImageUrl] : []),
+ ...(reference.rerunReferenceImages || [])
+ .map((image) => String(image.url || "").trim())
+ .filter((url) => /^https?:\/\//i.test(url))
+ .slice(0, 6),
+ reference.imageUrl,
+ ]),
+ );
+
+ try {
const replacementExecutionPlan = await analyzeReplacementExecutionPlan({
modeType: "detail_replacement",
replacementStrategy: detailReplacementStrategy,
@@ -6067,6 +6089,29 @@ export async function POST(request: Request) {
});
successCount += 1;
}
+ } catch (replacementPlanError) {
+ chatApiLogger.error("[Chat API] Direct detail replacement item failed before generation:", {
+ referenceIndex,
+ referenceLabel: reference.label,
+ error: replacementPlanError,
+ });
+ await emitReplacementResultMessage({
+ reference,
+ bodyText: `生成失败,当前先沿用参考图:${
+ replacementPlanError instanceof Error
+ ? replacementPlanError.message
+ : "请稍后重试"
+ }`,
+ image: {
+ originalUrl: reference.imageUrl,
+ thumbnailUrl: reference.imageUrl,
+ },
+ modelDisplayName: null,
+ sourceImageUrls: fallbackSourceImageUrls,
+ streamLabel: "agent-mode:image:detail-replacement:item-fallback",
+ });
+ successCount += 1;
+ }
});
} finally {
stopKeepalive();
diff --git a/app/(chat)/api/files/direct-upload/route.ts b/app/(chat)/api/files/direct-upload/route.ts
index af795c131..b26335114 100644
--- a/app/(chat)/api/files/direct-upload/route.ts
+++ b/app/(chat)/api/files/direct-upload/route.ts
@@ -21,6 +21,7 @@ import {
maybeEnqueueStorageMigrationTask,
resolveS3UploadTarget,
} from "@/lib/storage/service";
+import { OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE } from "@/lib/openclaw-chat/media-attachments";
const logger = createLogger("direct-upload-api");
const DIRECT_UPLOAD_MAX_BYTES = 500 * 1024 * 1024;
@@ -39,6 +40,13 @@ const allowedDirectUploadTypes = new Set([
"image/png",
"image/webp",
]);
+const allowedOpenClawVideoUploadTypes = new Set([
+ "video/mp4",
+ "video/quicktime",
+ "video/webm",
+ "video/x-m4v",
+ "video/x-matroska",
+]);
const InitiateSchema = z.object({
action: z.literal("initiate"),
@@ -109,8 +117,12 @@ function assertUserOwnedKey(key: string, userId: string) {
return normalized;
}
-function isAllowedDirectUpload(contentType: string) {
- return allowedDirectUploadTypes.has(contentType);
+function isAllowedDirectUpload(contentType: string, purpose?: string) {
+ return (
+ allowedDirectUploadTypes.has(contentType) ||
+ (purpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE &&
+ allowedOpenClawVideoUploadTypes.has(contentType))
+ );
}
function computePartSize(size: number) {
@@ -204,7 +216,7 @@ export async function POST(request: Request) {
payload.filename
);
- if (!isAllowedDirectUpload(contentType)) {
+ if (!isAllowedDirectUpload(contentType, payload.purpose)) {
logger.warn("unsupported_direct_upload_type", {
contentType,
filenameExtension: payload.filename.toLowerCase().split(".").pop() || null,
@@ -213,7 +225,10 @@ export async function POST(request: Request) {
clientUploadId: payload.clientUploadId || null,
});
return NextResponse.json(
- { error: "Direct upload only supports JPG, PNG and WebP images" },
+ {
+ error:
+ "直传仅支持 JPG、PNG、WebP 图片以及 Claw 对话的 MP4、MOV、WebM、M4V、MKV 视频",
+ },
{ status: 400 }
);
}
@@ -329,9 +344,12 @@ export async function POST(request: Request) {
const key = assertUserOwnedKey(payload.key, session.user.id);
const contentType = normalizeContentType(payload.contentType, payload.filename);
- if (!isAllowedDirectUpload(contentType)) {
+ if (!isAllowedDirectUpload(contentType, payload.purpose)) {
return NextResponse.json(
- { error: "Direct upload only supports JPG, PNG and WebP images" },
+ {
+ error:
+ "直传仅支持 JPG、PNG、WebP 图片以及 Claw 对话的 MP4、MOV、WebM、M4V、MKV 视频",
+ },
{ status: 400 }
);
}
@@ -390,15 +408,21 @@ export async function POST(request: Request) {
const assetStartedAt = Date.now();
const asset = await upsertMediaAsset({
userId: session.user.id,
- mediaType: "image",
- sourceType: "upload",
+ mediaType: resolvedContentType.startsWith("video/") ? "video" : "image",
+ sourceType:
+ payload.purpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE
+ ? "openclaw_chat_video_upload"
+ : "upload",
status: "ready",
originalUrl: url,
previewUrl: url,
thumbnailUrl: url,
mimeType: resolvedContentType,
displayName: payload.filename,
- sourceTable: "direct_upload",
+ sourceTable:
+ payload.purpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE
+ ? "openclaw_chat_video_upload"
+ : "direct_upload",
metadata: {
pathname: key,
size: actualSize ?? payload.size,
diff --git a/app/(chat)/api/files/upload/route.ts b/app/(chat)/api/files/upload/route.ts
index 6c11efe0b..c15e6f9e3 100644
--- a/app/(chat)/api/files/upload/route.ts
+++ b/app/(chat)/api/files/upload/route.ts
@@ -3,10 +3,24 @@ import { z } from "zod";
import { auth } from "@/app/(auth)/auth";
import { createAgentWorkflowPreviewImageBuffer } from "@/lib/agents/workflow-preview-images";
-import { upsertMediaAsset } from "@/lib/db/queries";
+import { getSystemSetting, upsertMediaAsset } from "@/lib/db/queries";
import { isSiteAdminType } from "@/lib/middleware/admin";
import { getContentType, uploadFile } from "@/lib/storage/service";
import { createLogger } from "@/lib/observability/logger";
+import {
+ defaultVideoCanvasFunctionConfig,
+ normalizeVideoCanvasFunctionConfig,
+ VIDEO_CANVAS_FUNCTION_CONFIG_KEY,
+} from "@/lib/video-canvas/config";
+import {
+ VIDEO_CANVAS_SUBJECT_UPLOAD_SOURCE_TYPE,
+ VIDEO_CANVAS_SUBJECT_VOICE_SOURCE_TYPE,
+} from "@/lib/video-canvas/subjects";
+import {
+ VIDEO_CANVAS_RESOURCE_UPLOAD_PURPOSE,
+ VIDEO_CANVAS_RESOURCE_UPLOAD_SOURCE_TYPE,
+} from "@/lib/video-canvas/uploaded-resource";
+import { OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE } from "@/lib/openclaw-chat/media-attachments";
const logger = createLogger("upload-api");
const ALLOWED_UPLOAD_IMAGE_TYPES = new Set([
@@ -18,11 +32,29 @@ const ALLOWED_UPLOAD_IMAGE_TYPES = new Set([
"image/heic",
"image/heif",
]);
+const ALLOWED_UPLOAD_AUDIO_TYPES = new Set([
+ "audio/mpeg",
+ "audio/mp3",
+ "audio/mp4",
+ "audio/wav",
+ "audio/x-wav",
+ "audio/webm",
+ "audio/ogg",
+ "audio/aac",
+]);
+const ALLOWED_UPLOAD_VIDEO_TYPES = new Set([
+ "video/mp4",
+ "video/quicktime",
+ "video/webm",
+ "video/x-m4v",
+ "video/x-matroska",
+]);
const HEIC_UPLOAD_TYPES = new Set(["image/heic", "image/heif"]);
+const GAUSSIAN_SPLAT_EXTENSION_RE = /\.(ply|splat|spz|ksplat)$/i;
function normalizeUploadContentType(contentType: string, filename: string) {
- const normalized = contentType.trim().toLowerCase();
+ const normalized = contentType.split(";", 1)[0].trim().toLowerCase();
if (normalized && normalized !== "application/octet-stream") {
return normalized;
}
@@ -30,6 +62,15 @@ function normalizeUploadContentType(contentType: string, filename: string) {
return getContentType(filename);
}
+function getAcceptedMimeTypes(accept: string) {
+ return new Set(
+ accept
+ .split(",")
+ .map((item) => item.trim().toLowerCase())
+ .filter((item) => item.includes("/") && !item.endsWith("/*")),
+ );
+}
+
function createUnsupportedImageTypeMessage(contentType: string, filename: string) {
const extension = filename.toLowerCase().split(".").pop() || "unknown";
if (contentType === "image/heic" || contentType === "image/heif" || extension === "heic" || extension === "heif") {
@@ -154,23 +195,16 @@ function toArrayBuffer(buffer: Buffer): ArrayBuffer {
}
// Use Blob instead of File since File is not available in Node.js environment
-const FileSchema = z.object({
- file: z
- .instanceof(Blob)
- .refine((file) => file.size <= 50 * 1024 * 1024, {
- message: "File size should be less than 50MB",
+function createFileSchema(maxBytes: number, maxSizeLabel: string) {
+ return z.object({
+ file: z.instanceof(Blob).refine((value) => value.size <= maxBytes, {
+ message: `File size should be less than ${maxSizeLabel}`,
}),
-});
+ });
+}
export async function POST(request: Request) {
logger.debug("[Upload API] Received upload request");
-
- const session = await auth();
-
- if (!session) {
- logger.debug("[Upload API] Unauthorized - no session");
- return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
- }
if (request.body === null) {
logger.debug("[Upload API] Request body is empty");
@@ -197,12 +231,88 @@ export async function POST(request: Request) {
let filename = originalFilename;
let contentType = originalContentType;
const uploadPurpose = String(formData.get("purpose") || "");
+ const isDirectorGaussianUpload = uploadPurpose === "director-gaussian";
+ const isCanvasAudioUpload = uploadPurpose === "canvas-audio-process";
+ const isCanvasVideoUpload = uploadPurpose === "canvas-video-process";
+ const isOpenClawChatVideoUpload =
+ uploadPurpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE;
+ const isVideoCanvasAssetLibraryUpload =
+ uploadPurpose === "video-canvas-asset-library";
+ const isVideoCanvasResourceUpload =
+ uploadPurpose === VIDEO_CANVAS_RESOURCE_UPLOAD_PURPOSE;
+ const isVideoCanvasSubjectUpload =
+ uploadPurpose === "video-canvas-subject";
+ const isVideoCanvasSubjectVoiceUpload =
+ uploadPurpose === "video-canvas-subject-voice";
+ const videoCanvasSetting =
+ isDirectorGaussianUpload ||
+ isVideoCanvasResourceUpload ||
+ isVideoCanvasSubjectUpload
+ ? await getSystemSetting({ key: VIDEO_CANVAS_FUNCTION_CONFIG_KEY }).catch(
+ () => undefined,
+ )
+ : undefined;
+ const videoCanvasConfig = videoCanvasSetting
+ ? normalizeVideoCanvasFunctionConfig(videoCanvasSetting.value)
+ : defaultVideoCanvasFunctionConfig;
+ const gaussianConfig = videoCanvasConfig.directorDesk.gaussian;
+ const subjectLibraryConfig = videoCanvasConfig.subjectLibrary;
+ if (
+ isDirectorGaussianUpload &&
+ !GAUSSIAN_SPLAT_EXTENSION_RE.test(originalFilename)
+ ) {
+ return NextResponse.json(
+ { error: `高斯泼溅仅支持 ${gaussianConfig.uploadAccept}` },
+ { status: 400 },
+ );
+ }
+ if (isVideoCanvasSubjectUpload) {
+ const acceptedTypes = new Set([
+ ...getAcceptedMimeTypes(subjectLibraryConfig.imageAccept),
+ ...getAcceptedMimeTypes(subjectLibraryConfig.videoAccept),
+ ]);
+ if (!acceptedTypes.has(originalContentType)) {
+ return NextResponse.json(
+ { error: "主体素材仅支持 JPG、PNG、WebP、MP4 或 MOV" },
+ { status: 400 },
+ );
+ }
+ }
+ if (
+ isVideoCanvasSubjectVoiceUpload &&
+ !ALLOWED_UPLOAD_AUDIO_TYPES.has(originalContentType) &&
+ !ALLOWED_UPLOAD_VIDEO_TYPES.has(originalContentType)
+ ) {
+ return NextResponse.json(
+ { error: "音色素材仅支持 MP3、WAV、MP4 或 MOV" },
+ { status: 400 },
+ );
+ }
+ const requestHostname = new URL(request.url).hostname.toLowerCase();
+ const isLoopbackRequest =
+ requestHostname === "localhost" ||
+ requestHostname === "127.0.0.1" ||
+ requestHostname === "::1";
+ const isLocalDevelopmentCanvasUpload =
+ (process.env.NODE_ENV !== "production" || isLoopbackRequest) &&
+ process.env.USE_REMOTE_STORAGE_IN_DEV !== "true" &&
+ (uploadPurpose === "canvas-upload" ||
+ isCanvasAudioUpload ||
+ isCanvasVideoUpload ||
+ isOpenClawChatVideoUpload ||
+ isDirectorGaussianUpload);
+ const session = isLocalDevelopmentCanvasUpload ? null : await auth();
+
+ if (!isLocalDevelopmentCanvasUpload && !session) {
+ logger.debug("[Upload API] Unauthorized - no session");
+ return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
+ }
const clientUploadId =
String(formData.get("clientUploadId") || "").trim() || null;
const isSvgUpload = contentType === "image/svg+xml";
if (
isSvgUpload &&
- (uploadPurpose !== "admin-svg-icon" || !isSiteAdminType(session.user.type))
+ (uploadPurpose !== "admin-svg-icon" || !isSiteAdminType(session?.user.type))
) {
return NextResponse.json(
{ error: "SVG uploads are only allowed for administrator icon settings" },
@@ -216,7 +326,19 @@ export async function POST(request: Request) {
sizeInMB: (file.size / 1024 / 1024).toFixed(2) + "MB"
});
- const validatedFile = FileSchema.safeParse({ file });
+ const maxFileSizeMb = isDirectorGaussianUpload
+ ? gaussianConfig.maxFileSizeMb
+ : isVideoCanvasResourceUpload
+ ? videoCanvasConfig.nodeCreation.upload.maxFileSizeMb
+ : isVideoCanvasSubjectUpload
+ ? originalContentType.startsWith("video/")
+ ? subjectLibraryConfig.videoMaxFileSizeMb
+ : subjectLibraryConfig.imageMaxFileSizeMb
+ : 50;
+ const validatedFile = createFileSchema(
+ maxFileSizeMb * 1024 * 1024,
+ `${maxFileSizeMb}MB`,
+ ).safeParse({ file });
if (!validatedFile.success) {
const errorMessage = validatedFile.error.errors
@@ -227,7 +349,36 @@ export async function POST(request: Request) {
return NextResponse.json({ error: errorMessage }, { status: 400 });
}
- if (!ALLOWED_UPLOAD_IMAGE_TYPES.has(contentType)) {
+ if (
+ !isDirectorGaussianUpload &&
+ !ALLOWED_UPLOAD_IMAGE_TYPES.has(contentType) &&
+ !(isCanvasAudioUpload && ALLOWED_UPLOAD_AUDIO_TYPES.has(contentType)) &&
+ !(isCanvasVideoUpload && ALLOWED_UPLOAD_VIDEO_TYPES.has(contentType)) &&
+ !(
+ isOpenClawChatVideoUpload &&
+ ALLOWED_UPLOAD_VIDEO_TYPES.has(contentType)
+ ) &&
+ !(
+ isVideoCanvasAssetLibraryUpload &&
+ (ALLOWED_UPLOAD_AUDIO_TYPES.has(contentType) ||
+ ALLOWED_UPLOAD_VIDEO_TYPES.has(contentType))
+ ) &&
+ !(
+ isVideoCanvasResourceUpload &&
+ (ALLOWED_UPLOAD_IMAGE_TYPES.has(contentType) ||
+ ALLOWED_UPLOAD_AUDIO_TYPES.has(contentType) ||
+ ALLOWED_UPLOAD_VIDEO_TYPES.has(contentType))
+ ) &&
+ !(
+ isVideoCanvasSubjectUpload &&
+ ALLOWED_UPLOAD_VIDEO_TYPES.has(contentType)
+ ) &&
+ !(
+ isVideoCanvasSubjectVoiceUpload &&
+ (ALLOWED_UPLOAD_AUDIO_TYPES.has(contentType) ||
+ ALLOWED_UPLOAD_VIDEO_TYPES.has(contentType))
+ )
+ ) {
const errorMessage = createUnsupportedImageTypeMessage(contentType, filename);
logger.warn("[Upload API] Unsupported upload content type", {
contentType,
@@ -248,7 +399,7 @@ export async function POST(request: Request) {
let uploadBuffer: Buffer = Buffer.from(fileBuffer);
let convertedFrom: string | null = null;
- if (isHeicUpload(contentType, filename)) {
+ if (!isDirectorGaussianUpload && isHeicUpload(contentType, filename)) {
try {
const convertedBuffer = await convertHeicToJpeg(uploadBuffer);
logger.info("[Upload API] HEIC image converted to JPEG", {
@@ -289,7 +440,7 @@ export async function POST(request: Request) {
let assetId: string | null = null;
try {
const asset = await upsertMediaAsset({
- userId: session.user.id,
+ userId: session?.user.id || "",
mediaType: "image",
sourceType: "agent_workflow_preview_upload",
status: "ready",
@@ -335,7 +486,10 @@ export async function POST(request: Request) {
// ✅ 优化原图(保持JPG/PNG格式)
// WebP格式不优化(用于预览图,保持原样)
const shouldOptimizeImage =
+ !isCanvasAudioUpload &&
+ !isCanvasVideoUpload &&
uploadPurpose !== "canvas-upload" &&
+ !isDirectorGaussianUpload &&
(contentType === 'image/jpeg' || contentType === 'image/jpg' || contentType === 'image/png');
if (shouldOptimizeImage) {
@@ -343,7 +497,13 @@ export async function POST(request: Request) {
} else if (contentType === 'image/webp' || contentType === 'image/svg+xml') {
// WebP/SVG 不优化;SVG 是矢量文件,交给前端按状态反色渲染。
logger.debug(`[Upload] ${contentType}格式,跳过优化,直接上传`);
- } else if (uploadPurpose === "canvas-upload") {
+ } else if (
+ uploadPurpose === "canvas-upload" ||
+ isCanvasAudioUpload ||
+ isCanvasVideoUpload ||
+ isOpenClawChatVideoUpload ||
+ isDirectorGaussianUpload
+ ) {
logger.debug("[Upload] Canvas upload, skip server-side image optimization for faster paste/drop");
}
@@ -358,33 +518,83 @@ export async function POST(request: Request) {
const data = await uploadFile(filename, optimizedArrayBuffer);
let assetId: string | null = null;
- try {
- const asset = await upsertMediaAsset({
- userId: session.user.id,
- mediaType: "image",
- sourceType: "upload",
- status: "ready",
- originalUrl: data.url,
- previewUrl: data.url,
- thumbnailUrl: data.url,
- mimeType: data.contentType || contentType || null,
- displayName: filename,
- sourceTable: "upload",
- metadata: {
- pathname: data.pathname,
- size: optimizedBuffer.length,
- clientUploadId,
- convertedFrom,
- },
- });
- assetId = asset?.id ?? null;
- } catch (assetError) {
- logger.warn("[Upload API] Failed to register uploaded image as media asset:", assetError);
+ const isLocalDevelopmentUpload = data.pathname.startsWith("uploads/local/");
+ if (
+ !isLocalDevelopmentUpload ||
+ isVideoCanvasResourceUpload ||
+ isVideoCanvasAssetLibraryUpload ||
+ isVideoCanvasSubjectUpload ||
+ isVideoCanvasSubjectVoiceUpload
+ ) {
+ try {
+ const uploadedMediaType = isDirectorGaussianUpload
+ ? "model"
+ : isCanvasAudioUpload || contentType.startsWith("audio/")
+ ? "audio"
+ : isCanvasVideoUpload || contentType.startsWith("video/")
+ ? "video"
+ : "image";
+ const asset = await upsertMediaAsset({
+ userId: session?.user.id || "",
+ mediaType: uploadedMediaType,
+ sourceType: isDirectorGaussianUpload
+ ? "director_gaussian_upload"
+ : isCanvasAudioUpload
+ ? "canvas_audio_process"
+ : isCanvasVideoUpload
+ ? "canvas_video_process"
+ : isOpenClawChatVideoUpload
+ ? "openclaw_chat_video_upload"
+ : isVideoCanvasResourceUpload
+ ? VIDEO_CANVAS_RESOURCE_UPLOAD_SOURCE_TYPE
+ : isVideoCanvasAssetLibraryUpload
+ ? "video_canvas_asset_library_upload"
+ : isVideoCanvasSubjectUpload
+ ? VIDEO_CANVAS_SUBJECT_UPLOAD_SOURCE_TYPE
+ : isVideoCanvasSubjectVoiceUpload
+ ? VIDEO_CANVAS_SUBJECT_VOICE_SOURCE_TYPE
+ : "upload",
+ status: "ready",
+ originalUrl: data.url,
+ previewUrl: isDirectorGaussianUpload ? null : data.url,
+ thumbnailUrl: isDirectorGaussianUpload ? null : data.url,
+ mimeType: contentType || data.contentType || null,
+ displayName: filename,
+ sourceTable: isDirectorGaussianUpload
+ ? "director_gaussian_upload"
+ : isCanvasAudioUpload
+ ? "canvas_audio_process"
+ : isCanvasVideoUpload
+ ? "canvas_video_process"
+ : isOpenClawChatVideoUpload
+ ? "openclaw_chat_video_upload"
+ : isVideoCanvasResourceUpload
+ ? VIDEO_CANVAS_RESOURCE_UPLOAD_SOURCE_TYPE
+ : isVideoCanvasSubjectUpload
+ ? VIDEO_CANVAS_SUBJECT_UPLOAD_SOURCE_TYPE
+ : isVideoCanvasSubjectVoiceUpload
+ ? VIDEO_CANVAS_SUBJECT_VOICE_SOURCE_TYPE
+ : "upload",
+ metadata: {
+ pathname: data.pathname,
+ size: optimizedBuffer.length,
+ clientUploadId,
+ convertedFrom,
+ purpose: uploadPurpose || null,
+ category:
+ String(formData.get("assetCategory") || "").trim() || null,
+ },
+ });
+ assetId = asset?.id ?? null;
+ } catch (assetError) {
+ logger.warn("[Upload API] Failed to register uploaded image as media asset:", assetError);
+ }
}
logger.debug("[Upload API] Upload successful:", data.url);
return NextResponse.json({
...data,
+ contentType,
assetId,
clientUploadId,
convertedFrom,
diff --git a/app/api/canvas/[id]/route.ts b/app/api/canvas/[id]/route.ts
index 5b8f76aa8..2aa384fe0 100644
--- a/app/api/canvas/[id]/route.ts
+++ b/app/api/canvas/[id]/route.ts
@@ -3,11 +3,17 @@ import { auth } from '@/app/(auth)/auth';
import { db } from '@/lib/db';
import { canvasProject } from '@/lib/db/schema';
import { syncCanvasNodesForProject } from '@/lib/db/queries';
-import { eq, and } from 'drizzle-orm';
+import { eq, and, inArray } from 'drizzle-orm';
import {
buildStaleCanvasSaveConflictPayload,
isStaleCanvasSaveBaseRevision,
} from '@/lib/canvas/canvas-save-conflict';
+import { normalizeCanvasProjectTitle } from '@/lib/canvas/project-title';
+import {
+ getExplicitVideoCanvasWorkspaceMeta,
+ getVideoCanvasWorkspaceProjects,
+ withVideoCanvasWorkspaceMeta,
+} from '@/lib/canvas/video-workspace';
function resolveWorkspaceType(input: unknown) {
return input === "agent" || input === "video" ? input : "canvas";
@@ -98,18 +104,56 @@ export async function PUT(
);
}
+ const explicitWorkspaceMeta = getExplicitVideoCanvasWorkspaceMeta(
+ existingProject.canvasData
+ );
+ const canvasDataWithWorkspace =
+ canvasData !== undefined && explicitWorkspaceMeta
+ ? withVideoCanvasWorkspaceMeta(canvasData, explicitWorkspaceMeta)
+ : canvasData;
const syncedCanvasData =
- canvasData !== undefined
+ canvasDataWithWorkspace !== undefined
? await syncCanvasNodesForProject({
projectId: id,
userId: session.user.id,
- canvasData,
+ canvasData: canvasDataWithWorkspace,
})
: undefined;
// 更新现有画布
const updateData: any = { updatedAt: new Date() };
- if (title !== undefined) updateData.title = title;
+ if (title !== undefined) {
+ updateData.title = normalizeCanvasProjectTitle(
+ title,
+ existingProject.title
+ );
+ if (existingProject.workspaceType === "video") {
+ const videoProjects = await db
+ .select({
+ canvasData: canvasProject.canvasData,
+ id: canvasProject.id,
+ })
+ .from(canvasProject)
+ .where(
+ and(
+ eq(canvasProject.userId, session.user.id),
+ eq(canvasProject.workspaceType, "video")
+ )
+ );
+ const workspaceProjectIds = getVideoCanvasWorkspaceProjects(
+ videoProjects,
+ existingProject
+ )
+ .map((project) => project.id)
+ .filter((projectId) => projectId !== id);
+ if (workspaceProjectIds.length > 0) {
+ await db
+ .update(canvasProject)
+ .set({ title: updateData.title, updatedAt: new Date() })
+ .where(inArray(canvasProject.id, workspaceProjectIds));
+ }
+ }
+ }
if (description !== undefined) updateData.description = description;
if (thumbnail !== undefined) updateData.thumbnail = thumbnail;
if (workspaceType !== undefined) updateData.workspaceType = workspaceType;
@@ -143,7 +187,10 @@ export async function PUT(
.values({
id, // 使用指定的 ID
userId: session.user.id,
- title: title || getDefaultWorkspaceTitle(workspaceType || "canvas"),
+ title: normalizeCanvasProjectTitle(
+ title,
+ getDefaultWorkspaceTitle(workspaceType || "canvas")
+ ),
description,
workspaceType: workspaceType || "canvas",
canvasData: syncedCanvasData || {
diff --git a/app/api/canvas/[id]/video-workspace/route.ts b/app/api/canvas/[id]/video-workspace/route.ts
new file mode 100644
index 000000000..32d2c0caa
--- /dev/null
+++ b/app/api/canvas/[id]/video-workspace/route.ts
@@ -0,0 +1,266 @@
+import { and, desc, eq } from "drizzle-orm";
+import { NextRequest, NextResponse } from "next/server";
+import { auth } from "@/app/(auth)/auth";
+import {
+ duplicateVideoCanvasData,
+ getExplicitVideoCanvasWorkspaceMeta,
+ getNextVideoCanvasName,
+ getVideoCanvasCopyName,
+ getVideoCanvasWorkspaceProjects,
+ normalizeVideoCanvasName,
+ resolveVideoCanvasWorkspaceMeta,
+ withVideoCanvasWorkspaceMeta,
+} from "@/lib/canvas/video-workspace";
+import { db } from "@/lib/db";
+import { syncCanvasNodesForProject } from "@/lib/db/queries";
+import { canvasProject } from "@/lib/db/schema";
+
+type CanvasProjectRow = typeof canvasProject.$inferSelect;
+
+function toCanvasItem(project: CanvasProjectRow, currentId: string) {
+ const meta = resolveVideoCanvasWorkspaceMeta(project);
+ return {
+ createdAt: project.createdAt.toISOString(),
+ id: project.id,
+ isCurrent: project.id === currentId,
+ name: meta.canvasName,
+ };
+}
+
+async function getWorkspaceContext(userId: string, currentId: string) {
+ const projects = await db
+ .select()
+ .from(canvasProject)
+ .where(
+ and(
+ eq(canvasProject.userId, userId),
+ eq(canvasProject.workspaceType, "video"),
+ ),
+ )
+ .orderBy(desc(canvasProject.createdAt));
+ const currentProject = projects.find((project) => project.id === currentId);
+ if (!currentProject) return null;
+ return {
+ currentProject,
+ projects: getVideoCanvasWorkspaceProjects(projects, currentProject),
+ };
+}
+
+async function ensureExplicitWorkspaceMetadata(projects: CanvasProjectRow[]) {
+ for (const project of projects) {
+ if (getExplicitVideoCanvasWorkspaceMeta(project.canvasData)) continue;
+ const canvasData = withVideoCanvasWorkspaceMeta(
+ project.canvasData,
+ resolveVideoCanvasWorkspaceMeta(project),
+ );
+ await db
+ .update(canvasProject)
+ .set({ canvasData })
+ .where(eq(canvasProject.id, project.id));
+ project.canvasData = canvasData;
+ }
+}
+
+export async function GET(
+ _request: NextRequest,
+ { params }: { params: Promise<{ id: string }> },
+) {
+ const session = await auth();
+ if (!session?.user?.id) {
+ return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
+ }
+ const { id } = await params;
+ const context = await getWorkspaceContext(session.user.id, id);
+ if (!context) {
+ return NextResponse.json(
+ { success: false, error: "画布不存在" },
+ { status: 404 },
+ );
+ }
+ return NextResponse.json({
+ success: true,
+ data: {
+ canvases: context.projects.map((project) => toCanvasItem(project, id)),
+ currentCanvasId: id,
+ workspaceTitle: context.currentProject.title,
+ },
+ });
+}
+
+export async function POST(
+ request: NextRequest,
+ { params }: { params: Promise<{ id: string }> },
+) {
+ const session = await auth();
+ if (!session?.user?.id) {
+ return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
+ }
+ const { id } = await params;
+ const body = await request.json();
+ if (body?.action !== "create" && body?.action !== "copy") {
+ return NextResponse.json(
+ { success: false, error: "不支持的画布操作" },
+ { status: 400 },
+ );
+ }
+ const action = body.action;
+ const context = await getWorkspaceContext(session.user.id, id);
+ if (!context) {
+ return NextResponse.json(
+ { success: false, error: "画布不存在" },
+ { status: 404 },
+ );
+ }
+ await ensureExplicitWorkspaceMetadata(context.projects);
+ const currentMeta = resolveVideoCanvasWorkspaceMeta(context.currentProject);
+ const existingNames = context.projects.map(
+ (project) => resolveVideoCanvasWorkspaceMeta(project).canvasName,
+ );
+ const source =
+ action === "copy"
+ ? context.projects.find((project) => project.id === body?.targetId)
+ : context.currentProject;
+ if (!source) {
+ return NextResponse.json(
+ { success: false, error: "要复制的画布不存在" },
+ { status: 404 },
+ );
+ }
+ const canvasName =
+ action === "copy"
+ ? getVideoCanvasCopyName(
+ resolveVideoCanvasWorkspaceMeta(source).canvasName,
+ existingNames,
+ )
+ : getNextVideoCanvasName(existingNames);
+ const meta = {
+ canvasName,
+ workspaceId: currentMeta.workspaceId,
+ };
+ const canvasData =
+ action === "copy"
+ ? duplicateVideoCanvasData(source.canvasData, meta)
+ : withVideoCanvasWorkspaceMeta(null, meta);
+ const [insertedProject] = await db
+ .insert(canvasProject)
+ .values({
+ canvasData,
+ description: context.currentProject.description,
+ title: context.currentProject.title,
+ userId: session.user.id,
+ workspaceType: "video",
+ })
+ .returning();
+ const syncedCanvasData = await syncCanvasNodesForProject({
+ canvasData,
+ projectId: insertedProject.id,
+ userId: session.user.id,
+ });
+ const [createdProject] = await db
+ .update(canvasProject)
+ .set({ canvasData: syncedCanvasData })
+ .where(eq(canvasProject.id, insertedProject.id))
+ .returning();
+
+ return NextResponse.json({
+ success: true,
+ data: toCanvasItem(createdProject || insertedProject, id),
+ });
+}
+
+export async function PATCH(
+ request: NextRequest,
+ { params }: { params: Promise<{ id: string }> },
+) {
+ const session = await auth();
+ if (!session?.user?.id) {
+ return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
+ }
+ const { id } = await params;
+ const body = await request.json();
+ const context = await getWorkspaceContext(session.user.id, id);
+ if (!context) {
+ return NextResponse.json(
+ { success: false, error: "画布不存在" },
+ { status: 404 },
+ );
+ }
+ const target = context.projects.find((project) => project.id === body?.targetId);
+ if (!target) {
+ return NextResponse.json(
+ { success: false, error: "画布不存在" },
+ { status: 404 },
+ );
+ }
+ const currentMeta = resolveVideoCanvasWorkspaceMeta(context.currentProject);
+ const targetMeta = resolveVideoCanvasWorkspaceMeta(target);
+ const canvasName = normalizeVideoCanvasName(body?.name, targetMeta.canvasName);
+ const [updatedProject] = await db
+ .update(canvasProject)
+ .set({
+ canvasData: withVideoCanvasWorkspaceMeta(target.canvasData, {
+ canvasName,
+ workspaceId: currentMeta.workspaceId,
+ }),
+ updatedAt: new Date(),
+ })
+ .where(
+ and(
+ eq(canvasProject.id, target.id),
+ eq(canvasProject.userId, session.user.id),
+ ),
+ )
+ .returning();
+
+ return NextResponse.json({
+ success: true,
+ data: toCanvasItem(updatedProject, id),
+ });
+}
+
+export async function DELETE(
+ request: NextRequest,
+ { params }: { params: Promise<{ id: string }> },
+) {
+ const session = await auth();
+ if (!session?.user?.id) {
+ return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
+ }
+ const { id } = await params;
+ const body = await request.json();
+ const context = await getWorkspaceContext(session.user.id, id);
+ if (!context) {
+ return NextResponse.json(
+ { success: false, error: "画布不存在" },
+ { status: 404 },
+ );
+ }
+ const target = context.projects.find((project) => project.id === body?.targetId);
+ if (!target) {
+ return NextResponse.json(
+ { success: false, error: "画布不存在" },
+ { status: 404 },
+ );
+ }
+ if (context.projects.length <= 1) {
+ return NextResponse.json(
+ { success: false, error: "至少保留一个画布" },
+ { status: 409 },
+ );
+ }
+ await db
+ .delete(canvasProject)
+ .where(
+ and(
+ eq(canvasProject.id, target.id),
+ eq(canvasProject.userId, session.user.id),
+ ),
+ );
+ const remaining = context.projects.filter((project) => project.id !== target.id);
+ return NextResponse.json({
+ success: true,
+ data: {
+ nextCanvasId: target.id === id ? remaining[0]?.id || null : id,
+ },
+ });
+}
diff --git a/app/api/canvas/audio-source/route.ts b/app/api/canvas/audio-source/route.ts
new file mode 100644
index 000000000..93c3ebe4c
--- /dev/null
+++ b/app/api/canvas/audio-source/route.ts
@@ -0,0 +1,16 @@
+import { auth } from "@/app/(auth)/auth";
+import { proxyCanvasAudioSource } from "@/lib/video-canvas/audio-source-proxy";
+
+export async function GET(request: Request) {
+ const requestHostname = new URL(request.url).hostname.toLowerCase();
+ const isLoopbackRequest =
+ requestHostname === "localhost" ||
+ requestHostname === "127.0.0.1" ||
+ requestHostname === "::1";
+ const session = isLoopbackRequest ? null : await auth();
+ if (!isLoopbackRequest && !session?.user?.id) {
+ return Response.json({ error: "Unauthorized" }, { status: 401 });
+ }
+
+ return proxyCanvasAudioSource(request);
+}
diff --git a/app/api/canvas/clawd-chat/route.ts b/app/api/canvas/clawd-chat/route.ts
index 01479493a..613dc2d00 100644
--- a/app/api/canvas/clawd-chat/route.ts
+++ b/app/api/canvas/clawd-chat/route.ts
@@ -26,6 +26,11 @@ import {
buildOpenClawMessageWithSpreadsheetMarkdown,
buildOpenClawSpreadsheetMarkdown,
} from "@/lib/openclaw-chat/spreadsheet-markdown";
+import {
+ normalizeOpenClawChatMediaAttachments,
+ OPENCLAW_CHAT_MEDIA_MAX_ATTACHMENTS,
+ type OpenClawChatMediaAttachment,
+} from "@/lib/openclaw-chat/media-attachments";
import { generateUUID } from "@/lib/utils";
const CLAW_UPGRADE_RETRY_MESSAGE = "claw正在升级更新稍等5分钟再试";
@@ -36,9 +41,18 @@ const requestSchema = z.object({
displayMessage: z.string().trim().max(8000).optional(),
userMessageId: z.string().uuid().optional(),
metadata: z.record(z.unknown()).optional(),
+ attachments: z
+ .array(z.unknown())
+ .max(OPENCLAW_CHAT_MEDIA_MAX_ATTACHMENTS)
+ .optional()
+ .default([]),
});
-type ParsedClawdChatRequest = z.infer
& {
+type ParsedClawdChatRequest = Omit<
+ z.infer,
+ "attachments"
+> & {
+ attachments: OpenClawChatMediaAttachment[];
tableFiles: File[];
};
@@ -86,14 +100,23 @@ async function parseClawdChatRequest(
const contentType = request.headers.get("content-type") || "";
if (!contentType.includes("multipart/form-data")) {
const parsed = requestSchema.safeParse(await request.json().catch(() => null));
- return parsed.success ? { ...parsed.data, tableFiles: [] } : null;
+ if (!parsed.success) return null;
+ const attachments = normalizeOpenClawChatMediaAttachments(
+ parsed.data.attachments
+ );
+ if (attachments.length !== parsed.data.attachments.length) {
+ return null;
+ }
+ return { ...parsed.data, attachments, tableFiles: [] };
}
const formData = await request.formData().catch(() => null);
if (!formData) return null;
const metadataValue = formData.get("metadata");
+ const attachmentsValue = formData.get("attachments");
let metadata: Record | undefined;
+ let attachments: unknown[] = [];
if (typeof metadataValue === "string" && metadataValue.trim()) {
try {
const parsedMetadata = JSON.parse(metadataValue);
@@ -106,6 +129,17 @@ async function parseClawdChatRequest(
return null;
}
}
+ if (typeof attachmentsValue === "string" && attachmentsValue.trim()) {
+ try {
+ const parsedAttachments = JSON.parse(attachmentsValue);
+ if (!Array.isArray(parsedAttachments)) {
+ return null;
+ }
+ attachments = parsedAttachments;
+ } catch {
+ return null;
+ }
+ }
const parsed = requestSchema.safeParse({
id: formData.get("id"),
@@ -113,14 +147,25 @@ async function parseClawdChatRequest(
displayMessage: formData.get("displayMessage") || undefined,
userMessageId: formData.get("userMessageId") || undefined,
metadata,
+ attachments,
});
if (!parsed.success) return null;
+ const normalizedAttachments = normalizeOpenClawChatMediaAttachments(
+ parsed.data.attachments
+ );
+ if (normalizedAttachments.length !== parsed.data.attachments.length) {
+ return null;
+ }
const tableFiles = formData
.getAll("tableFiles")
.filter((entry): entry is File => isRequestFile(entry) && entry.size > 0);
- return { ...parsed.data, tableFiles };
+ return {
+ ...parsed.data,
+ attachments: normalizedAttachments,
+ tableFiles,
+ };
}
async function readOpenClawConfig() {
@@ -164,8 +209,15 @@ export async function POST(request: Request) {
if (!parsed) {
return NextResponse.json({ error: "请求参数不正确" }, { status: 400 });
}
- if (!parsed.message && parsed.tableFiles.length === 0) {
- return NextResponse.json({ error: "请输入问题或上传表格" }, { status: 400 });
+ if (
+ !parsed.message &&
+ parsed.tableFiles.length === 0 &&
+ parsed.attachments.length === 0
+ ) {
+ return NextResponse.json(
+ { error: "请输入问题或上传附件" },
+ { status: 400 }
+ );
}
const config = await readOpenClawConfig();
@@ -280,6 +332,8 @@ export async function POST(request: Request) {
skill_tag_id: skillTagId || undefined,
skill_tag_name: skillTagName || undefined,
message: upstreamMessage,
+ attachments:
+ parsed.attachments.length > 0 ? parsed.attachments : undefined,
metadata: {
source: "ai_chatbot_canvas",
page: "/canvas",
@@ -293,6 +347,8 @@ export async function POST(request: Request) {
user_department: openClawDepartmentName,
skill_tag_id: skillTagId || undefined,
skill_tag_name: skillTagName || undefined,
+ media_attachments:
+ parsed.attachments.length > 0 ? parsed.attachments : undefined,
spreadsheet_files: spreadsheetMarkdownResult?.files || undefined,
spreadsheet_markdown_truncated:
spreadsheetMarkdownResult?.truncated || undefined,
@@ -384,7 +440,15 @@ export async function POST(request: Request) {
chatId: parsed.id,
role: "user",
parts: [{ type: "text", text: displayMessage }],
- attachments: [],
+ attachments: parsed.attachments.map((attachment) => ({
+ name:
+ attachment.name ||
+ (attachment.type === "video" ? "视频附件" : "图片附件"),
+ url: attachment.url,
+ contentType:
+ attachment.content_type ||
+ (attachment.type === "video" ? "video/mp4" : "image/jpeg"),
+ })),
createdAt: userCreatedAt,
videoStorageStatus: null,
temporaryVideoUrl: null,
@@ -393,6 +457,12 @@ export async function POST(request: Request) {
videoMetadata: {
composerMode: "clawd",
source: "openclaw",
+ imageAttachmentCount: parsed.attachments.filter(
+ (attachment) => attachment.type === "image"
+ ).length,
+ videoAttachmentCount: parsed.attachments.filter(
+ (attachment) => attachment.type === "video"
+ ).length,
skillTagId: skillTagId || null,
skillTagName: skillTagName || null,
},
@@ -437,6 +507,7 @@ export async function POST(request: Request) {
createdAt: userMessage.createdAt.toISOString(),
composerMode: "clawd",
},
+ attachments: userMessage.attachments,
videoMetadata: userMessage.videoMetadata,
},
{
diff --git a/app/api/canvas/generate/image/route.ts b/app/api/canvas/generate/image/route.ts
index 4d605a07a..cc1ad22d6 100644
--- a/app/api/canvas/generate/image/route.ts
+++ b/app/api/canvas/generate/image/route.ts
@@ -30,6 +30,12 @@ function normalizeBillingFeature(
}
function toGenerateImagePayload(body: GenerateImageRequest): CanvasGenerateImageRequest {
+ const requestedQuality =
+ body.providerImageParams &&
+ typeof body.providerImageParams === "object" &&
+ !Array.isArray(body.providerImageParams)
+ ? String(body.providerImageParams.quality || "").trim().toLowerCase()
+ : "";
return {
prompt: String(body.prompt || "").trim(),
negativePrompt:
@@ -49,6 +55,11 @@ function toGenerateImagePayload(body: GenerateImageRequest): CanvasGenerateImage
: 1,
promptExtend:
typeof body.promptExtend === "boolean" ? body.promptExtend : undefined,
+ providerImageParams: new Set(["low", "medium", "high"]).has(
+ requestedQuality,
+ )
+ ? { quality: requestedQuality }
+ : undefined,
referenceImage:
typeof body.referenceImage === "string" ? body.referenceImage : undefined,
referenceImages: Array.isArray(body.referenceImages)
diff --git a/app/api/canvas/generate/video/route.ts b/app/api/canvas/generate/video/route.ts
index aac3f4a9f..73b2c006f 100644
--- a/app/api/canvas/generate/video/route.ts
+++ b/app/api/canvas/generate/video/route.ts
@@ -1,23 +1,61 @@
+import { and, eq, inArray } from "drizzle-orm";
import { NextResponse } from "next/server";
import { auth } from "@/app/(auth)/auth";
import { executeDirectVideoGeneration } from "@/lib/ai/tools/generate-video";
+import {
+ resolveVideoReferenceSelectionRule,
+ selectVideoReferenceImages,
+} from "@/lib/canvas/video-reference-selection";
+import { getSystemSetting } from "@/lib/db/queries";
+import { db } from "@/lib/db";
+import { mediaAsset } from "@/lib/db/schema";
import { filterReachableRemoteUrls } from "@/lib/net/url-access";
+import {
+ VIDEO_CANVAS_FUNCTION_CONFIG_KEY,
+ defaultVideoCanvasFunctionConfig,
+ normalizeVideoCanvasFunctionConfig,
+} from "@/lib/video-canvas/config";
+import { normalizeVideoCanvasGenerationRequestCount } from "@/lib/video-canvas/video-generation-parameters";
+import {
+ buildVideoCanvasSubjectProviderReferences,
+ composeVideoCanvasSubjectPrompt,
+ normalizeVideoCanvasAttachedSubjects,
+ supportsVideoCanvasSubjectUsage,
+ toVideoCanvasAttachedSubject,
+ VIDEO_CANVAS_SUBJECT_SOURCE_TYPE,
+ type VideoCanvasAttachedSubject,
+} from "@/lib/video-canvas/subjects";
+import {
+ composeVideoCanvasSmartStoryboardPrompt,
+ normalizeVideoCanvasSmartStoryboard,
+ supportsVideoCanvasSmartStoryboard,
+ validateVideoCanvasSmartStoryboard,
+ type VideoCanvasSmartStoryboard,
+} from "@/lib/video-canvas/smart-storyboard";
import type { VideoGenerationMode, VideoRatio, VideoResolution } from "@/types/ai";
export const maxDuration = 300;
interface GenerateVideoRequest {
+ autoValidateMaterial?: boolean;
prompt?: string;
mode?: VideoGenerationMode;
images?: string[];
+ audioUrls?: string[];
resolution?: VideoResolution;
ratio?: VideoRatio;
duration?: number;
+ generateAudio?: boolean;
+ count?: number;
watermark?: boolean;
cameraFixed?: boolean;
seed?: number;
returnLastFrame?: boolean;
+ modelLabel?: string;
preferredModelRef?: string;
+ searchEnabled?: boolean;
+ smartStoryboard?: VideoCanvasSmartStoryboard;
+ subjects?: VideoCanvasAttachedSubject[];
}
function inferMode(images: string[]): VideoGenerationMode {
@@ -39,18 +77,142 @@ export async function POST(request: Request) {
const body = (await request.json().catch(() => ({}))) as GenerateVideoRequest;
const prompt = String(body.prompt || "").trim();
- if (!prompt) {
+ const count = normalizeVideoCanvasGenerationRequestCount(body.count, 1);
+ if (count === null) {
+ return NextResponse.json(
+ { success: false, error: "生成数量仅支持 1、2 或 4 个" },
+ { status: 400 },
+ );
+ }
+ const generateAudio = body.generateAudio !== false;
+ const setting = await getSystemSetting({
+ key: VIDEO_CANVAS_FUNCTION_CONFIG_KEY,
+ }).catch(() => null);
+ const config = setting?.value
+ ? normalizeVideoCanvasFunctionConfig(setting.value)
+ : defaultVideoCanvasFunctionConfig;
+ const autoValidateMaterial =
+ typeof body.autoValidateMaterial === "boolean"
+ ? body.autoValidateMaterial
+ : config.videoAdvancedSettings.defaultAutoValidateMaterial;
+ const searchEnabled =
+ typeof body.searchEnabled === "boolean"
+ ? body.searchEnabled
+ : config.videoAdvancedSettings.defaultSearchEnabled;
+ const subjectUsageEnabled = supportsVideoCanvasSubjectUsage(
+ config.subjectLibrary,
+ `${body.modelLabel || ""} ${body.preferredModelRef || ""}`,
+ );
+ const requestedSubjects = subjectUsageEnabled
+ ? normalizeVideoCanvasAttachedSubjects(
+ body.subjects,
+ config.subjectLibrary.videoUsage.maxSubjects,
+ )
+ : [];
+ const requestedSubjectIds = requestedSubjects
+ .map((subject) => subject.id)
+ .filter((id) => /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(id));
+ const ownedSubjectRows = requestedSubjectIds.length > 0
+ ? await db
+ .select({
+ description: mediaAsset.summary,
+ displayName: mediaAsset.displayName,
+ id: mediaAsset.id,
+ metadata: mediaAsset.metadata,
+ originalUrl: mediaAsset.originalUrl,
+ previewUrl: mediaAsset.previewUrl,
+ thumbnailUrl: mediaAsset.thumbnailUrl,
+ })
+ .from(mediaAsset)
+ .where(
+ and(
+ inArray(mediaAsset.id, requestedSubjectIds),
+ eq(mediaAsset.userId, session.user.id),
+ eq(mediaAsset.mediaType, "subject"),
+ eq(mediaAsset.sourceType, VIDEO_CANVAS_SUBJECT_SOURCE_TYPE),
+ eq(mediaAsset.status, "ready"),
+ ),
+ )
+ : [];
+ const ownedSubjectsById = new Map(
+ ownedSubjectRows.map((subject) => [subject.id, subject]),
+ );
+ const subjects = requestedSubjects.flatMap((requestedSubject) => {
+ const ownedSubject = ownedSubjectsById.get(requestedSubject.id);
+ const normalized = ownedSubject
+ ? toVideoCanvasAttachedSubject(ownedSubject)
+ : null;
+ return normalized ? [normalized] : [];
+ });
+ const subjectReferences = buildVideoCanvasSubjectProviderReferences(subjects);
+ const smartStoryboardSupported = supportsVideoCanvasSmartStoryboard(
+ config.smartStoryboard,
+ `${body.modelLabel || ""} ${body.preferredModelRef || ""}`,
+ );
+ const smartStoryboard = normalizeVideoCanvasSmartStoryboard(
+ body.smartStoryboard,
+ {
+ config: config.smartStoryboard,
+ fallbackPrompt: prompt,
+ fallbackTotalDurationSeconds: body.duration,
+ },
+ );
+ const storyboardValidation = validateVideoCanvasSmartStoryboard(
+ smartStoryboard,
+ );
+ if (
+ smartStoryboardSupported &&
+ smartStoryboard.enabled &&
+ !storyboardValidation.valid
+ ) {
+ const message =
+ storyboardValidation.emptyShotIndexes.length > 0
+ ? "智能分镜存在未填写的镜头描述"
+ : storyboardValidation.status === "over"
+ ? "智能分镜总时长超出"
+ : "智能分镜总时长不足";
+ return NextResponse.json(
+ { success: false, error: message },
+ { status: 400 },
+ );
+ }
+ const promptWithStoryboard = smartStoryboardSupported
+ ? composeVideoCanvasSmartStoryboardPrompt(
+ prompt,
+ smartStoryboard,
+ config.smartStoryboard,
+ )
+ : prompt;
+ const generationPrompt = composeVideoCanvasSubjectPrompt(
+ promptWithStoryboard,
+ subjects,
+ config.subjectLibrary.videoUsage.promptTemplate,
+ );
+ if (!generationPrompt) {
return NextResponse.json(
{ success: false, error: "请输入视频提示词" },
{ status: 400 }
);
}
-
- const inputImages = Array.isArray(body.images)
- ? body.images.map((item) => String(item || "").trim()).filter(Boolean).slice(0, 4)
+ const referenceRule = resolveVideoReferenceSelectionRule(
+ config.videoReferenceSelection,
+ `${body.modelLabel || ""} ${body.preferredModelRef || ""}`,
+ );
+ const rawImages = Array.isArray(body.images)
+ ? body.images.map((item) => String(item || "").trim()).filter(Boolean).slice(0, 20)
: [];
+ const inputImages = body.mode
+ ? body.mode === "text-to-video" || body.mode === "audio-to-video"
+ ? rawImages.slice(0, 4)
+ : selectVideoReferenceImages(rawImages, body.mode, referenceRule.maxImages)
+ : rawImages.slice(0, referenceRule.maxImages);
const { kept: reachableImages, dropped: unreachableImages } =
await filterReachableRemoteUrls(inputImages, { timeoutMs: 3000 });
+ const inputAudioUrls = Array.isArray(body.audioUrls)
+ ? body.audioUrls.map((item) => String(item || "").trim()).filter(Boolean).slice(0, 3)
+ : [];
+ const { kept: reachableAudioUrls, dropped: unreachableAudioUrls } =
+ await filterReachableRemoteUrls(inputAudioUrls, { timeoutMs: 3000 });
if (inputImages.length > 0 && reachableImages.length === 0) {
return NextResponse.json(
@@ -59,17 +221,35 @@ export async function POST(request: Request) {
);
}
- const mode = body.mode || inferMode(reachableImages);
+ const mode = body.mode || (reachableAudioUrls.length > 0 ? "audio-to-video" : inferMode(reachableImages));
+ const selectedImages =
+ mode === "text-to-video" || mode === "audio-to-video"
+ ? reachableImages
+ : selectVideoReferenceImages(
+ reachableImages,
+ mode,
+ referenceRule.maxImages,
+ );
+ if (mode === "audio-to-video" && reachableAudioUrls.length === 0) {
+ return NextResponse.json(
+ { success: false, error: "音频生视频至少需要 1 个可访问的音频素材" },
+ { status: 400 }
+ );
+ }
const result = await executeDirectVideoGeneration({
- prompt,
+ prompt: generationPrompt,
mode,
- images: reachableImages,
+ images: selectedImages,
+ audioUrls: reachableAudioUrls,
resolution: body.resolution,
ratio: body.ratio,
duration:
- typeof body.duration === "number" && Number.isFinite(body.duration)
- ? body.duration
- : undefined,
+ smartStoryboardSupported && smartStoryboard.enabled
+ ? smartStoryboard.totalDurationSeconds
+ : typeof body.duration === "number" && Number.isFinite(body.duration)
+ ? body.duration
+ : undefined,
+ generateAudio,
watermark: body.watermark,
cameraFixed: body.cameraFixed,
seed:
@@ -95,8 +275,20 @@ export async function POST(request: Request) {
modelId: result.modelId,
modelRef: result.modelRef,
mode,
- images: reachableImages,
+ images: selectedImages,
+ audioUrls: reachableAudioUrls,
+ autoValidateMaterial,
+ count,
+ generateAudio,
droppedImages: unreachableImages,
+ droppedAudioUrls: unreachableAudioUrls,
+ searchEnabled,
+ subjects,
+ subjectReferences,
+ smartStoryboard:
+ smartStoryboardSupported && smartStoryboard.enabled
+ ? smartStoryboard
+ : undefined,
},
});
} catch (error) {
diff --git a/app/api/canvas/video-source/route.ts b/app/api/canvas/video-source/route.ts
new file mode 100644
index 000000000..7b85a6e45
--- /dev/null
+++ b/app/api/canvas/video-source/route.ts
@@ -0,0 +1,16 @@
+import { auth } from "@/app/(auth)/auth";
+import { proxyCanvasVideoSource } from "@/lib/video-canvas/audio-source-proxy";
+
+export async function GET(request: Request) {
+ const requestHostname = new URL(request.url).hostname.toLowerCase();
+ const isLoopbackRequest =
+ requestHostname === "localhost" ||
+ requestHostname === "127.0.0.1" ||
+ requestHostname === "::1";
+ const session = isLoopbackRequest ? null : await auth();
+ if (!isLoopbackRequest && !session?.user?.id) {
+ return Response.json({ error: "Unauthorized" }, { status: 401 });
+ }
+
+ return proxyCanvasVideoSource(request);
+}
diff --git a/app/api/canvas/workflow/analyze/route.ts b/app/api/canvas/workflow/analyze/route.ts
index ed2ff3f2b..a001a5cd1 100644
--- a/app/api/canvas/workflow/analyze/route.ts
+++ b/app/api/canvas/workflow/analyze/route.ts
@@ -9,12 +9,26 @@ import sharp from "sharp";
import { z } from "zod";
import { auth } from "@/app/(auth)/auth";
import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
+import { getCompatibleTextGenerationTemperature } from "@/lib/ai/model-compat";
+import {
+ isImageGenerationModelDescriptor,
+ isVideoModelDescriptor,
+} from "@/lib/ai/model-classification";
import { inlineVisionImages } from "@/lib/ai/vision-inline";
import { getAiModelAgentCodingModelId, isAiModelMultimodal } from "@/lib/db/ai-model-flags";
import {
getAgentAiModelWithProvider,
getEnabledAiModelsWithProvider,
+ getSystemSetting,
} from "@/lib/db/queries";
+import { createLogger } from "@/lib/observability/logger";
+import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge";
+import {
+ VIDEO_CANVAS_FUNCTION_CONFIG_KEY,
+ defaultVideoCanvasFunctionConfig,
+ normalizeVideoCanvasFunctionConfig,
+} from "@/lib/video-canvas/config";
+import { normalizeVideoAnalysisTimeline } from "@/lib/video-canvas/video-analysis";
export const runtime = "nodejs";
export const maxDuration = 180;
@@ -23,6 +37,11 @@ const execFileAsync = promisify(execFile);
const VIDEO_ANALYSIS_FRAME_COUNT = 4;
const VIDEO_ANALYSIS_MAX_SIDE = 960;
const VIDEO_ANALYSIS_TIMEOUT_MS = 15000;
+const logger = createLogger("canvas-workflow-analyze");
+
+type ModelCandidate = Awaited<
+ ReturnType
+>[number];
const requestSchema = z.object({
mode: z.enum(["video-analysis", "character-scene-extraction", "story-script", "agent-plan"]),
@@ -30,6 +49,7 @@ const requestSchema = z.object({
imageUrls: z.array(z.string().trim()).optional(),
videoUrls: z.array(z.string().trim()).optional(),
nodeTitle: z.string().trim().optional(),
+ durationSeconds: z.number().positive().max(3600).optional(),
preferredModelRef: z.string().trim().optional(),
model: z.string().trim().optional(),
});
@@ -56,6 +76,24 @@ function supportsVision(model: {
);
}
+function isAnalysisModelCandidate(candidate: ModelCandidate) {
+ const descriptor = {
+ capabilities: candidate.model.capabilities,
+ modelId: candidate.model.modelId,
+ providerType: candidate.provider.type,
+ supportsImageGen: candidate.model.supportsImageGen,
+ };
+
+ return (
+ candidate.model.isEnabled &&
+ candidate.provider.isEnabled &&
+ !candidate.model.isSubModel &&
+ candidate.model.healthStatus !== "auto_disabled" &&
+ !isImageGenerationModelDescriptor(descriptor) &&
+ !isVideoModelDescriptor(descriptor)
+ );
+}
+
function extractJsonBlock>(text: string): T | null {
const matched = String(text || "").match(/\{[\s\S]*\}/);
if (!matched) return null;
@@ -107,9 +145,10 @@ function normalizeShotIdeas(value: unknown) {
function normalizeVideoAnalysisRows(
value: unknown,
framePreviews: string[],
+ totalDurationSeconds?: number | null,
) {
if (!Array.isArray(value)) return [];
- return value
+ const rows = value
.map((item, index) => {
const raw =
item && typeof item === "object" && !Array.isArray(item)
@@ -210,6 +249,8 @@ function normalizeVideoAnalysisRows(
} => Boolean(item),
)
.slice(0, 8);
+
+ return normalizeVideoAnalysisTimeline(rows, totalDurationSeconds);
}
function normalizeStoryShot(value: unknown, index: number) {
@@ -489,6 +530,7 @@ function buildVideoAnalysisPrompt(params: {
frameCount: number;
imageCount: number;
videoCount: number;
+ durationSeconds?: number | null;
}) {
return [
"你是视频画布工作流里的视频分析节点,需要为后续分镜和生成节点提供结构化输出。",
@@ -496,6 +538,9 @@ function buildVideoAnalysisPrompt(params: {
`上游参考图数量:${params.imageCount}`,
`上游视频数量:${params.videoCount}`,
`已提供按时间顺序抽取的视频关键帧数量:${params.frameCount}`,
+ params.durationSeconds
+ ? `视频总时长:${Number(params.durationSeconds.toFixed(2))} 秒`
+ : "视频总时长未知,请按关键帧顺序稳健估算。",
params.prompt ? `上游脚本/说明:\n${params.prompt}` : "上游没有额外脚本,请重点依据参考画面分析。",
"",
"请只输出 JSON,不要输出 markdown,不要输出解释。",
@@ -530,6 +575,9 @@ function buildVideoAnalysisPrompt(params: {
"recommendedNextSteps": ["2-4条建议的下游动作"]
}`,
"analysisRows 请按镜头时间顺序输出;短视频可以输出 1 行,复杂视频输出 2-6 行。",
+ params.durationSeconds
+ ? `所有镜头必须从 0 秒连续覆盖到 ${Number(params.durationSeconds.toFixed(2))} 秒,不能重叠、留空或超出视频总时长。`
+ : "镜头时间不能重叠,开始时间、结束时间和时长必须相互一致。",
"每行必须覆盖镜号、开始时间、结束时间、时长、画面描述、叙事内容、景别、摄影机角度、摄影机运动、焦距与景深、光线、背景音乐、人声/音效、图像生成提示词、视频运动提示词和关键帧。",
"imagePrompt 和 motionPrompt 优先使用英文;其他字段使用中文;keyframeIndex 使用 1 开始的序号,对应你看到的关键帧顺序。",
"如果没有真实视频帧,只能依据参考图和脚本推断,也要正常输出,但措辞要稳健。",
@@ -594,8 +642,9 @@ async function downscaleVideoFrameForVision(inputBuffer: Buffer) {
async function extractVideoFrameDataUrls(videoUrl: string, frameCount = VIDEO_ANALYSIS_FRAME_COUNT) {
const tempDir = await mkdtemp(path.join(os.tmpdir(), "canvas-video-analysis-"));
+ let durationSeconds: number | null = null;
try {
- const durationSeconds = await getVideoDurationSeconds(videoUrl);
+ durationSeconds = await getVideoDurationSeconds(videoUrl);
const timestamps = buildVideoFrameTimestamps(durationSeconds, frameCount);
const frames = await Promise.all(
timestamps.map(async (timestamp, index) => {
@@ -627,13 +676,13 @@ async function extractVideoFrameDataUrls(videoUrl: string, frameCount = VIDEO_AN
return `data:image/jpeg;base64,${processed.toString("base64")}`;
}),
);
- return frames.filter(Boolean);
+ return { durationSeconds, frames: frames.filter(Boolean) };
} catch (error) {
console.warn("[Canvas Workflow Analyze] extract video frames failed:", {
message: error instanceof Error ? error.message : String(error),
videoUrl,
});
- return [];
+ return { durationSeconds, frames: [] as string[] };
} finally {
await rm(tempDir, { force: true, recursive: true }).catch(() => undefined);
}
@@ -780,10 +829,13 @@ export async function POST(request: Request) {
const prompt = normalizeText(parsed.data.prompt);
const imageUrls = sanitizeUrls(parsed.data.imageUrls, 4);
const videoUrls = sanitizeUrls(parsed.data.videoUrls, 4);
- const extractedVideoFrames =
+ const extractedVideo =
parsed.data.mode === "video-analysis" && videoUrls.length > 0
? await extractVideoFrameDataUrls(videoUrls[0], VIDEO_ANALYSIS_FRAME_COUNT)
- : [];
+ : { durationSeconds: null, frames: [] as string[] };
+ const extractedVideoFrames = extractedVideo.frames;
+ const videoDurationSeconds =
+ extractedVideo.durationSeconds || parsed.data.durationSeconds || null;
const preferredModelRef = normalizeText(parsed.data.preferredModelRef || parsed.data.model);
if (!prompt && imageUrls.length === 0 && videoUrls.length === 0) {
return NextResponse.json(
@@ -797,60 +849,76 @@ export async function POST(request: Request) {
return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
}
+ const setting = await getSystemSetting({
+ key: VIDEO_CANVAS_FUNCTION_CONFIG_KEY,
+ }).catch(() => null);
+ const videoCanvasConfig = setting?.value
+ ? normalizeVideoCanvasFunctionConfig(setting.value)
+ : defaultVideoCanvasFunctionConfig;
+ const functionDefinition = videoCanvasConfig.functions[parsed.data.mode];
+ if (!videoCanvasConfig.enabled || functionDefinition?.enabled === false) {
+ return NextResponse.json(
+ { success: false, error: `${functionDefinition?.label || "当前功能"}已停用` },
+ { status: 403 },
+ );
+ }
+ const pointsCharge = await prepareFeaturePointsCharge({
+ forceEnabled:
+ parsed.data.mode === "video-analysis" && (functionDefinition?.cost || 0) > 0,
+ userId: session.user.id,
+ unitCost: parsed.data.mode === "video-analysis" ? functionDefinition?.cost || 0 : 0,
+ reason: parsed.data.mode,
+ note: functionDefinition?.label || "视频画布工作流分析",
+ });
+ if (!pointsCharge.ok) {
+ return NextResponse.json(pointsCharge.payload, { status: pointsCharge.status });
+ }
+
const [allModels, agentModel] = await Promise.all([
getEnabledAiModelsWithProvider(false),
getAgentAiModelWithProvider(),
]);
- const compatibleModels = allModels.filter(
- (item) => !item.model.isSubModel && item.model.isEnabled
- );
+ const compatibleModels = allModels.filter(isAnalysisModelCandidate);
+ const hasVisionInputs = imageUrls.length > 0 || extractedVideoFrames.length > 0;
+ const requestCandidates = hasVisionInputs
+ ? compatibleModels.filter((item) => supportsVision(item.model))
+ : compatibleModels;
const preferredModel =
preferredModelRef
- ? compatibleModels.find(
+ ? requestCandidates.find(
(item) =>
item.model.id === preferredModelRef ||
item.model.modelId === preferredModelRef,
)
: null;
+ const compatibleAgentModel =
+ agentModel &&
+ isAnalysisModelCandidate(agentModel) &&
+ (!hasVisionInputs || supportsVision(agentModel.model))
+ ? agentModel
+ : null;
+ const orderedCandidates = [
+ preferredModel,
+ compatibleAgentModel,
+ ...requestCandidates,
+ ].filter(
+ (candidate, index, items): candidate is ModelCandidate =>
+ Boolean(candidate) &&
+ items.findIndex((item) => item?.model.id === candidate?.model.id) === index,
+ );
- const hasVisionInputs = imageUrls.length > 0 || extractedVideoFrames.length > 0;
-
- let selectedModel =
- preferredModel && (!hasVisionInputs || supportsVision(preferredModel.model))
- ? preferredModel
- : hasVisionInputs &&
- agentModel &&
- supportsVision(agentModel.model)
- ? agentModel
- : hasVisionInputs
- ? compatibleModels.find((item) => supportsVision(item.model))
- : agentModel || compatibleModels[0];
-
- if (!selectedModel) {
- selectedModel = compatibleModels[0];
- }
-
- if (!selectedModel) {
+ if (orderedCandidates.length === 0) {
return NextResponse.json(
{ success: false, error: "当前站点没有可用的分析模型" },
{ status: 500 }
);
}
- const useVision = hasVisionInputs && supportsVision(selectedModel.model);
- const codingModelId = getAiModelAgentCodingModelId(selectedModel.model);
- const model = await getLanguageModelById(selectedModel.model.id, codingModelId);
- if (!model) {
- return NextResponse.json(
- { success: false, error: "分析模型初始化失败" },
- { status: 500 }
- );
- }
-
const promptText =
parsed.data.mode === "video-analysis"
- ? buildVideoAnalysisPrompt({
+ ? buildVideoAnalysisPrompt({
+ durationSeconds: videoDurationSeconds,
frameCount: extractedVideoFrames.length,
nodeTitle: normalizeText(parsed.data.nodeTitle),
prompt,
@@ -877,7 +945,7 @@ export async function POST(request: Request) {
videoCount: videoUrls.length,
});
- const inlinedImageReferences = useVision
+ const inlinedImageReferences = hasVisionInputs
? (
await inlineVisionImages(imageUrls, {
maxImages: 4,
@@ -888,26 +956,79 @@ export async function POST(request: Request) {
})
).inlineDataUrls
: [];
- const referenceImages = useVision
- ? [...inlinedImageReferences, ...extractedVideoFrames].slice(0, 6)
- : [];
+ const referenceImages = [...inlinedImageReferences, ...extractedVideoFrames].slice(0, 6);
+ let generatedResult: {
+ candidate: ModelCandidate;
+ parsedJson: Record;
+ text: string;
+ } | null = null;
- const { text } = await generateText({
- model,
- messages: [
- {
- role: "user",
- content: [
- { type: "text", text: promptText },
- ...referenceImages.map((image) => ({ type: "image" as const, image })),
+ for (const candidate of orderedCandidates.slice(0, 3)) {
+ try {
+ const model = await getLanguageModelById(
+ candidate.model.id,
+ getAiModelAgentCodingModelId(candidate.model),
+ );
+ if (!model) continue;
+ const { text } = await generateText({
+ abortSignal: AbortSignal.timeout(55_000),
+ model,
+ messages: [
+ {
+ role: "user",
+ content: [
+ { type: "text", text: promptText },
+ ...referenceImages.map((image) => ({ type: "image" as const, image })),
+ ],
+ },
],
- },
- ],
- temperature: 0.2,
- maxTokens: parsed.data.mode === "story-script" ? 4200 : parsed.data.mode === "video-analysis" ? 2200 : 1600,
- } as any);
+ temperature: getCompatibleTextGenerationTemperature(candidate, 0.2),
+ maxTokens:
+ parsed.data.mode === "story-script"
+ ? 4200
+ : parsed.data.mode === "video-analysis"
+ ? 2200
+ : 1600,
+ } as any);
+ const parsedJson = extractJsonBlock>(text);
+ if (!parsedJson) throw new Error("分析模型没有返回有效 JSON");
+ if (
+ parsed.data.mode === "video-analysis" &&
+ normalizeVideoAnalysisRows(
+ parsedJson.analysisRows,
+ extractedVideoFrames,
+ videoDurationSeconds,
+ ).length === 0
+ ) {
+ throw new Error("分析模型没有返回镜头数据");
+ }
+ generatedResult = { candidate, parsedJson, text };
+ break;
+ } catch (error) {
+ logger.warn("workflow analysis model candidate failed", {
+ error,
+ modelId: candidate.model.id,
+ });
+ }
+ }
- const parsedJson = extractJsonBlock>(text) || {};
+ if (!generatedResult) {
+ return NextResponse.json(
+ { success: false, error: "分析模型暂时不可用,请稍后重试" },
+ { status: 502 },
+ );
+ }
+
+ const { candidate: selectedModel, parsedJson, text } = generatedResult;
+ const chargeResult = await pointsCharge.consume({
+ modelId: selectedModel.model.modelId,
+ });
+ if (pointsCharge.chargeable && chargeResult?.ok === false) {
+ return NextResponse.json(
+ { success: false, error: "积分扣除失败,请刷新余额后重试" },
+ { status: 402 },
+ );
+ }
const summary =
normalizeText(parsedJson.summary) ||
normalizeText(text).slice(0, 160);
@@ -916,6 +1037,7 @@ export async function POST(request: Request) {
const analysisRows = normalizeVideoAnalysisRows(
parsedJson.analysisRows,
extractedVideoFrames,
+ videoDurationSeconds,
);
const shotIdeas =
normalizeShotIdeas(parsedJson.shotIdeas).length > 0
diff --git a/app/api/client-performance/agent-style/route.ts b/app/api/client-performance/agent-style/route.ts
index 798477532..37894f2a4 100644
--- a/app/api/client-performance/agent-style/route.ts
+++ b/app/api/client-performance/agent-style/route.ts
@@ -2,6 +2,7 @@ import { sql } from "drizzle-orm";
import { NextResponse } from "next/server";
import { z } from "zod";
import { auth } from "@/app/(auth)/auth";
+import { isAgentModeScopedChatId } from "@/lib/chat/messages-route-access";
import { db } from "@/lib/db";
import { clientPerformanceEvent } from "@/lib/db/schema";
@@ -167,9 +168,12 @@ export async function POST(request: Request) {
console.warn("[ClientPerformance] prune failed", error);
});
+ const persistedChatId =
+ event.chatId && !isAgentModeScopedChatId(event.chatId) ? event.chatId : null;
+
await db.insert(clientPerformanceEvent).values({
userId: session?.user?.id || null,
- chatId: event.chatId || null,
+ chatId: persistedChatId,
projectId: clampText(event.projectId, 64),
eventType: clampText(event.eventType, 64) || "unknown",
section: event.section || "unknown",
diff --git a/app/api/pick-object/route.ts b/app/api/pick-object/route.ts
index b1afb81cd..f7031e31d 100644
--- a/app/api/pick-object/route.ts
+++ b/app/api/pick-object/route.ts
@@ -1,9 +1,15 @@
import { NextResponse } from "next/server";
+import sharp from "sharp";
import { auth } from "@/app/(auth)/auth";
import { callQwen3Vl } from "@/lib/ai/qwen3-vl";
+import { getSystemSetting } from "@/lib/db/queries";
import { createLogger } from "@/lib/observability/logger";
-import { uploadFile } from "@/lib/storage/service";
-import sharp from "sharp";
+import {
+ VIDEO_CANVAS_FUNCTION_CONFIG_KEY,
+ defaultVideoCanvasFunctionConfig,
+ normalizeVideoCanvasFunctionConfig,
+} from "@/lib/video-canvas/config";
+import { readTrustedLocalImage } from "@/lib/video-canvas/local-image-input";
export const runtime = "nodejs";
export const maxDuration = 30;
@@ -21,10 +27,12 @@ interface PickObjectRequest {
imageData: string; // 带红色标记的图片 base64
clickX: number; // 点击位置 X (0-1)
clickY: number; // 点击位置 Y (0-1)
+ mode?: string;
}
interface PickObjectResult {
name: string;
+ labels: string[];
position: string;
confidence: number;
boundingBox: {
@@ -72,7 +80,7 @@ function ensureImageSizeAllowed(byteLength: number) {
}
}
-async function downscaleImageForVision(buffer: Buffer) {
+async function prepareImageForVision(buffer: Buffer, clickX: number, clickY: number) {
const source = sharp(buffer, { failOn: "none" });
const meta = await source.metadata();
const hasAlpha = Boolean(meta.hasAlpha);
@@ -88,41 +96,50 @@ async function downscaleImageForVision(buffer: Buffer) {
withoutEnlargement: true,
});
- if (hasAlpha) {
- return {
- buffer: await resized.png({ compressionLevel: 9 }).toBuffer(),
- ext: "png",
- width,
- height,
- };
- }
-
- return {
- buffer: await resized.jpeg({ quality: 84, mozjpeg: true }).toBuffer(),
- ext: "jpg",
- width,
- height,
- };
-}
-
-async function uploadVisionAnalysisImage(buffer: Buffer, filenamePrefix: string) {
- const { buffer: processedBuffer, ext, width, height } = await downscaleImageForVision(buffer);
- const filename = `${filenamePrefix}-${Date.now()}.${ext}`;
- const uploaded = await uploadFile(filename, processedBuffer as any);
+ const resizedBuffer = hasAlpha
+ ? await resized.png({ compressionLevel: 9 }).toBuffer()
+ : await resized.jpeg({ quality: 84, mozjpeg: true }).toBuffer();
+ const resizedMeta = await sharp(resizedBuffer).metadata();
+ const resizedWidth = resizedMeta.width || MAX_VISION_SIDE;
+ const resizedHeight = resizedMeta.height || MAX_VISION_SIDE;
+ const markerRadius = Math.max(10, Math.round(Math.min(resizedWidth, resizedHeight) * 0.035));
+ const markerX = Math.round(Math.max(0, Math.min(1, clickX)) * resizedWidth);
+ const markerY = Math.round(Math.max(0, Math.min(1, clickY)) * resizedHeight);
+ const markerSvg = Buffer.from(
+ ``,
+ );
+ const outputBuffer = await sharp(resizedBuffer)
+ .composite([{ input: markerSvg, left: 0, top: 0 }])
+ .png({ compressionLevel: 9 })
+ .toBuffer();
logger.info("vision image prepared", {
originalWidth: width,
originalHeight: height,
maxSide: MAX_VISION_SIDE,
- outputExt: ext,
- outputBytes: processedBuffer.length,
+ outputExt: "png",
+ outputBytes: outputBuffer.length,
});
- return uploaded.url;
+ return `data:image/png;base64,${outputBuffer.toString("base64")}`;
}
-async function normalizeImageInput(imageData: string): Promise {
+async function normalizeImageInput(
+ imageData: string,
+ clickX: number,
+ clickY: number,
+): Promise {
const raw = String(imageData || "").trim();
if (!raw) throw new Error("缺少图片数据");
+ const localImage = await readTrustedLocalImage(raw);
+ if (localImage) {
+ ensureImageSizeAllowed(localImage.byteLength);
+ return prepareImageForVision(localImage, clickX, clickY);
+ }
+
if (/^https?:\/\//i.test(raw)) {
const url = new URL(raw);
if (!["http:", "https:"].includes(url.protocol) || isBlockedImageHost(url.hostname)) {
@@ -145,7 +162,7 @@ async function normalizeImageInput(imageData: string): Promise {
}
const arrayBuffer = await response.arrayBuffer();
ensureImageSizeAllowed(arrayBuffer.byteLength);
- return uploadVisionAnalysisImage(Buffer.from(arrayBuffer), "pick-object-url");
+ return prepareImageForVision(Buffer.from(arrayBuffer), clickX, clickY);
} finally {
clearTimeout(timeout);
}
@@ -160,7 +177,7 @@ async function normalizeImageInput(imageData: string): Promise {
ensureImageSizeAllowed(Math.ceil((base64Body.length * 3) / 4));
const buffer = Buffer.from(base64Body, "base64");
ensureImageSizeAllowed(buffer.byteLength);
- return uploadVisionAnalysisImage(buffer, "pick-object-base64");
+ return prepareImageForVision(buffer, clickX, clickY);
}
export async function POST(request: Request) {
@@ -204,12 +221,19 @@ export async function POST(request: Request) {
}, { status: 400 });
}
- const normalizedImage = await normalizeImageInput(imageData);
+ const normalizedImage = await normalizeImageInput(imageData, clickX, clickY);
+ const setting = await getSystemSetting({ key: VIDEO_CANVAS_FUNCTION_CONFIG_KEY }).catch(
+ () => null,
+ );
+ const config = setting?.value
+ ? normalizeVideoCanvasFunctionConfig(setting.value)
+ : defaultVideoCanvasFunctionConfig;
+ const definition = config.functions["element-mark"];
- logger.info("vision model selected", { model: "qwen3-vl" });
+ logger.info("vision model selected", { model: definition?.model || "qwen3-vl-plus" });
// 构建 prompt - AI 通过红色标记来识别用户选择的物品
- const prompt = `
+ const fallbackPrompt = `
图片中有一个红色圆形标记(带十字线),请识别这个红色标记所指向的具体物品。
【重要】
@@ -232,12 +256,17 @@ export async function POST(request: Request) {
【position 格式】固定4字,从以下选择:
${POSITION_OPTIONS.join('、')}
`.trim();
+ const prompt = String(definition?.prompt || fallbackPrompt).replace(
+ "{{positionOptions}}",
+ POSITION_OPTIONS.join("、"),
+ );
logger.info("vision model call started");
// 调用视觉模型,发送带红色标记的图片
const { text: responseText, model } = await callQwen3Vl({
imageUrl: normalizedImage,
+ modelIds: definition?.model ? [definition.model] : undefined,
prompt,
temperature: 0.1,
maxTokens: 300,
@@ -292,6 +321,13 @@ ${POSITION_OPTIONS.join('、')}
result = {
name,
+ labels: Array.from(
+ new Set(
+ (Array.isArray(parsed.labels) ? parsed.labels : [name])
+ .map((label: unknown) => String(label || "").trim().slice(0, 8))
+ .filter(Boolean),
+ ),
+ ).slice(0, 4),
position,
confidence,
boundingBox: { x: clickX, y: clickY, width: 0, height: 0 }, // 不再使用,保持兼容
@@ -312,6 +348,7 @@ ${POSITION_OPTIONS.join('、')}
result = {
name: '物品',
+ labels: ['物品'],
position,
confidence: 0.5,
boundingBox: { x: clickX, y: clickY, width: 0, height: 0 },
diff --git a/app/api/user/account-center/route.ts b/app/api/user/account-center/route.ts
index 10e9ccb35..4a2425860 100644
--- a/app/api/user/account-center/route.ts
+++ b/app/api/user/account-center/route.ts
@@ -6,6 +6,7 @@ import { model3dAsset, paymentOrder, systemSettings } from "@/lib/db/schema";
import {
getPointsLedgerByUserId,
getPointsSystemConfig,
+ getSystemSetting,
getUserById,
} from "@/lib/db/queries";
import { resolveThreeDModelDisplayName } from "@/lib/model3d/model-alias-config";
@@ -18,6 +19,12 @@ import {
normalizeMembershipUserOverrides,
resolveMembershipPlanIdForUser,
} from "@/lib/membership/config";
+import {
+ VIDEO_CANVAS_FUNCTION_CONFIG_KEY,
+ defaultVideoCanvasFunctionConfig,
+ hasEnabledFixedVideoCanvasPointCosts,
+ normalizeVideoCanvasFunctionConfig,
+} from "@/lib/video-canvas/config";
const MEMBERSHIP_CONFIG_KEY = "membership_config";
const MEMBERSHIP_USER_OVERRIDES_KEY = "membership_user_plan_overrides";
@@ -129,6 +136,7 @@ export async function GET() {
paymentOrders,
configSetting,
overridesSetting,
+ videoCanvasSetting,
] =
await Promise.all([
getUserById({ id: session.user.id }),
@@ -157,6 +165,7 @@ export async function GET() {
.where(eq(systemSettings.key, MEMBERSHIP_USER_OVERRIDES_KEY))
.limit(1)
.then((rows) => rows[0]),
+ getSystemSetting({ key: VIDEO_CANVAS_FUNCTION_CONFIG_KEY }).catch(() => null),
]);
const threeDModelAliasConfig = await getThreeDModelAliasConfigForRuntime();
@@ -274,6 +283,11 @@ export async function GET() {
].sort((a, b) => {
return new Date(b.date).getTime() - new Date(a.date).getTime();
});
+ const videoCanvasConfig = videoCanvasSetting?.value
+ ? normalizeVideoCanvasFunctionConfig(videoCanvasSetting.value)
+ : defaultVideoCanvasFunctionConfig;
+ const fixedVideoCanvasPointsEnabled =
+ hasEnabledFixedVideoCanvasPointCosts(videoCanvasConfig);
return NextResponse.json({
profile: {
@@ -281,7 +295,7 @@ export async function GET() {
email: dbUser?.email || session.user.email || "",
currentPlanName: currentPlan.name,
currentPlanId,
- availablePoints: pointsConfig.enabled
+ availablePoints: pointsConfig.enabled || fixedVideoCanvasPointsEnabled
? Math.max(0, Number(dbUser?.pointsBalance || 0))
: currentPlan.monthlyCredits,
monthlyCredits: currentPlan.monthlyCredits,
diff --git a/app/api/user/membership-summary/route.ts b/app/api/user/membership-summary/route.ts
index a2c5716ed..eb85608c6 100644
--- a/app/api/user/membership-summary/route.ts
+++ b/app/api/user/membership-summary/route.ts
@@ -6,9 +6,16 @@ import { systemSettings } from "@/lib/db/schema";
import {
checkUserImageQuota,
getPointsSystemConfig,
+ getSystemSetting,
getUserById,
getUserMonthlyImageCount,
} from "@/lib/db/queries";
+import {
+ VIDEO_CANVAS_FUNCTION_CONFIG_KEY,
+ defaultVideoCanvasFunctionConfig,
+ hasEnabledFixedVideoCanvasPointCosts,
+ normalizeVideoCanvasFunctionConfig,
+} from "@/lib/video-canvas/config";
import {
defaultMembershipConfig,
defaultMembershipUserOverrides,
@@ -29,7 +36,15 @@ export async function GET() {
return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
}
- const [configSetting, overridesSetting, quota, totalImages, pointsConfig, dbUser] = await Promise.all([
+ const [
+ configSetting,
+ overridesSetting,
+ quota,
+ totalImages,
+ pointsConfig,
+ dbUser,
+ videoCanvasSetting,
+ ] = await Promise.all([
db
.select()
.from(systemSettings)
@@ -46,6 +61,7 @@ export async function GET() {
getUserMonthlyImageCount(session.user.id),
getPointsSystemConfig(),
getUserById({ id: session.user.id }),
+ getSystemSetting({ key: VIDEO_CANVAS_FUNCTION_CONFIG_KEY }).catch(() => null),
]);
const config = configSetting?.value
@@ -63,7 +79,15 @@ export async function GET() {
);
const currentPlan = getMembershipPlanById(config, currentPlanId);
- const currentCredits = pointsConfig.enabled
+ const videoCanvasConfig = videoCanvasSetting?.value
+ ? normalizeVideoCanvasFunctionConfig(videoCanvasSetting.value)
+ : defaultVideoCanvasFunctionConfig;
+ const fixedVideoCanvasPointsEnabled =
+ hasEnabledFixedVideoCanvasPointCosts(videoCanvasConfig);
+ const effectivePointsConfig = fixedVideoCanvasPointsEnabled
+ ? { ...pointsConfig, enabled: true }
+ : pointsConfig;
+ const currentCredits = effectivePointsConfig.enabled
? Math.max(0, Number(dbUser?.pointsBalance || 0))
: quota.remaining !== null
? Math.max(0, quota.remaining)
@@ -71,7 +95,7 @@ export async function GET() {
return NextResponse.json({
config,
- pointsSystemConfig: pointsConfig,
+ pointsSystemConfig: effectivePointsConfig,
isHighVip: Boolean(dbUser?.isHighVip),
currentPlanId,
currentPlan,
diff --git a/app/api/video-canvas/assets/route.ts b/app/api/video-canvas/assets/route.ts
new file mode 100644
index 000000000..feeab90c6
--- /dev/null
+++ b/app/api/video-canvas/assets/route.ts
@@ -0,0 +1,267 @@
+import { and, desc, eq, ilike, inArray, ne, sql } from "drizzle-orm";
+import { NextResponse } from "next/server";
+import { auth } from "@/app/(auth)/auth";
+import { db } from "@/lib/db";
+import { mediaAsset } from "@/lib/db/schema";
+import { createLogger } from "@/lib/observability/logger";
+import {
+ VIDEO_CANVAS_SUBJECT_UPLOAD_SOURCE_TYPE,
+ VIDEO_CANVAS_SUBJECT_VOICE_SOURCE_TYPE,
+} from "@/lib/video-canvas/subjects";
+
+export const runtime = "nodejs";
+
+const logger = createLogger("video-canvas-assets");
+const SUPPORTED_MEDIA_TYPES = new Set(["image", "video", "audio", "model"]);
+const ASSET_CATEGORIES = new Set(["其它", "人物", "场景", "物品", "风格", "音效"]);
+
+function normalizeAssetIds(value: unknown) {
+ if (!Array.isArray(value)) return [];
+ return Array.from(
+ new Set(
+ value
+ .map((item) => String(item || "").trim())
+ .filter(Boolean)
+ .slice(0, 50),
+ ),
+ );
+}
+
+export async function GET(request: Request) {
+ try {
+ const session = await auth();
+ if (!session?.user?.id) {
+ return NextResponse.json(
+ { success: false, error: "未登录" },
+ { status: 401 },
+ );
+ }
+
+ const url = new URL(request.url);
+ const query = url.searchParams.get("q")?.trim().slice(0, 120) || "";
+ const requestedType = url.searchParams.get("mediaType")?.trim() || "";
+ const mediaType = SUPPORTED_MEDIA_TYPES.has(requestedType)
+ ? requestedType
+ : "";
+ const filters = [
+ eq(mediaAsset.userId, session.user.id),
+ eq(mediaAsset.status, "ready"),
+ ne(mediaAsset.mediaType, "subject"),
+ ne(mediaAsset.sourceType, VIDEO_CANVAS_SUBJECT_UPLOAD_SOURCE_TYPE),
+ ne(mediaAsset.sourceType, VIDEO_CANVAS_SUBJECT_VOICE_SOURCE_TYPE),
+ ];
+ if (mediaType) filters.push(eq(mediaAsset.mediaType, mediaType));
+ if (query) filters.push(ilike(mediaAsset.displayName, `%${query}%`));
+
+ const assets = await db
+ .select({
+ createdAt: mediaAsset.createdAt,
+ displayName: mediaAsset.displayName,
+ durationMs: mediaAsset.durationMs,
+ id: mediaAsset.id,
+ mediaType: mediaAsset.mediaType,
+ metadata: mediaAsset.metadata,
+ mimeType: mediaAsset.mimeType,
+ originalUrl: mediaAsset.originalUrl,
+ previewUrl: mediaAsset.previewUrl,
+ thumbnailUrl: mediaAsset.thumbnailUrl,
+ })
+ .from(mediaAsset)
+ .where(and(...filters))
+ .orderBy(desc(mediaAsset.updatedAt))
+ .limit(100);
+
+ return NextResponse.json({ success: true, assets });
+ } catch (error) {
+ logger.error("asset list failed", { error });
+ return NextResponse.json(
+ { success: false, error: "资产加载失败" },
+ { status: 500 },
+ );
+ }
+}
+
+export async function POST(request: Request) {
+ try {
+ const session = await auth();
+ if (!session?.user?.id) {
+ return NextResponse.json(
+ { success: false, error: "未登录" },
+ { status: 401 },
+ );
+ }
+
+ const body = (await request.json().catch(() => null)) as {
+ action?: string;
+ name?: string;
+ } | null;
+ if (body?.action !== "create-folder") {
+ return NextResponse.json(
+ { success: false, error: "不支持的操作" },
+ { status: 400 },
+ );
+ }
+
+ const name = String(body.name || "").trim().slice(0, 20);
+ if (!name) {
+ return NextResponse.json(
+ { success: false, error: "请输入文件夹名称" },
+ { status: 400 },
+ );
+ }
+
+ const folderUrl = `asset-folder://${crypto.randomUUID()}`;
+ const [folder] = await db
+ .insert(mediaAsset)
+ .values({
+ displayName: name,
+ mediaType: "folder",
+ metadata: { isFolder: true },
+ normalizedUrl: folderUrl,
+ originalUrl: folderUrl,
+ sourceTable: "video_canvas_asset_library",
+ sourceType: "video_canvas_asset_folder",
+ status: "ready",
+ userId: session.user.id,
+ })
+ .returning();
+
+ return NextResponse.json({ success: true, asset: folder });
+ } catch (error) {
+ logger.error("asset folder creation failed", { error });
+ return NextResponse.json(
+ { success: false, error: "文件夹创建失败" },
+ { status: 500 },
+ );
+ }
+}
+
+export async function PATCH(request: Request) {
+ try {
+ const session = await auth();
+ if (!session?.user?.id) {
+ return NextResponse.json(
+ { success: false, error: "未登录" },
+ { status: 401 },
+ );
+ }
+
+ const body = (await request.json().catch(() => null)) as {
+ action?: string;
+ assetIds?: unknown;
+ category?: string;
+ folderId?: string | null;
+ } | null;
+ const assetIds = normalizeAssetIds(body?.assetIds);
+ if (!assetIds.length) {
+ return NextResponse.json(
+ { success: false, error: "请选择资产" },
+ { status: 400 },
+ );
+ }
+
+ let metadataPatch: Record;
+ if (body?.action === "update-category") {
+ const category = String(body.category || "").trim();
+ if (!ASSET_CATEGORIES.has(category)) {
+ return NextResponse.json(
+ { success: false, error: "资产标签无效" },
+ { status: 400 },
+ );
+ }
+ metadataPatch = { category };
+ } else if (body?.action === "move-to-folder") {
+ const folderId = String(body.folderId || "").trim() || null;
+ if (folderId) {
+ const [folder] = await db
+ .select({ id: mediaAsset.id })
+ .from(mediaAsset)
+ .where(
+ and(
+ eq(mediaAsset.id, folderId),
+ eq(mediaAsset.userId, session.user.id),
+ eq(mediaAsset.mediaType, "folder"),
+ ),
+ )
+ .limit(1);
+ if (!folder) {
+ return NextResponse.json(
+ { success: false, error: "文件夹不存在" },
+ { status: 404 },
+ );
+ }
+ }
+ metadataPatch = { folderId };
+ } else {
+ return NextResponse.json(
+ { success: false, error: "不支持的操作" },
+ { status: 400 },
+ );
+ }
+
+ const updated = await db
+ .update(mediaAsset)
+ .set({
+ metadata: sql`coalesce(${mediaAsset.metadata}, '{}'::jsonb) || ${JSON.stringify(metadataPatch)}::jsonb`,
+ updatedAt: new Date(),
+ })
+ .where(
+ and(
+ eq(mediaAsset.userId, session.user.id),
+ inArray(mediaAsset.id, assetIds),
+ ne(mediaAsset.mediaType, "folder"),
+ ),
+ )
+ .returning({ id: mediaAsset.id });
+
+ return NextResponse.json({ success: true, updated: updated.length });
+ } catch (error) {
+ logger.error("asset batch update failed", { error });
+ return NextResponse.json(
+ { success: false, error: "批量更新失败" },
+ { status: 500 },
+ );
+ }
+}
+
+export async function DELETE(request: Request) {
+ try {
+ const session = await auth();
+ if (!session?.user?.id) {
+ return NextResponse.json(
+ { success: false, error: "未登录" },
+ { status: 401 },
+ );
+ }
+
+ const body = (await request.json().catch(() => null)) as {
+ assetIds?: unknown;
+ } | null;
+ const assetIds = normalizeAssetIds(body?.assetIds);
+ if (!assetIds.length) {
+ return NextResponse.json(
+ { success: false, error: "请选择资产" },
+ { status: 400 },
+ );
+ }
+
+ const removed = await db
+ .delete(mediaAsset)
+ .where(
+ and(
+ eq(mediaAsset.userId, session.user.id),
+ inArray(mediaAsset.id, assetIds),
+ ne(mediaAsset.mediaType, "folder"),
+ ),
+ )
+ .returning({ id: mediaAsset.id });
+
+ return NextResponse.json({ success: true, removed: removed.length });
+ } catch (error) {
+ logger.error("asset batch delete failed", { error });
+ return NextResponse.json(
+ { success: false, error: "批量删除失败" },
+ { status: 500 },
+ );
+ }
+}
diff --git a/app/api/video-canvas/director-ai-import/route.ts b/app/api/video-canvas/director-ai-import/route.ts
new file mode 100644
index 000000000..674125470
--- /dev/null
+++ b/app/api/video-canvas/director-ai-import/route.ts
@@ -0,0 +1,309 @@
+import { generateText } from "ai";
+import { NextResponse } from "next/server";
+import { z } from "zod";
+import { auth } from "@/app/(auth)/auth";
+import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
+import { inlineVisionImages } from "@/lib/ai/vision-inline";
+import {
+ getAiModelAgentCodingModelId,
+ isAiModelMultimodal,
+} from "@/lib/db/ai-model-flags";
+import {
+ getAgentAiModelWithProvider,
+ getEnabledAiModelsWithProvider,
+ getSystemSetting,
+} from "@/lib/db/queries";
+import { createLogger } from "@/lib/observability/logger";
+import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge";
+import {
+ VIDEO_CANVAS_FUNCTION_CONFIG_KEY,
+ defaultVideoCanvasFunctionConfig,
+ normalizeVideoCanvasFunctionConfig,
+} from "@/lib/video-canvas/config";
+
+export const runtime = "nodejs";
+export const maxDuration = 90;
+
+const logger = createLogger("video-canvas-director-ai-import");
+
+const requestSchema = z.object({
+ imageUrl: z.string().trim().min(1).max(8_000_000),
+ mode: z.enum(["insert", "overwrite"]),
+});
+
+type ModelCandidate = Awaited<
+ ReturnType
+>[number];
+
+const BODY_TYPES = new Set([
+ "mannequin",
+ "female",
+ "broad",
+ "muscular",
+ "slim",
+ "teen",
+ "child",
+ "chibi",
+]);
+const GEOMETRY_TYPES = new Set([
+ "box",
+ "sphere",
+ "cylinder",
+ "torus",
+ "cone",
+ "pyramid",
+]);
+
+function supportsVision(candidate: ModelCandidate) {
+ return (
+ candidate.model.supportsVision === true ||
+ isAiModelMultimodal({
+ capabilities: candidate.model.capabilities,
+ defaultParams: candidate.model.defaultParams,
+ })
+ );
+}
+
+function normalizeModelRef(value: unknown) {
+ const candidate = String(value || "").trim();
+ return candidate && candidate !== "auto" ? candidate : "";
+}
+
+function extractJson(value: unknown) {
+ const match = String(value || "").match(/\{[\s\S]*\}/);
+ if (!match) return null;
+ try {
+ const parsed = JSON.parse(match[0]);
+ return parsed && typeof parsed === "object" && !Array.isArray(parsed)
+ ? (parsed as Record)
+ : null;
+ } catch {
+ return null;
+ }
+}
+
+function clampNumber(value: unknown, fallback: number, min: number, max: number) {
+ const parsed = Number(value);
+ return Number.isFinite(parsed)
+ ? Number(Math.min(max, Math.max(min, parsed)).toFixed(4))
+ : fallback;
+}
+
+function normalizeVector(
+ value: unknown,
+ fallback: [number, number, number],
+ min = -20,
+ max = 20,
+): [number, number, number] {
+ if (!Array.isArray(value) || value.length < 3) return fallback;
+ return [
+ clampNumber(value[0], fallback[0], min, max),
+ clampNumber(value[1], fallback[1], min, max),
+ clampNumber(value[2], fallback[2], min, max),
+ ];
+}
+
+function normalizeColor(value: unknown, fallback: string) {
+ const candidate = String(value || "").trim();
+ return /^#[0-9a-f]{6}$/i.test(candidate) ? candidate : fallback;
+}
+
+function normalizeName(value: unknown, fallback: string) {
+ return String(value || fallback).trim().slice(0, 80) || fallback;
+}
+
+function normalizeResult(value: Record) {
+ const rawCharacters = Array.isArray(value.characters) ? value.characters : [];
+ const rawProps = Array.isArray(value.props) ? value.props : [];
+ const rawCameras = Array.isArray(value.cameras) ? value.cameras : [];
+
+ return {
+ summary: String(value.summary || "").trim().slice(0, 600),
+ characters: rawCharacters.slice(0, 8).flatMap((item, index) => {
+ if (!item || typeof item !== "object" || Array.isArray(item)) return [];
+ const record = item as Record;
+ const bodyType = String(record.bodyType || "mannequin");
+ return [{
+ bodyType: BODY_TYPES.has(bodyType) ? bodyType : "mannequin",
+ color: normalizeColor(record.color, ["#7AA7FF", "#E56C5B", "#6CDB7A"][index % 3]!),
+ name: normalizeName(record.name, `角色${String.fromCharCode(65 + index)}`),
+ position: normalizeVector(record.position, [0, 0, index * 1.2]),
+ }];
+ }),
+ props: rawProps.slice(0, 12).flatMap((item, index) => {
+ if (!item || typeof item !== "object" || Array.isArray(item)) return [];
+ const record = item as Record;
+ const geometryType = String(record.geometryType || "box");
+ return [{
+ color: normalizeColor(record.color, "#d7e7ff"),
+ geometryType: GEOMETRY_TYPES.has(geometryType) ? geometryType : "box",
+ name: normalizeName(record.name, `道具${index + 1}`),
+ position: normalizeVector(record.position, [0, 0, 0]),
+ scale: normalizeVector(record.scale, [1, 1, 1], 0.05, 20),
+ }];
+ }),
+ cameras: rawCameras.slice(0, 4).flatMap((item, index) => {
+ if (!item || typeof item !== "object" || Array.isArray(item)) return [];
+ const record = item as Record;
+ return [{
+ fov: clampNumber(record.fov, 50, 10, 120),
+ name: normalizeName(record.name, `机位${index + 1}`),
+ position: normalizeVector(record.position, [0, 2.2, 8 + index * 1.5], -50, 50),
+ target: normalizeVector(record.target, [0, 1.2, 0], -50, 50),
+ }];
+ }),
+ };
+}
+
+export async function POST(request: Request) {
+ try {
+ const session = await auth();
+ if (!session?.user?.id) {
+ return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
+ }
+
+ const parsed = requestSchema.safeParse(await request.json());
+ if (!parsed.success) {
+ return NextResponse.json(
+ { success: false, error: parsed.error.issues[0]?.message || "请求参数无效" },
+ { status: 400 },
+ );
+ }
+ let imageUrl = parsed.data.imageUrl;
+ if (!imageUrl.startsWith("data:image/")) {
+ try {
+ const resolved = new URL(imageUrl, request.url);
+ imageUrl = resolved.protocol === "http:" || resolved.protocol === "https:"
+ ? resolved.toString()
+ : "";
+ } catch {
+ imageUrl = "";
+ }
+ }
+ if (!imageUrl) {
+ return NextResponse.json({ success: false, error: "图片地址无效" }, { status: 400 });
+ }
+
+ const setting = await getSystemSetting({ key: VIDEO_CANVAS_FUNCTION_CONFIG_KEY }).catch(() => null);
+ const config = setting?.value
+ ? normalizeVideoCanvasFunctionConfig(setting.value)
+ : defaultVideoCanvasFunctionConfig;
+ const aiImportConfig = config.directorDesk.aiImport;
+ const definition = config.functions[aiImportConfig.functionKey];
+ if (!config.enabled || !aiImportConfig.enabled || !definition?.enabled) {
+ return NextResponse.json({ success: false, error: "AI 识图导入已停用" }, { status: 403 });
+ }
+
+ const pointsCharge = await prepareFeaturePointsCharge({
+ userId: session.user.id,
+ unitCost: aiImportConfig.cost,
+ reason: "director-ai-import",
+ note: "3D导演台 AI 识图导入",
+ });
+ if (!pointsCharge.ok) {
+ return NextResponse.json(pointsCharge.payload, { status: pointsCharge.status });
+ }
+
+ const [models, agentModel] = await Promise.all([
+ getEnabledAiModelsWithProvider(false),
+ getAgentAiModelWithProvider(),
+ ]);
+ const compatible = models.filter(
+ (candidate) =>
+ candidate.model.isEnabled &&
+ candidate.provider.isEnabled &&
+ !candidate.model.isSubModel &&
+ candidate.model.healthStatus !== "auto_disabled" &&
+ supportsVision(candidate),
+ );
+ const preferredRef =
+ normalizeModelRef(definition.model) || normalizeModelRef(config.defaultMultimodalModel);
+ const preferred = preferredRef
+ ? compatible.find(
+ (candidate) =>
+ candidate.model.id === preferredRef ||
+ candidate.model.modelId === preferredRef,
+ )
+ : null;
+ const ordered = [
+ preferred,
+ agentModel && supportsVision(agentModel) ? agentModel : null,
+ ...compatible,
+ ].filter(
+ (candidate, index, items): candidate is ModelCandidate =>
+ Boolean(candidate) &&
+ items.findIndex((item) => item?.model.id === candidate?.model.id) === index,
+ );
+ if (ordered.length === 0) {
+ return NextResponse.json({ success: false, error: "当前站点没有可用的多模态模型" }, { status: 500 });
+ }
+
+ const referenceImages = imageUrl.startsWith("data:image/")
+ ? [imageUrl]
+ : (
+ await inlineVisionImages([imageUrl], {
+ maxImages: 1,
+ maxSide: 1280,
+ timeoutMs: 15_000,
+ retries: 1,
+ loggerPrefix: "[Director AI Import]",
+ })
+ ).inlineDataUrls;
+ if (referenceImages.length === 0) {
+ return NextResponse.json({ success: false, error: "识图素材读取失败" }, { status: 400 });
+ }
+
+ for (const candidate of ordered.slice(0, 3)) {
+ try {
+ const model = await getLanguageModelById(
+ candidate.model.id,
+ getAiModelAgentCodingModelId(candidate.model),
+ );
+ if (!model) continue;
+ const { text } = await generateText({
+ abortSignal: AbortSignal.timeout(45_000),
+ maxTokens: 2600,
+ messages: [{
+ role: "user",
+ content: [
+ {
+ type: "text",
+ text: `${definition.prompt}\n\n导入模式:${parsed.data.mode === "overwrite" ? "覆盖当前导演台" : "插入当前导演台"}\n只输出 JSON,不要 Markdown。`,
+ },
+ { type: "image", image: referenceImages[0]! },
+ ],
+ }],
+ model,
+ temperature: 0.15,
+ } as any);
+ const json = extractJson(text);
+ if (!json) continue;
+ const result = normalizeResult(json);
+ await pointsCharge.consume({ modelId: candidate.model.modelId });
+ return NextResponse.json({
+ success: true,
+ data: {
+ ...result,
+ model: candidate.model.displayName,
+ },
+ });
+ } catch (error) {
+ logger.warn("director AI import model failed", {
+ error,
+ modelId: candidate.model.id,
+ });
+ }
+ }
+
+ return NextResponse.json(
+ { success: false, error: "识图模型暂时不可用,请稍后重试" },
+ { status: 502 },
+ );
+ } catch (error) {
+ logger.error("director AI import failed", { error });
+ return NextResponse.json(
+ { success: false, error: error instanceof Error ? error.message : "识图导入失败" },
+ { status: 500 },
+ );
+ }
+}
diff --git a/app/api/video-canvas/director-captures/route.ts b/app/api/video-canvas/director-captures/route.ts
new file mode 100644
index 000000000..454d11d89
--- /dev/null
+++ b/app/api/video-canvas/director-captures/route.ts
@@ -0,0 +1,113 @@
+import { unlink } from "node:fs/promises";
+import path from "node:path";
+import { del } from "@vercel/blob";
+import { and, eq } from "drizzle-orm";
+import { NextResponse } from "next/server";
+import { z } from "zod";
+import { auth } from "@/app/(auth)/auth";
+import { db } from "@/lib/db";
+import { canvasProject, mediaAsset } from "@/lib/db/schema";
+import { createLogger } from "@/lib/observability/logger";
+import {
+ deleteS3ObjectByPath,
+ resolveS3UploadTarget,
+} from "@/lib/storage/service";
+
+export const runtime = "nodejs";
+
+const logger = createLogger("video-canvas-director-captures");
+const deleteSchema = z.object({
+ assetId: z.string().uuid().optional(),
+ canvasId: z.string().uuid(),
+ imageUrl: z.string().trim().max(4096).optional(),
+ pathname: z
+ .string()
+ .trim()
+ .max(1024)
+ .transform((value) => value.replace(/^\/+/, ""))
+ .refine((value) => !value.includes("..") && !value.includes("\\"), "文件路径无效")
+ .optional(),
+});
+
+async function deleteStoredCapture(pathname: string, imageUrl: string) {
+ if (pathname.startsWith("uploads/local/")) {
+ const publicRoot = path.resolve(process.cwd(), "public");
+ const absolutePath = path.resolve(publicRoot, pathname);
+ if (!absolutePath.startsWith(`${publicRoot}${path.sep}`)) {
+ throw new Error("文件路径越界");
+ }
+ await unlink(absolutePath).catch((error: NodeJS.ErrnoException) => {
+ if (error.code !== "ENOENT") throw error;
+ });
+ return;
+ }
+
+ if (
+ imageUrl.includes(".public.blob.vercel-storage.com/") &&
+ process.env.BLOB_READ_WRITE_TOKEN
+ ) {
+ await del(imageUrl, { token: process.env.BLOB_READ_WRITE_TOKEN });
+ return;
+ }
+
+ const target = await resolveS3UploadTarget();
+ if (!target) throw new Error("没有可用的对象存储配置");
+ await deleteS3ObjectByPath(pathname, target.config);
+}
+
+export async function DELETE(request: Request) {
+ try {
+ const session = await auth();
+ if (!session?.user?.id) {
+ return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
+ }
+ const parsed = deleteSchema.safeParse(await request.json());
+ if (!parsed.success) {
+ return NextResponse.json(
+ { success: false, error: parsed.error.issues[0]?.message || "请求参数无效" },
+ { status: 400 },
+ );
+ }
+
+ const [project] = await db
+ .select({ id: canvasProject.id })
+ .from(canvasProject)
+ .where(
+ and(
+ eq(canvasProject.id, parsed.data.canvasId),
+ eq(canvasProject.userId, session.user.id),
+ ),
+ )
+ .limit(1);
+ if (!project) {
+ return NextResponse.json({ success: false, error: "画布不存在" }, { status: 404 });
+ }
+
+ if (parsed.data.pathname) {
+ await deleteStoredCapture(
+ parsed.data.pathname,
+ parsed.data.imageUrl || "",
+ ).catch((error) => {
+ logger.warn("capture file cleanup failed", { error });
+ });
+ }
+ if (parsed.data.assetId) {
+ await db
+ .delete(mediaAsset)
+ .where(
+ and(
+ eq(mediaAsset.id, parsed.data.assetId),
+ eq(mediaAsset.userId, session.user.id),
+ ),
+ );
+ }
+
+ return NextResponse.json({ success: true });
+ } catch (error) {
+ logger.error("capture cleanup failed", { error });
+ return NextResponse.json(
+ { success: false, error: "截图清理失败" },
+ { status: 500 },
+ );
+ }
+}
diff --git a/app/api/video-canvas/effects/search/route.ts b/app/api/video-canvas/effects/search/route.ts
new file mode 100644
index 000000000..064951a12
--- /dev/null
+++ b/app/api/video-canvas/effects/search/route.ts
@@ -0,0 +1,56 @@
+import { NextResponse } from "next/server";
+import { z } from "zod";
+import { auth } from "@/app/(auth)/auth";
+import { getSystemSetting } from "@/lib/db/queries";
+import {
+ VIDEO_CANVAS_FUNCTION_CONFIG_KEY,
+ defaultVideoCanvasFunctionConfig,
+ normalizeVideoCanvasFunctionConfig,
+} from "@/lib/video-canvas/config";
+import { rankVideoCanvasEffectPresets } from "@/lib/video-canvas/video-effects";
+
+export const dynamic = "force-dynamic";
+
+const requestSchema = z.object({
+ query: z.string().trim().max(100).default(""),
+});
+
+export async function POST(request: Request) {
+ const session = await auth();
+ if (!session?.user?.id) {
+ return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
+ }
+
+ const parsed = requestSchema.safeParse(await request.json().catch(() => ({})));
+ if (!parsed.success) {
+ return NextResponse.json(
+ { success: false, error: parsed.error.issues[0]?.message || "搜索参数无效" },
+ { status: 400 },
+ );
+ }
+
+ const setting = await getSystemSetting({
+ key: VIDEO_CANVAS_FUNCTION_CONFIG_KEY,
+ }).catch(() => null);
+ const config = setting?.value
+ ? normalizeVideoCanvasFunctionConfig(setting.value)
+ : defaultVideoCanvasFunctionConfig;
+
+ if (!config.enabled) {
+ return NextResponse.json(
+ { success: false, error: "视频画布功能已停用" },
+ { status: 403 },
+ );
+ }
+
+ return NextResponse.json({
+ success: true,
+ data: {
+ presets: rankVideoCanvasEffectPresets(
+ config.effectPresets,
+ parsed.data.query,
+ ),
+ query: parsed.data.query,
+ },
+ });
+}
diff --git a/app/api/video-canvas/history-assets/route.ts b/app/api/video-canvas/history-assets/route.ts
new file mode 100644
index 000000000..cdcb91911
--- /dev/null
+++ b/app/api/video-canvas/history-assets/route.ts
@@ -0,0 +1,153 @@
+import { and, desc, eq, inArray, ne, sql } from "drizzle-orm";
+import { NextResponse } from "next/server";
+import { auth } from "@/app/(auth)/auth";
+import { db } from "@/lib/db";
+import { getSystemSetting } from "@/lib/db/queries";
+import { mediaAsset } from "@/lib/db/schema";
+import { createLogger } from "@/lib/observability/logger";
+import {
+ VIDEO_CANVAS_FUNCTION_CONFIG_KEY,
+ defaultVideoCanvasFunctionConfig,
+ normalizeVideoCanvasFunctionConfig,
+} from "@/lib/video-canvas/config";
+import {
+ VIDEO_CANVAS_HISTORY_HIDDEN_AT_KEY,
+ VIDEO_CANVAS_HISTORY_MEDIA_TYPES,
+ normalizeVideoCanvasHistoryAssetIds,
+} from "@/lib/video-canvas/history-assets";
+import {
+ VIDEO_CANVAS_SUBJECT_UPLOAD_SOURCE_TYPE,
+ VIDEO_CANVAS_SUBJECT_VOICE_SOURCE_TYPE,
+} from "@/lib/video-canvas/subjects";
+
+export const dynamic = "force-dynamic";
+export const runtime = "nodejs";
+
+const logger = createLogger("video-canvas-history-assets");
+
+async function getHistoryConfig() {
+ const setting = await getSystemSetting({
+ key: VIDEO_CANVAS_FUNCTION_CONFIG_KEY,
+ }).catch(() => null);
+ return setting?.value
+ ? normalizeVideoCanvasFunctionConfig(setting.value).historyAssets
+ : defaultVideoCanvasFunctionConfig.historyAssets;
+}
+
+export async function GET() {
+ try {
+ const session = await auth();
+ if (!session?.user?.id) {
+ return NextResponse.json(
+ { success: false, error: "未登录" },
+ { status: 401 },
+ );
+ }
+
+ const [config, assets] = await Promise.all([
+ getHistoryConfig(),
+ db
+ .select({
+ createdAt: mediaAsset.createdAt,
+ displayName: mediaAsset.displayName,
+ durationMs: mediaAsset.durationMs,
+ height: mediaAsset.height,
+ id: mediaAsset.id,
+ mediaType: mediaAsset.mediaType,
+ metadata: mediaAsset.metadata,
+ mimeType: mediaAsset.mimeType,
+ modelId: mediaAsset.modelId,
+ originalUrl: mediaAsset.originalUrl,
+ previewUrl: mediaAsset.previewUrl,
+ prompt: mediaAsset.prompt,
+ sourceNodeId: mediaAsset.sourceNodeId,
+ sourceType: mediaAsset.sourceType,
+ thumbnailUrl: mediaAsset.thumbnailUrl,
+ width: mediaAsset.width,
+ })
+ .from(mediaAsset)
+ .where(
+ and(
+ eq(mediaAsset.userId, session.user.id),
+ eq(mediaAsset.status, "ready"),
+ inArray(mediaAsset.mediaType, [
+ ...VIDEO_CANVAS_HISTORY_MEDIA_TYPES,
+ ]),
+ ne(
+ mediaAsset.sourceType,
+ VIDEO_CANVAS_SUBJECT_UPLOAD_SOURCE_TYPE,
+ ),
+ ne(
+ mediaAsset.sourceType,
+ VIDEO_CANVAS_SUBJECT_VOICE_SOURCE_TYPE,
+ ),
+ sql`coalesce(${mediaAsset.metadata}->>${VIDEO_CANVAS_HISTORY_HIDDEN_AT_KEY}, '') = ''`,
+ ),
+ )
+ .orderBy(desc(mediaAsset.createdAt))
+ .limit(500),
+ ]);
+
+ return NextResponse.json({
+ success: true,
+ assets: config.enabled ? assets.slice(0, config.maxItems) : [],
+ });
+ } catch (error) {
+ logger.error("history asset list failed", { error });
+ return NextResponse.json(
+ { success: false, error: "历史资产加载失败" },
+ { status: 500 },
+ );
+ }
+}
+
+export async function DELETE(request: Request) {
+ try {
+ const session = await auth();
+ if (!session?.user?.id) {
+ return NextResponse.json(
+ { success: false, error: "未登录" },
+ { status: 401 },
+ );
+ }
+
+ const body = (await request.json().catch(() => null)) as {
+ assetIds?: unknown;
+ } | null;
+ const assetIds = normalizeVideoCanvasHistoryAssetIds(body?.assetIds);
+ if (assetIds.length === 0) {
+ return NextResponse.json(
+ { success: false, error: "请选择历史资产" },
+ { status: 400 },
+ );
+ }
+
+ const hiddenAt = new Date().toISOString();
+ const hidden = await db
+ .update(mediaAsset)
+ .set({
+ metadata: sql`coalesce(${mediaAsset.metadata}, '{}'::jsonb) || ${JSON.stringify({
+ [VIDEO_CANVAS_HISTORY_HIDDEN_AT_KEY]: hiddenAt,
+ })}::jsonb`,
+ updatedAt: new Date(),
+ })
+ .where(
+ and(
+ eq(mediaAsset.userId, session.user.id),
+ eq(mediaAsset.status, "ready"),
+ inArray(mediaAsset.id, assetIds),
+ inArray(mediaAsset.mediaType, [...VIDEO_CANVAS_HISTORY_MEDIA_TYPES]),
+ sql`coalesce(${mediaAsset.metadata}->>${VIDEO_CANVAS_HISTORY_HIDDEN_AT_KEY}, '') = ''`,
+ ),
+ )
+ .returning({ id: mediaAsset.id });
+
+ return NextResponse.json({ success: true, removed: hidden.length });
+ } catch (error) {
+ logger.error("history asset delete failed", { error });
+ return NextResponse.json(
+ { success: false, error: "历史记录删除失败" },
+ { status: 500 },
+ );
+ }
+}
diff --git a/app/api/video-canvas/prompt-tools/route.ts b/app/api/video-canvas/prompt-tools/route.ts
new file mode 100644
index 000000000..4c09d8055
--- /dev/null
+++ b/app/api/video-canvas/prompt-tools/route.ts
@@ -0,0 +1,226 @@
+import { generateText } from "ai";
+import { NextResponse } from "next/server";
+import { z } from "zod";
+import { auth } from "@/app/(auth)/auth";
+import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
+import { getCompatibleTextGenerationTemperature } from "@/lib/ai/model-compat";
+import {
+ isImageGenerationModelDescriptor,
+ isVideoModelDescriptor,
+} from "@/lib/ai/model-classification";
+import { getAiModelAgentCodingModelId } from "@/lib/db/ai-model-flags";
+import {
+ getAgentAiModelWithProvider,
+ getEnabledAiModelsWithProvider,
+ getSystemSetting,
+} from "@/lib/db/queries";
+import { createLogger } from "@/lib/observability/logger";
+import {
+ VIDEO_CANVAS_FUNCTION_CONFIG_KEY,
+ defaultVideoCanvasFunctionConfig,
+ normalizeVideoCanvasFunctionConfig,
+} from "@/lib/video-canvas/config";
+import {
+ buildVideoPromptStructureInstructions,
+ injectVideoPromptStructure,
+ resolveVideoPromptStructure,
+} from "@/lib/video-canvas/prompt-structure";
+
+export const runtime = "nodejs";
+export const maxDuration = 60;
+
+const logger = createLogger("video-canvas-prompt-tools");
+
+const requestSchema = z.object({
+ marks: z
+ .array(
+ z.object({
+ id: z.string().trim().min(1).max(100),
+ label: z.string().trim().min(1).max(100),
+ })
+ )
+ .max(12)
+ .default([]),
+ mode: z.enum(["optimize", "translate"]),
+ prompt: z.string().trim().min(1, "请输入提示词").max(50_000, "提示词过长"),
+ references: z
+ .array(
+ z.object({
+ id: z.string().trim().min(1).max(100),
+ index: z.number().int().min(1).max(10),
+ label: z.string().trim().min(1).max(100),
+ mediaType: z.enum(["audio", "image", "video"]),
+ offset: z.number().int().min(0).max(50_000),
+ })
+ )
+ .max(20)
+ .default([]),
+});
+
+type ModelCandidate = Awaited<
+ ReturnType
+>[number];
+
+function isTextModelCandidate(candidate: ModelCandidate) {
+ const descriptor = {
+ capabilities: candidate.model.capabilities,
+ modelId: candidate.model.modelId,
+ providerType: candidate.provider.type,
+ supportsImageGen: candidate.model.supportsImageGen,
+ };
+
+ return (
+ candidate.model.isEnabled &&
+ candidate.provider.isEnabled &&
+ !candidate.model.isSubModel &&
+ candidate.model.healthStatus !== "auto_disabled" &&
+ !isImageGenerationModelDescriptor(descriptor) &&
+ !isVideoModelDescriptor(descriptor)
+ );
+}
+
+function normalizeModelRef(value: unknown) {
+ const normalized = String(value || "").trim();
+ return normalized && normalized !== "auto" ? normalized : "";
+}
+
+function normalizeGeneratedText(value: unknown) {
+ return String(value || "")
+ .trim()
+ .replace(/^```(?:text|markdown|md)?\s*/i, "")
+ .replace(/\s*```$/i, "")
+ .replace(/^(?:优化后的提示词|译文|translation)\s*[::]\s*/i, "")
+ .trim();
+}
+
+export async function POST(request: Request) {
+ try {
+ const session = await auth();
+ if (!session?.user?.id) {
+ return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
+ }
+
+ const parsed = requestSchema.safeParse(await request.json());
+ if (!parsed.success) {
+ return NextResponse.json(
+ { success: false, error: parsed.error.issues[0]?.message || "请求参数无效" },
+ { status: 400 },
+ );
+ }
+
+ const setting = await getSystemSetting({ key: VIDEO_CANVAS_FUNCTION_CONFIG_KEY }).catch(
+ () => null,
+ );
+ const config = setting?.value
+ ? normalizeVideoCanvasFunctionConfig(setting.value)
+ : defaultVideoCanvasFunctionConfig;
+ const functionKey =
+ parsed.data.mode === "optimize"
+ ? "video-prompt-optimize"
+ : "video-prompt-translate";
+ const definition = config.functions[functionKey];
+
+ if (!definition?.enabled) {
+ return NextResponse.json(
+ { success: false, error: `${definition?.label || "当前功能"}已停用` },
+ { status: 403 },
+ );
+ }
+
+ const [allModels, agentModel] = await Promise.all([
+ getEnabledAiModelsWithProvider(false),
+ getAgentAiModelWithProvider(),
+ ]);
+ const candidates = allModels.filter(isTextModelCandidate);
+ const preferredModelRef =
+ normalizeModelRef(definition.model) || normalizeModelRef(config.defaultMultimodalModel);
+ const preferredModel = preferredModelRef
+ ? candidates.find(
+ (candidate) =>
+ candidate.model.id === preferredModelRef ||
+ candidate.model.modelId === preferredModelRef,
+ )
+ : null;
+ const orderedCandidates = [
+ preferredModel,
+ agentModel && isTextModelCandidate(agentModel) ? agentModel : null,
+ ...candidates,
+ ].filter(
+ (candidate, index, items): candidate is ModelCandidate =>
+ Boolean(candidate) &&
+ items.findIndex((item) => item?.model.id === candidate?.model.id) === index,
+ );
+
+ if (orderedCandidates.length === 0) {
+ return NextResponse.json(
+ { success: false, error: "当前站点没有可用的提示词处理模型" },
+ { status: 500 },
+ );
+ }
+
+ for (const candidate of orderedCandidates.slice(0, 3)) {
+ try {
+ const codingModelId = getAiModelAgentCodingModelId(candidate.model);
+ const model = await getLanguageModelById(candidate.model.id, codingModelId);
+ if (!model) continue;
+ const structuredPrompt = injectVideoPromptStructure(
+ parsed.data.prompt,
+ parsed.data.references,
+ parsed.data.marks,
+ );
+ const structureInstructions = buildVideoPromptStructureInstructions(
+ parsed.data.references,
+ parsed.data.marks,
+ );
+
+ const { text } = await generateText({
+ abortSignal: AbortSignal.timeout(18_000),
+ maxTokens: parsed.data.mode === "optimize" ? 2200 : 1600,
+ model,
+ prompt:
+ parsed.data.mode === "optimize"
+ ? `原始视频提示词:\n${structuredPrompt}\n\n请按照系统规则输出优化后的提示词正文。`
+ : `待翻译的视频提示词:\n${structuredPrompt}\n\n请按照系统规则只输出译文。`,
+ system: [definition.prompt, structureInstructions].filter(Boolean).join("\n\n"),
+ temperature: getCompatibleTextGenerationTemperature(
+ candidate,
+ parsed.data.mode === "optimize" ? 0.65 : 0.25,
+ ),
+ } as any);
+ const generatedText = normalizeGeneratedText(text);
+ if (!generatedText) continue;
+ const result = resolveVideoPromptStructure(
+ generatedText,
+ parsed.data.references,
+ parsed.data.marks,
+ { flatten: parsed.data.mode === "optimize" },
+ );
+ if (!result.text) continue;
+
+ return NextResponse.json({
+ success: true,
+ data: {
+ mentionOffsets: result.mentionOffsets,
+ model: candidate.model.displayName,
+ preservedMarkIds: result.preservedMarkIds,
+ text: result.text,
+ },
+ });
+ } catch (error) {
+ logger.warn("prompt model candidate failed", {
+ error,
+ modelId: candidate.model.id,
+ });
+ }
+ }
+
+ return NextResponse.json(
+ { success: false, error: "提示词处理模型暂时不可用,请稍后重试" },
+ { status: 502 },
+ );
+ } catch (error) {
+ logger.error("prompt tool failed", { error });
+ const message = error instanceof Error ? error.message : "提示词处理失败";
+ return NextResponse.json({ success: false, error: message }, { status: 500 });
+ }
+}
diff --git a/app/api/video-canvas/subjects/complete-views/route.ts b/app/api/video-canvas/subjects/complete-views/route.ts
new file mode 100644
index 000000000..8f53f437e
--- /dev/null
+++ b/app/api/video-canvas/subjects/complete-views/route.ts
@@ -0,0 +1,226 @@
+import { and, eq } from "drizzle-orm";
+import { NextResponse } from "next/server";
+import { z } from "zod";
+import { auth } from "@/app/(auth)/auth";
+import { generateCanvasImage } from "@/lib/canvas/generate-image";
+import { db } from "@/lib/db";
+import { getPointsSystemConfig, getSystemSetting } from "@/lib/db/queries";
+import { mediaAsset } from "@/lib/db/schema";
+import { createLogger } from "@/lib/observability/logger";
+import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge";
+import {
+ VIDEO_CANVAS_FUNCTION_CONFIG_KEY,
+ defaultVideoCanvasFunctionConfig,
+ normalizeVideoCanvasFunctionConfig,
+} from "@/lib/video-canvas/config";
+import { VIDEO_CANVAS_SUBJECT_UPLOAD_SOURCE_TYPE } from "@/lib/video-canvas/subjects";
+
+export const runtime = "nodejs";
+export const maxDuration = 1200;
+
+const logger = createLogger("video-canvas-subject-view-completion");
+
+const requestSchema = z.object({
+ primaryAssetId: z.string().uuid(),
+});
+
+const SUBJECT_COMPLETION_VIEW_DIRECTIONS = [
+ "正面近景,突出面部与头部特征",
+ "正面全身,完整展示服装与体态",
+ "左前方四十五度全身视角",
+ "右前方四十五度全身视角",
+ "左侧面全身视角",
+ "右侧面全身视角",
+ "背面全身视角",
+ "背面回首的全身视角",
+ "轻微俯视的正面全身视角",
+] as const;
+
+export async function POST(request: Request) {
+ try {
+ const session = await auth();
+ if (!session?.user?.id) {
+ return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
+ }
+ const parsed = requestSchema.safeParse(await request.json());
+ if (!parsed.success) {
+ return NextResponse.json(
+ { success: false, error: parsed.error.issues[0]?.message || "请求参数无效" },
+ { status: 400 },
+ );
+ }
+
+ const setting = await getSystemSetting({
+ key: VIDEO_CANVAS_FUNCTION_CONFIG_KEY,
+ }).catch(() => null);
+ const config = setting?.value
+ ? normalizeVideoCanvasFunctionConfig(setting.value)
+ : defaultVideoCanvasFunctionConfig;
+ const completionConfig = config.subjectLibrary.smartCompletion;
+ if (!config.subjectLibrary.enabled || !completionConfig.enabled) {
+ return NextResponse.json(
+ { success: false, error: "智能补全已停用" },
+ { status: 403 },
+ );
+ }
+
+ const [primaryAsset] = await db
+ .select({
+ id: mediaAsset.id,
+ originalUrl: mediaAsset.originalUrl,
+ })
+ .from(mediaAsset)
+ .where(
+ and(
+ eq(mediaAsset.id, parsed.data.primaryAssetId),
+ eq(mediaAsset.userId, session.user.id),
+ eq(mediaAsset.mediaType, "image"),
+ eq(mediaAsset.status, "ready"),
+ ),
+ )
+ .limit(1);
+ if (!primaryAsset) {
+ return NextResponse.json(
+ { success: false, error: "主参考图不存在或已失效" },
+ { status: 404 },
+ );
+ }
+
+ const pointsConfig = await getPointsSystemConfig();
+ const pointsCharge = await prepareFeaturePointsCharge({
+ modelId: "video-canvas-subject-view-completion",
+ note: "主体其他视角智能补全消耗积分",
+ pointsConfig,
+ reason: "video_canvas_subject_view_completion",
+ unitCost: completionConfig.cost,
+ userId: session.user.id,
+ });
+ if (!pointsCharge.ok) {
+ return NextResponse.json(pointsCharge.payload, { status: pointsCharge.status });
+ }
+
+ const referenceImage = new URL(primaryAsset.originalUrl, request.url).toString();
+ const selectedModel =
+ completionConfig.model === "auto"
+ ? config.defaultImageModel
+ : completionConfig.model;
+ const generatedCandidates = (
+ await Promise.all(
+ Array.from({ length: completionConfig.candidateCount }, async (_, index) => {
+ const viewDirection =
+ SUBJECT_COMPLETION_VIEW_DIRECTIONS[
+ index % SUBJECT_COMPLETION_VIEW_DIRECTIONS.length
+ ];
+ const candidatePrompt = `${completionConfig.prompt}\n本次只生成一张独立候选图。候选方向:${viewDirection}。不要生成拼图、九宫格、文字或边框。`;
+ for (let attempt = 1; attempt <= 2; attempt += 1) {
+ try {
+ const result = await generateCanvasImage({
+ aspectRatio: completionConfig.aspectRatio,
+ debugLabel: "video-canvas-subject-view-completion",
+ debugMeta: {
+ candidateIndex: index,
+ candidateViewDirection: viewDirection,
+ primaryAssetId: primaryAsset.id,
+ subjectViewCompletion: true,
+ },
+ imageCount: 1,
+ imageSize: completionConfig.imageSize,
+ model: selectedModel,
+ ownerKey: `${session.user.id}:subject-view-${index}`,
+ prompt: candidatePrompt,
+ promptExtend: false,
+ referenceImage,
+ });
+ const image = result.images[0];
+ if (!image) throw new Error("模型未返回图片");
+ return {
+ image,
+ model: result.model,
+ modelRef: result.modelRef,
+ prompt: result.prompt,
+ };
+ } catch (error) {
+ logger.warn("subject completion candidate failed", {
+ attempt,
+ candidateIndex: index,
+ error,
+ });
+ }
+ }
+ return null;
+ }),
+ )
+ ).filter((candidate): candidate is NonNullable => Boolean(candidate));
+ if (generatedCandidates.length !== completionConfig.candidateCount) {
+ return NextResponse.json(
+ {
+ success: false,
+ error: `智能补全未能生成全部 ${completionConfig.candidateCount} 张候选图,请重试`,
+ },
+ { status: 502 },
+ );
+ }
+
+ const generatedRows = await db
+ .insert(mediaAsset)
+ .values(
+ generatedCandidates.map((candidate, index) => ({
+ displayName: `主体补充视角 ${index + 1}`,
+ mediaType: "image",
+ metadata: {
+ candidateIndex: index,
+ generatedBy: "subject-smart-completion",
+ primaryAssetId: primaryAsset.id,
+ },
+ mimeType: "image/png",
+ modelId: candidate.modelRef || candidate.model,
+ normalizedUrl: candidate.image.originalUrl,
+ originalUrl: candidate.image.originalUrl,
+ previewUrl: candidate.image.thumbnailUrl || candidate.image.originalUrl,
+ prompt: candidate.prompt,
+ rootAssetId: primaryAsset.id,
+ sourceTable: "video_canvas_subject",
+ sourceType: VIDEO_CANVAS_SUBJECT_UPLOAD_SOURCE_TYPE,
+ status: "ready",
+ thumbnailUrl: candidate.image.thumbnailUrl || candidate.image.originalUrl,
+ userId: session.user.id,
+ })),
+ )
+ .returning({
+ assetId: mediaAsset.id,
+ name: mediaAsset.displayName,
+ url: mediaAsset.originalUrl,
+ });
+
+ const primaryGeneratedCandidate = generatedCandidates[0];
+ await pointsCharge.consume({
+ modelId: primaryGeneratedCandidate.modelRef || primaryGeneratedCandidate.model,
+ }).catch((error) => {
+ logger.warn("subject completion points consumption failed", { error });
+ });
+
+ return NextResponse.json({
+ success: true,
+ data: {
+ candidates: generatedRows.map((item) => ({
+ assetId: item.assetId,
+ mediaType: "image" as const,
+ name: item.name || "补充视角",
+ url: item.url,
+ })),
+ cost: pointsCharge.unitCost,
+ maxSelected: completionConfig.maxSelected,
+ model: primaryGeneratedCandidate.model,
+ },
+ });
+ } catch (error) {
+ logger.error("subject view completion failed", { error });
+ return NextResponse.json(
+ {
+ success: false,
+ error: error instanceof Error ? error.message : "智能补全失败",
+ },
+ { status: 500 },
+ );
+ }
+}
diff --git a/app/api/video-canvas/subjects/generate-description/route.ts b/app/api/video-canvas/subjects/generate-description/route.ts
new file mode 100644
index 000000000..86eb38ada
--- /dev/null
+++ b/app/api/video-canvas/subjects/generate-description/route.ts
@@ -0,0 +1,274 @@
+import { generateText } from "ai";
+import { and, eq, inArray } from "drizzle-orm";
+import { NextResponse } from "next/server";
+import { z } from "zod";
+import { auth } from "@/app/(auth)/auth";
+import { getLanguageModelById } from "@/lib/ai/dynamic-provider";
+import { getCompatibleTextGenerationTemperature } from "@/lib/ai/model-compat";
+import {
+ isImageGenerationModelDescriptor,
+ isVideoModelDescriptor,
+} from "@/lib/ai/model-classification";
+import { inlineVisionImages } from "@/lib/ai/vision-inline";
+import { db } from "@/lib/db";
+import {
+ getAiModelAgentCodingModelId,
+ isAiModelMultimodal,
+} from "@/lib/db/ai-model-flags";
+import {
+ getAgentAiModelWithProvider,
+ getEnabledAiModelsWithProvider,
+ getPointsSystemConfig,
+ getSystemSetting,
+} from "@/lib/db/queries";
+import { mediaAsset } from "@/lib/db/schema";
+import { createLogger } from "@/lib/observability/logger";
+import {
+ VIDEO_CANVAS_FUNCTION_CONFIG_KEY,
+ defaultVideoCanvasFunctionConfig,
+ normalizeVideoCanvasFunctionConfig,
+} from "@/lib/video-canvas/config";
+import {
+ normalizeGeneratedSubjectDescription,
+ resolveVideoCanvasSubjectCategory,
+} from "@/lib/video-canvas/subjects";
+import { prepareFeaturePointsCharge } from "@/lib/points/feature-charge";
+
+export const runtime = "nodejs";
+export const maxDuration = 60;
+
+const logger = createLogger("video-canvas-subject-description");
+
+const requestSchema = z.object({
+ assetIds: z.array(z.string().uuid()).min(1).max(4),
+ category: z.string().trim().max(20).default("无标签"),
+ frameDataUrl: z
+ .string()
+ .max(12_000_000)
+ .refine(
+ (value) => !value || /^data:image\/(?:jpeg|png|webp);base64,/i.test(value),
+ "视频预览帧无效",
+ )
+ .optional(),
+});
+
+type ModelCandidate = Awaited<
+ ReturnType
+>[number];
+
+function isDescriptionModelCandidate(candidate: ModelCandidate) {
+ const descriptor = {
+ capabilities: candidate.model.capabilities,
+ modelId: candidate.model.modelId,
+ providerType: candidate.provider.type,
+ supportsImageGen: candidate.model.supportsImageGen,
+ };
+ return (
+ candidate.model.isEnabled &&
+ candidate.provider.isEnabled &&
+ !candidate.model.isSubModel &&
+ candidate.model.healthStatus !== "auto_disabled" &&
+ !isImageGenerationModelDescriptor(descriptor) &&
+ !isVideoModelDescriptor(descriptor) &&
+ (Boolean(candidate.model.supportsVision) ||
+ isAiModelMultimodal({
+ capabilities: candidate.model.capabilities,
+ defaultParams: candidate.model.defaultParams,
+ }))
+ );
+}
+
+function normalizeModelRef(value: unknown) {
+ const normalized = String(value || "").trim();
+ return normalized && normalized !== "auto" ? normalized : "";
+}
+
+export async function POST(request: Request) {
+ try {
+ const session = await auth();
+ if (!session?.user?.id) {
+ return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
+ }
+ const parsed = requestSchema.safeParse(await request.json());
+ if (!parsed.success) {
+ return NextResponse.json(
+ { success: false, error: parsed.error.issues[0]?.message || "请求参数无效" },
+ { status: 400 },
+ );
+ }
+
+ const setting = await getSystemSetting({
+ key: VIDEO_CANVAS_FUNCTION_CONFIG_KEY,
+ }).catch(() => null);
+ const config = setting?.value
+ ? normalizeVideoCanvasFunctionConfig(setting.value)
+ : defaultVideoCanvasFunctionConfig;
+ const subjectConfig = config.subjectLibrary;
+ if (!subjectConfig.enabled || !subjectConfig.smartDescription.enabled) {
+ return NextResponse.json(
+ { success: false, error: "智能生成已停用" },
+ { status: 403 },
+ );
+ }
+
+ const assets = await db
+ .select({
+ id: mediaAsset.id,
+ mediaType: mediaAsset.mediaType,
+ originalUrl: mediaAsset.originalUrl,
+ })
+ .from(mediaAsset)
+ .where(
+ and(
+ eq(mediaAsset.userId, session.user.id),
+ eq(mediaAsset.status, "ready"),
+ inArray(mediaAsset.id, parsed.data.assetIds),
+ ),
+ );
+ const assetById = new Map(assets.map((asset) => [asset.id, asset]));
+ const orderedAssets = parsed.data.assetIds.flatMap((id) => {
+ const asset = assetById.get(id);
+ return asset ? [asset] : [];
+ });
+ if (orderedAssets.length !== parsed.data.assetIds.length) {
+ return NextResponse.json(
+ { success: false, error: "主体素材不存在或已失效" },
+ { status: 400 },
+ );
+ }
+
+ const imageUrls = orderedAssets
+ .filter((asset) => asset.mediaType === "image")
+ .map((asset) => new URL(asset.originalUrl, request.url).toString());
+ const inlinedImages = imageUrls.length
+ ? (
+ await inlineVisionImages(imageUrls, {
+ loggerPrefix: "[Video Canvas Subject]",
+ maxImages: subjectConfig.imageMaxCount,
+ maxSide: 1024,
+ retries: 1,
+ timeoutMs: 12_000,
+ })
+ ).inlineDataUrls
+ : [];
+ const references = [
+ ...inlinedImages,
+ ...(parsed.data.frameDataUrl ? [parsed.data.frameDataUrl] : []),
+ ].slice(0, subjectConfig.imageMaxCount);
+ if (references.length === 0) {
+ return NextResponse.json(
+ { success: false, error: "未能读取主体画面" },
+ { status: 400 },
+ );
+ }
+
+ const [allModels, agentModel] = await Promise.all([
+ getEnabledAiModelsWithProvider(false),
+ getAgentAiModelWithProvider(),
+ ]);
+ const candidates = allModels.filter(isDescriptionModelCandidate);
+ const preferredRef = normalizeModelRef(config.defaultMultimodalModel);
+ const preferredModel = preferredRef
+ ? candidates.find(
+ (candidate) =>
+ candidate.model.id === preferredRef ||
+ candidate.model.modelId === preferredRef,
+ )
+ : null;
+ const orderedCandidates = [
+ preferredModel,
+ agentModel && isDescriptionModelCandidate(agentModel) ? agentModel : null,
+ ...candidates,
+ ].filter(
+ (candidate, index, items): candidate is ModelCandidate =>
+ Boolean(candidate) &&
+ items.findIndex((item) => item?.model.id === candidate?.model.id) === index,
+ );
+ if (orderedCandidates.length === 0) {
+ return NextResponse.json(
+ { success: false, error: "当前站点没有可用的多模态模型" },
+ { status: 500 },
+ );
+ }
+
+ const category = resolveVideoCanvasSubjectCategory(
+ parsed.data.category,
+ subjectConfig,
+ );
+ const pointsConfig = await getPointsSystemConfig();
+ const pointsCharge = await prepareFeaturePointsCharge({
+ modelId: "video-canvas-subject-description",
+ note: "主体描述智能生成消耗积分",
+ pointsConfig,
+ reason: "video_canvas_subject_description",
+ unitCost: subjectConfig.smartDescription.cost,
+ userId: session.user.id,
+ });
+ if (!pointsCharge.ok) {
+ return NextResponse.json(pointsCharge.payload, { status: pointsCharge.status });
+ }
+ for (const candidate of orderedCandidates.slice(0, 3)) {
+ try {
+ const model = await getLanguageModelById(
+ candidate.model.id,
+ getAiModelAgentCodingModelId(candidate.model),
+ );
+ if (!model) continue;
+ const { text } = await generateText({
+ abortSignal: AbortSignal.timeout(35_000),
+ maxTokens: subjectConfig.smartDescription.maxTokens,
+ messages: [
+ {
+ role: "user",
+ content: [
+ {
+ type: "text",
+ text: `主体标签:${category}\n参考画面数量:${references.length}\n请根据系统规则生成主体描述。`,
+ },
+ ...references.map((image) => ({ type: "image" as const, image })),
+ ],
+ },
+ ],
+ model,
+ system: subjectConfig.smartDescription.prompt,
+ temperature: getCompatibleTextGenerationTemperature(
+ candidate,
+ subjectConfig.smartDescription.temperature,
+ ),
+ } as any);
+ const description = normalizeGeneratedSubjectDescription(
+ text,
+ subjectConfig.descriptionMaxLength,
+ );
+ if (!description) continue;
+ await pointsCharge.consume({ modelId: candidate.model.id }).catch((error) => {
+ logger.warn("subject description points consumption failed", { error });
+ });
+ return NextResponse.json({
+ success: true,
+ data: {
+ cost: pointsCharge.unitCost,
+ description,
+ model: candidate.model.displayName,
+ },
+ });
+ } catch (error) {
+ logger.warn("subject description model failed", {
+ error,
+ modelId: candidate.model.id,
+ });
+ }
+ }
+
+ return NextResponse.json(
+ { success: false, error: "智能描述模型暂时不可用,请稍后重试" },
+ { status: 502 },
+ );
+ } catch (error) {
+ logger.error("subject description generation failed", { error });
+ return NextResponse.json(
+ { success: false, error: "主体描述生成失败" },
+ { status: 500 },
+ );
+ }
+}
diff --git a/app/api/video-canvas/subjects/route.ts b/app/api/video-canvas/subjects/route.ts
new file mode 100644
index 000000000..f9c770bae
--- /dev/null
+++ b/app/api/video-canvas/subjects/route.ts
@@ -0,0 +1,569 @@
+import { and, desc, eq, inArray } from "drizzle-orm";
+import { randomUUID } from "node:crypto";
+import { NextResponse } from "next/server";
+import { z } from "zod";
+import { auth } from "@/app/(auth)/auth";
+import { db } from "@/lib/db";
+import { getSystemSetting } from "@/lib/db/queries";
+import { mediaAsset } from "@/lib/db/schema";
+import { createLogger } from "@/lib/observability/logger";
+import {
+ VIDEO_CANVAS_FUNCTION_CONFIG_KEY,
+ defaultVideoCanvasFunctionConfig,
+ normalizeVideoCanvasFunctionConfig,
+} from "@/lib/video-canvas/config";
+import {
+ VIDEO_CANVAS_SUBJECT_SOURCE_TYPE,
+ VIDEO_CANVAS_SUBJECT_UPLOAD_SOURCE_TYPE,
+ VIDEO_CANVAS_SUBJECT_VOICE_SOURCE_TYPE,
+ normalizeVideoCanvasSubjectText,
+ resolveVideoCanvasSubjectCategory,
+ validateVideoCanvasSubjectDraft,
+} from "@/lib/video-canvas/subjects";
+
+export const runtime = "nodejs";
+
+const logger = createLogger("video-canvas-subjects");
+
+const createSubjectSchema = z.object({
+ assetIds: z.array(z.string().uuid()).min(1).max(10),
+ category: z.string().trim().max(20),
+ description: z.string().trim().min(1).max(1000),
+ durationSeconds: z.number().finite().positive().max(3600).nullish(),
+ mediaMode: z.enum(["images", "video"]),
+ name: z.string().trim().min(1).max(100),
+ voiceId: z.string().trim().min(1).max(160).default("none"),
+});
+const updateSubjectSchema = createSubjectSchema.extend({
+ subjectId: z.string().uuid(),
+});
+
+async function getSubjectLibraryConfig() {
+ const setting = await getSystemSetting({
+ key: VIDEO_CANVAS_FUNCTION_CONFIG_KEY,
+ }).catch(() => null);
+ return setting?.value
+ ? normalizeVideoCanvasFunctionConfig(setting.value).subjectLibrary
+ : defaultVideoCanvasFunctionConfig.subjectLibrary;
+}
+
+function readMetadataAssetIds(value: unknown) {
+ if (!value || typeof value !== "object" || Array.isArray(value)) return [];
+ const assetIds = (value as Record).assetIds;
+ return Array.isArray(assetIds)
+ ? assetIds.map((item) => String(item || "")).filter(Boolean)
+ : [];
+}
+
+export async function GET(request: Request) {
+ try {
+ const session = await auth();
+ if (!session?.user?.id) {
+ return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
+ }
+
+ const config = await getSubjectLibraryConfig();
+ const requestedCategory = new URL(request.url).searchParams.get("category");
+ const category = requestedCategory
+ ? resolveVideoCanvasSubjectCategory(requestedCategory, config)
+ : "";
+ const subjects = await db
+ .select({
+ createdAt: mediaAsset.createdAt,
+ description: mediaAsset.summary,
+ displayName: mediaAsset.displayName,
+ durationMs: mediaAsset.durationMs,
+ id: mediaAsset.id,
+ metadata: mediaAsset.metadata,
+ originalUrl: mediaAsset.originalUrl,
+ previewUrl: mediaAsset.previewUrl,
+ thumbnailUrl: mediaAsset.thumbnailUrl,
+ })
+ .from(mediaAsset)
+ .where(
+ and(
+ eq(mediaAsset.userId, session.user.id),
+ eq(mediaAsset.mediaType, "subject"),
+ eq(mediaAsset.sourceType, VIDEO_CANVAS_SUBJECT_SOURCE_TYPE),
+ eq(mediaAsset.status, "ready"),
+ ),
+ )
+ .orderBy(desc(mediaAsset.updatedAt))
+ .limit(100);
+ const filteredSubjects = category
+ ? subjects.filter((subject) => {
+ const metadata = subject.metadata;
+ return (
+ metadata &&
+ typeof metadata === "object" &&
+ !Array.isArray(metadata) &&
+ (metadata as Record).category === category
+ );
+ })
+ : subjects;
+
+ return NextResponse.json({ success: true, subjects: filteredSubjects });
+ } catch (error) {
+ logger.error("subject list failed", { error });
+ return NextResponse.json(
+ { success: false, error: "主体加载失败" },
+ { status: 500 },
+ );
+ }
+}
+
+export async function POST(request: Request) {
+ try {
+ const session = await auth();
+ if (!session?.user?.id) {
+ return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
+ }
+ const parsed = createSubjectSchema.safeParse(await request.json());
+ if (!parsed.success) {
+ return NextResponse.json(
+ { success: false, error: parsed.error.issues[0]?.message || "请求参数无效" },
+ { status: 400 },
+ );
+ }
+
+ const config = await getSubjectLibraryConfig();
+ const category = resolveVideoCanvasSubjectCategory(parsed.data.category, config);
+ const validationError = validateVideoCanvasSubjectDraft(
+ {
+ assetCount: parsed.data.assetIds.length,
+ category,
+ description: parsed.data.description,
+ durationSeconds: parsed.data.durationSeconds,
+ mediaMode: parsed.data.mediaMode,
+ name: parsed.data.name,
+ voiceId: parsed.data.voiceId,
+ },
+ config,
+ );
+ if (validationError) {
+ return NextResponse.json(
+ { success: false, error: validationError },
+ { status: config.enabled ? 400 : 403 },
+ );
+ }
+
+ const uploadedAssets = await db
+ .select({
+ id: mediaAsset.id,
+ mediaType: mediaAsset.mediaType,
+ mimeType: mediaAsset.mimeType,
+ originalUrl: mediaAsset.originalUrl,
+ previewUrl: mediaAsset.previewUrl,
+ thumbnailUrl: mediaAsset.thumbnailUrl,
+ })
+ .from(mediaAsset)
+ .where(
+ and(
+ eq(mediaAsset.userId, session.user.id),
+ eq(mediaAsset.status, "ready"),
+ inArray(mediaAsset.id, parsed.data.assetIds),
+ ),
+ );
+ const assetById = new Map(uploadedAssets.map((asset) => [asset.id, asset]));
+ const orderedAssets = parsed.data.assetIds.flatMap((id) => {
+ const asset = assetById.get(id);
+ return asset ? [asset] : [];
+ });
+ if (orderedAssets.length !== parsed.data.assetIds.length) {
+ return NextResponse.json(
+ { success: false, error: "主体素材不存在或已失效" },
+ { status: 400 },
+ );
+ }
+ const expectedMediaType = parsed.data.mediaMode === "images" ? "image" : "video";
+ if (orderedAssets.some((asset) => asset.mediaType !== expectedMediaType)) {
+ return NextResponse.json(
+ { success: false, error: "主体素材类型不一致" },
+ { status: 400 },
+ );
+ }
+
+ let customVoiceAssetId: string | null = null;
+ if (parsed.data.voiceId.startsWith("custom:")) {
+ customVoiceAssetId = parsed.data.voiceId.slice("custom:".length);
+ if (!z.string().uuid().safeParse(customVoiceAssetId).success) {
+ return NextResponse.json(
+ { success: false, error: "自定义音色无效" },
+ { status: 400 },
+ );
+ }
+ const [voiceAsset] = await db
+ .select({ id: mediaAsset.id })
+ .from(mediaAsset)
+ .where(
+ and(
+ eq(mediaAsset.id, customVoiceAssetId),
+ eq(mediaAsset.userId, session.user.id),
+ eq(mediaAsset.sourceType, VIDEO_CANVAS_SUBJECT_VOICE_SOURCE_TYPE),
+ eq(mediaAsset.status, "ready"),
+ ),
+ )
+ .limit(1);
+ if (!voiceAsset) {
+ return NextResponse.json(
+ { success: false, error: "自定义音色不存在或已失效" },
+ { status: 400 },
+ );
+ }
+ }
+
+ const primaryAsset = orderedAssets[0];
+ const description = normalizeVideoCanvasSubjectText(
+ parsed.data.description,
+ config.descriptionMaxLength,
+ );
+ const name = normalizeVideoCanvasSubjectText(
+ parsed.data.name,
+ config.nameMaxLength,
+ );
+ const voice = config.voices.find((item) => item.id === parsed.data.voiceId);
+ const durationMs =
+ parsed.data.mediaMode === "video" && parsed.data.durationSeconds
+ ? Math.round(parsed.data.durationSeconds * 1000)
+ : null;
+ const subjectId = randomUUID();
+ const [subject] = await db
+ .insert(mediaAsset)
+ .values({
+ displayName: name,
+ durationMs,
+ id: subjectId,
+ mediaType: "subject",
+ metadata: {
+ assetIds: parsed.data.assetIds,
+ category,
+ customVoiceAssetId,
+ durationSeconds: parsed.data.durationSeconds || null,
+ mediaMode: parsed.data.mediaMode,
+ mediaUrls: orderedAssets.map((asset) => asset.originalUrl),
+ voiceBadge: voice?.badge || (customVoiceAssetId ? "自定义" : ""),
+ voiceId: parsed.data.voiceId,
+ voiceName: voice?.name || (customVoiceAssetId ? "自定义音色" : "无配音"),
+ },
+ mimeType: "application/x-video-canvas-subject",
+ normalizedUrl: `video-canvas-subject://${subjectId}`,
+ originalUrl: primaryAsset.originalUrl,
+ previewUrl: primaryAsset.previewUrl || primaryAsset.originalUrl,
+ sourceTable: "video_canvas_subject",
+ sourceType: VIDEO_CANVAS_SUBJECT_SOURCE_TYPE,
+ status: "ready",
+ summary: { description },
+ thumbnailUrl:
+ primaryAsset.thumbnailUrl || primaryAsset.previewUrl || primaryAsset.originalUrl,
+ userId: session.user.id,
+ })
+ .returning();
+
+ return NextResponse.json({ success: true, subject });
+ } catch (error) {
+ logger.error("subject creation failed", { error });
+ return NextResponse.json(
+ { success: false, error: "主体创建失败" },
+ { status: 500 },
+ );
+ }
+}
+
+export async function PATCH(request: Request) {
+ try {
+ const session = await auth();
+ if (!session?.user?.id) {
+ return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
+ }
+ const parsed = updateSubjectSchema.safeParse(await request.json());
+ if (!parsed.success) {
+ return NextResponse.json(
+ { success: false, error: parsed.error.issues[0]?.message || "请求参数无效" },
+ { status: 400 },
+ );
+ }
+
+ const [existingSubject] = await db
+ .select({ id: mediaAsset.id, metadata: mediaAsset.metadata })
+ .from(mediaAsset)
+ .where(
+ and(
+ eq(mediaAsset.id, parsed.data.subjectId),
+ eq(mediaAsset.userId, session.user.id),
+ eq(mediaAsset.sourceType, VIDEO_CANVAS_SUBJECT_SOURCE_TYPE),
+ ),
+ )
+ .limit(1);
+ if (!existingSubject) {
+ return NextResponse.json(
+ { success: false, error: "主体不存在" },
+ { status: 404 },
+ );
+ }
+
+ const config = await getSubjectLibraryConfig();
+ const category = resolveVideoCanvasSubjectCategory(parsed.data.category, config);
+ const validationError = validateVideoCanvasSubjectDraft(
+ {
+ assetCount: parsed.data.assetIds.length,
+ category,
+ description: parsed.data.description,
+ durationSeconds: parsed.data.durationSeconds,
+ mediaMode: parsed.data.mediaMode,
+ name: parsed.data.name,
+ voiceId: parsed.data.voiceId,
+ },
+ config,
+ );
+ if (validationError) {
+ return NextResponse.json(
+ { success: false, error: validationError },
+ { status: config.enabled ? 400 : 403 },
+ );
+ }
+
+ const ownedAssets = await db
+ .select({
+ id: mediaAsset.id,
+ mediaType: mediaAsset.mediaType,
+ originalUrl: mediaAsset.originalUrl,
+ previewUrl: mediaAsset.previewUrl,
+ thumbnailUrl: mediaAsset.thumbnailUrl,
+ })
+ .from(mediaAsset)
+ .where(
+ and(
+ eq(mediaAsset.userId, session.user.id),
+ eq(mediaAsset.status, "ready"),
+ inArray(mediaAsset.id, parsed.data.assetIds),
+ ),
+ );
+ const assetById = new Map(ownedAssets.map((asset) => [asset.id, asset]));
+ const orderedAssets = parsed.data.assetIds.flatMap((id) => {
+ const asset = assetById.get(id);
+ return asset ? [asset] : [];
+ });
+ const expectedMediaType = parsed.data.mediaMode === "images" ? "image" : "video";
+ if (
+ orderedAssets.length !== parsed.data.assetIds.length ||
+ orderedAssets.some((asset) => asset.mediaType !== expectedMediaType)
+ ) {
+ return NextResponse.json(
+ { success: false, error: "主体素材不存在、已失效或类型不一致" },
+ { status: 400 },
+ );
+ }
+
+ let customVoiceAssetId: string | null = null;
+ if (parsed.data.voiceId.startsWith("custom:")) {
+ customVoiceAssetId = parsed.data.voiceId.slice("custom:".length);
+ if (!z.string().uuid().safeParse(customVoiceAssetId).success) {
+ return NextResponse.json(
+ { success: false, error: "自定义音色无效" },
+ { status: 400 },
+ );
+ }
+ const [voiceAsset] = await db
+ .select({ id: mediaAsset.id })
+ .from(mediaAsset)
+ .where(
+ and(
+ eq(mediaAsset.id, customVoiceAssetId),
+ eq(mediaAsset.userId, session.user.id),
+ eq(mediaAsset.sourceType, VIDEO_CANVAS_SUBJECT_VOICE_SOURCE_TYPE),
+ eq(mediaAsset.status, "ready"),
+ ),
+ )
+ .limit(1);
+ if (!voiceAsset) {
+ return NextResponse.json(
+ { success: false, error: "自定义音色不存在或已失效" },
+ { status: 400 },
+ );
+ }
+ }
+
+ const primaryAsset = orderedAssets[0];
+ const voice = config.voices.find((item) => item.id === parsed.data.voiceId);
+ const previousMetadata =
+ existingSubject.metadata &&
+ typeof existingSubject.metadata === "object" &&
+ !Array.isArray(existingSubject.metadata)
+ ? (existingSubject.metadata as Record)
+ : {};
+ const previousAssetIds = readMetadataAssetIds(existingSubject.metadata);
+ const previousCustomVoiceId = String(previousMetadata.customVoiceAssetId || "").trim();
+ const [subject] = await db
+ .update(mediaAsset)
+ .set({
+ displayName: normalizeVideoCanvasSubjectText(
+ parsed.data.name,
+ config.nameMaxLength,
+ ),
+ durationMs:
+ parsed.data.mediaMode === "video" && parsed.data.durationSeconds
+ ? Math.round(parsed.data.durationSeconds * 1000)
+ : null,
+ metadata: {
+ assetIds: parsed.data.assetIds,
+ category,
+ customVoiceAssetId,
+ durationSeconds: parsed.data.durationSeconds || null,
+ mediaMode: parsed.data.mediaMode,
+ mediaUrls: orderedAssets.map((asset) => asset.originalUrl),
+ voiceBadge: voice?.badge || (customVoiceAssetId ? "自定义" : ""),
+ voiceId: parsed.data.voiceId,
+ voiceName: voice?.name || (customVoiceAssetId ? "自定义音色" : "无配音"),
+ },
+ originalUrl: primaryAsset.originalUrl,
+ previewUrl: primaryAsset.previewUrl || primaryAsset.originalUrl,
+ summary: {
+ description: normalizeVideoCanvasSubjectText(
+ parsed.data.description,
+ config.descriptionMaxLength,
+ ),
+ },
+ thumbnailUrl:
+ primaryAsset.thumbnailUrl || primaryAsset.previewUrl || primaryAsset.originalUrl,
+ updatedAt: new Date(),
+ })
+ .where(eq(mediaAsset.id, existingSubject.id))
+ .returning();
+
+ const removedAssetIds = previousAssetIds.filter(
+ (assetId) => !parsed.data.assetIds.includes(assetId),
+ );
+ if (removedAssetIds.length) {
+ await db.delete(mediaAsset).where(
+ and(
+ eq(mediaAsset.userId, session.user.id),
+ eq(mediaAsset.sourceType, VIDEO_CANVAS_SUBJECT_UPLOAD_SOURCE_TYPE),
+ inArray(mediaAsset.id, removedAssetIds),
+ ),
+ );
+ }
+ if (
+ previousCustomVoiceId &&
+ previousCustomVoiceId !== customVoiceAssetId &&
+ z.string().uuid().safeParse(previousCustomVoiceId).success
+ ) {
+ await db.delete(mediaAsset).where(
+ and(
+ eq(mediaAsset.id, previousCustomVoiceId),
+ eq(mediaAsset.userId, session.user.id),
+ eq(mediaAsset.sourceType, VIDEO_CANVAS_SUBJECT_VOICE_SOURCE_TYPE),
+ ),
+ );
+ }
+
+ return NextResponse.json({ success: true, subject });
+ } catch (error) {
+ logger.error("subject update failed", { error });
+ return NextResponse.json(
+ { success: false, error: "主体保存失败" },
+ { status: 500 },
+ );
+ }
+}
+
+export async function DELETE(request: Request) {
+ try {
+ const session = await auth();
+ if (!session?.user?.id) {
+ return NextResponse.json({ success: false, error: "未登录" }, { status: 401 });
+ }
+ const body = (await request.json().catch(() => null)) as {
+ subjectId?: string;
+ uploadAssetIds?: unknown;
+ voiceAssetIds?: unknown;
+ } | null;
+ const uploadAssetIds = Array.isArray(body?.uploadAssetIds)
+ ? body.uploadAssetIds.map((item) => String(item || "")).filter(Boolean).slice(0, 10)
+ : [];
+ const voiceAssetIds = Array.isArray(body?.voiceAssetIds)
+ ? body.voiceAssetIds.map((item) => String(item || "")).filter(Boolean).slice(0, 10)
+ : [];
+ const subjectId = String(body?.subjectId || "").trim();
+
+ if (subjectId) {
+ const [subject] = await db
+ .select({ id: mediaAsset.id, metadata: mediaAsset.metadata })
+ .from(mediaAsset)
+ .where(
+ and(
+ eq(mediaAsset.id, subjectId),
+ eq(mediaAsset.userId, session.user.id),
+ eq(mediaAsset.sourceType, VIDEO_CANVAS_SUBJECT_SOURCE_TYPE),
+ ),
+ )
+ .limit(1);
+ if (!subject) {
+ return NextResponse.json(
+ { success: false, error: "主体不存在" },
+ { status: 404 },
+ );
+ }
+ const referencedAssetIds = readMetadataAssetIds(subject.metadata);
+ const subjectMetadata =
+ subject.metadata &&
+ typeof subject.metadata === "object" &&
+ !Array.isArray(subject.metadata)
+ ? (subject.metadata as Record)
+ : {};
+ const customVoiceAssetId = String(
+ subjectMetadata.customVoiceAssetId || "",
+ ).trim();
+ await db.delete(mediaAsset).where(eq(mediaAsset.id, subject.id));
+ if (referencedAssetIds.length > 0) {
+ await db
+ .delete(mediaAsset)
+ .where(
+ and(
+ eq(mediaAsset.userId, session.user.id),
+ eq(mediaAsset.sourceType, VIDEO_CANVAS_SUBJECT_UPLOAD_SOURCE_TYPE),
+ inArray(mediaAsset.id, referencedAssetIds),
+ ),
+ );
+ }
+ if (z.string().uuid().safeParse(customVoiceAssetId).success) {
+ await db
+ .delete(mediaAsset)
+ .where(
+ and(
+ eq(mediaAsset.id, customVoiceAssetId),
+ eq(mediaAsset.userId, session.user.id),
+ eq(mediaAsset.sourceType, VIDEO_CANVAS_SUBJECT_VOICE_SOURCE_TYPE),
+ ),
+ );
+ }
+ return NextResponse.json({ success: true, removed: 1 });
+ }
+
+ const cleanupGroups = [
+ { ids: uploadAssetIds, sourceType: VIDEO_CANVAS_SUBJECT_UPLOAD_SOURCE_TYPE },
+ { ids: voiceAssetIds, sourceType: VIDEO_CANVAS_SUBJECT_VOICE_SOURCE_TYPE },
+ ];
+ let removed = 0;
+ for (const group of cleanupGroups) {
+ if (group.ids.length === 0) continue;
+ const deleted = await db
+ .delete(mediaAsset)
+ .where(
+ and(
+ eq(mediaAsset.userId, session.user.id),
+ eq(mediaAsset.sourceType, group.sourceType),
+ inArray(mediaAsset.id, group.ids),
+ ),
+ )
+ .returning({ id: mediaAsset.id });
+ removed += deleted.length;
+ }
+ return NextResponse.json({ success: true, removed });
+ } catch (error) {
+ logger.error("subject deletion failed", { error });
+ return NextResponse.json(
+ { success: false, error: "主体删除失败" },
+ { status: 500 },
+ );
+ }
+}
diff --git a/app/canvas/[id]/CanvasPageClientRouter.tsx b/app/canvas/[id]/CanvasPageClientRouter.tsx
index 0eb394c91..17f87b36a 100644
--- a/app/canvas/[id]/CanvasPageClientRouter.tsx
+++ b/app/canvas/[id]/CanvasPageClientRouter.tsx
@@ -42,7 +42,10 @@ export function CanvasPageClientRouter({
const id = Array.isArray(routeId) ? routeId[0] || initialId : typeof routeId === "string" ? routeId : initialId;
const mode = searchParams.get("mode") || initialMode;
const requestedWorkspaceType =
- searchParams.get("workspaceType") || initialWorkspaceType;
+ searchParams.get("workspaceType") ||
+ initialWorkspaceType ||
+ initialProject?.workspaceType ||
+ "";
const requestedAgentMode = mode === "agent";
const isVideoMode = !requestedAgentMode && (mode === "video" || requestedWorkspaceType === "video");
const legacyParams = useMemo(() => Promise.resolve({ id }), [id]);
diff --git a/components/agents/AgentGenerationWorkspaceDarkV2.tsx b/components/agents/AgentGenerationWorkspaceDarkV2.tsx
index a21973f1d..727a8b286 100644
--- a/components/agents/AgentGenerationWorkspaceDarkV2.tsx
+++ b/components/agents/AgentGenerationWorkspaceDarkV2.tsx
@@ -4037,7 +4037,15 @@ export function AgentGenerationWorkspaceDarkV2({
? "agent-anchor-materials"
: "agent-anchor-generate";
- const { messages, setMessages, sendMessage, status, stop } = useChat({
+ const {
+ messages,
+ setMessages,
+ sendMessage,
+ status,
+ stop,
+ error: chatError,
+ clearError,
+ } = useChat({
id: chatId,
messages: historyMessages,
experimental_throttle: 100,
@@ -5236,6 +5244,56 @@ export function AgentGenerationWorkspaceDarkV2({
}
}, [pendingGeneration, pendingGenerationProgressCount]);
+ useEffect(() => {
+ if (!pendingGeneration || !chatError) {
+ return;
+ }
+
+ generationRequestSnapshotRef.current = {
+ hiddenPrompt: null,
+ replacementExecutionCardIds: null,
+ replacementRerunReferenceImages: null,
+ };
+ setPendingGeneration(null);
+ toast.error(chatError.message || "生成请求失败,请稍后重试");
+ clearError();
+ }, [chatError, clearError, pendingGeneration]);
+
+ useEffect(() => {
+ if (!pendingGeneration) {
+ return;
+ }
+
+ const elapsedMs = Date.now() - pendingGeneration.timestamp;
+ const remainingMs = AGENT_PENDING_GENERATION_EXPIRY_MS - elapsedMs;
+ const clearStalePendingGeneration = () => {
+ generationRequestSnapshotRef.current = {
+ hiddenPrompt: null,
+ replacementExecutionCardIds: null,
+ replacementRerunReferenceImages: null,
+ };
+ setPendingGeneration((current) => {
+ if (
+ !current ||
+ current.chatId !== pendingGeneration.chatId ||
+ current.timestamp !== pendingGeneration.timestamp
+ ) {
+ return current;
+ }
+ return null;
+ });
+ toast.warning("生成请求超过 30 分钟未回传,已结束处理中状态,可重新生成未回传结果。");
+ };
+
+ if (remainingMs <= 0) {
+ clearStalePendingGeneration();
+ return;
+ }
+
+ const timerId = window.setTimeout(clearStalePendingGeneration, remainingMs);
+ return () => window.clearTimeout(timerId);
+ }, [pendingGeneration]);
+
useEffect(() => {
if (!pendingGeneration || transportIsGenerating) {
return;
@@ -6059,7 +6117,15 @@ export function AgentGenerationWorkspaceDarkV2({
toast.success(`继续处理剩余 ${replacementExecutionCards.length} 张结果图`);
}
try {
- sendMessage({ role: "user", parts });
+ void Promise.resolve(sendMessage({ role: "user", parts })).catch((error) => {
+ generationRequestSnapshotRef.current = {
+ hiddenPrompt: null,
+ replacementExecutionCardIds: null,
+ replacementRerunReferenceImages: null,
+ };
+ setPendingGeneration(null);
+ toast.error(error instanceof Error ? error.message : "发送生成请求失败");
+ });
if (selectedAgentPreset?.type !== "image") {
setInput("");
}
diff --git a/components/agents/use-agent-style-performance-monitor.ts b/components/agents/use-agent-style-performance-monitor.ts
index 699293098..7ac380900 100644
--- a/components/agents/use-agent-style-performance-monitor.ts
+++ b/components/agents/use-agent-style-performance-monitor.ts
@@ -1,6 +1,7 @@
"use client";
import { useEffect, type RefObject } from "react";
+import { isAgentModeScopedChatId } from "@/lib/chat/messages-route-access";
export type AgentStyleSection =
| "brand"
@@ -140,6 +141,9 @@ export function useAgentStylePerformanceMonitor({
chatId,
projectId,
}: UseAgentStylePerformanceMonitorParams) {
+ const persistedChatId =
+ chatId && !isAgentModeScopedChatId(chatId) ? chatId : null;
+
useEffect(() => {
if (
!getBooleanEnvFlag(process.env.NEXT_PUBLIC_AGENT_STYLE_PERF_MONITOR, true) ||
@@ -200,7 +204,7 @@ export function useAgentStylePerformanceMonitor({
browser,
path: window.location.pathname,
projectId: projectId || null,
- chatId: chatId || null,
+ chatId: persistedChatId,
timestamp: new Date().toISOString(),
});
};
@@ -237,5 +241,5 @@ export function useAgentStylePerformanceMonitor({
root.removeEventListener("keydown", markAction, true);
root.removeEventListener("scroll", markAction, true);
};
- }, [chatId, projectId, rootRef]);
+ }, [persistedChatId, projectId, rootRef]);
}
diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx
index d04dab8b3..b5c8cebd5 100644
--- a/components/canvas/CanvasChatPanel.tsx
+++ b/components/canvas/CanvasChatPanel.tsx
@@ -83,6 +83,10 @@ import {
import { resolveCanvasAgentRequestPolicy } from "@/lib/ai/runtime/canvas-agent-kernel";
import { getVideoToolOptionsForMode } from "@/lib/ai/video-tool-capability";
import type { ChatMessage } from "@/lib/types";
+import {
+ normalizeVideoCanvasStoryboardReference,
+ type VideoCanvasStoryboardReferenceRequest,
+} from "@/lib/video-canvas/storyboard-reference";
import type {
AgentPreset,
AgentWorkflowComposerContext,
@@ -281,11 +285,24 @@ import {
stripCanvasModelRoutingLines,
stripMediaMarkup,
} from "@/components/canvas/chat-panel/canvas-chat-media";
+import {
+ buildOpenClawVideoDisplayText,
+ buildOpenClawVideoReferenceText,
+ getOpenClawChatVideoContentType,
+ isOpenClawChatVideoAttachment,
+ isOpenClawChatVideoFile,
+ normalizeOpenClawChatMediaAttachments,
+ OPENCLAW_CHAT_VIDEO_ACCEPT,
+ OPENCLAW_CHAT_VIDEO_MAX_BYTES,
+ OPENCLAW_CHAT_VIDEO_MAX_FILES,
+ OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE,
+} from "@/lib/openclaw-chat/media-attachments";
// 标签悬停预览信息
interface TagPreviewInfo {
url: string;
name: string;
+ contentType?: string;
x: number; // 标签中心x位置
y: number; // 标签顶部y位置
// 聊天面板容器边界
@@ -317,6 +334,12 @@ interface CanvasChatPanelProps {
workspaceVariant?: "canvas" | "video";
canvasToolbarConfig?: CanvasToolbarConfig;
canvasAgentFeatureSettings?: CanvasAgentFeatureSettings;
+ composerReferenceRequest?: VideoCanvasStoryboardReferenceRequest | null;
+ onComposerReferenceHandled?: (requestId: string) => void;
+ onStoryboardReferenceActivate?: (
+ nodeId: string,
+ mediaType: "audio" | "image" | "text" | "video",
+ ) => void;
}
type CanvasComposerMode = "agent" | "image" | "video" | "detail" | "clawd";
@@ -553,9 +576,10 @@ function writeStoredClawdComposerSubmissionSnapshot(
function shouldShowClawdProcessingHint(
message: string,
hasTableFiles: boolean,
- hasImageAttachments: boolean
+ hasImageAttachments: boolean,
+ hasVideoAttachments: boolean
) {
- if (hasTableFiles || hasImageAttachments) {
+ if (hasTableFiles || hasImageAttachments || hasVideoAttachments) {
return true;
}
@@ -616,17 +640,19 @@ function buildClawdImageReferenceText(attachments: PickedObjectAttachment[]) {
return `用户还添加了以下图片作为参考,请按图片链接理解:\n${lines.join("\n")}`;
}
-function getClawdDefaultPrompt(hasTables: boolean, hasImages: boolean) {
- if (hasTables && hasImages) {
- return "请分析上传的表格和图片内容";
- }
- if (hasTables) {
- return "请分析上传的表格内容";
- }
- if (hasImages) {
- return "请分析上传的图片内容";
- }
- return "请分析";
+function getClawdDefaultPrompt(
+ hasTables: boolean,
+ hasImages: boolean,
+ hasVideos: boolean
+) {
+ const mediaNames = [
+ hasTables ? "表格" : "",
+ hasImages ? "图片" : "",
+ hasVideos ? "视频" : "",
+ ].filter(Boolean);
+ return mediaNames.length > 0
+ ? `请分析上传的${mediaNames.join("、")}内容`
+ : "请分析";
}
type VideoStoryboardStreamData =
@@ -2255,6 +2281,9 @@ export function CanvasChatPanel({
workspaceVariant = "canvas",
canvasToolbarConfig: rawCanvasToolbarConfig,
canvasAgentFeatureSettings: propCanvasAgentFeatureSettings,
+ composerReferenceRequest = null,
+ onComposerReferenceHandled,
+ onStoryboardReferenceActivate,
}: CanvasChatPanelProps) {
const isWorkspacePresentation = presentation === "workspace";
const isVideoWorkspace = workspaceVariant === "video";
@@ -2275,6 +2304,9 @@ export function CanvasChatPanel({
const [input, setInput] = useState("");
const inputRef = useRef(input);
const [attachments, setAttachments] = useState([]);
+ const [storyboardContextReferences, setStoryboardContextReferences] = useState<
+ VideoCanvasStoryboardReferenceRequest[]
+ >([]);
const [selectedModelId, setSelectedModelId] = useState(
effectiveAgentEnabled ? "auto" : ""
);
@@ -3729,8 +3761,13 @@ export function CanvasChatPanel({
[]
);
- // 🆕 辅助函数:在 contenteditable 中插入图片标签
- const insertImageTagInEditor = useCallback((url: string, name: string, index: number) => {
+ // 在 contenteditable 中插入图片或视频附件标签
+ const insertImageTagInEditor = useCallback((
+ url: string,
+ name: string,
+ index: number,
+ contentType?: string
+ ) => {
const editableDiv = textareaRef.current as HTMLDivElement | null;
if (!editableDiv) return;
if (!isValidChatHttpUrl(url)) {
@@ -3752,13 +3789,27 @@ export function CanvasChatPanel({
tagSpan.setAttribute('data-attachment-index', String(index));
tagSpan.setAttribute('data-attachment-url', url);
tagSpan.setAttribute('data-attachment-name', name);
+ if (contentType) {
+ tagSpan.setAttribute('data-attachment-content-type', contentType);
+ }
tagSpan.className = 'inline-flex items-center gap-1 bg-muted/60 hover:bg-muted dark:bg-white/10 dark:hover:bg-white/15 dark:text-zinc-100 rounded-full pl-1 pr-2 py-0.5 mx-0.5 align-middle cursor-default select-none';
const shortName = name.length > 10 ? name.substring(0, 8) + '...' : name;
+ const isVideo = isOpenClawChatVideoAttachment(url, contentType);
- const image = document.createElement('img');
- image.src = buildPreviewImageUrl(url, 48);
- image.alt = '';
- image.className = 'h-5 w-5 object-cover rounded-full border border-border/50';
+ const mediaPreview = isVideo
+ ? document.createElement('video')
+ : document.createElement('img');
+ mediaPreview.src = isVideo ? url : buildPreviewImageUrl(url, 48);
+ mediaPreview.className =
+ 'h-5 w-5 object-cover rounded-full border border-border/50 bg-black/5';
+ if (mediaPreview instanceof HTMLImageElement) {
+ mediaPreview.alt = '';
+ } else {
+ mediaPreview.muted = true;
+ mediaPreview.preload = 'metadata';
+ mediaPreview.playsInline = true;
+ mediaPreview.setAttribute('aria-label', '视频附件');
+ }
const label = document.createElement('span');
label.className = 'text-xs text-foreground/80';
@@ -3771,7 +3822,7 @@ export function CanvasChatPanel({
removeButton.className = 'ml-0.5 inline-flex h-4 w-4 items-center justify-center rounded-full text-[11px] leading-none text-foreground/45 hover:bg-black/10 hover:text-foreground dark:hover:bg-white/15';
removeButton.textContent = '×';
- tagSpan.appendChild(image);
+ tagSpan.appendChild(mediaPreview);
tagSpan.appendChild(label);
tagSpan.appendChild(removeButton);
@@ -3795,6 +3846,157 @@ export function CanvasChatPanel({
console.log('[CanvasChat] 已插入图片标签:', url.substring(0, 50));
}, []);
+ const insertStoryboardReferenceTagInEditor = useCallback(
+ (reference: VideoCanvasStoryboardReferenceRequest) => {
+ const editableDiv = textareaRef.current as HTMLDivElement | null;
+ if (!editableDiv) return false;
+
+ const tagSpan = document.createElement("span");
+ tagSpan.contentEditable = "false";
+ if (reference.mediaUrl) {
+ tagSpan.setAttribute("data-attachment-url", reference.mediaUrl);
+ }
+ tagSpan.setAttribute("data-attachment-name", reference.title);
+ tagSpan.setAttribute("data-composer-tag-id", reference.requestId);
+ tagSpan.setAttribute("data-storyboard-reference", "true");
+ tagSpan.setAttribute("data-storyboard-node-id", reference.nodeId);
+ tagSpan.setAttribute(
+ "data-storyboard-media-type",
+ reference.mediaType || "image",
+ );
+ tagSpan.setAttribute(
+ "data-storyboard-activation",
+ reference.activation || "preview",
+ );
+ tagSpan.setAttribute(
+ "title",
+ reference.activation === "locate"
+ ? `${reference.title} · 点击定位到画布节点`
+ : `${reference.title} · 点击查看关键元素`,
+ );
+ tagSpan.className =
+ "node-chatInlineTag mx-0.5 inline-flex max-w-[180px] cursor-pointer select-none items-center gap-1 rounded-[6px] border border-white/12 bg-white/8 px-1 py-0.5 align-middle text-white/82 transition-colors hover:bg-white/12";
+
+ const thumbnail = document.createElement("span");
+ thumbnail.className =
+ "flex h-5 w-5 shrink-0 items-center justify-center overflow-hidden rounded-[4px] bg-black/20";
+ if (reference.mediaType === "audio") {
+ thumbnail.className += " text-[13px] text-white/72";
+ thumbnail.textContent = "♫";
+ } else if (reference.mediaType === "text") {
+ thumbnail.className += " text-[11px] font-medium text-white/72";
+ thumbnail.textContent = "文";
+ } else if (reference.mediaType === "video") {
+ if (reference.previewUrl) {
+ const image = document.createElement("img");
+ image.src = buildPreviewImageUrl(reference.previewUrl, 48);
+ image.alt = "";
+ image.className = "h-full w-full object-cover";
+ thumbnail.appendChild(image);
+ } else {
+ thumbnail.className += " text-[10px] font-medium text-white/72";
+ thumbnail.textContent = "视频";
+ }
+ } else if (!reference.mediaUrl) {
+ thumbnail.className += " text-[11px] font-medium text-white/72";
+ thumbnail.textContent = "图";
+ } else {
+ const image = document.createElement("img");
+ image.src = buildPreviewImageUrl(
+ reference.previewUrl || reference.mediaUrl || "",
+ 48,
+ );
+ image.alt = "";
+ image.className = "h-full w-full object-cover";
+ thumbnail.appendChild(image);
+ }
+
+ const label = document.createElement("span");
+ label.className = "truncate text-[12px] leading-5";
+ label.textContent = reference.title;
+ tagSpan.appendChild(thumbnail);
+ tagSpan.appendChild(label);
+
+ const selection = window.getSelection();
+ if (
+ selection &&
+ selection.rangeCount > 0 &&
+ editableDiv.contains(selection.anchorNode)
+ ) {
+ const range = selection.getRangeAt(0);
+ range.deleteContents();
+ const spaceNode = document.createTextNode("\u200b");
+ range.insertNode(spaceNode);
+ range.insertNode(tagSpan);
+ range.setStartAfter(spaceNode);
+ range.collapse(true);
+ selection.removeAllRanges();
+ selection.addRange(range);
+ } else {
+ editableDiv.appendChild(tagSpan);
+ editableDiv.appendChild(document.createTextNode("\u200b"));
+ }
+
+ return true;
+ },
+ [],
+ );
+
+ const handledStoryboardReferenceIdsRef = useRef>(new Set());
+ useEffect(() => {
+ if (!composerReferenceRequest) return;
+ const reference = normalizeVideoCanvasStoryboardReference(
+ composerReferenceRequest,
+ );
+ if (!reference) {
+ onComposerReferenceHandled?.(composerReferenceRequest.requestId);
+ return;
+ }
+ if (handledStoryboardReferenceIdsRef.current.has(reference.requestId)) {
+ return;
+ }
+ if (!insertStoryboardReferenceTagInEditor(reference)) return;
+
+ handledStoryboardReferenceIdsRef.current.add(reference.requestId);
+ if (reference.activation === "locate") {
+ setStoryboardContextReferences((current) => [
+ ...current.filter((item) => item.requestId !== reference.requestId),
+ reference,
+ ]);
+ }
+ if (reference.mediaUrl) {
+ setAttachments((current) => [
+ ...current,
+ {
+ composerTagId: reference.requestId,
+ contentType: reference.contentType || "image/jpeg",
+ name: reference.title,
+ storyboardReference: {
+ nodeId: reference.nodeId,
+ title: reference.title,
+ },
+ url: reference.mediaUrl || "",
+ },
+ ]);
+ }
+ window.setTimeout(() => {
+ const editor = textareaRef.current as HTMLDivElement | null;
+ editor?.focus();
+ const selection = window.getSelection();
+ if (!(editor && selection)) return;
+ const range = document.createRange();
+ range.selectNodeContents(editor);
+ range.collapse(false);
+ selection.removeAllRanges();
+ selection.addRange(range);
+ }, 0);
+ onComposerReferenceHandled?.(reference.requestId);
+ }, [
+ composerReferenceRequest,
+ insertStoryboardReferenceTagInEditor,
+ onComposerReferenceHandled,
+ ]);
+
const handleCopyUserMessage = useCallback(async (message: ChatMessage) => {
const payload = getUserMessageCopyPayload(message);
if (!payload.text && payload.attachments.length === 0) {
@@ -3945,7 +4147,12 @@ export function CanvasChatPanel({
}
normalizedAttachments.forEach((attachment, index) => {
- insertImageTagInEditor(attachment.url, attachment.name, index);
+ insertImageTagInEditor(
+ attachment.url,
+ attachment.name,
+ index,
+ attachment.contentType
+ );
});
}, 50);
}
@@ -4118,6 +4325,10 @@ export function CanvasChatPanel({
setTagPreview({
url,
name: objectName || attachmentName || tagSpan.textContent || '图片',
+ contentType:
+ attachment?.contentType ||
+ tagSpan.getAttribute('data-attachment-content-type') ||
+ undefined,
x: rect.left + rect.width / 2,
y: rect.top,
containerBounds,
@@ -4168,6 +4379,7 @@ export function CanvasChatPanel({
videoToolCapability?: VideoToolCapabilityContract | null;
imageGenConfig?: {
maxImages?: number;
+ maxVideoReferences?: number;
supportsSize?: boolean;
supports1K?: boolean;
supports2K?: boolean;
@@ -4196,6 +4408,7 @@ export function CanvasChatPanel({
isMultimodal?: boolean;
imageGenConfig?: {
maxImages?: number;
+ maxVideoReferences?: number;
supportsSize?: boolean;
supports1K?: boolean;
supports2K?: boolean;
@@ -4335,7 +4548,6 @@ export function CanvasChatPanel({
() =>
String(siteSettings?.lovartComposerPrefix || "").trim() ||
CANVAS_COMPOSER_PREFIX_DEFAULT,
- maxVideoReferences?: number;
[siteSettings?.lovartComposerPrefix]
);
const canvasComposerPreviewTexts = useMemo(() => {
@@ -4364,7 +4576,6 @@ export function CanvasChatPanel({
};
}
- maxVideoReferences?: number;
let textIndex = 0;
let charIndex = 0;
@@ -4695,6 +4906,25 @@ export function CanvasChatPanel({
[visualModels]
);
const currentModel = allModels.find((model) => model.id === selectedModelId);
+ const atlasVideoReferenceLimit =
+ currentModel?.providerType === "atlascloud"
+ ? Math.max(
+ 0,
+ Math.min(
+ 4,
+ Number(
+ currentModel.capabilities?.maxVideoReferences ||
+ currentModel.imageGenConfig?.maxVideoReferences ||
+ 0
+ )
+ )
+ )
+ : 0;
+ const supportsAtlasVideoReference = atlasVideoReferenceLimit > 0;
+ const requiresAtlasVideoReference = Boolean(
+ supportsAtlasVideoReference &&
+ currentModel?.capabilities?.requiresVideoReference === true
+ );
const isCanvasVideoModel = useCallback(
(model?: (typeof allModels)[number] | null) =>
Boolean(
@@ -4862,25 +5092,6 @@ export function CanvasChatPanel({
);
const selectableModels = useMemo(() => {
if (activeComposerMode === "detail") {
- const atlasVideoReferenceLimit =
- currentModel?.providerType === "atlascloud"
- ? Math.max(
- 0,
- Math.min(
- 4,
- Number(
- currentModel.capabilities?.maxVideoReferences ||
- currentModel.imageGenConfig?.maxVideoReferences ||
- 0
- )
- )
- )
- : 0;
- const supportsAtlasVideoReference = atlasVideoReferenceLimit > 0;
- const requiresAtlasVideoReference = Boolean(
- supportsAtlasVideoReference &&
- currentModel?.capabilities?.requiresVideoReference === true
- );
return [SMART_AUTO_MODEL];
}
if (activeComposerMode === "video") {
@@ -5057,7 +5268,12 @@ export function CanvasChatPanel({
if (toAdd.length > 0) {
setTimeout(() => {
toAdd.forEach((item, index) => {
- insertImageTagInEditor(item.url, item.name, attachments.length + index);
+ insertImageTagInEditor(
+ item.url,
+ item.name,
+ attachments.length + index,
+ item.contentType
+ );
});
(textareaRef.current as HTMLDivElement | null)?.focus();
}, 0);
@@ -5248,7 +5464,12 @@ export function CanvasChatPanel({
if (toAdd.length > 0) {
toAdd.forEach((item, index) => {
- insertImageTagInEditor(item.url, item.name, currentAttachments.length + index);
+ insertImageTagInEditor(
+ item.url,
+ item.name,
+ currentAttachments.length + index,
+ item.contentType
+ );
});
setAttachments([...currentAttachments, ...toAdd]);
@@ -5408,10 +5629,15 @@ export function CanvasChatPanel({
useEffect(() => {
if (isDetailGenerationMode) {
setAttachments([]);
+ setStoryboardContextReferences([]);
setDetailSections({ required: true, strategy: false, assets: true, archives: true });
const editableDiv = textareaRef.current as HTMLDivElement | null;
if (editableDiv) {
- editableDiv.querySelectorAll("[data-attachment-url]").forEach((element) => element.remove());
+ editableDiv
+ .querySelectorAll(
+ '[data-attachment-url], [data-storyboard-reference="true"]',
+ )
+ .forEach((element) => element.remove());
}
} else {
setDetailProgress({ status: "idle", stage: "", current: 0, total: 0 });
@@ -7364,7 +7590,8 @@ export function CanvasChatPanel({
insertImageTagInEditor(
attachment.url,
normalizeChatAttachmentName(attachment.name),
- index
+ index,
+ attachment.contentType
);
});
editableDiv.focus();
@@ -7479,6 +7706,7 @@ export function CanvasChatPanel({
pendingLaunchRef.current = null;
setInput("");
setAttachments([]);
+ setStoryboardContextReferences([]);
setClawdTableFiles([]);
clearSelection();
@@ -7628,6 +7856,7 @@ export function CanvasChatPanel({
setMessages([]);
setInput("");
setAttachments([]);
+ setStoryboardContextReferences([]);
setClawdTableFiles([]);
setIsSessionMenuOpen(false);
setDetailSlotPickerIndex(null);
@@ -10921,15 +11150,44 @@ export function CanvasChatPanel({
: "";
const hasPendingClawdSkillTagPrompt = Boolean(clawdSkillTagPrompt);
const attachmentsSnapshot = isProgrammaticClawdQuickSend ? [] : [...attachments];
- const imageAttachmentsSnapshot = attachmentsSnapshot
+ const sanitizedAttachmentsSnapshot = attachmentsSnapshot
.map((attachment) => sanitizeChatAttachment(attachment))
.filter((attachment): attachment is PickedObjectAttachment => Boolean(attachment));
+ const videoAttachmentsSnapshot = sanitizedAttachmentsSnapshot.filter(
+ (attachment) =>
+ isOpenClawChatVideoAttachment(
+ attachment.url,
+ attachment.contentType
+ )
+ );
+ const imageAttachmentsSnapshot = sanitizedAttachmentsSnapshot.filter(
+ (attachment) =>
+ !isOpenClawChatVideoAttachment(
+ attachment.url,
+ attachment.contentType
+ )
+ );
const hasImageAttachments = imageAttachmentsSnapshot.length > 0;
+ const hasVideoAttachments = videoAttachmentsSnapshot.length > 0;
+ const openClawMediaAttachments = normalizeOpenClawChatMediaAttachments(
+ sanitizedAttachmentsSnapshot.map((attachment) => ({
+ type: isOpenClawChatVideoAttachment(
+ attachment.url,
+ attachment.contentType
+ )
+ ? "video"
+ : "image",
+ url: attachment.url,
+ name: normalizeChatAttachmentName(attachment.name),
+ content_type: attachment.contentType,
+ }))
+ );
if (
!textContent &&
!hasTableFiles &&
!hasImageAttachments &&
+ !hasVideoAttachments &&
!hasPendingClawdSkillTagPrompt
) {
return;
@@ -10939,14 +11197,22 @@ export function CanvasChatPanel({
textContent ||
(clawdSkillTagForRequest && hasPendingClawdSkillTagPrompt
? `请按「${clawdSkillTagForRequest.name}」快捷功能处理。`
- : getClawdDefaultPrompt(hasTableFiles, hasImageAttachments));
+ : getClawdDefaultPrompt(
+ hasTableFiles,
+ hasImageAttachments,
+ hasVideoAttachments
+ ));
const imageReferenceText = buildClawdImageReferenceText(
imageAttachmentsSnapshot
);
+ const videoReferenceText = buildOpenClawVideoReferenceText(
+ openClawMediaAttachments
+ );
const requestMessage = [
baseRequestMessage,
clawdSkillTagPrompt,
imageReferenceText,
+ videoReferenceText,
]
.filter(Boolean)
.join("\n\n");
@@ -10965,6 +11231,7 @@ export function CanvasChatPanel({
const optimisticMessageText = [
buildClawdTableDisplayText(optimisticBaseMessage, tableFilesSnapshot),
buildClawdImageDisplayText(imageAttachmentsSnapshot),
+ buildOpenClawVideoDisplayText(openClawMediaAttachments),
]
.filter(Boolean)
.join("\n\n");
@@ -10981,6 +11248,10 @@ export function CanvasChatPanel({
.map((attachment) => normalizeMediaUrl(attachment.url))
.filter(Boolean)
.sort(),
+ videoAttachments: videoAttachmentsSnapshot
+ .map((attachment) => normalizeMediaUrl(attachment.url))
+ .filter(Boolean)
+ .sort(),
});
if (shouldBlockDuplicateSend(activeChatId, sendSignature)) {
@@ -10991,7 +11262,8 @@ export function CanvasChatPanel({
const nextProcessingHint = shouldShowClawdProcessingHint(
[textContent, clawdSkillTagPrompt].filter(Boolean).join("\n\n"),
hasTableFiles,
- hasImageAttachments
+ hasImageAttachments,
+ hasVideoAttachments
)
? CLAWD_PROCESSING_HINT
: null;
@@ -11094,11 +11366,18 @@ export function CanvasChatPanel({
composerMode: "clawd",
activeChatId,
imageAttachmentCount: imageAttachmentsSnapshot.length || undefined,
+ videoAttachmentCount: videoAttachmentsSnapshot.length || undefined,
skillTagId: clawdSkillTagForRequest?.id || undefined,
skillTagName: clawdSkillTagForRequest?.name || undefined,
skillTagPromptIncluded: hasPendingClawdSkillTagPrompt || undefined,
})
);
+ if (openClawMediaAttachments.length > 0) {
+ formData.append(
+ "attachments",
+ JSON.stringify(openClawMediaAttachments)
+ );
+ }
tableFilesSnapshot.forEach((file) => {
formData.append("tableFiles", file, file.name);
});
@@ -11205,7 +11484,8 @@ export function CanvasChatPanel({
insertImageTagInEditor(
attachment.url,
normalizeChatAttachmentName(attachment.name),
- index
+ index,
+ attachment.contentType
);
});
}, 0);
@@ -11241,7 +11521,13 @@ export function CanvasChatPanel({
return;
}
- if (!composerInput.trim() && attachments.length === 0 && !selectedSkillTag && !hasWorkflowContext) {
+ if (
+ !composerInput.trim() &&
+ attachments.length === 0 &&
+ storyboardContextReferences.length === 0 &&
+ !selectedSkillTag &&
+ !hasWorkflowContext
+ ) {
return;
}
const effectiveModelId = currentModelIdRef.current || selectedModelId;
@@ -11260,6 +11546,20 @@ export function CanvasChatPanel({
let effectiveAttachments = attachments
.map((attachment) => sanitizeChatAttachment(attachment))
.filter((attachment): attachment is PickedObjectAttachment => Boolean(attachment));
+ if (supportsAtlasVideoReference) {
+ let keptVideoCount = 0;
+ effectiveAttachments = effectiveAttachments.filter((attachment) => {
+ if (!isLikelyVideoAttachment(attachment.url, attachment.contentType)) {
+ return true;
+ }
+ keptVideoCount += 1;
+ return keptVideoCount <= atlasVideoReferenceLimit;
+ });
+ if (requiresAtlasVideoReference && keptVideoCount === 0) {
+ toast.error("当前模型需要先上传 1 个参考视频");
+ return;
+ }
+ }
if (hasAgentUploadSlots) {
if (effectiveAttachments.length > agentRequiredUploadCount) {
effectiveAttachments = effectiveAttachments.slice(0, agentRequiredUploadCount);
@@ -11357,6 +11657,12 @@ export function CanvasChatPanel({
if (!textContent && hasWorkflowContext && workflowContext?.summaryText) {
textContent = `请基于当前工作流设定生成方案:${workflowContext.summaryText}`;
}
+ if (!textContent && storyboardContextReferences.length > 0) {
+ textContent = storyboardContextReferences
+ .map((reference) => reference.title)
+ .filter(Boolean)
+ .join(" ");
+ }
const effectiveComposerMode = activeComposerModeRef.current || activeComposerMode;
const effectiveAgentEntryMode = agentEntryModeRef.current || agentEntryMode;
if (!textContent && hasAgentUploadSlots && selectedAgentPreset) {
@@ -11438,22 +11744,27 @@ export function CanvasChatPanel({
hiddenSegments.push("【注意:图片中有标注(箭头、圆圈、文字等)标记的区域是需要重点关注或修改的部分,请根据标注进行处理】");
console.log('[CanvasChat] 检测到标注图片,已生成隐藏提示');
}
+ if (storyboardContextReferences.length > 0) {
+ hiddenSegments.push(
+ storyboardContextReferences
+ .map((reference) => {
+ const typeLabel =
+ reference.mediaType === "text"
+ ? "文本"
+ : reference.mediaType === "audio"
+ ? "音频"
+ : reference.mediaType === "video"
+ ? "视频"
+ : "图片";
+ const context = String(reference.textContent || "").trim();
+ return `【故事板${typeLabel}引用:${reference.title}】${context ? `\n${context}` : ""}`;
+ })
+ .filter(Boolean)
+ .join("\n\n"),
+ );
+ }
if (isDetailGenerationMode && canUseDetailGeneration) {
hiddenSegments.push(
- if (supportsAtlasVideoReference) {
- let keptVideoCount = 0;
- effectiveAttachments = effectiveAttachments.filter((attachment) => {
- if (!isLikelyVideoAttachment(attachment.url, attachment.contentType)) {
- return true;
- }
- keptVideoCount += 1;
- return keptVideoCount <= atlasVideoReferenceLimit;
- });
- if (requiresAtlasVideoReference && keptVideoCount === 0) {
- toast.error("当前模型需要先上传 1 个参考视频");
- return;
- }
- }
[
"【AI详情生成模式】",
"你正在处理电商详情页任务,请先规划分镜结构,再执行生图。",
@@ -11554,6 +11865,7 @@ export function CanvasChatPanel({
// 清空状态
setInput("");
setAttachments([]);
+ setStoryboardContextReferences([]);
// 🆕 清空画布选中状态(防止 useEffect 重新添加标签)
clearSelection();
@@ -11574,6 +11886,7 @@ export function CanvasChatPanel({
rawChatId,
input,
attachments,
+ storyboardContextReferences,
clawdTableFiles,
activeClawdSkillTag,
activeClawdSkillTagPromptApplied,
@@ -11695,7 +12008,12 @@ export function CanvasChatPanel({
}
const newIndex = prev.length;
setTimeout(() => {
- insertImageTagInEditor(uploaded.imageUrl, uploaded.name, newIndex);
+ insertImageTagInEditor(
+ uploaded.imageUrl,
+ uploaded.name,
+ newIndex,
+ uploaded.contentType
+ );
(textareaRef.current as HTMLDivElement | null)?.focus();
}, 0);
@@ -11896,6 +12214,117 @@ export function CanvasChatPanel({
}
}, [clearComposerUploadProgress, updateComposerUploadProgress]);
+ const uploadVideoFileForComposerAttachment = useCallback(async (
+ file: File,
+ progressOptions?: CanvasUploadProgressOptions
+ ): Promise => {
+ if (!file.type.startsWith("video/")) {
+ toast.error("请选择视频文件");
+ return null;
+ }
+ if (file.size > 15 * 1024 * 1024) {
+ toast.error("参考视频大小不能超过 15MB");
+ return null;
+ }
+
+ try {
+ const formData = new FormData();
+ formData.append("file", file);
+ formData.append("purpose", "canvas-video-process");
+ if (progressOptions) updateComposerUploadProgress(progressOptions);
+ const response = await fetchUploadWithRetry(
+ formData,
+ progressOptions
+ ? {
+ onProgress: (progress) => {
+ updateComposerUploadProgress(progressOptions, progress);
+ },
+ }
+ : undefined
+ );
+ if (!response.ok) throw new Error("上传失败");
+
+ const data = await response.json();
+ clearComposerUploadProgress(250);
+ return {
+ imageUrl: String(data.url || ""),
+ name: file.name || `参考视频-${Date.now()}.mp4`,
+ contentType: file.type || "video/mp4",
+ assetId:
+ typeof data.assetId === "string" && data.assetId.trim()
+ ? data.assetId.trim()
+ : undefined,
+ };
+ } catch (_error) {
+ toast.error("参考视频上传失败");
+ return null;
+ }
+ }, [clearComposerUploadProgress, updateComposerUploadProgress]);
+
+ const uploadClawdVideoFileForComposerAttachment = useCallback(async (
+ file: File,
+ progressOptions: CanvasUploadProgressOptions
+ ): Promise => {
+ if (!isOpenClawChatVideoFile(file)) {
+ toast.error("仅支持 MP4、MOV、WebM、M4V 或 MKV 视频");
+ return null;
+ }
+ if (file.size <= 0) {
+ toast.error(`${file.name} 是空文件`);
+ return null;
+ }
+ if (file.size > OPENCLAW_CHAT_VIDEO_MAX_BYTES) {
+ toast.error(`${file.name} 超过 500MB,请先压缩视频`);
+ return null;
+ }
+
+ try {
+ const normalizedContentType =
+ getOpenClawChatVideoContentType(file) || "video/mp4";
+ const uploadFile =
+ file.type === normalizedContentType
+ ? file
+ : new File([file], file.name, {
+ type: normalizedContentType,
+ lastModified: file.lastModified,
+ });
+ const formData = new FormData();
+ formData.append("file", uploadFile);
+ formData.append("purpose", OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE);
+ updateComposerUploadProgress(progressOptions);
+ const response = await fetchUploadWithRetry(formData, {
+ onProgress: (progress) => {
+ updateComposerUploadProgress(progressOptions, progress);
+ },
+ });
+ const data = await response.json().catch(() => null);
+ if (!response.ok) {
+ throw new Error(String(data?.error || data?.details || "视频上传失败"));
+ }
+
+ const imageUrl = String(data?.url || "").trim();
+ if (!isValidChatHttpUrl(imageUrl)) {
+ throw new Error("储存桶未返回可访问的视频地址");
+ }
+
+ clearComposerUploadProgress(250);
+ return {
+ imageUrl,
+ name: file.name || `视频-${Date.now()}.mp4`,
+ contentType:
+ String(data?.contentType || "").trim() ||
+ normalizedContentType,
+ assetId:
+ typeof data?.assetId === "string" && data.assetId.trim()
+ ? data.assetId.trim()
+ : undefined,
+ };
+ } catch (error) {
+ toast.error(error instanceof Error ? error.message : "视频上传失败");
+ return null;
+ }
+ }, [clearComposerUploadProgress, updateComposerUploadProgress]);
+
const handlePastedImages = useCallback(async (files: File[]) => {
if (files.length === 0) return;
@@ -12103,53 +12532,6 @@ export function CanvasChatPanel({
clearComposerUploadProgress,
hasAgentUploadSlots,
isClawdMode,
- const uploadVideoFileForComposerAttachment = useCallback(async (
- file: File,
- progressOptions?: CanvasUploadProgressOptions
- ): Promise => {
- if (!file.type.startsWith("video/")) {
- toast.error("请选择视频文件");
- return null;
- }
- if (file.size > 15 * 1024 * 1024) {
- toast.error("参考视频大小不能超过 15MB");
- return null;
- }
-
- try {
- const formData = new FormData();
- formData.append("file", file);
- formData.append("purpose", "canvas-video-process");
- if (progressOptions) updateComposerUploadProgress(progressOptions);
- const response = await fetchUploadWithRetry(
- formData,
- progressOptions
- ? {
- onProgress: (progress) => {
- updateComposerUploadProgress(progressOptions, progress);
- },
- }
- : undefined
- );
- if (!response.ok) throw new Error("上传失败");
-
- const data = await response.json();
- clearComposerUploadProgress(250);
- return {
- imageUrl: String(data.url || ""),
- name: file.name || `参考视频-${Date.now()}.mp4`,
- contentType: file.type || "video/mp4",
- assetId:
- typeof data.assetId === "string" && data.assetId.trim()
- ? data.assetId.trim()
- : undefined,
- };
- } catch (_error) {
- toast.error("参考视频上传失败");
- return null;
- }
- }, [clearComposerUploadProgress, updateComposerUploadProgress]);
-
isVideoModel,
maybeAutoSetCanvasTitle,
nodes.length,
@@ -12225,18 +12607,27 @@ export function CanvasChatPanel({
const imageFiles = selectedFiles.filter((file) =>
file.type.startsWith("image/")
);
+ const videoFiles = selectedFiles.filter((file) =>
+ isOpenClawChatVideoFile(file)
+ );
const unsupportedFiles = selectedFiles.filter(
- (file) => !isClawdTableFile(file) && !file.type.startsWith("image/")
+ (file) =>
+ !isClawdTableFile(file) &&
+ !file.type.startsWith("image/") &&
+ !isOpenClawChatVideoFile(file)
);
unsupportedFiles.forEach((file) => {
- toast.error(`${file.name} 格式不支持,请上传图片或 .xlsx/.csv/.tsv`);
+ toast.error(
+ `${file.name} 格式不支持,请上传图片、视频或 .xlsx/.csv/.tsv`
+ );
});
if (tableFiles.length > 0) {
addClawdTableFiles(tableFiles);
}
let handledImageCount = 0;
+ let handledVideoCount = 0;
if (imageFiles.length > 0) {
try {
for (const [index, file] of imageFiles.entries()) {
@@ -12265,16 +12656,118 @@ export function CanvasChatPanel({
}
}
+ if (videoFiles.length > 0) {
+ const existingVideoCount = attachments.filter((attachment) =>
+ isOpenClawChatVideoAttachment(
+ attachment.url,
+ attachment.contentType
+ )
+ ).length;
+ const availableVideoSlots = Math.max(
+ 0,
+ OPENCLAW_CHAT_VIDEO_MAX_FILES - existingVideoCount
+ );
+ const acceptedVideoFiles = videoFiles.slice(0, availableVideoSlots);
+ if (acceptedVideoFiles.length < videoFiles.length) {
+ toast.info(
+ `${CLAW_MODE_LABEL} 对话一次最多添加 ${OPENCLAW_CHAT_VIDEO_MAX_FILES} 个视频`
+ );
+ }
+
+ try {
+ for (const [index, file] of acceptedVideoFiles.entries()) {
+ const uploaded = await uploadClawdVideoFileForComposerAttachment(
+ file,
+ {
+ title:
+ acceptedVideoFiles.length === 1
+ ? "正在上传视频到储存桶"
+ : `正在上传 ${acceptedVideoFiles.length} 个视频`,
+ fileName: file.name,
+ fileIndex: index,
+ fileTotal: acceptedVideoFiles.length,
+ target: "composer-reference",
+ }
+ );
+ if (!uploaded) continue;
+
+ handledVideoCount += 1;
+ addUploadedImageAttachmentToComposer(uploaded);
+ }
+ } finally {
+ clearComposerUploadProgress(450);
+ }
+ }
+
if (handledImageCount > 0) {
toast.success(
handledImageCount === 1
? `图片已添加为 ${CLAW_MODE_LABEL} 附件`
: `已添加 ${handledImageCount} 张图片为 ${CLAW_MODE_LABEL} 附件`
+ );
+ }
+ if (handledVideoCount > 0) {
+ toast.success(
+ handledVideoCount === 1
+ ? "视频已上传并添加到 Claw 对话"
+ : `已上传并添加 ${handledVideoCount} 个视频`
);
}
e.target.value = "";
return;
}
+
+ if (supportsAtlasVideoReference) {
+ const selectedVideoFiles = selectedFiles.filter((file) =>
+ file.type.startsWith("video/")
+ );
+ const existingVideoCount = attachments.filter((attachment) =>
+ isLikelyVideoAttachment(attachment.url, attachment.contentType)
+ ).length;
+ const availableVideoSlots = Math.max(
+ 0,
+ atlasVideoReferenceLimit - existingVideoCount
+ );
+ const acceptedVideoFiles = selectedVideoFiles.slice(0, availableVideoSlots);
+
+ if (selectedVideoFiles.length > acceptedVideoFiles.length) {
+ toast.info(`当前模型最多支持 ${atlasVideoReferenceLimit} 个参考视频`);
+ }
+ if (acceptedVideoFiles.length > 0) {
+ let uploadedCount = 0;
+ try {
+ for (const [index, file] of acceptedVideoFiles.entries()) {
+ const uploaded = await uploadVideoFileForComposerAttachment(file, {
+ title: "正在上传参考视频",
+ fileName: file.name,
+ fileIndex: index,
+ fileTotal: acceptedVideoFiles.length,
+ target: "composer-reference",
+ });
+ if (!uploaded?.imageUrl) continue;
+ addUploadedImageAttachmentToComposer(uploaded);
+ uploadedCount += 1;
+ }
+ } finally {
+ clearComposerUploadProgress(450);
+ }
+ if (uploadedCount > 0) {
+ toast.success(
+ uploadedCount === 1
+ ? "参考视频已添加到对话"
+ : `已添加 ${uploadedCount} 个参考视频`
+ );
+ }
+ }
+
+ selectedFiles = selectedFiles.filter(
+ (file) => !file.type.startsWith("video/")
+ );
+ if (selectedFiles.length === 0) {
+ e.target.value = "";
+ return;
+ }
+ }
let firstImageUrl: string | null = null;
// 记录本次上传开始时的节点数量,用于计算新图片位置
@@ -12482,58 +12975,6 @@ export function CanvasChatPanel({
if (acceptedFiles.length === 0) return;
-
- if (supportsAtlasVideoReference) {
- const selectedVideoFiles = selectedFiles.filter((file) =>
- file.type.startsWith("video/")
- );
- const existingVideoCount = attachments.filter((attachment) =>
- isLikelyVideoAttachment(attachment.url, attachment.contentType)
- ).length;
- const availableVideoSlots = Math.max(
- 0,
- atlasVideoReferenceLimit - existingVideoCount
- );
- const acceptedVideoFiles = selectedVideoFiles.slice(0, availableVideoSlots);
-
- if (selectedVideoFiles.length > acceptedVideoFiles.length) {
- toast.info(`当前模型最多支持 ${atlasVideoReferenceLimit} 个参考视频`);
- }
- if (acceptedVideoFiles.length > 0) {
- let uploadedCount = 0;
- try {
- for (const [index, file] of acceptedVideoFiles.entries()) {
- const uploaded = await uploadVideoFileForComposerAttachment(file, {
- title: "正在上传参考视频",
- fileName: file.name,
- fileIndex: index,
- fileTotal: acceptedVideoFiles.length,
- target: "composer-reference",
- });
- if (!uploaded?.imageUrl) continue;
- addUploadedImageAttachmentToComposer(uploaded);
- uploadedCount += 1;
- }
- } finally {
- clearComposerUploadProgress(450);
- }
- if (uploadedCount > 0) {
- toast.success(
- uploadedCount === 1
- ? "参考视频已添加到对话"
- : `已添加 ${uploadedCount} 个参考视频`
- );
- }
- }
-
- selectedFiles = selectedFiles.filter(
- (file) => !file.type.startsWith("video/")
- );
- if (selectedFiles.length === 0) {
- e.target.value = "";
- return;
- }
- }
setClawdTableFiles((currentFiles) => {
const existingSignatures = new Set(
currentFiles.map(
@@ -12567,10 +13008,12 @@ export function CanvasChatPanel({
// 移除附件(通过标签元素删除,支持同图多物品)
const removeAttachmentByTagElement = (tagEl: HTMLElement) => {
const url = tagEl.getAttribute('data-attachment-url') || '';
+ const composerTagId = tagEl.getAttribute('data-composer-tag-id') || '';
const isPicked = tagEl.getAttribute('data-picked-object') === 'true';
const objectName = tagEl.getAttribute('data-object-name') || '';
const objectPosition = tagEl.getAttribute('data-object-position') || '';
setAttachments(prev => prev.filter(a => {
+ if (composerTagId) return a.composerTagId !== composerTagId;
if (!url) return true;
if (isPicked) {
return !(
@@ -12592,9 +13035,15 @@ export function CanvasChatPanel({
if (!container) return;
const tagElements = Array.from(container.querySelectorAll('[data-attachment-url]')) as HTMLElement[];
const tagKeys = new Set();
+ const composerTagIds = new Set();
tagElements.forEach((el) => {
const url = el.getAttribute('data-attachment-url') || '';
if (!url) return;
+ const composerTagId = el.getAttribute('data-composer-tag-id') || '';
+ if (composerTagId) {
+ composerTagIds.add(composerTagId);
+ return;
+ }
const isPicked = el.getAttribute('data-picked-object') === 'true';
if (isPicked) {
const objectName = el.getAttribute('data-object-name') || '';
@@ -12605,6 +13054,7 @@ export function CanvasChatPanel({
}
});
setAttachments(prev => prev.filter((att) => {
+ if (att.composerTagId) return composerTagIds.has(att.composerTagId);
if (att.pickedObject) {
const key = `picked|${att.url}|${att.pickedObject.objectName}|${att.pickedObject.objectPosition}`;
return tagKeys.has(key);
@@ -12641,7 +13091,10 @@ export function CanvasChatPanel({
? composerUploadProgress
: null;
const hasInputOrAttachments = Boolean(
- input.trim() || attachments.length > 0 || (!isDetailGenerationMode && selectedSkillTag)
+ input.trim() ||
+ attachments.length > 0 ||
+ storyboardContextReferences.length > 0 ||
+ (!isDetailGenerationMode && selectedSkillTag)
);
const hasEnoughAgentUploads = !hasAgentUploadSlots || attachments.length >= agentRequiredUploadCount;
const composerPlaceholder = isWorkspacePresentation
@@ -14005,6 +14458,10 @@ export function CanvasChatPanel({
: "";
if (part.type === 'file' && fileUrl) {
+ const isVideoAttachment = isLikelyVideoAttachment(
+ fileUrl,
+ String(part.mediaType || part.contentType || "").toLowerCase()
+ );
renderedUserParts.push(
{
const target = event.target as HTMLElement;
const removeButton = target.closest('[data-attachment-remove]');
- if (!removeButton) return;
- const tagSpan = removeButton.closest('[data-attachment-url]') as HTMLElement | null;
- if (!tagSpan) return;
- event.preventDefault();
- event.stopPropagation();
- removeAttachmentByTagElement(tagSpan);
- tagSpan.remove();
- setTagPreview(null);
+ if (removeButton) {
+ const tagSpan = removeButton.closest('[data-attachment-url]') as HTMLElement | null;
+ if (!tagSpan) return;
+ event.preventDefault();
+ event.stopPropagation();
+ removeAttachmentByTagElement(tagSpan);
+ tagSpan.remove();
+ setTagPreview(null);
+ return;
+ }
+
+ const storyboardTag = target.closest(
+ '[data-storyboard-reference="true"]',
+ ) as HTMLElement | null;
+ if (!storyboardTag) return;
+ const storyboardMediaType = storyboardTag.getAttribute(
+ 'data-storyboard-media-type',
+ );
+ const storyboardActivation = storyboardTag.getAttribute(
+ 'data-storyboard-activation',
+ );
+ if (storyboardActivation === 'locate') {
+ const nodeId = storyboardTag.getAttribute(
+ 'data-storyboard-node-id',
+ );
+ if (!nodeId) return;
+ event.preventDefault();
+ event.stopPropagation();
+ setTagPreview(null);
+ onStoryboardReferenceActivate?.(
+ nodeId,
+ storyboardMediaType === 'audio' ||
+ storyboardMediaType === 'text' ||
+ storyboardMediaType === 'video'
+ ? storyboardMediaType
+ : 'image',
+ );
+ return;
+ }
+ const url = storyboardTag.getAttribute('data-attachment-url');
+ if (!url) return;
+ const rect = storyboardTag.getBoundingClientRect();
+ setTagPreview({
+ name:
+ storyboardTag.getAttribute('data-attachment-name') ||
+ storyboardTag.textContent ||
+ '关键元素',
+ url,
+ x: rect.left + rect.width / 2,
+ y: rect.top,
+ });
}}
onInput={(e) => {
// 从 contenteditable 中提取纯文本(排除附件标签文本)
const target = e.target as HTMLDivElement;
const clone = target.cloneNode(true) as HTMLDivElement;
- clone.querySelectorAll("[data-attachment-url]").forEach((el) => el.remove());
+ const activeContextReferenceIds = new Set(
+ Array.from(
+ target.querySelectorAll(
+ '[data-storyboard-activation="locate"]',
+ ),
+ )
+ .map((element) => element.getAttribute('data-composer-tag-id') || '')
+ .filter(Boolean),
+ );
+ setStoryboardContextReferences((current) =>
+ current.filter((reference) =>
+ activeContextReferenceIds.has(reference.requestId),
+ ),
+ );
+ clone
+ .querySelectorAll(
+ '[data-attachment-url], [data-storyboard-reference="true"]',
+ )
+ .forEach((el) => el.remove());
const textContent = (clone.textContent || "")
.replace(/\u200b/g, " ")
.replace(/\s+/g, " ")
@@ -15877,15 +16391,34 @@ export function CanvasChatPanel({
const selection = window.getSelection();
if (selection && selection.rangeCount > 0) {
const range = selection.getRangeAt(0);
- if (range.collapsed && range.startOffset === 0) {
- // 检查前一个兄弟节点是否是图片标签
- const prevSibling = range.startContainer.previousSibling as HTMLElement;
- if (prevSibling && prevSibling.getAttribute?.('data-attachment-url')) {
+ if (range.collapsed) {
+ let previousNode: ChildNode | null = null;
+ if (range.startContainer.nodeType === Node.TEXT_NODE) {
+ const textNode = range.startContainer as Text;
+ const textBeforeCursor = textNode.data.slice(0, range.startOffset);
+ if (/^[\s\u200b]*$/.test(textBeforeCursor)) {
+ previousNode = textNode.previousSibling;
+ }
+ } else if (range.startContainer instanceof HTMLElement) {
+ previousNode =
+ range.startContainer.childNodes[range.startOffset - 1] || null;
+ }
+ while (
+ previousNode?.nodeType === Node.TEXT_NODE &&
+ /^[\s\u200b]*$/.test(previousNode.textContent || '')
+ ) {
+ previousNode = previousNode.previousSibling;
+ }
+ const previousTag = previousNode as HTMLElement | null;
+ if (previousTag?.getAttribute?.('data-attachment-url')) {
e.preventDefault();
e.stopPropagation(); // 阻止事件冒泡,防止触发全局删除
- // 使用 URL 来删除附件,更可靠
- removeAttachmentByTagElement(prevSibling);
- prevSibling.remove();
+ removeAttachmentByTagElement(previousTag);
+ previousTag.remove();
+ syncAttachmentsWithEditor(
+ textareaRef.current as HTMLDivElement | null,
+ );
+ setTagPreview(null);
}
}
}
@@ -16045,7 +16578,7 @@ export function CanvasChatPanel({
type="file"
accept={
isClawdMode
- ? `image/*,${CLAWD_TABLE_ACCEPT}`
+ ? `image/*,${OPENCLAW_CHAT_VIDEO_ACCEPT},${CLAWD_TABLE_ACCEPT}`
: supportsAtlasVideoReference
? "image/*,video/mp4,video/quicktime,video/webm"
: "image/*"
@@ -16527,7 +17060,9 @@ export function CanvasChatPanel({
return createPortal(
- {/* 图片全貌 - 使用 inline-block + 居中,让标注点相对于图片本身定位 */}
+ {/* 附件预览 */}
-

{
- if (event.currentTarget.src !== tagPreview.url) {
- event.currentTarget.src = tagPreview.url;
- }
- }}
- />
+ {isOpenClawChatVideoAttachment(
+ tagPreview.url,
+ tagPreview.contentType
+ ) ? (
+
+ ) : (
+

{
+ if (event.currentTarget.src !== tagPreview.url) {
+ event.currentTarget.src = tagPreview.url;
+ }
+ }}
+ />
+ )}
{/* 如果是物品标签,显示标注位置 */}
{tagPreview.pickedObject && (
<>
diff --git a/components/canvas/LiblibVideoCanvasReplica.tsx b/components/canvas/LiblibVideoCanvasReplica.tsx
index 0a226d80d..00b3815e5 100644
--- a/components/canvas/LiblibVideoCanvasReplica.tsx
+++ b/components/canvas/LiblibVideoCanvasReplica.tsx
@@ -1,6 +1,8 @@
"use client";
import {
+ AudioLines,
+ ArrowLeft,
ArrowUp,
Bell,
BookOpen,
@@ -10,13 +12,18 @@ import {
ChevronDown,
ChevronRight,
Clock3,
+ CircleDot,
+ Command,
Copy,
+ Delete as DeleteIcon,
Download,
Ellipsis,
FileText,
FolderOpen,
GitBranch,
+ Gem,
Headphones,
+ Hand,
HelpCircle,
Image as ImageIcon,
Keyboard,
@@ -24,31 +31,42 @@ import {
Layers,
Loader2,
Maximize2,
+ MessageCircle,
Minimize2,
+ Minus,
Music2,
Move,
+ Mouse,
+ PanelLeftClose,
Pause,
+ Pencil,
Plus,
Play,
Redo2,
+ Repeat2,
RotateCcw,
Search,
Scissors,
Settings2,
SlidersHorizontal,
Sparkles,
+ Spline,
Star,
Trash2,
Upload,
UserRound,
Video,
+ Volume2,
+ VolumeX,
Wand2,
+ Waves,
Undo2,
Zap,
X,
type LucideIcon,
} from "lucide-react";
import { useRouter } from "next/navigation";
+import dynamic from "next/dynamic";
import { createPortal } from "react-dom";
import {
type ChangeEvent as ReactChangeEvent,
@@ -57,6 +75,7 @@ import {
type MouseEvent as ReactMouseEvent,
type PointerEvent as ReactPointerEvent,
type ReactNode,
+ type SetStateAction,
type WheelEvent as ReactWheelEvent,
useCallback,
useEffect,
@@ -65,8 +84,17 @@ import {
useState,
} from "react";
import { toast } from "sonner";
+import { Streamdown } from "streamdown";
import useSWR from "swr";
+import { Button } from "@/components/ui/button";
import {
+ Tooltip,
+ TooltipContent,
+ TooltipProvider,
+ TooltipTrigger,
+} from "@/components/ui/tooltip";
+import {
+ applyImageModelCreditDiscount,
getGenerationCreditCostFromImageGenConfig,
getImageCreditCostFromConfig,
} from "@/lib/ai/image-model-pricing";
@@ -81,14 +109,149 @@ import {
type ImageToolCreditFeature,
type PointsSystemConfig,
} from "@/lib/points/config";
-import { cn, fetcher } from "@/lib/utils";
import {
+ removeVideoReferenceEdge,
+ resolveVideoReferenceSelectionRule,
+ resolveVideoReferenceToggleAction,
+ selectVideoReferenceImages,
+ toggleVideoReferenceEdge,
+} from "@/lib/canvas/video-reference-selection";
+import {
+ CANVAS_GRID_SNAP_STORAGE_KEY,
+ resolveCanvasGridSnapPosition,
+} from "@/lib/canvas/grid-snap";
+import {
+ CANVAS_PROJECT_TITLE_MAX_LENGTH,
+ normalizeCanvasProjectTitle,
+} from "@/lib/canvas/project-title";
+import {
+ DEFAULT_VIDEO_CANVAS_NAME,
+ normalizeVideoCanvasName,
+} from "@/lib/canvas/video-workspace";
+import {
+ applyCanvasArrangeUpdates,
+ arrangeCanvasNodes,
+ getArrangeCanvasViewport,
+} from "@/lib/canvas/arrange";
+import {
+ clampWorkspacePanelWidth,
+ getInitialWorkspacePanelWidth,
+ WORKSPACE_PANEL_MAX_WIDTH,
+ WORKSPACE_PANEL_MIN_WIDTH,
+ WORKSPACE_PANEL_WIDTH_STORAGE_KEY,
+} from "@/lib/canvas/asset-manager";
+import {
+ CANVAS_MAX_ZOOM,
+ CANVAS_MIN_ZOOM,
+ type CanvasZoomDirection,
+ clampCanvasZoom,
+ sanitizeCanvasZoomInput,
+ stepCanvasZoom,
+} from "@/lib/canvas/zoom";
+import {
+ type CanvasShortcutHelpToken,
+ VIDEO_CANVAS_SHORTCUT_HELP_GROUPS,
+} from "@/lib/canvas/shortcut-help";
+import {
+ attachCanvasNodesToGroup,
+ collectCanvasGroupDescendantIds,
+ detachCanvasGroups,
+ duplicateCanvasNodeSelection,
+ duplicateCanvasNodeTree,
+ expandCanvasGroupSelection,
+ getCanvasGroupBounds,
+ getCanvasGroupChildIds,
+ isCanvasGroupNode,
+ mergeCanvasStoryboardGroups,
+} from "@/lib/canvas/grouping";
+import {
+ isCanvasKeyboardPanKey,
+ shouldStartCanvasPointerPan,
+} from "@/lib/canvas/pan";
+import { cn, fetcher } from "@/lib/utils";
+import { VideoCanvasStoryboardCompositionEditor } from "@/components/canvas/VideoCanvasStoryboardCompositionEditor";
+import {
+ audioResourceService,
canvasService,
imageGenerationService,
videoGenerationService,
+ videoResourceService,
workflowAnalysisService,
} from "@/services/canvas";
-import type { CanvasInitialProject } from "@/types/canvas";
+import {
+ AUDIO_PLAYBACK_RATE_STEP,
+ MAX_AUDIO_PLAYBACK_RATE,
+ MIN_AUDIO_PLAYBACK_RATE,
+ formatAudioPlaybackRate,
+ getAudioDownloadExtension,
+ getDefaultAudioTrimRange,
+ getProcessedAudioTitle,
+ normalizeAudioPlaybackRate,
+ normalizeAudioTrimRange,
+ stepAudioPlaybackRate,
+ type AudioToolRequest,
+} from "@/lib/video-canvas/audio-tools";
+import { processCanvasAudio } from "@/lib/video-canvas/audio-processing";
+import {
+ DEFAULT_VIDEO_CLIP_RANGE,
+ getVideoClipDuration,
+ normalizeVideoClipRange,
+ snapVideoClipRatio,
+ type VideoClipRange,
+} from "@/lib/video-canvas/video-clip";
+import {
+ extractCanvasVideoFrame,
+ loadCanvasVideoClipFrames,
+ inspectCanvasVideoAudioTrack,
+ processCanvasVideoClip,
+ processCanvasVideoAudioSplit,
+ processCanvasVideoCrop,
+ processCanvasVideoSubtitleErase,
+ processCanvasVideoUpscale,
+} from "@/lib/video-canvas/video-processing";
+import {
+ getVideoFrameExtractionCopy,
+ type VideoFrameExtractionKind,
+} from "@/lib/video-canvas/video-frame-extraction";
+import {
+ getVideoDownloadFilename,
+ resolveVideoDownloadRuleCopy,
+ type VideoCanvasVideoDownloadConfig,
+ VIDEO_WATERMARK_RULES_STORAGE_KEY,
+} from "@/lib/video-canvas/video-download";
+import {
+ getVideoAudioSplitResultCopy,
+ type VideoAudioSplitMode,
+} from "@/lib/video-canvas/video-audio-separation";
+import {
+ DEFAULT_VIDEO_CROP_RECT,
+ getVideoCropOutputDimensions,
+ moveVideoCropRect,
+ normalizeVideoCropRect,
+ resizeVideoCropRect,
+ type VideoCropHandle,
+ type VideoCropRect,
+} from "@/lib/video-canvas/video-crop";
+import {
+ formatVideoHdFrameRate,
+ getVideoHdCost,
+ type VideoHdInterpolationMode,
+ type VideoHdResolution,
+} from "@/lib/video-canvas/video-hd";
+import {
+ moveSubtitleEraseRegion,
+ normalizeSubtitleEraseRegion,
+ resizeSubtitleEraseRegion,
+ type SubtitleEraseMode,
+ type SubtitleEraseRegion,
+ type SubtitleEraseRegionHandle,
+} from "@/lib/video-canvas/subtitle-erase";
+import {
+ VIDEO_STORY_NODE_HEIGHT,
+ VIDEO_STORY_NODE_WIDTH,
+ getVideoStoryPlacement,
+} from "@/lib/video-canvas/video-analysis";
+import type { CanvasInitialProject, CanvasState } from "@/types/canvas";
import {
getVideoCanvasFunctionBillingCopy,
type VideoCanvasFunctionConfig,
@@ -96,12 +259,159 @@ import {
type VideoCanvasEffectPreset,
type VideoCanvasMotionPreset,
type VideoCanvasRolePreset,
+ type VideoCanvasSmartStoryboardConfig,
+ type VideoCanvasSubjectLibraryConfig,
+ type VideoCanvasSubjectVoicePreset,
defaultVideoCanvasFunctionConfig,
defaultVideoCanvasMotionPresets,
normalizeVideoCanvasFunctionConfig,
} from "@/lib/video-canvas/config";
+import {
+ buildVideoCanvasImageCreationParams,
+ buildVideoCanvasVideoCreationParams,
+ defaultVideoCanvasNodeCreationConfig,
+ getInitialVideoCanvasNodeCreationOrdinal,
+ type VideoCanvasNodeCreationConfig,
+} from "@/lib/video-canvas/node-creation";
+import {
+ VIDEO_CANVAS_HISTORY_PICKER_MEDIA_LABELS,
+ VIDEO_CANVAS_HISTORY_ZOOM_LEVELS,
+ allocateVideoCanvasHistoryMediaNodeTitle,
+ getInitialVideoCanvasHistoryMediaOrdinals,
+ getVideoCanvasHistoryMediaNodeDimensions,
+ getVideoCanvasHistoryMediaNodePosition,
+ getVideoCanvasHistoryPickerMediaTypes,
+ getVideoCanvasHistoryPickerTitle,
+ getVideoCanvasHistoryZoomIndex,
+ groupVideoCanvasHistoryItems,
+ paginateVideoCanvasHistoryItems,
+ type VideoCanvasHistoryMediaOrdinals,
+ type VideoCanvasHistoryPickerMediaMode,
+ type VideoCanvasHistoryMediaType,
+} from "@/lib/video-canvas/history-assets";
+import {
+ VIDEO_CANVAS_RESOURCE_UPLOAD_PURPOSE,
+ getVideoCanvasUploadedResourceUrl,
+ validateVideoCanvasUploadedResource,
+} from "@/lib/video-canvas/uploaded-resource";
+import {
+ getVideoCanvasTutorialSupportQr,
+ type VideoCanvasTutorialSupportConfig,
+ type VideoCanvasTutorialSupportQrKind,
+} from "@/lib/video-canvas/tutorial-support";
+import {
+ mergeVideoCanvasAdvancedSettings,
+ resolveVideoCanvasAdvancedSettings,
+} from "@/lib/video-canvas/video-advanced-settings";
+import {
+ defaultVideoCanvasGenerationParametersConfig,
+ estimateVideoCanvasGenerationParameterPoints,
+ getVideoCanvasGenerationCount,
+ mergeVideoCanvasGenerationParameterSettings,
+ resolveVideoCanvasGenerationParameterSettings,
+ type VideoCanvasGenerationParameterSettings,
+ type VideoCanvasGenerationParametersConfig,
+} from "@/lib/video-canvas/video-generation-parameters";
+import {
+ estimateVideoCanvasModelCatalogPoints,
+ getVideoCanvasModelCatalogEntry,
+ resolveVideoCanvasCatalogModelRef,
+ type VideoCanvasModelCatalogEntry,
+ type VideoCanvasModelIconKey,
+} from "@/lib/video-canvas/video-model-catalog";
+import {
+ getVideoCanvasToolboxGroupOrigin,
+ type VideoCanvasToolboxConfig,
+ type VideoCanvasToolboxTemplate,
+} from "@/lib/video-canvas/toolbox-workflows";
+import {
+ getVideoCanvasAttachedEffect,
+ setVideoCanvasAttachedEffect,
+ type VideoCanvasAttachedEffect,
+} from "@/lib/video-canvas/video-effects";
+import {
+ composeVideoCanvasRolePrompt,
+ getVideoCanvasAttachedRole,
+ setVideoCanvasAttachedRole,
+ type VideoCanvasAttachedRole,
+ type VideoCanvasRoleMaterialKind,
+ VIDEO_CANVAS_ROLE_MATERIALS,
+} from "@/lib/video-canvas/video-roles";
+import {
+ addVideoCanvasAttachedSubject,
+ getVideoCanvasAttachedSubjects,
+ removeVideoCanvasAttachedSubject,
+ supportsVideoCanvasSubjectUsage,
+ toVideoCanvasAttachedSubject,
+ type VideoCanvasAttachedSubject,
+} from "@/lib/video-canvas/subjects";
+import {
+ getVideoCanvasSmartStoryboard,
+ normalizeVideoCanvasSmartStoryboard,
+ setVideoCanvasSmartStoryboard,
+ supportsVideoCanvasSmartStoryboard,
+ validateVideoCanvasSmartStoryboard,
+ type VideoCanvasSmartStoryboard,
+ type VideoCanvasSmartStoryboardShot,
+} from "@/lib/video-canvas/smart-storyboard";
+import {
+ normalizeVideoPromptMentions,
+ type VideoPromptMention,
+} from "@/lib/video-canvas/prompt-structure";
+import {
+ composeVideoCanvasMotionPrompt,
+ normalizeVideoCanvasMotionMentions,
+ type VideoCanvasMotionMention,
+} from "@/lib/video-canvas/video-motion";
+import { composeVideoMarkGenerationPrompt } from "@/lib/video-canvas/video-marks";
+import {
+ filterVideoCanvasKeyElements,
+ filterVideoCanvasStoryboardVideos,
+ getVideoCanvasStoryboardColumnLayout,
+ getVideoCanvasKeyElementCategoryLabel,
+ projectVideoCanvasStoryboard,
+ projectVideoCanvasStoryboardCompositions,
+ resolveVideoCanvasViewMode,
+ type VideoCanvasKeyElementCategory,
+ type VideoCanvasKeyElementFilter,
+ type VideoCanvasStoryboardExpandedPanel,
+ type VideoCanvasStoryboardItem,
+ type VideoCanvasStoryboardVideoFilter,
+ type VideoCanvasViewMode,
+ VIDEO_CANVAS_KEY_ELEMENT_CATEGORY_OPTIONS,
+ VIDEO_CANVAS_STORYBOARD_VIDEO_FILTER_OPTIONS,
+ VIDEO_CANVAS_VIEW_MODE_KEY,
+} from "@/lib/video-canvas/storyboard-overview";
+import {
+ normalizeVideoCanvasStoryboardComposition,
+ type VideoCanvasStoryboardCompositionProject,
+} from "@/lib/video-canvas/storyboard-composition";
+import { renderVideoCanvasStoryboardComposition } from "@/lib/video-canvas/storyboard-composition-render";
+import type { VideoCanvasStoryboardReferenceRequest } from "@/lib/video-canvas/storyboard-reference";
+import {
+ collectVideoCanvasDirectorCaptures,
+ isPortableDirectorCaptureUrl,
+ normalizeVideoCanvasDirectorProject,
+ type VideoCanvasDirectorProject,
+} from "@/lib/video-canvas/director-project";
import type { VideoGenerationMode, VideoRatio, VideoResolution } from "@/types/ai";
+const VideoCanvasAgentPanel = dynamic(
+ () =>
+ import("@/components/canvas/CanvasChatPanel").then(
+ (module) => module.CanvasChatPanel,
+ ),
+ {
+ loading: () => (
+
+
+ 正在加载对话面板...
+
+ ),
+ ssr: false,
+ },
+);
+
type ActivePanel =
| "add"
| "workflow"
@@ -113,6 +423,7 @@ type ActivePanel =
| "help"
| "roleLibrary"
| "videoRoleLibrary"
+ | "videoSubjectLibrary"
| "shortcuts"
| "styleLibrary"
| "support"
@@ -159,17 +470,7 @@ type ComposerMenu =
type MemberStoreTab = "vip" | "history" | "credits";
type VideoToolbarMode = "clip" | "crop" | "subtitleErase" | "analysis" | null;
type VideoToolbarMenu = "hd" | "subtitle" | "audio" | null;
-type VideoFrameExtractionKind = "first" | "last" | "current";
-type SubtitleEraseMode = "smart" | "region";
-type VideoAudioSplitMode = "voice" | "background" | "audio-video";
-
-type SubtitleEraseRegion = {
- height: number;
- id: string;
- width: number;
- x: number;
- y: number;
-};
+type ProcessAudioNode = (node: CloneNode, request: AudioToolRequest) => Promise
;
type SubtitleEraseRegionHistory = {
future: SubtitleEraseRegion[][];
@@ -214,17 +515,6 @@ type WorkflowPreset = {
cover?: string;
};
-type ToolboxTemplatePreset = WorkflowPreset & {
- accent: string;
- author: string;
- badge?: string;
- category: string;
- duration: string;
- steps: string[];
- templateKey: string;
- usageCount: string;
-};
-
type RoleLibraryItem = VideoCanvasRolePreset;
type RoleLibraryFilterKey = keyof RoleLibraryItem["filters"];
@@ -315,11 +605,9 @@ type EndpointConnectionDragState = {
type DragState =
| {
kind: "node";
- id: string;
+ nodes: Array<{ id: string; x: number; y: number }>;
startX: number;
startY: number;
- nodeX: number;
- nodeY: number;
}
| {
kind: "canvas";
@@ -336,6 +624,13 @@ type CanvasCommandState = {
screenY: number;
} | null;
+type CanvasShortcutSnapshot = {
+ edges: CloneEdge[];
+ nodes: CloneNode[];
+ selectedNodeIds: string[];
+ viewport: typeof DEFAULT_VIEWPORT;
+};
+
type StoryWorkflowStep = "shots" | "assets" | "prompts";
type NodeContextState = {
@@ -384,6 +679,7 @@ type VideoComposerTabKey =
type VideoTargetComposerSubmitPayload = {
magicMarks: VideoCanvasMagicMark[];
mode: VideoGenerationMode;
+ motionMentions: VideoCanvasMotionMention[];
prompt: string;
settings: ComposerSettings & {
duration: string;
@@ -515,6 +811,23 @@ type MembershipSummaryResponse = {
pointsSystemConfig?: PointsSystemConfig | null;
};
+type VideoWorkspaceCanvasItem = {
+ createdAt: string;
+ id: string;
+ isCurrent: boolean;
+ name: string;
+};
+
+type VideoWorkspaceCanvasResponse = {
+ data?: {
+ canvases: VideoWorkspaceCanvasItem[];
+ currentCanvasId: string;
+ workspaceTitle: string;
+ };
+ error?: string;
+ success: boolean;
+};
+
type MembershipPlan = {
id: string;
name: string;
@@ -624,11 +937,6 @@ type AngleEditorSettings = {
zoom: number;
};
-type VideoClipRange = {
- start: number;
- end: number;
-};
-
type VideoAnalysisRow = {
id: string;
shot: string;
@@ -715,6 +1023,8 @@ type VideoAudioSplitState = {
mode: VideoAudioSplitMode;
};
+type VideoAudioTrackAvailability = "checking" | "available" | "unavailable";
+
const capturedTools: CapturedTool[] = [];
const capturedCounts: { byType?: Record } = {};
const capturedWorkflows: WorkflowPreset[] = [];
@@ -813,158 +1123,6 @@ const STARTER_CARDS = [
},
];
-const TOOLBOX_TEMPLATE_CATEGORIES = ["周星驰经典名场面", "电商展示", "镜头运动", "人物动画", "全部"] as const;
-
-const TOOLBOX_TEMPLATE_PRESETS: ToolboxTemplatePreset[] = [
- {
- accent: "linear-gradient(135deg,#f5d789,#f08b62)",
- author: "LibTV 官方",
- badge: "预设",
- category: "周星驰经典名场面",
- cover:
- "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-first-frame-video-cover.png",
- description: "左弧滑行,镜头从人物左侧弧线推进,带轻微喜剧节奏和夸张表情停顿。",
- duration: "5s",
- id: "left-arc-slide",
- name: "【预设】左弧滑行",
- steps: ["参考图", "镜头轨迹", "Seedance 2.0", "视频"],
- templateKey: "left-arc-slide",
- usageCount: "4.2k",
- },
- {
- accent: "linear-gradient(135deg,#98d5ff,#6c84ff)",
- author: "电商模板库",
- badge: "商用",
- category: "电商展示",
- cover:
- "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-character-three-view-cover.png",
- description: "电商手机弹出效果,产品从画面中心弹出并旋转定格,适合主图视频开场。",
- duration: "5s",
- id: "phone-pop-product",
- name: "电商手机弹出效果",
- steps: ["产品图", "弹出动画", "高光", "视频"],
- templateKey: "phone-pop-product",
- usageCount: "3.6k",
- },
- {
- accent: "linear-gradient(135deg,#dfc192,#8e623c)",
- author: "商品展示",
- badge: "商用",
- category: "电商展示",
- cover:
- "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-story-script-cover.png",
- description: "咖啡杯出场,杯身从桌面滑入,蒸汽和柔光形成温暖的品牌氛围。",
- duration: "5s",
- id: "coffee-cup-entry",
- name: "咖啡杯出场",
- steps: ["商品图", "场景光", "蒸汽", "视频"],
- templateKey: "coffee-cup-entry",
- usageCount: "2.8k",
- },
- {
- accent: "linear-gradient(135deg,#d3e5ff,#9fb3cc)",
- author: "运镜模板",
- badge: "热门",
- category: "镜头运动",
- cover:
- "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-audio-to-video-cover.png",
- description: "360旋转展示,围绕主体做一圈稳定环绕,适合产品、人物和空间展示。",
- duration: "6s",
- id: "orbit-360-showcase",
- name: "360旋转展示",
- steps: ["主体图", "环绕镜头", "景深", "视频"],
- templateKey: "orbit-360-showcase",
- usageCount: "5.1k",
- },
- {
- accent: "linear-gradient(135deg,#d7d8dc,#737d89)",
- author: "工业镜头",
- category: "镜头运动",
- cover:
- "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/character-image.png",
- description: "机械臂视角,以第一视角穿过机械臂与产品线,增强科技感和空间纵深。",
- duration: "5s",
- id: "robot-arm-pov",
- name: "机械臂视角",
- steps: ["参考图", "机械臂", "穿行", "视频"],
- templateKey: "robot-arm-pov",
- usageCount: "1.9k",
- },
- {
- accent: "linear-gradient(135deg,#f9c8d8,#c693ff)",
- author: "人物动画",
- badge: "NEW",
- category: "人物动画",
- cover:
- "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/character-three-view.png",
- description: "Live 2D,让单张角色图产生眨眼、微笑和轻微身体摆动,适合口播开场。",
- duration: "4s",
- id: "live-2d-character",
- name: "Live 2D",
- steps: ["角色图", "表情", "轻动作", "视频"],
- templateKey: "live-2d-character",
- usageCount: "6.4k",
- },
- {
- accent: "linear-gradient(135deg,#bad5ff,#2e375d)",
- author: "电影运镜",
- category: "镜头运动",
- cover:
- "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-first-frame-video-cover.png",
- description: "瞳孔拉近,镜头快速推向眼部并在反光里展示下一场景,适合悬疑转场。",
- duration: "5s",
- id: "pupil-zoom",
- name: "瞳孔拉近",
- steps: ["人物图", "特写", "反光", "转场"],
- templateKey: "pupil-zoom",
- usageCount: "2.2k",
- },
- {
- accent: "linear-gradient(135deg,#f7e6b0,#ef6b62)",
- author: "创意玩法",
- badge: "热门",
- category: "人物动画",
- cover:
- "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-story-script-cover.png",
- description: "飞鸟解体,主体由细碎羽片散开再重组,适合视觉冲击强的短视频片头。",
- duration: "6s",
- id: "bird-dissolve",
- name: "飞鸟解体",
- steps: ["主体图", "粒子", "重组", "视频"],
- templateKey: "bird-dissolve",
- usageCount: "3.4k",
- },
- {
- accent: "linear-gradient(135deg,#f3db93,#ef5e50)",
- author: "电商模板库",
- badge: "商用",
- category: "电商展示",
- cover:
- "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-character-three-view-cover.png",
- description: "商品震撼登场,产品带冲击波与局部高光进入画面,适合活动主推款。",
- duration: "5s",
- id: "product-hero-entry",
- name: "商品震撼登场",
- steps: ["产品图", "冲击波", "高光", "视频"],
- templateKey: "product-hero-entry",
- usageCount: "4.8k",
- },
- {
- accent: "linear-gradient(135deg,#a6e7d8,#6da3ff)",
- author: "旅拍模板",
- category: "镜头运动",
- cover:
- "https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/starter-audio-to-video-cover.png",
- description: "旅拍转场,前景遮挡切换到新地点,带自然手持感和轻微速度变化。",
- duration: "5s",
- id: "travel-whip-transition",
- name: "旅拍转场",
- steps: ["首帧", "遮挡", "转场", "视频"],
- templateKey: "travel-whip-transition",
- usageCount: "2.6k",
- },
-];
-
const ROLE_LIBRARY_FILTER_GROUPS: Array<{
key: RoleLibraryFilterKey;
label: string;
@@ -997,177 +1155,13 @@ type VideoCanvasCustomMotionPreset = VideoCanvasMotionPreset & {
custom: true;
};
-const LIBLIB_VIDEO_MODEL_OPTIONS = [
- {
- label: "Seedance 2.0 VIP",
- duration: "2min",
- description: "最强视频模型,会员专属通道,15s音画同步",
- },
- {
- label: "Seedance 2.0 Fast VIP",
- duration: "2min",
- description: "最强视频模型快速版,会员专属通道,15s音画同步",
- },
- {
- label: "Seedance 2.0 Mini",
- duration: "2min",
- description: "最强视频模型mini版,高性价比生成,15s音画同步",
- },
- {
- label: "Happy Horse 1.1",
- duration: "3min",
- description: "阿里最新视频模型,一致性与视听质量更可控",
- },
- {
- label: "Happy Horse 1.0",
- duration: "3min",
- description: "阿里视频模型,支持多参生成",
- },
- {
- label: "Kling O3",
- duration: "3min",
- description: "视频编辑模型、参考一致性、音画同出、多镜头",
- },
- {
- label: "Kling 3.0 Turbo",
- duration: "3min",
- description: "视频生成模型,高质感、支持多镜头",
- },
- {
- label: "Kling 3.0",
- duration: "3min",
- description: "视频生成模型,高质感、支持多镜头",
- },
- {
- label: "Wan 2.7",
- duration: "3min",
- description: "全能参考,支持修改视频画面、剧情、环境",
- },
- {
- label: "Kling O1",
- duration: "3min",
- description: "可灵一代编辑模型、支持多模态输入",
- },
- {
- label: "Wan 2.6",
- duration: "3min",
- description: "音画同步,支持多机位镜头,最长可生15秒视频",
- },
- {
- label: "Hailuo 2.3",
- duration: "2min",
- description: "善于表达动作、表情、镜头,更高质感",
- },
- {
- label: "Seedance1.5 Pro",
- duration: "2min",
- description: "音画同步,支持多机位镜头,最长可生12秒视频",
- },
- {
- label: "Seedance 1.0 Pro",
- duration: "2min",
- description: "高精度提示词理解,40秒生成1080P视频",
- },
- {
- label: "Seedance 1.0 Lite",
- duration: "1min",
- description: "轻量快速,一键进行日常视频生成",
- },
- {
- label: "Kling 2.6",
- duration: "2min",
- description: "视频生成模型、直出音画同步",
- },
- {
- label: "Hailuo 02",
- duration: "2min",
- description: "画质稳定,适合打造运动特效场景",
- },
- {
- label: "Vidu Q2",
- duration: "3min",
- description: "多图主体参考,精确控制效果佳",
- },
- {
- label: "Vidu Q2 Pro",
- description: "Vidu Q2 Pro",
- },
- {
- label: "Vidu Q2 Turbo",
- description: "Vidu Q2 Turbo",
- },
- {
- label: "Vidu Q3 Pro",
- duration: "2min",
- description: "支持主体参考,精确控制效果佳",
- },
- {
- label: "OmniHuman 1.5",
- duration: "3min",
- description: "多模态数字人视频生成",
- },
- {
- label: "Kling 2.5",
- duration: "2min",
- description: "速度快、效果稳定、性价比高",
- },
- {
- label: "Kling 2.1",
- duration: "3min",
- description: "支持首尾帧,图生视频效果更出色",
- },
- {
- label: "Wan 2.2",
- duration: "3min",
- description: "支持特效、玩法千变万化",
- },
- {
- label: "Wan 2.5",
- duration: "3min",
- description: "支持特效、直出音画同步",
- },
- {
- label: "Pixverse V5.5",
- duration: "3min",
- description: "支持特效、玩法丰富",
- },
- {
- label: "Pixverse V5",
- duration: "3min",
- description: "支持特效、玩法丰富",
- },
- {
- label: "Hailuo 2.3 Fast",
- duration: "1min",
- description: "善于表达动作、表情、镜头,更快速",
- disabled: true,
- },
- {
- label: "Kling3.0 动作迁移",
- duration: "8min",
- description: "动作控制模型,需输入1张图片、1条视频",
- disabled: true,
- },
- {
- label: "MJ Video",
- duration: "2min",
- description: "图生视频效果稳定,画面表现力强",
- disabled: true,
- },
-] as const;
+function isVideoCanvasCustomMotionPreset(
+ preset: VideoCanvasMotionPreset | VideoCanvasCustomMotionPreset,
+): preset is VideoCanvasCustomMotionPreset {
+ return "custom" in preset && preset.custom === true;
+}
-const VIDEO_RATIO_OPTIONS = [
- { label: "Auto", value: "adaptive" },
- { label: "16:9", value: "16:9" },
- { label: "4:3", value: "4:3" },
- { label: "1:1", value: "1:1" },
- { label: "3:4", value: "3:4" },
- { label: "9:16", value: "9:16" },
- { label: "21:9", value: "21:9" },
-] as const;
-const VIDEO_RESOLUTION_OPTIONS = ["480P", "720P", "1080P", "4K"] as const;
-const VIDEO_COUNT_OPTIONS = ["1个", "2个", "4个"] as const;
const VIDEO_AUDIO_OPTIONS = ["开启", "关闭"] as const;
const VIDEO_EFFECT_PRESETS = [
@@ -1206,22 +1200,11 @@ const TOOLBAR_ITEMS: Array<{ id: Exclude; label: string; icon
const DEFAULT_VIEWPORT = { x: 0, y: 0, zoom: 1 };
const PRESET_VIEWPORT = { x: 144, y: 100, zoom: 0.77 };
const PRESET_WORKFLOW_CENTER = { x: 1024, y: 512 };
-const CANVAS_EXTENT = 20000;
-const CANVAS_MIN_ZOOM = 0.1;
-const CANVAS_MAX_ZOOM = 8;
-const CANVAS_ZOOM_STEP = 0.2;
+const CANVAS_EXTENT = 12000;
+const CANVAS_CONTROL_FONT_FAMILY =
+ '-apple-system, system-ui, "PingFang SC", Inter, "Noto Sans SC", "Microsoft YaHei", sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol"';
const ASSET_MANAGER_DEFAULT_WIDTH = 240;
-const ASSET_MANAGER_MIN_WIDTH = 240;
-const ASSET_MANAGER_MAX_WIDTH = 520;
const OFFICIAL_NODE_GAP = 100;
-const TOOLBOX_TEMPLATE_VIEWPORT = { x: 137.4, y: 529, zoom: 0.1 };
-const TOOLBOX_TEMPLATE_GROUP_FRAME = {
- height: 658,
- width: 1493,
- x: 6010,
- y: -1440,
-};
-const VIDEO_SUBTITLE_ERASE_SIMULATED_MS = 42_000;
const OFFICIAL_TEXT_RESOURCE_HEIGHT = 330;
const OFFICIAL_TEXT_RESOURCE_WIDTH = 560;
const OFFICIAL_IMAGE_TOOLBAR_SCALE = 0.7;
@@ -1242,43 +1225,10 @@ const LOCAL_EDIT_COLORS = [
"#8d6e63",
];
const VIDEO_CLIP_FRAME_COUNT = 9;
-const VIDEO_CROP_INLINE_LABEL = "1024 x 576";
-const VIDEO_CROP_OUTPUT_LABEL = "1024 × 576";
-const VIDEO_CROP_RECT = { x: 0.1, y: 0.1, width: 0.8, height: 0.8 };
const STORY_BEAT_GRID_FUNCTION_KEY = "story-beat-grid-4";
const MULTI_VIEW_NINE_GRID_FUNCTION_KEY = "multi-view-nine-grid";
const CONTINUOUS_STORYBOARD_GRID_FUNCTION_KEY = "continuous-storyboard-grid-25";
-function clampCanvasZoom(value: number) {
- return Math.min(CANVAS_MAX_ZOOM, Math.max(CANVAS_MIN_ZOOM, Number(value.toFixed(2))));
-}
-
-function clampAssetManagerWidth(value: number) {
- return Math.min(ASSET_MANAGER_MAX_WIDTH, Math.max(ASSET_MANAGER_MIN_WIDTH, Math.round(value)));
-}
-
-function arrangeNodesForPreview(nodes: CloneNode[]) {
- if (nodes.length <= 1) return nodes;
- const orderedNodes = [...nodes].sort((first, second) => first.y - second.y || first.x - second.x);
- const columns = Math.max(1, Math.ceil(Math.sqrt(nodes.length * 1.2)));
- const cellWidth = Math.max(360, ...nodes.map((node) => node.width)) + OFFICIAL_NODE_GAP;
- const cellHeight = Math.max(260, ...nodes.map((node) => node.height)) + OFFICIAL_NODE_GAP;
- const arrangedById = new Map>();
-
- orderedNodes.forEach((node, index) => {
- const column = index % columns;
- const row = Math.floor(index / columns);
- arrangedById.set(node.id, {
- x: column * cellWidth,
- y: row * cellHeight,
- });
- });
-
- return nodes.map((node) => {
- const arranged = arrangedById.get(node.id);
- return arranged ? { ...node, ...arranged } : node;
- });
-}
const CINEMATIC_LIGHTING_CORRECTION_FUNCTION_KEY = "cinematic-lighting-correction";
const CHARACTER_THREE_VIEW_FUNCTION_KEY = "character-three-view";
const FRAME_PREDICTION_PLUS_3S_FUNCTION_KEY = "frame-prediction-plus-3s";
@@ -1314,7 +1264,7 @@ const PUBLISH_MODAL_BG_URL =
"https://demo-sparkai.oss-cn-shanghai.aliyuncs.com/video-canvas-demo/publish-modal-bg.png";
const STORY_SCRIPT_PROMPT = "根据我上传的剧本生成一个完整的故事脚本";
const AUDIO_TO_VIDEO_PROMPT =
- "根据上传的音频生成对应场景画面,镜头语言、节奏、音乐匹配情绪变化,电影级质感。";
+ defaultVideoCanvasFunctionConfig.functions["audio-to-video"]?.prompt || "";
const TEXT_TO_VIDEO_PROMPT =
"爆款情绪短片\n傍晚的城市天台,一个年轻人迎着晚风张开双手,镜头从远景缓慢推进到中景,头发被风吹起,夕阳逆光,光晕明显,空气中有漂浮微尘,电影级色彩分级,浅景深,情绪张力强,慢动作";
const TEXT_TO_VIDEO_GENERATE_PROMPT = "根据文字描述生成视频。";
@@ -1560,6 +1510,60 @@ function makeId(prefix: string) {
return `${prefix}-${crypto.randomUUID()}`;
}
+function formatStoryboardTextGenerationResult(
+ result: {
+ analysisRows?: Array<{
+ backgroundMusic?: string;
+ cameraMovement?: string;
+ dialogue?: string;
+ duration?: string | number;
+ narrative?: string;
+ sceneDescription?: string;
+ shot?: string | number;
+ story?: string;
+ voiceAndSfx?: string;
+ }>;
+ raw?: string;
+ summary?: string;
+ },
+ title: string,
+) {
+ const raw = String(result.raw || "").trim();
+ if (raw && !/^[{[]/.test(raw)) return raw;
+
+ const rows = Array.isArray(result.analysisRows) ? result.analysisRows : [];
+ if (rows.length > 0) {
+ const sections = rows.map((row, index) => {
+ const shot = String(row.shot || index + 1).trim();
+ const scene = String(
+ row.sceneDescription || row.narrative || row.story || "",
+ ).trim();
+ const sound = String(
+ row.dialogue || row.voiceAndSfx || row.backgroundMusic || "",
+ ).trim();
+ const movement = String(row.cameraMovement || "").trim();
+ const duration = String(row.duration || "").trim();
+ return [
+ `## 镜头 ${shot}${duration ? ` · ${duration}` : ""}`,
+ scene,
+ sound ? `**声音:** ${sound}` : "",
+ movement ? `**运镜:** ${movement}` : "",
+ ]
+ .filter(Boolean)
+ .join("\n\n");
+ });
+ return [
+ `# ${title}`,
+ String(result.summary || "").trim(),
+ ...sections,
+ ]
+ .filter(Boolean)
+ .join("\n\n");
+ }
+
+ return String(result.summary || raw).trim();
+}
+
function serializeNodes(nodes: CloneNode[]) {
return nodes.map((node) => ({
id: node.id,
@@ -1614,6 +1618,43 @@ function restoreEdges(canvasData: unknown): CloneEdge[] {
.filter((edge) => edge.source && edge.target);
}
+function restoreSelectedNodeIds(canvasData: unknown, nodes: CloneNode[]) {
+ const selectedNodeIds = (canvasData as { selectedNodeIds?: unknown[] } | null)
+ ?.selectedNodeIds;
+ if (!Array.isArray(selectedNodeIds)) return [];
+ const availableNodeIds = new Set(nodes.map((node) => node.id));
+ return selectedNodeIds
+ .map((nodeId) => (typeof nodeId === "string" ? nodeId : ""))
+ .filter((nodeId) => availableNodeIds.has(nodeId));
+}
+
+function restoreViewport(canvasData: unknown) {
+ const value = (canvasData as { viewport?: unknown } | null)?.viewport;
+ if (!value || typeof value !== "object" || Array.isArray(value)) {
+ return DEFAULT_VIEWPORT;
+ }
+ const record = value as Record;
+ const x = Number(record.x);
+ const y = Number(record.y);
+ const zoom = Number(record.zoom);
+ return {
+ x: Number.isFinite(x) ? x : DEFAULT_VIEWPORT.x,
+ y: Number.isFinite(y) ? y : DEFAULT_VIEWPORT.y,
+ zoom: Number.isFinite(zoom)
+ ? Math.max(CANVAS_MIN_ZOOM, Math.min(CANVAS_MAX_ZOOM, zoom))
+ : DEFAULT_VIEWPORT.zoom,
+ };
+}
+
+function getVideoCanvasPersistenceSignature(
+ nodes: CloneNode[],
+ edges: CloneEdge[],
+ viewport: typeof DEFAULT_VIEWPORT,
+ viewMode: VideoCanvasViewMode,
+) {
+ return JSON.stringify({ edges, nodes, viewMode, viewport });
+}
+
function createInputNode(inputType: CloneNode["inputType"], x: number, y: number): CloneNode {
const title =
inputType === "video"
@@ -1685,16 +1726,26 @@ function createWorkflowNode(workflow: WorkflowPreset, x: number, y: number): Clo
};
}
-function createToolboxTemplateGroupNode(title: string): CloneNode {
+function createToolboxTemplateGroupNode({
+ config,
+ title,
+ x,
+ y,
+}: {
+ config: VideoCanvasToolboxConfig;
+ title: string;
+ x: number;
+ y: number;
+}): CloneNode {
return {
id: makeId("group"),
kind: "workflow",
title: "分组 3 个节点",
subtitle: title,
- x: TOOLBOX_TEMPLATE_GROUP_FRAME.x,
- y: TOOLBOX_TEMPLATE_GROUP_FRAME.y,
- width: TOOLBOX_TEMPLATE_GROUP_FRAME.width,
- height: TOOLBOX_TEMPLATE_GROUP_FRAME.height,
+ x,
+ y,
+ width: config.layout.groupWidth,
+ height: config.layout.groupHeight,
accent: "#ffffff",
draftValues: {
action: "TOOLBOX_TEMPLATE_GROUP",
@@ -1720,7 +1771,16 @@ function createSimpleNode(title: string, kind: CloneNodeKind, x: number, y: numb
};
}
-function createTargetModuleNode(kind: AddModuleKind, x: number, y: number, index: number): CloneNode {
+function createTargetModuleNode(
+ kind: AddModuleKind,
+ x: number,
+ y: number,
+ index: number,
+ options: {
+ nodeCreation?: VideoCanvasNodeCreationConfig;
+ videoModelCatalog?: VideoCanvasModelCatalogEntry[];
+ } = {},
+): CloneNode {
if (kind === "script") {
return {
id: makeId("script"),
@@ -1853,8 +1913,18 @@ function createTargetModuleNode(kind: AddModuleKind, x: number, y: number, index
},
};
const item = meta[kind];
- const isSquareModule = item.nodeType === "text" || item.nodeType === "image" || item.nodeType === "audio";
+ const nodeCreation = options.nodeCreation || defaultVideoCanvasNodeCreationConfig;
+ const isSquareModule = item.nodeType === "text" || item.nodeType === "audio";
+ const isImageModule = item.nodeType === "image";
const isVideoModule = item.nodeType === "video";
+ const params = isImageModule
+ ? buildVideoCanvasImageCreationParams(nodeCreation)
+ : isVideoModule
+ ? buildVideoCanvasVideoCreationParams(
+ nodeCreation,
+ options.videoModelCatalog || [],
+ )
+ : undefined;
return {
id: makeId(kind),
kind: item.kind,
@@ -1863,8 +1933,20 @@ function createTargetModuleNode(kind: AddModuleKind, x: number, y: number, index
inputType: item.nodeType,
x,
y,
- width: isSquareModule ? 350 : isVideoModule ? 622 : 436,
- height: isSquareModule || isVideoModule ? 350 : 470,
+ width: isImageModule
+ ? nodeCreation.image.width
+ : isVideoModule
+ ? nodeCreation.video.width
+ : isSquareModule
+ ? 350
+ : 436,
+ height: isImageModule
+ ? nodeCreation.image.height
+ : isVideoModule
+ ? nodeCreation.video.height
+ : isSquareModule
+ ? 350
+ : 470,
accent: item.accent,
draftValues: {
action: item.action,
@@ -1872,13 +1954,21 @@ function createTargetModuleNode(kind: AddModuleKind, x: number, y: number, index
content: item.nodeType === "text" ? [] : undefined,
generatorType: "DEFAULT",
nodeType: item.nodeType,
+ params,
poster: item.nodeType === "video" ? "" : undefined,
url: item.nodeType === "text" ? undefined : [],
},
};
}
-function createUploadedMediaNode(file: File, mediaUrl: string, x: number, y: number, index: number): CloneNode {
+function createUploadedMediaNode(
+ file: File,
+ mediaUrl: string,
+ x: number,
+ y: number,
+ index: number,
+ assetId?: string | null,
+): CloneNode {
const mediaType: NonNullable = file.type.startsWith("video/")
? "video"
: file.type.startsWith("audio/")
@@ -1902,6 +1992,7 @@ function createUploadedMediaNode(file: File, mediaUrl: string, x: number, y: num
draftValues: {
action: `${mediaType.toUpperCase()}_RESOURCE`,
alt: file.name,
+ assetId: assetId || undefined,
generated: false,
mediaType,
mediaUrl,
@@ -1914,6 +2005,28 @@ function createUploadedMediaNode(file: File, mediaUrl: string, x: number, y: num
};
}
+async function uploadVideoCanvasResourceFile(file: File) {
+ const body = new FormData();
+ body.append("file", file);
+ body.append("purpose", VIDEO_CANVAS_RESOURCE_UPLOAD_PURPOSE);
+ body.append("clientUploadId", crypto.randomUUID());
+ const response = await fetch("/api/files/upload", {
+ body,
+ method: "POST",
+ });
+ const payload = (await response.json().catch(() => null)) as
+ | { assetId?: string | null; error?: string; url?: string }
+ | null;
+ const mediaUrl = getVideoCanvasUploadedResourceUrl(payload);
+ if (!response.ok || !mediaUrl) {
+ throw new Error(payload?.error || "上传失败");
+ }
+ return {
+ assetId: payload?.assetId || null,
+ mediaUrl,
+ };
+}
+
function createMediaResourceNode({
action,
height,
@@ -2079,77 +2192,164 @@ function createMaterialRoleNode({
};
}
-function createHistoryDemoAssetNode({
- id,
- mediaType,
- mediaUrl,
- title,
-}: {
+type VideoCanvasHistoryAsset = {
+ createdAt: string;
+ displayName: string | null;
+ durationMs: number | null;
+ height: number | null;
id: string;
- mediaType: NonNullable;
- mediaUrl: string;
- title: string;
-}): CloneNode {
- const label = mediaType === "video" ? "视频" : mediaType === "audio" ? "音频" : "图片";
- return {
- id,
- kind: "media",
- title,
- subtitle: `${label}资源`,
- inputType: mediaType,
- x: 0,
- y: 0,
- width: mediaType === "audio" ? 430 : mediaType === "video" ? 560 : 420,
- height: mediaType === "audio" ? 170 : mediaType === "video" ? 315 : 280,
- accent: mediaType === "audio" ? "#9ee6bb" : mediaType === "video" ? "#8b96ff" : "#8790ff",
- draftValues: {
- action: `${mediaType.toUpperCase()}_RESOURCE`,
- alt: title,
- generated: true,
- historyDemo: true,
- mediaType,
- mediaUrl,
- nodeType: mediaType,
- url: [mediaUrl],
- },
- };
+ mediaType: VideoCanvasHistoryMediaType;
+ metadata: unknown;
+ mimeType: string | null;
+ modelId: string | null;
+ originalUrl: string;
+ previewUrl: string | null;
+ prompt: string | null;
+ sourceNodeId: string | null;
+ sourceType: string;
+ thumbnailUrl: string | null;
+ width: number | null;
+};
+
+type VideoCanvasHistoryAssetsResponse = {
+ assets?: VideoCanvasHistoryAsset[];
+ error?: string;
+ success?: boolean;
+};
+
+type VideoCanvasHistoryItem = {
+ asset: VideoCanvasHistoryAsset;
+ createdAt: string;
+ node: CloneNode;
+};
+
+function readVideoCanvasHistoryAssetMetadata(asset: VideoCanvasHistoryAsset) {
+ return asset.metadata &&
+ typeof asset.metadata === "object" &&
+ !Array.isArray(asset.metadata)
+ ? (asset.metadata as Record)
+ : {};
}
-const DEFAULT_IMAGE_HISTORY_NODES: CloneNode[] = [
- ["history-demo-image-1", "角色图", CHARACTER_IMAGE_URL],
- ["history-demo-image-2", "角色三视图", CHARACTER_THREE_VIEW_URL],
- ["history-demo-image-3", "首帧", FIRST_FRAME_IMAGE_URL],
- ["history-demo-image-4", "音频参考图", AUDIO_REFERENCE_IMAGE_URL],
- ["history-demo-image-5", "高清图片", HD_IMAGE_SOURCE_URL],
- ["history-demo-image-6", "视频首帧", VIDEO_FIRST_FRAME_SOURCE_URL],
- ["history-demo-image-7", "首尾帧-首帧", VIDEO_FIRST_LAST_FIRST_URL],
- ["history-demo-image-8", "首尾帧-尾帧", VIDEO_FIRST_LAST_LAST_URL],
-].map(([id, title, mediaUrl]) => createHistoryDemoAssetNode({ id, mediaType: "image", mediaUrl, title }));
+function getVideoCanvasHistoryAssetUrl(asset: VideoCanvasHistoryAsset) {
+ return asset.previewUrl || asset.originalUrl;
+}
-const DEFAULT_VIDEO_HISTORY_NODES: CloneNode[] = Array.from({ length: 10 }, (_, index) =>
- createHistoryDemoAssetNode({
- id: `history-demo-video-${index + 1}`,
- mediaType: "video",
- mediaUrl: SCRIPT_VIDEO_REFERENCE_URL,
- title: `分镜视频-#${index + 1}`,
- })
-);
+function getVideoCanvasHistoryAssetThumbnailUrl(
+ asset: VideoCanvasHistoryAsset,
+) {
+ return asset.thumbnailUrl || asset.previewUrl || asset.originalUrl;
+}
-const DEFAULT_AUDIO_HISTORY_NODES: CloneNode[] = Array.from({ length: 6 }, (_, index) =>
- createHistoryDemoAssetNode({
- id: `history-demo-audio-${index + 1}`,
- mediaType: "audio",
- mediaUrl: AUDIO_SOURCE_URL,
- title: `示例音频-#${index + 1}`,
- })
-);
+function createVideoCanvasHistoryAssetNode({
+ asset,
+ canvasNodes,
+ index,
+}: {
+ asset: VideoCanvasHistoryAsset;
+ canvasNodes: CloneNode[];
+ index: number;
+}): CloneNode {
+ const mediaUrl = getVideoCanvasHistoryAssetUrl(asset);
+ const metadata = readVideoCanvasHistoryAssetMetadata(asset);
+ const matchingCanvasNode = canvasNodes.find(
+ (node) =>
+ (asset.sourceNodeId && node.id === asset.sourceNodeId) ||
+ getNodeMediaUrl(node) === asset.originalUrl ||
+ getNodeReferenceMediaUrl(node) === asset.originalUrl,
+ );
+ const label =
+ asset.mediaType === "video"
+ ? "视频"
+ : asset.mediaType === "audio"
+ ? "音频"
+ : "图片";
+ const displayName = String(
+ metadata.nodeTitle || metadata.title || asset.displayName || "",
+ )
+ .replace(/\.[^.]+$/, "")
+ .trim();
+ const title = displayName || `${label}${index + 1}`;
+ const historyDraft = {
+ historyAssetCreatedAt: asset.createdAt,
+ historyAssetHeight: asset.height,
+ historyAssetId: asset.id,
+ historySourceNodeId: asset.sourceNodeId,
+ historyAssetWidth: asset.width,
+ mediaType: asset.mediaType,
+ mediaUrl,
+ model: asset.modelId || undefined,
+ pathname: asset.originalUrl,
+ prompt: asset.prompt || "",
+ thumbnailUrl: getVideoCanvasHistoryAssetThumbnailUrl(asset),
+ url: [mediaUrl],
+ };
-const HISTORY_ASSET_ZOOM_LEVELS = [
- { cardSize: 130, label: "75%" },
- { cardSize: 144, label: "100%" },
- { cardSize: 182, label: "125%" },
- { cardSize: 248, label: "150%" },
-] as const;
+ if (matchingCanvasNode) {
+ return {
+ ...matchingCanvasNode,
+ draftValues: {
+ ...(matchingCanvasNode.draftValues || {}),
+ ...historyDraft,
+ },
+ };
+ }
+
+ const generated = asset.sourceType !== "upload";
+ const aspectRatio =
+ asset.width && asset.height ? asset.width / asset.height : null;
+ const generatedHeight =
+ asset.mediaType === "image" && aspectRatio
+ ? Math.max(280, Math.min(560, Math.round(350 / aspectRatio)))
+ : 350;
+ const width = generated
+ ? asset.mediaType === "video"
+ ? 622
+ : 350
+ : asset.mediaType === "audio"
+ ? 430
+ : asset.mediaType === "video"
+ ? 560
+ : 420;
+ const height = generated
+ ? asset.mediaType === "image"
+ ? generatedHeight
+ : 350
+ : asset.mediaType === "audio"
+ ? 170
+ : asset.mediaType === "video"
+ ? 315
+ : aspectRatio
+ ? Math.max(240, Math.min(560, Math.round(420 / aspectRatio)))
+ : 280;
+
+ return {
+ accent:
+ asset.mediaType === "audio"
+ ? "#9ee6bb"
+ : asset.mediaType === "video"
+ ? "#8b96ff"
+ : "#8790ff",
+ draftValues: {
+ action: `${asset.mediaType.toUpperCase()}_${generated ? "GENERATE" : "RESOURCE"}`,
+ alt: title,
+ generated,
+ generatorType: generated ? "DEFAULT" : undefined,
+ nodeType: asset.mediaType,
+ params: asset.prompt ? { prompt: asset.prompt } : undefined,
+ ...historyDraft,
+ },
+ height,
+ id: `history-asset-${asset.id}`,
+ inputType: asset.mediaType,
+ kind: generated ? "model" : "media",
+ subtitle: `${label}${generated ? "生成" : "资源"}`,
+ title,
+ width,
+ x: 0,
+ y: 0,
+ };
+}
function createNineGridCompositeNode({
anchor,
@@ -2299,6 +2499,66 @@ function preventNativeImageDrag(event: ReactDragEvent) {
event.preventDefault();
}
+function RetriableCanvasImage({
+ alt,
+ className,
+ rotation = 0,
+ src,
+}: {
+ alt: string;
+ className: string;
+ rotation?: number;
+ src: string;
+}) {
+ const [retryAttempt, setRetryAttempt] = useState(0);
+ const retryTimerRef = useRef(null);
+
+ useEffect(() => {
+ setRetryAttempt(0);
+ return () => {
+ if (retryTimerRef.current !== null) {
+ window.clearTimeout(retryTimerRef.current);
+ }
+ };
+ }, [src]);
+
+ const retrySrc = useMemo(() => {
+ if (retryAttempt === 0 || !/^https?:\/\//i.test(src)) return src;
+ const separator = src.includes("?") ? "&" : "?";
+ return `${src}${separator}canvas_image_retry=${retryAttempt}`;
+ }, [retryAttempt, src]);
+
+ const clearRetryTimer = useCallback(() => {
+ if (retryTimerRef.current === null) return;
+ window.clearTimeout(retryTimerRef.current);
+ retryTimerRef.current = null;
+ }, []);
+
+ const handleLoadError = useCallback(() => {
+ if (retryAttempt >= 4 || retryTimerRef.current !== null) return;
+ retryTimerRef.current = window.setTimeout(
+ () => {
+ retryTimerRef.current = null;
+ setRetryAttempt((current) => Math.min(4, current + 1));
+ },
+ 750 * 2 ** retryAttempt,
+ );
+ }, [retryAttempt]);
+
+ return (
+
+ );
+}
+
function normalizeGridOption(option?: string) {
const text = String(option || "").trim();
const explicitGrid = text.match(/(\d+)\s*[x×]\s*(\d+)/i);
@@ -2332,6 +2592,48 @@ function getNodeReferenceMediaUrl(node: CloneNode | null | undefined) {
return getNodeMediaUrl(node) || getDraftString(node, "sourceUrl");
}
+function isVideoReferenceImageNode(node: CloneNode | null | undefined) {
+ const mediaType = getDraftString(node, "mediaType") || node?.inputType;
+ return Boolean(
+ node &&
+ node.kind === "media" &&
+ mediaType === "image" &&
+ !isVideoEffectMaterialNode(node) &&
+ !isVideoRoleMaterialNode(node) &&
+ getNodeReferenceMediaUrl(node),
+ );
+}
+
+function getVideoReferenceModelIdentity(node: CloneNode | null | undefined) {
+ const params = (node?.draftValues?.params || {}) as Record;
+ const settings =
+ params.settings && typeof params.settings === "object"
+ ? (params.settings as Record)
+ : {};
+ return `${String(settings.modelLabel || "")} ${String(
+ params.model || settings.model || "",
+ )}`.trim();
+}
+
+function withVideoGenerationMode(node: CloneNode, mode: VideoGenerationMode): CloneNode {
+ const params = (node.draftValues?.params || {}) as Record;
+ const settings =
+ params.settings && typeof params.settings === "object"
+ ? (params.settings as Record)
+ : {};
+ return {
+ ...node,
+ draftValues: {
+ ...(node.draftValues || {}),
+ params: {
+ ...params,
+ mode,
+ settings: { ...settings, mode },
+ },
+ },
+ };
+}
+
function normalizeVideoCanvasMagicMarks(value: unknown): VideoCanvasMagicMark[] {
if (!Array.isArray(value)) return [];
return value
@@ -2375,6 +2677,22 @@ function getVideoCanvasMagicMarks(node: CloneNode | null | undefined) {
);
}
+function getVideoPromptMentions(node: CloneNode | null | undefined) {
+ if (!node) return [];
+ const params = getNodeParams(node);
+ return normalizeVideoPromptMentions(
+ node.draftValues?.promptMentions || params.promptMentions
+ );
+}
+
+function getVideoCanvasMotionMentions(node: CloneNode | null | undefined) {
+ if (!node) return [];
+ const params = getNodeParams(node);
+ return normalizeVideoCanvasMotionMentions(
+ node.draftValues?.motionMentions || params.motionMentions,
+ );
+}
+
function getNodeGridSplitSourceUrl(node: CloneNode | null | undefined) {
return getNodeMediaUrl(node) || getNodeSourceMediaUrl(node);
}
@@ -2396,8 +2714,9 @@ function getVideoNodeDurationSeconds(node: CloneNode | null | undefined) {
? (params.settings as Record)
: {};
const numericValue = Number(
- params.duration ||
+ params.duration ||
paramsSettings.duration ||
+ node?.draftValues?.durationSeconds ||
getDraftString(node, "duration") ||
(node?.draftValues?.videoMeta && typeof node.draftValues.videoMeta === "object"
? (node.draftValues.videoMeta as Record).duration
@@ -2413,13 +2732,28 @@ function getVideoNodeDurationSeconds(node: CloneNode | null | undefined) {
return 5;
}
+function getVideoNodePixelDimensions(node: CloneNode | null | undefined) {
+ const outputLabel = getDraftString(node, "outputSizeLabel");
+ const match = outputLabel.match(/(\d+)\s*[×x]\s*(\d+)/i);
+ const width = Number(match?.[1] || 1280);
+ const height = Number(match?.[2] || 720);
+ return {
+ width: Number.isFinite(width) && width > 0 ? width : 1280,
+ height: Number.isFinite(height) && height > 0 ? height : 720,
+ };
+}
+
+function formatVideoCropDimensions(width: number, height: number, separator: "x" | "×") {
+ return `${Math.max(2, Math.round(width))} ${separator} ${Math.max(2, Math.round(height))}`;
+}
+
function formatClipDurationLabel(seconds: number) {
const normalized = Math.max(0.1, seconds);
return `${normalized.toFixed(2)} s`;
}
function formatVideoTimestamp(seconds: number) {
- const normalized = Math.max(0, Math.round(seconds));
+ const normalized = Math.max(0, Math.floor(seconds));
const minutes = Math.floor(normalized / 60);
const remainder = normalized % 60;
return `${minutes}:${String(remainder).padStart(2, "0")}`;
@@ -2432,20 +2766,6 @@ function formatAudioTimestamp(seconds: number) {
return `${String(minutes).padStart(2, "0")}:${String(remainder).padStart(2, "0")}`;
}
-function normalizeVideoClipRange(value: unknown): VideoClipRange {
- if (!value || typeof value !== "object") return { start: 0.18, end: 0.52 };
- const candidate = value as Partial;
- const start = Number(candidate.start);
- const end = Number(candidate.end);
- if (!Number.isFinite(start) || !Number.isFinite(end)) {
- return { start: 0.18, end: 0.52 };
- }
- return {
- start: Math.max(0, Math.min(start, 0.94)),
- end: Math.max(Math.min(start + 0.04, 1), Math.min(end, 1)),
- };
-}
-
function formatVideoStoryNumber(value: unknown, fallback: number) {
const text = String(value ?? "").trim();
if (text) return text.replace(/\s*秒$/i, "").replace(/\s*s$/i, "");
@@ -2472,7 +2792,6 @@ function buildVideoAnalysisRows(node: CloneNode, responseData?: VideoAnalysisRes
? responseData.framePreviews.map((item) => String(item || "").trim()).filter(Boolean)
: [];
const videoDuration = Math.max(1, Math.round(getVideoNodeDurationSeconds(node)));
- const defaultEndTime = formatVideoStoryNumber(undefined, videoDuration);
const summary = responseData?.summary?.trim() || "";
const fallbackPrompt = summary || responseData?.raw?.trim() || `${node.title} 的连续镜头推演`;
const createRow = (
@@ -2521,74 +2840,51 @@ function buildVideoAnalysisRows(node: CloneNode, responseData?: VideoAnalysisRes
}
const highlights = responseData?.highlights || [];
const shotIdeas = responseData?.shotIdeas?.slice(0, 3) || [];
- const rows =
- shotIdeas.length > 0
- ? shotIdeas.map((idea, index) =>
- createRow(
- {
- backgroundMusic: highlights[index] || highlights[0] || "氛围音乐待接入",
- imagePrompt: idea.description || fallbackPrompt,
- motionPrompt:
- responseData?.recommendedNextSteps?.[index] ||
- "The camera moves smoothly around the subject and keeps the action continuous.",
- narrative: idea.description || fallbackPrompt,
- sceneDescription: idea.title || `镜头 ${index + 1}`,
- voiceAndSfx: highlights[index + 1] || "无",
- },
- index
- )
- )
- : [
- createRow(
- {
- backgroundMusic: highlights[0] || "低沉的氛围电子乐,带有缓慢的机械节拍感。",
- endTime: defaultEndTime,
- imagePrompt: fallbackPrompt,
- motionPrompt:
- "The subject moves subtly while the camera slowly dollies in, keeping the scene cinematic and continuous.",
- narrative: summary || "通过主要角色、环境和镜头运动呈现当前视频的叙事情绪。",
- sceneDescription: `${node.title} 开场镜头`,
- startTime: "0",
- voiceAndSfx: highlights[1] || "环境声与轻微动作声。",
- },
- 0
- ),
- ];
-
- return rows;
+ if (shotIdeas.length === 0) return [];
+ return shotIdeas.map((idea, index) =>
+ createRow(
+ {
+ backgroundMusic: highlights[index] || highlights[0] || "无",
+ imagePrompt: idea.description || fallbackPrompt,
+ motionPrompt:
+ responseData?.recommendedNextSteps?.[index] ||
+ "The camera moves smoothly around the subject and keeps the action continuous.",
+ narrative: idea.description || fallbackPrompt,
+ sceneDescription: idea.title || `镜头 ${index + 1}`,
+ voiceAndSfx: highlights[index + 1] || "无",
+ },
+ index,
+ ),
+ );
}
function createVideoStoryAnalysisNode({
- existingNode,
framePreviews = [],
+ position,
sourceNode,
rows,
summary,
- error,
}: {
- error?: string;
- existingNode?: CloneNode;
framePreviews?: string[];
+ position?: { x: number; y: number };
sourceNode: CloneNode;
rows: VideoAnalysisRow[];
summary: string;
}): CloneNode {
const previewUrl = getVideoNodePreviewUrl(sourceNode) || getNodeMediaUrl(sourceNode) || "";
return {
- id: existingNode?.id || makeId("video-story"),
+ id: makeId("video-story"),
kind: "video-story",
title: "视频故事",
subtitle: "视频解析结果",
inputType: "text",
- x: existingNode?.x ?? Math.round(sourceNode.x - 720),
- y: existingNode?.y ?? Math.round(sourceNode.y + sourceNode.height + 620),
- width: existingNode?.width ?? 800,
- height: existingNode?.height ?? 400,
+ x: position?.x ?? Math.round(sourceNode.x),
+ y: position?.y ?? Math.round(sourceNode.y + sourceNode.height + 94),
+ width: VIDEO_STORY_NODE_WIDTH,
+ height: VIDEO_STORY_NODE_HEIGHT,
accent: "#8b96ff",
draftValues: {
- ...(existingNode?.draftValues || {}),
action: "VIDEO_STORY_ANALYSIS",
- error: error || undefined,
framePreviews,
generated: true,
keyframeUrl: previewUrl,
@@ -2597,7 +2893,7 @@ function createVideoStoryAnalysisNode({
rows,
sourceVideoNodeId: sourceNode.id,
sourceVideoTitle: sourceNode.title,
- status: error ? "error" : "ready",
+ status: "ready",
summary,
},
};
@@ -2605,43 +2901,55 @@ function createVideoStoryAnalysisNode({
function normalizeVideoHdResolutionOption(option: string) {
const text = option.trim();
- if (/^(1080P|2K|4K)$/i.test(text)) return text.toUpperCase();
- return "1080P";
+ if (text.toUpperCase() === "2K") return "2K" as const;
+ if (text.toUpperCase() === "4K") return "4K" as const;
+ return "1080P" as const;
}
function createVideoHdUpscaleNode({
+ config,
existingNode,
option,
sourceNode,
}: {
+ config: VideoCanvasFunctionConfig["videoHd"];
existingNode?: CloneNode;
option: string;
sourceNode: CloneNode;
}): CloneNode {
const resolution = normalizeVideoHdResolutionOption(option);
- const sourceUrl = getNodeReferenceMediaUrl(sourceNode);
+ const sourceUrl = getNodeReferenceMediaUrl(sourceNode) || getNodeMediaUrl(sourceNode);
+ const interpolationMode = (
+ getDraftString(existingNode, "interpolationMode") || "none"
+ ) as VideoHdInterpolationMode;
+ const slowMultiplier = getDraftNumber(existingNode, "slowMultiplier", 1);
+ const resolutionConfig = config.resolutions[resolution];
return {
id: existingNode?.id || makeId("video-hd"),
kind: "model",
title: `高清(${resolution})`,
subtitle: "配置参数",
inputType: "video",
- x: existingNode?.x ?? Math.round(sourceNode.x),
- y: existingNode?.y ?? Math.round(sourceNode.y + sourceNode.height + OFFICIAL_NODE_GAP),
+ x: existingNode?.x ?? Math.round(sourceNode.x + sourceNode.width + OFFICIAL_NODE_GAP),
+ y: existingNode?.y ?? Math.round(sourceNode.y),
width: existingNode?.width ?? 350,
height: existingNode?.height ?? 350,
accent: "#8b96ff",
draftValues: {
...(existingNode?.draftValues || {}),
action: "VIDEO_HD_UPSCALE",
- frameInterpolation: option === "不补帧" ? "不补帧" : existingNode?.draftValues?.frameInterpolation || "不补帧",
+ cost: getVideoHdCost(config, resolution, interpolationMode, slowMultiplier),
+ frameInterpolation: interpolationMode === "high-quality" ? "高质量补帧" : "不补帧",
+ frameRate: getDraftNumber(existingNode, "frameRate", 30),
generated: existingNode?.draftValues?.generated || false,
- hdEngine: option === "Topazlabs" ? "Topazlabs" : existingNode?.draftValues?.hdEngine || "Topazlabs",
+ hdEngine: config.model,
hdResolution: resolution,
+ interpolationMode,
mediaType: "video",
nodeType: "video-hd",
- outputSizeLabel: resolution === "4K" ? "3840 × 2160" : resolution === "2K" ? "2560 × 1440" : "1920 × 1080",
- scale: option === "1x" ? "1x" : existingNode?.draftValues?.scale || "1x",
+ outputSizeLabel: `${resolutionConfig.width} × ${resolutionConfig.height}`,
+ scale: `${slowMultiplier}x`,
+ slowMultiplier,
sourceVideoNodeId: sourceNode.id,
sourceVideoTitle: sourceNode.title,
sourceUrl,
@@ -2650,69 +2958,173 @@ function createVideoHdUpscaleNode({
};
}
+type VideoHdSettingsPatch = Partial<{
+ frameRate: number;
+ interpolationMode: VideoHdInterpolationMode;
+ resolution: VideoHdResolution;
+ slowMultiplier: number;
+}>;
+
+function updateVideoHdUpscaleNodeSettings(
+ node: CloneNode,
+ config: VideoCanvasFunctionConfig["videoHd"],
+ patch: VideoHdSettingsPatch,
+) {
+ const resolution = patch.resolution || normalizeVideoHdResolutionOption(
+ getDraftString(node, "hdResolution") || "1080P",
+ );
+ const interpolationMode = patch.interpolationMode || (
+ getDraftString(node, "interpolationMode") || "none"
+ ) as VideoHdInterpolationMode;
+ const slowMultiplier = patch.slowMultiplier ?? getDraftNumber(node, "slowMultiplier", 1);
+ const frameRate = patch.frameRate ?? getDraftNumber(node, "frameRate", 30);
+ const resolutionConfig = config.resolutions[resolution];
+ return {
+ ...node,
+ title: `高清(${resolution})`,
+ draftValues: {
+ ...(node.draftValues || {}),
+ cost: getVideoHdCost(config, resolution, interpolationMode, slowMultiplier),
+ frameInterpolation: interpolationMode === "high-quality" ? "高质量补帧" : "不补帧",
+ frameRate,
+ hdEngine: config.model,
+ hdResolution: resolution,
+ interpolationMode,
+ outputSizeLabel: `${resolutionConfig.width} × ${resolutionConfig.height}`,
+ scale: `${slowMultiplier}x`,
+ slowMultiplier,
+ },
+ } satisfies CloneNode;
+}
+
function createVideoCropResultNode({
- existingIndex,
+ cropRect,
+ durationSeconds,
+ mediaUrl,
+ outputHeight,
+ outputWidth,
+ pathname,
sourceNode,
}: {
- existingIndex: number;
+ cropRect: VideoCropRect;
+ durationSeconds: number;
+ mediaUrl: string;
+ outputHeight: number;
+ outputWidth: number;
+ pathname: string;
sourceNode: CloneNode;
}): CloneNode {
- const sourceUrl = getNodeReferenceMediaUrl(sourceNode) || SCRIPT_VIDEO_REFERENCE_URL;
- const previewUrl = getVideoNodePreviewUrl(sourceNode) || sourceUrl;
+ const sourceUrl = getNodeReferenceMediaUrl(sourceNode) || getNodeMediaUrl(sourceNode);
+ const previewUrl = getVideoNodePreviewUrl(sourceNode) || mediaUrl;
const width = sourceNode.width || 560;
const height = sourceNode.height || 315;
return {
id: makeId("video"),
kind: "media",
title: `${sourceNode.title || "视频"}_裁剪`,
- subtitle: "裁剪中",
+ subtitle: "裁剪结果",
inputType: "video",
x: Math.round(sourceNode.x + sourceNode.width + OFFICIAL_NODE_GAP),
- y: Math.round(sourceNode.y + existingIndex * (height + OFFICIAL_NODE_GAP)),
+ y: Math.round(sourceNode.y),
width,
height,
accent: "#8b96ff",
draftValues: {
action: "VIDEO_CROP",
- cropRect: VIDEO_CROP_RECT,
- durationSeconds: getVideoNodeDurationSeconds(sourceNode),
- generated: false,
- generationProgress: 37,
- generationStatus: "pending",
+ cropRect,
+ durationSeconds,
+ generated: true,
+ generationProgress: 100,
+ generationStatus: "complete",
mediaType: "video",
- mediaUrl: "",
+ mediaUrl,
nodeType: "video",
- outputSizeLabel: VIDEO_CROP_OUTPUT_LABEL,
+ outputSizeLabel: formatVideoCropDimensions(outputWidth, outputHeight, "×"),
+ pathname,
poster: previewUrl,
sourceUrl,
sourceVideoNodeId: sourceNode.id,
sourceVideoTitle: sourceNode.title,
- uploadStatus: "uploading",
- url: [],
+ uploadStatus: "complete",
+ url: [mediaUrl],
+ },
+ };
+}
+
+function createVideoClipResultNode({
+ durationSeconds,
+ mediaUrl,
+ pathname,
+ range,
+ sourceNode,
+}: {
+ durationSeconds: number;
+ mediaUrl: string;
+ pathname: string;
+ range: VideoClipRange;
+ sourceNode: CloneNode;
+}): CloneNode {
+ const previewUrl = getVideoNodePreviewUrl(sourceNode) || mediaUrl;
+ return {
+ id: makeId("video"),
+ kind: "media",
+ title: "剪辑",
+ subtitle: "生成结果",
+ inputType: "video",
+ x: Math.round(sourceNode.x + sourceNode.width + OFFICIAL_NODE_GAP),
+ y: Math.round(sourceNode.y),
+ width: sourceNode.width || 560,
+ height: sourceNode.height || 315,
+ accent: "#8b96ff",
+ draftValues: {
+ action: "VIDEO_CLIP",
+ clipRange: range,
+ durationSeconds,
+ generated: true,
+ generationProgress: 100,
+ generationStatus: "complete",
+ mediaType: "video",
+ mediaUrl,
+ nodeType: "video",
+ outputSizeLabel: getDraftString(sourceNode, "outputSizeLabel") || "1280 × 720",
+ pathname,
+ poster: previewUrl,
+ sourceVideoNodeId: sourceNode.id,
+ sourceVideoTitle: sourceNode.title,
+ sourceUrl: getNodeReferenceMediaUrl(sourceNode) || getNodeMediaUrl(sourceNode),
+ uploadStatus: "complete",
+ url: [mediaUrl],
},
};
}
function createVideoSubtitleEraseNode({
+ definition,
existingIndex,
mode = "smart",
regions = [],
sourceNode,
+ subtitleEraseConfig,
}: {
+ definition: VideoCanvasFunctionDefinition;
existingIndex: number;
mode?: SubtitleEraseMode;
regions?: SubtitleEraseRegion[];
sourceNode: CloneNode;
+ subtitleEraseConfig: VideoCanvasFunctionConfig["videoSubtitleErase"];
}): CloneNode {
const sourceUrl = getNodeReferenceMediaUrl(sourceNode) || SCRIPT_VIDEO_REFERENCE_URL;
const previewUrl = getVideoNodePreviewUrl(sourceNode) || sourceUrl;
const outputSizeLabel = getDraftString(sourceNode, "outputSizeLabel") || "1280 × 720";
- const normalizedRegions = regions.map(({ height, width, x, y }) => ({
- height: Number(height.toFixed(4)),
- width: Number(width.toFixed(4)),
- x: Number(x.toFixed(4)),
- y: Number(y.toFixed(4)),
- }));
+ const normalizedRegions = regions.map((region) => {
+ const normalized = normalizeSubtitleEraseRegion(region);
+ return {
+ height: Number(normalized.height.toFixed(4)),
+ width: Number(normalized.width.toFixed(4)),
+ x: Number(normalized.x.toFixed(4)),
+ y: Number(normalized.y.toFixed(4)),
+ };
+ });
const isRegionErase = mode === "region";
return {
id: makeId("video"),
@@ -2727,7 +3139,7 @@ function createVideoSubtitleEraseNode({
accent: "#8b96ff",
draftValues: {
action: "VIDEO_SUBTITLE_ERASE",
- cost: 0,
+ cost: Math.max(0, Math.round(definition.cost || 0)),
durationSeconds: getVideoNodeDurationSeconds(sourceNode),
generated: false,
generationProgress: 2,
@@ -2738,10 +3150,11 @@ function createVideoSubtitleEraseNode({
eraseMode: mode,
eraseRatioLocation: normalizedRegions,
erase_ratio_location: normalizedRegions,
- model: "volcano-subtitle-eraser",
+ model: definition.model || subtitleEraseConfig.model,
nodeType: "video",
outputSizeLabel,
poster: previewUrl,
+ prompt: definition.prompt,
sourceUrl,
sourceVideoNodeId: sourceNode.id,
sourceVideoTitle: sourceNode.title,
@@ -3611,10 +4024,7 @@ function getToolReferenceModuleTemplates(
if (!requiresMedia(tool) && !functionRequiresMedia(definition)) return [];
if (definition.key === "video-analysis") return [{ kind: "video", title: "视频参考" }];
if (definition.key === "audio-to-video") {
- return [
- { kind: "audio", title: "音频参考" },
- { kind: "image", title: "画面参考" },
- ];
+ return [{ kind: "audio", title: "音频参考" }];
}
if (definition.key === "first-last-frame-video") {
return [
@@ -3721,7 +4131,8 @@ function normalizeVideoGenerationMode(value: unknown): VideoGenerationMode | nul
mode === "text-to-video" ||
mode === "first-frame" ||
mode === "first-last-frame" ||
- mode === "reference-images"
+ mode === "reference-images" ||
+ mode === "audio-to-video"
) {
return mode;
}
@@ -3756,12 +4167,14 @@ function normalizeVideoResolutionValue(value: unknown): VideoResolution {
.toLowerCase()
.replace(/\s+/g, "");
if (normalized === "480p") return "480p";
- if (normalized === "1080p" || normalized === "1k" || normalized === "4k") return "1080p";
+ if (normalized === "4k") return "4k";
+ if (normalized === "1080p" || normalized === "1k") return "1080p";
return "720p";
}
function formatVideoResolutionLabel(value: unknown) {
const resolution = normalizeVideoResolutionValue(value);
+ if (resolution === "4k") return "4K";
return resolution === "480p" ? "480P" : resolution === "1080p" ? "1080P" : "720P";
}
@@ -3798,20 +4211,14 @@ function getVideoModeIcon(tab: VideoComposerTabKey): LucideIcon {
}
function getVideoFunctionKeyForMode(mode: VideoGenerationMode) {
+ if (mode === "audio-to-video") return "audio-to-video" as const;
if (mode === "text-to-video") return "text-to-video" as const;
if (mode === "first-last-frame") return "first-last-frame-video" as const;
return "first-frame-video" as const;
}
-function selectVideoReferenceImages(images: string[], mode: VideoGenerationMode) {
- if (mode === "text-to-video") return [];
- if (mode === "first-frame") return images.slice(0, 1);
- if (mode === "first-last-frame") return images.slice(0, 2);
- return images.slice(0, 4);
-}
-
-function getCanvasModelRef(model: CanvasModelOption) {
- return String(model.modelId || model.id || "").trim();
+function getCanvasModelRef(model?: CanvasModelOption) {
+ return String(model?.modelId || model?.id || "").trim();
}
function isCanvasImageModel(model: CanvasModelOption) {
@@ -3863,6 +4270,7 @@ function getCanvasModelDisplayName(
type VideoCanvasBillingDisplay = {
label: string;
message: string;
+ originalLabel?: string;
};
function getCanvasModelOptionByRef(
@@ -3904,6 +4312,9 @@ function resolveVideoCanvasBillingDisplay(params: {
imageCount?: number;
duration?: number | null;
resolution?: string | null;
+ parameterPricingCost?: number | null;
+ videoCatalogBilling?: { cost: number; originalCost?: number } | null;
+ videoCount?: number | null;
billingFeature?: ImageToolCreditFeature | null;
functionDefinition?: Pick | null;
}): VideoCanvasBillingDisplay {
@@ -3941,28 +4352,61 @@ function resolveVideoCanvasBillingDisplay(params: {
params.modelRef,
params.modelOptions || [],
);
- if (!matchedModel?.imageGenConfig) {
- return fallback;
- }
if (params.modelKind === "video") {
- const videoCost = getGenerationCreditCostFromImageGenConfig(
- matchedModel.imageGenConfig,
- "video",
- {
- duration: params.duration,
- resolution: params.resolution,
- },
+ if (params.videoCatalogBilling && params.videoCatalogBilling.cost > 0) {
+ const originalCost = params.videoCatalogBilling.originalCost;
+ return {
+ label: String(params.videoCatalogBilling.cost),
+ message: originalCost
+ ? `当前视频参数原价 ${originalCost} 积分,按模型折扣预计消耗 ${params.videoCatalogBilling.cost} 积分。`
+ : `当前视频参数预计消耗 ${params.videoCatalogBilling.cost} 积分。`,
+ originalLabel: originalCost ? String(originalCost) : undefined,
+ };
+ }
+ const modelPricing = matchedModel?.imageGenConfig;
+ const hasGranularModelPricing = Boolean(
+ modelPricing &&
+ ((modelPricing.creditCostPerVideoByDuration &&
+ Object.keys(modelPricing.creditCostPerVideoByDuration).length > 0) ||
+ (modelPricing.creditCostPerVideoByResolution &&
+ Object.keys(modelPricing.creditCostPerVideoByResolution).length > 0)),
);
+ const videoCount = Math.max(1, Math.round(Number(params.videoCount || 1)));
+ const rawModelUnitCost = modelPricing
+ ? getGenerationCreditCostFromImageGenConfig(
+ { ...modelPricing, creditDiscountRate: undefined },
+ "video",
+ {
+ duration: params.duration,
+ resolution: params.resolution,
+ },
+ )
+ : 0;
+ const rawVideoCost = hasGranularModelPricing
+ ? rawModelUnitCost * videoCount
+ : Math.max(0, Math.round(Number(params.parameterPricingCost || 0))) ||
+ rawModelUnitCost * videoCount;
+ const videoCost = modelPricing
+ ? applyImageModelCreditDiscount(rawVideoCost, modelPricing)
+ : rawVideoCost;
if (videoCost > 0) {
return {
label: String(videoCost),
- message: `当前视频模型按站点配置预计消耗 ${videoCost} 积分。`,
+ message:
+ rawVideoCost !== videoCost
+ ? `当前视频参数原价 ${rawVideoCost} 积分,按模型折扣预计消耗 ${videoCost} 积分。`
+ : `当前视频参数预计消耗 ${videoCost} 积分。`,
+ originalLabel: rawVideoCost !== videoCost ? String(rawVideoCost) : undefined,
};
}
return fallback;
}
+ if (!matchedModel?.imageGenConfig) {
+ return fallback;
+ }
+
if (params.modelKind === "image") {
const unitCost = getImageCreditCostFromConfig(matchedModel.imageGenConfig);
if (unitCost > 0) {
@@ -4025,6 +4469,37 @@ function wait(ms: number) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
+async function resolveGeneratedCanvasImageUrl(
+ response: Awaited>,
+) {
+ const immediateUrl = getFirstGeneratedImageUrl(response);
+ if (immediateUrl) return { success: true as const, url: immediateUrl };
+
+ const taskId = String(response.data?.taskId || "").trim();
+ if (!response.success || !taskId) {
+ return { error: response.error || "生成图片失败", success: false as const, url: "" };
+ }
+
+ for (let attempt = 0; attempt < 60; attempt += 1) {
+ await wait(attempt < 8 ? 1500 : 2500);
+ const taskResponse = await imageGenerationService.getImageTask(taskId);
+ const task = taskResponse.data?.task;
+ if (!taskResponse.success || !task) continue;
+ if (task.status === "failed") {
+ return { error: task.error || "生成图片失败", success: false as const, url: "" };
+ }
+ if (task.status !== "succeeded") continue;
+
+ const image = task.result?.images?.[0];
+ const url = image?.originalUrl || image?.thumbnailUrl || "";
+ return url
+ ? { success: true as const, url }
+ : { error: "生成任务未返回图片", success: false as const, url: "" };
+ }
+
+ return { error: "生成任务等待超时", success: false as const, url: "" };
+}
+
function createEqualGridRects(rows: number, columns: number): GridCropRect[] {
return Array.from({ length: rows * columns }).map((_, index) => {
const column = index % columns;
@@ -4356,12 +4831,21 @@ async function createGridSplitReferenceAsset({
return upload.success ? upload.data?.url || "" : "";
}
-async function ensureRemoteCanvasImageUrl(sourceUrl: string, filenamePrefix: string) {
- if (/^https?:\/\//i.test(sourceUrl)) return sourceUrl;
- if (!sourceUrl || typeof window === "undefined") return "";
+type StoredCanvasImage = {
+ assetId?: string;
+ pathname?: string;
+ url: string;
+};
+
+async function ensureStoredCanvasImage(
+ sourceUrl: string,
+ filenamePrefix: string,
+): Promise {
+ if (isPortableDirectorCaptureUrl(sourceUrl)) return { url: sourceUrl };
+ if (!sourceUrl || typeof window === "undefined") return null;
const response = await fetch(sourceUrl);
- if (!response.ok) return "";
+ if (!response.ok) return null;
const blob = await response.blob();
const extension = blob.type.includes("jpeg") || blob.type.includes("jpg")
? "jpg"
@@ -4372,7 +4856,16 @@ async function ensureRemoteCanvasImageUrl(sourceUrl: string, filenamePrefix: str
type: blob.type || `image/${extension}`,
});
const upload = await imageGenerationService.uploadImage(file);
- return upload.success ? upload.data?.url || "" : "";
+ if (!upload.success || !upload.data?.url) return null;
+ return {
+ assetId: upload.data.assetId,
+ pathname: upload.data.pathname,
+ url: upload.data.url,
+ };
+}
+
+async function ensureRemoteCanvasImageUrl(sourceUrl: string, filenamePrefix: string) {
+ return (await ensureStoredCanvasImage(sourceUrl, filenamePrefix))?.url || "";
}
async function resolveNodeReferenceImages(node: CloneNode | null | undefined, limit = 12): Promise {
@@ -4427,6 +4920,7 @@ function getPresetMode(node: CloneNode | null | undefined): ActiveStarterScene {
function getPresetControlMode(node: CloneNode | null | undefined): ActivePresetControlMode {
const starterMode = getPresetMode(node);
if (starterMode) return starterMode;
+ if (getNodeParams(node).functionKey === "audio-to-video") return "audio-to-video";
const functionKey = resolveOfficialImagePresetFunctionKey({
fallbackKey: getNodeFunctionKey(node),
option: getNodeParams(node).option || node?.draftValues?.toolbarOption,
@@ -4436,11 +4930,17 @@ function getPresetControlMode(node: CloneNode | null | undefined): ActivePresetC
}
function isOfficialImageControlNode(node: CloneNode | null | undefined): node is CloneNode {
- return !!node && (node.kind === "media" || node.kind === "script") && !!getPresetControlMode(node);
+ if (!node) return false;
+ const mode = getPresetControlMode(node);
+ return !!mode && (node.kind === "media" || node.kind === "script" || mode === "audio-to-video");
}
function getTargetModuleKind(node: CloneNode): AddModuleKind | null {
- if (node.kind === "media" || getPresetMode(node)) return null;
+ if (node.kind === "media") {
+ const mediaType = getDraftString(node, "mediaType") || node.inputType;
+ return mediaType === "video" ? "video" : null;
+ }
+ if (getPresetControlMode(node)) return null;
if (getDraftString(node, "action") === "TEXT_RESOURCE" || getDraftString(node, "scriptRole") === "resource") {
return null;
}
@@ -4467,7 +4967,7 @@ function extractTrailingNumber(value: string | undefined) {
}
function inferLegacyTargetModuleKind(node: CloneNode): AddModuleKind | null {
- if (node.kind === "media" || getPresetMode(node)) return null;
+ if (node.kind === "media" || getPresetControlMode(node)) return null;
const action = getDraftString(node, "action");
if (action.endsWith("_RESOURCE") || action === "TEXT_RESOURCE") return null;
@@ -4564,6 +5064,24 @@ function normalizeTargetModuleNode(node: CloneNode, kind: AddModuleKind, fallbac
function normalizeRestoredNode(node: CloneNode, fallbackIndex: number): CloneNode {
const action = getDraftString(node, "action");
+ if (node.kind === "video-story" || action === "VIDEO_STORY_ANALYSIS") {
+ return {
+ ...node,
+ kind: "video-story",
+ title: "视频故事",
+ subtitle: "视频解析结果",
+ inputType: "text",
+ width: VIDEO_STORY_NODE_WIDTH,
+ height: VIDEO_STORY_NODE_HEIGHT,
+ draftValues: {
+ ...(node.draftValues || {}),
+ action: "VIDEO_STORY_ANALYSIS",
+ mediaType: "text",
+ nodeType: "video-story",
+ status: "ready",
+ },
+ };
+ }
const isCharacterThreeViewResultNode = isCharacterThreeViewNode(node);
const isNineGridResultNode =
!isCharacterThreeViewResultNode &&
@@ -4734,7 +5252,8 @@ function makePresetEdges(leftNodes: CloneNode[], rightNode: CloneNode): CloneEdg
function createStarterWorkflow(
starterTitle: string,
- center: { x: number; y: number }
+ center: { x: number; y: number },
+ audioToVideoPrompt = "",
): { nodes: CloneNode[]; edges: CloneEdge[]; selectedId: string } | null {
if (starterTitle === "故事脚本生成") {
const preset = "story-script";
@@ -4841,30 +5360,19 @@ function createStarterWorkflow(
x: leftX,
y: leftY,
});
- const image = createPresetMediaNode({
- action: "IMAGE_RESOURCE",
- height: 400,
- mediaType: "image",
- mediaUrl: AUDIO_REFERENCE_IMAGE_URL,
- preset,
- title: "图片",
- width: 720,
- x: leftX,
- y: leftY + 300,
- });
const right = createPresetMediaNode({
action: "VIDEO_GENERATE",
generated: true,
height: 430,
mediaType: "video",
preset,
- prompt: AUDIO_TO_VIDEO_PROMPT,
+ prompt: audioToVideoPrompt,
title: "视频",
width: 760,
x: Math.round(center.x + 90),
y: Math.round(center.y - 300),
});
- return { nodes: [audio, image, right], edges: makePresetEdges([audio, image], right), selectedId: right.id };
+ return { nodes: [audio, right], edges: makePresetEdges([audio], right), selectedId: right.id };
}
return null;
@@ -4880,7 +5388,9 @@ export function LiblibVideoCanvasReplica({
const connectionDragCleanupRef = useRef<(() => void) | null>(null);
const connectionDragRef = useRef(null);
const assetManagerResizeCleanupRef = useRef<(() => void) | null>(null);
+ const assetManagerWidthRef = useRef(ASSET_MANAGER_DEFAULT_WIDTH);
const dragRef = useRef(null);
+ const keyboardPanPressedRef = useRef(false);
const suppressConnectionClickRef = useRef(false);
const { data: siteBrandSettings } = useSWR(
"/api/site-settings?scope=branding",
@@ -4894,7 +5404,7 @@ export function LiblibVideoCanvasReplica({
"/api/models",
fetcher
);
- const { data: membershipSummary } = useSWR(
+ const { data: membershipSummary, mutate: mutateMembershipSummary } = useSWR(
"/api/user/membership-summary",
fetcher
);
@@ -4914,10 +5424,46 @@ export function LiblibVideoCanvasReplica({
() => resolveSiteName(siteBrandSettings?.siteName),
[siteBrandSettings?.siteName]
);
- const [nodes, setNodes] = useState([]);
- const [edges, setEdges] = useState([]);
- const [selectedNodeId, setSelectedNodeId] = useState(null);
+ const localCanvasRevisionRef = useRef(0);
+ const hydrationWriteDepthRef = useRef(0);
+ const lastSavedCanvasSignatureRef = useRef("");
+ const nodeCreationOrdinalRef = useRef(1);
+ const historyMediaOrdinalsRef = useRef({
+ audio: 1,
+ image: 1,
+ video: 1,
+ });
+ const directorCaptureDeleteTimersRef = useRef