From fb8d8232dcb93280fa859b50c6cdcc0a7dd5213a Mon Sep 17 00:00:00 2001 From: 134 Date: Tue, 28 Jul 2026 15:50:47 +0800 Subject: [PATCH] feat: support video attachments in Claw chat --- app/(chat)/api/files/direct-upload/route.ts | 42 +++- app/(chat)/api/files/upload/route.ts | 37 ++- app/api/canvas/clawd-chat/route.ts | 83 ++++++- components/canvas/CanvasChatPanel.tsx | 234 ++++++++++++++++-- lib/client/upload-with-retry.ts | 18 +- lib/openclaw-chat/media-attachments.ts | 183 ++++++++++++++ lib/storage/service.ts | 5 + playwright.helpers.config.ts | 1 + .../routes/openclaw-video-attachments.test.ts | 92 +++++++ 9 files changed, 650 insertions(+), 45 deletions(-) create mode 100644 lib/openclaw-chat/media-attachments.ts create mode 100644 tests/routes/openclaw-video-attachments.test.ts diff --git a/app/(chat)/api/files/direct-upload/route.ts b/app/(chat)/api/files/direct-upload/route.ts index 7236ed280..6b168b7a6 100644 --- a/app/(chat)/api/files/direct-upload/route.ts +++ b/app/(chat)/api/files/direct-upload/route.ts @@ -21,6 +21,7 @@ import { maybeEnqueueStorageMigrationTask, resolveS3UploadTarget, } from "@/lib/storage/service"; +import { OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE } from "@/lib/openclaw-chat/media-attachments"; const logger = createLogger("direct-upload-api"); const DIRECT_UPLOAD_MAX_BYTES = 500 * 1024 * 1024; @@ -39,6 +40,13 @@ const allowedDirectUploadTypes = new Set([ "image/png", "image/webp", ]); +const allowedOpenClawVideoUploadTypes = new Set([ + "video/mp4", + "video/quicktime", + "video/webm", + "video/x-m4v", + "video/x-matroska", +]); const InitiateSchema = z.object({ action: z.literal("initiate"), @@ -109,8 +117,12 @@ function assertUserOwnedKey(key: string, userId: string) { return normalized; } -function isAllowedDirectUpload(contentType: string) { - return allowedDirectUploadTypes.has(contentType); +function isAllowedDirectUpload(contentType: string, purpose?: string) { + return ( + allowedDirectUploadTypes.has(contentType) || + (purpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE && + allowedOpenClawVideoUploadTypes.has(contentType)) + ); } function computePartSize(size: number) { @@ -204,7 +216,7 @@ export async function POST(request: Request) { payload.filename ); - if (!isAllowedDirectUpload(contentType)) { + if (!isAllowedDirectUpload(contentType, payload.purpose)) { logger.warn("unsupported_direct_upload_type", { contentType, filenameExtension: payload.filename.toLowerCase().split(".").pop() || null, @@ -213,7 +225,10 @@ export async function POST(request: Request) { clientUploadId: payload.clientUploadId || null, }); return NextResponse.json( - { error: "Direct upload only supports JPG, PNG and WebP images" }, + { + error: + "直传仅支持 JPG、PNG、WebP 图片以及 Claw 对话的 MP4、MOV、WebM、M4V、MKV 视频", + }, { status: 400 } ); } @@ -328,9 +343,12 @@ export async function POST(request: Request) { const key = assertUserOwnedKey(payload.key, session.user.id); const contentType = normalizeContentType(payload.contentType, payload.filename); - if (!isAllowedDirectUpload(contentType)) { + if (!isAllowedDirectUpload(contentType, payload.purpose)) { return NextResponse.json( - { error: "Direct upload only supports JPG, PNG and WebP images" }, + { + error: + "直传仅支持 JPG、PNG、WebP 图片以及 Claw 对话的 MP4、MOV、WebM、M4V、MKV 视频", + }, { status: 400 } ); } @@ -389,15 +407,21 @@ export async function POST(request: Request) { const assetStartedAt = Date.now(); const asset = await upsertMediaAsset({ userId: session.user.id, - mediaType: "image", - sourceType: "upload", + mediaType: resolvedContentType.startsWith("video/") ? "video" : "image", + sourceType: + payload.purpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE + ? "openclaw_chat_video_upload" + : "upload", status: "ready", originalUrl: url, previewUrl: url, thumbnailUrl: url, mimeType: resolvedContentType, displayName: payload.filename, - sourceTable: "direct_upload", + sourceTable: + payload.purpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE + ? "openclaw_chat_video_upload" + : "direct_upload", metadata: { pathname: key, size: actualSize ?? payload.size, diff --git a/app/(chat)/api/files/upload/route.ts b/app/(chat)/api/files/upload/route.ts index 831484ff4..26adc7335 100644 --- a/app/(chat)/api/files/upload/route.ts +++ b/app/(chat)/api/files/upload/route.ts @@ -7,6 +7,7 @@ import { upsertMediaAsset } from "@/lib/db/queries"; import { isSiteAdminType } from "@/lib/middleware/admin"; import { getContentType, uploadFile } from "@/lib/storage/service"; import { createLogger } from "@/lib/observability/logger"; +import { OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE } from "@/lib/openclaw-chat/media-attachments"; const logger = createLogger("upload-api"); const ALLOWED_UPLOAD_IMAGE_TYPES = new Set([ @@ -26,6 +27,13 @@ const ALLOWED_ATLASCLOUD_VIDEO_TYPES = new Set([ "video/quicktime", "video/webm", ]); +const ALLOWED_OPENCLAW_VIDEO_TYPES = new Set([ + "video/mp4", + "video/quicktime", + "video/webm", + "video/x-m4v", + "video/x-matroska", +]); const HEIC_UPLOAD_TYPES = new Set(["image/heic", "image/heif"]); const SHARP_COMPATIBILITY_UPLOAD_TYPES = new Set([ @@ -246,6 +254,9 @@ export async function POST(request: Request) { const isAtlasCloudReferenceVideo = uploadPurpose === "canvas-video-process" && ALLOWED_ATLASCLOUD_VIDEO_TYPES.has(contentType); + const isOpenClawChatVideoUpload = + uploadPurpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE && + ALLOWED_OPENCLAW_VIDEO_TYPES.has(contentType); const clientUploadId = String(formData.get("clientUploadId") || "").trim() || null; const isSvgUpload = contentType === "image/svg+xml"; @@ -278,7 +289,8 @@ export async function POST(request: Request) { if ( !ALLOWED_UPLOAD_IMAGE_TYPES.has(contentType) && - !isAtlasCloudReferenceVideo + !isAtlasCloudReferenceVideo && + !isOpenClawChatVideoUpload ) { const errorMessage = createUnsupportedImageTypeMessage(contentType, filename); logger.warn("[Upload API] Unsupported upload content type", { @@ -445,15 +457,28 @@ export async function POST(request: Request) { try { const asset = await upsertMediaAsset({ userId: session.user.id, - mediaType: isAtlasCloudReferenceVideo ? "video" : "image", - sourceType: "upload", + mediaType: + isAtlasCloudReferenceVideo || isOpenClawChatVideoUpload + ? "video" + : "image", + sourceType: isOpenClawChatVideoUpload + ? "openclaw_chat_video_upload" + : "upload", status: "ready", originalUrl: data.url, - previewUrl: isAtlasCloudReferenceVideo ? null : data.url, - thumbnailUrl: isAtlasCloudReferenceVideo ? null : data.url, + previewUrl: + isAtlasCloudReferenceVideo || isOpenClawChatVideoUpload + ? null + : data.url, + thumbnailUrl: + isAtlasCloudReferenceVideo || isOpenClawChatVideoUpload + ? null + : data.url, mimeType: data.contentType || contentType || null, displayName: filename, - sourceTable: "upload", + sourceTable: isOpenClawChatVideoUpload + ? "openclaw_chat_video_upload" + : "upload", metadata: { pathname: data.pathname, size: optimizedBuffer.length, diff --git a/app/api/canvas/clawd-chat/route.ts b/app/api/canvas/clawd-chat/route.ts index d360ece3e..fb336cbc7 100644 --- a/app/api/canvas/clawd-chat/route.ts +++ b/app/api/canvas/clawd-chat/route.ts @@ -25,6 +25,11 @@ import { buildOpenClawMessageWithSpreadsheetMarkdown, buildOpenClawSpreadsheetMarkdown, } from "@/lib/openclaw-chat/spreadsheet-markdown"; +import { + normalizeOpenClawChatMediaAttachments, + OPENCLAW_CHAT_MEDIA_MAX_ATTACHMENTS, + type OpenClawChatMediaAttachment, +} from "@/lib/openclaw-chat/media-attachments"; import { generateUUID } from "@/lib/utils"; const CLAW_UPGRADE_RETRY_MESSAGE = "claw正在升级更新稍等5分钟再试"; @@ -35,9 +40,18 @@ const requestSchema = z.object({ displayMessage: z.string().trim().max(8000).optional(), userMessageId: z.string().uuid().optional(), metadata: z.record(z.unknown()).optional(), + attachments: z + .array(z.unknown()) + .max(OPENCLAW_CHAT_MEDIA_MAX_ATTACHMENTS) + .optional() + .default([]), }); -type ParsedClawdChatRequest = z.infer & { +type ParsedClawdChatRequest = Omit< + z.infer, + "attachments" +> & { + attachments: OpenClawChatMediaAttachment[]; tableFiles: File[]; }; @@ -85,14 +99,23 @@ async function parseClawdChatRequest( const contentType = request.headers.get("content-type") || ""; if (!contentType.includes("multipart/form-data")) { const parsed = requestSchema.safeParse(await request.json().catch(() => null)); - return parsed.success ? { ...parsed.data, tableFiles: [] } : null; + if (!parsed.success) return null; + const attachments = normalizeOpenClawChatMediaAttachments( + parsed.data.attachments + ); + if (attachments.length !== parsed.data.attachments.length) { + return null; + } + return { ...parsed.data, attachments, tableFiles: [] }; } const formData = await request.formData().catch(() => null); if (!formData) return null; const metadataValue = formData.get("metadata"); + const attachmentsValue = formData.get("attachments"); let metadata: Record | undefined; + let attachments: unknown[] = []; if (typeof metadataValue === "string" && metadataValue.trim()) { try { const parsedMetadata = JSON.parse(metadataValue); @@ -105,6 +128,17 @@ async function parseClawdChatRequest( return null; } } + if (typeof attachmentsValue === "string" && attachmentsValue.trim()) { + try { + const parsedAttachments = JSON.parse(attachmentsValue); + if (!Array.isArray(parsedAttachments)) { + return null; + } + attachments = parsedAttachments; + } catch { + return null; + } + } const parsed = requestSchema.safeParse({ id: formData.get("id"), @@ -112,14 +146,25 @@ async function parseClawdChatRequest( displayMessage: formData.get("displayMessage") || undefined, userMessageId: formData.get("userMessageId") || undefined, metadata, + attachments, }); if (!parsed.success) return null; + const normalizedAttachments = normalizeOpenClawChatMediaAttachments( + parsed.data.attachments + ); + if (normalizedAttachments.length !== parsed.data.attachments.length) { + return null; + } const tableFiles = formData .getAll("tableFiles") .filter((entry): entry is File => isRequestFile(entry) && entry.size > 0); - return { ...parsed.data, tableFiles }; + return { + ...parsed.data, + attachments: normalizedAttachments, + tableFiles, + }; } async function readOpenClawConfig() { @@ -163,8 +208,15 @@ export async function POST(request: Request) { if (!parsed) { return NextResponse.json({ error: "请求参数不正确" }, { status: 400 }); } - if (!parsed.message && parsed.tableFiles.length === 0) { - return NextResponse.json({ error: "请输入问题或上传表格" }, { status: 400 }); + if ( + !parsed.message && + parsed.tableFiles.length === 0 && + parsed.attachments.length === 0 + ) { + return NextResponse.json( + { error: "请输入问题或上传附件" }, + { status: 400 } + ); } const config = await readOpenClawConfig(); @@ -275,6 +327,8 @@ export async function POST(request: Request) { skill_tag_id: skillTagId || undefined, skill_tag_name: skillTagName || undefined, message: upstreamMessage, + attachments: + parsed.attachments.length > 0 ? parsed.attachments : undefined, metadata: { source: "ai_chatbot_canvas", page: "/canvas", @@ -288,6 +342,8 @@ export async function POST(request: Request) { user_department: openClawDepartmentName, skill_tag_id: skillTagId || undefined, skill_tag_name: skillTagName || undefined, + media_attachments: + parsed.attachments.length > 0 ? parsed.attachments : undefined, spreadsheet_files: spreadsheetMarkdownResult?.files || undefined, spreadsheet_markdown_truncated: spreadsheetMarkdownResult?.truncated || undefined, @@ -378,7 +434,15 @@ export async function POST(request: Request) { chatId: parsed.id, role: "user", parts: [{ type: "text", text: displayMessage }], - attachments: [], + attachments: parsed.attachments.map((attachment) => ({ + name: + attachment.name || + (attachment.type === "video" ? "视频附件" : "图片附件"), + url: attachment.url, + contentType: + attachment.content_type || + (attachment.type === "video" ? "video/mp4" : "image/jpeg"), + })), createdAt: userCreatedAt, videoStorageStatus: null, temporaryVideoUrl: null, @@ -387,6 +451,12 @@ export async function POST(request: Request) { videoMetadata: { composerMode: "clawd", source: "openclaw", + imageAttachmentCount: parsed.attachments.filter( + (attachment) => attachment.type === "image" + ).length, + videoAttachmentCount: parsed.attachments.filter( + (attachment) => attachment.type === "video" + ).length, skillTagId: skillTagId || null, skillTagName: skillTagName || null, }, @@ -431,6 +501,7 @@ export async function POST(request: Request) { createdAt: userMessage.createdAt.toISOString(), composerMode: "clawd", }, + attachments: userMessage.attachments, videoMetadata: userMessage.videoMetadata, }, { diff --git a/components/canvas/CanvasChatPanel.tsx b/components/canvas/CanvasChatPanel.tsx index acb17211f..a77225837 100644 --- a/components/canvas/CanvasChatPanel.tsx +++ b/components/canvas/CanvasChatPanel.tsx @@ -287,6 +287,18 @@ import { stripCanvasModelRoutingLines, stripMediaMarkup, } from "@/components/canvas/chat-panel/canvas-chat-media"; +import { + buildOpenClawVideoDisplayText, + buildOpenClawVideoReferenceText, + getOpenClawChatVideoContentType, + isOpenClawChatVideoAttachment, + isOpenClawChatVideoFile, + normalizeOpenClawChatMediaAttachments, + OPENCLAW_CHAT_VIDEO_ACCEPT, + OPENCLAW_CHAT_VIDEO_MAX_BYTES, + OPENCLAW_CHAT_VIDEO_MAX_FILES, + OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE, +} from "@/lib/openclaw-chat/media-attachments"; type ResponseModule = typeof import("@/components/elements/response"); type ResponseProps = ComponentProps; @@ -612,9 +624,10 @@ function writeStoredClawdComposerSubmissionSnapshot( function shouldShowClawdProcessingHint( message: string, hasTableFiles: boolean, - hasImageAttachments: boolean + hasImageAttachments: boolean, + hasVideoAttachments: boolean ) { - if (hasTableFiles || hasImageAttachments) { + if (hasTableFiles || hasImageAttachments || hasVideoAttachments) { return true; } @@ -675,17 +688,19 @@ function buildClawdImageReferenceText(attachments: PickedObjectAttachment[]) { return `用户还添加了以下图片作为参考，请按图片链接理解：\n${lines.join("\n")}`; } -function getClawdDefaultPrompt(hasTables: boolean, hasImages: boolean) { - if (hasTables && hasImages) { - return "请分析上传的表格和图片内容"; - } - if (hasTables) { - return "请分析上传的表格内容"; - } - if (hasImages) { - return "请分析上传的图片内容"; - } - return "请分析"; +function getClawdDefaultPrompt( + hasTables: boolean, + hasImages: boolean, + hasVideos: boolean +) { + const mediaNames = [ + hasTables ? "表格" : "", + hasImages ? "图片" : "", + hasVideos ? "视频" : "", + ].filter(Boolean); + return mediaNames.length > 0 + ? `请分析上传的${mediaNames.join("、")}内容` + : "请分析"; } type VideoStoryboardStreamData = @@ -11034,15 +11049,44 @@ export function CanvasChatPanel({ : ""; const hasPendingClawdSkillTagPrompt = Boolean(clawdSkillTagPrompt); const attachmentsSnapshot = isProgrammaticClawdQuickSend ? [] : [...attachments]; - const imageAttachmentsSnapshot = attachmentsSnapshot + const sanitizedAttachmentsSnapshot = attachmentsSnapshot .map((attachment) => sanitizeChatAttachment(attachment)) .filter((attachment): attachment is PickedObjectAttachment => Boolean(attachment)); + const videoAttachmentsSnapshot = sanitizedAttachmentsSnapshot.filter( + (attachment) => + isOpenClawChatVideoAttachment( + attachment.url, + attachment.contentType + ) + ); + const imageAttachmentsSnapshot = sanitizedAttachmentsSnapshot.filter( + (attachment) => + !isOpenClawChatVideoAttachment( + attachment.url, + attachment.contentType + ) + ); const hasImageAttachments = imageAttachmentsSnapshot.length > 0; + const hasVideoAttachments = videoAttachmentsSnapshot.length > 0; + const openClawMediaAttachments = normalizeOpenClawChatMediaAttachments( + sanitizedAttachmentsSnapshot.map((attachment) => ({ + type: isOpenClawChatVideoAttachment( + attachment.url, + attachment.contentType + ) + ? "video" + : "image", + url: attachment.url, + name: normalizeChatAttachmentName(attachment.name), + content_type: attachment.contentType, + })) + ); if ( !textContent && !hasTableFiles && !hasImageAttachments && + !hasVideoAttachments && !hasPendingClawdSkillTagPrompt ) { return; @@ -11052,14 +11096,22 @@ export function CanvasChatPanel({ textContent || (clawdSkillTagForRequest && hasPendingClawdSkillTagPrompt ? `请按「${clawdSkillTagForRequest.name}」快捷功能处理。` - : getClawdDefaultPrompt(hasTableFiles, hasImageAttachments)); + : getClawdDefaultPrompt( + hasTableFiles, + hasImageAttachments, + hasVideoAttachments + )); const imageReferenceText = buildClawdImageReferenceText( imageAttachmentsSnapshot ); + const videoReferenceText = buildOpenClawVideoReferenceText( + openClawMediaAttachments + ); const requestMessage = [ baseRequestMessage, clawdSkillTagPrompt, imageReferenceText, + videoReferenceText, ] .filter(Boolean) .join("\n\n"); @@ -11078,6 +11130,7 @@ export function CanvasChatPanel({ const optimisticMessageText = [ buildClawdTableDisplayText(optimisticBaseMessage, tableFilesSnapshot), buildClawdImageDisplayText(imageAttachmentsSnapshot), + buildOpenClawVideoDisplayText(openClawMediaAttachments), ] .filter(Boolean) .join("\n\n"); @@ -11094,6 +11147,10 @@ export function CanvasChatPanel({ .map((attachment) => normalizeMediaUrl(attachment.url)) .filter(Boolean) .sort(), + videoAttachments: videoAttachmentsSnapshot + .map((attachment) => normalizeMediaUrl(attachment.url)) + .filter(Boolean) + .sort(), }); if (shouldBlockDuplicateSend(activeChatId, sendSignature)) { @@ -11104,7 +11161,8 @@ export function CanvasChatPanel({ const nextProcessingHint = shouldShowClawdProcessingHint( [textContent, clawdSkillTagPrompt].filter(Boolean).join("\n\n"), hasTableFiles, - hasImageAttachments + hasImageAttachments, + hasVideoAttachments ) ? CLAWD_PROCESSING_HINT : null; @@ -11207,11 +11265,18 @@ export function CanvasChatPanel({ composerMode: "clawd", activeChatId, imageAttachmentCount: imageAttachmentsSnapshot.length || undefined, + videoAttachmentCount: videoAttachmentsSnapshot.length || undefined, skillTagId: clawdSkillTagForRequest?.id || undefined, skillTagName: clawdSkillTagForRequest?.name || undefined, skillTagPromptIncluded: hasPendingClawdSkillTagPrompt || undefined, }) ); + if (openClawMediaAttachments.length > 0) { + formData.append( + "attachments", + JSON.stringify(openClawMediaAttachments) + ); + } tableFilesSnapshot.forEach((file) => { formData.append("tableFiles", file, file.name); }); @@ -12042,6 +12107,70 @@ export function CanvasChatPanel({ } }, [clearComposerUploadProgress, updateComposerUploadProgress]); + const uploadClawdVideoFileForComposerAttachment = useCallback(async ( + file: File, + progressOptions: CanvasUploadProgressOptions + ): Promise => { + if (!isOpenClawChatVideoFile(file)) { + toast.error("仅支持 MP4、MOV、WebM、M4V 或 MKV 视频"); + return null; + } + if (file.size <= 0) { + toast.error(`${file.name} 是空文件`); + return null; + } + if (file.size > OPENCLAW_CHAT_VIDEO_MAX_BYTES) { + toast.error(`${file.name} 超过 500MB，请先压缩视频`); + return null; + } + + try { + const normalizedContentType = + getOpenClawChatVideoContentType(file) || "video/mp4"; + const uploadFile = + file.type === normalizedContentType + ? file + : new File([file], file.name, { + type: normalizedContentType, + lastModified: file.lastModified, + }); + const formData = new FormData(); + formData.append("file", uploadFile); + formData.append("purpose", OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE); + updateComposerUploadProgress(progressOptions); + const response = await fetchUploadWithRetry(formData, { + onProgress: (progress) => { + updateComposerUploadProgress(progressOptions, progress); + }, + }); + const data = await response.json().catch(() => null); + if (!response.ok) { + throw new Error(String(data?.error || data?.details || "视频上传失败")); + } + + const imageUrl = String(data?.url || "").trim(); + if (!isValidChatHttpUrl(imageUrl)) { + throw new Error("储存桶未返回可访问的视频地址"); + } + + clearComposerUploadProgress(250); + return { + imageUrl, + name: file.name || `视频-${Date.now()}.mp4`, + contentType: + String(data?.contentType || "").trim() || + normalizedContentType, + assetId: + typeof data?.assetId === "string" && data.assetId.trim() + ? data.assetId.trim() + : undefined, + }; + } catch (error) { + toast.error(error instanceof Error ? error.message : "视频上传失败"); + return null; + } + }, [clearComposerUploadProgress, updateComposerUploadProgress]); + const handlePastedImages = useCallback(async (files: File[]) => { if (files.length === 0) return; @@ -12321,19 +12450,30 @@ export function CanvasChatPanel({ if (isClawdMode) { const tableFiles = selectedFiles.filter((file) => isClawdTableFile(file)); - const imageFiles = selectedFiles.filter((file) => isClawdImageFile(file)); + const imageFiles = selectedFiles.filter((file) => + file.type.startsWith("image/") + ); + const videoFiles = selectedFiles.filter((file) => + isOpenClawChatVideoFile(file) + ); const unsupportedFiles = selectedFiles.filter( - (file) => !isClawdTableFile(file) && !isClawdImageFile(file) + (file) => + !isClawdTableFile(file) && + !file.type.startsWith("image/") && + !isOpenClawChatVideoFile(file) ); unsupportedFiles.forEach((file) => { - toast.error(`${file.name} 格式不支持，请上传图片或 .xlsx/.csv/.tsv`); + toast.error( + `${file.name} 格式不支持，请上传图片、视频或 .xlsx/.csv/.tsv` + ); }); if (tableFiles.length > 0) { addClawdTableFiles(tableFiles); } let handledImageCount = 0; + let handledVideoCount = 0; if (imageFiles.length > 0) { try { for (const [index, file] of imageFiles.entries()) { @@ -12362,6 +12502,49 @@ export function CanvasChatPanel({ } } + if (videoFiles.length > 0) { + const existingVideoCount = attachments.filter((attachment) => + isOpenClawChatVideoAttachment( + attachment.url, + attachment.contentType + ) + ).length; + const availableVideoSlots = Math.max( + 0, + OPENCLAW_CHAT_VIDEO_MAX_FILES - existingVideoCount + ); + const acceptedVideoFiles = videoFiles.slice(0, availableVideoSlots); + if (acceptedVideoFiles.length < videoFiles.length) { + toast.info( + `${CLAW_MODE_LABEL} 对话一次最多添加 ${OPENCLAW_CHAT_VIDEO_MAX_FILES} 个视频` + ); + } + + try { + for (const [index, file] of acceptedVideoFiles.entries()) { + const uploaded = await uploadClawdVideoFileForComposerAttachment( + file, + { + title: + acceptedVideoFiles.length === 1 + ? "正在上传视频到储存桶" + : `正在上传 ${acceptedVideoFiles.length} 个视频`, + fileName: file.name, + fileIndex: index, + fileTotal: acceptedVideoFiles.length, + target: "composer-reference", + } + ); + if (!uploaded) continue; + + handledVideoCount += 1; + addUploadedImageAttachmentToComposer(uploaded); + } + } finally { + clearComposerUploadProgress(450); + } + } + if (handledImageCount > 0) { toast.success( handledImageCount === 1 @@ -12369,6 +12552,13 @@ export function CanvasChatPanel({ : `已添加 ${handledImageCount} 张图片为 ${CLAW_MODE_LABEL} 附件` ); } + if (handledVideoCount > 0) { + toast.success( + handledVideoCount === 1 + ? `视频已添加为 ${CLAW_MODE_LABEL} 附件` + : `已添加 ${handledVideoCount} 个视频为 ${CLAW_MODE_LABEL} 附件` + ); + } e.target.value = ""; return; } @@ -12746,7 +12936,7 @@ export function CanvasChatPanel({ ? activeClawdSkillTag ? activeClawdSkillTag.placeholder?.trim() || `继续向 ${CLAW_MODE_LABEL} 提问，当前使用「${activeClawdSkillTag.name}」` - : `输入要交给 ${CLAW_MODE_LABEL} 的问题，可上传图片或表格` + : `输入要交给 ${CLAW_MODE_LABEL} 的问题，可上传图片、视频或表格` : isDetailGenerationMode ? "可选补充：风格、情绪氛围、禁用元素、期望生成张数（如：生成8张）" : isVideoWorkspace @@ -12760,7 +12950,7 @@ export function CanvasChatPanel({ ? activeClawdSkillTag ? activeClawdSkillTag.placeholder?.trim() || `继续向 ${CLAW_MODE_LABEL} 提问，当前使用「${activeClawdSkillTag.name}」` - : `输入要交给 ${CLAW_MODE_LABEL} 的问题，可上传图片或表格` + : `输入要交给 ${CLAW_MODE_LABEL} 的问题，可上传图片、视频或表格` : isDetailGenerationMode ? "可选补充：风格、情绪氛围、禁用元素、期望生成张数（如：生成8张）" : isVideoWorkspace @@ -16125,7 +16315,7 @@ export function CanvasChatPanel({ type="file" accept={ isClawdMode - ? `${CLAWD_IMAGE_ACCEPT},${CLAWD_TABLE_ACCEPT}` + ? `${CLAWD_IMAGE_ACCEPT},${OPENCLAW_CHAT_VIDEO_ACCEPT},${CLAWD_TABLE_ACCEPT}` : supportsAtlasVideoReference ? "image/*,video/mp4,video/quicktime,video/webm" : "image/*" diff --git a/lib/client/upload-with-retry.ts b/lib/client/upload-with-retry.ts index 4690aad1c..09a816e44 100644 --- a/lib/client/upload-with-retry.ts +++ b/lib/client/upload-with-retry.ts @@ -1,3 +1,5 @@ +import { OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE } from "@/lib/openclaw-chat/media-attachments"; + const DEFAULT_MAX_RETRIES = 3; const MAX_BACKOFF_MS = 8000; const MAX_CONCURRENT_UPLOADS = 3; @@ -30,6 +32,13 @@ const DIRECT_UPLOAD_IMAGE_TYPES = new Set([ "image/png", "image/webp", ]); +const DIRECT_UPLOAD_OPENCLAW_VIDEO_TYPES = new Set([ + "video/mp4", + "video/quicktime", + "video/webm", + "video/x-m4v", + "video/x-matroska", +]); const BROWSER_IMAGE_TYPES_BY_EXTENSION: Record = { jpg: "image/jpeg", @@ -298,7 +307,12 @@ function shouldTryDirectUpload(file: UploadFormFile) { if (file.blob.size <= 0 || file.blob.size > DIRECT_UPLOAD_MAX_BYTES) { return false; } - if (!DIRECT_UPLOAD_IMAGE_TYPES.has(file.contentType.toLowerCase())) { + const normalizedContentType = file.contentType.toLowerCase(); + const supportsDirectUpload = + DIRECT_UPLOAD_IMAGE_TYPES.has(normalizedContentType) || + (file.purpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE && + DIRECT_UPLOAD_OPENCLAW_VIDEO_TYPES.has(normalizedContentType)); + if (!supportsDirectUpload) { return false; } if (file.purpose === "agent-workflow-preview") { @@ -848,7 +862,7 @@ async function fetchDirectUpload( return createJsonResponse( { error: - "图片超过 50MB，必须使用浏览器直传；当前直传失败，请刷新后重试，或联系管理员检查存储桶 CORS/ETag 配置。", + "文件超过 50MB，必须使用浏览器直传；当前直传失败，请刷新后重试，或联系管理员检查存储桶 CORS/ETag 配置。", details: error instanceof Error ? error.message : String(error), clientUploadId: file.clientUploadId, }, diff --git a/lib/openclaw-chat/media-attachments.ts b/lib/openclaw-chat/media-attachments.ts new file mode 100644 index 000000000..0156b52e9 --- /dev/null +++ b/lib/openclaw-chat/media-attachments.ts @@ -0,0 +1,183 @@ +export const OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE = "openclaw-chat-video"; +export const OPENCLAW_CHAT_VIDEO_MAX_FILES = 2; +export const OPENCLAW_CHAT_VIDEO_MAX_BYTES = 500 * 1024 * 1024; +export const OPENCLAW_CHAT_MEDIA_MAX_ATTACHMENTS = 16; +export const OPENCLAW_CHAT_VIDEO_ACCEPT = + "video/mp4,video/quicktime,video/webm,video/x-m4v,video/x-matroska,.mp4,.mov,.webm,.m4v,.mkv"; + +const VIDEO_CONTENT_TYPES = new Set([ + "video/mp4", + "video/quicktime", + "video/webm", + "video/x-m4v", + "video/x-matroska", +]); + +const VIDEO_CONTENT_TYPE_BY_EXTENSION: Record = { + m4v: "video/x-m4v", + mkv: "video/x-matroska", + mov: "video/quicktime", + mp4: "video/mp4", + webm: "video/webm", +}; + +const VIDEO_EXTENSION_PATTERN = /\.(mp4|mov|webm|m4v|mkv)(?:$|[?#])/i; + +export type OpenClawChatMediaAttachment = { + type: "image" | "video"; + url: string; + name?: string; + content_type?: string; +}; + +type OpenClawChatAttachmentLike = { + type?: unknown; + url?: unknown; + name?: unknown; + contentType?: unknown; + content_type?: unknown; +}; + +type OpenClawChatVideoFileLike = { + name: string; + type: string; +}; + +function normalizeContentType(value: unknown) { + return String(value || "") + .split(";", 1)[0] + .trim() + .toLowerCase(); +} + +function getExtension(value: string) { + const cleanValue = String(value || "").split(/[?#]/, 1)[0]; + return cleanValue.split(".").pop()?.trim().toLowerCase() || ""; +} + +function isHttpUrl(value: string) { + try { + const parsed = new URL(value); + return parsed.protocol === "http:" || parsed.protocol === "https:"; + } catch { + return false; + } +} + +export function getOpenClawChatVideoContentType( + file: OpenClawChatVideoFileLike +) { + const contentType = normalizeContentType(file.type); + if (VIDEO_CONTENT_TYPES.has(contentType)) { + return contentType; + } + return VIDEO_CONTENT_TYPE_BY_EXTENSION[getExtension(file.name)] || ""; +} + +export function isOpenClawChatVideoFile(file: OpenClawChatVideoFileLike) { + return Boolean(getOpenClawChatVideoContentType(file)); +} + +export function isOpenClawChatVideoAttachment( + url: string, + contentType?: string | null +) { + return ( + normalizeContentType(contentType).startsWith("video/") || + VIDEO_EXTENSION_PATTERN.test(String(url || "")) + ); +} + +export function normalizeOpenClawChatMediaAttachments( + value: unknown +): OpenClawChatMediaAttachment[] { + if (!Array.isArray(value)) { + return []; + } + + const result: OpenClawChatMediaAttachment[] = []; + const seenUrls = new Set(); + let videoCount = 0; + + for (const item of value) { + if (!item || typeof item !== "object") { + continue; + } + + const raw = item as OpenClawChatAttachmentLike; + const url = String(raw.url || "").trim(); + if (!url || !isHttpUrl(url) || seenUrls.has(url)) { + continue; + } + + const requestedType = String(raw.type || "").trim().toLowerCase(); + const rawContentType = normalizeContentType( + raw.content_type || raw.contentType + ); + const isVideo = + requestedType === "video" || + isOpenClawChatVideoAttachment(url, rawContentType); + + if (isVideo && videoCount >= OPENCLAW_CHAT_VIDEO_MAX_FILES) { + continue; + } + if (result.length >= OPENCLAW_CHAT_MEDIA_MAX_ATTACHMENTS) { + break; + } + + const type = isVideo ? "video" : "image"; + const fallbackContentType = isVideo + ? VIDEO_CONTENT_TYPE_BY_EXTENSION[getExtension(url)] || "video/mp4" + : "image/jpeg"; + const contentType = + rawContentType.startsWith(`${type}/`) ? rawContentType : fallbackContentType; + const name = String(raw.name || "") + .trim() + .slice(0, 200); + + result.push({ + type, + url, + ...(name ? { name } : {}), + content_type: contentType, + }); + seenUrls.add(url); + if (isVideo) { + videoCount += 1; + } + } + + return result; +} + +export function buildOpenClawVideoDisplayText( + attachments: OpenClawChatMediaAttachment[] +) { + const videos = attachments.filter((attachment) => attachment.type === "video"); + if (videos.length === 0) { + return ""; + } + + const lines = videos.map( + (attachment, index) => `- ${attachment.name || `视频${index + 1}`}` + ); + return `已上传视频：\n${lines.join("\n")}`; +} + +export function buildOpenClawVideoReferenceText( + attachments: OpenClawChatMediaAttachment[] +) { + const videos = attachments.filter((attachment) => attachment.type === "video"); + if (videos.length === 0) { + return ""; + } + + const lines = videos.map( + (attachment, index) => + `- ${attachment.name || `视频${index + 1}`}: ${attachment.url}` + ); + return [ + "用户上传了以下视频，请调用视频理解能力并按视频文件地址处理：", + lines.join("\n"), + ].join("\n"); +} diff --git a/lib/storage/service.ts b/lib/storage/service.ts index 7e6da8547..d90b4dc5f 100644 --- a/lib/storage/service.ts +++ b/lib/storage/service.ts @@ -129,6 +129,11 @@ export function getContentType(filename: string): string { woff2: "font/woff2", glb: "model/gltf-binary", gltf: "model/gltf+json", + mp4: "video/mp4", + mov: "video/quicktime", + webm: "video/webm", + m4v: "video/x-m4v", + mkv: "video/x-matroska", usdz: "model/vnd.usdz+zip", fbx: "application/octet-stream", obj: "text/plain", diff --git a/playwright.helpers.config.ts b/playwright.helpers.config.ts index b0d63cfa4..d44dbaac4 100644 --- a/playwright.helpers.config.ts +++ b/playwright.helpers.config.ts @@ -14,6 +14,7 @@ export default defineConfig({ /routes\/canvas-runtime-internal-endpoint\.test\.ts/, /routes\/model-classification\.test\.ts/, /routes\/clawd-request-lifecycle\.test\.ts/, + /routes\/openclaw-video-attachments\.test\.ts/, /routes\/hicad-embedded-performance\.test\.ts/, /routes\/hicad-manufacturing-extension\.test\.ts/, ], diff --git a/tests/routes/openclaw-video-attachments.test.ts b/tests/routes/openclaw-video-attachments.test.ts new file mode 100644 index 000000000..8dddad8f6 --- /dev/null +++ b/tests/routes/openclaw-video-attachments.test.ts @@ -0,0 +1,92 @@ +import { expect, test } from "@playwright/test"; +import { + buildOpenClawVideoDisplayText, + buildOpenClawVideoReferenceText, + getOpenClawChatVideoContentType, + isOpenClawChatVideoFile, + normalizeOpenClawChatMediaAttachments, + OPENCLAW_CHAT_VIDEO_MAX_FILES, +} from "@/lib/openclaw-chat/media-attachments"; + +test.describe("OpenClaw video attachments", () => { + test("accepts the supported browser video formats by MIME type or extension", () => { + expect( + isOpenClawChatVideoFile({ + name: "市场分析.mp4", + type: "video/mp4", + }) + ).toBe(true); + expect( + isOpenClawChatVideoFile({ + name: "素材.MOV", + type: "application/octet-stream", + }) + ).toBe(true); + expect( + getOpenClawChatVideoContentType({ + name: "演示.m4v", + type: "", + }) + ).toBe("video/x-m4v"); + expect( + isOpenClawChatVideoFile({ + name: "说明.pdf", + type: "application/pdf", + }) + ).toBe(false); + }); + + test("normalizes public video URLs to the OpenClaw attachment contract", () => { + expect( + normalizeOpenClawChatMediaAttachments([ + { + type: "video", + url: "https://cdn.example.com/demo.mp4", + name: "demo.mp4", + contentType: "video/mp4", + }, + ]) + ).toEqual([ + { + type: "video", + url: "https://cdn.example.com/demo.mp4", + name: "demo.mp4", + content_type: "video/mp4", + }, + ]); + }); + + test("rejects private schemes, duplicates, and videos beyond the gateway limit", () => { + const attachments = normalizeOpenClawChatMediaAttachments([ + { type: "video", url: "https://cdn.example.com/one.mp4" }, + { type: "video", url: "https://cdn.example.com/two.mov" }, + { type: "video", url: "https://cdn.example.com/three.webm" }, + { type: "video", url: "file:///tmp/private.mp4" }, + { type: "video", url: "https://cdn.example.com/one.mp4" }, + ]); + + expect(attachments).toHaveLength(OPENCLAW_CHAT_VIDEO_MAX_FILES); + expect(attachments.map((attachment) => attachment.url)).toEqual([ + "https://cdn.example.com/one.mp4", + "https://cdn.example.com/two.mov", + ]); + }); + + test("builds a readable fallback message without exposing storage credentials", () => { + const attachments = normalizeOpenClawChatMediaAttachments([ + { + type: "video", + url: "https://cdn.example.com/review.mp4", + name: "review.mp4", + }, + ]); + + expect(buildOpenClawVideoDisplayText(attachments)).toContain("review.mp4"); + expect(buildOpenClawVideoReferenceText(attachments)).toContain( + "https://cdn.example.com/review.mp4" + ); + expect(buildOpenClawVideoReferenceText(attachments)).not.toContain( + "Authorization" + ); + }); +});