feat: support video attachments in Claw chat

This commit is contained in:
134 2026-07-28 15:50:47 +08:00
parent 21ba23c4ab
commit fb8d8232dc
9 changed files with 650 additions and 45 deletions

View File

@ -21,6 +21,7 @@ import {
maybeEnqueueStorageMigrationTask, maybeEnqueueStorageMigrationTask,
resolveS3UploadTarget, resolveS3UploadTarget,
} from "@/lib/storage/service"; } from "@/lib/storage/service";
import { OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE } from "@/lib/openclaw-chat/media-attachments";
const logger = createLogger("direct-upload-api"); const logger = createLogger("direct-upload-api");
const DIRECT_UPLOAD_MAX_BYTES = 500 * 1024 * 1024; const DIRECT_UPLOAD_MAX_BYTES = 500 * 1024 * 1024;
@ -39,6 +40,13 @@ const allowedDirectUploadTypes = new Set([
"image/png", "image/png",
"image/webp", "image/webp",
]); ]);
const allowedOpenClawVideoUploadTypes = new Set([
"video/mp4",
"video/quicktime",
"video/webm",
"video/x-m4v",
"video/x-matroska",
]);
const InitiateSchema = z.object({ const InitiateSchema = z.object({
action: z.literal("initiate"), action: z.literal("initiate"),
@ -109,8 +117,12 @@ function assertUserOwnedKey(key: string, userId: string) {
return normalized; return normalized;
} }
function isAllowedDirectUpload(contentType: string) { function isAllowedDirectUpload(contentType: string, purpose?: string) {
return allowedDirectUploadTypes.has(contentType); return (
allowedDirectUploadTypes.has(contentType) ||
(purpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE &&
allowedOpenClawVideoUploadTypes.has(contentType))
);
} }
function computePartSize(size: number) { function computePartSize(size: number) {
@ -204,7 +216,7 @@ export async function POST(request: Request) {
payload.filename payload.filename
); );
if (!isAllowedDirectUpload(contentType)) { if (!isAllowedDirectUpload(contentType, payload.purpose)) {
logger.warn("unsupported_direct_upload_type", { logger.warn("unsupported_direct_upload_type", {
contentType, contentType,
filenameExtension: payload.filename.toLowerCase().split(".").pop() || null, filenameExtension: payload.filename.toLowerCase().split(".").pop() || null,
@ -213,7 +225,10 @@ export async function POST(request: Request) {
clientUploadId: payload.clientUploadId || null, clientUploadId: payload.clientUploadId || null,
}); });
return NextResponse.json( return NextResponse.json(
{ error: "Direct upload only supports JPG, PNG and WebP images" }, {
error:
"直传仅支持 JPG、PNG、WebP 图片以及 Claw 对话的 MP4、MOV、WebM、M4V、MKV 视频",
},
{ status: 400 } { status: 400 }
); );
} }
@ -328,9 +343,12 @@ export async function POST(request: Request) {
const key = assertUserOwnedKey(payload.key, session.user.id); const key = assertUserOwnedKey(payload.key, session.user.id);
const contentType = normalizeContentType(payload.contentType, payload.filename); const contentType = normalizeContentType(payload.contentType, payload.filename);
if (!isAllowedDirectUpload(contentType)) { if (!isAllowedDirectUpload(contentType, payload.purpose)) {
return NextResponse.json( return NextResponse.json(
{ error: "Direct upload only supports JPG, PNG and WebP images" }, {
error:
"直传仅支持 JPG、PNG、WebP 图片以及 Claw 对话的 MP4、MOV、WebM、M4V、MKV 视频",
},
{ status: 400 } { status: 400 }
); );
} }
@ -389,15 +407,21 @@ export async function POST(request: Request) {
const assetStartedAt = Date.now(); const assetStartedAt = Date.now();
const asset = await upsertMediaAsset({ const asset = await upsertMediaAsset({
userId: session.user.id, userId: session.user.id,
mediaType: "image", mediaType: resolvedContentType.startsWith("video/") ? "video" : "image",
sourceType: "upload", sourceType:
payload.purpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE
? "openclaw_chat_video_upload"
: "upload",
status: "ready", status: "ready",
originalUrl: url, originalUrl: url,
previewUrl: url, previewUrl: url,
thumbnailUrl: url, thumbnailUrl: url,
mimeType: resolvedContentType, mimeType: resolvedContentType,
displayName: payload.filename, displayName: payload.filename,
sourceTable: "direct_upload", sourceTable:
payload.purpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE
? "openclaw_chat_video_upload"
: "direct_upload",
metadata: { metadata: {
pathname: key, pathname: key,
size: actualSize ?? payload.size, size: actualSize ?? payload.size,

View File

@ -7,6 +7,7 @@ import { upsertMediaAsset } from "@/lib/db/queries";
import { isSiteAdminType } from "@/lib/middleware/admin"; import { isSiteAdminType } from "@/lib/middleware/admin";
import { getContentType, uploadFile } from "@/lib/storage/service"; import { getContentType, uploadFile } from "@/lib/storage/service";
import { createLogger } from "@/lib/observability/logger"; import { createLogger } from "@/lib/observability/logger";
import { OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE } from "@/lib/openclaw-chat/media-attachments";
const logger = createLogger("upload-api"); const logger = createLogger("upload-api");
const ALLOWED_UPLOAD_IMAGE_TYPES = new Set([ const ALLOWED_UPLOAD_IMAGE_TYPES = new Set([
@ -26,6 +27,13 @@ const ALLOWED_ATLASCLOUD_VIDEO_TYPES = new Set([
"video/quicktime", "video/quicktime",
"video/webm", "video/webm",
]); ]);
const ALLOWED_OPENCLAW_VIDEO_TYPES = new Set([
"video/mp4",
"video/quicktime",
"video/webm",
"video/x-m4v",
"video/x-matroska",
]);
const HEIC_UPLOAD_TYPES = new Set(["image/heic", "image/heif"]); const HEIC_UPLOAD_TYPES = new Set(["image/heic", "image/heif"]);
const SHARP_COMPATIBILITY_UPLOAD_TYPES = new Set([ const SHARP_COMPATIBILITY_UPLOAD_TYPES = new Set([
@ -246,6 +254,9 @@ export async function POST(request: Request) {
const isAtlasCloudReferenceVideo = const isAtlasCloudReferenceVideo =
uploadPurpose === "canvas-video-process" && uploadPurpose === "canvas-video-process" &&
ALLOWED_ATLASCLOUD_VIDEO_TYPES.has(contentType); ALLOWED_ATLASCLOUD_VIDEO_TYPES.has(contentType);
const isOpenClawChatVideoUpload =
uploadPurpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE &&
ALLOWED_OPENCLAW_VIDEO_TYPES.has(contentType);
const clientUploadId = const clientUploadId =
String(formData.get("clientUploadId") || "").trim() || null; String(formData.get("clientUploadId") || "").trim() || null;
const isSvgUpload = contentType === "image/svg+xml"; const isSvgUpload = contentType === "image/svg+xml";
@ -278,7 +289,8 @@ export async function POST(request: Request) {
if ( if (
!ALLOWED_UPLOAD_IMAGE_TYPES.has(contentType) && !ALLOWED_UPLOAD_IMAGE_TYPES.has(contentType) &&
!isAtlasCloudReferenceVideo !isAtlasCloudReferenceVideo &&
!isOpenClawChatVideoUpload
) { ) {
const errorMessage = createUnsupportedImageTypeMessage(contentType, filename); const errorMessage = createUnsupportedImageTypeMessage(contentType, filename);
logger.warn("[Upload API] Unsupported upload content type", { logger.warn("[Upload API] Unsupported upload content type", {
@ -445,15 +457,28 @@ export async function POST(request: Request) {
try { try {
const asset = await upsertMediaAsset({ const asset = await upsertMediaAsset({
userId: session.user.id, userId: session.user.id,
mediaType: isAtlasCloudReferenceVideo ? "video" : "image", mediaType:
sourceType: "upload", isAtlasCloudReferenceVideo || isOpenClawChatVideoUpload
? "video"
: "image",
sourceType: isOpenClawChatVideoUpload
? "openclaw_chat_video_upload"
: "upload",
status: "ready", status: "ready",
originalUrl: data.url, originalUrl: data.url,
previewUrl: isAtlasCloudReferenceVideo ? null : data.url, previewUrl:
thumbnailUrl: isAtlasCloudReferenceVideo ? null : data.url, isAtlasCloudReferenceVideo || isOpenClawChatVideoUpload
? null
: data.url,
thumbnailUrl:
isAtlasCloudReferenceVideo || isOpenClawChatVideoUpload
? null
: data.url,
mimeType: data.contentType || contentType || null, mimeType: data.contentType || contentType || null,
displayName: filename, displayName: filename,
sourceTable: "upload", sourceTable: isOpenClawChatVideoUpload
? "openclaw_chat_video_upload"
: "upload",
metadata: { metadata: {
pathname: data.pathname, pathname: data.pathname,
size: optimizedBuffer.length, size: optimizedBuffer.length,

View File

@ -25,6 +25,11 @@ import {
buildOpenClawMessageWithSpreadsheetMarkdown, buildOpenClawMessageWithSpreadsheetMarkdown,
buildOpenClawSpreadsheetMarkdown, buildOpenClawSpreadsheetMarkdown,
} from "@/lib/openclaw-chat/spreadsheet-markdown"; } from "@/lib/openclaw-chat/spreadsheet-markdown";
import {
normalizeOpenClawChatMediaAttachments,
OPENCLAW_CHAT_MEDIA_MAX_ATTACHMENTS,
type OpenClawChatMediaAttachment,
} from "@/lib/openclaw-chat/media-attachments";
import { generateUUID } from "@/lib/utils"; import { generateUUID } from "@/lib/utils";
const CLAW_UPGRADE_RETRY_MESSAGE = "claw正在升级更新稍等5分钟再试"; const CLAW_UPGRADE_RETRY_MESSAGE = "claw正在升级更新稍等5分钟再试";
@ -35,9 +40,18 @@ const requestSchema = z.object({
displayMessage: z.string().trim().max(8000).optional(), displayMessage: z.string().trim().max(8000).optional(),
userMessageId: z.string().uuid().optional(), userMessageId: z.string().uuid().optional(),
metadata: z.record(z.unknown()).optional(), metadata: z.record(z.unknown()).optional(),
attachments: z
.array(z.unknown())
.max(OPENCLAW_CHAT_MEDIA_MAX_ATTACHMENTS)
.optional()
.default([]),
}); });
type ParsedClawdChatRequest = z.infer<typeof requestSchema> & { type ParsedClawdChatRequest = Omit<
z.infer<typeof requestSchema>,
"attachments"
> & {
attachments: OpenClawChatMediaAttachment[];
tableFiles: File[]; tableFiles: File[];
}; };
@ -85,14 +99,23 @@ async function parseClawdChatRequest(
const contentType = request.headers.get("content-type") || ""; const contentType = request.headers.get("content-type") || "";
if (!contentType.includes("multipart/form-data")) { if (!contentType.includes("multipart/form-data")) {
const parsed = requestSchema.safeParse(await request.json().catch(() => null)); const parsed = requestSchema.safeParse(await request.json().catch(() => null));
return parsed.success ? { ...parsed.data, tableFiles: [] } : null; if (!parsed.success) return null;
const attachments = normalizeOpenClawChatMediaAttachments(
parsed.data.attachments
);
if (attachments.length !== parsed.data.attachments.length) {
return null;
}
return { ...parsed.data, attachments, tableFiles: [] };
} }
const formData = await request.formData().catch(() => null); const formData = await request.formData().catch(() => null);
if (!formData) return null; if (!formData) return null;
const metadataValue = formData.get("metadata"); const metadataValue = formData.get("metadata");
const attachmentsValue = formData.get("attachments");
let metadata: Record<string, unknown> | undefined; let metadata: Record<string, unknown> | undefined;
let attachments: unknown[] = [];
if (typeof metadataValue === "string" && metadataValue.trim()) { if (typeof metadataValue === "string" && metadataValue.trim()) {
try { try {
const parsedMetadata = JSON.parse(metadataValue); const parsedMetadata = JSON.parse(metadataValue);
@ -105,6 +128,17 @@ async function parseClawdChatRequest(
return null; return null;
} }
} }
if (typeof attachmentsValue === "string" && attachmentsValue.trim()) {
try {
const parsedAttachments = JSON.parse(attachmentsValue);
if (!Array.isArray(parsedAttachments)) {
return null;
}
attachments = parsedAttachments;
} catch {
return null;
}
}
const parsed = requestSchema.safeParse({ const parsed = requestSchema.safeParse({
id: formData.get("id"), id: formData.get("id"),
@ -112,14 +146,25 @@ async function parseClawdChatRequest(
displayMessage: formData.get("displayMessage") || undefined, displayMessage: formData.get("displayMessage") || undefined,
userMessageId: formData.get("userMessageId") || undefined, userMessageId: formData.get("userMessageId") || undefined,
metadata, metadata,
attachments,
}); });
if (!parsed.success) return null; if (!parsed.success) return null;
const normalizedAttachments = normalizeOpenClawChatMediaAttachments(
parsed.data.attachments
);
if (normalizedAttachments.length !== parsed.data.attachments.length) {
return null;
}
const tableFiles = formData const tableFiles = formData
.getAll("tableFiles") .getAll("tableFiles")
.filter((entry): entry is File => isRequestFile(entry) && entry.size > 0); .filter((entry): entry is File => isRequestFile(entry) && entry.size > 0);
return { ...parsed.data, tableFiles }; return {
...parsed.data,
attachments: normalizedAttachments,
tableFiles,
};
} }
async function readOpenClawConfig() { async function readOpenClawConfig() {
@ -163,8 +208,15 @@ export async function POST(request: Request) {
if (!parsed) { if (!parsed) {
return NextResponse.json({ error: "请求参数不正确" }, { status: 400 }); return NextResponse.json({ error: "请求参数不正确" }, { status: 400 });
} }
if (!parsed.message && parsed.tableFiles.length === 0) { if (
return NextResponse.json({ error: "请输入问题或上传表格" }, { status: 400 }); !parsed.message &&
parsed.tableFiles.length === 0 &&
parsed.attachments.length === 0
) {
return NextResponse.json(
{ error: "请输入问题或上传附件" },
{ status: 400 }
);
} }
const config = await readOpenClawConfig(); const config = await readOpenClawConfig();
@ -275,6 +327,8 @@ export async function POST(request: Request) {
skill_tag_id: skillTagId || undefined, skill_tag_id: skillTagId || undefined,
skill_tag_name: skillTagName || undefined, skill_tag_name: skillTagName || undefined,
message: upstreamMessage, message: upstreamMessage,
attachments:
parsed.attachments.length > 0 ? parsed.attachments : undefined,
metadata: { metadata: {
source: "ai_chatbot_canvas", source: "ai_chatbot_canvas",
page: "/canvas", page: "/canvas",
@ -288,6 +342,8 @@ export async function POST(request: Request) {
user_department: openClawDepartmentName, user_department: openClawDepartmentName,
skill_tag_id: skillTagId || undefined, skill_tag_id: skillTagId || undefined,
skill_tag_name: skillTagName || undefined, skill_tag_name: skillTagName || undefined,
media_attachments:
parsed.attachments.length > 0 ? parsed.attachments : undefined,
spreadsheet_files: spreadsheetMarkdownResult?.files || undefined, spreadsheet_files: spreadsheetMarkdownResult?.files || undefined,
spreadsheet_markdown_truncated: spreadsheet_markdown_truncated:
spreadsheetMarkdownResult?.truncated || undefined, spreadsheetMarkdownResult?.truncated || undefined,
@ -378,7 +434,15 @@ export async function POST(request: Request) {
chatId: parsed.id, chatId: parsed.id,
role: "user", role: "user",
parts: [{ type: "text", text: displayMessage }], parts: [{ type: "text", text: displayMessage }],
attachments: [], attachments: parsed.attachments.map((attachment) => ({
name:
attachment.name ||
(attachment.type === "video" ? "视频附件" : "图片附件"),
url: attachment.url,
contentType:
attachment.content_type ||
(attachment.type === "video" ? "video/mp4" : "image/jpeg"),
})),
createdAt: userCreatedAt, createdAt: userCreatedAt,
videoStorageStatus: null, videoStorageStatus: null,
temporaryVideoUrl: null, temporaryVideoUrl: null,
@ -387,6 +451,12 @@ export async function POST(request: Request) {
videoMetadata: { videoMetadata: {
composerMode: "clawd", composerMode: "clawd",
source: "openclaw", source: "openclaw",
imageAttachmentCount: parsed.attachments.filter(
(attachment) => attachment.type === "image"
).length,
videoAttachmentCount: parsed.attachments.filter(
(attachment) => attachment.type === "video"
).length,
skillTagId: skillTagId || null, skillTagId: skillTagId || null,
skillTagName: skillTagName || null, skillTagName: skillTagName || null,
}, },
@ -431,6 +501,7 @@ export async function POST(request: Request) {
createdAt: userMessage.createdAt.toISOString(), createdAt: userMessage.createdAt.toISOString(),
composerMode: "clawd", composerMode: "clawd",
}, },
attachments: userMessage.attachments,
videoMetadata: userMessage.videoMetadata, videoMetadata: userMessage.videoMetadata,
}, },
{ {

View File

@ -287,6 +287,18 @@ import {
stripCanvasModelRoutingLines, stripCanvasModelRoutingLines,
stripMediaMarkup, stripMediaMarkup,
} from "@/components/canvas/chat-panel/canvas-chat-media"; } from "@/components/canvas/chat-panel/canvas-chat-media";
import {
buildOpenClawVideoDisplayText,
buildOpenClawVideoReferenceText,
getOpenClawChatVideoContentType,
isOpenClawChatVideoAttachment,
isOpenClawChatVideoFile,
normalizeOpenClawChatMediaAttachments,
OPENCLAW_CHAT_VIDEO_ACCEPT,
OPENCLAW_CHAT_VIDEO_MAX_BYTES,
OPENCLAW_CHAT_VIDEO_MAX_FILES,
OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE,
} from "@/lib/openclaw-chat/media-attachments";
type ResponseModule = typeof import("@/components/elements/response"); type ResponseModule = typeof import("@/components/elements/response");
type ResponseProps = ComponentProps<ResponseModule["Response"]>; type ResponseProps = ComponentProps<ResponseModule["Response"]>;
@ -612,9 +624,10 @@ function writeStoredClawdComposerSubmissionSnapshot(
function shouldShowClawdProcessingHint( function shouldShowClawdProcessingHint(
message: string, message: string,
hasTableFiles: boolean, hasTableFiles: boolean,
hasImageAttachments: boolean hasImageAttachments: boolean,
hasVideoAttachments: boolean
) { ) {
if (hasTableFiles || hasImageAttachments) { if (hasTableFiles || hasImageAttachments || hasVideoAttachments) {
return true; return true;
} }
@ -675,17 +688,19 @@ function buildClawdImageReferenceText(attachments: PickedObjectAttachment[]) {
return `用户还添加了以下图片作为参考,请按图片链接理解:\n${lines.join("\n")}`; return `用户还添加了以下图片作为参考,请按图片链接理解:\n${lines.join("\n")}`;
} }
function getClawdDefaultPrompt(hasTables: boolean, hasImages: boolean) { function getClawdDefaultPrompt(
if (hasTables && hasImages) { hasTables: boolean,
return "请分析上传的表格和图片内容"; hasImages: boolean,
} hasVideos: boolean
if (hasTables) { ) {
return "请分析上传的表格内容"; const mediaNames = [
} hasTables ? "表格" : "",
if (hasImages) { hasImages ? "图片" : "",
return "请分析上传的图片内容"; hasVideos ? "视频" : "",
} ].filter(Boolean);
return "请分析"; return mediaNames.length > 0
? `请分析上传的${mediaNames.join("、")}内容`
: "请分析";
} }
type VideoStoryboardStreamData = type VideoStoryboardStreamData =
@ -11034,15 +11049,44 @@ export function CanvasChatPanel({
: ""; : "";
const hasPendingClawdSkillTagPrompt = Boolean(clawdSkillTagPrompt); const hasPendingClawdSkillTagPrompt = Boolean(clawdSkillTagPrompt);
const attachmentsSnapshot = isProgrammaticClawdQuickSend ? [] : [...attachments]; const attachmentsSnapshot = isProgrammaticClawdQuickSend ? [] : [...attachments];
const imageAttachmentsSnapshot = attachmentsSnapshot const sanitizedAttachmentsSnapshot = attachmentsSnapshot
.map((attachment) => sanitizeChatAttachment(attachment)) .map((attachment) => sanitizeChatAttachment(attachment))
.filter((attachment): attachment is PickedObjectAttachment => Boolean(attachment)); .filter((attachment): attachment is PickedObjectAttachment => Boolean(attachment));
const videoAttachmentsSnapshot = sanitizedAttachmentsSnapshot.filter(
(attachment) =>
isOpenClawChatVideoAttachment(
attachment.url,
attachment.contentType
)
);
const imageAttachmentsSnapshot = sanitizedAttachmentsSnapshot.filter(
(attachment) =>
!isOpenClawChatVideoAttachment(
attachment.url,
attachment.contentType
)
);
const hasImageAttachments = imageAttachmentsSnapshot.length > 0; const hasImageAttachments = imageAttachmentsSnapshot.length > 0;
const hasVideoAttachments = videoAttachmentsSnapshot.length > 0;
const openClawMediaAttachments = normalizeOpenClawChatMediaAttachments(
sanitizedAttachmentsSnapshot.map((attachment) => ({
type: isOpenClawChatVideoAttachment(
attachment.url,
attachment.contentType
)
? "video"
: "image",
url: attachment.url,
name: normalizeChatAttachmentName(attachment.name),
content_type: attachment.contentType,
}))
);
if ( if (
!textContent && !textContent &&
!hasTableFiles && !hasTableFiles &&
!hasImageAttachments && !hasImageAttachments &&
!hasVideoAttachments &&
!hasPendingClawdSkillTagPrompt !hasPendingClawdSkillTagPrompt
) { ) {
return; return;
@ -11052,14 +11096,22 @@ export function CanvasChatPanel({
textContent || textContent ||
(clawdSkillTagForRequest && hasPendingClawdSkillTagPrompt (clawdSkillTagForRequest && hasPendingClawdSkillTagPrompt
? `请按「${clawdSkillTagForRequest.name}」快捷功能处理。` ? `请按「${clawdSkillTagForRequest.name}」快捷功能处理。`
: getClawdDefaultPrompt(hasTableFiles, hasImageAttachments)); : getClawdDefaultPrompt(
hasTableFiles,
hasImageAttachments,
hasVideoAttachments
));
const imageReferenceText = buildClawdImageReferenceText( const imageReferenceText = buildClawdImageReferenceText(
imageAttachmentsSnapshot imageAttachmentsSnapshot
); );
const videoReferenceText = buildOpenClawVideoReferenceText(
openClawMediaAttachments
);
const requestMessage = [ const requestMessage = [
baseRequestMessage, baseRequestMessage,
clawdSkillTagPrompt, clawdSkillTagPrompt,
imageReferenceText, imageReferenceText,
videoReferenceText,
] ]
.filter(Boolean) .filter(Boolean)
.join("\n\n"); .join("\n\n");
@ -11078,6 +11130,7 @@ export function CanvasChatPanel({
const optimisticMessageText = [ const optimisticMessageText = [
buildClawdTableDisplayText(optimisticBaseMessage, tableFilesSnapshot), buildClawdTableDisplayText(optimisticBaseMessage, tableFilesSnapshot),
buildClawdImageDisplayText(imageAttachmentsSnapshot), buildClawdImageDisplayText(imageAttachmentsSnapshot),
buildOpenClawVideoDisplayText(openClawMediaAttachments),
] ]
.filter(Boolean) .filter(Boolean)
.join("\n\n"); .join("\n\n");
@ -11094,6 +11147,10 @@ export function CanvasChatPanel({
.map((attachment) => normalizeMediaUrl(attachment.url)) .map((attachment) => normalizeMediaUrl(attachment.url))
.filter(Boolean) .filter(Boolean)
.sort(), .sort(),
videoAttachments: videoAttachmentsSnapshot
.map((attachment) => normalizeMediaUrl(attachment.url))
.filter(Boolean)
.sort(),
}); });
if (shouldBlockDuplicateSend(activeChatId, sendSignature)) { if (shouldBlockDuplicateSend(activeChatId, sendSignature)) {
@ -11104,7 +11161,8 @@ export function CanvasChatPanel({
const nextProcessingHint = shouldShowClawdProcessingHint( const nextProcessingHint = shouldShowClawdProcessingHint(
[textContent, clawdSkillTagPrompt].filter(Boolean).join("\n\n"), [textContent, clawdSkillTagPrompt].filter(Boolean).join("\n\n"),
hasTableFiles, hasTableFiles,
hasImageAttachments hasImageAttachments,
hasVideoAttachments
) )
? CLAWD_PROCESSING_HINT ? CLAWD_PROCESSING_HINT
: null; : null;
@ -11207,11 +11265,18 @@ export function CanvasChatPanel({
composerMode: "clawd", composerMode: "clawd",
activeChatId, activeChatId,
imageAttachmentCount: imageAttachmentsSnapshot.length || undefined, imageAttachmentCount: imageAttachmentsSnapshot.length || undefined,
videoAttachmentCount: videoAttachmentsSnapshot.length || undefined,
skillTagId: clawdSkillTagForRequest?.id || undefined, skillTagId: clawdSkillTagForRequest?.id || undefined,
skillTagName: clawdSkillTagForRequest?.name || undefined, skillTagName: clawdSkillTagForRequest?.name || undefined,
skillTagPromptIncluded: hasPendingClawdSkillTagPrompt || undefined, skillTagPromptIncluded: hasPendingClawdSkillTagPrompt || undefined,
}) })
); );
if (openClawMediaAttachments.length > 0) {
formData.append(
"attachments",
JSON.stringify(openClawMediaAttachments)
);
}
tableFilesSnapshot.forEach((file) => { tableFilesSnapshot.forEach((file) => {
formData.append("tableFiles", file, file.name); formData.append("tableFiles", file, file.name);
}); });
@ -12042,6 +12107,70 @@ export function CanvasChatPanel({
} }
}, [clearComposerUploadProgress, updateComposerUploadProgress]); }, [clearComposerUploadProgress, updateComposerUploadProgress]);
const uploadClawdVideoFileForComposerAttachment = useCallback(async (
file: File,
progressOptions: CanvasUploadProgressOptions
): Promise<ComposerUploadedImage | null> => {
if (!isOpenClawChatVideoFile(file)) {
toast.error("仅支持 MP4、MOV、WebM、M4V 或 MKV 视频");
return null;
}
if (file.size <= 0) {
toast.error(`${file.name} 是空文件`);
return null;
}
if (file.size > OPENCLAW_CHAT_VIDEO_MAX_BYTES) {
toast.error(`${file.name} 超过 500MB,请先压缩视频`);
return null;
}
try {
const normalizedContentType =
getOpenClawChatVideoContentType(file) || "video/mp4";
const uploadFile =
file.type === normalizedContentType
? file
: new File([file], file.name, {
type: normalizedContentType,
lastModified: file.lastModified,
});
const formData = new FormData();
formData.append("file", uploadFile);
formData.append("purpose", OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE);
updateComposerUploadProgress(progressOptions);
const response = await fetchUploadWithRetry(formData, {
onProgress: (progress) => {
updateComposerUploadProgress(progressOptions, progress);
},
});
const data = await response.json().catch(() => null);
if (!response.ok) {
throw new Error(String(data?.error || data?.details || "视频上传失败"));
}
const imageUrl = String(data?.url || "").trim();
if (!isValidChatHttpUrl(imageUrl)) {
throw new Error("储存桶未返回可访问的视频地址");
}
clearComposerUploadProgress(250);
return {
imageUrl,
name: file.name || `视频-${Date.now()}.mp4`,
contentType:
String(data?.contentType || "").trim() ||
normalizedContentType,
assetId:
typeof data?.assetId === "string" && data.assetId.trim()
? data.assetId.trim()
: undefined,
};
} catch (error) {
toast.error(error instanceof Error ? error.message : "视频上传失败");
return null;
}
}, [clearComposerUploadProgress, updateComposerUploadProgress]);
const handlePastedImages = useCallback(async (files: File[]) => { const handlePastedImages = useCallback(async (files: File[]) => {
if (files.length === 0) return; if (files.length === 0) return;
@ -12321,19 +12450,30 @@ export function CanvasChatPanel({
if (isClawdMode) { if (isClawdMode) {
const tableFiles = selectedFiles.filter((file) => isClawdTableFile(file)); const tableFiles = selectedFiles.filter((file) => isClawdTableFile(file));
const imageFiles = selectedFiles.filter((file) => isClawdImageFile(file)); const imageFiles = selectedFiles.filter((file) =>
file.type.startsWith("image/")
);
const videoFiles = selectedFiles.filter((file) =>
isOpenClawChatVideoFile(file)
);
const unsupportedFiles = selectedFiles.filter( const unsupportedFiles = selectedFiles.filter(
(file) => !isClawdTableFile(file) && !isClawdImageFile(file) (file) =>
!isClawdTableFile(file) &&
!file.type.startsWith("image/") &&
!isOpenClawChatVideoFile(file)
); );
unsupportedFiles.forEach((file) => { unsupportedFiles.forEach((file) => {
toast.error(`${file.name} 格式不支持,请上传图片或 .xlsx/.csv/.tsv`); toast.error(
`${file.name} 格式不支持,请上传图片、视频或 .xlsx/.csv/.tsv`
);
}); });
if (tableFiles.length > 0) { if (tableFiles.length > 0) {
addClawdTableFiles(tableFiles); addClawdTableFiles(tableFiles);
} }
let handledImageCount = 0; let handledImageCount = 0;
let handledVideoCount = 0;
if (imageFiles.length > 0) { if (imageFiles.length > 0) {
try { try {
for (const [index, file] of imageFiles.entries()) { for (const [index, file] of imageFiles.entries()) {
@ -12362,6 +12502,49 @@ export function CanvasChatPanel({
} }
} }
if (videoFiles.length > 0) {
const existingVideoCount = attachments.filter((attachment) =>
isOpenClawChatVideoAttachment(
attachment.url,
attachment.contentType
)
).length;
const availableVideoSlots = Math.max(
0,
OPENCLAW_CHAT_VIDEO_MAX_FILES - existingVideoCount
);
const acceptedVideoFiles = videoFiles.slice(0, availableVideoSlots);
if (acceptedVideoFiles.length < videoFiles.length) {
toast.info(
`${CLAW_MODE_LABEL} 对话一次最多添加 ${OPENCLAW_CHAT_VIDEO_MAX_FILES} 个视频`
);
}
try {
for (const [index, file] of acceptedVideoFiles.entries()) {
const uploaded = await uploadClawdVideoFileForComposerAttachment(
file,
{
title:
acceptedVideoFiles.length === 1
? "正在上传视频到储存桶"
: `正在上传 ${acceptedVideoFiles.length} 个视频`,
fileName: file.name,
fileIndex: index,
fileTotal: acceptedVideoFiles.length,
target: "composer-reference",
}
);
if (!uploaded) continue;
handledVideoCount += 1;
addUploadedImageAttachmentToComposer(uploaded);
}
} finally {
clearComposerUploadProgress(450);
}
}
if (handledImageCount > 0) { if (handledImageCount > 0) {
toast.success( toast.success(
handledImageCount === 1 handledImageCount === 1
@ -12369,6 +12552,13 @@ export function CanvasChatPanel({
: `已添加 ${handledImageCount} 张图片为 ${CLAW_MODE_LABEL} 附件` : `已添加 ${handledImageCount} 张图片为 ${CLAW_MODE_LABEL} 附件`
); );
} }
if (handledVideoCount > 0) {
toast.success(
handledVideoCount === 1
? `视频已添加为 ${CLAW_MODE_LABEL} 附件`
: `已添加 ${handledVideoCount} 个视频为 ${CLAW_MODE_LABEL} 附件`
);
}
e.target.value = ""; e.target.value = "";
return; return;
} }
@ -12746,7 +12936,7 @@ export function CanvasChatPanel({
? activeClawdSkillTag ? activeClawdSkillTag
? activeClawdSkillTag.placeholder?.trim() || ? activeClawdSkillTag.placeholder?.trim() ||
`继续向 ${CLAW_MODE_LABEL} 提问,当前使用「${activeClawdSkillTag.name}」` `继续向 ${CLAW_MODE_LABEL} 提问,当前使用「${activeClawdSkillTag.name}」`
: `输入要交给 ${CLAW_MODE_LABEL} 的问题,可上传图片或表格` : `输入要交给 ${CLAW_MODE_LABEL} 的问题,可上传图片、视频或表格`
: isDetailGenerationMode : isDetailGenerationMode
? "可选补充:风格、情绪氛围、禁用元素、期望生成张数(如:生成8张)" ? "可选补充:风格、情绪氛围、禁用元素、期望生成张数(如:生成8张)"
: isVideoWorkspace : isVideoWorkspace
@ -12760,7 +12950,7 @@ export function CanvasChatPanel({
? activeClawdSkillTag ? activeClawdSkillTag
? activeClawdSkillTag.placeholder?.trim() || ? activeClawdSkillTag.placeholder?.trim() ||
`继续向 ${CLAW_MODE_LABEL} 提问,当前使用「${activeClawdSkillTag.name}」` `继续向 ${CLAW_MODE_LABEL} 提问,当前使用「${activeClawdSkillTag.name}」`
: `输入要交给 ${CLAW_MODE_LABEL} 的问题,可上传图片或表格` : `输入要交给 ${CLAW_MODE_LABEL} 的问题,可上传图片、视频或表格`
: isDetailGenerationMode : isDetailGenerationMode
? "可选补充:风格、情绪氛围、禁用元素、期望生成张数(如:生成8张)" ? "可选补充:风格、情绪氛围、禁用元素、期望生成张数(如:生成8张)"
: isVideoWorkspace : isVideoWorkspace
@ -16125,7 +16315,7 @@ export function CanvasChatPanel({
type="file" type="file"
accept={ accept={
isClawdMode isClawdMode
? `${CLAWD_IMAGE_ACCEPT},${CLAWD_TABLE_ACCEPT}` ? `${CLAWD_IMAGE_ACCEPT},${OPENCLAW_CHAT_VIDEO_ACCEPT},${CLAWD_TABLE_ACCEPT}`
: supportsAtlasVideoReference : supportsAtlasVideoReference
? "image/*,video/mp4,video/quicktime,video/webm" ? "image/*,video/mp4,video/quicktime,video/webm"
: "image/*" : "image/*"

View File

@ -1,3 +1,5 @@
import { OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE } from "@/lib/openclaw-chat/media-attachments";
const DEFAULT_MAX_RETRIES = 3; const DEFAULT_MAX_RETRIES = 3;
const MAX_BACKOFF_MS = 8000; const MAX_BACKOFF_MS = 8000;
const MAX_CONCURRENT_UPLOADS = 3; const MAX_CONCURRENT_UPLOADS = 3;
@ -30,6 +32,13 @@ const DIRECT_UPLOAD_IMAGE_TYPES = new Set([
"image/png", "image/png",
"image/webp", "image/webp",
]); ]);
const DIRECT_UPLOAD_OPENCLAW_VIDEO_TYPES = new Set([
"video/mp4",
"video/quicktime",
"video/webm",
"video/x-m4v",
"video/x-matroska",
]);
const BROWSER_IMAGE_TYPES_BY_EXTENSION: Record<string, string> = { const BROWSER_IMAGE_TYPES_BY_EXTENSION: Record<string, string> = {
jpg: "image/jpeg", jpg: "image/jpeg",
@ -298,7 +307,12 @@ function shouldTryDirectUpload(file: UploadFormFile) {
if (file.blob.size <= 0 || file.blob.size > DIRECT_UPLOAD_MAX_BYTES) { if (file.blob.size <= 0 || file.blob.size > DIRECT_UPLOAD_MAX_BYTES) {
return false; return false;
} }
if (!DIRECT_UPLOAD_IMAGE_TYPES.has(file.contentType.toLowerCase())) { const normalizedContentType = file.contentType.toLowerCase();
const supportsDirectUpload =
DIRECT_UPLOAD_IMAGE_TYPES.has(normalizedContentType) ||
(file.purpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE &&
DIRECT_UPLOAD_OPENCLAW_VIDEO_TYPES.has(normalizedContentType));
if (!supportsDirectUpload) {
return false; return false;
} }
if (file.purpose === "agent-workflow-preview") { if (file.purpose === "agent-workflow-preview") {
@ -848,7 +862,7 @@ async function fetchDirectUpload(
return createJsonResponse( return createJsonResponse(
{ {
error: error:
"图片超过 50MB,必须使用浏览器直传;当前直传失败,请刷新后重试,或联系管理员检查存储桶 CORS/ETag 配置。", "文件超过 50MB,必须使用浏览器直传;当前直传失败,请刷新后重试,或联系管理员检查存储桶 CORS/ETag 配置。",
details: error instanceof Error ? error.message : String(error), details: error instanceof Error ? error.message : String(error),
clientUploadId: file.clientUploadId, clientUploadId: file.clientUploadId,
}, },

View File

@ -0,0 +1,183 @@
export const OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE = "openclaw-chat-video";
export const OPENCLAW_CHAT_VIDEO_MAX_FILES = 2;
export const OPENCLAW_CHAT_VIDEO_MAX_BYTES = 500 * 1024 * 1024;
export const OPENCLAW_CHAT_MEDIA_MAX_ATTACHMENTS = 16;
export const OPENCLAW_CHAT_VIDEO_ACCEPT =
"video/mp4,video/quicktime,video/webm,video/x-m4v,video/x-matroska,.mp4,.mov,.webm,.m4v,.mkv";
const VIDEO_CONTENT_TYPES = new Set([
"video/mp4",
"video/quicktime",
"video/webm",
"video/x-m4v",
"video/x-matroska",
]);
const VIDEO_CONTENT_TYPE_BY_EXTENSION: Record<string, string> = {
m4v: "video/x-m4v",
mkv: "video/x-matroska",
mov: "video/quicktime",
mp4: "video/mp4",
webm: "video/webm",
};
const VIDEO_EXTENSION_PATTERN = /\.(mp4|mov|webm|m4v|mkv)(?:$|[?#])/i;
export type OpenClawChatMediaAttachment = {
type: "image" | "video";
url: string;
name?: string;
content_type?: string;
};
type OpenClawChatAttachmentLike = {
type?: unknown;
url?: unknown;
name?: unknown;
contentType?: unknown;
content_type?: unknown;
};
type OpenClawChatVideoFileLike = {
name: string;
type: string;
};
function normalizeContentType(value: unknown) {
return String(value || "")
.split(";", 1)[0]
.trim()
.toLowerCase();
}
function getExtension(value: string) {
const cleanValue = String(value || "").split(/[?#]/, 1)[0];
return cleanValue.split(".").pop()?.trim().toLowerCase() || "";
}
function isHttpUrl(value: string) {
try {
const parsed = new URL(value);
return parsed.protocol === "http:" || parsed.protocol === "https:";
} catch {
return false;
}
}
export function getOpenClawChatVideoContentType(
file: OpenClawChatVideoFileLike
) {
const contentType = normalizeContentType(file.type);
if (VIDEO_CONTENT_TYPES.has(contentType)) {
return contentType;
}
return VIDEO_CONTENT_TYPE_BY_EXTENSION[getExtension(file.name)] || "";
}
export function isOpenClawChatVideoFile(file: OpenClawChatVideoFileLike) {
return Boolean(getOpenClawChatVideoContentType(file));
}
export function isOpenClawChatVideoAttachment(
url: string,
contentType?: string | null
) {
return (
normalizeContentType(contentType).startsWith("video/") ||
VIDEO_EXTENSION_PATTERN.test(String(url || ""))
);
}
export function normalizeOpenClawChatMediaAttachments(
value: unknown
): OpenClawChatMediaAttachment[] {
if (!Array.isArray(value)) {
return [];
}
const result: OpenClawChatMediaAttachment[] = [];
const seenUrls = new Set<string>();
let videoCount = 0;
for (const item of value) {
if (!item || typeof item !== "object") {
continue;
}
const raw = item as OpenClawChatAttachmentLike;
const url = String(raw.url || "").trim();
if (!url || !isHttpUrl(url) || seenUrls.has(url)) {
continue;
}
const requestedType = String(raw.type || "").trim().toLowerCase();
const rawContentType = normalizeContentType(
raw.content_type || raw.contentType
);
const isVideo =
requestedType === "video" ||
isOpenClawChatVideoAttachment(url, rawContentType);
if (isVideo && videoCount >= OPENCLAW_CHAT_VIDEO_MAX_FILES) {
continue;
}
if (result.length >= OPENCLAW_CHAT_MEDIA_MAX_ATTACHMENTS) {
break;
}
const type = isVideo ? "video" : "image";
const fallbackContentType = isVideo
? VIDEO_CONTENT_TYPE_BY_EXTENSION[getExtension(url)] || "video/mp4"
: "image/jpeg";
const contentType =
rawContentType.startsWith(`${type}/`) ? rawContentType : fallbackContentType;
const name = String(raw.name || "")
.trim()
.slice(0, 200);
result.push({
type,
url,
...(name ? { name } : {}),
content_type: contentType,
});
seenUrls.add(url);
if (isVideo) {
videoCount += 1;
}
}
return result;
}
export function buildOpenClawVideoDisplayText(
attachments: OpenClawChatMediaAttachment[]
) {
const videos = attachments.filter((attachment) => attachment.type === "video");
if (videos.length === 0) {
return "";
}
const lines = videos.map(
(attachment, index) => `- ${attachment.name || `视频${index + 1}`}`
);
return `已上传视频:\n${lines.join("\n")}`;
}
export function buildOpenClawVideoReferenceText(
attachments: OpenClawChatMediaAttachment[]
) {
const videos = attachments.filter((attachment) => attachment.type === "video");
if (videos.length === 0) {
return "";
}
const lines = videos.map(
(attachment, index) =>
`- ${attachment.name || `视频${index + 1}`}: ${attachment.url}`
);
return [
"用户上传了以下视频,请调用视频理解能力并按视频文件地址处理:",
lines.join("\n"),
].join("\n");
}

View File

@ -129,6 +129,11 @@ export function getContentType(filename: string): string {
woff2: "font/woff2", woff2: "font/woff2",
glb: "model/gltf-binary", glb: "model/gltf-binary",
gltf: "model/gltf+json", gltf: "model/gltf+json",
mp4: "video/mp4",
mov: "video/quicktime",
webm: "video/webm",
m4v: "video/x-m4v",
mkv: "video/x-matroska",
usdz: "model/vnd.usdz+zip", usdz: "model/vnd.usdz+zip",
fbx: "application/octet-stream", fbx: "application/octet-stream",
obj: "text/plain", obj: "text/plain",

View File

@ -14,6 +14,7 @@ export default defineConfig({
/routes\/canvas-runtime-internal-endpoint\.test\.ts/, /routes\/canvas-runtime-internal-endpoint\.test\.ts/,
/routes\/model-classification\.test\.ts/, /routes\/model-classification\.test\.ts/,
/routes\/clawd-request-lifecycle\.test\.ts/, /routes\/clawd-request-lifecycle\.test\.ts/,
/routes\/openclaw-video-attachments\.test\.ts/,
/routes\/hicad-embedded-performance\.test\.ts/, /routes\/hicad-embedded-performance\.test\.ts/,
/routes\/hicad-manufacturing-extension\.test\.ts/, /routes\/hicad-manufacturing-extension\.test\.ts/,
], ],

View File

@ -0,0 +1,92 @@
import { expect, test } from "@playwright/test";
import {
buildOpenClawVideoDisplayText,
buildOpenClawVideoReferenceText,
getOpenClawChatVideoContentType,
isOpenClawChatVideoFile,
normalizeOpenClawChatMediaAttachments,
OPENCLAW_CHAT_VIDEO_MAX_FILES,
} from "@/lib/openclaw-chat/media-attachments";
test.describe("OpenClaw video attachments", () => {
test("accepts the supported browser video formats by MIME type or extension", () => {
expect(
isOpenClawChatVideoFile({
name: "市场分析.mp4",
type: "video/mp4",
})
).toBe(true);
expect(
isOpenClawChatVideoFile({
name: "素材.MOV",
type: "application/octet-stream",
})
).toBe(true);
expect(
getOpenClawChatVideoContentType({
name: "演示.m4v",
type: "",
})
).toBe("video/x-m4v");
expect(
isOpenClawChatVideoFile({
name: "说明.pdf",
type: "application/pdf",
})
).toBe(false);
});
test("normalizes public video URLs to the OpenClaw attachment contract", () => {
expect(
normalizeOpenClawChatMediaAttachments([
{
type: "video",
url: "https://cdn.example.com/demo.mp4",
name: "demo.mp4",
contentType: "video/mp4",
},
])
).toEqual([
{
type: "video",
url: "https://cdn.example.com/demo.mp4",
name: "demo.mp4",
content_type: "video/mp4",
},
]);
});
test("rejects private schemes, duplicates, and videos beyond the gateway limit", () => {
const attachments = normalizeOpenClawChatMediaAttachments([
{ type: "video", url: "https://cdn.example.com/one.mp4" },
{ type: "video", url: "https://cdn.example.com/two.mov" },
{ type: "video", url: "https://cdn.example.com/three.webm" },
{ type: "video", url: "file:///tmp/private.mp4" },
{ type: "video", url: "https://cdn.example.com/one.mp4" },
]);
expect(attachments).toHaveLength(OPENCLAW_CHAT_VIDEO_MAX_FILES);
expect(attachments.map((attachment) => attachment.url)).toEqual([
"https://cdn.example.com/one.mp4",
"https://cdn.example.com/two.mov",
]);
});
test("builds a readable fallback message without exposing storage credentials", () => {
const attachments = normalizeOpenClawChatMediaAttachments([
{
type: "video",
url: "https://cdn.example.com/review.mp4",
name: "review.mp4",
},
]);
expect(buildOpenClawVideoDisplayText(attachments)).toContain("review.mp4");
expect(buildOpenClawVideoReferenceText(attachments)).toContain(
"https://cdn.example.com/review.mp4"
);
expect(buildOpenClawVideoReferenceText(attachments)).not.toContain(
"Authorization"
);
});
});