lunanting/ai-chatbot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: support video attachments in Claw chat

Browse Source
This commit is contained in:
134 2026-07-28 15:50:47 +08:00
parent 21ba23c4ab
commit fb8d8232dc
9 changed files with 650 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
app/(chat)/api/files/direct-upload/route.ts
View File

@ -21,6 +21,7 @@ import {
maybeEnqueueStorageMigrationTask,
resolveS3UploadTarget,
} from "@/lib/storage/service";
import { OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE } from "@/lib/openclaw-chat/media-attachments";
const logger = createLogger("direct-upload-api");
const DIRECT_UPLOAD_MAX_BYTES = 500 * 1024 * 1024;
@ -39,6 +40,13 @@ const allowedDirectUploadTypes = new Set([
"image/png",
"image/webp",
]);
const allowedOpenClawVideoUploadTypes = new Set([
"video/mp4",
"video/quicktime",
"video/webm",
"video/x-m4v",
"video/x-matroska",
]);
const InitiateSchema = z.object({
action: z.literal("initiate"),
@ -109,8 +117,12 @@ function assertUserOwnedKey(key: string, userId: string) {
return normalized;
}
function isAllowedDirectUpload(contentType: string) {
return allowedDirectUploadTypes.has(contentType);
function isAllowedDirectUpload(contentType: string, purpose?: string) {
return (
allowedDirectUploadTypes.has(contentType) ||
(purpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE &&
allowedOpenClawVideoUploadTypes.has(contentType))
);
}
function computePartSize(size: number) {
@ -204,7 +216,7 @@ export async function POST(request: Request) {
payload.filename
);
if (!isAllowedDirectUpload(contentType)) {
if (!isAllowedDirectUpload(contentType, payload.purpose)) {
logger.warn("unsupported_direct_upload_type", {
contentType,
filenameExtension: payload.filename.toLowerCase().split(".").pop() || null,
@ -213,7 +225,10 @@ export async function POST(request: Request) {
clientUploadId: payload.clientUploadId || null,
});
return NextResponse.json(
{ error: "Direct upload only supports JPG, PNG and WebP images" },
{
error:
"直传仅支持 JPG、PNG、WebP 图片以及 Claw 对话的 MP4、MOV、WebM、M4V、MKV 视频",
},
{ status: 400 }
);
}
@ -328,9 +343,12 @@ export async function POST(request: Request) {
const key = assertUserOwnedKey(payload.key, session.user.id);
const contentType = normalizeContentType(payload.contentType, payload.filename);
if (!isAllowedDirectUpload(contentType)) {
if (!isAllowedDirectUpload(contentType, payload.purpose)) {
return NextResponse.json(
{ error: "Direct upload only supports JPG, PNG and WebP images" },
{
error:
"直传仅支持 JPG、PNG、WebP 图片以及 Claw 对话的 MP4、MOV、WebM、M4V、MKV 视频",
},
{ status: 400 }
);
}
@ -389,15 +407,21 @@ export async function POST(request: Request) {
const assetStartedAt = Date.now();
const asset = await upsertMediaAsset({
userId: session.user.id,
mediaType: "image",
sourceType: "upload",
mediaType: resolvedContentType.startsWith("video/") ? "video" : "image",
sourceType:
payload.purpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE
? "openclaw_chat_video_upload"
: "upload",
status: "ready",
originalUrl: url,
previewUrl: url,
thumbnailUrl: url,
mimeType: resolvedContentType,
displayName: payload.filename,
sourceTable: "direct_upload",
sourceTable:
payload.purpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE
? "openclaw_chat_video_upload"
: "direct_upload",
metadata: {
pathname: key,
size: actualSize ?? payload.size,

37
app/(chat)/api/files/upload/route.ts
View File

@ -7,6 +7,7 @@ import { upsertMediaAsset } from "@/lib/db/queries";
import { isSiteAdminType } from "@/lib/middleware/admin";
import { getContentType, uploadFile } from "@/lib/storage/service";
import { createLogger } from "@/lib/observability/logger";
import { OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE } from "@/lib/openclaw-chat/media-attachments";
const logger = createLogger("upload-api");
const ALLOWED_UPLOAD_IMAGE_TYPES = new Set([
@ -26,6 +27,13 @@ const ALLOWED_ATLASCLOUD_VIDEO_TYPES = new Set([
"video/quicktime",
"video/webm",
]);
const ALLOWED_OPENCLAW_VIDEO_TYPES = new Set([
"video/mp4",
"video/quicktime",
"video/webm",
"video/x-m4v",
"video/x-matroska",
]);
const HEIC_UPLOAD_TYPES = new Set(["image/heic", "image/heif"]);
const SHARP_COMPATIBILITY_UPLOAD_TYPES = new Set([
@ -246,6 +254,9 @@ export async function POST(request: Request) {
const isAtlasCloudReferenceVideo =
uploadPurpose === "canvas-video-process" &&
ALLOWED_ATLASCLOUD_VIDEO_TYPES.has(contentType);
const isOpenClawChatVideoUpload =
uploadPurpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE &&
ALLOWED_OPENCLAW_VIDEO_TYPES.has(contentType);
const clientUploadId =
String(formData.get("clientUploadId") || "").trim() || null;
const isSvgUpload = contentType === "image/svg+xml";
@ -278,7 +289,8 @@ export async function POST(request: Request) {
if (
!ALLOWED_UPLOAD_IMAGE_TYPES.has(contentType) &&
!isAtlasCloudReferenceVideo
!isAtlasCloudReferenceVideo &&
!isOpenClawChatVideoUpload
) {
const errorMessage = createUnsupportedImageTypeMessage(contentType, filename);
logger.warn("[Upload API] Unsupported upload content type", {
@ -445,15 +457,28 @@ export async function POST(request: Request) {
try {
const asset = await upsertMediaAsset({
userId: session.user.id,
mediaType: isAtlasCloudReferenceVideo ? "video" : "image",
sourceType: "upload",
mediaType:
isAtlasCloudReferenceVideo || isOpenClawChatVideoUpload
? "video"
: "image",
sourceType: isOpenClawChatVideoUpload
? "openclaw_chat_video_upload"
: "upload",
status: "ready",
originalUrl: data.url,
previewUrl: isAtlasCloudReferenceVideo ? null : data.url,
thumbnailUrl: isAtlasCloudReferenceVideo ? null : data.url,
previewUrl:
isAtlasCloudReferenceVideo || isOpenClawChatVideoUpload
? null
: data.url,
thumbnailUrl:
isAtlasCloudReferenceVideo || isOpenClawChatVideoUpload
? null
: data.url,
mimeType: data.contentType || contentType || null,
displayName: filename,
sourceTable: "upload",
sourceTable: isOpenClawChatVideoUpload
? "openclaw_chat_video_upload"
: "upload",
metadata: {
pathname: data.pathname,
size: optimizedBuffer.length,

83
app/api/canvas/clawd-chat/route.ts
View File

@ -25,6 +25,11 @@ import {
buildOpenClawMessageWithSpreadsheetMarkdown,
buildOpenClawSpreadsheetMarkdown,
} from "@/lib/openclaw-chat/spreadsheet-markdown";
import {
normalizeOpenClawChatMediaAttachments,
OPENCLAW_CHAT_MEDIA_MAX_ATTACHMENTS,
type OpenClawChatMediaAttachment,
} from "@/lib/openclaw-chat/media-attachments";
import { generateUUID } from "@/lib/utils";
const CLAW_UPGRADE_RETRY_MESSAGE = "claw正在升级更新稍等5分钟再试";
@ -35,9 +40,18 @@ const requestSchema = z.object({
displayMessage: z.string().trim().max(8000).optional(),
userMessageId: z.string().uuid().optional(),
metadata: z.record(z.unknown()).optional(),
attachments: z
.array(z.unknown())
.max(OPENCLAW_CHAT_MEDIA_MAX_ATTACHMENTS)
.optional()
.default([]),
});
type ParsedClawdChatRequest = z.infer<typeof requestSchema> & {
type ParsedClawdChatRequest = Omit<
z.infer<typeof requestSchema>,
"attachments"
> & {
attachments: OpenClawChatMediaAttachment[];
tableFiles: File[];
};
@ -85,14 +99,23 @@ async function parseClawdChatRequest(
const contentType = request.headers.get("content-type") || "";
if (!contentType.includes("multipart/form-data")) {
const parsed = requestSchema.safeParse(await request.json().catch(() => null));
return parsed.success ? { ...parsed.data, tableFiles: [] } : null;
if (!parsed.success) return null;
const attachments = normalizeOpenClawChatMediaAttachments(
parsed.data.attachments
);
if (attachments.length !== parsed.data.attachments.length) {
return null;
}
return { ...parsed.data, attachments, tableFiles: [] };
}
const formData = await request.formData().catch(() => null);
if (!formData) return null;
const metadataValue = formData.get("metadata");
const attachmentsValue = formData.get("attachments");
let metadata: Record<string, unknown> | undefined;
let attachments: unknown[] = [];
if (typeof metadataValue === "string" && metadataValue.trim()) {
try {
const parsedMetadata = JSON.parse(metadataValue);
@ -105,6 +128,17 @@ async function parseClawdChatRequest(
return null;
}
}
if (typeof attachmentsValue === "string" && attachmentsValue.trim()) {
try {
const parsedAttachments = JSON.parse(attachmentsValue);
if (!Array.isArray(parsedAttachments)) {
return null;
}
attachments = parsedAttachments;
} catch {
return null;
}
}
const parsed = requestSchema.safeParse({
id: formData.get("id"),
@ -112,14 +146,25 @@ async function parseClawdChatRequest(
displayMessage: formData.get("displayMessage") || undefined,
userMessageId: formData.get("userMessageId") || undefined,
metadata,
attachments,
});
if (!parsed.success) return null;
const normalizedAttachments = normalizeOpenClawChatMediaAttachments(
parsed.data.attachments
);
if (normalizedAttachments.length !== parsed.data.attachments.length) {
return null;
}
const tableFiles = formData
.getAll("tableFiles")
.filter((entry): entry is File => isRequestFile(entry) && entry.size > 0);
return { ...parsed.data, tableFiles };
return {
...parsed.data,
attachments: normalizedAttachments,
tableFiles,
};
}
async function readOpenClawConfig() {
@ -163,8 +208,15 @@ export async function POST(request: Request) {
if (!parsed) {
return NextResponse.json({ error: "请求参数不正确" }, { status: 400 });
}
if (!parsed.message && parsed.tableFiles.length === 0) {
return NextResponse.json({ error: "请输入问题或上传表格" }, { status: 400 });
if (
!parsed.message &&
parsed.tableFiles.length === 0 &&
parsed.attachments.length === 0
) {
return NextResponse.json(
{ error: "请输入问题或上传附件" },
{ status: 400 }
);
}
const config = await readOpenClawConfig();
@ -275,6 +327,8 @@ export async function POST(request: Request) {
skill_tag_id: skillTagId || undefined,
skill_tag_name: skillTagName || undefined,
message: upstreamMessage,
attachments:
parsed.attachments.length > 0 ? parsed.attachments : undefined,
metadata: {
source: "ai_chatbot_canvas",
page: "/canvas",
@ -288,6 +342,8 @@ export async function POST(request: Request) {
user_department: openClawDepartmentName,
skill_tag_id: skillTagId || undefined,
skill_tag_name: skillTagName || undefined,
media_attachments:
parsed.attachments.length > 0 ? parsed.attachments : undefined,
spreadsheet_files: spreadsheetMarkdownResult?.files || undefined,
spreadsheet_markdown_truncated:
spreadsheetMarkdownResult?.truncated || undefined,
@ -378,7 +434,15 @@ export async function POST(request: Request) {
chatId: parsed.id,
role: "user",
parts: [{ type: "text", text: displayMessage }],
attachments: [],
attachments: parsed.attachments.map((attachment) => ({
name:
attachment.name ||
(attachment.type === "video" ? "视频附件" : "图片附件"),
url: attachment.url,
contentType:
attachment.content_type ||
(attachment.type === "video" ? "video/mp4" : "image/jpeg"),
})),
createdAt: userCreatedAt,
videoStorageStatus: null,
temporaryVideoUrl: null,
@ -387,6 +451,12 @@ export async function POST(request: Request) {
videoMetadata: {
composerMode: "clawd",
source: "openclaw",
imageAttachmentCount: parsed.attachments.filter(
(attachment) => attachment.type === "image"
).length,
videoAttachmentCount: parsed.attachments.filter(
(attachment) => attachment.type === "video"
).length,
skillTagId: skillTagId || null,
skillTagName: skillTagName || null,
},
@ -431,6 +501,7 @@ export async function POST(request: Request) {
createdAt: userMessage.createdAt.toISOString(),
composerMode: "clawd",
},
attachments: userMessage.attachments,
videoMetadata: userMessage.videoMetadata,
},
{

234
components/canvas/CanvasChatPanel.tsx
View File

@ -287,6 +287,18 @@ import {
stripCanvasModelRoutingLines,
stripMediaMarkup,
} from "@/components/canvas/chat-panel/canvas-chat-media";
import {
buildOpenClawVideoDisplayText,
buildOpenClawVideoReferenceText,
getOpenClawChatVideoContentType,
isOpenClawChatVideoAttachment,
isOpenClawChatVideoFile,
normalizeOpenClawChatMediaAttachments,
OPENCLAW_CHAT_VIDEO_ACCEPT,
OPENCLAW_CHAT_VIDEO_MAX_BYTES,
OPENCLAW_CHAT_VIDEO_MAX_FILES,
OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE,
} from "@/lib/openclaw-chat/media-attachments";
type ResponseModule = typeof import("@/components/elements/response");
type ResponseProps = ComponentProps<ResponseModule["Response"]>;
@ -612,9 +624,10 @@ function writeStoredClawdComposerSubmissionSnapshot(
function shouldShowClawdProcessingHint(
message: string,
hasTableFiles: boolean,
hasImageAttachments: boolean
hasImageAttachments: boolean,
hasVideoAttachments: boolean
) {
if (hasTableFiles || hasImageAttachments) {
if (hasTableFiles || hasImageAttachments || hasVideoAttachments) {
return true;
}
@ -675,17 +688,19 @@ function buildClawdImageReferenceText(attachments: PickedObjectAttachment[]) {
return `用户还添加了以下图片作为参考，请按图片链接理解：\n${lines.join("\n")}`;
}
function getClawdDefaultPrompt(hasTables: boolean, hasImages: boolean) {
if (hasTables && hasImages) {
return "请分析上传的表格和图片内容";
}
if (hasTables) {
return "请分析上传的表格内容";
}
if (hasImages) {
return "请分析上传的图片内容";
}
return "请分析";
function getClawdDefaultPrompt(
hasTables: boolean,
hasImages: boolean,
hasVideos: boolean
) {
const mediaNames = [
hasTables ? "表格" : "",
hasImages ? "图片" : "",
hasVideos ? "视频" : "",
].filter(Boolean);
return mediaNames.length > 0
? `请分析上传的${mediaNames.join("、")}内容`
: "请分析";
}
type VideoStoryboardStreamData =
@ -11034,15 +11049,44 @@ export function CanvasChatPanel({
: "";
const hasPendingClawdSkillTagPrompt = Boolean(clawdSkillTagPrompt);
const attachmentsSnapshot = isProgrammaticClawdQuickSend ? [] : [...attachments];
const imageAttachmentsSnapshot = attachmentsSnapshot
const sanitizedAttachmentsSnapshot = attachmentsSnapshot
.map((attachment) => sanitizeChatAttachment(attachment))
.filter((attachment): attachment is PickedObjectAttachment => Boolean(attachment));
const videoAttachmentsSnapshot = sanitizedAttachmentsSnapshot.filter(
(attachment) =>
isOpenClawChatVideoAttachment(
attachment.url,
attachment.contentType
)
);
const imageAttachmentsSnapshot = sanitizedAttachmentsSnapshot.filter(
(attachment) =>
!isOpenClawChatVideoAttachment(
attachment.url,
attachment.contentType
)
);
const hasImageAttachments = imageAttachmentsSnapshot.length > 0;
const hasVideoAttachments = videoAttachmentsSnapshot.length > 0;
const openClawMediaAttachments = normalizeOpenClawChatMediaAttachments(
sanitizedAttachmentsSnapshot.map((attachment) => ({
type: isOpenClawChatVideoAttachment(
attachment.url,
attachment.contentType
)
? "video"
: "image",
url: attachment.url,
name: normalizeChatAttachmentName(attachment.name),
content_type: attachment.contentType,
}))
);
if (
!textContent &&
!hasTableFiles &&
!hasImageAttachments &&
!hasVideoAttachments &&
!hasPendingClawdSkillTagPrompt
) {
return;
@ -11052,14 +11096,22 @@ export function CanvasChatPanel({
textContent ||
(clawdSkillTagForRequest && hasPendingClawdSkillTagPrompt
? `请按「${clawdSkillTagForRequest.name}」快捷功能处理。`
: getClawdDefaultPrompt(hasTableFiles, hasImageAttachments));
: getClawdDefaultPrompt(
hasTableFiles,
hasImageAttachments,
hasVideoAttachments
));
const imageReferenceText = buildClawdImageReferenceText(
imageAttachmentsSnapshot
);
const videoReferenceText = buildOpenClawVideoReferenceText(
openClawMediaAttachments
);
const requestMessage = [
baseRequestMessage,
clawdSkillTagPrompt,
imageReferenceText,
videoReferenceText,
]
.filter(Boolean)
.join("\n\n");
@ -11078,6 +11130,7 @@ export function CanvasChatPanel({
const optimisticMessageText = [
buildClawdTableDisplayText(optimisticBaseMessage, tableFilesSnapshot),
buildClawdImageDisplayText(imageAttachmentsSnapshot),
buildOpenClawVideoDisplayText(openClawMediaAttachments),
]
.filter(Boolean)
.join("\n\n");
@ -11094,6 +11147,10 @@ export function CanvasChatPanel({
.map((attachment) => normalizeMediaUrl(attachment.url))
.filter(Boolean)
.sort(),
videoAttachments: videoAttachmentsSnapshot
.map((attachment) => normalizeMediaUrl(attachment.url))
.filter(Boolean)
.sort(),
});
if (shouldBlockDuplicateSend(activeChatId, sendSignature)) {
@ -11104,7 +11161,8 @@ export function CanvasChatPanel({
const nextProcessingHint = shouldShowClawdProcessingHint(
[textContent, clawdSkillTagPrompt].filter(Boolean).join("\n\n"),
hasTableFiles,
hasImageAttachments
hasImageAttachments,
hasVideoAttachments
)
? CLAWD_PROCESSING_HINT
: null;
@ -11207,11 +11265,18 @@ export function CanvasChatPanel({
composerMode: "clawd",
activeChatId,
imageAttachmentCount: imageAttachmentsSnapshot.length || undefined,
videoAttachmentCount: videoAttachmentsSnapshot.length || undefined,
skillTagId: clawdSkillTagForRequest?.id || undefined,
skillTagName: clawdSkillTagForRequest?.name || undefined,
skillTagPromptIncluded: hasPendingClawdSkillTagPrompt || undefined,
})
);
if (openClawMediaAttachments.length > 0) {
formData.append(
"attachments",
JSON.stringify(openClawMediaAttachments)
);
}
tableFilesSnapshot.forEach((file) => {
formData.append("tableFiles", file, file.name);
});
@ -12042,6 +12107,70 @@ export function CanvasChatPanel({
}
}, [clearComposerUploadProgress, updateComposerUploadProgress]);
const uploadClawdVideoFileForComposerAttachment = useCallback(async (
file: File,
progressOptions: CanvasUploadProgressOptions
): Promise<ComposerUploadedImage | null> => {
if (!isOpenClawChatVideoFile(file)) {
toast.error("仅支持 MP4、MOV、WebM、M4V 或 MKV 视频");
return null;
}
if (file.size <= 0) {
toast.error(`${file.name} 是空文件`);
return null;
}
if (file.size > OPENCLAW_CHAT_VIDEO_MAX_BYTES) {
toast.error(`${file.name} 超过 500MB，请先压缩视频`);
return null;
}
try {
const normalizedContentType =
getOpenClawChatVideoContentType(file) || "video/mp4";
const uploadFile =
file.type === normalizedContentType
? file
: new File([file], file.name, {
type: normalizedContentType,
lastModified: file.lastModified,
});
const formData = new FormData();
formData.append("file", uploadFile);
formData.append("purpose", OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE);
updateComposerUploadProgress(progressOptions);
const response = await fetchUploadWithRetry(formData, {
onProgress: (progress) => {
updateComposerUploadProgress(progressOptions, progress);
},
});
const data = await response.json().catch(() => null);
if (!response.ok) {
throw new Error(String(data?.error || data?.details || "视频上传失败"));
}
const imageUrl = String(data?.url || "").trim();
if (!isValidChatHttpUrl(imageUrl)) {
throw new Error("储存桶未返回可访问的视频地址");
}
clearComposerUploadProgress(250);
return {
imageUrl,
name: file.name || `视频-${Date.now()}.mp4`,
contentType:
String(data?.contentType || "").trim() ||
normalizedContentType,
assetId:
typeof data?.assetId === "string" && data.assetId.trim()
? data.assetId.trim()
: undefined,
};
} catch (error) {
toast.error(error instanceof Error ? error.message : "视频上传失败");
return null;
}
}, [clearComposerUploadProgress, updateComposerUploadProgress]);
const handlePastedImages = useCallback(async (files: File[]) => {
if (files.length === 0) return;
@ -12321,19 +12450,30 @@ export function CanvasChatPanel({
if (isClawdMode) {
const tableFiles = selectedFiles.filter((file) => isClawdTableFile(file));
const imageFiles = selectedFiles.filter((file) => isClawdImageFile(file));
const imageFiles = selectedFiles.filter((file) =>
file.type.startsWith("image/")
);
const videoFiles = selectedFiles.filter((file) =>
isOpenClawChatVideoFile(file)
);
const unsupportedFiles = selectedFiles.filter(
(file) => !isClawdTableFile(file) && !isClawdImageFile(file)
(file) =>
!isClawdTableFile(file) &&
!file.type.startsWith("image/") &&
!isOpenClawChatVideoFile(file)
);
unsupportedFiles.forEach((file) => {
toast.error(`${file.name} 格式不支持，请上传图片或 .xlsx/.csv/.tsv`);
toast.error(
`${file.name} 格式不支持，请上传图片、视频或 .xlsx/.csv/.tsv`
);
});
if (tableFiles.length > 0) {
addClawdTableFiles(tableFiles);
}
let handledImageCount = 0;
let handledVideoCount = 0;
if (imageFiles.length > 0) {
try {
for (const [index, file] of imageFiles.entries()) {
@ -12362,6 +12502,49 @@ export function CanvasChatPanel({
}
}
if (videoFiles.length > 0) {
const existingVideoCount = attachments.filter((attachment) =>
isOpenClawChatVideoAttachment(
attachment.url,
attachment.contentType
)
).length;
const availableVideoSlots = Math.max(
0,
OPENCLAW_CHAT_VIDEO_MAX_FILES - existingVideoCount
);
const acceptedVideoFiles = videoFiles.slice(0, availableVideoSlots);
if (acceptedVideoFiles.length < videoFiles.length) {
toast.info(
`${CLAW_MODE_LABEL} 对话一次最多添加 ${OPENCLAW_CHAT_VIDEO_MAX_FILES} 个视频`
);
}
try {
for (const [index, file] of acceptedVideoFiles.entries()) {
const uploaded = await uploadClawdVideoFileForComposerAttachment(
file,
{
title:
acceptedVideoFiles.length === 1
? "正在上传视频到储存桶"
: `正在上传 ${acceptedVideoFiles.length} 个视频`,
fileName: file.name,
fileIndex: index,
fileTotal: acceptedVideoFiles.length,
target: "composer-reference",
}
);
if (!uploaded) continue;
handledVideoCount += 1;
addUploadedImageAttachmentToComposer(uploaded);
}
} finally {
clearComposerUploadProgress(450);
}
}
if (handledImageCount > 0) {
toast.success(
handledImageCount === 1
@ -12369,6 +12552,13 @@ export function CanvasChatPanel({
: `已添加 ${handledImageCount} 张图片为 ${CLAW_MODE_LABEL} 附件`
);
}
if (handledVideoCount > 0) {
toast.success(
handledVideoCount === 1
? `视频已添加为 ${CLAW_MODE_LABEL} 附件`
: `已添加 ${handledVideoCount} 个视频为 ${CLAW_MODE_LABEL} 附件`
);
}
e.target.value = "";
return;
}
@ -12746,7 +12936,7 @@ export function CanvasChatPanel({
? activeClawdSkillTag
? activeClawdSkillTag.placeholder?.trim() ||
`继续向 ${CLAW_MODE_LABEL} 提问，当前使用「${activeClawdSkillTag.name}」`
: `输入要交给 ${CLAW_MODE_LABEL} 的问题，可上传图片或表格`
: `输入要交给 ${CLAW_MODE_LABEL} 的问题，可上传图片、视频或表格`
: isDetailGenerationMode
? "可选补充：风格、情绪氛围、禁用元素、期望生成张数（如：生成8张）"
: isVideoWorkspace
@ -12760,7 +12950,7 @@ export function CanvasChatPanel({
? activeClawdSkillTag
? activeClawdSkillTag.placeholder?.trim() ||
`继续向 ${CLAW_MODE_LABEL} 提问，当前使用「${activeClawdSkillTag.name}」`
: `输入要交给 ${CLAW_MODE_LABEL} 的问题，可上传图片或表格`
: `输入要交给 ${CLAW_MODE_LABEL} 的问题，可上传图片、视频或表格`
: isDetailGenerationMode
? "可选补充：风格、情绪氛围、禁用元素、期望生成张数（如：生成8张）"
: isVideoWorkspace
@ -16125,7 +16315,7 @@ export function CanvasChatPanel({
type="file"
accept={
isClawdMode
? `${CLAWD_IMAGE_ACCEPT},${CLAWD_TABLE_ACCEPT}`
? `${CLAWD_IMAGE_ACCEPT},${OPENCLAW_CHAT_VIDEO_ACCEPT},${CLAWD_TABLE_ACCEPT}`
: supportsAtlasVideoReference
? "image/*,video/mp4,video/quicktime,video/webm"
: "image/*"

18
lib/client/upload-with-retry.ts
View File

@ -1,3 +1,5 @@
import { OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE } from "@/lib/openclaw-chat/media-attachments";
const DEFAULT_MAX_RETRIES = 3;
const MAX_BACKOFF_MS = 8000;
const MAX_CONCURRENT_UPLOADS = 3;
@ -30,6 +32,13 @@ const DIRECT_UPLOAD_IMAGE_TYPES = new Set([
"image/png",
"image/webp",
]);
const DIRECT_UPLOAD_OPENCLAW_VIDEO_TYPES = new Set([
"video/mp4",
"video/quicktime",
"video/webm",
"video/x-m4v",
"video/x-matroska",
]);
const BROWSER_IMAGE_TYPES_BY_EXTENSION: Record<string, string> = {
jpg: "image/jpeg",
@ -298,7 +307,12 @@ function shouldTryDirectUpload(file: UploadFormFile) {
if (file.blob.size <= 0 || file.blob.size > DIRECT_UPLOAD_MAX_BYTES) {
return false;
}
if (!DIRECT_UPLOAD_IMAGE_TYPES.has(file.contentType.toLowerCase())) {
const normalizedContentType = file.contentType.toLowerCase();
const supportsDirectUpload =
DIRECT_UPLOAD_IMAGE_TYPES.has(normalizedContentType) ||
(file.purpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE &&
DIRECT_UPLOAD_OPENCLAW_VIDEO_TYPES.has(normalizedContentType));
if (!supportsDirectUpload) {
return false;
}
if (file.purpose === "agent-workflow-preview") {
@ -848,7 +862,7 @@ async function fetchDirectUpload(
return createJsonResponse(
{
error:
"图片超过 50MB，必须使用浏览器直传；当前直传失败，请刷新后重试，或联系管理员检查存储桶 CORS/ETag 配置。",
"文件超过 50MB，必须使用浏览器直传；当前直传失败，请刷新后重试，或联系管理员检查存储桶 CORS/ETag 配置。",
details: error instanceof Error ? error.message : String(error),
clientUploadId: file.clientUploadId,
},

183
lib/openclaw-chat/media-attachments.ts Normal file
View File

@ -0,0 +1,183 @@
export const OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE = "openclaw-chat-video";
export const OPENCLAW_CHAT_VIDEO_MAX_FILES = 2;
export const OPENCLAW_CHAT_VIDEO_MAX_BYTES = 500 * 1024 * 1024;
export const OPENCLAW_CHAT_MEDIA_MAX_ATTACHMENTS = 16;
export const OPENCLAW_CHAT_VIDEO_ACCEPT =
"video/mp4,video/quicktime,video/webm,video/x-m4v,video/x-matroska,.mp4,.mov,.webm,.m4v,.mkv";
const VIDEO_CONTENT_TYPES = new Set([
"video/mp4",
"video/quicktime",
"video/webm",
"video/x-m4v",
"video/x-matroska",
]);
const VIDEO_CONTENT_TYPE_BY_EXTENSION: Record<string, string> = {
m4v: "video/x-m4v",
mkv: "video/x-matroska",
mov: "video/quicktime",
mp4: "video/mp4",
webm: "video/webm",
};
const VIDEO_EXTENSION_PATTERN = /\.(mp4|mov|webm|m4v|mkv)(?:$|[?#])/i;
export type OpenClawChatMediaAttachment = {
type: "image" | "video";
url: string;
name?: string;
content_type?: string;
};
type OpenClawChatAttachmentLike = {
type?: unknown;
url?: unknown;
name?: unknown;
contentType?: unknown;
content_type?: unknown;
};
type OpenClawChatVideoFileLike = {
name: string;
type: string;
};
function normalizeContentType(value: unknown) {
return String(value || "")
.split(";", 1)[0]
.trim()
.toLowerCase();
}
function getExtension(value: string) {
const cleanValue = String(value || "").split(/[?#]/, 1)[0];
return cleanValue.split(".").pop()?.trim().toLowerCase() || "";
}
function isHttpUrl(value: string) {
try {
const parsed = new URL(value);
return parsed.protocol === "http:" || parsed.protocol === "https:";
} catch {
return false;
}
}
export function getOpenClawChatVideoContentType(
file: OpenClawChatVideoFileLike
) {
const contentType = normalizeContentType(file.type);
if (VIDEO_CONTENT_TYPES.has(contentType)) {
return contentType;
}
return VIDEO_CONTENT_TYPE_BY_EXTENSION[getExtension(file.name)] || "";
}
export function isOpenClawChatVideoFile(file: OpenClawChatVideoFileLike) {
return Boolean(getOpenClawChatVideoContentType(file));
}
export function isOpenClawChatVideoAttachment(
url: string,
contentType?: string | null
) {
return (
normalizeContentType(contentType).startsWith("video/") ||
VIDEO_EXTENSION_PATTERN.test(String(url || ""))
);
}
export function normalizeOpenClawChatMediaAttachments(
value: unknown
): OpenClawChatMediaAttachment[] {
if (!Array.isArray(value)) {
return [];
}
const result: OpenClawChatMediaAttachment[] = [];
const seenUrls = new Set<string>();
let videoCount = 0;
for (const item of value) {
if (!item || typeof item !== "object") {
continue;
}
const raw = item as OpenClawChatAttachmentLike;
const url = String(raw.url || "").trim();
if (!url || !isHttpUrl(url) || seenUrls.has(url)) {
continue;
}
const requestedType = String(raw.type || "").trim().toLowerCase();
const rawContentType = normalizeContentType(
raw.content_type || raw.contentType
);
const isVideo =
requestedType === "video" ||
isOpenClawChatVideoAttachment(url, rawContentType);
if (isVideo && videoCount >= OPENCLAW_CHAT_VIDEO_MAX_FILES) {
continue;
}
if (result.length >= OPENCLAW_CHAT_MEDIA_MAX_ATTACHMENTS) {
break;
}
const type = isVideo ? "video" : "image";
const fallbackContentType = isVideo
? VIDEO_CONTENT_TYPE_BY_EXTENSION[getExtension(url)] || "video/mp4"
: "image/jpeg";
const contentType =
rawContentType.startsWith(`${type}/`) ? rawContentType : fallbackContentType;
const name = String(raw.name || "")
.trim()
.slice(0, 200);
result.push({
type,
url,
...(name ? { name } : {}),
content_type: contentType,
});
seenUrls.add(url);
if (isVideo) {
videoCount += 1;
}
}
return result;
}
export function buildOpenClawVideoDisplayText(
attachments: OpenClawChatMediaAttachment[]
) {
const videos = attachments.filter((attachment) => attachment.type === "video");
if (videos.length === 0) {
return "";
}
const lines = videos.map(
(attachment, index) => `- ${attachment.name || `视频${index + 1}`}`
);
return `已上传视频：\n${lines.join("\n")}`;
}
export function buildOpenClawVideoReferenceText(
attachments: OpenClawChatMediaAttachment[]
) {
const videos = attachments.filter((attachment) => attachment.type === "video");
if (videos.length === 0) {
return "";
}
const lines = videos.map(
(attachment, index) =>
`- ${attachment.name || `视频${index + 1}`}: ${attachment.url}`
);
return [
"用户上传了以下视频，请调用视频理解能力并按视频文件地址处理：",
lines.join("\n"),
].join("\n");
}

5
lib/storage/service.ts
View File

@ -129,6 +129,11 @@ export function getContentType(filename: string): string {
woff2: "font/woff2",
glb: "model/gltf-binary",
gltf: "model/gltf+json",
mp4: "video/mp4",
mov: "video/quicktime",
webm: "video/webm",
m4v: "video/x-m4v",
mkv: "video/x-matroska",
usdz: "model/vnd.usdz+zip",
fbx: "application/octet-stream",
obj: "text/plain",

1
playwright.helpers.config.ts
View File

@ -14,6 +14,7 @@ export default defineConfig({
/routes\/canvas-runtime-internal-endpoint\.test\.ts/,
/routes\/model-classification\.test\.ts/,
/routes\/clawd-request-lifecycle\.test\.ts/,
/routes\/openclaw-video-attachments\.test\.ts/,
/routes\/hicad-embedded-performance\.test\.ts/,
/routes\/hicad-manufacturing-extension\.test\.ts/,
],

92
tests/routes/openclaw-video-attachments.test.ts Normal file
View File

@ -0,0 +1,92 @@
import { expect, test } from "@playwright/test";
import {
buildOpenClawVideoDisplayText,
buildOpenClawVideoReferenceText,
getOpenClawChatVideoContentType,
isOpenClawChatVideoFile,
normalizeOpenClawChatMediaAttachments,
OPENCLAW_CHAT_VIDEO_MAX_FILES,
} from "@/lib/openclaw-chat/media-attachments";
test.describe("OpenClaw video attachments", () => {
test("accepts the supported browser video formats by MIME type or extension", () => {
expect(
isOpenClawChatVideoFile({
name: "市场分析.mp4",
type: "video/mp4",
})
).toBe(true);
expect(
isOpenClawChatVideoFile({
name: "素材.MOV",
type: "application/octet-stream",
})
).toBe(true);
expect(
getOpenClawChatVideoContentType({
name: "演示.m4v",
type: "",
})
).toBe("video/x-m4v");
expect(
isOpenClawChatVideoFile({
name: "说明.pdf",
type: "application/pdf",
})
).toBe(false);
});
test("normalizes public video URLs to the OpenClaw attachment contract", () => {
expect(
normalizeOpenClawChatMediaAttachments([
{
type: "video",
url: "https://cdn.example.com/demo.mp4",
name: "demo.mp4",
contentType: "video/mp4",
},
])
).toEqual([
{
type: "video",
url: "https://cdn.example.com/demo.mp4",
name: "demo.mp4",
content_type: "video/mp4",
},
]);
});
test("rejects private schemes, duplicates, and videos beyond the gateway limit", () => {
const attachments = normalizeOpenClawChatMediaAttachments([
{ type: "video", url: "https://cdn.example.com/one.mp4" },
{ type: "video", url: "https://cdn.example.com/two.mov" },
{ type: "video", url: "https://cdn.example.com/three.webm" },
{ type: "video", url: "file:///tmp/private.mp4" },
{ type: "video", url: "https://cdn.example.com/one.mp4" },
]);
expect(attachments).toHaveLength(OPENCLAW_CHAT_VIDEO_MAX_FILES);
expect(attachments.map((attachment) => attachment.url)).toEqual([
"https://cdn.example.com/one.mp4",
"https://cdn.example.com/two.mov",
]);
});
test("builds a readable fallback message without exposing storage credentials", () => {
const attachments = normalizeOpenClawChatMediaAttachments([
{
type: "video",
url: "https://cdn.example.com/review.mp4",
name: "review.mp4",
},
]);
expect(buildOpenClawVideoDisplayText(attachments)).toContain("review.mp4");
expect(buildOpenClawVideoReferenceText(attachments)).toContain(
"https://cdn.example.com/review.mp4"
);
expect(buildOpenClawVideoReferenceText(attachments)).not.toContain(
"Authorization"
);
});
});