feat: support video attachments in Claw chat
This commit is contained in:
parent
21ba23c4ab
commit
fb8d8232dc
@ -21,6 +21,7 @@ import {
|
|||||||
maybeEnqueueStorageMigrationTask,
|
maybeEnqueueStorageMigrationTask,
|
||||||
resolveS3UploadTarget,
|
resolveS3UploadTarget,
|
||||||
} from "@/lib/storage/service";
|
} from "@/lib/storage/service";
|
||||||
|
import { OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE } from "@/lib/openclaw-chat/media-attachments";
|
||||||
|
|
||||||
const logger = createLogger("direct-upload-api");
|
const logger = createLogger("direct-upload-api");
|
||||||
const DIRECT_UPLOAD_MAX_BYTES = 500 * 1024 * 1024;
|
const DIRECT_UPLOAD_MAX_BYTES = 500 * 1024 * 1024;
|
||||||
@ -39,6 +40,13 @@ const allowedDirectUploadTypes = new Set([
|
|||||||
"image/png",
|
"image/png",
|
||||||
"image/webp",
|
"image/webp",
|
||||||
]);
|
]);
|
||||||
|
const allowedOpenClawVideoUploadTypes = new Set([
|
||||||
|
"video/mp4",
|
||||||
|
"video/quicktime",
|
||||||
|
"video/webm",
|
||||||
|
"video/x-m4v",
|
||||||
|
"video/x-matroska",
|
||||||
|
]);
|
||||||
|
|
||||||
const InitiateSchema = z.object({
|
const InitiateSchema = z.object({
|
||||||
action: z.literal("initiate"),
|
action: z.literal("initiate"),
|
||||||
@ -109,8 +117,12 @@ function assertUserOwnedKey(key: string, userId: string) {
|
|||||||
return normalized;
|
return normalized;
|
||||||
}
|
}
|
||||||
|
|
||||||
function isAllowedDirectUpload(contentType: string) {
|
function isAllowedDirectUpload(contentType: string, purpose?: string) {
|
||||||
return allowedDirectUploadTypes.has(contentType);
|
return (
|
||||||
|
allowedDirectUploadTypes.has(contentType) ||
|
||||||
|
(purpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE &&
|
||||||
|
allowedOpenClawVideoUploadTypes.has(contentType))
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
function computePartSize(size: number) {
|
function computePartSize(size: number) {
|
||||||
@ -204,7 +216,7 @@ export async function POST(request: Request) {
|
|||||||
payload.filename
|
payload.filename
|
||||||
);
|
);
|
||||||
|
|
||||||
if (!isAllowedDirectUpload(contentType)) {
|
if (!isAllowedDirectUpload(contentType, payload.purpose)) {
|
||||||
logger.warn("unsupported_direct_upload_type", {
|
logger.warn("unsupported_direct_upload_type", {
|
||||||
contentType,
|
contentType,
|
||||||
filenameExtension: payload.filename.toLowerCase().split(".").pop() || null,
|
filenameExtension: payload.filename.toLowerCase().split(".").pop() || null,
|
||||||
@ -213,7 +225,10 @@ export async function POST(request: Request) {
|
|||||||
clientUploadId: payload.clientUploadId || null,
|
clientUploadId: payload.clientUploadId || null,
|
||||||
});
|
});
|
||||||
return NextResponse.json(
|
return NextResponse.json(
|
||||||
{ error: "Direct upload only supports JPG, PNG and WebP images" },
|
{
|
||||||
|
error:
|
||||||
|
"直传仅支持 JPG、PNG、WebP 图片以及 Claw 对话的 MP4、MOV、WebM、M4V、MKV 视频",
|
||||||
|
},
|
||||||
{ status: 400 }
|
{ status: 400 }
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@ -328,9 +343,12 @@ export async function POST(request: Request) {
|
|||||||
|
|
||||||
const key = assertUserOwnedKey(payload.key, session.user.id);
|
const key = assertUserOwnedKey(payload.key, session.user.id);
|
||||||
const contentType = normalizeContentType(payload.contentType, payload.filename);
|
const contentType = normalizeContentType(payload.contentType, payload.filename);
|
||||||
if (!isAllowedDirectUpload(contentType)) {
|
if (!isAllowedDirectUpload(contentType, payload.purpose)) {
|
||||||
return NextResponse.json(
|
return NextResponse.json(
|
||||||
{ error: "Direct upload only supports JPG, PNG and WebP images" },
|
{
|
||||||
|
error:
|
||||||
|
"直传仅支持 JPG、PNG、WebP 图片以及 Claw 对话的 MP4、MOV、WebM、M4V、MKV 视频",
|
||||||
|
},
|
||||||
{ status: 400 }
|
{ status: 400 }
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@ -389,15 +407,21 @@ export async function POST(request: Request) {
|
|||||||
const assetStartedAt = Date.now();
|
const assetStartedAt = Date.now();
|
||||||
const asset = await upsertMediaAsset({
|
const asset = await upsertMediaAsset({
|
||||||
userId: session.user.id,
|
userId: session.user.id,
|
||||||
mediaType: "image",
|
mediaType: resolvedContentType.startsWith("video/") ? "video" : "image",
|
||||||
sourceType: "upload",
|
sourceType:
|
||||||
|
payload.purpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE
|
||||||
|
? "openclaw_chat_video_upload"
|
||||||
|
: "upload",
|
||||||
status: "ready",
|
status: "ready",
|
||||||
originalUrl: url,
|
originalUrl: url,
|
||||||
previewUrl: url,
|
previewUrl: url,
|
||||||
thumbnailUrl: url,
|
thumbnailUrl: url,
|
||||||
mimeType: resolvedContentType,
|
mimeType: resolvedContentType,
|
||||||
displayName: payload.filename,
|
displayName: payload.filename,
|
||||||
sourceTable: "direct_upload",
|
sourceTable:
|
||||||
|
payload.purpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE
|
||||||
|
? "openclaw_chat_video_upload"
|
||||||
|
: "direct_upload",
|
||||||
metadata: {
|
metadata: {
|
||||||
pathname: key,
|
pathname: key,
|
||||||
size: actualSize ?? payload.size,
|
size: actualSize ?? payload.size,
|
||||||
|
|||||||
@ -7,6 +7,7 @@ import { upsertMediaAsset } from "@/lib/db/queries";
|
|||||||
import { isSiteAdminType } from "@/lib/middleware/admin";
|
import { isSiteAdminType } from "@/lib/middleware/admin";
|
||||||
import { getContentType, uploadFile } from "@/lib/storage/service";
|
import { getContentType, uploadFile } from "@/lib/storage/service";
|
||||||
import { createLogger } from "@/lib/observability/logger";
|
import { createLogger } from "@/lib/observability/logger";
|
||||||
|
import { OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE } from "@/lib/openclaw-chat/media-attachments";
|
||||||
|
|
||||||
const logger = createLogger("upload-api");
|
const logger = createLogger("upload-api");
|
||||||
const ALLOWED_UPLOAD_IMAGE_TYPES = new Set([
|
const ALLOWED_UPLOAD_IMAGE_TYPES = new Set([
|
||||||
@ -26,6 +27,13 @@ const ALLOWED_ATLASCLOUD_VIDEO_TYPES = new Set([
|
|||||||
"video/quicktime",
|
"video/quicktime",
|
||||||
"video/webm",
|
"video/webm",
|
||||||
]);
|
]);
|
||||||
|
const ALLOWED_OPENCLAW_VIDEO_TYPES = new Set([
|
||||||
|
"video/mp4",
|
||||||
|
"video/quicktime",
|
||||||
|
"video/webm",
|
||||||
|
"video/x-m4v",
|
||||||
|
"video/x-matroska",
|
||||||
|
]);
|
||||||
|
|
||||||
const HEIC_UPLOAD_TYPES = new Set(["image/heic", "image/heif"]);
|
const HEIC_UPLOAD_TYPES = new Set(["image/heic", "image/heif"]);
|
||||||
const SHARP_COMPATIBILITY_UPLOAD_TYPES = new Set([
|
const SHARP_COMPATIBILITY_UPLOAD_TYPES = new Set([
|
||||||
@ -246,6 +254,9 @@ export async function POST(request: Request) {
|
|||||||
const isAtlasCloudReferenceVideo =
|
const isAtlasCloudReferenceVideo =
|
||||||
uploadPurpose === "canvas-video-process" &&
|
uploadPurpose === "canvas-video-process" &&
|
||||||
ALLOWED_ATLASCLOUD_VIDEO_TYPES.has(contentType);
|
ALLOWED_ATLASCLOUD_VIDEO_TYPES.has(contentType);
|
||||||
|
const isOpenClawChatVideoUpload =
|
||||||
|
uploadPurpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE &&
|
||||||
|
ALLOWED_OPENCLAW_VIDEO_TYPES.has(contentType);
|
||||||
const clientUploadId =
|
const clientUploadId =
|
||||||
String(formData.get("clientUploadId") || "").trim() || null;
|
String(formData.get("clientUploadId") || "").trim() || null;
|
||||||
const isSvgUpload = contentType === "image/svg+xml";
|
const isSvgUpload = contentType === "image/svg+xml";
|
||||||
@ -278,7 +289,8 @@ export async function POST(request: Request) {
|
|||||||
|
|
||||||
if (
|
if (
|
||||||
!ALLOWED_UPLOAD_IMAGE_TYPES.has(contentType) &&
|
!ALLOWED_UPLOAD_IMAGE_TYPES.has(contentType) &&
|
||||||
!isAtlasCloudReferenceVideo
|
!isAtlasCloudReferenceVideo &&
|
||||||
|
!isOpenClawChatVideoUpload
|
||||||
) {
|
) {
|
||||||
const errorMessage = createUnsupportedImageTypeMessage(contentType, filename);
|
const errorMessage = createUnsupportedImageTypeMessage(contentType, filename);
|
||||||
logger.warn("[Upload API] Unsupported upload content type", {
|
logger.warn("[Upload API] Unsupported upload content type", {
|
||||||
@ -445,15 +457,28 @@ export async function POST(request: Request) {
|
|||||||
try {
|
try {
|
||||||
const asset = await upsertMediaAsset({
|
const asset = await upsertMediaAsset({
|
||||||
userId: session.user.id,
|
userId: session.user.id,
|
||||||
mediaType: isAtlasCloudReferenceVideo ? "video" : "image",
|
mediaType:
|
||||||
sourceType: "upload",
|
isAtlasCloudReferenceVideo || isOpenClawChatVideoUpload
|
||||||
|
? "video"
|
||||||
|
: "image",
|
||||||
|
sourceType: isOpenClawChatVideoUpload
|
||||||
|
? "openclaw_chat_video_upload"
|
||||||
|
: "upload",
|
||||||
status: "ready",
|
status: "ready",
|
||||||
originalUrl: data.url,
|
originalUrl: data.url,
|
||||||
previewUrl: isAtlasCloudReferenceVideo ? null : data.url,
|
previewUrl:
|
||||||
thumbnailUrl: isAtlasCloudReferenceVideo ? null : data.url,
|
isAtlasCloudReferenceVideo || isOpenClawChatVideoUpload
|
||||||
|
? null
|
||||||
|
: data.url,
|
||||||
|
thumbnailUrl:
|
||||||
|
isAtlasCloudReferenceVideo || isOpenClawChatVideoUpload
|
||||||
|
? null
|
||||||
|
: data.url,
|
||||||
mimeType: data.contentType || contentType || null,
|
mimeType: data.contentType || contentType || null,
|
||||||
displayName: filename,
|
displayName: filename,
|
||||||
sourceTable: "upload",
|
sourceTable: isOpenClawChatVideoUpload
|
||||||
|
? "openclaw_chat_video_upload"
|
||||||
|
: "upload",
|
||||||
metadata: {
|
metadata: {
|
||||||
pathname: data.pathname,
|
pathname: data.pathname,
|
||||||
size: optimizedBuffer.length,
|
size: optimizedBuffer.length,
|
||||||
|
|||||||
@ -25,6 +25,11 @@ import {
|
|||||||
buildOpenClawMessageWithSpreadsheetMarkdown,
|
buildOpenClawMessageWithSpreadsheetMarkdown,
|
||||||
buildOpenClawSpreadsheetMarkdown,
|
buildOpenClawSpreadsheetMarkdown,
|
||||||
} from "@/lib/openclaw-chat/spreadsheet-markdown";
|
} from "@/lib/openclaw-chat/spreadsheet-markdown";
|
||||||
|
import {
|
||||||
|
normalizeOpenClawChatMediaAttachments,
|
||||||
|
OPENCLAW_CHAT_MEDIA_MAX_ATTACHMENTS,
|
||||||
|
type OpenClawChatMediaAttachment,
|
||||||
|
} from "@/lib/openclaw-chat/media-attachments";
|
||||||
import { generateUUID } from "@/lib/utils";
|
import { generateUUID } from "@/lib/utils";
|
||||||
|
|
||||||
const CLAW_UPGRADE_RETRY_MESSAGE = "claw正在升级更新稍等5分钟再试";
|
const CLAW_UPGRADE_RETRY_MESSAGE = "claw正在升级更新稍等5分钟再试";
|
||||||
@ -35,9 +40,18 @@ const requestSchema = z.object({
|
|||||||
displayMessage: z.string().trim().max(8000).optional(),
|
displayMessage: z.string().trim().max(8000).optional(),
|
||||||
userMessageId: z.string().uuid().optional(),
|
userMessageId: z.string().uuid().optional(),
|
||||||
metadata: z.record(z.unknown()).optional(),
|
metadata: z.record(z.unknown()).optional(),
|
||||||
|
attachments: z
|
||||||
|
.array(z.unknown())
|
||||||
|
.max(OPENCLAW_CHAT_MEDIA_MAX_ATTACHMENTS)
|
||||||
|
.optional()
|
||||||
|
.default([]),
|
||||||
});
|
});
|
||||||
|
|
||||||
type ParsedClawdChatRequest = z.infer<typeof requestSchema> & {
|
type ParsedClawdChatRequest = Omit<
|
||||||
|
z.infer<typeof requestSchema>,
|
||||||
|
"attachments"
|
||||||
|
> & {
|
||||||
|
attachments: OpenClawChatMediaAttachment[];
|
||||||
tableFiles: File[];
|
tableFiles: File[];
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -85,14 +99,23 @@ async function parseClawdChatRequest(
|
|||||||
const contentType = request.headers.get("content-type") || "";
|
const contentType = request.headers.get("content-type") || "";
|
||||||
if (!contentType.includes("multipart/form-data")) {
|
if (!contentType.includes("multipart/form-data")) {
|
||||||
const parsed = requestSchema.safeParse(await request.json().catch(() => null));
|
const parsed = requestSchema.safeParse(await request.json().catch(() => null));
|
||||||
return parsed.success ? { ...parsed.data, tableFiles: [] } : null;
|
if (!parsed.success) return null;
|
||||||
|
const attachments = normalizeOpenClawChatMediaAttachments(
|
||||||
|
parsed.data.attachments
|
||||||
|
);
|
||||||
|
if (attachments.length !== parsed.data.attachments.length) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return { ...parsed.data, attachments, tableFiles: [] };
|
||||||
}
|
}
|
||||||
|
|
||||||
const formData = await request.formData().catch(() => null);
|
const formData = await request.formData().catch(() => null);
|
||||||
if (!formData) return null;
|
if (!formData) return null;
|
||||||
|
|
||||||
const metadataValue = formData.get("metadata");
|
const metadataValue = formData.get("metadata");
|
||||||
|
const attachmentsValue = formData.get("attachments");
|
||||||
let metadata: Record<string, unknown> | undefined;
|
let metadata: Record<string, unknown> | undefined;
|
||||||
|
let attachments: unknown[] = [];
|
||||||
if (typeof metadataValue === "string" && metadataValue.trim()) {
|
if (typeof metadataValue === "string" && metadataValue.trim()) {
|
||||||
try {
|
try {
|
||||||
const parsedMetadata = JSON.parse(metadataValue);
|
const parsedMetadata = JSON.parse(metadataValue);
|
||||||
@ -105,6 +128,17 @@ async function parseClawdChatRequest(
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (typeof attachmentsValue === "string" && attachmentsValue.trim()) {
|
||||||
|
try {
|
||||||
|
const parsedAttachments = JSON.parse(attachmentsValue);
|
||||||
|
if (!Array.isArray(parsedAttachments)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
attachments = parsedAttachments;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const parsed = requestSchema.safeParse({
|
const parsed = requestSchema.safeParse({
|
||||||
id: formData.get("id"),
|
id: formData.get("id"),
|
||||||
@ -112,14 +146,25 @@ async function parseClawdChatRequest(
|
|||||||
displayMessage: formData.get("displayMessage") || undefined,
|
displayMessage: formData.get("displayMessage") || undefined,
|
||||||
userMessageId: formData.get("userMessageId") || undefined,
|
userMessageId: formData.get("userMessageId") || undefined,
|
||||||
metadata,
|
metadata,
|
||||||
|
attachments,
|
||||||
});
|
});
|
||||||
if (!parsed.success) return null;
|
if (!parsed.success) return null;
|
||||||
|
const normalizedAttachments = normalizeOpenClawChatMediaAttachments(
|
||||||
|
parsed.data.attachments
|
||||||
|
);
|
||||||
|
if (normalizedAttachments.length !== parsed.data.attachments.length) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
const tableFiles = formData
|
const tableFiles = formData
|
||||||
.getAll("tableFiles")
|
.getAll("tableFiles")
|
||||||
.filter((entry): entry is File => isRequestFile(entry) && entry.size > 0);
|
.filter((entry): entry is File => isRequestFile(entry) && entry.size > 0);
|
||||||
|
|
||||||
return { ...parsed.data, tableFiles };
|
return {
|
||||||
|
...parsed.data,
|
||||||
|
attachments: normalizedAttachments,
|
||||||
|
tableFiles,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
async function readOpenClawConfig() {
|
async function readOpenClawConfig() {
|
||||||
@ -163,8 +208,15 @@ export async function POST(request: Request) {
|
|||||||
if (!parsed) {
|
if (!parsed) {
|
||||||
return NextResponse.json({ error: "请求参数不正确" }, { status: 400 });
|
return NextResponse.json({ error: "请求参数不正确" }, { status: 400 });
|
||||||
}
|
}
|
||||||
if (!parsed.message && parsed.tableFiles.length === 0) {
|
if (
|
||||||
return NextResponse.json({ error: "请输入问题或上传表格" }, { status: 400 });
|
!parsed.message &&
|
||||||
|
parsed.tableFiles.length === 0 &&
|
||||||
|
parsed.attachments.length === 0
|
||||||
|
) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: "请输入问题或上传附件" },
|
||||||
|
{ status: 400 }
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const config = await readOpenClawConfig();
|
const config = await readOpenClawConfig();
|
||||||
@ -275,6 +327,8 @@ export async function POST(request: Request) {
|
|||||||
skill_tag_id: skillTagId || undefined,
|
skill_tag_id: skillTagId || undefined,
|
||||||
skill_tag_name: skillTagName || undefined,
|
skill_tag_name: skillTagName || undefined,
|
||||||
message: upstreamMessage,
|
message: upstreamMessage,
|
||||||
|
attachments:
|
||||||
|
parsed.attachments.length > 0 ? parsed.attachments : undefined,
|
||||||
metadata: {
|
metadata: {
|
||||||
source: "ai_chatbot_canvas",
|
source: "ai_chatbot_canvas",
|
||||||
page: "/canvas",
|
page: "/canvas",
|
||||||
@ -288,6 +342,8 @@ export async function POST(request: Request) {
|
|||||||
user_department: openClawDepartmentName,
|
user_department: openClawDepartmentName,
|
||||||
skill_tag_id: skillTagId || undefined,
|
skill_tag_id: skillTagId || undefined,
|
||||||
skill_tag_name: skillTagName || undefined,
|
skill_tag_name: skillTagName || undefined,
|
||||||
|
media_attachments:
|
||||||
|
parsed.attachments.length > 0 ? parsed.attachments : undefined,
|
||||||
spreadsheet_files: spreadsheetMarkdownResult?.files || undefined,
|
spreadsheet_files: spreadsheetMarkdownResult?.files || undefined,
|
||||||
spreadsheet_markdown_truncated:
|
spreadsheet_markdown_truncated:
|
||||||
spreadsheetMarkdownResult?.truncated || undefined,
|
spreadsheetMarkdownResult?.truncated || undefined,
|
||||||
@ -378,7 +434,15 @@ export async function POST(request: Request) {
|
|||||||
chatId: parsed.id,
|
chatId: parsed.id,
|
||||||
role: "user",
|
role: "user",
|
||||||
parts: [{ type: "text", text: displayMessage }],
|
parts: [{ type: "text", text: displayMessage }],
|
||||||
attachments: [],
|
attachments: parsed.attachments.map((attachment) => ({
|
||||||
|
name:
|
||||||
|
attachment.name ||
|
||||||
|
(attachment.type === "video" ? "视频附件" : "图片附件"),
|
||||||
|
url: attachment.url,
|
||||||
|
contentType:
|
||||||
|
attachment.content_type ||
|
||||||
|
(attachment.type === "video" ? "video/mp4" : "image/jpeg"),
|
||||||
|
})),
|
||||||
createdAt: userCreatedAt,
|
createdAt: userCreatedAt,
|
||||||
videoStorageStatus: null,
|
videoStorageStatus: null,
|
||||||
temporaryVideoUrl: null,
|
temporaryVideoUrl: null,
|
||||||
@ -387,6 +451,12 @@ export async function POST(request: Request) {
|
|||||||
videoMetadata: {
|
videoMetadata: {
|
||||||
composerMode: "clawd",
|
composerMode: "clawd",
|
||||||
source: "openclaw",
|
source: "openclaw",
|
||||||
|
imageAttachmentCount: parsed.attachments.filter(
|
||||||
|
(attachment) => attachment.type === "image"
|
||||||
|
).length,
|
||||||
|
videoAttachmentCount: parsed.attachments.filter(
|
||||||
|
(attachment) => attachment.type === "video"
|
||||||
|
).length,
|
||||||
skillTagId: skillTagId || null,
|
skillTagId: skillTagId || null,
|
||||||
skillTagName: skillTagName || null,
|
skillTagName: skillTagName || null,
|
||||||
},
|
},
|
||||||
@ -431,6 +501,7 @@ export async function POST(request: Request) {
|
|||||||
createdAt: userMessage.createdAt.toISOString(),
|
createdAt: userMessage.createdAt.toISOString(),
|
||||||
composerMode: "clawd",
|
composerMode: "clawd",
|
||||||
},
|
},
|
||||||
|
attachments: userMessage.attachments,
|
||||||
videoMetadata: userMessage.videoMetadata,
|
videoMetadata: userMessage.videoMetadata,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
@ -287,6 +287,18 @@ import {
|
|||||||
stripCanvasModelRoutingLines,
|
stripCanvasModelRoutingLines,
|
||||||
stripMediaMarkup,
|
stripMediaMarkup,
|
||||||
} from "@/components/canvas/chat-panel/canvas-chat-media";
|
} from "@/components/canvas/chat-panel/canvas-chat-media";
|
||||||
|
import {
|
||||||
|
buildOpenClawVideoDisplayText,
|
||||||
|
buildOpenClawVideoReferenceText,
|
||||||
|
getOpenClawChatVideoContentType,
|
||||||
|
isOpenClawChatVideoAttachment,
|
||||||
|
isOpenClawChatVideoFile,
|
||||||
|
normalizeOpenClawChatMediaAttachments,
|
||||||
|
OPENCLAW_CHAT_VIDEO_ACCEPT,
|
||||||
|
OPENCLAW_CHAT_VIDEO_MAX_BYTES,
|
||||||
|
OPENCLAW_CHAT_VIDEO_MAX_FILES,
|
||||||
|
OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE,
|
||||||
|
} from "@/lib/openclaw-chat/media-attachments";
|
||||||
|
|
||||||
type ResponseModule = typeof import("@/components/elements/response");
|
type ResponseModule = typeof import("@/components/elements/response");
|
||||||
type ResponseProps = ComponentProps<ResponseModule["Response"]>;
|
type ResponseProps = ComponentProps<ResponseModule["Response"]>;
|
||||||
@ -612,9 +624,10 @@ function writeStoredClawdComposerSubmissionSnapshot(
|
|||||||
function shouldShowClawdProcessingHint(
|
function shouldShowClawdProcessingHint(
|
||||||
message: string,
|
message: string,
|
||||||
hasTableFiles: boolean,
|
hasTableFiles: boolean,
|
||||||
hasImageAttachments: boolean
|
hasImageAttachments: boolean,
|
||||||
|
hasVideoAttachments: boolean
|
||||||
) {
|
) {
|
||||||
if (hasTableFiles || hasImageAttachments) {
|
if (hasTableFiles || hasImageAttachments || hasVideoAttachments) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -675,17 +688,19 @@ function buildClawdImageReferenceText(attachments: PickedObjectAttachment[]) {
|
|||||||
return `用户还添加了以下图片作为参考,请按图片链接理解:\n${lines.join("\n")}`;
|
return `用户还添加了以下图片作为参考,请按图片链接理解:\n${lines.join("\n")}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
function getClawdDefaultPrompt(hasTables: boolean, hasImages: boolean) {
|
function getClawdDefaultPrompt(
|
||||||
if (hasTables && hasImages) {
|
hasTables: boolean,
|
||||||
return "请分析上传的表格和图片内容";
|
hasImages: boolean,
|
||||||
}
|
hasVideos: boolean
|
||||||
if (hasTables) {
|
) {
|
||||||
return "请分析上传的表格内容";
|
const mediaNames = [
|
||||||
}
|
hasTables ? "表格" : "",
|
||||||
if (hasImages) {
|
hasImages ? "图片" : "",
|
||||||
return "请分析上传的图片内容";
|
hasVideos ? "视频" : "",
|
||||||
}
|
].filter(Boolean);
|
||||||
return "请分析";
|
return mediaNames.length > 0
|
||||||
|
? `请分析上传的${mediaNames.join("、")}内容`
|
||||||
|
: "请分析";
|
||||||
}
|
}
|
||||||
|
|
||||||
type VideoStoryboardStreamData =
|
type VideoStoryboardStreamData =
|
||||||
@ -11034,15 +11049,44 @@ export function CanvasChatPanel({
|
|||||||
: "";
|
: "";
|
||||||
const hasPendingClawdSkillTagPrompt = Boolean(clawdSkillTagPrompt);
|
const hasPendingClawdSkillTagPrompt = Boolean(clawdSkillTagPrompt);
|
||||||
const attachmentsSnapshot = isProgrammaticClawdQuickSend ? [] : [...attachments];
|
const attachmentsSnapshot = isProgrammaticClawdQuickSend ? [] : [...attachments];
|
||||||
const imageAttachmentsSnapshot = attachmentsSnapshot
|
const sanitizedAttachmentsSnapshot = attachmentsSnapshot
|
||||||
.map((attachment) => sanitizeChatAttachment(attachment))
|
.map((attachment) => sanitizeChatAttachment(attachment))
|
||||||
.filter((attachment): attachment is PickedObjectAttachment => Boolean(attachment));
|
.filter((attachment): attachment is PickedObjectAttachment => Boolean(attachment));
|
||||||
|
const videoAttachmentsSnapshot = sanitizedAttachmentsSnapshot.filter(
|
||||||
|
(attachment) =>
|
||||||
|
isOpenClawChatVideoAttachment(
|
||||||
|
attachment.url,
|
||||||
|
attachment.contentType
|
||||||
|
)
|
||||||
|
);
|
||||||
|
const imageAttachmentsSnapshot = sanitizedAttachmentsSnapshot.filter(
|
||||||
|
(attachment) =>
|
||||||
|
!isOpenClawChatVideoAttachment(
|
||||||
|
attachment.url,
|
||||||
|
attachment.contentType
|
||||||
|
)
|
||||||
|
);
|
||||||
const hasImageAttachments = imageAttachmentsSnapshot.length > 0;
|
const hasImageAttachments = imageAttachmentsSnapshot.length > 0;
|
||||||
|
const hasVideoAttachments = videoAttachmentsSnapshot.length > 0;
|
||||||
|
const openClawMediaAttachments = normalizeOpenClawChatMediaAttachments(
|
||||||
|
sanitizedAttachmentsSnapshot.map((attachment) => ({
|
||||||
|
type: isOpenClawChatVideoAttachment(
|
||||||
|
attachment.url,
|
||||||
|
attachment.contentType
|
||||||
|
)
|
||||||
|
? "video"
|
||||||
|
: "image",
|
||||||
|
url: attachment.url,
|
||||||
|
name: normalizeChatAttachmentName(attachment.name),
|
||||||
|
content_type: attachment.contentType,
|
||||||
|
}))
|
||||||
|
);
|
||||||
|
|
||||||
if (
|
if (
|
||||||
!textContent &&
|
!textContent &&
|
||||||
!hasTableFiles &&
|
!hasTableFiles &&
|
||||||
!hasImageAttachments &&
|
!hasImageAttachments &&
|
||||||
|
!hasVideoAttachments &&
|
||||||
!hasPendingClawdSkillTagPrompt
|
!hasPendingClawdSkillTagPrompt
|
||||||
) {
|
) {
|
||||||
return;
|
return;
|
||||||
@ -11052,14 +11096,22 @@ export function CanvasChatPanel({
|
|||||||
textContent ||
|
textContent ||
|
||||||
(clawdSkillTagForRequest && hasPendingClawdSkillTagPrompt
|
(clawdSkillTagForRequest && hasPendingClawdSkillTagPrompt
|
||||||
? `请按「${clawdSkillTagForRequest.name}」快捷功能处理。`
|
? `请按「${clawdSkillTagForRequest.name}」快捷功能处理。`
|
||||||
: getClawdDefaultPrompt(hasTableFiles, hasImageAttachments));
|
: getClawdDefaultPrompt(
|
||||||
|
hasTableFiles,
|
||||||
|
hasImageAttachments,
|
||||||
|
hasVideoAttachments
|
||||||
|
));
|
||||||
const imageReferenceText = buildClawdImageReferenceText(
|
const imageReferenceText = buildClawdImageReferenceText(
|
||||||
imageAttachmentsSnapshot
|
imageAttachmentsSnapshot
|
||||||
);
|
);
|
||||||
|
const videoReferenceText = buildOpenClawVideoReferenceText(
|
||||||
|
openClawMediaAttachments
|
||||||
|
);
|
||||||
const requestMessage = [
|
const requestMessage = [
|
||||||
baseRequestMessage,
|
baseRequestMessage,
|
||||||
clawdSkillTagPrompt,
|
clawdSkillTagPrompt,
|
||||||
imageReferenceText,
|
imageReferenceText,
|
||||||
|
videoReferenceText,
|
||||||
]
|
]
|
||||||
.filter(Boolean)
|
.filter(Boolean)
|
||||||
.join("\n\n");
|
.join("\n\n");
|
||||||
@ -11078,6 +11130,7 @@ export function CanvasChatPanel({
|
|||||||
const optimisticMessageText = [
|
const optimisticMessageText = [
|
||||||
buildClawdTableDisplayText(optimisticBaseMessage, tableFilesSnapshot),
|
buildClawdTableDisplayText(optimisticBaseMessage, tableFilesSnapshot),
|
||||||
buildClawdImageDisplayText(imageAttachmentsSnapshot),
|
buildClawdImageDisplayText(imageAttachmentsSnapshot),
|
||||||
|
buildOpenClawVideoDisplayText(openClawMediaAttachments),
|
||||||
]
|
]
|
||||||
.filter(Boolean)
|
.filter(Boolean)
|
||||||
.join("\n\n");
|
.join("\n\n");
|
||||||
@ -11094,6 +11147,10 @@ export function CanvasChatPanel({
|
|||||||
.map((attachment) => normalizeMediaUrl(attachment.url))
|
.map((attachment) => normalizeMediaUrl(attachment.url))
|
||||||
.filter(Boolean)
|
.filter(Boolean)
|
||||||
.sort(),
|
.sort(),
|
||||||
|
videoAttachments: videoAttachmentsSnapshot
|
||||||
|
.map((attachment) => normalizeMediaUrl(attachment.url))
|
||||||
|
.filter(Boolean)
|
||||||
|
.sort(),
|
||||||
});
|
});
|
||||||
|
|
||||||
if (shouldBlockDuplicateSend(activeChatId, sendSignature)) {
|
if (shouldBlockDuplicateSend(activeChatId, sendSignature)) {
|
||||||
@ -11104,7 +11161,8 @@ export function CanvasChatPanel({
|
|||||||
const nextProcessingHint = shouldShowClawdProcessingHint(
|
const nextProcessingHint = shouldShowClawdProcessingHint(
|
||||||
[textContent, clawdSkillTagPrompt].filter(Boolean).join("\n\n"),
|
[textContent, clawdSkillTagPrompt].filter(Boolean).join("\n\n"),
|
||||||
hasTableFiles,
|
hasTableFiles,
|
||||||
hasImageAttachments
|
hasImageAttachments,
|
||||||
|
hasVideoAttachments
|
||||||
)
|
)
|
||||||
? CLAWD_PROCESSING_HINT
|
? CLAWD_PROCESSING_HINT
|
||||||
: null;
|
: null;
|
||||||
@ -11207,11 +11265,18 @@ export function CanvasChatPanel({
|
|||||||
composerMode: "clawd",
|
composerMode: "clawd",
|
||||||
activeChatId,
|
activeChatId,
|
||||||
imageAttachmentCount: imageAttachmentsSnapshot.length || undefined,
|
imageAttachmentCount: imageAttachmentsSnapshot.length || undefined,
|
||||||
|
videoAttachmentCount: videoAttachmentsSnapshot.length || undefined,
|
||||||
skillTagId: clawdSkillTagForRequest?.id || undefined,
|
skillTagId: clawdSkillTagForRequest?.id || undefined,
|
||||||
skillTagName: clawdSkillTagForRequest?.name || undefined,
|
skillTagName: clawdSkillTagForRequest?.name || undefined,
|
||||||
skillTagPromptIncluded: hasPendingClawdSkillTagPrompt || undefined,
|
skillTagPromptIncluded: hasPendingClawdSkillTagPrompt || undefined,
|
||||||
})
|
})
|
||||||
);
|
);
|
||||||
|
if (openClawMediaAttachments.length > 0) {
|
||||||
|
formData.append(
|
||||||
|
"attachments",
|
||||||
|
JSON.stringify(openClawMediaAttachments)
|
||||||
|
);
|
||||||
|
}
|
||||||
tableFilesSnapshot.forEach((file) => {
|
tableFilesSnapshot.forEach((file) => {
|
||||||
formData.append("tableFiles", file, file.name);
|
formData.append("tableFiles", file, file.name);
|
||||||
});
|
});
|
||||||
@ -12042,6 +12107,70 @@ export function CanvasChatPanel({
|
|||||||
}
|
}
|
||||||
}, [clearComposerUploadProgress, updateComposerUploadProgress]);
|
}, [clearComposerUploadProgress, updateComposerUploadProgress]);
|
||||||
|
|
||||||
|
const uploadClawdVideoFileForComposerAttachment = useCallback(async (
|
||||||
|
file: File,
|
||||||
|
progressOptions: CanvasUploadProgressOptions
|
||||||
|
): Promise<ComposerUploadedImage | null> => {
|
||||||
|
if (!isOpenClawChatVideoFile(file)) {
|
||||||
|
toast.error("仅支持 MP4、MOV、WebM、M4V 或 MKV 视频");
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (file.size <= 0) {
|
||||||
|
toast.error(`${file.name} 是空文件`);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (file.size > OPENCLAW_CHAT_VIDEO_MAX_BYTES) {
|
||||||
|
toast.error(`${file.name} 超过 500MB,请先压缩视频`);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const normalizedContentType =
|
||||||
|
getOpenClawChatVideoContentType(file) || "video/mp4";
|
||||||
|
const uploadFile =
|
||||||
|
file.type === normalizedContentType
|
||||||
|
? file
|
||||||
|
: new File([file], file.name, {
|
||||||
|
type: normalizedContentType,
|
||||||
|
lastModified: file.lastModified,
|
||||||
|
});
|
||||||
|
const formData = new FormData();
|
||||||
|
formData.append("file", uploadFile);
|
||||||
|
formData.append("purpose", OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE);
|
||||||
|
updateComposerUploadProgress(progressOptions);
|
||||||
|
const response = await fetchUploadWithRetry(formData, {
|
||||||
|
onProgress: (progress) => {
|
||||||
|
updateComposerUploadProgress(progressOptions, progress);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const data = await response.json().catch(() => null);
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new Error(String(data?.error || data?.details || "视频上传失败"));
|
||||||
|
}
|
||||||
|
|
||||||
|
const imageUrl = String(data?.url || "").trim();
|
||||||
|
if (!isValidChatHttpUrl(imageUrl)) {
|
||||||
|
throw new Error("储存桶未返回可访问的视频地址");
|
||||||
|
}
|
||||||
|
|
||||||
|
clearComposerUploadProgress(250);
|
||||||
|
return {
|
||||||
|
imageUrl,
|
||||||
|
name: file.name || `视频-${Date.now()}.mp4`,
|
||||||
|
contentType:
|
||||||
|
String(data?.contentType || "").trim() ||
|
||||||
|
normalizedContentType,
|
||||||
|
assetId:
|
||||||
|
typeof data?.assetId === "string" && data.assetId.trim()
|
||||||
|
? data.assetId.trim()
|
||||||
|
: undefined,
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
toast.error(error instanceof Error ? error.message : "视频上传失败");
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}, [clearComposerUploadProgress, updateComposerUploadProgress]);
|
||||||
|
|
||||||
const handlePastedImages = useCallback(async (files: File[]) => {
|
const handlePastedImages = useCallback(async (files: File[]) => {
|
||||||
if (files.length === 0) return;
|
if (files.length === 0) return;
|
||||||
|
|
||||||
@ -12321,19 +12450,30 @@ export function CanvasChatPanel({
|
|||||||
|
|
||||||
if (isClawdMode) {
|
if (isClawdMode) {
|
||||||
const tableFiles = selectedFiles.filter((file) => isClawdTableFile(file));
|
const tableFiles = selectedFiles.filter((file) => isClawdTableFile(file));
|
||||||
const imageFiles = selectedFiles.filter((file) => isClawdImageFile(file));
|
const imageFiles = selectedFiles.filter((file) =>
|
||||||
|
file.type.startsWith("image/")
|
||||||
|
);
|
||||||
|
const videoFiles = selectedFiles.filter((file) =>
|
||||||
|
isOpenClawChatVideoFile(file)
|
||||||
|
);
|
||||||
const unsupportedFiles = selectedFiles.filter(
|
const unsupportedFiles = selectedFiles.filter(
|
||||||
(file) => !isClawdTableFile(file) && !isClawdImageFile(file)
|
(file) =>
|
||||||
|
!isClawdTableFile(file) &&
|
||||||
|
!file.type.startsWith("image/") &&
|
||||||
|
!isOpenClawChatVideoFile(file)
|
||||||
);
|
);
|
||||||
|
|
||||||
unsupportedFiles.forEach((file) => {
|
unsupportedFiles.forEach((file) => {
|
||||||
toast.error(`${file.name} 格式不支持,请上传图片或 .xlsx/.csv/.tsv`);
|
toast.error(
|
||||||
|
`${file.name} 格式不支持,请上传图片、视频或 .xlsx/.csv/.tsv`
|
||||||
|
);
|
||||||
});
|
});
|
||||||
if (tableFiles.length > 0) {
|
if (tableFiles.length > 0) {
|
||||||
addClawdTableFiles(tableFiles);
|
addClawdTableFiles(tableFiles);
|
||||||
}
|
}
|
||||||
|
|
||||||
let handledImageCount = 0;
|
let handledImageCount = 0;
|
||||||
|
let handledVideoCount = 0;
|
||||||
if (imageFiles.length > 0) {
|
if (imageFiles.length > 0) {
|
||||||
try {
|
try {
|
||||||
for (const [index, file] of imageFiles.entries()) {
|
for (const [index, file] of imageFiles.entries()) {
|
||||||
@ -12362,6 +12502,49 @@ export function CanvasChatPanel({
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (videoFiles.length > 0) {
|
||||||
|
const existingVideoCount = attachments.filter((attachment) =>
|
||||||
|
isOpenClawChatVideoAttachment(
|
||||||
|
attachment.url,
|
||||||
|
attachment.contentType
|
||||||
|
)
|
||||||
|
).length;
|
||||||
|
const availableVideoSlots = Math.max(
|
||||||
|
0,
|
||||||
|
OPENCLAW_CHAT_VIDEO_MAX_FILES - existingVideoCount
|
||||||
|
);
|
||||||
|
const acceptedVideoFiles = videoFiles.slice(0, availableVideoSlots);
|
||||||
|
if (acceptedVideoFiles.length < videoFiles.length) {
|
||||||
|
toast.info(
|
||||||
|
`${CLAW_MODE_LABEL} 对话一次最多添加 ${OPENCLAW_CHAT_VIDEO_MAX_FILES} 个视频`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
for (const [index, file] of acceptedVideoFiles.entries()) {
|
||||||
|
const uploaded = await uploadClawdVideoFileForComposerAttachment(
|
||||||
|
file,
|
||||||
|
{
|
||||||
|
title:
|
||||||
|
acceptedVideoFiles.length === 1
|
||||||
|
? "正在上传视频到储存桶"
|
||||||
|
: `正在上传 ${acceptedVideoFiles.length} 个视频`,
|
||||||
|
fileName: file.name,
|
||||||
|
fileIndex: index,
|
||||||
|
fileTotal: acceptedVideoFiles.length,
|
||||||
|
target: "composer-reference",
|
||||||
|
}
|
||||||
|
);
|
||||||
|
if (!uploaded) continue;
|
||||||
|
|
||||||
|
handledVideoCount += 1;
|
||||||
|
addUploadedImageAttachmentToComposer(uploaded);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
clearComposerUploadProgress(450);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (handledImageCount > 0) {
|
if (handledImageCount > 0) {
|
||||||
toast.success(
|
toast.success(
|
||||||
handledImageCount === 1
|
handledImageCount === 1
|
||||||
@ -12369,6 +12552,13 @@ export function CanvasChatPanel({
|
|||||||
: `已添加 ${handledImageCount} 张图片为 ${CLAW_MODE_LABEL} 附件`
|
: `已添加 ${handledImageCount} 张图片为 ${CLAW_MODE_LABEL} 附件`
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
if (handledVideoCount > 0) {
|
||||||
|
toast.success(
|
||||||
|
handledVideoCount === 1
|
||||||
|
? `视频已添加为 ${CLAW_MODE_LABEL} 附件`
|
||||||
|
: `已添加 ${handledVideoCount} 个视频为 ${CLAW_MODE_LABEL} 附件`
|
||||||
|
);
|
||||||
|
}
|
||||||
e.target.value = "";
|
e.target.value = "";
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@ -12746,7 +12936,7 @@ export function CanvasChatPanel({
|
|||||||
? activeClawdSkillTag
|
? activeClawdSkillTag
|
||||||
? activeClawdSkillTag.placeholder?.trim() ||
|
? activeClawdSkillTag.placeholder?.trim() ||
|
||||||
`继续向 ${CLAW_MODE_LABEL} 提问,当前使用「${activeClawdSkillTag.name}」`
|
`继续向 ${CLAW_MODE_LABEL} 提问,当前使用「${activeClawdSkillTag.name}」`
|
||||||
: `输入要交给 ${CLAW_MODE_LABEL} 的问题,可上传图片或表格`
|
: `输入要交给 ${CLAW_MODE_LABEL} 的问题,可上传图片、视频或表格`
|
||||||
: isDetailGenerationMode
|
: isDetailGenerationMode
|
||||||
? "可选补充:风格、情绪氛围、禁用元素、期望生成张数(如:生成8张)"
|
? "可选补充:风格、情绪氛围、禁用元素、期望生成张数(如:生成8张)"
|
||||||
: isVideoWorkspace
|
: isVideoWorkspace
|
||||||
@ -12760,7 +12950,7 @@ export function CanvasChatPanel({
|
|||||||
? activeClawdSkillTag
|
? activeClawdSkillTag
|
||||||
? activeClawdSkillTag.placeholder?.trim() ||
|
? activeClawdSkillTag.placeholder?.trim() ||
|
||||||
`继续向 ${CLAW_MODE_LABEL} 提问,当前使用「${activeClawdSkillTag.name}」`
|
`继续向 ${CLAW_MODE_LABEL} 提问,当前使用「${activeClawdSkillTag.name}」`
|
||||||
: `输入要交给 ${CLAW_MODE_LABEL} 的问题,可上传图片或表格`
|
: `输入要交给 ${CLAW_MODE_LABEL} 的问题,可上传图片、视频或表格`
|
||||||
: isDetailGenerationMode
|
: isDetailGenerationMode
|
||||||
? "可选补充:风格、情绪氛围、禁用元素、期望生成张数(如:生成8张)"
|
? "可选补充:风格、情绪氛围、禁用元素、期望生成张数(如:生成8张)"
|
||||||
: isVideoWorkspace
|
: isVideoWorkspace
|
||||||
@ -16125,7 +16315,7 @@ export function CanvasChatPanel({
|
|||||||
type="file"
|
type="file"
|
||||||
accept={
|
accept={
|
||||||
isClawdMode
|
isClawdMode
|
||||||
? `${CLAWD_IMAGE_ACCEPT},${CLAWD_TABLE_ACCEPT}`
|
? `${CLAWD_IMAGE_ACCEPT},${OPENCLAW_CHAT_VIDEO_ACCEPT},${CLAWD_TABLE_ACCEPT}`
|
||||||
: supportsAtlasVideoReference
|
: supportsAtlasVideoReference
|
||||||
? "image/*,video/mp4,video/quicktime,video/webm"
|
? "image/*,video/mp4,video/quicktime,video/webm"
|
||||||
: "image/*"
|
: "image/*"
|
||||||
|
|||||||
@ -1,3 +1,5 @@
|
|||||||
|
import { OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE } from "@/lib/openclaw-chat/media-attachments";
|
||||||
|
|
||||||
const DEFAULT_MAX_RETRIES = 3;
|
const DEFAULT_MAX_RETRIES = 3;
|
||||||
const MAX_BACKOFF_MS = 8000;
|
const MAX_BACKOFF_MS = 8000;
|
||||||
const MAX_CONCURRENT_UPLOADS = 3;
|
const MAX_CONCURRENT_UPLOADS = 3;
|
||||||
@ -30,6 +32,13 @@ const DIRECT_UPLOAD_IMAGE_TYPES = new Set([
|
|||||||
"image/png",
|
"image/png",
|
||||||
"image/webp",
|
"image/webp",
|
||||||
]);
|
]);
|
||||||
|
const DIRECT_UPLOAD_OPENCLAW_VIDEO_TYPES = new Set([
|
||||||
|
"video/mp4",
|
||||||
|
"video/quicktime",
|
||||||
|
"video/webm",
|
||||||
|
"video/x-m4v",
|
||||||
|
"video/x-matroska",
|
||||||
|
]);
|
||||||
|
|
||||||
const BROWSER_IMAGE_TYPES_BY_EXTENSION: Record<string, string> = {
|
const BROWSER_IMAGE_TYPES_BY_EXTENSION: Record<string, string> = {
|
||||||
jpg: "image/jpeg",
|
jpg: "image/jpeg",
|
||||||
@ -298,7 +307,12 @@ function shouldTryDirectUpload(file: UploadFormFile) {
|
|||||||
if (file.blob.size <= 0 || file.blob.size > DIRECT_UPLOAD_MAX_BYTES) {
|
if (file.blob.size <= 0 || file.blob.size > DIRECT_UPLOAD_MAX_BYTES) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (!DIRECT_UPLOAD_IMAGE_TYPES.has(file.contentType.toLowerCase())) {
|
const normalizedContentType = file.contentType.toLowerCase();
|
||||||
|
const supportsDirectUpload =
|
||||||
|
DIRECT_UPLOAD_IMAGE_TYPES.has(normalizedContentType) ||
|
||||||
|
(file.purpose === OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE &&
|
||||||
|
DIRECT_UPLOAD_OPENCLAW_VIDEO_TYPES.has(normalizedContentType));
|
||||||
|
if (!supportsDirectUpload) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (file.purpose === "agent-workflow-preview") {
|
if (file.purpose === "agent-workflow-preview") {
|
||||||
@ -848,7 +862,7 @@ async function fetchDirectUpload(
|
|||||||
return createJsonResponse(
|
return createJsonResponse(
|
||||||
{
|
{
|
||||||
error:
|
error:
|
||||||
"图片超过 50MB,必须使用浏览器直传;当前直传失败,请刷新后重试,或联系管理员检查存储桶 CORS/ETag 配置。",
|
"文件超过 50MB,必须使用浏览器直传;当前直传失败,请刷新后重试,或联系管理员检查存储桶 CORS/ETag 配置。",
|
||||||
details: error instanceof Error ? error.message : String(error),
|
details: error instanceof Error ? error.message : String(error),
|
||||||
clientUploadId: file.clientUploadId,
|
clientUploadId: file.clientUploadId,
|
||||||
},
|
},
|
||||||
|
|||||||
183
lib/openclaw-chat/media-attachments.ts
Normal file
183
lib/openclaw-chat/media-attachments.ts
Normal file
@ -0,0 +1,183 @@
|
|||||||
|
export const OPENCLAW_CHAT_VIDEO_UPLOAD_PURPOSE = "openclaw-chat-video";
|
||||||
|
export const OPENCLAW_CHAT_VIDEO_MAX_FILES = 2;
|
||||||
|
export const OPENCLAW_CHAT_VIDEO_MAX_BYTES = 500 * 1024 * 1024;
|
||||||
|
export const OPENCLAW_CHAT_MEDIA_MAX_ATTACHMENTS = 16;
|
||||||
|
export const OPENCLAW_CHAT_VIDEO_ACCEPT =
|
||||||
|
"video/mp4,video/quicktime,video/webm,video/x-m4v,video/x-matroska,.mp4,.mov,.webm,.m4v,.mkv";
|
||||||
|
|
||||||
|
const VIDEO_CONTENT_TYPES = new Set([
|
||||||
|
"video/mp4",
|
||||||
|
"video/quicktime",
|
||||||
|
"video/webm",
|
||||||
|
"video/x-m4v",
|
||||||
|
"video/x-matroska",
|
||||||
|
]);
|
||||||
|
|
||||||
|
const VIDEO_CONTENT_TYPE_BY_EXTENSION: Record<string, string> = {
|
||||||
|
m4v: "video/x-m4v",
|
||||||
|
mkv: "video/x-matroska",
|
||||||
|
mov: "video/quicktime",
|
||||||
|
mp4: "video/mp4",
|
||||||
|
webm: "video/webm",
|
||||||
|
};
|
||||||
|
|
||||||
|
const VIDEO_EXTENSION_PATTERN = /\.(mp4|mov|webm|m4v|mkv)(?:$|[?#])/i;
|
||||||
|
|
||||||
|
export type OpenClawChatMediaAttachment = {
|
||||||
|
type: "image" | "video";
|
||||||
|
url: string;
|
||||||
|
name?: string;
|
||||||
|
content_type?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
type OpenClawChatAttachmentLike = {
|
||||||
|
type?: unknown;
|
||||||
|
url?: unknown;
|
||||||
|
name?: unknown;
|
||||||
|
contentType?: unknown;
|
||||||
|
content_type?: unknown;
|
||||||
|
};
|
||||||
|
|
||||||
|
type OpenClawChatVideoFileLike = {
|
||||||
|
name: string;
|
||||||
|
type: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
function normalizeContentType(value: unknown) {
|
||||||
|
return String(value || "")
|
||||||
|
.split(";", 1)[0]
|
||||||
|
.trim()
|
||||||
|
.toLowerCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
function getExtension(value: string) {
|
||||||
|
const cleanValue = String(value || "").split(/[?#]/, 1)[0];
|
||||||
|
return cleanValue.split(".").pop()?.trim().toLowerCase() || "";
|
||||||
|
}
|
||||||
|
|
||||||
|
function isHttpUrl(value: string) {
|
||||||
|
try {
|
||||||
|
const parsed = new URL(value);
|
||||||
|
return parsed.protocol === "http:" || parsed.protocol === "https:";
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getOpenClawChatVideoContentType(
|
||||||
|
file: OpenClawChatVideoFileLike
|
||||||
|
) {
|
||||||
|
const contentType = normalizeContentType(file.type);
|
||||||
|
if (VIDEO_CONTENT_TYPES.has(contentType)) {
|
||||||
|
return contentType;
|
||||||
|
}
|
||||||
|
return VIDEO_CONTENT_TYPE_BY_EXTENSION[getExtension(file.name)] || "";
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isOpenClawChatVideoFile(file: OpenClawChatVideoFileLike) {
|
||||||
|
return Boolean(getOpenClawChatVideoContentType(file));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isOpenClawChatVideoAttachment(
|
||||||
|
url: string,
|
||||||
|
contentType?: string | null
|
||||||
|
) {
|
||||||
|
return (
|
||||||
|
normalizeContentType(contentType).startsWith("video/") ||
|
||||||
|
VIDEO_EXTENSION_PATTERN.test(String(url || ""))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeOpenClawChatMediaAttachments(
|
||||||
|
value: unknown
|
||||||
|
): OpenClawChatMediaAttachment[] {
|
||||||
|
if (!Array.isArray(value)) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
const result: OpenClawChatMediaAttachment[] = [];
|
||||||
|
const seenUrls = new Set<string>();
|
||||||
|
let videoCount = 0;
|
||||||
|
|
||||||
|
for (const item of value) {
|
||||||
|
if (!item || typeof item !== "object") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const raw = item as OpenClawChatAttachmentLike;
|
||||||
|
const url = String(raw.url || "").trim();
|
||||||
|
if (!url || !isHttpUrl(url) || seenUrls.has(url)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const requestedType = String(raw.type || "").trim().toLowerCase();
|
||||||
|
const rawContentType = normalizeContentType(
|
||||||
|
raw.content_type || raw.contentType
|
||||||
|
);
|
||||||
|
const isVideo =
|
||||||
|
requestedType === "video" ||
|
||||||
|
isOpenClawChatVideoAttachment(url, rawContentType);
|
||||||
|
|
||||||
|
if (isVideo && videoCount >= OPENCLAW_CHAT_VIDEO_MAX_FILES) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (result.length >= OPENCLAW_CHAT_MEDIA_MAX_ATTACHMENTS) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
const type = isVideo ? "video" : "image";
|
||||||
|
const fallbackContentType = isVideo
|
||||||
|
? VIDEO_CONTENT_TYPE_BY_EXTENSION[getExtension(url)] || "video/mp4"
|
||||||
|
: "image/jpeg";
|
||||||
|
const contentType =
|
||||||
|
rawContentType.startsWith(`${type}/`) ? rawContentType : fallbackContentType;
|
||||||
|
const name = String(raw.name || "")
|
||||||
|
.trim()
|
||||||
|
.slice(0, 200);
|
||||||
|
|
||||||
|
result.push({
|
||||||
|
type,
|
||||||
|
url,
|
||||||
|
...(name ? { name } : {}),
|
||||||
|
content_type: contentType,
|
||||||
|
});
|
||||||
|
seenUrls.add(url);
|
||||||
|
if (isVideo) {
|
||||||
|
videoCount += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildOpenClawVideoDisplayText(
|
||||||
|
attachments: OpenClawChatMediaAttachment[]
|
||||||
|
) {
|
||||||
|
const videos = attachments.filter((attachment) => attachment.type === "video");
|
||||||
|
if (videos.length === 0) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
const lines = videos.map(
|
||||||
|
(attachment, index) => `- ${attachment.name || `视频${index + 1}`}`
|
||||||
|
);
|
||||||
|
return `已上传视频:\n${lines.join("\n")}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildOpenClawVideoReferenceText(
|
||||||
|
attachments: OpenClawChatMediaAttachment[]
|
||||||
|
) {
|
||||||
|
const videos = attachments.filter((attachment) => attachment.type === "video");
|
||||||
|
if (videos.length === 0) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
const lines = videos.map(
|
||||||
|
(attachment, index) =>
|
||||||
|
`- ${attachment.name || `视频${index + 1}`}: ${attachment.url}`
|
||||||
|
);
|
||||||
|
return [
|
||||||
|
"用户上传了以下视频,请调用视频理解能力并按视频文件地址处理:",
|
||||||
|
lines.join("\n"),
|
||||||
|
].join("\n");
|
||||||
|
}
|
||||||
@ -129,6 +129,11 @@ export function getContentType(filename: string): string {
|
|||||||
woff2: "font/woff2",
|
woff2: "font/woff2",
|
||||||
glb: "model/gltf-binary",
|
glb: "model/gltf-binary",
|
||||||
gltf: "model/gltf+json",
|
gltf: "model/gltf+json",
|
||||||
|
mp4: "video/mp4",
|
||||||
|
mov: "video/quicktime",
|
||||||
|
webm: "video/webm",
|
||||||
|
m4v: "video/x-m4v",
|
||||||
|
mkv: "video/x-matroska",
|
||||||
usdz: "model/vnd.usdz+zip",
|
usdz: "model/vnd.usdz+zip",
|
||||||
fbx: "application/octet-stream",
|
fbx: "application/octet-stream",
|
||||||
obj: "text/plain",
|
obj: "text/plain",
|
||||||
|
|||||||
@ -14,6 +14,7 @@ export default defineConfig({
|
|||||||
/routes\/canvas-runtime-internal-endpoint\.test\.ts/,
|
/routes\/canvas-runtime-internal-endpoint\.test\.ts/,
|
||||||
/routes\/model-classification\.test\.ts/,
|
/routes\/model-classification\.test\.ts/,
|
||||||
/routes\/clawd-request-lifecycle\.test\.ts/,
|
/routes\/clawd-request-lifecycle\.test\.ts/,
|
||||||
|
/routes\/openclaw-video-attachments\.test\.ts/,
|
||||||
/routes\/hicad-embedded-performance\.test\.ts/,
|
/routes\/hicad-embedded-performance\.test\.ts/,
|
||||||
/routes\/hicad-manufacturing-extension\.test\.ts/,
|
/routes\/hicad-manufacturing-extension\.test\.ts/,
|
||||||
],
|
],
|
||||||
|
|||||||
92
tests/routes/openclaw-video-attachments.test.ts
Normal file
92
tests/routes/openclaw-video-attachments.test.ts
Normal file
@ -0,0 +1,92 @@
|
|||||||
|
import { expect, test } from "@playwright/test";
|
||||||
|
import {
|
||||||
|
buildOpenClawVideoDisplayText,
|
||||||
|
buildOpenClawVideoReferenceText,
|
||||||
|
getOpenClawChatVideoContentType,
|
||||||
|
isOpenClawChatVideoFile,
|
||||||
|
normalizeOpenClawChatMediaAttachments,
|
||||||
|
OPENCLAW_CHAT_VIDEO_MAX_FILES,
|
||||||
|
} from "@/lib/openclaw-chat/media-attachments";
|
||||||
|
|
||||||
|
test.describe("OpenClaw video attachments", () => {
|
||||||
|
test("accepts the supported browser video formats by MIME type or extension", () => {
|
||||||
|
expect(
|
||||||
|
isOpenClawChatVideoFile({
|
||||||
|
name: "市场分析.mp4",
|
||||||
|
type: "video/mp4",
|
||||||
|
})
|
||||||
|
).toBe(true);
|
||||||
|
expect(
|
||||||
|
isOpenClawChatVideoFile({
|
||||||
|
name: "素材.MOV",
|
||||||
|
type: "application/octet-stream",
|
||||||
|
})
|
||||||
|
).toBe(true);
|
||||||
|
expect(
|
||||||
|
getOpenClawChatVideoContentType({
|
||||||
|
name: "演示.m4v",
|
||||||
|
type: "",
|
||||||
|
})
|
||||||
|
).toBe("video/x-m4v");
|
||||||
|
expect(
|
||||||
|
isOpenClawChatVideoFile({
|
||||||
|
name: "说明.pdf",
|
||||||
|
type: "application/pdf",
|
||||||
|
})
|
||||||
|
).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("normalizes public video URLs to the OpenClaw attachment contract", () => {
|
||||||
|
expect(
|
||||||
|
normalizeOpenClawChatMediaAttachments([
|
||||||
|
{
|
||||||
|
type: "video",
|
||||||
|
url: "https://cdn.example.com/demo.mp4",
|
||||||
|
name: "demo.mp4",
|
||||||
|
contentType: "video/mp4",
|
||||||
|
},
|
||||||
|
])
|
||||||
|
).toEqual([
|
||||||
|
{
|
||||||
|
type: "video",
|
||||||
|
url: "https://cdn.example.com/demo.mp4",
|
||||||
|
name: "demo.mp4",
|
||||||
|
content_type: "video/mp4",
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("rejects private schemes, duplicates, and videos beyond the gateway limit", () => {
|
||||||
|
const attachments = normalizeOpenClawChatMediaAttachments([
|
||||||
|
{ type: "video", url: "https://cdn.example.com/one.mp4" },
|
||||||
|
{ type: "video", url: "https://cdn.example.com/two.mov" },
|
||||||
|
{ type: "video", url: "https://cdn.example.com/three.webm" },
|
||||||
|
{ type: "video", url: "file:///tmp/private.mp4" },
|
||||||
|
{ type: "video", url: "https://cdn.example.com/one.mp4" },
|
||||||
|
]);
|
||||||
|
|
||||||
|
expect(attachments).toHaveLength(OPENCLAW_CHAT_VIDEO_MAX_FILES);
|
||||||
|
expect(attachments.map((attachment) => attachment.url)).toEqual([
|
||||||
|
"https://cdn.example.com/one.mp4",
|
||||||
|
"https://cdn.example.com/two.mov",
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("builds a readable fallback message without exposing storage credentials", () => {
|
||||||
|
const attachments = normalizeOpenClawChatMediaAttachments([
|
||||||
|
{
|
||||||
|
type: "video",
|
||||||
|
url: "https://cdn.example.com/review.mp4",
|
||||||
|
name: "review.mp4",
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
expect(buildOpenClawVideoDisplayText(attachments)).toContain("review.mp4");
|
||||||
|
expect(buildOpenClawVideoReferenceText(attachments)).toContain(
|
||||||
|
"https://cdn.example.com/review.mp4"
|
||||||
|
);
|
||||||
|
expect(buildOpenClawVideoReferenceText(attachments)).not.toContain(
|
||||||
|
"Authorization"
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
Loading…
x
Reference in New Issue
Block a user