1222 lines
40 KiB
TypeScript
1222 lines
40 KiB
TypeScript
"use client";
|
||
|
||
import type { UseChatHelpers } from "@ai-sdk/react";
|
||
|
||
// 动态模型类型定义
|
||
type DynamicModel = {
|
||
id: string;
|
||
displayName: string;
|
||
description: string | null;
|
||
supportsVision: boolean;
|
||
isDefault: boolean;
|
||
providerName: string;
|
||
isSubModel?: boolean; // 🆕 是否为子模型
|
||
parentModelId?: string | null; // 🆕 父模型ID
|
||
capabilities?: any; // 🆕 模型能力配置
|
||
};
|
||
|
||
const fetcher = (url: string) => fetch(url).then((res) => res.json());
|
||
import { Trigger } from "@radix-ui/react-select";
|
||
import type { UIMessage } from "ai";
|
||
import equal from "fast-deep-equal";
|
||
import {
|
||
type ChangeEvent,
|
||
type Dispatch,
|
||
memo,
|
||
type SetStateAction,
|
||
startTransition,
|
||
useCallback,
|
||
useEffect,
|
||
useMemo,
|
||
useRef,
|
||
useState,
|
||
} from "react";
|
||
import { toast } from "sonner";
|
||
import useSWR from "swr";
|
||
import { useLocalStorage, useWindowSize } from "usehooks-ts";
|
||
import { saveChatModelAsCookie } from "@/app/(chat)/actions";
|
||
import { SelectItem } from "@/components/ui/select";
|
||
import type { Attachment, ChatMessage } from "@/lib/types";
|
||
import type { AppUsage } from "@/lib/usage";
|
||
import { cn } from "@/lib/utils";
|
||
import { Context } from "./elements/context";
|
||
import {
|
||
PromptInput,
|
||
PromptInputModelSelect,
|
||
PromptInputModelSelectContent,
|
||
PromptInputSubmit,
|
||
PromptInputTextarea,
|
||
PromptInputToolbar,
|
||
PromptInputTools,
|
||
} from "./elements/prompt-input";
|
||
import {
|
||
ArrowUpIcon,
|
||
ChevronDownIcon,
|
||
CpuIcon,
|
||
PaperclipIcon,
|
||
StopIcon,
|
||
} from "./icons";
|
||
import { PreviewAttachment } from "./preview-attachment";
|
||
import { SuggestedActions } from "./suggested-actions";
|
||
import { Button } from "./ui/button";
|
||
import type { VisibilityType } from "./visibility-selector";
|
||
import { VideoParamsBar, type VideoGenerationParams } from "./video-params-bar";
|
||
import { ImageParamsBar, type ImageGenerationParams } from "./image-params-bar";
|
||
|
||
function PureMultimodalInput({
|
||
chatId,
|
||
input,
|
||
setInput,
|
||
status,
|
||
stop,
|
||
attachments,
|
||
setAttachments,
|
||
messages,
|
||
setMessages,
|
||
sendMessage,
|
||
className,
|
||
selectedVisibilityType,
|
||
selectedModelId,
|
||
onModelChange,
|
||
usage,
|
||
isVideoModel,
|
||
videoModelName,
|
||
videoParams,
|
||
onVideoParamsChange,
|
||
isImageModel,
|
||
imageModelName,
|
||
imageParams,
|
||
onImageParamsChange,
|
||
imageProviderType,
|
||
}: {
|
||
chatId: string;
|
||
input: string;
|
||
setInput: Dispatch<SetStateAction<string>>;
|
||
status: UseChatHelpers<ChatMessage>["status"];
|
||
stop: () => void;
|
||
attachments: Attachment[];
|
||
setAttachments: Dispatch<SetStateAction<Attachment[]>>;
|
||
messages: UIMessage[];
|
||
setMessages: UseChatHelpers<ChatMessage>["setMessages"];
|
||
sendMessage: UseChatHelpers<ChatMessage>["sendMessage"];
|
||
className?: string;
|
||
selectedVisibilityType: VisibilityType;
|
||
selectedModelId: string;
|
||
onModelChange?: (modelId: string) => void;
|
||
usage?: AppUsage;
|
||
isVideoModel?: boolean;
|
||
videoModelName?: string;
|
||
videoParams?: VideoGenerationParams | null;
|
||
onVideoParamsChange?: (params: VideoGenerationParams) => void;
|
||
isImageModel?: boolean;
|
||
imageModelName?: string;
|
||
imageParams?: ImageGenerationParams | null;
|
||
onImageParamsChange?: (params: ImageGenerationParams) => void;
|
||
imageProviderType?: string; // 🆕 图片模型提供商类型
|
||
}) {
|
||
// 🔍 调试日志：检查图片模型相关props
|
||
console.log('[MultimodalInput] Render - isImageModel:', isImageModel);
|
||
console.log('[MultimodalInput] Render - imageParams:', imageParams);
|
||
console.log('[MultimodalInput] Render - onImageParamsChange:', !!onImageParamsChange);
|
||
|
||
const textareaRef = useRef<HTMLTextAreaElement>(null);
|
||
const { width } = useWindowSize();
|
||
|
||
// 获取网站设置（用于时间限制检查）
|
||
const { data: siteSettings } = useSWR("/api/site-settings", fetcher);
|
||
|
||
// 检查当前时间是否在允许的使用时间段内
|
||
const isWithinUsageTime = useMemo(() => {
|
||
if (!siteSettings?.usageTimeEnabled) return true;
|
||
|
||
const now = new Date();
|
||
const currentTime = `${String(now.getHours()).padStart(2, "0")}:${String(now.getMinutes()).padStart(2, "0")}`;
|
||
const startTime = siteSettings.usageTimeStart || "08:00";
|
||
const endTime = siteSettings.usageTimeEnd || "20:00";
|
||
|
||
return currentTime >= startTime && currentTime <= endTime;
|
||
}, [siteSettings]);
|
||
|
||
// 每分钟更新一次检查
|
||
useEffect(() => {
|
||
if (!siteSettings?.usageTimeEnabled) return;
|
||
|
||
const interval = setInterval(() => {
|
||
// 强制重新渲染以更新 isWithinUsageTime
|
||
startTransition(() => {
|
||
setInput((prev) => prev);
|
||
});
|
||
}, 60000); // 每分钟检查一次
|
||
|
||
return () => clearInterval(interval);
|
||
}, [siteSettings?.usageTimeEnabled, setInput]);
|
||
|
||
const adjustHeight = useCallback(() => {
|
||
if (textareaRef.current) {
|
||
textareaRef.current.style.height = "44px";
|
||
}
|
||
}, []);
|
||
|
||
useEffect(() => {
|
||
if (textareaRef.current) {
|
||
adjustHeight();
|
||
}
|
||
}, [adjustHeight]);
|
||
|
||
const resetHeight = useCallback(() => {
|
||
if (textareaRef.current) {
|
||
textareaRef.current.style.height = "44px";
|
||
}
|
||
}, []);
|
||
|
||
const [localStorageInput, setLocalStorageInput] = useLocalStorage(
|
||
"input",
|
||
""
|
||
);
|
||
|
||
useEffect(() => {
|
||
if (textareaRef.current) {
|
||
const domValue = textareaRef.current.value;
|
||
// Prefer DOM value over localStorage to handle hydration
|
||
const finalValue = domValue || localStorageInput || "";
|
||
setInput(finalValue);
|
||
adjustHeight();
|
||
}
|
||
// Only run once after hydration
|
||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||
}, [adjustHeight, localStorageInput, setInput]);
|
||
|
||
useEffect(() => {
|
||
setLocalStorageInput(input);
|
||
}, [input, setLocalStorageInput]);
|
||
|
||
const handleInput = (event: React.ChangeEvent<HTMLTextAreaElement>) => {
|
||
setInput(event.target.value);
|
||
};
|
||
|
||
const fileInputRef = useRef<HTMLInputElement>(null);
|
||
const [uploadQueue, setUploadQueue] = useState<string[]>([]);
|
||
const [isDragging, setIsDragging] = useState(false);
|
||
const [draggedIndex, setDraggedIndex] = useState<number | null>(null);
|
||
|
||
const submitForm = useCallback(() => {
|
||
window.history.pushState({}, "", `/chat/${chatId}`);
|
||
|
||
sendMessage({
|
||
role: "user",
|
||
parts: [
|
||
...attachments.map((attachment) => ({
|
||
type: "file" as const,
|
||
url: attachment.url,
|
||
name: attachment.name,
|
||
mediaType: attachment.contentType,
|
||
})),
|
||
{
|
||
type: "text",
|
||
text: input,
|
||
},
|
||
],
|
||
});
|
||
|
||
setAttachments([]);
|
||
setLocalStorageInput("");
|
||
resetHeight();
|
||
setInput("");
|
||
|
||
if (width && width > 768) {
|
||
textareaRef.current?.focus();
|
||
}
|
||
}, [
|
||
input,
|
||
setInput,
|
||
attachments,
|
||
sendMessage,
|
||
setAttachments,
|
||
setLocalStorageInput,
|
||
width,
|
||
chatId,
|
||
resetHeight,
|
||
]);
|
||
|
||
// 压缩图片函数（保持原格式：JPG保持JPG，PNG保持PNG）
|
||
// 不限制分辨率，优先保留原图质量
|
||
const compressImage = useCallback(async (file: File, maxSizeMB = 50): Promise<File> => {
|
||
// 如果不是图片，直接返回
|
||
if (!file.type.startsWith('image/')) {
|
||
return file;
|
||
}
|
||
|
||
// ✅ 保持原格式：只处理JPG和PNG，其他格式（如WebP）直接返回
|
||
const isJPEG = file.type === 'image/jpeg' || file.type === 'image/jpg';
|
||
const isPNG = file.type === 'image/png';
|
||
|
||
// 如果不是JPG/PNG，直接返回（WebP等格式不压缩）
|
||
if (!isJPEG && !isPNG) {
|
||
if (file.size <= maxSizeMB * 1024 * 1024) {
|
||
return file;
|
||
}
|
||
// 如果WebP等格式太大，也返回（让服务器端处理或提示用户）
|
||
console.log(`[压缩] 非JPG/PNG格式，跳过压缩: ${file.type}`);
|
||
return file;
|
||
}
|
||
|
||
return new Promise((resolve, reject) => {
|
||
const img = document.createElement('img');
|
||
const objectUrl = URL.createObjectURL(file);
|
||
|
||
img.onload = () => {
|
||
URL.revokeObjectURL(objectUrl);
|
||
const width = img.width;
|
||
const height = img.height;
|
||
|
||
// ✅ 免检通道：50MB以内直接使用原图，避免分辨率与画质损失
|
||
if (file.size <= maxSizeMB * 1024 * 1024) {
|
||
console.log(`[压缩] 免检通过: ${(file.size / 1024 / 1024).toFixed(2)}MB, ${width}x${height}，使用原图`);
|
||
resolve(file);
|
||
return;
|
||
}
|
||
|
||
const canvas = document.createElement('canvas');
|
||
let newWidth = width;
|
||
let newHeight = height;
|
||
|
||
canvas.width = newWidth;
|
||
canvas.height = newHeight;
|
||
|
||
const ctx = canvas.getContext('2d', {
|
||
alpha: isPNG, // PNG需要透明通道
|
||
desynchronized: false, // 同步渲染，保证质量
|
||
willReadFrequently: false,
|
||
});
|
||
|
||
if (!ctx) {
|
||
reject(new Error('无法创建 Canvas 上下文'));
|
||
return;
|
||
}
|
||
|
||
// ✅ 启用高质量图像平滑，减少锯齿
|
||
ctx.imageSmoothingEnabled = true;
|
||
ctx.imageSmoothingQuality = 'high';
|
||
|
||
ctx.drawImage(img, 0, 0, newWidth, newHeight);
|
||
|
||
// ✅ 保持原格式压缩
|
||
// 策略：优先缩小尺寸，尽量保持高质量，让服务器端做最终优化
|
||
const outputFormat = isPNG ? 'image/png' : 'image/jpeg';
|
||
const TARGET_SIZE_MB = maxSizeMB * 0.9; // 目标大小（留10%余量给服务器端优化）
|
||
|
||
// 如果尺寸已经调整过，使用高质量直接输出（让服务器端优化）
|
||
const wasResized = newWidth !== width || newHeight !== height;
|
||
let quality = wasResized ? 0.95 : 0.92; // 如果已缩小尺寸，使用更高质量
|
||
const minQuality = 0.90; // 最低质量（提高，避免画质损失）
|
||
|
||
const tryCompress = () => {
|
||
canvas.toBlob(
|
||
(blob) => {
|
||
if (!blob) {
|
||
reject(new Error('图片压缩失败'));
|
||
return;
|
||
}
|
||
|
||
console.log(`[压缩] 格式: ${outputFormat}, 质量: ${quality}, 大小: ${(blob.size / 1024 / 1024).toFixed(2)}MB`);
|
||
|
||
// 目标 TARGET_SIZE_MB（留余量给服务器端优化）
|
||
// 如果还是太大且质量可以降低，继续压缩
|
||
if (blob.size > TARGET_SIZE_MB * 1024 * 1024 && quality > minQuality) {
|
||
quality -= 0.02; // 每次减少0.02（更精细的调整）
|
||
tryCompress();
|
||
} else if (blob.size > TARGET_SIZE_MB * 1024 * 1024 && quality <= minQuality) {
|
||
// 如果质量已到最低还是太大，缩小尺寸而不是继续降低质量
|
||
if (newWidth > 1000 || newHeight > 1000) {
|
||
newWidth = Math.floor(newWidth * 0.95);
|
||
newHeight = Math.floor(newHeight * 0.95);
|
||
canvas.width = newWidth;
|
||
canvas.height = newHeight;
|
||
ctx.imageSmoothingEnabled = true;
|
||
ctx.imageSmoothingQuality = 'high';
|
||
ctx.drawImage(img, 0, 0, newWidth, newHeight);
|
||
quality = 0.95; // 重置质量（缩小尺寸后使用高质量）
|
||
tryCompress();
|
||
} else {
|
||
// 已经很小了，使用当前结果
|
||
const compressedFile = new File([blob], file.name, {
|
||
type: outputFormat, // 保持原格式
|
||
lastModified: Date.now(),
|
||
});
|
||
console.log(`[压缩] 原始: ${(file.size / 1024 / 1024).toFixed(2)}MB → 压缩后: ${(compressedFile.size / 1024 / 1024).toFixed(2)}MB`);
|
||
resolve(compressedFile);
|
||
}
|
||
} else {
|
||
// 大小合适，使用当前结果（让服务器端做最终优化）
|
||
const compressedFile = new File([blob], file.name, {
|
||
type: outputFormat, // 保持原格式
|
||
lastModified: Date.now(),
|
||
});
|
||
console.log(`[压缩] 原始: ${(file.size / 1024 / 1024).toFixed(2)}MB → 压缩后: ${(compressedFile.size / 1024 / 1024).toFixed(2)}MB (将交由服务器端进一步优化)`);
|
||
resolve(compressedFile);
|
||
}
|
||
},
|
||
outputFormat, // 保持原格式
|
||
isPNG ? undefined : quality // PNG不需要quality参数
|
||
);
|
||
};
|
||
|
||
tryCompress();
|
||
};
|
||
|
||
img.onerror = () => {
|
||
URL.revokeObjectURL(objectUrl);
|
||
reject(new Error('图片加载失败'));
|
||
};
|
||
|
||
img.src = objectUrl;
|
||
});
|
||
}, []);
|
||
|
||
const uploadFile = useCallback(async (file: File) => {
|
||
try {
|
||
// 先压缩图片
|
||
console.log(`[上传] 开始处理文件: ${file.name}, 大小: ${(file.size / 1024 / 1024).toFixed(2)}MB`);
|
||
const processedFile = await compressImage(file);
|
||
|
||
const formData = new FormData();
|
||
formData.append("file", processedFile);
|
||
|
||
const response = await fetch("/api/files/upload", {
|
||
method: "POST",
|
||
body: formData,
|
||
});
|
||
|
||
if (response.ok) {
|
||
const data = await response.json();
|
||
const { url, pathname, contentType } = data;
|
||
|
||
console.log(`[上传] 成功:`, { url, pathname, contentType });
|
||
toast.success("图片上传成功");
|
||
|
||
return {
|
||
url, // 完整的可访问 URL
|
||
name: file.name, // 使用原始文件名，不是 pathname
|
||
contentType,
|
||
};
|
||
}
|
||
const { error } = await response.json();
|
||
console.error(`[上传] 服务器返回错误:`, error);
|
||
toast.error(error);
|
||
} catch (_error) {
|
||
console.error("[上传] 失败:", _error);
|
||
toast.error(_error instanceof Error ? _error.message : "文件上传失败，请稍后重试");
|
||
}
|
||
}, [compressImage]);
|
||
|
||
const contextProps = useMemo(
|
||
() => ({
|
||
usage,
|
||
}),
|
||
[usage]
|
||
);
|
||
|
||
// 🆕 视频模式下的图片数量限制
|
||
const getVideoImageLimit = useCallback(() => {
|
||
if (!isVideoModel || !videoParams) {
|
||
return { max: Number.POSITIVE_INFINITY, label: "" as const };
|
||
}
|
||
switch (videoParams.mode) {
|
||
case "text-to-video":
|
||
return { max: 0, label: "文生视频模式不支持图片" as const };
|
||
case "first-frame":
|
||
return { max: 4, label: "智能多帧模式最多 4 张图" as const };
|
||
case "first-last-frame":
|
||
return { max: 2, label: "首尾帧模式仅支持 2 张图" as const };
|
||
case "reference-images":
|
||
return { max: 1, label: "主体参考模式仅支持 1 张图" as const };
|
||
default:
|
||
return { max: Number.POSITIVE_INFINITY, label: "" as const };
|
||
}
|
||
}, [isVideoModel, videoParams]);
|
||
|
||
const enforceVideoImageLimit = useCallback(
|
||
(incomingCount: number) => {
|
||
const { max, label } = getVideoImageLimit();
|
||
if (!Number.isFinite(max)) {
|
||
return incomingCount;
|
||
}
|
||
const remaining = max - attachments.length;
|
||
if (max === 0) {
|
||
toast.error(label || "当前模式不支持上传图片");
|
||
return 0;
|
||
}
|
||
if (remaining <= 0) {
|
||
toast.error(`已达到图片上限（${max} 张）`);
|
||
return 0;
|
||
}
|
||
if (incomingCount > remaining) {
|
||
toast.error(`最多再添加 ${remaining} 张图片（总上限 ${max} 张）`);
|
||
return remaining;
|
||
}
|
||
return incomingCount;
|
||
},
|
||
[attachments.length, getVideoImageLimit]
|
||
);
|
||
|
||
// 模式切换时自动清理不合法的图片
|
||
useEffect(() => {
|
||
if (!isVideoModel || !videoParams) {
|
||
return;
|
||
}
|
||
const { max, label } = getVideoImageLimit();
|
||
if (!Number.isFinite(max)) {
|
||
return;
|
||
}
|
||
if (max === 0 && attachments.length > 0) {
|
||
setAttachments([]);
|
||
toast.error(label || "当前模式不支持图片，已移除附件");
|
||
return;
|
||
}
|
||
if (attachments.length > max) {
|
||
setAttachments((curr) => curr.slice(0, max));
|
||
toast.error(`当前模式最多 ${max} 张图片，已自动保留前 ${max} 张`);
|
||
}
|
||
}, [attachments.length, getVideoImageLimit, isVideoModel, setAttachments, videoParams]);
|
||
|
||
const handleFileChange = useCallback(
|
||
async (event: ChangeEvent<HTMLInputElement>) => {
|
||
const files = Array.from(event.target.files || []);
|
||
const allowedCount = enforceVideoImageLimit(files.length);
|
||
if (allowedCount <= 0) {
|
||
if (fileInputRef.current) {
|
||
fileInputRef.current.value = "";
|
||
}
|
||
return;
|
||
}
|
||
const filesToUpload = files.slice(0, allowedCount);
|
||
|
||
setUploadQueue(filesToUpload.map((file) => file.name));
|
||
|
||
try {
|
||
const uploadPromises = filesToUpload.map((file) => uploadFile(file));
|
||
const uploadedAttachments = await Promise.all(uploadPromises);
|
||
const successfullyUploadedAttachments = uploadedAttachments.filter(
|
||
(attachment) => attachment !== undefined
|
||
);
|
||
|
||
setAttachments((currentAttachments) => [
|
||
...currentAttachments,
|
||
...successfullyUploadedAttachments,
|
||
]);
|
||
} catch (error) {
|
||
console.error("Error uploading files!", error);
|
||
} finally {
|
||
setUploadQueue([]);
|
||
if (fileInputRef.current) {
|
||
fileInputRef.current.value = "";
|
||
}
|
||
}
|
||
},
|
||
[enforceVideoImageLimit, setAttachments, uploadFile]
|
||
);
|
||
|
||
const handlePaste = useCallback(
|
||
async (event: ClipboardEvent) => {
|
||
const items = event.clipboardData?.items;
|
||
if (!items) {
|
||
return;
|
||
}
|
||
|
||
const imageItems = Array.from(items).filter((item) =>
|
||
item.type.startsWith("image/")
|
||
);
|
||
|
||
if (imageItems.length === 0) {
|
||
return;
|
||
}
|
||
|
||
// Prevent default paste behavior for images
|
||
event.preventDefault();
|
||
|
||
setUploadQueue((prev) => [...prev, "已粘贴图片"]);
|
||
|
||
try {
|
||
const allowedCount = enforceVideoImageLimit(imageItems.length);
|
||
if (allowedCount <= 0) {
|
||
setUploadQueue([]);
|
||
return;
|
||
}
|
||
const uploadPromises = imageItems
|
||
.map((item) => item.getAsFile())
|
||
.filter((file): file is File => file !== null)
|
||
.slice(0, allowedCount)
|
||
.map((file) => uploadFile(file));
|
||
|
||
const uploadedAttachments = await Promise.all(uploadPromises);
|
||
const successfullyUploadedAttachments = uploadedAttachments.filter(
|
||
(attachment) =>
|
||
attachment !== undefined &&
|
||
attachment.url !== undefined &&
|
||
attachment.contentType !== undefined
|
||
);
|
||
|
||
setAttachments((curr) => [
|
||
...curr,
|
||
...(successfullyUploadedAttachments as Attachment[]),
|
||
]);
|
||
} catch (error) {
|
||
console.error("Error uploading pasted images:", error);
|
||
toast.error("粘贴的图片上传失败");
|
||
} finally {
|
||
setUploadQueue([]);
|
||
}
|
||
},
|
||
[enforceVideoImageLimit, setAttachments, uploadFile]
|
||
);
|
||
|
||
// Add paste event listener to textarea
|
||
useEffect(() => {
|
||
const textarea = textareaRef.current;
|
||
if (!textarea) {
|
||
return;
|
||
}
|
||
|
||
textarea.addEventListener("paste", handlePaste);
|
||
return () => textarea.removeEventListener("paste", handlePaste);
|
||
}, [handlePaste]);
|
||
|
||
// Handle drag and drop
|
||
const handleDragOver = useCallback((event: React.DragEvent) => {
|
||
event.preventDefault();
|
||
event.stopPropagation();
|
||
// 🆕 只有拖动外部文件时才显示上传覆盖层（不影响图片排序）
|
||
const isInternalDrag = event.dataTransfer.types.includes("application/x-attachment-sort");
|
||
if (!isInternalDrag) {
|
||
setIsDragging(true);
|
||
}
|
||
}, []);
|
||
|
||
const handleDragEnter = useCallback((event: React.DragEvent) => {
|
||
event.preventDefault();
|
||
event.stopPropagation();
|
||
// 🆕 只有拖动外部文件时才显示上传覆盖层（不影响图片排序）
|
||
const isInternalDrag = event.dataTransfer.types.includes("application/x-attachment-sort");
|
||
if (!isInternalDrag) {
|
||
setIsDragging(true);
|
||
}
|
||
}, []);
|
||
|
||
const handleDragLeave = useCallback((event: React.DragEvent) => {
|
||
event.preventDefault();
|
||
event.stopPropagation();
|
||
// 只在离开整个容器时隐藏
|
||
if (event.currentTarget === event.target) {
|
||
setIsDragging(false);
|
||
}
|
||
}, []);
|
||
|
||
const handleDrop = useCallback(
|
||
async (event: React.DragEvent) => {
|
||
event.preventDefault();
|
||
event.stopPropagation();
|
||
setIsDragging(false);
|
||
|
||
// 🆕 如果是内部拖动（图片排序），不处理上传逻辑
|
||
const isInternalDrag = event.dataTransfer.types.includes("application/x-attachment-sort");
|
||
if (isInternalDrag) {
|
||
return;
|
||
}
|
||
|
||
const files = Array.from(event.dataTransfer.files);
|
||
const imageFiles = files.filter((file) => file.type.startsWith("image/"));
|
||
|
||
if (imageFiles.length === 0) {
|
||
toast.error("请拖拽图片文件");
|
||
return;
|
||
}
|
||
|
||
const allowedCount = enforceVideoImageLimit(imageFiles.length);
|
||
if (allowedCount <= 0) {
|
||
setUploadQueue([]);
|
||
return;
|
||
}
|
||
|
||
const filesToUpload = imageFiles.slice(0, allowedCount);
|
||
|
||
console.log("[拖拽] 开始处理文件:", imageFiles.length, "个");
|
||
|
||
// 添加到上传队列
|
||
setUploadQueue((prev) => [...prev, ...filesToUpload.map((f) => f.name)]);
|
||
|
||
try {
|
||
const uploadedAttachments = await Promise.all(
|
||
filesToUpload.map((file) => uploadFile(file))
|
||
);
|
||
|
||
const validAttachments = uploadedAttachments.filter(
|
||
(attachment): attachment is Attachment => attachment !== undefined
|
||
);
|
||
|
||
setAttachments((currentAttachments) => [
|
||
...currentAttachments,
|
||
...validAttachments,
|
||
]);
|
||
} catch (error) {
|
||
console.error("拖拽上传失败:", error);
|
||
toast.error("拖拽的图片上传失败");
|
||
} finally {
|
||
setUploadQueue([]);
|
||
}
|
||
},
|
||
[uploadFile]
|
||
);
|
||
|
||
// 🆕 拖拽排序处理函数
|
||
const handleAttachmentDragStart = useCallback((index: number) => (e: React.DragEvent) => {
|
||
setDraggedIndex(index);
|
||
e.dataTransfer.effectAllowed = "move";
|
||
// 🆕 设置标记，表明这是内部拖动（图片排序）
|
||
e.dataTransfer.setData("application/x-attachment-sort", String(index));
|
||
}, []);
|
||
|
||
const handleAttachmentDragEnd = useCallback(() => {
|
||
setDraggedIndex(null);
|
||
}, []);
|
||
|
||
const handleAttachmentDragOver = useCallback((index: number) => (e: React.DragEvent) => {
|
||
e.preventDefault();
|
||
e.dataTransfer.dropEffect = "move";
|
||
}, []);
|
||
|
||
const handleAttachmentDrop = useCallback((dropIndex: number) => (e: React.DragEvent) => {
|
||
e.preventDefault();
|
||
|
||
if (draggedIndex === null || draggedIndex === dropIndex) {
|
||
return;
|
||
}
|
||
|
||
setAttachments((currentAttachments) => {
|
||
const newAttachments = [...currentAttachments];
|
||
const draggedItem = newAttachments[draggedIndex];
|
||
|
||
// 移除被拖拽的项
|
||
newAttachments.splice(draggedIndex, 1);
|
||
// 插入到新位置
|
||
newAttachments.splice(dropIndex, 0, draggedItem);
|
||
|
||
return newAttachments;
|
||
});
|
||
|
||
setDraggedIndex(null);
|
||
}, [draggedIndex, setAttachments]);
|
||
|
||
return (
|
||
<div
|
||
className={cn("relative flex w-full flex-col gap-4", className)}
|
||
onDragOver={handleDragOver}
|
||
onDragEnter={handleDragEnter}
|
||
onDragLeave={handleDragLeave}
|
||
onDrop={handleDrop}
|
||
>
|
||
{messages.length === 0 &&
|
||
attachments.length === 0 &&
|
||
uploadQueue.length === 0 && (
|
||
<SuggestedActions
|
||
chatId={chatId}
|
||
selectedVisibilityType={selectedVisibilityType}
|
||
sendMessage={sendMessage}
|
||
/>
|
||
)}
|
||
|
||
{/* 拖拽覆盖层提示 */}
|
||
{isDragging && (
|
||
<div className="absolute inset-0 z-50 flex items-center justify-center rounded-xl border-2 border-dashed border-primary bg-primary/10 backdrop-blur-sm">
|
||
<div className="flex flex-col items-center gap-2 text-primary">
|
||
<ArrowUpIcon className="h-8 w-8" />
|
||
<p className="text-lg font-medium">释放以上传图片</p>
|
||
</div>
|
||
</div>
|
||
)}
|
||
|
||
<input
|
||
className="-top-4 -left-4 pointer-events-none fixed size-0.5 opacity-0"
|
||
multiple
|
||
onChange={handleFileChange}
|
||
ref={fileInputRef}
|
||
tabIndex={-1}
|
||
type="file"
|
||
/>
|
||
|
||
<PromptInput
|
||
className="rounded-xl border border-border bg-background p-3 shadow-xs transition-all duration-200 focus-within:border-border hover:border-muted-foreground/50"
|
||
onSubmit={(event) => {
|
||
event.preventDefault();
|
||
if (status === "submitted" || status === "streaming") {
|
||
toast.error("请等待模型返回当前回复后再继续");
|
||
} else {
|
||
submitForm();
|
||
}
|
||
}}
|
||
>
|
||
{(attachments.length > 0 || uploadQueue.length > 0) && (
|
||
<div
|
||
className="flex flex-row items-end gap-2 overflow-x-scroll"
|
||
data-testid="attachments-preview"
|
||
>
|
||
{attachments.map((attachment, index) => (
|
||
<PreviewAttachment
|
||
attachment={attachment}
|
||
key={attachment.url}
|
||
draggable={true}
|
||
onDragStart={handleAttachmentDragStart(index)}
|
||
onDragEnd={handleAttachmentDragEnd}
|
||
onDragOver={handleAttachmentDragOver(index)}
|
||
onDrop={handleAttachmentDrop(index)}
|
||
onRemove={() => {
|
||
setAttachments((currentAttachments) =>
|
||
currentAttachments.filter((a) => a.url !== attachment.url)
|
||
);
|
||
if (fileInputRef.current) {
|
||
fileInputRef.current.value = "";
|
||
}
|
||
}}
|
||
onAnnotate={(annotation) => {
|
||
// 自动插入标注文本到输入框（用「」包裹，用于高亮显示）
|
||
setInput((prev) => {
|
||
const currentValue = prev || "";
|
||
const separator = currentValue ? " " : "";
|
||
return `${currentValue}${separator}「${annotation.formattedText}」`;
|
||
});
|
||
toast.success(`已添加标注：${annotation.description}`);
|
||
}}
|
||
onDrawingComplete={async (annotatedImageUrl) => {
|
||
// 将标注后的图片替换原图片（或添加为新附件）
|
||
const fileName = `annotated-${Date.now()}.png`;
|
||
|
||
// 如果仍然是 base64 格式，先上传到服务器
|
||
let finalUrl = annotatedImageUrl;
|
||
if (annotatedImageUrl.startsWith("data:")) {
|
||
try {
|
||
toast.info("正在上传图片...");
|
||
const response = await fetch(annotatedImageUrl);
|
||
const blob = await response.blob();
|
||
const formData = new FormData();
|
||
formData.append("file", blob, fileName);
|
||
const uploadResponse = await fetch("/api/files/upload", {
|
||
method: "POST",
|
||
body: formData,
|
||
});
|
||
if (uploadResponse.ok) {
|
||
const result = await uploadResponse.json();
|
||
finalUrl = result.url;
|
||
}
|
||
} catch (error) {
|
||
console.error("上传标注图片失败:", error);
|
||
toast.error("上传失败");
|
||
return;
|
||
}
|
||
}
|
||
|
||
setAttachments((prev) => [
|
||
...prev,
|
||
{
|
||
url: finalUrl,
|
||
name: fileName,
|
||
contentType: "image/png",
|
||
},
|
||
]);
|
||
// 如果输入框为空，自动填入提示词
|
||
setInput((prev) => prev || "请根据我的标注修改这张图片：");
|
||
}}
|
||
imageIndex={index + 1}
|
||
/>
|
||
))}
|
||
|
||
{uploadQueue.map((filename) => (
|
||
<PreviewAttachment
|
||
attachment={{
|
||
url: "",
|
||
name: filename,
|
||
contentType: "",
|
||
}}
|
||
isUploading={true}
|
||
key={filename}
|
||
/>
|
||
))}
|
||
</div>
|
||
)}
|
||
{/* 超时提示 */}
|
||
{!isWithinUsageTime && siteSettings?.usageTimeMessage && (
|
||
<div className="px-3 py-2 mb-2 text-sm text-center text-muted-foreground bg-muted/50 rounded-md">
|
||
{siteSettings.usageTimeMessage}
|
||
</div>
|
||
)}
|
||
<div className="flex flex-row items-start gap-1 sm:gap-2">
|
||
<PromptInputTextarea
|
||
autoFocus
|
||
className={cn(
|
||
"grow resize-none border-0! border-none! bg-transparent p-2 text-sm outline-none ring-0 [-ms-overflow-style:none] [scrollbar-width:none] placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-0 focus-visible:ring-offset-0 [&::-webkit-scrollbar]:hidden",
|
||
!isWithinUsageTime && "opacity-50 cursor-not-allowed"
|
||
)}
|
||
data-testid="multimodal-input"
|
||
disableAutoResize={true}
|
||
maxHeight={200}
|
||
minHeight={44}
|
||
onChange={handleInput}
|
||
placeholder={isWithinUsageTime ? "输入消息..." : "当前不在使用时间段内"}
|
||
ref={textareaRef}
|
||
rows={1}
|
||
value={input}
|
||
disabled={!isWithinUsageTime}
|
||
/>{" "}
|
||
<Context {...contextProps} />
|
||
</div>
|
||
<PromptInputToolbar className="!border-top-0 border-t-0! p-0 shadow-none dark:border-0 dark:border-transparent!">
|
||
<PromptInputTools className="gap-0 sm:gap-0.5 flex-1 overflow-x-auto">
|
||
<AttachmentsButton
|
||
fileInputRef={fileInputRef}
|
||
selectedModelId={selectedModelId}
|
||
status={status}
|
||
/>
|
||
<ModelSelectorCompact
|
||
onModelChange={onModelChange}
|
||
selectedModelId={selectedModelId}
|
||
attachments={attachments}
|
||
/>
|
||
{/* 🆕 视频参数栏（内嵌在模型选择器后面，同一行） */}
|
||
{isVideoModel && onVideoParamsChange && (
|
||
<VideoParamsBar
|
||
key={`video-params-${selectedModelId}-${isVideoModel}`}
|
||
modelName={videoModelName || "视频 3.0"}
|
||
initialParams={videoParams || undefined}
|
||
onParamsChange={onVideoParamsChange}
|
||
inline={true}
|
||
/>
|
||
)}
|
||
{/* 🆕 图片参数栏（内嵌在模型选择器后面，同一行） */}
|
||
{isImageModel && onImageParamsChange && (
|
||
<ImageParamsBar
|
||
key={`image-params-${selectedModelId}-${isImageModel}`}
|
||
modelName={imageModelName || "Gemini 图片生成"}
|
||
initialParams={imageParams || undefined}
|
||
onParamsChange={onImageParamsChange}
|
||
inline={true}
|
||
providerType={imageProviderType}
|
||
/>
|
||
)}
|
||
</PromptInputTools>
|
||
|
||
{status === "submitted" || status === "streaming" ? (
|
||
<StopButton setMessages={setMessages} stop={stop} />
|
||
) : (
|
||
<PromptInputSubmit
|
||
className="size-8 rounded-full bg-primary text-primary-foreground transition-colors duration-200 hover:bg-primary/90 disabled:bg-muted disabled:text-muted-foreground"
|
||
data-testid="send-button"
|
||
disabled={!input.trim() || uploadQueue.length > 0 || !isWithinUsageTime}
|
||
status={status}
|
||
>
|
||
<ArrowUpIcon size={14} />
|
||
</PromptInputSubmit>
|
||
)}
|
||
</PromptInputToolbar>
|
||
</PromptInput>
|
||
</div>
|
||
);
|
||
}
|
||
|
||
export const MultimodalInput = memo(
|
||
PureMultimodalInput,
|
||
(prevProps, nextProps) => {
|
||
if (prevProps.input !== nextProps.input) {
|
||
return false;
|
||
}
|
||
if (prevProps.status !== nextProps.status) {
|
||
return false;
|
||
}
|
||
if (!equal(prevProps.attachments, nextProps.attachments)) {
|
||
return false;
|
||
}
|
||
if (prevProps.selectedVisibilityType !== nextProps.selectedVisibilityType) {
|
||
return false;
|
||
}
|
||
if (prevProps.selectedModelId !== nextProps.selectedModelId) {
|
||
return false;
|
||
}
|
||
if (prevProps.isVideoModel !== nextProps.isVideoModel) {
|
||
return false;
|
||
}
|
||
if (!equal(prevProps.videoParams, nextProps.videoParams)) {
|
||
return false;
|
||
}
|
||
if (prevProps.isImageModel !== nextProps.isImageModel) {
|
||
return false;
|
||
}
|
||
if (!equal(prevProps.imageParams, nextProps.imageParams)) {
|
||
return false;
|
||
}
|
||
|
||
return true;
|
||
}
|
||
);
|
||
|
||
function PureAttachmentsButton({
|
||
fileInputRef,
|
||
status,
|
||
selectedModelId,
|
||
}: {
|
||
fileInputRef: React.MutableRefObject<HTMLInputElement | null>;
|
||
status: UseChatHelpers<ChatMessage>["status"];
|
||
selectedModelId: string;
|
||
}) {
|
||
const isReasoningModel = selectedModelId === "chat-model-reasoning";
|
||
|
||
return (
|
||
<Button
|
||
className="aspect-square h-8 rounded-lg p-1 transition-colors hover:bg-accent"
|
||
data-testid="attachments-button"
|
||
disabled={status !== "ready" || isReasoningModel}
|
||
onClick={(event) => {
|
||
event.preventDefault();
|
||
fileInputRef.current?.click();
|
||
}}
|
||
variant="ghost"
|
||
>
|
||
<PaperclipIcon size={14} style={{ width: 14, height: 14 }} />
|
||
</Button>
|
||
);
|
||
}
|
||
|
||
const AttachmentsButton = memo(PureAttachmentsButton);
|
||
|
||
function PureModelSelectorCompact({
|
||
selectedModelId,
|
||
onModelChange,
|
||
attachments,
|
||
}: {
|
||
selectedModelId: string;
|
||
onModelChange?: (modelId: string) => void;
|
||
attachments: Attachment[];
|
||
}) {
|
||
const [optimisticModelId, setOptimisticModelId] = useState(selectedModelId);
|
||
|
||
// 从 API 获取动态模型配置
|
||
const { data: dynamicModelsData, isLoading } = useSWR<{
|
||
models: DynamicModel[];
|
||
defaultModelId: string | null;
|
||
}>("/api/models", fetcher);
|
||
|
||
// 获取站点设置
|
||
const { data: siteSettings } = useSWR<{
|
||
showAutoModel: boolean;
|
||
}>("/api/site-settings", fetcher);
|
||
|
||
const allModels = dynamicModelsData?.models ?? [];
|
||
|
||
// 🆕 将模型分为主模型和子模型，构建层级结构
|
||
const { mainModels, subModelsMap } = useMemo(() => {
|
||
const main: DynamicModel[] = [];
|
||
const subMap: Record<string, DynamicModel[]> = {};
|
||
|
||
allModels.forEach(model => {
|
||
if (model.isSubModel && model.parentModelId) {
|
||
if (!subMap[model.parentModelId]) {
|
||
subMap[model.parentModelId] = [];
|
||
}
|
||
subMap[model.parentModelId].push(model);
|
||
} else {
|
||
main.push(model);
|
||
}
|
||
});
|
||
|
||
return { mainModels: main, subModelsMap: subMap };
|
||
}, [allModels]);
|
||
|
||
// 🆕 展开显示：主模型 + 其子模型（缩进显示）
|
||
const dynamicModels = useMemo(() => {
|
||
const result: DynamicModel[] = [];
|
||
mainModels.forEach(mainModel => {
|
||
result.push(mainModel);
|
||
// 如果有子模型，添加到主模型后面
|
||
if (subModelsMap[mainModel.id]) {
|
||
result.push(...subModelsMap[mainModel.id]);
|
||
}
|
||
});
|
||
return result;
|
||
}, [mainModels, subModelsMap]);
|
||
|
||
// 定义自动模式
|
||
const autoModel: DynamicModel = {
|
||
id: "auto",
|
||
displayName: "Agent模式",
|
||
description: "根据您的意图自动选择最合适的模型",
|
||
supportsVision: true,
|
||
isDefault: false,
|
||
providerName: "System",
|
||
};
|
||
|
||
// 合并模型列表（根据设置决定是否包含自动模型）
|
||
const showAutoModel = siteSettings?.showAutoModel ?? true;
|
||
const displayModels = showAutoModel ? [autoModel, ...dynamicModels] : dynamicModels;
|
||
|
||
useEffect(() => {
|
||
setOptimisticModelId(selectedModelId);
|
||
}, [selectedModelId]);
|
||
|
||
useEffect(() => {
|
||
if (displayModels.length === 0) return;
|
||
|
||
const currentExists = displayModels.some((m) => m.id === selectedModelId);
|
||
if (currentExists) return;
|
||
|
||
const fallbackModelId =
|
||
dynamicModels.find((m) => m.isDefault)?.id ??
|
||
dynamicModels[0]?.id ??
|
||
(showAutoModel ? "auto" : "");
|
||
|
||
if (!fallbackModelId) return;
|
||
|
||
setOptimisticModelId(fallbackModelId);
|
||
onModelChange?.(fallbackModelId);
|
||
startTransition(() => {
|
||
saveChatModelAsCookie(fallbackModelId);
|
||
});
|
||
}, [displayModels, dynamicModels, onModelChange, selectedModelId, showAutoModel]);
|
||
|
||
// 自动选择模型的逻辑 - 必须在所有条件 return 之前定义
|
||
const selectBestModel = useCallback(() => {
|
||
const hasAttachments = attachments && attachments.length > 0;
|
||
|
||
// 优先选择支持视觉的模型（如果有附件）
|
||
if (hasAttachments) {
|
||
const visionModel = dynamicModels.find((m) => m.supportsVision);
|
||
if (visionModel) return visionModel;
|
||
}
|
||
|
||
// 否则选择默认模型
|
||
const defaultModel = dynamicModels.find((m) => m.isDefault);
|
||
if (defaultModel) return defaultModel;
|
||
|
||
// 最后选择第一个可用模型
|
||
return dynamicModels[0];
|
||
}, [dynamicModels, attachments]);
|
||
|
||
// 从动态模型中查找当前选中的模型
|
||
const selectedModel = useMemo(() => {
|
||
const dynamicModel = displayModels.find((m) => m.id === optimisticModelId);
|
||
if (dynamicModel) {
|
||
return { id: dynamicModel.id, name: dynamicModel.displayName, description: dynamicModel.description };
|
||
}
|
||
if (displayModels.length > 0) {
|
||
return { id: displayModels[0].id, name: displayModels[0].displayName, description: displayModels[0].description };
|
||
}
|
||
return { id: "", name: "未配置模型", description: null };
|
||
}, [optimisticModelId, displayModels]);
|
||
|
||
if (isLoading) {
|
||
return (
|
||
<Button className="h-8 px-2" variant="ghost" disabled>
|
||
<CpuIcon size={16} />
|
||
<span className="hidden font-medium text-xs sm:block">加载中...</span>
|
||
</Button>
|
||
);
|
||
}
|
||
|
||
if (displayModels.length === 0) {
|
||
return (
|
||
<Button className="h-8 px-2" variant="ghost" disabled>
|
||
<CpuIcon size={16} />
|
||
<span className="hidden font-medium text-xs sm:block">⚠️ 未配置</span>
|
||
</Button>
|
||
);
|
||
}
|
||
|
||
return (
|
||
<PromptInputModelSelect
|
||
onValueChange={(modelName) => {
|
||
const model = displayModels.find((m) => m.displayName === modelName);
|
||
if (model) {
|
||
// 🆕 如果选择的是"自动"模式，直接传递 "auto" 给后端
|
||
// 后端会使用智能分析器进行意图识别和模型选择
|
||
if (model.id === "auto") {
|
||
console.log("[自动模式] 选择Agent模式，后端将进行意图分析和模型路由");
|
||
setOptimisticModelId("auto");
|
||
onModelChange?.("auto");
|
||
startTransition(() => {
|
||
saveChatModelAsCookie("auto");
|
||
});
|
||
return;
|
||
}
|
||
|
||
setOptimisticModelId(model.id);
|
||
onModelChange?.(model.id);
|
||
startTransition(() => {
|
||
saveChatModelAsCookie(model.id);
|
||
});
|
||
}
|
||
}}
|
||
value={selectedModel?.name}
|
||
>
|
||
<Trigger asChild>
|
||
<Button className="h-8 px-2" variant="ghost">
|
||
<CpuIcon size={16} />
|
||
<span className="hidden font-medium text-xs sm:block">
|
||
{selectedModel?.name}
|
||
</span>
|
||
<ChevronDownIcon size={16} />
|
||
</Button>
|
||
</Trigger>
|
||
<PromptInputModelSelectContent className="min-w-[260px] p-0">
|
||
<div className="flex flex-col gap-px">
|
||
{displayModels.map((model) => (
|
||
<SelectItem
|
||
key={model.id}
|
||
value={model.displayName}
|
||
className={model.isSubModel ? "pl-6" : ""}
|
||
>
|
||
<div className="truncate font-medium text-xs">
|
||
{model.isSubModel && "└ "}{model.displayName}
|
||
</div>
|
||
<div className="mt-px truncate text-[10px] text-muted-foreground leading-tight">
|
||
{model.description ?? model.providerName}
|
||
</div>
|
||
</SelectItem>
|
||
))}
|
||
</div>
|
||
</PromptInputModelSelectContent>
|
||
</PromptInputModelSelect>
|
||
);
|
||
}
|
||
|
||
const ModelSelectorCompact = memo(
|
||
PureModelSelectorCompact,
|
||
(prevProps, nextProps) => {
|
||
if (prevProps.selectedModelId !== nextProps.selectedModelId) {
|
||
return false;
|
||
}
|
||
if (!equal(prevProps.attachments, nextProps.attachments)) {
|
||
return false;
|
||
}
|
||
return true;
|
||
}
|
||
);
|
||
|
||
function PureStopButton({
|
||
stop,
|
||
setMessages,
|
||
}: {
|
||
stop: () => void;
|
||
setMessages: UseChatHelpers<ChatMessage>["setMessages"];
|
||
}) {
|
||
return (
|
||
<Button
|
||
className="size-8 rounded-full bg-foreground p-1 text-background transition-colors duration-200 hover:bg-foreground/90 disabled:bg-muted disabled:text-muted-foreground"
|
||
data-testid="stop-button"
|
||
onClick={(event) => {
|
||
event.preventDefault();
|
||
stop();
|
||
setMessages((messages) => messages);
|
||
}}
|
||
>
|
||
<StopIcon size={14} />
|
||
</Button>
|
||
);
|
||
}
|
||
|
||
const StopButton = memo(PureStopButton);
|