初始提交:JmalCloud公共空间版改造完成

- 改造为公共空间模式,所有用户共享文件
- 移除用户文件隔离机制
- 保持前端API兼容性
- 更新配置文件和文档
- 创建完整启动脚本工具链
This commit is contained in:
chenpei 2025-08-24 11:16:01 +08:00
commit c26105b684
993 changed files with 96104 additions and 0 deletions

9
.env-example Normal file
View File

@ -0,0 +1,9 @@
APP_PORT=7070
BLOG_PORT=7071
SERVER_PORT=7072
FTP_SERVER_PORT=7073
CONTAINER_NAME_PREFIX=jmalcloud
RESOURCE_DB_PATH=./docker/jmalcloud/mongodb
RESOURCE_FILE_PATH=./docker/jmalcloud/files
OFFICE_IMAGE_VERSION=7.0.0.132
DRAWIO_IMAGE_VERSION=20.2.3

39
.github/release-drafter.yml vendored Normal file
View File

@ -0,0 +1,39 @@
# .github/release-drafter.yml
name-template: 'v$NEXT_PATCH_VERSION'
tag-template: 'v$NEXT_PATCH_VERSION'
categories:
- title: '新功能 ✨'
labels:
- 'feature'
- title: '修复 🐛'
labels:
- 'bug'
- title: '优化 ⚡️'
labels:
- 'improvement'
change-template: '- $TITLE (#$NUMBER)'
no-changes-template: '- No changes'
template: |
[docker-docker.yml](https://github.com/jamebal/jmal-cloud-server/blob/master/docker-compose.base.yml)
#### 更新前备份数据库
```shell
docker exec -it jmalcloud_mongodb mongodump -d jmalcloud -o /dump/$PREVIOUS_TAG --gzip --quiet
```
```shell
docker-compose pull && docker-compose up -d --remove-orphans
```
#### 清理旧镜像
```shell
docker rmi $(docker images --filter "dangling=true" | grep "jmalcloud" | awk '{print $3}')
```
## Changes in this release:
$CHANGES
**Full Changelog**: [$PREVIOUS_TAG...v$NEXT_PATCH_VERSION](https://github.com/jamebal/jmal-cloud-server/compare/$PREVIOUS_TAG...v$NEXT_PATCH_VERSION)

59
.github/workflows/build.yml vendored Normal file
View File

@ -0,0 +1,59 @@
name: Build jmalcloud Docker Image
on:
workflow_dispatch:
release:
types: [published]
tags: [v*]
jobs:
build-and-push:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Set up JDK 17
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '17'
cache: 'maven'
- name: Build with Maven
run: mvn clean package -DskipTests
- name: Extract project version from pom.xml
run: |
VERSION=$(mvn -q -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive exec:exec)
echo "PROJECT_VERSION=$VERSION" >> $GITHUB_ENV
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ secrets.GHCR_IO_USERNAME }}
password: ${{ secrets.GHCR_IO_TOKEN }}
- name: Build and push Docker image
uses: docker/build-push-action@v5
with:
context: .
file: ./Dockerfile
push: true
tags: |
jmal/jmalcloud:${{ env.PROJECT_VERSION }},jmal/jmalcloud:latest
ghcr.io/${{ secrets.GHCR_IO_USERNAME }}/jmalcloud:${{ env.PROJECT_VERSION }},ghcr.io/${{ secrets.GHCR_IO_USERNAME }}/jmalcloud:latest
platforms: linux/amd64,linux/arm64
build-args: |
VERSION=${{ env.PROJECT_VERSION }}

48
.github/workflows/build_jdk_ffmpeg.yml vendored Normal file
View File

@ -0,0 +1,48 @@
name: Build jdk21_ffmpeg Docker Image
on:
workflow_dispatch:
jobs:
build-and-push:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Set up JDK 21
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '21'
cache: 'maven'
- name: Build with Maven
run: mvn clean package -DskipTests
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ secrets.GHCR_IO_USERNAME }}
password: ${{ secrets.GHCR_IO_TOKEN }}
- name: Build and push Docker image
uses: docker/build-push-action@v5
with:
context: .
file: docker/Dockerfile
push: true
tags: |
jmal/jdk21_ffmpeg:latest
ghcr.io/${{ secrets.GHCR_IO_USERNAME }}/jdk21_ffmpeg:latest
platforms: linux/amd64,linux/arm64

View File

@ -0,0 +1,38 @@
name: Build jmalcloud Base Docker Image
on:
workflow_dispatch:
jobs:
build-and-push:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ secrets.GHCR_IO_USERNAME }}
password: ${{ secrets.GHCR_IO_TOKEN }}
- name: Build and push Docker image
uses: docker/build-push-action@v5
with:
context: .
file: docker/Dockerfile
push: true
tags: |
jmal/jmalcloud_base:latest
ghcr.io/${{ secrets.GHCR_IO_USERNAME }}/jmalcloud_base:latest
platforms: linux/amd64,linux/arm64

41
.github/workflows/merge-manifests.yml vendored Normal file
View File

@ -0,0 +1,41 @@
name: merge manifests
on:
push:
branches:
- 'master'
workflow_dispatch:
jobs:
merge-manifests:
# 等待所有架构的构建都完成后再运行
runs-on: ubuntu-latest # 合并操作不耗资源,用哪个都行
steps:
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ secrets.GHCR_IO_USERNAME }}
password: ${{ secrets.GHCR_IO_TOKEN }}
- name: Create and push Docker Hub manifest
run: |
docker manifest create jmal/jmalcloud_native:test \
--amend jmal/jmalcloud_native:test-amd64 \
--amend jmal/jmalcloud_native:test-arm64
docker manifest push jmal/jmalcloud_native:test
- name: Create and push GHCR manifest
run: |
docker manifest create ghcr.io/${{ secrets.GHCR_IO_USERNAME }}/jmalcloud_native:test \
--amend ghcr.io/${{ secrets.GHCR_IO_USERNAME }}/jmalcloud_native:test-amd64 \
--amend ghcr.io/${{ secrets.GHCR_IO_USERNAME }}/jmalcloud_native:test-arm64
docker manifest push ghcr.io/${{ secrets.GHCR_IO_USERNAME }}/jmalcloud_native:test

86
.github/workflows/pushAliyun.yml vendored Normal file
View File

@ -0,0 +1,86 @@
name: Build and Push Docker Image to Aliyun
on:
workflow_dispatch:
release:
types: [ published ]
tags: [ v* ]
jobs:
build_and_push:
runs-on: ubuntu-latest
strategy:
matrix:
arch:
- amd64
- arm64
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Set up JDK 21
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '21'
cache: 'maven'
- name: Build with Maven
run: mvn clean package -DskipTests
- name: Extract project version from pom.xml
run: |
VERSION=$(mvn -q -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive exec:exec)
echo "PROJECT_VERSION=$VERSION" >> $GITHUB_ENV
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log in to Aliyun Container Registry guangzhou
uses: docker/login-action@v3
with:
registry: registry.cn-guangzhou.aliyuncs.com
username: ${{ secrets.ALIYUN_DOCKER_USERNAME }}
password: ${{ secrets.ALIYUN_DOCKER_PASSWORD }}
- name: Log in to Aliyun Container Registry beijing
uses: docker/login-action@v3
with:
registry: registry.cn-beijing.aliyuncs.com
username: ${{ secrets.ALIYUN_DOCKER_USERNAME }}
password: ${{ secrets.ALIYUN_DOCKER_PASSWORD }}
- name: Set tag for amd64
if: matrix.arch == 'amd64'
run: |
echo "TAG_LATEST_0=registry.cn-guangzhou.aliyuncs.com/jmalcloud/jmalcloud:latest" >> $GITHUB_ENV
echo "TAG_VERSION_0=registry.cn-guangzhou.aliyuncs.com/jmalcloud/jmalcloud:${{ env.PROJECT_VERSION }}" >> $GITHUB_ENV
echo "TAG_LATEST_1=registry.cn-beijing.aliyuncs.com/jmalcloud/jmalcloud:latest" >> $GITHUB_ENV
echo "TAG_VERSION_1=registry.cn-beijing.aliyuncs.com/jmalcloud/jmalcloud:${{ env.PROJECT_VERSION }}" >> $GITHUB_ENV
- name: Set tag for non-amd64
if: matrix.arch != 'amd64'
run: |
echo "TAG_LATEST_0=registry.cn-guangzhou.aliyuncs.com/jmalcloud/jmalcloud:latest-${{ matrix.arch }}" >> $GITHUB_ENV
echo "TAG_VERSION_0=registry.cn-guangzhou.aliyuncs.com/jmalcloud/jmalcloud:${{ env.PROJECT_VERSION }}-${{ matrix.arch }}" >> $GITHUB_ENV
echo "TAG_LATEST_1=registry.cn-beijing.aliyuncs.com/jmalcloud/jmalcloud:latest-${{ matrix.arch }}" >> $GITHUB_ENV
echo "TAG_VERSION_1=registry.cn-beijing.aliyuncs.com/jmalcloud/jmalcloud:${{ env.PROJECT_VERSION }}-${{ matrix.arch }}" >> $GITHUB_ENV
- name: Build and push Docker image
uses: docker/build-push-action@v5
with:
context: .
file: ./Dockerfile
platforms: linux/${{ matrix.arch }}
push: true
tags: |
${{ env.TAG_LATEST_0 }}
${{ env.TAG_VERSION_0 }}
${{ env.TAG_LATEST_1 }}
${{ env.TAG_VERSION_1 }}
build-args: |
VERSION=${{ env.PROJECT_VERSION }}

18
.github/workflows/release-drafter.yml vendored Normal file
View File

@ -0,0 +1,18 @@
# .github/workflows/release-drafter.yml
name: Release Drafter
on:
push:
branches:
- master
jobs:
update_release_draft:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: release-drafter/release-drafter@v5
with:
config-name: release-drafter.yml
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

59
.github/workflows/test-build.yml vendored Normal file
View File

@ -0,0 +1,59 @@
name: Build test
on:
push:
branches:
- 'master'
workflow_dispatch:
jobs:
build-and-push:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Set up JDK 21
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '21'
cache: 'maven'
- name: Build with Maven
run: mvn clean package -DskipTests
- name: Extract project version from pom.xml
run: |
VERSION=$(mvn -q -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive exec:exec)
echo "PROJECT_VERSION=$VERSION" >> $GITHUB_ENV
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ secrets.GHCR_IO_USERNAME }}
password: ${{ secrets.GHCR_IO_TOKEN }}
- name: Build and push Docker image to multiple registries
uses: docker/build-push-action@v5
with:
context: .
file: ./Dockerfile
push: true
tags: |
jmal/jmalcloud:test
ghcr.io/${{ secrets.GHCR_IO_USERNAME }}/jmalcloud:test
platforms: linux/amd64,linux/arm64
build-args: |
VERSION=${{ env.PROJECT_VERSION }}

66
.github/workflows/test-native-build.yml vendored Normal file
View File

@ -0,0 +1,66 @@
name: Build test native
on:
push:
branches:
- 'master'
workflow_dispatch:
jobs:
build-and-push:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Extract project version from pom.xml
id: project_version # 给步骤一个id以便后续引用
run: |
VERSION=$(mvn -q -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive exec:exec)
echo "PROJECT_VERSION=$VERSION" >> $GITHUB_ENV
echo "version=$VERSION" >> "$GITHUB_OUTPUT" # 使用 GITHUB_OUTPUT 在步骤间传递值
# ------------------- 阶段二: 使用 GraalVM 构建 Native Image -------------------
- name: Set up GraalVM for Native Image
uses: graalvm/setup-graalvm@v1
with:
java-version: '24' # GraalVM 24.x 基于 JDK 24
distribution: 'graalvm'
version: '24' # 指定使用 GraalVM 24.x
github-token: ${{ secrets.GITHUB_TOKEN }}
cache: 'maven' # GraalVM action 也支持缓存 Maven 依赖
- name: Build Native Image with Maven
run: mvn clean package -Pnative -DskipTests
# -Pnative 激活 pom.xml 中为 native image 定义的 profile
# native:compile 是 GraalVM 插件提供的编译目标
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ secrets.GHCR_IO_USERNAME }}
password: ${{ secrets.GHCR_IO_TOKEN }}
- name: Build and push Docker image to multiple registries
uses: docker/build-push-action@v5
with:
context: .
file: ./Dockerfile.native
push: true
tags: |
jmal/jmalcloud:test
ghcr.io/${{ secrets.GHCR_IO_USERNAME }}/jmalcloud:test
platforms: linux/amd64,linux/arm64
build-args: |
VERSION=${{ env.PROJECT_VERSION }}

39
.gitignore vendored Normal file
View File

@ -0,0 +1,39 @@
## myproject
push.sh
update.sh
HELP.md
.env
/log/
/lucene/
/docker/
target/
!.mvn/wrapper/maven-wrapper.jar
!**/src/main/**
!**/src/test/**
### STS ###
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache
### IntelliJ IDEA ###
.idea
*.iws
*.iml
*.ipr
### NetBeans ###
/nbproject/private/
/nbbuild/
/dist/
/nbdist/
/.nb-gradle/
build/
### VS Code ###
.vscode/
src/main/resources/application-dev.yml

51
Dockerfile Normal file
View File

@ -0,0 +1,51 @@
FROM ghcr.io/jamebal/jdk21_ffmpeg:latest
ARG VERSION
ENV MONGODB_URI "mongodb://mongo:27017/jmalcloud"
ENV RUN_ENVIRONMENT prod
ENV JVM_OPTS ""
ENV LOG_LEVEL warn
# 是否开启精确搜索
ENV EXACT_SEARCH false
ENV NGRAM_MAX_CONTENT_LENGTH_MB "5"
ENV NGRAM_MIN_SIZE "2"
ENV NGRAM_MAX_SIZE "6"
ENV FILE_MONITOR true
ENV MONITOR_IGNORE_FILE_PREFIX ".DS_Store,._"
ENV FILE_ROOT_DIR /jmalcloud/files
ENV TESS4J_DATA_PATH /jmalcloud/tess4j/datapath
ADD target/clouddisk-${VERSION}.jar /usr/local/
VOLUME /jmalcloud/
# 设置支持的平台
ARG TARGETPLATFORM
RUN echo "Building for platform: $TARGETPLATFORM"
LABEL org.label-schema.build.multi-platform=true
ENV PLATFORM=$TARGETPLATFORM
ENV VERSION=${VERSION}
# 将 Linux/arm64/v8 架构设置为默认平台
# 如果需要,可以根据需要更改此设置
ENV DOCKER_DEFAULT_PLATFORM=linux/amd64,linux/arm64
COPY docker-entrypoint.sh /docker-entrypoint.sh
RUN apt-get update && \
apt-get install -y gosu && \
chmod +x /docker-entrypoint.sh && \
rm -rf /var/lib/apt/lists/*
EXPOSE 8088
# FTP Server
EXPOSE 8089
HEALTHCHECK --interval=30s --timeout=3s --start-period=60s --retries=3 \
CMD curl -f http://localhost:8088/public/health > /dev/null || exit 1
ENTRYPOINT ["/docker-entrypoint.sh"]

21
LICENSE Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2022 jmal
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

134
README.md Normal file
View File

@ -0,0 +1,134 @@
# JmalCloud 公共空间版 - 个人网盘服务端
> 基于 [JmalCloud](https://github.com/jamebal/jmal-cloud-view) 改造的公共空间版本
## 🚀 项目简介
这是一个基于原版 JmalCloud 改造的**公共空间版本**,将原来的用户独立文件空间模式改为**所有用户共享同一文件空间**的模式。
### 📋 主要变更
- ✅ **公共文件空间**:所有用户登录后看到相同的文件列表
- ✅ **统一存储**:文件存储在统一的公共目录中,不再按用户分离
- ✅ **共享操作**:任何用户都可以上传、下载、删除、重命名所有文件
- ✅ **保留认证**:用户登录验证和基本权限控制仍然有效
- ✅ **元数据保留**:文件的创建者信息仍保存在数据库中
## 🔧 核心改造内容
### 后端修改
1. **文件存储路径统一化** - 移除用户名路径组件
2. **数据库查询去用户化** - 移除 userId 过滤条件
3. **文件操作公共化** - 所有文件操作适配公共空间
4. **用户目录管理适配** - 不再创建用户专属目录
### 前端兼容性
- ✅ **无需修改** - 前端代码完全兼容,无需任何变更
- ✅ **API向后兼容** - 保持原有接口参数,仅后端逻辑调整
## 🎯 适用场景
- **家庭网盘** - 家庭成员共享文件存储空间
- **小团队协作** - 团队内部文件共享和协作
- **企业内网盘** - 企业内部公共文件存储
- **学校/机构** - 教学资源共享平台
## ⚠️ 重要注意事项
### 数据迁移
```bash
# 需要手动将现有用户文件迁移到公共目录
# 原来的结构: /rootDir/user1/file1.txt
# 现在的结构: /rootDir/file1.txt
```
### 安全考虑
- **文件安全性**:所有登录用户都能访问所有文件
- **权限控制**:需要在应用层面控制用户访问权限
- **数据备份**:建议在使用前备份原始数据
## 🔄 从原版迁移
1. **备份数据**
```bash
# 备份原有文件和数据库
cp -r /your/original/files /backup/location
```
2. **部署改造版**
```bash
# 部署修改后的代码
# 使用相同的配置文件和数据库
```
3. **文件迁移**
```bash
# 将用户目录中的文件移动到公共根目录
# 注意处理文件名冲突
```
## 📁 文件结构对比
### 改造前
```
/rootDir/
├── user1/
│ ├── file1.txt
│ └── folder1/
└── user2/
├── file2.txt
└── folder2/
```
### 改造后
```
/rootDir/
├── file1.txt
├── file2.txt
├── folder1/
└── folder2/
```
## 🛠️ 部署说明
### 环境要求
- Java 21+
- MongoDB 4.4+
- Maven 3.6+
### 构建和运行
```bash
# 构建项目
mvn clean package -DskipTests
# 运行
java -jar target/clouddisk-*.jar
```
### Docker 部署
```bash
# 使用提供的 docker-compose.yml
docker-compose up -d
```
## 📝 变更日志
### v2.15.0-public-space
- 改造为公共空间模式
- 移除用户文件隔离机制
- 统一文件存储路径
- 保持前端API兼容性
## 🤝 贡献
如果你发现问题或有改进建议,欢迎提交 Issue 或 Pull Request。
## 📄 许可
[MIT](https://github.com/jamebal/jmal-cloud-view/blob/master/LICENSE) license.
Copyright (c) 2023-present jmal
---
> **免责声明**: 这是基于开源项目 JmalCloud 的自定义改造版本,使用前请仔细评估安全性和适用性。原作者不对此改造版本承担责任。

View File

@ -0,0 +1,51 @@
services:
mongo:
container_name: jmalcloud_mongodb
image: registry.cn-beijing.aliyuncs.com/jmalcloud/mongo:4.4
environment:
TZ: "Asia/Shanghai"
volumes:
- ./docker/jmalcloud/mongodb/data/db:/data/db
- ./docker/jmalcloud/mongodb/backup:/dump
restart: unless-stopped
healthcheck:
test: ["CMD", "mongo", "--eval", "db.adminCommand('ping')"]
interval: 10s
timeout: 5s
retries: 3
command: --wiredTigerCacheSizeGB 0.5
jmalcloud:
container_name: jmalcloud_server
image: registry.cn-beijing.aliyuncs.com/jmalcloud/jmalcloud:latest
environment:
MONGODB_URI: "mongodb://mongo:27017/jmalcloud"
TZ: "Asia/Shanghai"
JVM_OPTS: "-Xms256m -Xmx1024m"
volumes:
- ./docker/jmalcloud/files:/jmalcloud/files/
restart: unless-stopped
depends_on:
mongo:
condition: service_healthy
nginx:
container_name: jmalcloud_nginx
image: registry.cn-beijing.aliyuncs.com/jmalcloud/jmalcloud-nginx:latest
ports:
- 7070:80
- 7071:8089
environment:
TZ: "Asia/Shanghai"
links:
- jmalcloud
- office
restart: unless-stopped
office: # Optional
container_name: jmalcloud_office
image: registry.cn-beijing.aliyuncs.com/jmalcloud/onlyoffice_documentserver:8.0.1
environment:
TZ: "Asia/Shanghai"
JWT_SECRET: "my_secret"
restart: unless-stopped

View File

@ -0,0 +1,51 @@
services:
mongo:
container_name: jmalcloud_mongodb
image: registry.cn-guangzhou.aliyuncs.com/jmalcloud/mongo:4.4
environment:
TZ: "Asia/Shanghai"
volumes:
- ./docker/jmalcloud/mongodb/data/db:/data/db
- ./docker/jmalcloud/mongodb/backup:/dump
restart: unless-stopped
healthcheck:
test: ["CMD", "mongo", "--eval", "db.adminCommand('ping')"]
interval: 10s
timeout: 5s
retries: 3
command: --wiredTigerCacheSizeGB 0.5
jmalcloud:
container_name: jmalcloud_server
image: registry.cn-guangzhou.aliyuncs.com/jmalcloud/jmalcloud:latest
environment:
MONGODB_URI: "mongodb://mongo:27017/jmalcloud"
TZ: "Asia/Shanghai"
JVM_OPTS: "-Xms256m -Xmx1024m"
volumes:
- ./docker/jmalcloud/files:/jmalcloud/files/
restart: unless-stopped
depends_on:
mongo:
condition: service_healthy
nginx:
container_name: jmalcloud_nginx
image: registry.cn-guangzhou.aliyuncs.com/jmalcloud/jmalcloud-nginx:latest
ports:
- 7070:80
- 7071:8089
environment:
TZ: "Asia/Shanghai"
links:
- jmalcloud
- office
restart: unless-stopped
office: # Optional
container_name: jmalcloud_office
image: registry.cn-guangzhou.aliyuncs.com/jmalcloud/onlyoffice_documentserver:8.0.1
environment:
TZ: "Asia/Shanghai"
JWT_SECRET: "my_secret"
restart: unless-stopped

View File

@ -0,0 +1,51 @@
services:
mongo:
container_name: jmalcloud_mongodb
image: docker.jmalx.com/mongo:4.4
environment:
TZ: "Asia/Shanghai"
volumes:
- ./docker/jmalcloud/mongodb/data/db:/data/db
- ./docker/jmalcloud/mongodb/backup:/dump
restart: unless-stopped
healthcheck:
test: ["CMD", "mongo", "--eval", "db.adminCommand('ping')"]
interval: 10s
timeout: 5s
retries: 3
command: --wiredTigerCacheSizeGB 0.5
jmalcloud:
container_name: jmalcloud_server
image: docker.jmalx.com/jmal/jmalcloud:latest
environment:
MONGODB_URI: "mongodb://mongo:27017/jmalcloud"
TZ: "Asia/Shanghai"
JVM_OPTS: "-Xms256m -Xmx1024m"
volumes:
- ./docker/jmalcloud/files:/jmalcloud/files/
restart: unless-stopped
depends_on:
mongo:
condition: service_healthy
nginx:
container_name: jmalcloud_nginx
image: docker.jmalx.com/jmal/jmalcloud-nginx:latest
ports:
- 7070:80
- 7071:8089
environment:
TZ: "Asia/Shanghai"
links:
- jmalcloud
- office
restart: unless-stopped
office: # Optional
container_name: jmalcloud_office
image: docker.jmalx.com/onlyoffice/documentserver:8.0.1
environment:
TZ: "Asia/Shanghai"
JWT_SECRET: "my_secret"
restart: unless-stopped

57
docker-compose.base.yml Normal file
View File

@ -0,0 +1,57 @@
services:
mongo:
container_name: jmalcloud_mongodb
image: mongo:7.0
environment:
TZ: "Asia/Shanghai"
volumes:
- ./docker/jmalcloud/mongodb/data/db:/data/db
- ./docker/jmalcloud/mongodb/backup:/dump
restart: unless-stopped
healthcheck:
test: ["CMD", "mongosh", "--eval", "db.adminCommand('ping')"]
interval: 10s
timeout: 5s
retries: 3
command: --wiredTigerCacheSizeGB 0.5
jmalcloud:
container_name: jmalcloud_server
image: jmal/jmalcloud:latest
environment:
MONGODB_URI: "mongodb://mongo:27017/jmalcloud"
TZ: "Asia/Shanghai"
JVM_OPTS: "-Xms512m -Xmx4g"
#EXACT_SEARCH: false
#PUID: 0
#PGID: 0
# 如果为空系统会自动生成(位置在/jmalcloud/files/.env)(openssl rand -hex 32)
ENCRYPTION_SECRET_KEY: ""
# 如果为空系统会自动生成(位置在/jmalcloud/files/.env)(openssl rand -hex 16)
ENCRYPTION_SALT: ""
volumes:
- ./docker/jmalcloud/files:/jmalcloud/files/
restart: unless-stopped
ports:
- 7072:8088
nginx:
container_name: jmalcloud_nginx
image: jmal/jmalcloud-nginx:latest
ports:
- 7070:80
- 7071:8089
environment:
TZ: "Asia/Shanghai"
restart: unless-stopped
links:
- jmalcloud
- office
office: # Optional
container_name: jmalcloud_office
image: onlyoffice/documentserver:9.0
environment:
TZ: "Asia/Shanghai"
JWT_SECRET: "my_secret"
restart: unless-stopped

View File

@ -0,0 +1,44 @@
services:
mongo:
container_name: ${CONTAINER_NAME_PREFIX}_mongodb
image: mongo:4.4
environment:
TZ: Asia/Shanghai
volumes:
- ${RESOURCE_DB_PATH}/data/db:/data/db
- ${RESOURCE_DB_PATH}/custom:/etc/mongo
- ${RESOURCE_DB_PATH}/backup:/dump
restart: unless-stopped
command: --wiredTigerCacheSizeGB 0.5
jmalcloud:
container_name: ${CONTAINER_NAME_PREFIX}_server
image: ghcr.io/jamebal/jmalcloud:latest
environment:
MONGODB_URI: mongodb://mongo:27017/jmalcloud
TZ: Asia/Shanghai
ports:
- ${SERVER_PORT}:8088
volumes:
- ${RESOURCE_FILE_PATH}:/jmalcloud/files/
restart: unless-stopped
nginx:
container_name: ${CONTAINER_NAME_PREFIX}_nginx
image: ghcr.io/jamebal/jmalcloud-nginx:latest
ports:
- ${APP_PORT}:80
- ${BLOG_PORT}:8089
environment:
TZ: Asia/Shanghai
links:
- jmalcloud
- office
restart: unless-stopped
office:
container_name: ${CONTAINER_NAME_PREFIX}_office
image: onlyoffice/documentserver:${OFFICE_IMAGE_VERSION}
environment:
TZ: Asia/Shanghai
JWT_ENABLED: false
ALLOW_PRIVATE_IP_ADDRESS: true
restart: unless-stopped
networks: {}

37
docker-entrypoint.sh Normal file
View File

@ -0,0 +1,37 @@
#!/bin/bash
set -e
USER_UID=${PUID:-0}
USER_GID=${PGID:-0}
TZ=${TZ:-Asia/Shanghai}
mkdir -p /log
chown ${USER_UID}:${USER_GID} /jmalcloud
chown -R ${USER_UID}:${USER_GID} /jmalcloud/models
chown -R ${USER_UID}:${USER_GID} /jmalcloud/tess4j
chown ${USER_UID}:${USER_GID} /jmalcloud/ip2region.xdb
chown -R ${USER_UID}:${USER_GID} /usr/local/mxcad
chown ${USER_UID}:${USER_GID} /usr/local/clouddisk-${VERSION}.jar
chown -R ${USER_UID}:${USER_GID} /usr/local/clouddisk-lib
chown -R ${USER_UID}:${USER_GID} log
exec gosu ${USER_UID}:${USER_GID} java ${JVM_OPTS} \
-Duser.timezone=${TZ} \
-Dfile.encoding=UTF-8 \
-Dloader.path=/usr/local/clouddisk-lib \
-jar /usr/local/clouddisk-${VERSION}.jar \
--spring.profiles.active=${RUN_ENVIRONMENT} \
--spring.data.mongodb.uri=${MONGODB_URI} \
--tess4j.data-path=${TESS4J_DATA_PATH} \
--file.exactSearch=${EXACT_SEARCH} \
--file.ngramMaxContentLengthMB=${NGRAM_MAX_CONTENT_LENGTH_MB} \
--file.ngramMinSize=${NGRAM_MIN_SIZE} \
--file.ngramMaxSize=${NGRAM_MAX_SIZE} \
--file.monitor=${FILE_MONITOR} \
--file.rootDir=${FILE_ROOT_DIR} \
--file.monitorIgnoreFilePrefix=${MONITOR_IGNORE_FILE_PREFIX} \
--logging.level.root=${LOG_LEVEL} \
--file.ip2region-db-path=/jmalcloud/ip2region.xdb \
--file.ocr-lite-onnx-model-path=/jmalcloud/models

335
jc.sh Executable file
View File

@ -0,0 +1,335 @@
#!/bin/bash
#fonts color
Green="\033[32m"
Red="\033[31m"
Font="\033[0m"
#notification information
OK="${Green}[OK]${Font}"
Error="${Red}[错误]${Font}"
cur_path="$(pwd)"
cur_arg=$@
COMPOSE="docker-compose"
check_docker() {
docker --version &>/dev/null
if [ $? -ne 0 ]; then
echo -e "${Error} ${Red}未安装 Docker!${Font}"
exit 1
fi
if ! docker ps > /dev/null 2>&1; then
echo -e "${Error} ${Red}请确保您有操作docker的权限。${Font}"
exit 1
fi
docker-compose version &>/dev/null
if [ $? -ne 0 ]; then
docker compose version &>/dev/null
if [ $? -ne 0 ]; then
echo -e "${Error} ${Red}未安装 Docker-compose!${Font}"
exit 1
fi
COMPOSE="docker compose"
fi
if [[ -n $($COMPOSE version | grep -E "\sv*1") ]]; then
$COMPOSE version
echo -e "${Error} ${Red}Docker-compose 的版本太低了,请升级到 v2+!${Font}"
exit 1
fi
if [ ! -f docker-compose.yml ]; then
cp docker-compose.example.yml docker-compose.yml
fi
}
run_exec() {
local container=$1
local cmd=$2
local name="$(env_get CONTAINER_NAME_PREFIX)_$container"
if [ -z "$name" ]; then
echo -e "${Error} ${Red}没有找到 $container 容器!${Font}"
exit 1
fi
docker exec -it "$name" /bin/sh -c "$cmd"
}
judge() {
if [[ 0 -eq $? ]]; then
echo -e "${OK} ${Green}$1 完成${Font}"
sleep 1
else
echo -e "${Error} ${Red}$1 失败${Font}"
exit 1
fi
}
rand() {
local min=$1
local max=$(($2 - $min + 1))
local num=$(($RANDOM + 1000000000))
echo $(($num % $max + $min))
}
local_ipv4() {
local ip="127.0.0.1"
if [[ $(uname) == 'Linux' ]]; then
ip=$(ip a | grep inet | grep -v 127.0.0.1 | grep -v inet6 | awk '{print $2}' | tr -d "addr:" | head -n 1)
fi
if [[ $(uname) == 'Darwin' ]]; then
ip=$(ifconfig -a | grep inet | grep -v 127.0.0.1 | grep -v inet6 | awk '{print $2}' | tr -d "addr:" | head -n 1)
fi
echo "${ip%/*}"
}
env_get() {
local key=$1
local value=$(cat ${cur_path}/.env | grep "^$key=" | awk -F '=' '{print $2}')
echo "$value"
}
env_set() {
local key=$1
local val=$2
local exist=$(cat ${cur_path}/.env | grep "^$key=")
if [ -z "$exist" ]; then
echo "$key=$val" >>$cur_path/.env
else
if [[ $(uname) == 'Linux' ]]; then
sed -i "/^${key}=/c\\${key}=${val}" ${cur_path}/.env
fi
if [ $? -ne 0 ]; then
echo -e "${Error} ${Red}设置env参数失败!${Font}"
exit 1
fi
fi
}
arg_get() {
local find="n"
local value=""
for var in $cur_arg; do
if [[ "$find" == "y" ]]; then
if [[ ! $var =~ "--" ]]; then
value=$var
fi
break
fi
if [[ "--$1" == "$var" ]] || [[ "-$1" == "$var" ]]; then
find="y"
value="yes"
fi
done
echo $value
}
run_mongo() {
if [ "$1" = "dump" ]; then
# 备份数据库
mkdir -p ${cur_path}/docker/mongodb/backup
run_exec mongodb "mongodump -d jmalcloud -o /dump --gzip --quiet"
judge "备份数据库"
elif [ "$1" = "restore" ]; then
# 恢复数据库
run_exec mongodb "mongorestore --gzip --nsInclude=jmalcloud.* --dir /dump/jmalcloud --quiet"
judge "恢复数据库"
fi
}
help() {
echo "
Usage: install.sh COMMAND [OPTIONS]
Options:
--port 网盘端口,默认7070
--blog_port 博客端口,默认7071
--server_port 服务端口,默认7072(API地址: http://127.0.0.1:7072/public/api)
--prefix 容器名称前缀,默认为jmalcloud
Management Commands:
init,install 安装
uninstall 卸载
pull 拉取最新的的程序和镜像
update 更新
start 启动
stop 停止
restart 重启
mongo 执行容器 mongodb 容器内的命令 example: ./jc.sh mongo mongostat
nginx 执行容器 nginx 容器内的命令 example: ./jc.sh nginx nginx -v
dump 备份数据库
restore 恢复数据库
reinstall 卸载后再安装
reset-password 重置初始管理员的密码
"
}
env_init() {
env_set OFFICE_IMAGE_VERSION "8.0.1"
env_set APP_PORT 7070
env_set BLOG_PORT 7071
env_set SERVER_PORT 7072
env_set FTP_SERVER_PORT 7073
env_set CONTAINER_NAME_PREFIX "jmalcloud"
local file_ptah=$(cat ${cur_path}/.env | grep "^RESOURCE_DB_PATH=")
if [ -z "$file_ptah" ]; then
env_set RESOURCE_DB_PATH "$cur_path/docker/jmalcloud/mongodb"
fi
local db_ptah=$(cat ${cur_path}/.env | grep "^RESOURCE_FILE_PATH=")
if [ -z "$db_ptah" ]; then
env_set RESOURCE_FILE_PATH "$cur_path/docker/jmalcloud/files"
fi
}
before_start() {
[[ "$(arg_get port)" -gt 0 ]] && env_set APP_PORT "$(arg_get port)"
[[ "$(arg_get blog_port)" -gt 0 ]] && env_set BLOG_PORT "$(arg_get blog_port)"
[[ "$(arg_get server_port)" -gt 0 ]] && env_set SERVER_PORT "$(arg_get server_port)"
[[ "$(arg_get ftp_server_port)" -gt 0 ]] && env_set FTP_SERVER_PORT "$(arg_get ftp_server_port)"
[[ "$(arg_get prefix)" -gt 0 ]] && env_set CONTAINER_NAME_PREFIX "$(arg_get prefix)"
}
install() {
# 启动容器
before_start
$COMPOSE up -d
# 检测服务是否启动
check_run "安装"
}
uninstall() {
read -rp "确定要卸载(含:删除容器、所有文件、数据库、日志)吗?(y/n): " uninstall
[[ -z ${uninstall} ]] && uninstall="N"
case $uninstall in
[yY][eE][sS] | [yY])
echo -e "${Red} 开始卸载... ${Font}"
;;
*)
echo -e "${OK} ${Green} 终止卸载 ${Font}"
exit 2
;;
esac
$COMPOSE down
rm -rf "./docker"
echo -e "${OK} ${Green} 卸载完成 ${Font}"
}
check_run() {
spin=('\' '|' '/' '-')
server_url="http://$(local_ipv4):$(env_get BLOG_PORT)"
for ((i = 0; i <= 350; i++)); do
url_status=$(curl -s -m 5 -IL "$server_url" | grep 200)
if [ "$url_status" != "" ]; then
echo -ne " \r"
echo -e "${OK} ${Green}$1完成${Font}"
echo "网盘地址: http://$(local_ipv4):$(env_get APP_PORT)"
echo "博客地址: $server_url"
echo "API 地址: http://$(local_ipv4):$(env_get SERVER_PORT)/public/api"
break
else
echo -n "正在启动服务, 请稍等... ${spin[$((i % 4))]}"
echo -n -e \\r
sleep 0.2
fi
done
url_status=$(curl -s -m 5 -IL $server_url | grep 200)
if [ "$url_status" == "" ]; then
echo -e "${Error} ${Red}$1失败,反馈给开发者: https://github.com/jamebal/jmal-cloud-view/issues${Font}"
fi
}
pull() {
git fetch --all
git reset --hard origin/$(git branch | sed -n -e 's/^\* \(.*\)/\1/p')
git pull origin master
$COMPOSE pull
}
backup_and_install() {
local container_mongodb="$(env_get CONTAINER_NAME_PREFIX)_mongodb"
if docker ps --format '{{.Names}}' | grep -q "$container_mongodb"; then
run_mongo "dump"
run_exec nginx "nginx -s reload"
install
else
install
fi
}
reset_password() {
local container_server="$(env_get CONTAINER_NAME_PREFIX)_server"
echo -e "正在重置密码..."
docker restart "$container_server" > /dev/null 2>&1
run_exec mongodb "mongo jmalcloud --eval \"db.getCollection('user').update({ 'creator': true }, {\\\$set: { 'password': '1000:c5b705ea13a1221f5e59110947ed806f8a978e955fbd2ed6:22508de12228c34a235454a0caf3bcaa5552858543258e56' }}, { 'multi': false, 'upsert': false })\" > /dev/null 2>&1"
check_run "重置"
echo -e "${OK} ${Green}重置后的密码为:jmalcloud${Font}"
}
########################################################################################################
check_docker
env_init
if [ $# -gt 0 ]; then
if [[ "$1" == "init" ]] || [[ "$1" == "install" ]]; then
shift 1
install
elif [[ "$1" == "pull" ]]; then
shift 1
pull
elif [[ "$1" == "update" ]]; then
shift 1
backup_and_install
elif [[ "$1" == "uninstall" ]]; then
shift 1
uninstall
elif [[ "$1" == "reinstall" ]]; then
shift 1
uninstall
sleep 3
install
elif [[ "$1" == "reset-password" ]]; then
shift 1
reset_password
elif [[ "$1" == "port" ]]; then
shift 1
env_set APP_PORT "$1"
before_start
$COMPOSE up -d
check_run "修改"
elif [[ "$1" == "nginx" ]]; then
shift 1
e="$@" && run_exec nginx "$e"
elif [[ "$1" == "mongo" ]]; then
shift 1
e="$@" && run_exec mongodb "$e"
elif [[ "$1" == "dump" ]]; then
shift 1
run_mongo "dump"
elif [[ "$1" == "restore" ]]; then
shift 1
run_mongo "restore"
elif [[ "$1" == "start" ]]; then
shift 1
# 启动容器
before_start
$COMPOSE start "$@"
elif [[ "$1" == "stop" ]]; then
shift 1
$COMPOSE stop "$@"
elif [[ "$1" == "restart" ]]; then
shift 1
$COMPOSE stop "$@"
# 启动容器
before_start
$COMPOSE start "$@"
# 检测服务是否启动
check_run "启动"
else
if [[ "$1" == "-h" ]] || [[ "$1" == "--help" ]] || [[ "$1" == "man" ]]; then
help
else
echo "$1 is not a support parameter."
fi
fi
else
help
fi

475
pom.xml Normal file
View File

@ -0,0 +1,475 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>3.5.3</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>com.jmal</groupId>
<artifactId>clouddisk</artifactId>
<version>2.15.0</version>
<name>clouddisk</name>
<description>Cloud Disk</description>
<packaging>jar</packaging>
<properties>
<java.version>21</java.version>
<maven-jar-plugin.version>3.0.0</maven-jar-plugin.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<lucene.version>10.2.2</lucene.version>
<zxing.version>3.5.3</zxing.version>
</properties>
<repositories>
<repository>
<id>openkm</id>
<url>https://maven.openkm.com/</url>
</repository>
<repository>
<id>e-iceblue</id>
<url>https://repo.e-iceblue.cn/repository/maven-public/</url>
</repository>
</repositories>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.google.zxing</groupId>
<artifactId>core</artifactId>
<version>${zxing.version}</version>
</dependency>
<dependency>
<groupId>com.google.zxing</groupId>
<artifactId>javase</artifactId>
<version>${zxing.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
<dependencies>
<!--向后兼容-->
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-backward-codecs</artifactId>
<version>${lucene.version}</version>
</dependency>
<!--lucene 核心包-->
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
<version>${lucene.version}</version>
</dependency>
<!--对分词索引查询解析-->
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-queryparser</artifactId>
<version>${lucene.version}</version>
</dependency>
<!--一般分词器,适用于英文分词-->
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analysis-common</artifactId>
<version>${lucene.version}</version>
</dependency>
<!--检索关键字高亮显示 -->
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-highlighter</artifactId>
<version>${lucene.version}</version>
</dependency>
<!-- smartcn中文分词器 -->
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analysis-smartcn</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-expressions</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>io.github.cdimascio</groupId>
<artifactId>dotenv-java</artifactId>
<version>3.2.0</version>
</dependency>
<!-- crypto -->
<dependency>
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-crypto</artifactId>
</dependency>
<!-- totp -->
<dependency>
<groupId>dev.samstevens.totp</groupId>
<artifactId>totp</artifactId>
<version>1.7.1</version>
</dependency>
<!-- spring-->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-mongodb</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<exclusions>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-ldap</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-configuration-processor</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-thymeleaf</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-validation</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-aop</artifactId>
</dependency>
<!-- 转换图片为webp-->
<dependency>
<groupId>com.github.gotson</groupId>
<artifactId>webp-imageio</artifactId>
<version>0.2.2</version>
</dependency>
<dependency>
<groupId>com.github.albfernandez</groupId>
<artifactId>juniversalchardet</artifactId>
<version>2.5.0</version>
</dependency>
<dependency>
<groupId>net.sourceforge.tess4j</groupId>
<artifactId>tess4j</artifactId>
<version>5.13.0</version>
<exclusions>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- 反射-->
<dependency>
<groupId>org.reflections</groupId>
<artifactId>reflections</artifactId>
<version>0.10.2</version>
</dependency>
<!-- fastjson-->
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>2.0.47</version>
</dependency>
<!-- 缩略图处理-->
<dependency>
<groupId>net.coobird</groupId>
<artifactId>thumbnailator</artifactId>
<version>0.4.20</version>
</dependency>
<!--解析MP3-->
<dependency>
<groupId>org</groupId>
<artifactId>jaudiotagger</artifactId>
<version>2.0.3</version>
</dependency>
<!-- 获取图片拍摄信息-->
<dependency>
<groupId>com.drewnoakes</groupId>
<artifactId>metadata-extractor</artifactId>
<version>2.19.0</version>
</dependency>
<!-- 文件的压缩和解压-->
<dependency>
<groupId>org.apache.ant</groupId>
<artifactId>ant</artifactId>
<version>1.10.11</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>1.26.1</version>
</dependency>
<!-- api文档-->
<dependency>
<groupId>org.springdoc</groupId>
<artifactId>springdoc-openapi-starter-webmvc-ui</artifactId>
<version>2.8.6</version>
<exclusions>
<exclusion>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-kotlin</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-kotlin</artifactId>
</dependency>
<dependency>
<groupId>io.reactivex.rxjava3</groupId>
<artifactId>rxjava</artifactId>
<version>3.1.8</version>
</dependency>
<!-- lombok-->
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<optional>true</optional>
</dependency>
<!-- tools-->
<dependency>
<groupId>org.apache.ftpserver</groupId>
<artifactId>ftpserver-core</artifactId>
<version>1.2.1</version>
</dependency>
<dependency>
<groupId>org.apache.ftpserver</groupId>
<artifactId>ftplet-api</artifactId>
<version>1.2.0</version>
</dependency>
<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-all</artifactId>
<version>5.8.25</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi</artifactId>
<version>5.4.1</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>5.4.1</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-scratchpad</artifactId>
<version>5.4.1</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.16.1</version>
</dependency>
<dependency>
<groupId>io.methvin</groupId>
<artifactId>directory-watcher</artifactId>
<version>0.18.0</version>
</dependency>
<dependency>
<groupId>e-iceblue</groupId>
<artifactId>spire.doc.free</artifactId>
<version>5.2.0</version>
</dependency>
<dependency>
<groupId>org.glassfish.jaxb</groupId>
<artifactId>jaxb-runtime</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-core</artifactId>
<version>2.9.2</version>
</dependency>
<dependency>
<groupId>nl.siegmann.epublib</groupId>
<artifactId>epublib-core</artifactId>
<version>3.1</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.18.1</version>
</dependency>
<dependency>
<groupId>org.xhtmlrenderer</groupId>
<artifactId>flying-saucer-pdf</artifactId>
<version>9.9.0</version>
</dependency>
<dependency>
<groupId>com.github.jai-imageio</groupId>
<artifactId>jai-imageio-core</artifactId>
<version>1.4.0</version>
</dependency>
<dependency>
<groupId>com.github.jai-imageio</groupId>
<artifactId>jai-imageio-jpeg2000</artifactId>
<version>1.4.0</version>
</dependency>
<dependency>
<groupId>com.itextpdf</groupId>
<artifactId>html2pdf</artifactId>
<version>5.0.5</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-xml</artifactId>
</dependency>
<!-- jwt-->
<dependency>
<groupId>com.auth0</groupId>
<artifactId>java-jwt</artifactId>
<version>3.18.2</version>
</dependency>
<!-- websocket-->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-websocket</artifactId>
</dependency>
<!-- springboot 缓存自动配置-->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-cache</artifactId>
</dependency>
<dependency>
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>caffeine</artifactId>
<version>2.8.0</version>
</dependency>
<dependency>
<groupId>io.projectreactor</groupId>
<artifactId>reactor-core</artifactId>
<version>3.2.12.RELEASE</version>
</dependency>
<dependency>
<groupId>com.github.oshi</groupId>
<artifactId>oshi-core</artifactId>
<version>6.2.2</version>
</dependency>
<!-- ip定位-->
<dependency>
<groupId>org.lionsoul</groupId>
<artifactId>ip2region</artifactId>
<version>2.7.0</version>
</dependency>
<!-- minio -->
<dependency>
<groupId>io.minio</groupId>
<artifactId>minio</artifactId>
<version>8.5.13</version>
</dependency>
<!-- 阿里云 OSS -->
<dependency>
<groupId>com.aliyun.oss</groupId>
<artifactId>aliyun-sdk-oss</artifactId>
<version>3.17.4</version>
<exclusions>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- 腾讯云 OSS -->
<dependency>
<groupId>com.qcloud</groupId>
<artifactId>cos_api</artifactId>
<version>5.6.247</version>
<exclusions>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- test-->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-model</artifactId>
<version>3.9.1</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<mainClass>com.jmal.clouddisk.JmalCloudApplication</mainClass>
<executable>true</executable>
<layout>ZIP</layout>
<includes>
<include>
<groupId>nothing</groupId>
<artifactId>nothing</artifactId>
</include>
</includes>
</configuration>
</plugin>
<!-- 将第三方包拷贝到lib目录 -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration><outputDirectory>${project.build.directory}/lib</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

25
release-www.sh Normal file
View File

@ -0,0 +1,25 @@
#!/bin/bash
# version
if [ -n "$1" ]; then
echo "version: $1"
else
echo "需要带上版本号, 例如 sh release-www.sh 2.5.1"
exit
fi
version=$1
# jmal-cloud-view Directory location
view_dir="/Users/jmal/studio/myProject/github/jmal-cloud-view"
# jmal-cloud-server Directory location
server_dir="/Users/jmal/studio/myProject/github/jmal-cloud-server"
# build jmal-cloud-view
cd $view_dir || exit
echo "location: ${view_dir} "
if [ ! -f "dist-$version.tar" ]; then
npm run build:prod
tar -czf "dist-$version.tar" dist
fi
echo "current $(pwd)"
echo "copy dist-$version.tar to $server_dir/www/releases/dist-latest.tar"
cp "dist-$version.tar" $server_dir"/www/releases/dist-latest.tar"

View File

@ -0,0 +1,34 @@
package com.jmal.clouddisk;
import cn.hutool.crypto.SecureUtil;
import com.jmal.clouddisk.util.TesseractUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.scheduling.annotation.EnableScheduling;
/**
* ClouddiskApplication
*
* @author jmal
*/
@SpringBootApplication
@EnableCaching
@EnableScheduling
@Slf4j
public class JmalCloudApplication {
public static void main(String[] args) {
SecureUtil.disableBouncyCastle();
SpringApplication application = new SpringApplication(JmalCloudApplication.class);
// 允许循环引用
application.setAllowCircularReferences(true);
// dev环境下设置tesseract的lib路径
TesseractUtil.setTesseractLibPath();
application.run(args);
}
}

View File

@ -0,0 +1,108 @@
package com.jmal.clouddisk.acpect;
import cn.hutool.core.text.CharSequenceUtil;
import com.jmal.clouddisk.annotation.LogOperatingFun;
import com.jmal.clouddisk.model.LogOperation;
import com.jmal.clouddisk.model.rbac.ConsumerDTO;
import com.jmal.clouddisk.service.impl.LogService;
import com.jmal.clouddisk.service.impl.UserLoginHolder;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.RequiredArgsConstructor;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Pointcut;
import org.aspectj.lang.reflect.MethodSignature;
import org.springframework.stereotype.Component;
import org.springframework.web.context.request.RequestContextHolder;
import org.springframework.web.context.request.ServletRequestAttributes;
import java.lang.reflect.Method;
/***
* @Description 操作日志切面现实
* @author jmal
**/
@Aspect
@Component
@RequiredArgsConstructor
public class LogOperatingAspect {
private final UserLoginHolder userLoginHolder;
private final LogService logService;
/**
* 切入点
**/
@Pointcut("@annotation(com.jmal.clouddisk.annotation.LogOperatingFun)")
public void addAdvice() {
}
/**
* 日志切面拦截器,用来记录每个请求端日志
* 根据LogOperatingFun注解判判断是否记录日志
* 如果LogOperatingFun注解没有value值,者使用swagger的ApiOperation注解的值
*/
@Around("addAdvice()")
public Object interceptor(ProceedingJoinPoint joinPoint) throws Throwable {
String username = userLoginHolder.getUsername();
long time = System.currentTimeMillis();
// 执行方法
Object result = joinPoint.proceed();
// 执行耗时
long timeConsuming = System.currentTimeMillis() - time;
// 接收到请求,记录请求内容
ServletRequestAttributes attributes = (ServletRequestAttributes) RequestContextHolder.getRequestAttributes();
if (attributes != null) {
LogOperation logOperation = new LogOperation();
logOperation.setTime(timeConsuming);
logOperation.setUsername(username);
parserLog(joinPoint, result, attributes, logOperation);
}
return result;
}
/***
* 解析日志
*/
private void parserLog(ProceedingJoinPoint joinPoint, Object result, ServletRequestAttributes attributes, LogOperation logOperation) {
// 获取类/方法上的操作日志注解
MethodSignature methodSignature = ((MethodSignature) joinPoint.getSignature());
//获取method对象
Method method = methodSignature.getMethod();
LogOperatingFun logOperatingFun = method.getAnnotation(LogOperatingFun.class);
// swagger的ApiOperation注解, 用来获取操作说明
Operation apiOperation = method.getAnnotation(Operation.class);
// swagger的Api注解, 用来获取操作模块
@SuppressWarnings("unchecked")
Tag api = (Tag) joinPoint.getSourceLocation().getWithinType().getAnnotation(Tag.class);
if(api != null){
logOperation.setOperationModule(api.name());
}
// 操作功能
String operationFun = logOperatingFun.value();
if(CharSequenceUtil.isBlank(operationFun)){
operationFun = apiOperation.summary();
}
logOperation.setOperationFun(operationFun);
String logType = logOperatingFun.logType().name();
logOperation.setType(logType);
// 不记录浏览日志
if(LogOperation.Type.BROWSE.name().equals(logType)){
return;
}
if(LogOperation.Type.LOGIN.name().equals(logType)){
// 登录日志
for (Object arg : joinPoint.getArgs()) {
if (arg instanceof ConsumerDTO consumerDTO) {
logOperation.setUsername(consumerDTO.getUsername());
break;
}
}
}
// 添加日志
logService.addLogBefore(logOperation, result, attributes.getRequest(), attributes.getResponse());
}
}

View File

@ -0,0 +1,87 @@
package com.jmal.clouddisk.acpect;
import cn.hutool.core.text.CharSequenceUtil;
import com.jmal.clouddisk.annotation.Permission;
import com.jmal.clouddisk.exception.ExceptionType;
import com.jmal.clouddisk.service.impl.UserLoginHolder;
import com.jmal.clouddisk.service.impl.UserServiceImpl;
import com.jmal.clouddisk.util.CaffeineUtil;
import com.jmal.clouddisk.util.ResultUtil;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Pointcut;
import org.aspectj.lang.reflect.MethodSignature;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.lang.reflect.Method;
import java.util.List;
/**
* @Description 授权切面
* @blame jmal
* @Date 2021/1/8 8:38 下午
*/
@Aspect
@Component
public class PermissionAspect {
@Autowired
private UserLoginHolder userLoginHolder;
@Autowired
private UserServiceImpl userService;
@Pointcut("@annotation(com.jmal.clouddisk.annotation.Permission)")
public void privilege() {
}
@Around("privilege()")
public Object before(ProceedingJoinPoint joinPoint) throws Throwable {
MethodSignature methodSignature = (MethodSignature) joinPoint.getSignature();
Method targetMethod = methodSignature.getMethod();
final Permission permission = privilegeParse(targetMethod);
if (permission == null) {
// 方法上没有权限注解, 直接调用方法
return joinPoint.proceed();
}
boolean onlyCreator = permission.onlyCreator();
if (onlyCreator) {
// 只有创建者才有权限
if (userService.getIsCreator(userLoginHolder.getUserId())) {
return joinPoint.proceed();
} else {
return ResultUtil.error(ExceptionType.PERMISSION_DENIED);
}
}
String authority = permission.value();
if (CharSequenceUtil.isBlank(authority)) {
// 方法上没有权限注解, 直接调用方法
return joinPoint.proceed();
}
// 上传前检查空间是否够用
if ("cloud:file:upload".equals(authority) && CaffeineUtil.spaceFull(userLoginHolder.getUserId())) {
return ResultUtil.error(ExceptionType.SPACE_FULL);
}
// 获取当前身份的权限
List<String> authorities = userLoginHolder.getAuthorities();
if (authorities != null && authorities.contains(authority)) {
return joinPoint.proceed();
}
return ResultUtil.error(ExceptionType.PERMISSION_DENIED);
}
/***
* 解析权限注解
* @param method Method
* @return Permission
*/
public static Permission privilegeParse(Method method) {
if (method.isAnnotationPresent(Permission.class)) {
return method.getAnnotation(Permission.class);
}
return null;
}
}

View File

@ -0,0 +1,58 @@
package com.jmal.clouddisk.annotation;
import cn.hutool.core.text.CharSequenceUtil;
import jakarta.annotation.PostConstruct;
import org.reflections.Reflections;
import org.springframework.stereotype.Component;
import org.springframework.web.bind.annotation.RestController;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
/**
* @Description 注解工具
* @blame jmal
* @Date 2021/1/8 8:38 下午
*/
@Component
public class AnnoManageUtil {
/***
* 权限标识列表
*/
public static final List<String> AUTHORITIES = new ArrayList<>();
/**
* 通过反射获取所有的权限标识
*
* @return
*/
@PostConstruct
public void getAllAuthorities() {
Reflections reflections = new Reflections("com.jmal.clouddisk.controller", "com.jmal.clouddisk.oss.web");
Set<Class<?>> classesList = reflections.getTypesAnnotatedWith(RestController.class);
List<String> arrayList = new ArrayList<>();
for (Class<?> classes : classesList) {
//得到该类下面的所有方法
Method[] methods = classes.getDeclaredMethods();
for (Method method : methods) {
//得到该类下面的Permission注解
Permission permission = method.getAnnotation(Permission.class);
if(permission == null){
continue;
}
if(CharSequenceUtil.isBlank(permission.value())){
continue;
}
if(arrayList.contains(permission.value())){
continue;
}
arrayList.add(permission.value());
}
}
AUTHORITIES.addAll(arrayList.stream().sorted().toList());
}
}

View File

@ -0,0 +1,25 @@
package com.jmal.clouddisk.annotation;
import com.jmal.clouddisk.model.LogOperation;
import java.lang.annotation.*;
/**
* 操作方法日志注解
* @author jmal
*/
@Documented
@Target({ElementType.METHOD, ElementType.TYPE})
@Retention(RetentionPolicy.RUNTIME)
public @interface LogOperatingFun {
/**
* 操作功能
*/
String value() default "";
/***
* 日志类型
*/
LogOperation.Type logType() default LogOperation.Type.OPERATION;
}

View File

@ -0,0 +1,26 @@
package com.jmal.clouddisk.annotation;
/**
* OperationType 数据操作类型枚举
*
* @author jmal
*/
public enum OperationType {
/***
* 添加
*/
create,
/***
* 查询
*/
read,
/***
* 更新
*/
update,
/***
* 删除
*/
delete
}

View File

@ -0,0 +1,26 @@
package com.jmal.clouddisk.annotation;
import java.lang.annotation.*;
/**
* 操作日志注解
* @author jmal
*/
@Documented
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.METHOD)
public @interface Permission {
/***
* @return 权限标识
*/
String value() default "";
/***
* @return 只有网盘创建者才能通过
*/
boolean onlyCreator() default false;
}

View File

@ -0,0 +1,27 @@
package com.jmal.clouddisk.config;
import cn.hutool.core.util.HexUtil;
import lombok.RequiredArgsConstructor;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.security.crypto.encrypt.Encryptors;
import org.springframework.security.crypto.encrypt.TextEncryptor;
import java.nio.charset.StandardCharsets;
@Configuration
@RequiredArgsConstructor
public class EncryptionConfig {
private final EncryptionKeyProvider encryptionKeyProvider;
@Bean
public TextEncryptor textEncryptor() {
String secretKey = encryptionKeyProvider.getSecretKey();
String salt = encryptionKeyProvider.getSalt();
if (!HexUtil.isHexNumber(salt)) {
salt = HexUtil.encodeHexStr(salt.getBytes(StandardCharsets.UTF_8));
}
return Encryptors.text(secretKey, salt);
}
}

View File

@ -0,0 +1,130 @@
package com.jmal.clouddisk.config;
import io.github.cdimascio.dotenv.Dotenv;
import jakarta.annotation.PostConstruct;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.io.PrintWriter;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.SecureRandom;
import java.util.HexFormat;
@Component
@RequiredArgsConstructor
@Getter
@Slf4j
public class EncryptionKeyProvider {
private static final String SECRET_KEY_NAME = "ENCRYPTION_SECRET_KEY";
private static final String SALT_NAME = "ENCRYPTION_SALT";
private final FileProperties fileProperties;
private final Environment environment;
private String secretKey;
private String salt;
@PostConstruct
public void init() throws IOException {
// 步骤 1: 优先从环境变量加载
boolean loadedFromEnv = loadKeysFromEnvironment();
// 步骤 2: 如果环境变量没有提供密钥,则回退到文件系统
if (!loadedFromEnv) {
log.info("Encryption keys not found in environment variables. Checking for .env file...");
loadKeysFromFileSystem();
}
// 步骤 3: 最终验证,确保密钥已成功加载
if (this.secretKey == null || this.secretKey.isBlank() || this.salt == null || this.salt.isBlank()) {
throw new IllegalStateException("Failed to load or generate encryption keys from any source (environment, .env file). Application cannot start securely.");
}
log.info("Encryption keys loaded successfully.");
}
/**
* 尝试从环境变量加载密钥。
* @return 如果成功从环境变量加载了两个密钥,则返回 true。
*/
private boolean loadKeysFromEnvironment() {
String envSecretKey = environment.getProperty(SECRET_KEY_NAME);
String envSalt = environment.getProperty(SALT_NAME);
if (envSecretKey != null && !envSecretKey.isBlank() && envSalt != null && !envSalt.isBlank()) {
log.info("Found {} and {} in environment variables. Using them for encryption.", SECRET_KEY_NAME, SALT_NAME);
this.secretKey = envSecretKey;
this.salt = envSalt;
return true;
}
return false;
}
/**
* 从 .env 文件加载或创建密钥。
*/
private void loadKeysFromFileSystem() throws IOException {
Path envPath = Paths.get(fileProperties.getRootDir(), ".env");
if (Files.exists(envPath)) {
log.info("Found existing .env file at [{}]. Loading keys from file.", envPath);
loadKeysFromDotenv(envPath.getParent().toString());
} else {
log.warn("`.env` file not found at [{}]. Generating new encryption keys and saving to file. THIS IS A SECURITY-SENSITIVE EVENT.", envPath);
createNewEnvFileAndSetKeys(envPath);
}
}
private void loadKeysFromDotenv(String directory) {
Dotenv dotenv = Dotenv.configure().directory(directory).load();
this.secretKey = dotenv.get(SECRET_KEY_NAME);
this.salt = dotenv.get(SALT_NAME);
}
private void createNewEnvFileAndSetKeys(Path envPath) throws IOException {
// 生成安全的随机密钥和盐
String newSecretKey = generateRandomHexString(32);
String newSalt = generateRandomHexString(16);
Files.createDirectories(envPath.getParent());
try (PrintWriter writer = new PrintWriter(envPath.toFile(), StandardCharsets.UTF_8)) {
writer.println("# 该文件由应用程序在首次运行时自动生成。");
writer.println("# =======================================================================");
writer.println("# !! 警告:这是一个极其敏感的安全配置文件,请勿随意修改或删除 !!");
writer.println("#");
writer.println("# 更改或丢失此文件中的密钥 (" + SECRET_KEY_NAME + " 或 " + SALT_NAME + ") 将导致:");
writer.println("# 1. 所有已启用MFA(两步验证)的用户将无法登录,需要重置MFA。");
writer.println("# 2. 已添加的OSS(对象存储)将无法使用, 需要重新添加。");
writer.println("#");
writer.println("# 请务必在安全的位置备份此文件。");
writer.println("# =======================================================================");
writer.println();
writer.println("# 如果您希望通过环境变量来覆盖这些值,请设置名为 " + SECRET_KEY_NAME + " 和 " + SALT_NAME + " 的环境变量。");
writer.println();
writer.println(SECRET_KEY_NAME + "=" + newSecretKey);
writer.println(SALT_NAME + "=" + newSalt);
}
log.info("Successfully created and populated new .env file at [{}].", envPath);
// 将新生成的密钥加载到当前实例中
this.secretKey = newSecretKey;
this.salt = newSalt;
}
private String generateRandomHexString(int byteLength) {
SecureRandom random = new SecureRandom();
byte[] bytes = new byte[byteLength];
random.nextBytes(bytes);
return HexFormat.of().formatHex(bytes);
}
}

View File

@ -0,0 +1,22 @@
package com.jmal.clouddisk.config;
import org.springframework.boot.web.server.ErrorPage;
import org.springframework.boot.web.server.ErrorPageRegistrar;
import org.springframework.boot.web.server.ErrorPageRegistry;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Component;
/**
* @author jmal
* @Description 错误页
* @Date 2020/12/16 3:39 下午
*/
@Component
public class ErrorConfig implements ErrorPageRegistrar {
@Override
public void registerErrorPages(ErrorPageRegistry registry) {
ErrorPage[] errorPages = new ErrorPage[1];
errorPages[0] = new ErrorPage(HttpStatus.NOT_FOUND, "/public/404");
registry.addErrorPages(errorPages);
}
}

View File

@ -0,0 +1,208 @@
package com.jmal.clouddisk.config;
import cn.hutool.core.io.file.PathUtil;
import cn.hutool.core.util.StrUtil;
import com.jmal.clouddisk.lucene.LuceneService;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.PropertySource;
import org.springframework.stereotype.Component;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
* @Description 文件存储配置类
* @Author jmal
* @Date 2020-03-24 11:15
*/
@Data
@Component
@PropertySource(value = "classpath:file.yml", factory = YamlPropertyLoaderFactory.class)
@ConfigurationProperties(prefix = "file")
@Slf4j
public class FileProperties {
/***
* 文件存储根目录 文件监控目录
*/
private String rootDir = System.getProperty("user.dir");
/***
* 断点续传的临时文件目录名称 位于rootDir下,文件监控扫描忽略的目录
*/
private String chunkFileDir = "chunkFileTemp";
/**
* 视频转码后的缓存目录, 位于 ${chunkFileDir}/${username}/${videoTranscodeCache}
*/
private String videoTranscodeCache = "videoTranscodeCache";
/**
* 回收站目录, 位于 ${chunkFileDir}/${username}/${jmalcloudTrashDir}
*/
private String jmalcloudTrashDir = ".jmalcloudTrash";
/**
* lucene索引存储目录
*/
private String luceneIndexDir = "luceneIndex";
/***
* 用户头像默认存储路径
*/
private String userImgDir = "/Image/usr/";
/***
* markdown类型文件(文章)默认存储的位置
*/
private String documentDir = "/Document/";
/***
* markdown类型文件(文章)里图片默认存储的位置
*/
private String documentImgDir = "/Image/Document/";
/**
* FTP server 端口号
*/
private Integer ftpServerPort = 8089;
/***
* 默认分隔符
*/
private String separator = "/";
/***
* 文本编辑器支持的文本类型
*/
private List<String> simText = new ArrayList<>();
/***
* 文档类型
*/
private List<String> document = new ArrayList<>();
/***
* 是否开启文件监控(默认开启)
* 开启文件监控会监控 ${rootDir} 目录下文件的变化
*/
private Boolean monitor = true;
/***
* 文件监控扫描时间间隔(秒)
*/
private Long timeInterval = 10L;
/**
* 文件监控忽略的文件前缀
*/
private List<String> monitorIgnoreFilePrefix;
/**
* webDAV协议前缀
*/
private String webDavPrefix;
/***
* ip2region-path
*/
private String ip2regionDbPath;
/**
* ocr-lite-onnx模型路径
*/
private String ocrLiteONNXModelPath;
/**
* 是否开启精确搜索
*/
private Boolean exactSearch = false;
/**
* ngram最大内容长度
*/
private Double ngramMaxContentLengthMB = 5.0;
/**
* ngram最小长度
*/
private Integer ngramMinSize = 2;
/**
* ngram最大长度
*/
private Integer ngramMaxSize = 6;
public void setIp2regionDbPath(String path) {
Path dbPath = Paths.get(path);
if (!PathUtil.exists(dbPath, true)) {
log.error("Error: Invalid ip2region.xdb file");
ip2regionDbPath = null;
return;
}
this.ip2regionDbPath = path;
}
private static final long GIGABYTE = 1024L * 1024 * 1024;
private static final double NGRAM_THRESHOLD_MB = 3.0;
private static final int MAX_MEMORY_GB = 4;
private static final double ONE_GB_CONFIG = 0.1;
private static final double TWO_GB_CONFIG = 1.0;
private static final double THREE_GB_CONFIG = 2.0;
public int getNgramMaxContentLength() {
double effectiveNgramMaxContentLengthMB = determineEffectiveNGramLength();
return (int) (effectiveNgramMaxContentLengthMB * LuceneService.BYTES_PER_MB);
}
private double determineEffectiveNGramLength() {
// 当可用内存小于4G,需要设置更小的ngramMaxContentLengthMB
long maxMemory = Runtime.getRuntime().maxMemory();
double effectiveNgramMaxContentLengthMB = this.ngramMaxContentLengthMB;
if (maxMemory < MAX_MEMORY_GB * GIGABYTE && effectiveNgramMaxContentLengthMB >= NGRAM_THRESHOLD_MB) {
if (maxMemory < GIGABYTE) {
effectiveNgramMaxContentLengthMB = ONE_GB_CONFIG;
} else if (maxMemory < 2 * GIGABYTE) {
effectiveNgramMaxContentLengthMB = TWO_GB_CONFIG;
} else {
effectiveNgramMaxContentLengthMB = THREE_GB_CONFIG;
}
}
return effectiveNgramMaxContentLengthMB;
}
public void setMonitorIgnoreFilePrefix(String monitorIgnoreFilePrefix) {
if (monitorIgnoreFilePrefix != null && !monitorIgnoreFilePrefix.isEmpty()) {
this.monitorIgnoreFilePrefix = Arrays.asList(monitorIgnoreFilePrefix.split(","));
} else {
this.monitorIgnoreFilePrefix = new ArrayList<>();
}
}
public String getWebDavPrefixPath() {
return "/" + webDavPrefix;
}
public String getRootDir() {
return Paths.get(rootDir).toString();
}
public String getUserImgDir() {
return Paths.get(userImgDir).toString();
}
public String getDocumentImgDir() {
return Paths.get(documentImgDir).toString();
}
public String getDocumentDir() {
return Paths.get(documentDir).toString();
}
/**
* 判断用户名是否不允许使用
*
* @param username 用户名
* @return true: 不允许使用
*/
public boolean notAllowUsername(String username) {
if (StrUtil.isBlank(username)) {
return false;
}
if (getChunkFileDir().equals(username)) {
return true;
}
if (getLuceneIndexDir().equals(username)) {
return true;
}
return username.startsWith("log-");
}
}

View File

@ -0,0 +1,13 @@
package com.jmal.clouddisk.config;
import org.springframework.context.annotation.Configuration;
/**
* GlobalCorsConfig
*
* @author jmal
*/
@Configuration
public class GlobalCorsConfig {
}

View File

@ -0,0 +1,33 @@
package com.jmal.clouddisk.config;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
/**
* @Description: 去掉返回值中为null的字段
* @author jmal
*/
@Configuration
public class JacksonConfig {
@Bean
@Primary
@ConditionalOnMissingBean(ObjectMapper.class)
public ObjectMapper jacksonObjectMapper(Jackson2ObjectMapperBuilder builder) {
ObjectMapper objectMapper = builder.createXmlMapper(false).build();
//通过该方法对mapper对象进行设置,所有序列化的对象都将按改规则进行系列化,属性为NULL 不序列化
objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
return objectMapper;
}
//Include.Include.ALWAYS 默认
//Include.NON_DEFAULT 属性为默认值不序列化
//nclude.NON_EMPTY 属性为 空("") 或者为 NULL 都不序列化
//Include.NON_NULL 属性为NULL 不序列化
}

View File

@ -0,0 +1,68 @@
package com.jmal.clouddisk.config;
import com.mongodb.ConnectionString;
import com.mongodb.MongoClientSettings;
import com.mongodb.client.MongoClient;
import com.mongodb.client.MongoClients;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.jetbrains.annotations.NotNull;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.env.Environment;
import org.springframework.data.mongodb.config.AbstractMongoClientConfiguration;
import org.springframework.data.mongodb.core.MongoTemplate;
import java.util.Objects;
@Configuration
@Slf4j
@RequiredArgsConstructor
public class MongoClientConfig extends AbstractMongoClientConfiguration {
private final Environment env;
@NotNull
@Override
protected String getDatabaseName() {
String mongoURI = env.getProperty("spring.data.mongodb.uri");
if (mongoURI != null && !mongoURI.isBlank()) {
try {
return Objects.requireNonNull(new ConnectionString(mongoURI).getDatabase());
} catch (Exception e) {
log.error("Invalid MongoDB URI", e);
}
}
return "jmalcloud";
}
@Override
public boolean autoIndexCreation() {
return true;
}
@NotNull
@Bean
@Override
public MongoClient mongoClient() {
String mongoURI = env.getProperty("spring.data.mongodb.uri");
if (mongoURI != null && !mongoURI.isBlank()) {
try {
ConnectionString connectionString = new ConnectionString(mongoURI);
MongoClientSettings mongoClientSettings = MongoClientSettings.builder()
.applyConnectionString(connectionString)
.build();
return MongoClients.create(mongoClientSettings);
} catch (Exception e) {
log.error("Failed to create MongoClient", e);
}
}
return MongoClients.create();
}
@Bean
public MongoTemplate mongoTemplate() {
return new MongoTemplate(mongoClient(), getDatabaseName());
}
}

View File

@ -0,0 +1,26 @@
package com.jmal.clouddisk.config;
import org.springframework.boot.autoconfigure.web.ServerProperties;
import org.springframework.boot.autoconfigure.web.servlet.TomcatServletWebServerFactoryCustomizer;
import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory;
import org.springframework.stereotype.Component;
/**
* @author jmal
*/
@Component
public class MyTomcatServletWebServerFactoryCustomizer extends TomcatServletWebServerFactoryCustomizer {
public MyTomcatServletWebServerFactoryCustomizer(ServerProperties serverProperties) {
super(serverProperties);
}
@Override
public void customize(TomcatServletWebServerFactory factory) {
super.customize(factory);
factory.addConnectorCustomizers(connector -> {
connector.setProperty("relaxedPathChars", "|{}[]");
connector.setProperty("relaxedQueryChars", "|{}[]");
});
}
}

View File

@ -0,0 +1,48 @@
package com.jmal.clouddisk.config;
import dev.samstevens.totp.code.*;
import dev.samstevens.totp.qr.QrGenerator;
import dev.samstevens.totp.qr.ZxingPngQrGenerator;
import dev.samstevens.totp.secret.DefaultSecretGenerator;
import dev.samstevens.totp.secret.SecretGenerator;
import dev.samstevens.totp.time.SystemTimeProvider;
import dev.samstevens.totp.time.TimeProvider;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class TotpConfig {
@Bean
public SecretGenerator secretGenerator() {
// 用于生成新的MFA密钥
return new DefaultSecretGenerator(64);
}
@Bean
public TimeProvider timeProvider() {
// 提供当前时间,用于TOTP计算
return new SystemTimeProvider();
}
@Bean
public CodeGenerator codeGenerator() {
// TOTP代码生成器,使用标准SHA1算法,6位数字,30秒周期
return new DefaultCodeGenerator(HashingAlgorithm.SHA1, 6);
}
@Bean
public CodeVerifier codeVerifier(TimeProvider timeProvider, CodeGenerator codeGenerator) {
// 代码验证器
DefaultCodeVerifier verifier = new DefaultCodeVerifier(codeGenerator, timeProvider);
// 关键:设置容错窗口,允许客户端和服务器之间有轻微的时间偏差
// 1 表示允许前后一个时间周期(即前后30秒)的代码也有效
verifier.setAllowedTimePeriodDiscrepancy(1);
return verifier;
}
@Bean
public QrGenerator qrGenerator() {
return new ZxingPngQrGenerator();
}
}

View File

@ -0,0 +1,103 @@
package com.jmal.clouddisk.config;
import com.jmal.clouddisk.interceptor.*;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.jetbrains.annotations.NotNull;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.task.AsyncTaskExecutor;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.web.servlet.LocaleResolver;
import org.springframework.web.servlet.config.annotation.*;
import org.springframework.web.servlet.i18n.CookieLocaleResolver;
import java.io.File;
import java.util.Locale;
/**
* WebConfig
*
* @author jmal
*/
@Configuration
@Slf4j
@RequiredArgsConstructor
public class WebConfig implements WebMvcConfigurer {
private final FileProperties fileProperties;
private final AuthInterceptor authInterceptor;
private final FileInterceptor fileInterceptor;
private final ShareFileInterceptor shareFileInterceptor;
private final DirectFileInterceptor directFileInterceptor;
private final PreFileInterceptor preFileInterceptor;
/**
* 注册拦截器
*/
@Override
public void addInterceptors(InterceptorRegistry registry) {
registry.addInterceptor(localeChangeInterceptor());
registry.addInterceptor(authInterceptor).addPathPatterns("/**").
excludePathPatterns("/", "/login/**", "/public/**", "/articles/**", "/error/**", "/file/**" , "/pre-file/**" , "/share-file/**", "/direct-file/**" , "/files/**","/swagger-ui/**");
registry.addInterceptor(fileInterceptor).addPathPatterns("/file/**").addPathPatterns("/files/**");
registry.addInterceptor(shareFileInterceptor).addPathPatterns("/share-file/{fileId}/{token}/**");
registry.addInterceptor(directFileInterceptor).addPathPatterns("/direct-file/{mark}/{filename}");
registry.addInterceptor(preFileInterceptor).addPathPatterns("/pre-file/{fileId}/**");
}
@Override
public void addResourceHandlers(ResourceHandlerRegistry registry) {
registry.addResourceHandler("/file/**")
.addResourceLocations("file:" + fileProperties.getRootDir() + File.separator);
log.info("静态资源目录:{}", fileProperties.getRootDir() + File.separator);
}
public AsyncTaskExecutor taskExecutor() {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
executor.setCorePoolSize(10);
executor.setMaxPoolSize(50);
executor.setQueueCapacity(100);
executor.setThreadNamePrefix("Custom-Executor-");
executor.initialize();
return executor;
}
@Override
public void addCorsMappings(CorsRegistry registry) {
registry.addMapping("/**")
.allowedOrigins("*")
.allowedMethods("*")
.allowedHeaders("*");
}
@Override
public void configureAsyncSupport(@NotNull AsyncSupportConfigurer configurer) {
configurer.setTaskExecutor(taskExecutor());
// configurer.setDefaultTimeout(3000);
WebMvcConfigurer.super.configureAsyncSupport(configurer);
}
@Bean
public LocaleResolver localeResolver() {
CookieLocaleResolver resolver = new CookieLocaleResolver("locale");
resolver.setDefaultLocale(Locale.US);
return resolver;
}
@Bean
public HeaderLocaleChangeInterceptor localeChangeInterceptor() {
return new HeaderLocaleChangeInterceptor();
}
}

View File

@ -0,0 +1,35 @@
package com.jmal.clouddisk.config;
import jakarta.servlet.*;
import jakarta.servlet.http.HttpServletRequest;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.util.regex.Pattern;
/**
* @author jmal
* @Description 路径过滤器
* @Date 2020/10/20 4:12 下午
*/
@Component
@Lazy
public class WebFilter implements Filter {
public static final String API = "/api";
public static final Pattern COMPILE = Pattern.compile(API);
@Override
public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException {
HttpServletRequest httpRequest = (HttpServletRequest) request;
String uri = httpRequest.getRequestURI();
if (uri.startsWith(API)) {
uri = COMPILE.matcher(uri).replaceFirst("");
httpRequest.getRequestDispatcher(uri).forward(request, response);
return;
}
chain.doFilter(request, response);
}
}

View File

@ -0,0 +1,36 @@
package com.jmal.clouddisk.config;
import org.springframework.boot.env.YamlPropertySourceLoader;
import org.springframework.core.env.PropertySource;
import org.springframework.core.io.support.DefaultPropertySourceFactory;
import org.springframework.core.io.support.EncodedResource;
import org.springframework.lang.Nullable;
import java.io.IOException;
import java.util.List;
/**
* yml 资源加载工厂类
*
* @author jmal
*/
public class YamlPropertyLoaderFactory extends DefaultPropertySourceFactory {
@Override
public PropertySource<?> createPropertySource(@Nullable String name, EncodedResource resource) throws IOException {
if (null == resource) {
super.createPropertySource(name, resource);
}
if (resource == null) {
throw new RuntimeException("请配置 file.yml");
}
List<PropertySource<?>> sourceList = new YamlPropertySourceLoader().load(resource.getResource().getFilename(), resource.getResource());
if(sourceList.size() < 1){
throw new RuntimeException("请配置 file.yml");
}
return sourceList.get(0);
}
}

View File

@ -0,0 +1,436 @@
package com.jmal.clouddisk.controller;
import cn.hutool.core.text.CharSequenceUtil;
import cn.hutool.core.util.BooleanUtil;
import cn.hutool.core.util.ReUtil;
import cn.hutool.core.util.StrUtil;
import com.jmal.clouddisk.annotation.LogOperatingFun;
import com.jmal.clouddisk.model.*;
import com.jmal.clouddisk.service.IMarkdownService;
import com.jmal.clouddisk.service.IUserService;
import com.jmal.clouddisk.service.impl.CategoryService;
import com.jmal.clouddisk.service.impl.LogService;
import com.jmal.clouddisk.service.impl.SettingService;
import com.jmal.clouddisk.service.impl.TagService;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.servlet.http.Cookie;
import jakarta.servlet.http.HttpServletRequest;
import lombok.RequiredArgsConstructor;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.*;
import java.util.ArrayList;
import java.util.List;
/**
* @author jmal
* @Description 文章页面
* @Date 2020/11/16 5:41 下午
*/
@Controller
@Tag(name = "文章")
@RequiredArgsConstructor
public class ArticlesController {
private final SettingService settingService;
private final CategoryService categoryService;
private final TagService tagService;
private final IMarkdownService fileService;
private final LogService logService;
private final IUserService userService;
private static final String X_PJAX = "X-PJAX";
private static final String X_PJAX_TRUE = "true";
@RequestMapping("/public/404")
@LogOperatingFun(value = "404", logType = LogOperation.Type.ARTICLE)
public String notFind(HttpServletRequest request, Model map) {
boolean isPjax = pjaxMap(request, map, "404");
map.addAttribute("titleName", "页面没有找到");
return isPjax ? "404" : "index";
}
@GetMapping(value = "/articles/sitemap.xml", produces = {"application/xml;charset=UTF-8"})
@LogOperatingFun(value = "sitemap.xml", logType = LogOperation.Type.ARTICLE)
@ResponseBody
public Urlset sitemapXml() {
return fileService.getSitemapXml();
}
@GetMapping(value = "/articles/sitemap.txt", produces = {"text/plain;charset=UTF-8"})
@LogOperatingFun(value = "sitemap.txt", logType = LogOperation.Type.ARTICLE)
@ResponseBody
public String sitemapTxt() {
return fileService.getSitemapTxt();
}
@GetMapping("/")
public String redirectToArticles() {
return "redirect:/public/api";
}
@GetMapping("/articles")
@LogOperatingFun(value = "文章列表", logType = LogOperation.Type.ARTICLE)
public String index(HttpServletRequest request, Model map) {
map.addAttribute("mark", "articles");
boolean isPjax = isPjax(request);
WebsiteSettingDTO websiteSettingDTO;
if (!isPjax) {
websiteSettingDTO = getSetting(request, map);
} else {
websiteSettingDTO = settingService.getWebsiteSetting();
map.addAttribute("setting", settingService.getWebsiteSetting());
}
map.addAttribute("titleName", websiteSettingDTO.getSiteName());
int page = 1, pageSize = 10;
String pIndex = request.getParameter("page");
if (!CharSequenceUtil.isBlank(pIndex)) {
page = Integer.parseInt(pIndex);
}
map.addAttribute("articlesData", fileService.getArticles(page, pageSize));
return isPjax ? "articles" : "index";
}
@GetMapping("/articles/o/{slug}")
@LogOperatingFun(value = "独立页面", logType = LogOperation.Type.ARTICLE)
public String alonePage(HttpServletRequest request, @PathVariable String slug, Model map) {
String url = "/articles/o/" + slug;
long visits = logService.getVisitsByUrl(url);
map.addAttribute("visits", visits);
map.addAttribute("url", url);
return articlePage(request, slug, map);
}
@GetMapping("/articles/s/{slug}")
@LogOperatingFun(value = "文章", logType = LogOperation.Type.ARTICLE)
public String article(HttpServletRequest request, @PathVariable String slug, Model map) {
String url = "/articles/s/" + slug;
long visits = logService.getVisitsByUrl(url);
map.addAttribute("visits", visits);
map.addAttribute("url", url);
return articlePage(request, slug, map);
}
private String articlePage(HttpServletRequest request, String slug, Model map) {
boolean isPjax = pjaxMap(request, map, "article");
ArticleVO articleVO = fileService.getMarkDownContentBySlug(slug);
if (articleVO == null || !BooleanUtil.isTrue(articleVO.getRelease())) {
return notFind(request, map);
}
Cookie[] cookies = request.getCookies();
if (cookies != null) {
for (Cookie cookie : cookies) {
if ("consumerId".equals(cookie.getName())) {
articleVO.setEditable(true);
}
}
}
map.addAttribute("keywords", setKeywords(articleVO));
map.addAttribute("description", setDescription(articleVO));
modifyHtml(map, articleVO);
map.addAttribute("markdown", articleVO);
map.addAttribute("titleName", articleVO.getName());
return isPjax ? "article" : "index";
}
private static void modifyHtml(Model map, ArticleVO articleVO) {
// 使用 Jsoup 解析 HTML 内容, 使用parseBodyFragment 避免 Jsoup 自动添加 <html> 和 <body> 标签
Document document = Jsoup.parseBodyFragment(articleVO.getHtml());
// 关闭 "Pretty Print"(美化排版) 避免 Jsoup 自动格式化 HTML
document.outputSettings().prettyPrint(false);
// 遍历所有 h 标签(h1, h2, h3, h4, h5, h6)
for (int i = 1; i <= 6; i++) {
Elements headers = document.select("h" + i);
for (Element header : headers) {
if (!header.hasAttr("id")) {
String textContent = header.text();
// 将非字母数字字符替换为下划线
String idValue = "header-" + textContent.replaceAll("[^a-zA-Z0-9一-龥\\s]", "").trim().replaceAll("\\s+", "_");
header.attr("id", idValue);
}
}
}
// default img
String cover = articleVO.getCover();
if (StrUtil.isBlank(cover)) {
Object object = map.getAttribute("setting");
if (object instanceof WebsiteSettingDTO settingDTO) {
cover = settingDTO.getBackgroundSite();
}
}
// 遍历所有 img 标签
Elements images = document.select("img");
for (Element img : images) {
if (img.hasAttr("src")) {
String srcValue = img.attr("src");
img.attr("data-src", srcValue);
img.addClass("lazy");
img.attr("src", cover);
}
}
// 获取修改后的 HTML 内容
String modifiedHtmlContent = document.body().html();
articleVO.setHtml(modifiedHtmlContent);
}
private String setDescription(ArticleVO articleVO) {
if (articleVO.getHtml() == null) {
return articleVO.getName();
}
return articleVO.getHtml().substring(0, Math.min(articleVO.getHtml().length(), 500)).replaceAll("<[^>]*>","");
}
private String setKeywords(ArticleVO articleVO) {
StringBuilder stringBuilder = new StringBuilder();
List<TagDO> list = articleVO.getTags();
if (list != null && !list.isEmpty()) {
for (int i = 0; i < list.size(); i++) {
if (i != 0) {
stringBuilder.append(", ");
}
stringBuilder.append(list.get(i).getName());
}
}
return stringBuilder.toString();
}
@GetMapping("/articles/categories")
@LogOperatingFun(value = "文章分类", logType = LogOperation.Type.ARTICLE)
public String categories(HttpServletRequest request, Model map) {
boolean isPjax = pjaxMap(request, map, "categories");
map.addAttribute("titleName", "分类");
map.addAttribute("categories", categoryService.list(null, null));
return isPjax ? "categories" : "index";
}
@GetMapping("/articles/archives")
@LogOperatingFun(value = "文章归档", logType = LogOperation.Type.ARTICLE)
public String archives(HttpServletRequest request, Model map) {
boolean isPjax = pjaxMap(request, map, "archives");
map.addAttribute("titleName", "归档");
int page = 1, pageSize = 100;
String pIndex = request.getParameter("page");
if (!CharSequenceUtil.isBlank(pIndex)) {
page = Integer.parseInt(pIndex);
}
map.addAttribute("articlesData", fileService.getArchives(page, pageSize));
return isPjax ? "archives" : "index";
}
@GetMapping("/articles/categories/{categorySlugName}")
@LogOperatingFun(value = "文章分类", logType = LogOperation.Type.ARTICLE)
public String getCategoryByName(HttpServletRequest request, Model map, @PathVariable String categorySlugName) {
boolean isPjax = pjaxMap(request, map, "articles-query");
if (CharSequenceUtil.isBlank(categorySlugName)) {
return notFind(request, map);
}
String categoryId = null;
if (!CharSequenceUtil.isBlank(categorySlugName)) {
CategoryDO categoryDO = categoryService.getCategoryInfoBySlug(null, categorySlugName);
if (categoryDO == null) {
return notFind(request, map);
}
ArticlesQueryVO query = categoryDO.toArticlesQuery();
map.addAttribute("titleName", query.getName());
map.addAttribute("query", query);
categoryId = categoryDO.getId();
}
int page = 1, pageSize = 10;
String pIndex = request.getParameter("page");
if (!CharSequenceUtil.isBlank(pIndex)) {
page = Integer.parseInt(pIndex);
}
map.addAttribute("articlesData", fileService.getArticlesByCategoryId(page, pageSize, categoryId));
return isPjax ? "articles-query" : "index";
}
@GetMapping("/articles/tags")
@LogOperatingFun(value = "文章标签", logType = LogOperation.Type.ARTICLE)
public String tags(HttpServletRequest request, Model map) {
boolean isPjax = pjaxMap(request, map, "tags");
map.addAttribute("titleName", "标签");
map.addAttribute("tags", tagService.listTagsOfArticle());
return isPjax ? "tags" : "index";
}
@GetMapping("/articles/tags/{tagSlugName}")
@LogOperatingFun(value = "文章标签", logType = LogOperation.Type.ARTICLE)
public String getTagByName(HttpServletRequest request, Model map, @PathVariable String tagSlugName) {
boolean isPjax = pjaxMap(request, map, "articles-query");
if (CharSequenceUtil.isBlank(tagSlugName)) {
return notFind(request, map);
}
String tagId = null;
if (!CharSequenceUtil.isBlank(tagSlugName)) {
TagDO tag = tagService.getTagInfoBySlug(null, tagSlugName);
if (tag == null) {
return notFind(request, map);
}
ArticlesQueryVO query = tag.toArticlesQuery();
map.addAttribute("titleName", query.getName());
map.addAttribute("query", query);
tagId = tag.getId();
}
int page = 1, pageSize = 10;
String pIndex = request.getParameter("page");
if (!CharSequenceUtil.isBlank(pIndex)) {
page = Integer.parseInt(pIndex);
}
map.addAttribute("articlesData", fileService.getArticlesByTagId(page, pageSize, tagId));
return isPjax ? "articles-query" : "index";
}
@GetMapping("/articles/search")
@LogOperatingFun(value = "文章搜索", logType = LogOperation.Type.ARTICLE)
public String search(HttpServletRequest request, Model map, @RequestParam String keyword) {
boolean isPjax = pjaxMap(request, map, "articles-query");
int page = 1, pageSize = 10;
String pIndex = request.getParameter("page");
if (!CharSequenceUtil.isBlank(pIndex)) {
page = Integer.parseInt(pIndex);
}
ArticlesQueryVO query = new ArticlesQueryVO();
Page<List<MarkdownVO>> articles = fileService.getArticlesByKeyword(page, pageSize, keyword);
if (!articles.isEmpty()) {
MarkdownVO markdownVO = articles.getData().get(0);
query.setBackground(markdownVO.getCover());
}
query.setName("包含关键字 " + keyword + " 的文章");
map.addAttribute("titleName", query.getName());
map.addAttribute("query", query);
map.addAttribute("articlesData", articles);
return isPjax ? "articles-query" : "index";
}
@GetMapping("/articles/author/{username}")
@LogOperatingFun(value = "文章作者", logType = LogOperation.Type.ARTICLE)
public String author(HttpServletRequest request, Model map, @PathVariable String username) {
boolean isPjax = pjaxMap(request, map, "articles-query");
String userId = userService.getUserIdByShowName(username);
if (CharSequenceUtil.isBlank(userId)) {
return notFind(request, map);
}
int page = 1, pageSize = 10;
String pIndex = request.getParameter("page");
if (!CharSequenceUtil.isBlank(pIndex)) {
page = Integer.parseInt(pIndex);
}
ArticlesQueryVO query = new ArticlesQueryVO();
Page<List<MarkdownVO>> articles = fileService.getArticlesByAuthor(page, pageSize, userId);
if (!articles.isEmpty()) {
MarkdownVO markdownVO = articles.getData().get(0);
query.setBackground(markdownVO.getCover());
}
query.setName(username + " 发布的文章");
map.addAttribute("titleName", query.getName());
map.addAttribute("query", query);
map.addAttribute("articlesData", articles);
return isPjax ? "articles-query" : "index";
}
private WebsiteSettingDTO getSetting(HttpServletRequest request, Model map) {
WebsiteSettingDTO websiteSettingDTO = getWebsiteSetting();
setOperatingButtonList(websiteSettingDTO);
List<MarkdownVO> markdownVOList = fileService.getAlonePages();
map.addAttribute("alonePages", markdownVOList);
map.addAttribute("setting", websiteSettingDTO);
int alonePageShowIndex = 4 - websiteSettingDTO.getAlonePages().size();
if (markdownVOList.size() > alonePageShowIndex) {
markdownVOList = markdownVOList.subList(0, alonePageShowIndex);
}
map.addAttribute("showAlonePages", markdownVOList);
map.addAttribute("darkTheme", darkTheme(request));
return websiteSettingDTO;
}
/***
* 解析设置里的文本为操作按钮列表
* @param websiteSettingDTO userSettingDTO
*/
private void setOperatingButtonList(WebsiteSettingDTO websiteSettingDTO) {
if (websiteSettingDTO != null && !CharSequenceUtil.isBlank(websiteSettingDTO.getOperatingButtons())) {
String operatingButtons = websiteSettingDTO.getOperatingButtons();
List<WebsiteSettingDTO.OperatingButton> operatingButtonList = new ArrayList<>();
for (String button : operatingButtons.split("[\\n]")) {
WebsiteSettingDTO.OperatingButton operatingButton = new WebsiteSettingDTO.OperatingButton();
int splitIndex = button.indexOf(":");
String label = button.substring(0, splitIndex);
String title = ReUtil.getGroup0("[^><]+(?=<\\/i>)", label);
if (CharSequenceUtil.isBlank(title)) {
title = "";
}
operatingButton.setTitle(title);
operatingButton.setStyle(ReUtil.getGroup0("[^=\"<]+(?=\">)", label));
operatingButton.setUrl(button.substring(splitIndex + 1));
operatingButtonList.add(operatingButton);
}
websiteSettingDTO.setOperatingButtonList(operatingButtonList);
}
}
private boolean darkTheme(HttpServletRequest request) {
Cookie[] cookies = request.getCookies();
if (cookies != null) {
for (Cookie cookie : cookies) {
if ("jmal-theme".equals(cookie.getName())) {
if ("dark".equals(cookie.getValue())) {
return true;
}
}
}
}
return false;
}
/***
* 判断请求是否为pjax
* @param request HttpServletRequest
* @return boolean
*/
private boolean isPjax(HttpServletRequest request) {
return X_PJAX_TRUE.equals(request.getHeader(X_PJAX));
}
private boolean pjaxMap(HttpServletRequest request, Model map, String viewName) {
boolean isPjax = isPjax(request);
if (!isPjax) {
getSetting(request, map);
} else {
map.addAttribute("setting", getWebsiteSetting());
}
map.addAttribute("mark", viewName);
return isPjax;
}
private WebsiteSettingDTO getWebsiteSetting() {
WebsiteSettingDTO websiteSettingDTO = settingService.getWebsiteSetting();
if (StrUtil.isNotBlank(websiteSettingDTO.getSiteUrl())) {
String siteUrl = websiteSettingDTO.getSiteUrl();
if (siteUrl.endsWith("/")) {
siteUrl = siteUrl.substring(0, siteUrl.length() - 1);
websiteSettingDTO.setSiteUrl(siteUrl);
}
}
return websiteSettingDTO;
}
}

View File

@ -0,0 +1,54 @@
package com.jmal.clouddisk.controller;
import com.jmal.clouddisk.annotation.LogOperatingFun;
import com.jmal.clouddisk.model.LogOperation;
import com.jmal.clouddisk.service.IFileService;
import com.jmal.clouddisk.service.IShareService;
import com.jmal.clouddisk.util.ResultUtil;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestParam;
/**
* @author jmal
* @Description 文件资源管理
* @Date 2020/10/19 3:55 下午
*/
@Tag(name = "文件资源管理")
@Controller
public class FileViewController {
@Autowired
private IFileService fileService;
@Autowired
IShareService shareService;
@Operation(summary = "预览文档里的图片")
@GetMapping("/public/view")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public String imageRelativePath(@RequestParam String relativePath,@RequestParam String userId) {
ResultUtil.checkParamIsNull(relativePath,userId);
return fileService.publicViewFile(relativePath, userId);
}
@Operation(summary = "分享:预览文件")
@GetMapping("/public/s/preview/{filename}")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public String publicPreview(@RequestParam String fileId, @RequestParam String shareId, @RequestParam String shareToken) {
shareService.validShare(shareToken, shareId);
return fileService.viewFile(fileId, fileId, shareToken, "preview");
}
@Operation(summary = "分享:下载单个文件")
@GetMapping("/public/s/download/{filename}")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public String publicDownload(@RequestParam String fileId, @RequestParam String shareId, @RequestParam String shareToken) {
shareService.validShare(shareToken, shareId);
return fileService.viewFile(fileId, fileId, shareToken, "download");
}
}

View File

@ -0,0 +1,3 @@
package com.jmal.clouddisk.controller.record;
public record LoginResponse(String token, boolean mfaRequired, String mfaToken) {}

View File

@ -0,0 +1,3 @@
package com.jmal.clouddisk.controller.record;
public record MfaSetupResponse(Boolean mfaEnable, String secret, String qrCodeImageUri) {}

View File

@ -0,0 +1,3 @@
package com.jmal.clouddisk.controller.record;
public record MfaVerifyRequest(String secret, String code) {}

View File

@ -0,0 +1,79 @@
package com.jmal.clouddisk.controller.rest;
import com.jmal.clouddisk.annotation.LogOperatingFun;
import com.jmal.clouddisk.annotation.Permission;
import com.jmal.clouddisk.interceptor.AuthInterceptor;
import com.jmal.clouddisk.model.LogOperation;
import com.jmal.clouddisk.model.rbac.ConsumerDTO;
import com.jmal.clouddisk.service.IAuthService;
import com.jmal.clouddisk.service.IUserService;
import com.jmal.clouddisk.util.ResponseResult;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RestController;
/**
* 登录、登出、验证
*
* @author jmal
*/
@RestController
@Tag(name = "登录认证")
public class AuthController {
@Autowired
IAuthService authService;
@Autowired
IUserService userService;
@Operation(summary = "登录")
@LogOperatingFun(logType = LogOperation.Type.LOGIN)
@PostMapping("/login")
public ResponseResult<Object> login(HttpServletRequest request, HttpServletResponse response, @RequestBody ConsumerDTO consumerDTO) {
return authService.login(request, response, consumerDTO);
}
@Operation(summary = "验证TOTP")
@LogOperatingFun(logType = LogOperation.Type.LOGIN)
@PostMapping("/public/verify-totp")
public ResponseResult<Object> verifyTotp(HttpServletRequest request, HttpServletResponse response, @RequestBody ConsumerDTO consumerDTO) {
return authService.verifyTotp(request, response, consumerDTO);
}
@Operation(summary = "校验旧密码")
@PostMapping("/valid-old-pass")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
@Permission("sys:user:list")
public ResponseResult<Object> validOldPass(@RequestBody ConsumerDTO consumerDTO) {
return authService.validOldPass(consumerDTO.getId(), consumerDTO.getPassword());
}
@Operation(summary = "登出")
@GetMapping("/logout")
@LogOperatingFun
public ResponseResult<Object> logout(HttpServletRequest request, HttpServletResponse response) {
String token = request.getHeader(AuthInterceptor.JMAL_TOKEN);
return authService.logout(token, response);
}
@Operation(summary = "是否有用户")
@GetMapping("/public/has_user")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<Boolean> hasUser() {
return userService.hasUser();
}
@Operation(summary = "初始化创建管理员")
@PostMapping("/public/initialization")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<Object> initialization(ConsumerDTO consumer) {
return userService.initialization(consumer);
}
}

View File

@ -0,0 +1,96 @@
package com.jmal.clouddisk.controller.rest;
import com.jmal.clouddisk.annotation.LogOperatingFun;
import com.jmal.clouddisk.annotation.Permission;
import com.jmal.clouddisk.model.CategoryDO;
import com.jmal.clouddisk.model.CategoryDTO;
import com.jmal.clouddisk.model.LogOperation;
import com.jmal.clouddisk.service.impl.CategoryService;
import com.jmal.clouddisk.util.ResponseResult;
import com.jmal.clouddisk.util.ResultUtil;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
import java.util.Arrays;
import java.util.List;
/**
* @author jmal
* @Description 分类管理
* @Date 2020/10/26 5:47 下午
*/
@RestController
@Tag(name = "分类管理")
public class CategoryController {
@Autowired
private CategoryService categoryService;
@Operation(summary = "分类列表")
@GetMapping("/category/list")
@Permission("website:set:list")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<List<CategoryDTO>> list(String userId, String parentCategoryId) {
return ResultUtil.success(categoryService.list(userId, parentCategoryId));
}
@Operation(summary = "分类树")
@GetMapping("/category/tree")
@Permission("website:set:list")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<List<CategoryDTO>> tree(String userId) {
return ResultUtil.success(categoryService.tree(userId, false));
}
@Operation(summary = "分类信息")
@GetMapping("/category/info")
@Permission("website:set:list")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<CategoryDTO> categoryInfo(@RequestParam String categoryId) {
CategoryDTO categoryDTO = new CategoryDTO();
CategoryDO categoryDO = categoryService.getCategoryInfo(categoryId);
if(categoryDO != null){
BeanUtils.copyProperties(categoryDO, categoryDTO);
}
return ResultUtil.success(categoryDTO);
}
@Operation(summary = "添加分类")
@PostMapping("/category/add")
@LogOperatingFun
@Permission("website:set:add")
public ResponseResult<Object> add(@RequestParam String name, @ModelAttribute @Validated CategoryDTO categoryDTO) {
categoryDTO.setName(name);
return categoryService.add(categoryDTO);
}
@Operation(summary = "更新分类")
@PutMapping("/category/update")
@Permission("website:set:update")
public ResponseResult<Object> update(@RequestParam String name, @ModelAttribute CategoryDTO categoryDTO) {
categoryDTO.setName(name);
return categoryService.update(categoryDTO);
}
@Operation(summary = "设置默认分类")
@PutMapping("/category/setDefault")
@LogOperatingFun
@Permission("website:set:update")
public ResponseResult<Object> setDefault(@RequestParam String categoryId) {
return categoryService.setDefault(categoryId);
}
@Operation(summary = "删除分类")
@DeleteMapping("/category/delete")
@LogOperatingFun
@Permission("website:set:delete")
public ResponseResult<Object> delete(@RequestParam String[] categoryIds) {
List<String> categoryList = Arrays.asList(categoryIds);
categoryService.delete(categoryList);
return ResultUtil.success();
}
}

View File

@ -0,0 +1,207 @@
package com.jmal.clouddisk.controller.rest;
import com.jmal.clouddisk.annotation.LogOperatingFun;
import com.jmal.clouddisk.annotation.Permission;
import com.jmal.clouddisk.lucene.RebuildIndexTaskService;
import com.jmal.clouddisk.lucene.TaskProgress;
import com.jmal.clouddisk.lucene.TaskProgressService;
import com.jmal.clouddisk.model.LdapConfigDTO;
import com.jmal.clouddisk.model.LogOperation;
import com.jmal.clouddisk.model.WebsiteSettingDTO;
import com.jmal.clouddisk.ocr.OcrConfig;
import com.jmal.clouddisk.ocr.OcrService;
import com.jmal.clouddisk.service.IAuthService;
import com.jmal.clouddisk.service.IUserService;
import com.jmal.clouddisk.service.impl.SettingService;
import com.jmal.clouddisk.util.ResponseResult;
import com.jmal.clouddisk.util.ResultUtil;
import com.jmal.clouddisk.media.TranscodeConfig;
import com.jmal.clouddisk.media.VideoProcessService;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.RequiredArgsConstructor;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import java.util.List;
import java.util.Map;
@RestController
@Tag(name = "网盘设置")
@RequiredArgsConstructor
public class CloudSettingController {
private final SettingService settingService;
private final TaskProgressService taskProgressService;
private final IUserService userService;
private final IAuthService authService;
private final RebuildIndexTaskService rebuildIndexTaskService;
private final VideoProcessService videoProcessService;
private final OcrService ocrService;
@Operation(summary = "重建索引-用户")
@GetMapping("/user/setting/sync")
@Permission(value = "cloud:file:upload")
@LogOperatingFun
public ResponseResult<Object> userSync(@RequestParam String username, String path) {
return rebuildIndexTaskService.sync(username, path, true);
}
@Operation(summary = "重建索引-全盘")
@GetMapping("/cloud/setting/sync")
@Permission(value = "cloud:set:sync")
@LogOperatingFun
public ResponseResult<Object> sync(@RequestParam String username) {
return rebuildIndexTaskService.sync(username, null, true);
}
@Operation(summary = "重新计算文件夹大小")
@GetMapping("/cloud/setting/recalculateFolderSize")
@Permission(value = "cloud:set:sync")
@LogOperatingFun
public ResponseResult<Object> recalculateFolderSize() {
return settingService.recalculateFolderSize();
}
@Operation(summary = "获取视频转码配置")
@GetMapping("/cloud/setting/transcode/config")
@Permission(value = "cloud:set:sync")
public ResponseResult<TranscodeConfig> getTranscodeConfig() {
return ResultUtil.success(videoProcessService.getTranscodeConfig());
}
@Operation(summary = "修改视频转码配置")
@PutMapping("/cloud/setting/transcode/config")
@Permission(value = "cloud:set:sync")
public ResponseResult<Object> setTranscodeConfig(@RequestBody @Validated TranscodeConfig transcodeConfig) {
return ResultUtil.success(videoProcessService.setTranscodeConfig(transcodeConfig));
}
@Operation(summary = "获取ocr配置")
@GetMapping("/cloud/setting/ocr/config")
@Permission(value = "cloud:set:sync")
public ResponseResult<OcrConfig> getOcrConfig() {
return ResultUtil.success(ocrService.getOcrConfig());
}
@Operation(summary = "修改ocr配置")
@PutMapping("/cloud/setting/ocr/config")
@Permission(value = "cloud:set:sync")
public ResponseResult<Object> setOcrConfig(@RequestBody @Validated OcrConfig ocrConfig) {
return ResultUtil.success(ocrService.setOcrConfig(ocrConfig));
}
@Operation(summary = "取消转码任务")
@PutMapping("/cloud/setting/transcode/cancel-task")
@Permission(value = "cloud:set:sync")
public ResponseResult<Object> cancelTranscodeTask() {
videoProcessService.cancelTranscodeTask();
return ResultUtil.success();
}
@Operation(summary = "是否正在同步")
@GetMapping("/user/setting/isSync")
public ResponseResult<Map<String, Double>> isSync() {
return ResultUtil.success(rebuildIndexTaskService.isSync());
}
@Operation(summary = "上传网盘logo")
@PostMapping("/user/setting/upload_logo")
@Permission(value = "cloud:set:sync")
@LogOperatingFun
public ResponseResult<Object> uploadLogo(MultipartFile file) {
return settingService.uploadLogo(file);
}
@Operation(summary = "修改网盘名称")
@PutMapping("/user/setting/update_netdisk_name")
@Permission(value = "cloud:set:sync")
@LogOperatingFun
public ResponseResult<Object> updateNetdiskName(@RequestParam String netdiskName) {
return settingService.updateNetdiskName(netdiskName);
}
@Operation(summary = "重置角色菜单")
@PutMapping("/user/setting/resetMenuAndRole")
@Permission(onlyCreator = true)
@LogOperatingFun
public ResponseResult<Object> resetMenuAndRole() {
settingService.resetMenuAndRole();
return ResultUtil.success();
}
@Operation(summary = "获取是否禁用webp状态")
@GetMapping("/user/setting/get/webp")
@Permission("sys:user:list")
public ResponseResult<Boolean> getDisabledWebp(@RequestParam String userId) {
return ResultUtil.success(userService.getDisabledWebp(userId));
}
@Operation(summary = "是否禁用webp(默认开启)")
@PutMapping("/user/setting/disabled/webp")
@Permission("sys:user:update")
@LogOperatingFun
public ResponseResult<Object> disabledWebp(@RequestParam String userId, @RequestParam Boolean disabled) {
userService.disabledWebp(userId, disabled);
return ResultUtil.success();
}
@Operation(summary = "加载ldap配置")
@GetMapping("/ldap/config")
public ResponseResult<Object> loadLdapConfig() {
return ResultUtil.success(authService.loadLdapConfig());
}
@Operation(summary = "ldap配置")
@LogOperatingFun(logType = LogOperation.Type.OPERATION)
@PutMapping("/ldap/config")
@Permission(value = "cloud:set:sync")
public ResponseResult<Object> updateLdapConfig(@RequestBody LdapConfigDTO ldapConfigDTO) {
return authService.updateLdapConfig(ldapConfigDTO);
}
@Operation(summary = "测试ldap配置")
@PutMapping("/ldap/test-config")
@Permission(value = "cloud:set:sync")
public ResponseResult<Object> testLdapConfig(@RequestBody LdapConfigDTO ldapConfigDTO) {
authService.testLdapConfig(ldapConfigDTO);
return ResultUtil.success();
}
@Operation(summary = "获取预览配置")
@GetMapping("/cloud/setting/preview/config")
public ResponseResult<Object> getPreviewConfig() {
return ResultUtil.success(settingService.getPreviewConfig());
}
@Operation(summary = "更新预览配置")
@LogOperatingFun(logType = LogOperation.Type.OPERATION)
@PutMapping("/cloud/setting/preview/config")
@Permission(value = "cloud:set:sync")
public ResponseResult<Object> updatePreviewConfig(@RequestBody WebsiteSettingDTO websiteSettingDTO) {
settingService.updatePreviewConfig(websiteSettingDTO);
return ResultUtil.success();
}
@Operation(summary = "加载任务进度")
@GetMapping("/cloud/task/progress")
@Permission(value = "cloud:file:upload")
public ResponseResult<List<TaskProgress>> getTaskProgress() {
return ResultUtil.success(taskProgressService.getTaskProgressList());
}
@Operation(summary = "加载转码状态")
@GetMapping("/cloud/transcode/status")
@Permission(value = "cloud:file:upload")
public ResponseResult<Map<String, Integer>> getTranscodeStatus() {
return ResultUtil.success(videoProcessService.getTranscodeStatus());
}
}

View File

@ -0,0 +1,45 @@
package com.jmal.clouddisk.controller.rest;
import com.jmal.clouddisk.annotation.LogOperatingFun;
import com.jmal.clouddisk.annotation.Permission;
import com.jmal.clouddisk.service.impl.DirectLinkService;
import com.jmal.clouddisk.util.ResponseResult;
import com.jmal.clouddisk.util.ResultUtil;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.*;
@RestController
@Tag(name = "直链管理")
@RequestMapping("direct-link")
@RequiredArgsConstructor
public class DirectLinkController {
private final DirectLinkService directLinkService;
@Operation(summary = "创建直链")
@PostMapping("/create")
@Permission(value = "cloud:file:upload")
@LogOperatingFun
public ResponseResult<String> createDirectLink(@RequestParam String fileId) {
return ResultUtil.success(directLinkService.createDirectLink(fileId));
}
@Operation(summary = "重置直链")
@PutMapping("/reset")
@Permission(value = "cloud:file:upload")
@LogOperatingFun
public ResponseResult<String> resetDirectLink(@RequestParam String fileId) {
return ResultUtil.success(directLinkService.resetDirectLink(fileId));
}
@Operation(summary = "重置所有直链")
@DeleteMapping("/reset-all")
@Permission(value = "cloud:file:upload")
@LogOperatingFun
public ResponseResult<String> resetAllDirectLink(@RequestParam String fileId) {
return ResultUtil.success(directLinkService.resetAllDirectLink(fileId));
}
}

View File

@ -0,0 +1,490 @@
package com.jmal.clouddisk.controller.rest;
import cn.hutool.core.text.CharSequenceUtil;
import cn.hutool.core.util.BooleanUtil;
import com.jmal.clouddisk.annotation.LogOperatingFun;
import com.jmal.clouddisk.annotation.Permission;
import com.jmal.clouddisk.exception.CommonException;
import com.jmal.clouddisk.exception.ExceptionType;
import com.jmal.clouddisk.model.*;
import com.jmal.clouddisk.oss.web.WebOssCommonService;
import com.jmal.clouddisk.oss.web.WebOssService;
import com.jmal.clouddisk.service.Constants;
import com.jmal.clouddisk.service.IFileService;
import com.jmal.clouddisk.service.IUserService;
import com.jmal.clouddisk.service.impl.UserLoginHolder;
import com.jmal.clouddisk.util.CaffeineUtil;
import com.jmal.clouddisk.util.FileNameUtils;
import com.jmal.clouddisk.util.ResponseResult;
import com.jmal.clouddisk.util.ResultUtil;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
/**
* @author jmal
* @Description 云文件管理控制器
* @Author jmal
* @Date 2020-01-27 12:59
*/
@Tag(name = "文件管理")
@RestController
@RequiredArgsConstructor
public class FileController {
private final IFileService fileService;
private final WebOssService webOssService;
private final UserLoginHolder userLoginHolder;
private final IUserService userService;
@Operation(summary = "根据id获取文件信息")
@GetMapping("/file_info")
@Permission("cloud:file:list")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<FileDocument> getFileById(@RequestParam String id) {
return ResultUtil.success(fileService.getById(id));
}
@Operation(summary = "文件列表")
@GetMapping("/list")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<Object> list(UploadApiParamDTO upload) {
return fileService.listFiles(upload);
}
@Operation(summary = "查找下级目录")
@GetMapping("/query-file-tree")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<Object> queryFileTree(UploadApiParamDTO upload, String fileId) {
return fileService.queryFileTree(upload, fileId);
}
@Operation(summary = "搜索文件")
@GetMapping("/search-file")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<List<FileIntroVO>> searchFile(UploadApiParamDTO upload, String keyword) {
return fileService.searchFile(upload, keyword);
}
@Operation(summary = "搜索文件并打开文件夹")
@GetMapping("/search-file-open")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<Object> searchFileAndOpenDir(UploadApiParamDTO upload, String id, String folder) {
return fileService.searchFileAndOpenDir(upload, id, folder);
}
@Operation(summary = "图片上传(Typora自定义命令上传图片接口)")
@PostMapping("/img-upload")
@LogOperatingFun
@Permission("cloud:file:upload")
public String imgUpload(HttpServletRequest request, MultipartFile file) {
if (file == null) {
throw new CommonException(ExceptionType.MISSING_PARAMETERS.getCode(), "缺少文件参数, file");
}
String filepath = request.getHeader(Constants.FILE_PATH);
if (CharSequenceUtil.isBlank(filepath)) {
throw new CommonException(ExceptionType.MISSING_PARAMETERS.getCode(), "headers里缺少参数, filepath: 远程目标文件夹, 例如: '/Image/Typora/Public/Images'");
}
String baseUrl = request.getHeader("baseurl");
if (CharSequenceUtil.isBlank(baseUrl)) {
throw new CommonException(ExceptionType.MISSING_PARAMETERS.getCode(), "headers里缺少参数, baseUrl: 远程服务器地址, 例如: 'https://www.jmal.top/api'");
}
return fileService.imgUpload(baseUrl, filepath, file);
}
@Operation(summary = "文件上传")
@PostMapping("upload")
@Permission("cloud:file:upload")
public ResponseResult<Object> uploadPost(UploadApiParamDTO upload) throws IOException {
return fileService.upload(upload);
}
@Operation(summary = "文件夹上传")
@PostMapping("upload-folder")
@Permission("cloud:file:upload")
public ResponseResult<Object> uploadFolder(UploadApiParamDTO upload) {
return fileService.uploadFolder(upload);
}
@Operation(summary = "新建文件夹")
@PostMapping("new_folder")
@Permission("cloud:file:upload")
public ResponseResult<Object> newFolder(UploadApiParamDTO upload) {
return fileService.newFolder(upload);
}
@Operation(summary = "检查文件/分片是否存在")
@GetMapping("upload")
@Permission("cloud:file:upload")
public ResponseResult<Object> checkUpload(UploadApiParamDTO upload) throws IOException {
return fileService.checkChunkUploaded(upload);
}
@Operation(summary = "检查文件是否存在")
@PostMapping("checkExist")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
@Permission("cloud:file:upload")
public ResponseResult<Object> checkFileExist(@RequestBody UploadApiParamDTO upload) throws IOException {
return fileService.checkChunkUploaded(upload);
}
@Operation(summary = "合并文件")
@PostMapping("merge")
@Permission("cloud:file:upload")
public ResponseResult<Object> merge(UploadApiParamDTO upload) throws IOException {
return fileService.merge(upload);
}
@Operation(summary = "读取simText文件")
@GetMapping("/preview/text")
@Permission("cloud:file:list")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<Object> previewText(@RequestParam String id, @RequestParam String path, @RequestParam String fileName, Boolean content) {
String ossPath = CaffeineUtil.getOssPath(Paths.get(id));
if (ossPath != null) {
Path prePth = Paths.get(WebOssCommonService.getUsernameByOssPath(ossPath), path, fileName);
return ResultUtil.success(webOssService.readToText(ossPath, prePth, content));
}
return ResultUtil.success(fileService.getById(id, content));
}
@Operation(summary = "流式读取simText文件")
@GetMapping("/preview/text/stream")
@Permission("cloud:file:list")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseEntity<StreamingResponseBody> previewTextStream(@RequestParam String id, @RequestParam String path, @RequestParam String fileName) {
String ossPath = CaffeineUtil.getOssPath(Paths.get(id));
StreamingResponseBody responseBody;
if (ossPath != null) {
Path prePth = Paths.get(WebOssCommonService.getUsernameByOssPath(ossPath), path, fileName);
responseBody = webOssService.readToTextStream(ossPath, prePth);
} else {
responseBody = fileService.getStreamById(id);
}
return new ResponseEntity<>(responseBody, HttpStatus.OK);
}
@Operation(summary = "根据path读取simText文件")
@GetMapping("/preview/path/text")
@Permission("cloud:file:list")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<Object> previewTextByPath(@RequestParam String path, @RequestParam String username) {
Path prePth = Paths.get(username, path);
String ossPath = CaffeineUtil.getOssPath(prePth);
if (ossPath != null) {
return ResultUtil.success(webOssService.readToText(ossPath, prePth, false));
}
return ResultUtil.success(fileService.previewTextByPath(FileNameUtils.safeDecode(path), username));
}
@Operation(summary = "根据path流式读取simText文件")
@GetMapping("/preview/path/text/stream")
@Permission("cloud:file:list")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseEntity<StreamingResponseBody> previewTextByPathStream(@RequestParam String path, @RequestParam String username) {
Path prePth = Paths.get(username, path);
String ossPath = CaffeineUtil.getOssPath(prePth);
StreamingResponseBody responseBody;
if (ossPath != null) {
responseBody = webOssService.readToTextStream(ossPath, prePth);
} else {
responseBody = fileService.previewTextByPathStream(FileNameUtils.safeDecode(path), username);
}
return new ResponseEntity<>(responseBody, HttpStatus.OK);
}
@Operation(summary = "是否允许下载")
@GetMapping("/isAllowDownload")
@Permission("cloud:file:download")
public ResponseResult<Object> isAllowDownload(@RequestParam List<String> fileIds) {
return fileService.isAllowDownload(fileIds);
}
@Operation(summary = "是否允许批量下载")
@GetMapping("/isAllowPackageDownload")
@Permission("cloud:file:packageDownload")
public ResponseResult<Object> isAllowPackageDownload(@RequestParam List<String> fileIds) {
return fileService.isAllowDownload(fileIds);
}
@Operation(summary = "打包下载")
@GetMapping("/packageDownload")
@Permission("cloud:file:packageDownload")
@LogOperatingFun
public void packageDownload(HttpServletRequest request, HttpServletResponse response, @RequestParam String[] fileIds) {
if (fileIds != null && fileIds.length > 0) {
List<String> fileIdList = Arrays.asList(fileIds);
fileService.packageDownload(request, response, fileIdList);
} else {
throw new CommonException(ExceptionType.MISSING_PARAMETERS.getCode(), ExceptionType.MISSING_PARAMETERS.getMsg());
}
}
@Operation(summary = "获取dwg文件对应的mxweb文件")
@GetMapping("/view/mxweb/{fileId}")
@Permission("cloud:file:list")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseEntity<Object> getMxweb(@PathVariable String fileId) {
Optional<FileDocument> file = fileService.getMxweb(fileId);
return file.map(fileService::getObjectResponseEntity).orElseGet(() -> ResponseEntity.status(HttpStatus.NOT_FOUND).body("找不到该文件"));
}
@Operation(summary = "显示缩略图")
@GetMapping("/view/thumbnail")
@Permission("cloud:file:list")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseEntity<Object> thumbnail(@RequestParam String id, Boolean showCover) {
String ossPath = CaffeineUtil.getOssPath(Paths.get(id));
if (ossPath != null) {
return webOssService.thumbnail(ossPath, id);
}
Optional<FileDocument> file = fileService.thumbnail(id, showCover);
return file.map(fileService::getObjectResponseEntity).orElseGet(() -> ResponseEntity.status(HttpStatus.NOT_FOUND).body("找不到该文件"));
}
@Operation(summary = "显示缩略图")
@GetMapping("/view/thumbnail/user/{username}")
@Permission("cloud:file:list")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseEntity<Object> thumbnailByUsername(@PathVariable String username, Boolean showCover) {
String avatar = userService.getAvatarByUsername(username);
return thumbnail(avatar, showCover);
}
@Operation(summary = "显示缩略图")
@GetMapping("/view/thumbnail/{filename}")
@Permission("cloud:file:list")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseEntity<Object> thumbnailName(@RequestParam String id, Boolean showCover) {
return thumbnail(id, showCover);
}
@Operation(summary = "显示缩略图(媒体封面)")
@GetMapping("/view/cover")
@Permission("cloud:file:list")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseEntity<Object> coverOfMedia(String id, String name) {
ResultUtil.checkParamIsNull(id, name);
Optional<FileDocument> file = fileService.coverOfMedia(id, name);
return file.map(fileService::getObjectResponseEntity).orElseGet(() -> ResponseEntity.status(HttpStatus.NOT_FOUND).body("找不到该文件"));
}
@Operation(summary = "收藏文件或文件夹")
@PostMapping("/favorite")
@LogOperatingFun
@Permission("cloud:file:update")
public ResponseResult<Object> favorite(@RequestParam String[] fileIds) {
if (fileIds != null && fileIds.length > 0) {
List<String> list = Arrays.asList(fileIds);
return fileService.favorite(list);
} else {
throw new CommonException(ExceptionType.MISSING_PARAMETERS.getCode(), ExceptionType.MISSING_PARAMETERS.getMsg());
}
}
@Operation(summary = "设置文件标签")
@PostMapping("/setTag")
@LogOperatingFun
@Permission("cloud:file:update")
public ResponseResult<Object> setTag(@RequestBody @Validated EditTagDTO editTagDTO) {
return fileService.setTag(editTagDTO);
}
@Operation(summary = "设置文件标签颜色")
@PutMapping("/updateTag")
@LogOperatingFun
@Permission("cloud:file:update")
public ResponseResult<Object> updateTag(@RequestParam String tagId, @RequestParam String name, @RequestParam String color) {
return fileService.setTag(tagId, name, color);
}
@Operation(summary = "删除文件标签")
@DeleteMapping("/deleteTag")
@LogOperatingFun
@Permission("cloud:file:update")
public ResponseResult<Object> deleteTag(@RequestParam String tagId) {
return fileService.deleteTag(tagId);
}
@Operation(summary = "设为公共文件")
@PutMapping("/setPublic")
@LogOperatingFun
@Permission("cloud:file:update")
public ResponseResult<Object> setPublic(@RequestParam String fileId) {
fileService.setPublic(fileId);
return ResultUtil.success();
}
@Operation(summary = "取消收藏")
@PostMapping("/unFavorite")
@LogOperatingFun
@Permission("cloud:file:update")
public ResponseResult<Object> unFavorite(@RequestParam String[] fileIds) {
if (fileIds != null && fileIds.length > 0) {
List<String> list = Arrays.asList(fileIds);
return fileService.unFavorite(list);
} else {
throw new CommonException(ExceptionType.MISSING_PARAMETERS.getCode(), ExceptionType.MISSING_PARAMETERS.getMsg());
}
}
@Operation(summary = "删除文件")
@DeleteMapping("/delete")
@Permission("cloud:file:delete")
public ResponseResult<Object> delete(@RequestParam String username, @RequestParam String[] fileIds, @RequestParam String currentDirectory, Boolean sweep) {
if (fileIds != null && fileIds.length > 0) {
List<String> list = Arrays.asList(fileIds);
return fileService.delete(username, currentDirectory, list, userLoginHolder.getUsername(), BooleanUtil.isTrue(sweep));
} else {
throw new CommonException(ExceptionType.MISSING_PARAMETERS.getCode(), ExceptionType.MISSING_PARAMETERS.getMsg());
}
}
@Operation(summary = "放回原处")
@PostMapping("/restore")
@Permission("cloud:file:upload")
public ResponseResult<Object> restore(@RequestParam String[] fileIds) {
if (fileIds != null && fileIds.length > 0) {
List<String> list = Arrays.asList(fileIds);
return fileService.restore(list, userLoginHolder.getUsername());
} else {
throw new CommonException(ExceptionType.MISSING_PARAMETERS.getCode(), ExceptionType.MISSING_PARAMETERS.getMsg());
}
}
@Operation(summary = "彻底删除")
@DeleteMapping("/sweep")
@Permission("cloud:file:delete")
public ResponseResult<Object> sweep(@RequestParam String[] fileIds) {
if (fileIds != null && fileIds.length > 0) {
List<String> list = Arrays.asList(fileIds);
return fileService.sweep(list, userLoginHolder.getUsername());
} else {
throw new CommonException(ExceptionType.MISSING_PARAMETERS.getCode(), ExceptionType.MISSING_PARAMETERS.getMsg());
}
}
@Operation(summary = "清空回收站")
@DeleteMapping("/clear-trash")
@LogOperatingFun
@Permission("cloud:file:delete")
public ResponseResult<Object> clearTrash() {
return fileService.clearTrash(userLoginHolder.getUsername());
}
@Operation(summary = "重命名")
@GetMapping("/rename")
@LogOperatingFun
@Permission("cloud:file:update")
public ResponseResult<Object> rename(@RequestParam String newFileName, @RequestParam String username, @RequestParam String id, String folder) {
return fileService.rename(FileNameUtils.safeDecode(newFileName), username, id, folder);
}
@Operation(summary = "移动或复制前检查目标目录是否存在要移动或复制的文件")
@GetMapping("/check-move-copy")
@Permission("cloud:file:update")
public ResponseResult<List<FileDocument>> checkMoveOrCopy(UploadApiParamDTO upload, @RequestParam String[] froms, String to) throws IOException {
if (froms != null && froms.length > 0) {
List<String> list = Arrays.asList(froms);
return fileService.checkMoveOrCopy(upload, list, to);
} else {
throw new CommonException(ExceptionType.MISSING_PARAMETERS.getCode(), ExceptionType.MISSING_PARAMETERS.getMsg());
}
}
@Operation(summary = "移动文件/文件夹")
@GetMapping("/move")
@Permission("cloud:file:update")
public ResponseResult<Object> move(UploadApiParamDTO upload, @RequestParam String[] froms, String to) throws IOException {
if (froms != null && froms.length > 0) {
List<String> list = Arrays.asList(froms);
return fileService.move(upload, list, to);
} else {
throw new CommonException(ExceptionType.MISSING_PARAMETERS.getCode(), ExceptionType.MISSING_PARAMETERS.getMsg());
}
}
@Operation(summary = "复制文件/文件夹")
@GetMapping("/copy")
@Permission("cloud:file:update")
public ResponseResult<Object> copy(UploadApiParamDTO upload, @RequestParam String[] froms, String to) throws IOException {
if (froms != null && froms.length > 0) {
List<String> list = Arrays.asList(froms);
return fileService.copy(upload, list, to);
} else {
throw new CommonException(ExceptionType.MISSING_PARAMETERS.getCode(), ExceptionType.MISSING_PARAMETERS.getMsg());
}
}
@Operation(summary = "创建副本")
@GetMapping("/duplicate")
@Permission("cloud:file:update")
public ResponseResult<Object> duplicate(@RequestParam String fileId, @RequestParam String newFilename) {
return fileService.duplicate(fileId, newFilename);
}
@Operation(summary = "解压zip文件")
@GetMapping("/unzip")
@Permission("cloud:file:update")
public ResponseResult<Object> unzip(@RequestParam String fileId, String destFileId) {
return fileService.unzip(fileId, destFileId);
}
@Operation(summary = "获取目录下的文件")
@GetMapping("/listfiles")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<Object> listFiles(@RequestParam String path, @RequestParam String username, Boolean tempDir) {
Boolean dir = tempDir;
if (dir == null) {
dir = false;
}
return fileService.listFiles(FileNameUtils.safeDecode(path), username, dir);
}
@Operation(summary = "获取上级文件列表")
@GetMapping("/upper-level-list")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<Object> upperLevelList(@RequestParam String path, @RequestParam String username) {
return fileService.upperLevelList(FileNameUtils.safeDecode(path), username);
}
@Operation(summary = "根据path删除文件/文件夹")
@DeleteMapping("/delFile")
@Permission("cloud:file:delete")
public ResponseResult<Object> delFile(@RequestParam String path, @RequestParam String username) {
return fileService.delFile(FileNameUtils.safeDecode(path), username);
}
@Operation(summary = "根据path重命名")
@GetMapping("/rename/path")
@Permission("cloud:file:update")
public ResponseResult<Object> renameByPath(@RequestParam String newFileName, @RequestParam String username, @RequestParam String path) {
return fileService.renameByPath(FileNameUtils.safeDecode(newFileName), username, FileNameUtils.safeDecode(path));
}
@Operation(summary = "根据path添加文件/文件夹")
@PostMapping("/addfile")
@Permission("cloud:file:upload")
public ResponseResult<FileIntroVO> addFile(@RequestParam String fileName, @RequestParam Boolean isFolder, @RequestParam String username, @RequestParam String parentPath, String folder) {
return fileService.addFile(FileNameUtils.safeDecode(fileName), isFolder, username, FileNameUtils.safeDecode(parentPath), folder);
}
}

View File

@ -0,0 +1,100 @@
package com.jmal.clouddisk.controller.rest;
import com.jmal.clouddisk.annotation.LogOperatingFun;
import com.jmal.clouddisk.annotation.Permission;
import com.jmal.clouddisk.model.GridFSBO;
import com.jmal.clouddisk.model.LogOperation;
import com.jmal.clouddisk.office.OfficeHistory;
import com.jmal.clouddisk.service.IFileVersionService;
import com.jmal.clouddisk.util.ResponseResult;
import com.jmal.clouddisk.util.ResultUtil;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.RequiredArgsConstructor;
import org.springframework.core.io.InputStreamResource;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
import java.util.List;
/**
* @author jmal
* @Description 文件历史版本
* @date 2023/5/10 16:56
*/
@Tag(name = "文件历史版本")
@RequestMapping("history")
@RestController
@RequiredArgsConstructor
public class FileVersionController {
private final IFileVersionService fileVersionService;
@Operation(summary = "历史文件列表(根据fileId)")
@GetMapping("/list")
@Permission("cloud:file:list")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<List<GridFSBO>> list(String fileId, @RequestParam Integer pageSize, @RequestParam Integer pageIndex) {
return fileVersionService.listFileVersion(fileId, pageSize, pageIndex);
}
@Operation(summary = "office历史文件列表(根据fileId)")
@GetMapping("/office/list")
@Permission("cloud:file:list")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<List<OfficeHistory>> officeList(String fileId, @RequestParam Integer pageSize, @RequestParam Integer pageIndex) {
return fileVersionService.officeListFileVersion(fileId, pageSize, pageIndex);
}
@Operation(summary = "历史文件列表(根据filepath)")
@GetMapping("/path/list")
@Permission("cloud:file:list")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<List<GridFSBO>> listByPath(@RequestParam String path, @RequestParam Integer pageSize, @RequestParam Integer pageIndex) {
return fileVersionService.listFileVersionByPath(path, pageSize, pageIndex);
}
@Operation(summary = "读取历史文件")
@GetMapping("/preview/file")
@Permission("cloud:file:list")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseEntity<InputStreamResource> readHistoryFile(@RequestParam String id) {
return fileVersionService.readHistoryFile(id);
}
@Operation(summary = "读取历史simText文件")
@GetMapping("/preview/text")
@Permission("cloud:file:list")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<Object> previewText(@RequestParam String id) {
return ResultUtil.success(fileVersionService.getFileById(id));
}
@Operation(summary = "恢复该历史版本")
@PutMapping("/recovery")
@Permission("cloud:file:update")
@LogOperatingFun(logType = LogOperation.Type.OPERATION)
public ResponseResult<Long> recovery(@RequestParam String id) {
return ResultUtil.success(fileVersionService.recovery(id));
}
@Operation(summary = "删除该历史版本")
@DeleteMapping("/delete")
@Permission("cloud:file:delete")
@LogOperatingFun(logType = LogOperation.Type.OPERATION)
public ResponseResult<Object> delete(@RequestParam String id) {
fileVersionService.deleteOne(id);
return ResultUtil.success();
}
@Operation(summary = "流式读取历史simText文件")
@GetMapping("/preview/text/stream")
@Permission("cloud:file:list")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseEntity<StreamingResponseBody> previewTextStream(@RequestParam String id) {
return new ResponseEntity<>(fileVersionService.getStreamFileById(id), HttpStatus.OK);
}
}

View File

@ -0,0 +1,18 @@
package com.jmal.clouddisk.controller.rest;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController
@Tag(name = "健康检查")
public class HealthCheckController {
@GetMapping("/public/health")
public ResponseEntity<String> healthCheck() {
return new ResponseEntity<>("UP", HttpStatus.OK);
}
}

View File

@ -0,0 +1,41 @@
package com.jmal.clouddisk.controller.rest;
import com.jmal.clouddisk.annotation.Permission;
import com.jmal.clouddisk.model.LogOperation;
import com.jmal.clouddisk.model.LogOperationDTO;
import com.jmal.clouddisk.service.impl.LogService;
import com.jmal.clouddisk.util.ResponseResult;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.*;
import java.util.List;
/**
* @author jmal
* @Description 日志管理
* @Date 2021/2/8 2:12 下午
*/
@RestController
@RequestMapping("log")
@Tag(name = "日志")
@RequiredArgsConstructor
public class LogController {
private final LogService logService;
@Operation(summary = "查询日志")
@GetMapping("/list")
@Permission("sys:log:list")
public ResponseResult<List<LogOperation>> list(@ModelAttribute LogOperationDTO logOperationDTO) {
return logService.list(logOperationDTO);
}
@Operation(summary = "查询文件操作记录")
@GetMapping("/getFileOperationHistory")
public ResponseResult<List<LogOperationDTO>> getFileOperationHistory(@ModelAttribute LogOperationDTO logOperationDTO, @RequestParam String fileId) {
return logService.getFileOperationHistory(logOperationDTO, fileId);
}
}

View File

@ -0,0 +1,56 @@
package com.jmal.clouddisk.controller.rest;
import com.jmal.clouddisk.lucene.SearchFileService;
import com.jmal.clouddisk.model.FileIntroVO;
import com.jmal.clouddisk.model.query.SearchDTO;
import com.jmal.clouddisk.util.ResponseResult;
import com.jmal.clouddisk.util.ResultUtil;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.RequiredArgsConstructor;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
import org.springframework.web.bind.annotation.*;
import java.io.IOException;
import java.util.List;
/**
* @author jmal
* @Description 全文搜索
* @Date 2021/4/27 5:17 下午
*/
@RestController
@RequestMapping("/search")
@Tag(name = "全文搜索")
@RequiredArgsConstructor
public class LuceneSearchController {
private final SearchFileService searchFileService;
@Operation(summary = "搜索")
@GetMapping("/")
public ResponseResult<List<FileIntroVO>> list(SearchDTO searchDTO) throws IOException, ParseException, InvalidTokenOffsetsException {
return searchFileService.searchFile(searchDTO);
}
@Operation(summary = "最近搜索记录")
@GetMapping("/recentlySearchHistory")
public ResponseResult<List<SearchDTO>> recentlySearchHistory(String keyword) {
return ResultUtil.success(searchFileService.recentlySearchHistory(keyword));
}
@Operation(summary = "删除搜索记录")
@DeleteMapping("/deleteSearchHistory")
public ResponseResult<String> deleteSearchHistory(String id) {
searchFileService.deleteSearchHistory(id);
return ResultUtil.success();
}
@Operation(summary = "删除所有搜索记录")
@DeleteMapping("/deleteAllSearchHistory")
public ResponseResult<String> deleteAllSearchHistory() {
searchFileService.deleteAllSearchHistory();
return ResultUtil.success();
}
}

View File

@ -0,0 +1,136 @@
package com.jmal.clouddisk.controller.rest;
import cn.hutool.core.text.CharSequenceUtil;
import com.jmal.clouddisk.annotation.LogOperatingFun;
import com.jmal.clouddisk.annotation.Permission;
import com.jmal.clouddisk.model.*;
import com.jmal.clouddisk.oss.web.WebOssService;
import com.jmal.clouddisk.service.IFileService;
import com.jmal.clouddisk.service.IMarkdownService;
import com.jmal.clouddisk.service.IUserService;
import com.jmal.clouddisk.util.CaffeineUtil;
import com.jmal.clouddisk.util.ResponseResult;
import com.jmal.clouddisk.util.ResultUtil;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.servlet.http.HttpServletRequest;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
/**
* @author jmal
* @Description markdown
*/
@Controller
@Tag(name = "markdown管理")
@RestController
public class MarkDownController {
@Autowired
private IMarkdownService markdownService;
@Autowired
private WebOssService webOssService;
@Autowired
private IFileService fileService;
@Autowired
private IUserService userService;
@Operation(summary = "获取markdown内容")
@GetMapping("/markdown/p")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<?> getMarkDownContent(ArticleDTO articleDTO, Integer pageIndex, Integer pageSize) {
articleDTO.setPageIndex(pageIndex);
articleDTO.setPageSize(pageSize);
if (CharSequenceUtil.isBlank(articleDTO.getMark())) {
return markdownService.getMarkdownList(articleDTO);
} else {
return markdownService.getMarkDownOne(articleDTO);
}
}
@Operation(summary = "编辑文档(根据fileId)")
@PostMapping("/markdown/edit")
@Permission("cloud:file:update")
@LogOperatingFun(logType = LogOperation.Type.OPERATION)
public ResponseResult<Object> editMarkdown(@RequestBody @Validated ArticleParamDTO upload) {
return markdownService.editMarkdown(upload);
}
@Operation(summary = "修改文档排序")
@PostMapping("/markdown/sort")
@Permission("cloud:file:update")
@LogOperatingFun
public ResponseResult<Object> sortMarkdown(@RequestBody String[] fileIdList) {
return markdownService.sortMarkdown(Arrays.asList(fileIdList));
}
@Operation(summary = "删除草稿")
@DeleteMapping("/markdown/deleteDraft")
@Permission("cloud:file:delete")
@LogOperatingFun(logType = LogOperation.Type.OPERATION)
public ResponseResult<Object> deleteDraft(@RequestParam String fileId, @RequestParam String username) {
return markdownService.deleteDraft(fileId, username);
}
@Operation(summary = "编辑文档(根据path)")
@PostMapping("/markdown/edit1")
@Permission("cloud:file:update")
public ResponseResult<Object> editTextByPath(@RequestBody UploadApiParamDTO upload) {
ResultUtil.checkParamIsNull(upload.getUsername(), upload.getUserId(), upload.getRelativePath(), upload.getContentText());
if (!CharSequenceUtil.isBlank(upload.getMountFileId())) {
FileDocument fileDocument = fileService.getById(upload.getMountFileId());
upload.setUserId(fileDocument.getUserId());
upload.setUsername(userService.getUserNameById(fileDocument.getUserId()));
upload.setRelativePath(fileDocument.getPath() + fileDocument.getName());
upload.setOperationPermissionList(fileDocument.getOperationPermissionList());
}
Path prePth = Paths.get(upload.getUsername(), upload.getRelativePath());
String ossPath = CaffeineUtil.getOssPath(prePth);
if (ossPath != null) {
webOssService.putObjectText(ossPath, prePth, upload.getContentText());
return ResultUtil.success();
}
return markdownService.editTextByPath(upload);
}
@Operation(summary = "上传文档里的图片")
@PostMapping("/upload-markdown-image")
@Permission("cloud:file:upload")
public ResponseResult<Object> uploadMarkdownImage(UploadImageDTO upload) {
if(CharSequenceUtil.isBlank(upload.getUserId()) || CharSequenceUtil.isBlank(upload.getUsername())) {
return ResultUtil.warning("参数里缺少 userId 或 username");
}
return markdownService.uploadMarkdownImage(upload);
}
@Operation(summary = "上传文档里链接图片")
@PostMapping("/upload-markdown-link-image")
@Permission("cloud:file:upload")
@LogOperatingFun
public ResponseResult<Object> uploadMarkdownLinkImage(HttpServletRequest request, @RequestBody UploadImageDTO uploadImageDTO) {
String userId = uploadImageDTO.getUserId();
String username = uploadImageDTO.getUsername();
if(CharSequenceUtil.isBlank(userId)){
userId = request.getHeader("userId");
uploadImageDTO.setUserId(userId);
}
if(CharSequenceUtil.isBlank(username)){
username = request.getHeader("username");
uploadImageDTO.setUsername(username);
}
if(CharSequenceUtil.isBlank(userId) || CharSequenceUtil.isBlank(username)) {
return ResultUtil.warning("请求头里或参数里必须含有userId和username");
}
return markdownService.uploadMarkdownLinkImage(uploadImageDTO);
}
}

View File

@ -0,0 +1,96 @@
package com.jmal.clouddisk.controller.rest;
import com.jmal.clouddisk.annotation.AnnoManageUtil;
import com.jmal.clouddisk.annotation.LogOperatingFun;
import com.jmal.clouddisk.annotation.Permission;
import com.jmal.clouddisk.model.LogOperation;
import com.jmal.clouddisk.model.query.QueryMenuDTO;
import com.jmal.clouddisk.model.rbac.MenuDO;
import com.jmal.clouddisk.model.rbac.MenuDTO;
import com.jmal.clouddisk.service.impl.MenuService;
import com.jmal.clouddisk.util.ResponseResult;
import com.jmal.clouddisk.util.ResultUtil;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
import java.util.Arrays;
import java.util.List;
/**
* @author jmal
* @Description 菜单管理
* @Date 2021/1/7 7:44 下午
*/
@RestController
@RequestMapping("menu")
@Tag(name = "菜单管理")
public class MenuController {
@Autowired
private MenuService menuService;
@Operation(summary = "菜单树")
@GetMapping("/tree")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<List<MenuDTO>> tree(String name, QueryMenuDTO queryDTO) {
queryDTO.setName(name);
return ResultUtil.success(menuService.tree(queryDTO));
}
@Operation(summary = "菜单信息")
@GetMapping("/info")
@Permission("sys:menu:list")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<MenuDTO> info(@RequestParam String menuId) {
MenuDTO menuDTO = new MenuDTO();
MenuDO menuDO = menuService.getMenuInfo(menuId);
if(menuDO != null){
BeanUtils.copyProperties(menuDO, menuDTO);
}
return ResultUtil.success(menuDTO);
}
@Operation(summary = "权限标识列表")
@GetMapping("/authorities")
@Permission("sys:menu:list")
public ResponseResult<List<String>> authorities() {
return ResultUtil.success(AnnoManageUtil.AUTHORITIES);
}
@Operation(summary = "添加菜单")
@PostMapping("/add")
@LogOperatingFun
@Permission("sys:menu:add")
public ResponseResult<Object> add(@RequestParam String name, @ModelAttribute @Validated MenuDTO menuDTO) {
menuDTO.setName(name);
return menuService.add(menuDTO);
}
@Operation(summary = "更新菜单")
@PutMapping("/update")
@LogOperatingFun
@Permission("sys:menu:update")
public ResponseResult<Object> update(@RequestParam String name, @ModelAttribute MenuDTO menuDTO) {
menuDTO.setName(name);
if(menuDTO.getId() == null){
return ResultUtil.warning("缺少参数menuId");
}
return menuService.update(menuDTO);
}
@Operation(summary = "删除菜单")
@DeleteMapping("/delete")
@LogOperatingFun
@Permission("sys:menu:delete")
public ResponseResult<Object> delete(@RequestParam String[] menuIds) {
List<String> categoryList = Arrays.asList(menuIds);
menuService.delete(categoryList);
return ResultUtil.success();
}
}

View File

@ -0,0 +1,82 @@
package com.jmal.clouddisk.controller.rest;
import com.jmal.clouddisk.annotation.Permission;
import com.jmal.clouddisk.controller.record.MfaSetupResponse;
import com.jmal.clouddisk.controller.record.MfaVerifyRequest;
import com.jmal.clouddisk.exception.CommonException;
import com.jmal.clouddisk.exception.ExceptionType;
import com.jmal.clouddisk.service.IUserService;
import com.jmal.clouddisk.service.impl.TotpService;
import com.jmal.clouddisk.service.impl.UserLoginHolder;
import com.jmal.clouddisk.util.MessageUtil;
import com.jmal.clouddisk.util.ResponseResult;
import com.jmal.clouddisk.util.ResultUtil;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@RestController
@RequiredArgsConstructor
@RequestMapping("/mfa")
@Tag(name = "两步验证")
public class MfaController {
private final TotpService totpService;
private final IUserService userService;
private final UserLoginHolder userLoginHolder;
private final MessageUtil messageUtil;
@PostMapping("/setup")
@Permission("sys:user:update")
@Operation(summary = "准备开启两步验证")
public ResponseResult<Object> initiateMfaSetup() {
if (userService.isMfaEnabled(userLoginHolder.getUsername())) {
return ResultUtil.success(new MfaSetupResponse(true, null, null));
}
final String secret = totpService.generateNewSecret();
final String qrCodeUri = totpService.generateQrCodeImageUri(secret, userLoginHolder.getUsername());
return ResultUtil.success(new MfaSetupResponse(false, secret, qrCodeUri));
}
@PostMapping("/enable")
@Permission("sys:user:update")
@Operation(summary = "验证并开启两步验证")
public ResponseResult<Object> verifyAndEnableMfa(@RequestBody MfaVerifyRequest request) {
if (userService.isMfaEnabled(userLoginHolder.getUsername())) {
throw new CommonException(ExceptionType.EXISTING_RESOURCES.getCode(), "MFA已启用");
}
// 使用前端传回的secret和用户输入的code进行验证
if (!totpService.isRawCodeValid(request.secret(), request.code())) {
return ResultUtil.error(messageUtil.getMessage("login.mfaError"));
}
// 验证成功!将密钥与用户绑定并启用MFA
userService.enableMfa(userLoginHolder.getUserId(), request.secret());
return ResultUtil.success();
}
@PostMapping("/disable")
@Permission("sys:user:update")
@Operation(summary = "验证并禁用两步验证")
public ResponseResult<Object> verifyDisableMfa(@RequestBody MfaVerifyRequest request) {
if (!userService.isMfaEnabled(userLoginHolder.getUsername())) {
throw new CommonException(ExceptionType.EXISTING_RESOURCES.getCode(), "MFA未启用");
}
// 对用户输入的code进行验证
if (totpService.isCodeInvalid(request.code(), userLoginHolder.getUsername())) {
return ResultUtil.error(messageUtil.getMessage("login.mfaError"));
}
// 验证成功!禁用MFA
userService.disableMfa(userLoginHolder.getUserId());
return ResultUtil.success();
}
}

View File

@ -0,0 +1,39 @@
package com.jmal.clouddisk.controller.rest;
import cn.hutool.core.date.TimeInterval;
import cn.hutool.core.io.FileUtil;
import cn.hutool.http.HttpUtil;
import com.jmal.clouddisk.ocr.OcrService;
import com.jmal.clouddisk.service.impl.UserLoginHolder;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
@RestController
@RequiredArgsConstructor
@Tag(name = "OCR")
@Slf4j
public class OcrController {
private final OcrService ocrService;
private final UserLoginHolder userLoginHolder;
@GetMapping("/ocr")
public String performOcr(@RequestParam String fileUrl) {
String tempImagePath = ocrService.generateOrcTempImagePath(userLoginHolder.getUsername());
try {
HttpUtil.downloadFile(fileUrl, tempImagePath);
TimeInterval timeInterval = new TimeInterval();
timeInterval.start();
String str = ocrService.doOCR(tempImagePath, null, "tesseract");
log.info("OCR time consuming: {}", timeInterval.intervalMs());
return str;
} finally {
// 删除临时文件
FileUtil.del(tempImagePath);
}
}
}

View File

@ -0,0 +1,71 @@
package com.jmal.clouddisk.controller.rest;
import com.jmal.clouddisk.annotation.LogOperatingFun;
import com.jmal.clouddisk.annotation.Permission;
import com.jmal.clouddisk.model.LogOperation;
import com.jmal.clouddisk.model.query.QueryRoleDTO;
import com.jmal.clouddisk.model.rbac.RoleDTO;
import com.jmal.clouddisk.service.impl.RoleService;
import com.jmal.clouddisk.util.ResponseResult;
import com.jmal.clouddisk.util.ResultUtil;
import io.swagger.v3.oas.annotations.tags.Tag;
import io.swagger.v3.oas.annotations.Operation;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
import java.util.List;
/**
* @Description 角色管理
* @blame jmal
* @Date 2021/1/7 7:44 下午
*/
@RestController
@RequestMapping("role")
@Tag(name = "角色管理")
public class RoleController {
@Autowired
private RoleService roleService;
@Operation(summary = "角色列表")
@GetMapping("/list")
@Permission("sys:role:list")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<List<RoleDTO>> list(String name, @ModelAttribute QueryRoleDTO queryRoleDTO) {
queryRoleDTO.setName(name);
return roleService.list(queryRoleDTO);
}
@Operation(summary = "添加角色")
@PostMapping("/add")
@LogOperatingFun
@Permission("sys:role:add")
public ResponseResult<Object> add(@RequestParam String name, @ModelAttribute @Validated RoleDTO roleDTO) {
roleDTO.setName(name);
return roleService.add(roleDTO);
}
@Operation(summary = "修改角色")
@PutMapping("/update")
@LogOperatingFun
@Permission("sys:role:update")
public ResponseResult<Object> update(@RequestParam String name, @RequestBody @ModelAttribute @Validated RoleDTO roleDTO) {
roleDTO.setName(name);
if (roleDTO.getId() == null) {
return ResultUtil.warning("缺少参数roleId");
}
return roleService.update(roleDTO);
}
@Operation(summary = "删除角色")
@DeleteMapping("/delete")
@LogOperatingFun
@Permission("sys:role:delete")
public ResponseResult<Object> delete(@RequestParam String[] roleIds) {
roleService.delete(roleIds);
return ResultUtil.success();
}
}

View File

@ -0,0 +1,309 @@
package com.jmal.clouddisk.controller.rest;
import com.jmal.clouddisk.annotation.LogOperatingFun;
import com.jmal.clouddisk.annotation.Permission;
import com.jmal.clouddisk.exception.CommonException;
import com.jmal.clouddisk.exception.ExceptionType;
import com.jmal.clouddisk.interceptor.FileInterceptor;
import com.jmal.clouddisk.model.*;
import com.jmal.clouddisk.model.rbac.ConsumerDO;
import com.jmal.clouddisk.oss.web.WebOssService;
import com.jmal.clouddisk.service.Constants;
import com.jmal.clouddisk.service.IFileService;
import com.jmal.clouddisk.service.IShareService;
import com.jmal.clouddisk.service.IUserService;
import com.jmal.clouddisk.util.CaffeineUtil;
import com.jmal.clouddisk.util.ResponseResult;
import com.jmal.clouddisk.util.ResultUtil;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import lombok.RequiredArgsConstructor;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
/**
* @Description share
* @Author jmal
* @Date 2020-03-17 16:22
*/
@Tag(name = "文件分享")
@RestController
@RequiredArgsConstructor
public class ShareController {
public static final String SHARE_EXPIRED = "该分享已过期";
private final IShareService shareService;
private final IFileService fileService;
private final IUserService userService;
private final FileInterceptor fileInterceptor;
private final WebOssService webOssService;
@Operation(summary = "该分享已失效")
@GetMapping("/public/s/invalid")
public String invalid() {
return "该分享已失效";
}
@Operation(summary = "生成分享链接")
@PostMapping("/share/generate")
@Permission("cloud:file:update")
@LogOperatingFun
public ResponseResult<Object> generateLink(@RequestBody ShareDO share) {
ResultUtil.checkParamIsNull(share.getFileId(), share.getUserId());
return shareService.generateLink(share);
}
@Operation(summary = "生成shareToken")
@GetMapping("/share/generate/share-token")
@Permission("cloud:file:list")
public ResponseResult<Object> generateShareToken(@RequestParam String fileId) {
return shareService.generateShareToken(fileId);
}
@Operation(summary = "获取分享信息")
@GetMapping("/get/share")
@Permission("cloud:file:list")
@LogOperatingFun
public ResponseResult<ShareDO> getShare(@RequestParam String shareId) {
return ResultUtil.success(shareService.getShare(shareId));
}
@Operation(summary = "获取分享信息")
@GetMapping("/get/share/by/fileId")
@Permission("cloud:file:list")
@LogOperatingFun
public ResponseResult<ShareDO> getShareByFileId(@RequestParam String fileId) {
return ResultUtil.success(shareService.getShareByFileId(fileId));
}
@Operation(summary = "是否含有子分享")
@GetMapping("/share/has-sub-share")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<Boolean> hasSubShare(@RequestParam List<String> shareIds) {
return ResultUtil.success(shareService.hasSubShare(shareIds));
}
@Operation(summary = "文件夹下是否含有子分享")
@GetMapping("/share/folder-sub-share")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<Boolean> folderSubShare(@RequestParam String fileId) {
return ResultUtil.success(shareService.folderSubShare(fileId));
}
@Operation(summary = "取消分享")
@DeleteMapping("/share/cancel")
@Permission("cloud:file:delete")
@LogOperatingFun
public ResponseResult<Object> cancelShare(@RequestParam String[] shareId) {
return shareService.cancelShare(shareId);
}
@Operation(summary = "分享列表")
@GetMapping("/share/list")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<Object> shareList(UploadApiParamDTO upload) {
return shareService.shareList(upload);
}
@Operation(summary = "挂载文件夹")
@PutMapping("/share/mount-folder")
@Permission("cloud:file:upload")
public ResponseResult<Object> mountFile(@RequestBody UploadApiParamDTO upload) {
shareService.mountFile(upload);
return ResultUtil.success();
}
@Operation(summary = "验证提取码")
@PostMapping("/public/valid-share-code")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<Object> validShareCode(@RequestBody Map<String, String> body) {
String shareId = body.get(Constants.SHARE_ID);
String shareCode = body.get("shareCode");
ResultUtil.checkParamIsNull(shareId, shareCode);
return shareService.validShareCode(shareId, shareCode);
}
@Operation(summary = "访问分享链接")
@GetMapping("/public/access-share")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<Object> accessShare(HttpServletRequest request, @RequestParam String share, Integer pageIndex, Integer pageSize, Boolean showFolderSize) {
ShareDO shareDO = shareService.getShare(share);
shareService.validShare(request.getHeader(Constants.SHARE_TOKEN), shareDO);
return shareService.accessShare(shareDO, pageIndex, pageSize, showFolderSize);
}
@Operation(summary = "获取分享者信息")
@GetMapping("/public/get/sharer")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<SharerDTO> getSharer(@RequestParam String shareId) {
return shareService.getSharer(shareId);
}
@Operation(summary = "访问分享链接里的目录")
@GetMapping("public/access-share/open")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<Object> accessShareOpenDir(HttpServletRequest request, @RequestParam String share, @RequestParam String fileId, Integer pageIndex, Integer pageSize, Boolean showFolderSize) {
ShareDO shareDO = shareService.getShare(share);
shareService.validShare(request.getHeader(Constants.SHARE_TOKEN), shareDO);
return shareService.accessShareOpenDir(shareDO, fileId, pageIndex, pageSize, showFolderSize);
}
@Operation(summary = "打包下载")
@GetMapping("/public/s/packageDownload")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public void publicPackageDownload(HttpServletRequest request, HttpServletResponse response, @RequestParam String shareId, @RequestParam String[] fileIds) {
shareService.validShare(request.getParameter(Constants.SHARE_TOKEN), shareId);
if (fileIds != null && fileIds.length > 0) {
List<String> fileIdList = Arrays.asList(fileIds);
fileService.publicPackageDownload(request, response, fileIdList);
} else {
throw new CommonException(ExceptionType.MISSING_PARAMETERS.getCode(), ExceptionType.MISSING_PARAMETERS.getMsg());
}
}
@Operation(summary = "打包下载")
@GetMapping("/public/s/{fileId}/packageDownload/{filename}")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public void publicPackageDownloadOne(HttpServletRequest request, HttpServletResponse response, @PathVariable String fileId, @PathVariable String filename) {
FileDocument fileDocument = fileService.getById(fileId);
if (fileInterceptor.isNotAllowAccess(fileDocument, request)) {
return;
}
List<String> fileIdList = Collections.singletonList(fileId);
fileService.publicPackageDownload(request, response, fileIdList);
}
@Operation(summary = "显示缩略图")
@GetMapping("/articles/s/view/thumbnail")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseEntity<Object> articlesThumbnail(String id, Boolean showCover) {
return thumbnail(id, showCover, null);
}
@Operation(summary = "显示缩略图")
@GetMapping("/public/s/view/thumbnail")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseEntity<Object> publicThumbnail(String id, Boolean showCover, HttpServletRequest request) {
return thumbnail(id, showCover, request);
}
@Operation(summary = "获取dwg文件对应的mxweb文件")
@GetMapping("/public/s/view/mxweb/{fileId}/{shareId}")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseEntity<Object> publicGetMxweb(HttpServletRequest request, @PathVariable String shareId, @PathVariable String fileId) {
validShare(request, shareId);
Optional<FileDocument> file = fileService.getMxweb(fileId);
return file.map(fileService::getObjectResponseEntity).orElseGet(() -> ResponseEntity.status(HttpStatus.NOT_FOUND).body("找不到该文件"));
}
@Operation(summary = "显示缩略图")
@GetMapping("/public/s/view/thumbnail/{filename}")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseEntity<Object> publicThumbnailName(String id, Boolean showCover, HttpServletRequest request) {
return publicThumbnail(id, showCover, request);
}
@Operation(summary = "显示缩略图(媒体封面)")
@GetMapping("/public/s/view/cover")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseEntity<Object> coverOfMedia(String id, String name) {
ResultUtil.checkParamIsNull(id, name);
Optional<FileDocument> file = fileService.coverOfMedia(id, name);
return file.map(fileService::getObjectResponseEntity).orElseGet(() -> ResponseEntity.status(HttpStatus.NOT_FOUND).body("找不到该文件"));
}
private ResponseEntity<Object> thumbnail(String id, Boolean showCover, HttpServletRequest request) {
ResultUtil.checkParamIsNull(id);
Optional<FileDocument> file = fileService.thumbnail(id, showCover);
if (fileInterceptor.isNotAllowAccess(file.orElse(null), request)) {
return null;
}
String ossPath = CaffeineUtil.getOssPath(Paths.get(id));
if (ossPath != null) {
return webOssService.thumbnail(ossPath, id);
}
return file.<ResponseEntity<Object>>map(fileDocument ->
ResponseEntity.ok()
.header(HttpHeaders.CONTENT_TYPE, fileDocument.getContentType())
.header(HttpHeaders.CONNECTION, "close")
.header(HttpHeaders.CONTENT_LENGTH, String.valueOf(fileDocument.getContent().length))
.header(HttpHeaders.CONTENT_ENCODING, "utf-8")
.header(HttpHeaders.CACHE_CONTROL, "public, max-age=604800")
.body(fileDocument.getContent())).orElseGet(() -> ResponseEntity.status(HttpStatus.NOT_FOUND).body("找不到该文件"));
}
@Operation(summary = "读取simText文件")
@GetMapping("/public/s/preview/text")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<Object> preview(HttpServletRequest request, @RequestParam String shareId, @RequestParam String fileId, Boolean content) {
validShare(request, shareId);
Path prePth = Paths.get(fileId);
String ossPath = CaffeineUtil.getOssPath(prePth);
if (ossPath != null) {
return ResultUtil.success(webOssService.readToText(ossPath, prePth, content));
}
return ResultUtil.success(fileService.getById(fileId, content));
}
@Operation(summary = "流式读取simText文件")
@GetMapping("/public/s/preview/text/stream")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseEntity<StreamingResponseBody> previewTextStream(HttpServletRequest request, @RequestParam String shareId, @RequestParam String fileId) {
validShare(request, shareId);
Path prePth = Paths.get(fileId);
String ossPath = CaffeineUtil.getOssPath(prePth);
StreamingResponseBody responseBody;
if (ossPath != null) {
responseBody = webOssService.readToTextStream(ossPath, prePth);
} else {
responseBody = fileService.getStreamById(fileId);
}
return new ResponseEntity<>(responseBody, HttpStatus.OK);
}
private void validShare(HttpServletRequest request, String shareId) {
ShareDO shareDO = shareService.getShare(shareId);
shareService.validShare(request.getHeader(Constants.SHARE_TOKEN), shareDO);
ConsumerDO consumer = userService.userInfoById(shareDO.getUserId());
if (consumer == null) {
throw new CommonException(ExceptionType.WARNING.getCode(), SHARE_EXPIRED);
}
}
@Operation(summary = "根据id获取分享的文件信息")
@GetMapping("/public/file_info")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<Object> getFileById(HttpServletRequest request, @RequestParam String fileId, @RequestParam String shareId) {
shareService.validShare(request.getHeader(Constants.SHARE_TOKEN), shareId);
return ResultUtil.success(fileService.getById(fileId));
}
@Operation(summary = "挂用户获取分享文件信息")
@GetMapping("/mount/file_info")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<Map<String, String>> getMountFileInfo(@RequestParam String fileId, @RequestParam String fileUserId) {
return ResultUtil.success(shareService.getMountFileInfo(fileId, fileUserId));
}
@Operation(summary = "挂用户获取目录Id")
@GetMapping("/mount/folder/id")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<String> getMountFolderId(@RequestParam String otherFileId, @RequestParam String fileUsername, @RequestParam String path) {
return ResultUtil.success(shareService.getMountFolderId(path, fileUsername, otherFileId));
}
}

View File

@ -0,0 +1,89 @@
package com.jmal.clouddisk.controller.rest;
import com.jmal.clouddisk.annotation.LogOperatingFun;
import com.jmal.clouddisk.annotation.Permission;
import com.jmal.clouddisk.model.LogOperation;
import com.jmal.clouddisk.model.TagDO;
import com.jmal.clouddisk.model.TagDTO;
import com.jmal.clouddisk.service.impl.TagService;
import com.jmal.clouddisk.util.ResponseResult;
import com.jmal.clouddisk.util.ResultUtil;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
import java.util.Arrays;
import java.util.List;
/**
* @author jmal
* @Description 标签管理
* @Date 2020/12/02 6:47 下午
*/
@RestController
@Tag(name = "标签管理")
public class TagController {
@Autowired
private TagService tagService;
@Operation(summary = "标签列表")
@GetMapping("/tag/list")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<List<TagDTO>> list(String userId) {
return ResultUtil.success(tagService.list(userId));
}
@Operation(summary = "标签信息")
@GetMapping("/tag/info")
@Permission("website:set:list")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<TagDTO> tagInfo(@RequestParam String tagId) {
TagDTO tagDTO = new TagDTO();
TagDO tag = tagService.getTagInfo(tagId);
if(tag != null){
BeanUtils.copyProperties(tag, tagDTO);
}
return ResultUtil.success(tagDTO);
}
@Operation(summary = "添加标签")
@PostMapping("/tag/add")
@Permission("website:set:add")
@LogOperatingFun
public ResponseResult<Object> add(@RequestParam String name, @ModelAttribute @Validated TagDTO tagDTO) {
tagDTO.setName(name);
return tagService.add(tagDTO);
}
@Operation(summary = "更新标签")
@PutMapping("/tag/update")
@Permission("website:set:update")
@LogOperatingFun
public ResponseResult<Object> update(@RequestParam String name, @ModelAttribute @Validated TagDTO tagDTO) {
tagDTO.setName(name);
return tagService.update(tagDTO);
}
@Operation(summary = "删除标签")
@DeleteMapping("/tag/delete")
@Permission("website:set:delete")
@LogOperatingFun
public ResponseResult<Object> delete(@RequestParam String[] tagIds) {
List<String> tagIdList = Arrays.asList(tagIds);
tagService.delete(tagIdList);
return ResultUtil.success();
}
@Operation(summary = "修改标签排序")
@PostMapping("/tag/sort")
@LogOperatingFun
@Permission("cloud:file:update")
public ResponseResult<Object> updateTagSort(@RequestBody List<String> tagIdList) {
tagService.updateTagSort(tagIdList);
return ResultUtil.success();
}
}

View File

@ -0,0 +1,104 @@
package com.jmal.clouddisk.controller.rest;
import com.jmal.clouddisk.annotation.LogOperatingFun;
import com.jmal.clouddisk.annotation.Permission;
import com.jmal.clouddisk.model.LogOperation;
import com.jmal.clouddisk.model.query.QueryUserDTO;
import com.jmal.clouddisk.model.rbac.ConsumerDO;
import com.jmal.clouddisk.model.rbac.ConsumerDTO;
import com.jmal.clouddisk.service.IUserService;
import com.jmal.clouddisk.util.ResponseResult;
import com.jmal.clouddisk.util.ResultUtil;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import java.util.Arrays;
import java.util.List;
/**
* @author jmal
* @Description ConsumerController
*/
@RestController
@RequestMapping("user")
@Tag(name = "用户管理")
public class UserController {
@Autowired
IUserService service;
@Operation(summary = "添加用户")
@PostMapping("/add")
@Permission("sys:user:add")
@LogOperatingFun
public ResponseResult<Object> add(@Validated ConsumerDTO consumerDTO) {
service.add(consumerDTO);
return ResultUtil.success();
}
@Operation(summary = "删除用户")
@DeleteMapping("/delete")
@Permission("sys:user:delete")
@LogOperatingFun
public ResponseResult<Object> delete(@RequestParam String[] ids) {
List<String> idList = Arrays.asList(ids);
return service.delete(idList);
}
@Operation(summary = "修改用户")
@PutMapping("/update")
@Permission("sys:user:update")
@LogOperatingFun
public ResponseResult<Object> update(ConsumerDTO consumerDTO, MultipartFile blobAvatar) {
return service.update(consumerDTO, blobAvatar);
}
@Operation(summary = "修改用户密码")
@PutMapping("/update-pass")
@Permission("sys:user:update")
@LogOperatingFun
public ResponseResult<Object> updatePass(ConsumerDO consumer) {
return service.updatePass(consumer);
}
@Operation(summary = "重置密码")
@PutMapping("/reset-pass")
@Permission(value = "sys:user:update")
@LogOperatingFun
public ResponseResult<Object> resetPass(ConsumerDO consumer) {
return service.resetPass(consumer);
}
@Operation(summary = "用户信息(自己)")
@GetMapping("/info")
public ResponseResult<ConsumerDTO> consumerInfo() {
return service.info();
}
@Operation(summary = "用户信息")
@GetMapping("/userInfo")
@Permission("sys:user:list")
public ResponseResult<ConsumerDTO> consumerInfo(@RequestParam String id) {
return service.userInfo(id);
}
@Operation(summary = "获取用户名")
@GetMapping("/username")
@Permission("cloud:file:list")
public ResponseResult<String> getUsernameByUserId(@RequestParam String userId) {
return ResultUtil.success(service.getUserNameById(userId));
}
@Operation(summary = "用户信息列表")
@GetMapping("/userList")
@Permission("sys:user:list")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<List<ConsumerDTO>> consumerList(QueryUserDTO queryDTO) {
return service.userList(queryDTO);
}
}

View File

@ -0,0 +1,57 @@
package com.jmal.clouddisk.controller.rest;
import com.jmal.clouddisk.annotation.LogOperatingFun;
import com.jmal.clouddisk.annotation.Permission;
import com.jmal.clouddisk.model.LogOperation;
import com.jmal.clouddisk.model.UserAccessTokenDTO;
import com.jmal.clouddisk.service.impl.SettingService;
import com.jmal.clouddisk.service.impl.UserLoginHolder;
import com.jmal.clouddisk.util.ResponseResult;
import com.jmal.clouddisk.util.ResultUtil;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.*;
import java.util.List;
/**
* @author jmal
* @Description 用户设置
* @Date 2020/12/24 11:21 上午
*/
@RestController
@Tag(name = "用户设置")
@RequiredArgsConstructor
public class UserSettingController {
private final SettingService settingService;
private final UserLoginHolder userLoginHolder;
@Operation(summary = "生成accessToken")
@PutMapping("/user/setting/generateAccessToken")
@Permission("sys:user:update")
@LogOperatingFun
ResponseResult<String> generateAccessToken(@RequestParam String tokenName) {
return settingService.generateAccessToken(userLoginHolder.getUsername(), tokenName);
}
@Operation(summary = "accessToken列表")
@GetMapping("/user/setting/accessTokenList")
@Permission("sys:user:list")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
ResponseResult<List<UserAccessTokenDTO>> accessTokenList() {
return settingService.accessTokenList(userLoginHolder.getUsername());
}
@Operation(summary = "删除accessToken")
@DeleteMapping("/user/setting/deleteAccessToken")
@Permission("sys:user:delete")
@LogOperatingFun
ResponseResult<Object> deleteAccessToken(@RequestParam String id) {
settingService.deleteAccessToken(id);
return ResultUtil.success();
}
}

View File

@ -0,0 +1,79 @@
package com.jmal.clouddisk.controller.rest;
import com.jmal.clouddisk.config.FileProperties;
import com.jmal.clouddisk.service.IShareService;
import com.jmal.clouddisk.util.FileContentTypeUtils;
import com.jmal.clouddisk.util.MyFileUtils;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.RequiredArgsConstructor;
import org.jetbrains.annotations.NotNull;
import org.springframework.core.io.UrlResource;
import org.springframework.http.HttpHeaders;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RestController;
import java.io.IOException;
import java.net.MalformedURLException;
import java.nio.file.Path;
import java.nio.file.Paths;
@Tag(name = "视频播放")
@RestController
@RequiredArgsConstructor
public class VideoController {
private final FileProperties fileProperties;
private final IShareService shareService;
@GetMapping("/video/hls/{username}/{fileId}.m3u8")
public ResponseEntity<UrlResource> m3u8(@PathVariable String username, @PathVariable String fileId) throws IOException {
return getUrlResourceResponseEntity(username, fileId, fileId + ".m3u8");
}
private @NotNull ResponseEntity<UrlResource> getUrlResourceResponseEntity(String username, String fileId, String suffix) throws MalformedURLException {
Path txtPath = Paths.get(fileProperties.getRootDir(), fileProperties.getChunkFileDir(), username, fileProperties.getVideoTranscodeCache(), fileId, suffix);
UrlResource videoResource = new UrlResource(txtPath.toUri());
return ResponseEntity.ok()
.header(HttpHeaders.CONTENT_TYPE, FileContentTypeUtils.getContentType(MyFileUtils.extName(suffix)))
.header(HttpHeaders.CACHE_CONTROL, "max-age=600")
.body(videoResource);
}
@GetMapping("/video/hls/{username}/{fileId}.vtt")
public ResponseEntity<UrlResource> vtt(@PathVariable String username, @PathVariable String fileId) throws IOException {
return getUrlResourceResponseEntity(username, fileId, fileId + ".vtt");
}
@GetMapping("/video/hls/{username}/{fileId}-vtt.jpg")
public ResponseEntity<UrlResource> vttPNG(@PathVariable String username, @PathVariable String fileId) throws IOException {
return getUrlResourceResponseEntity(username, fileId, fileId + "-vtt.jpg");
}
@GetMapping("/video/hls/{username}/{fileId}-{index}.ts")
public ResponseEntity<UrlResource> ts(@PathVariable String username, @PathVariable String fileId, @PathVariable String index) throws IOException {
return getUrlResourceResponseEntity(username, fileId, fileId + "-" + index + ".ts");
}
@GetMapping("/public/video/hls/{shareId}/{shareToken}/{username}/{fileId}.m3u8")
public ResponseEntity<UrlResource> publicM3u8(@PathVariable String username, @PathVariable String fileId, @PathVariable String shareId, @PathVariable String shareToken) throws IOException {
shareService.validShare(shareToken, shareId);
return m3u8(username, fileId);
}
@GetMapping("/public/video/hls/{shareId}/{shareToken}/{username}/{fileId}.vtt")
public ResponseEntity<UrlResource> publicVtt(@PathVariable String username, @PathVariable String fileId, @PathVariable String shareId, @PathVariable String shareToken) throws IOException {
shareService.validShare(shareToken, shareId);
return vtt(username, fileId);
}
@GetMapping("/public/video/hls/{shareId}/{shareToken}/{username}/{fileId}-{index}.ts")
public ResponseEntity<UrlResource> publicTs(@PathVariable String username, @PathVariable String fileId, @PathVariable String index, @PathVariable String shareId, @PathVariable String shareToken) throws IOException {
shareService.validShare(shareToken, shareId);
return ts(username, fileId, index);
}
}

View File

@ -0,0 +1,61 @@
package com.jmal.clouddisk.controller.rest;
import com.jmal.clouddisk.annotation.LogOperatingFun;
import com.jmal.clouddisk.annotation.Permission;
import com.jmal.clouddisk.model.HeartwingsDO;
import com.jmal.clouddisk.model.LogOperation;
import com.jmal.clouddisk.model.WebsiteSettingDO;
import com.jmal.clouddisk.model.WebsiteSettingDTO;
import com.jmal.clouddisk.service.impl.SettingService;
import com.jmal.clouddisk.util.ResponseResult;
import com.jmal.clouddisk.util.ResultUtil;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.*;
import java.util.List;
/**
* @author jmal
* @Description 网站设置
* @Date 2020/10/28 5:29 下午
*/
@RestController
@Tag(name = "网站设置")
@RequiredArgsConstructor
public class WebsiteSettingController {
private final SettingService settingService;
@Operation(summary = "获取网站设置")
@GetMapping("/website/setting")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<WebsiteSettingDTO> getWebsiteSetting() {
return ResultUtil.success(settingService.getWebsiteSetting());
}
@Operation(summary = "获取网站备案信息")
@GetMapping("/public/website/record")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<WebsiteSettingDTO> getWebsiteRecord() {
return ResultUtil.success(settingService.getWebsiteRecord());
}
@Operation(summary = "获取网站心语记录")
@GetMapping("/website/heartwings")
@LogOperatingFun(logType = LogOperation.Type.BROWSE)
public ResponseResult<List<HeartwingsDO>> getWebsiteHeartwings(@RequestParam Integer page, @RequestParam Integer pageSize, @RequestParam String order) {
return settingService.getWebsiteHeartwings(page, pageSize, order);
}
@Operation(summary = "更新网站设置")
@PutMapping("/website/setting/update")
@Permission("website:set:update")
@LogOperatingFun
public ResponseResult<Object> update(@RequestBody WebsiteSettingDO websiteSettingDO) {
return settingService.websiteUpdate(websiteSettingDO);
}
}

View File

@ -0,0 +1,13 @@
package com.jmal.clouddisk.controller.sse;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
public class Message {
private String username;
private Object body;
private String url;
private Long space;
}

View File

@ -0,0 +1,179 @@
package com.jmal.clouddisk.controller.sse;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.extern.slf4j.Slf4j;
import org.jetbrains.annotations.NotNull;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
import java.io.IOException;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArraySet;
@RestController
@Slf4j
@Tag(name = "SSE")
public class SseController {
private static final long SSE_TIMEOUT = 30 * 60 * 1000L; // 30分钟
private static final int MAX_CONNECTIONS_PER_USER = 128;
/**
* SseEmitter 容器
* key: uuid (客户端唯一标识)
* value: SseEmitter
*/
private final Map<String, SseEmitter> emitters = new ConcurrentHashMap<>();
/**
* 用户连接映射
* key: username
* value: 该用户所有连接的 uuid 集合
*/
private final Map<String, Set<String>> userConnections = new ConcurrentHashMap<>();
/**
* uuid到用户的反向映射,用于高效清理
* key: uuid
* value: username
*/
private final Map<String, String> uuidToUser = new ConcurrentHashMap<>();
@GetMapping(value = "/events", produces = "text/event-stream")
public SseEmitter events(@RequestParam String username, @RequestParam String uuid) {
// 使用 computeIfAbsent 保证原子性操作,为用户创建连接集合
Set<String> uuids = userConnections.computeIfAbsent(username, k -> new CopyOnWriteArraySet<>());
// 检查用户连接数是否超限
if (uuids.size() >= MAX_CONNECTIONS_PER_USER) {
log.warn("User '{}' has reached the maximum connection limit ({})", username, MAX_CONNECTIONS_PER_USER);
// 可以返回一个立即完成的 SseEmitter 并携带错误信息
SseEmitter errorEmitter = new SseEmitter();
try {
errorEmitter.send(SseEmitter.event().name("error").data("Connection limit exceeded"));
errorEmitter.complete();
} catch (IOException e) {
// ignore
}
return errorEmitter;
}
SseEmitter emitter = getSseEmitter(uuid);
// 存储连接信息
uuids.add(uuid);
uuidToUser.put(uuid, username);
emitters.put(uuid, emitter);
log.debug("Client connected: user='{}', uuid='{}'. Total connections for user: {}", username, uuid, uuids.size());
// 连接成功后可以立即发送一条欢迎消息
try {
emitter.send(SseEmitter.event().name("connected").data("Connection established"));
} catch (IOException e) {
log.debug("Failed to send initial connection message to {}: {}", uuid, e.getMessage());
this.removeClient(uuid);
}
return emitter;
}
@NotNull
private SseEmitter getSseEmitter(String uuid) {
SseEmitter emitter = new SseEmitter(SSE_TIMEOUT);
// 注册回调,统一处理连接关闭的所有情况(完成、超时、错误)
emitter.onCompletion(() -> {
log.debug("SseEmitter completed for uuid: {}", uuid);
this.removeClient(uuid);
});
emitter.onTimeout(() -> {
log.debug("SseEmitter timed out for uuid: {}", uuid);
// SseEmitter 超时会自动触发 onCompletion,所以这里只需要日志记录,无需重复调用 removeClient
// 如果希望超时后立即完成,可以调用 emitter.complete(),它会触发 onCompletion
});
emitter.onError(throwable -> {
log.debug("SseEmitter error for uuid: {}. Error: {}", uuid, throwable.getMessage());
// onError 发生后,Spring 也会确保 onCompletion 被调用,因此也无需重复调用 removeClient
});
return emitter;
}
@PostMapping("/send")
public void sendEvent(@RequestBody Message message) {
String username = message.getUsername();
Set<String> uuids = userConnections.get(username);
if (uuids != null && !uuids.isEmpty()) {
// 遍历时复制集合,防止在发送过程中集合被修改
for (String uuid : Set.copyOf(uuids)) {
sendMessage(uuid, message);
}
}
}
private void sendMessage(String uuid, Object message) {
SseEmitter emitter = emitters.get(uuid);
if (emitter != null) {
try {
emitter.send(SseEmitter.event().data(message));
} catch (Exception e) {
// 发送失败,不需要手动清理,因为 onError 和 onCompletion 回调会处理
log.debug("Failed to send message to client {}. It might be disconnected. Error: {}", uuid, e.getMessage());
}
}
}
/**
* 统一的客户端移除方法,保证线程安全和数据一致性
* @param uuid 客户端唯一标识
*/
private void removeClient(String uuid) {
// 1. 移除 emitter
emitters.remove(uuid);
// 2. 移除 uuid 到 username 的映射
String username = uuidToUser.remove(uuid);
// 3. 从用户的连接集合中移除 uuid
if (username != null) {
Set<String> uuids = userConnections.get(username);
if (uuids != null) {
uuids.remove(uuid);
// 4. 如果用户已无任何连接,则从 userConnections 中移除该用户
if (uuids.isEmpty()) {
userConnections.remove(username);
log.debug("User '{}' has no active connections, removed from tracking.", username);
}
}
}
log.debug("Client disconnected and cleaned up: uuid='{}'", uuid);
}
/**
* 每30秒发送一次心跳,以保持连接活跃并检测断开的客户端
*/
@Scheduled(fixedRate = 30000)
public void heartbeat() {
if (emitters.isEmpty()) {
return;
}
log.trace("Sending heartbeat to {} clients...", emitters.size());
// 遍历所有 emitter 发送心跳
for (Map.Entry<String, SseEmitter> entry : emitters.entrySet()) {
String uuid = entry.getKey();
SseEmitter emitter = entry.getValue();
try {
// 发送一个 SSE 注释行,这不会在客户端触发 "message" 事件
emitter.send(SseEmitter.event().comment("heartbeat"));
} catch (Exception e) {
// 发送失败意味着客户端已断开
// 无需在此处执行清理,SseEmitter 的 onError/onCompletion 回调会负责
log.debug("Heartbeat failed for client {}, will be removed by callbacks.", uuid);
}
}
}
}

View File

@ -0,0 +1,7 @@
package com.jmal.clouddisk.exception;
@FunctionalInterface
public interface CheckedFunction<T,R> {
R apply(T t) throws Exception;
}

View File

@ -0,0 +1,56 @@
package com.jmal.clouddisk.exception;
import lombok.Getter;
import org.springframework.data.annotation.Transient;
import java.io.Serial;
/**
* @Description 通用异常处理类
* @Author jmal
* @Date 2019-08-15 10:54
* @author jmal
*/
@Getter
public class CommonException extends RuntimeException {
/**
*
*/
@Serial
private static final long serialVersionUID = 1L;
private final int code;
private final String msg;
@Transient
private final Object data;
public CommonException(int code, String msg) {
this.code = code;
this.msg = msg;
this.data = null;
}
public CommonException(ExceptionType type, Object data) {
this.code = type.getCode();
this.msg = type.getMsg();
this.data = data;
}
public CommonException(ExceptionType type) {
this.code = type.getCode();
this.msg = type.getMsg();
this.data = null;
}
public CommonException(String msg) {
this.code = -1;
this.msg = msg;
this.data = null;
}
public String getMessage() {
return msg;
}
}

View File

@ -0,0 +1,100 @@
package com.jmal.clouddisk.exception;
import com.jmal.clouddisk.util.ResponseResult;
import com.jmal.clouddisk.util.ResultUtil;
import lombok.extern.slf4j.Slf4j;
import org.apache.catalina.connector.ClientAbortException;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.validation.BindException;
import org.springframework.web.ErrorResponse;
import org.springframework.web.ErrorResponseException;
import org.springframework.web.bind.MethodArgumentNotValidException;
import org.springframework.web.bind.MissingServletRequestParameterException;
import org.springframework.web.bind.annotation.ControllerAdvice;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.context.request.async.AsyncRequestNotUsableException;
import org.springframework.web.context.request.async.AsyncRequestTimeoutException;
import org.springframework.web.servlet.resource.NoResourceFoundException;
import java.io.IOException;
/**
* 统一异常处理
*
* @author jmal
*/
@ControllerAdvice
@Slf4j
public class CommonExceptionHandler {
@ExceptionHandler(IOException.class)
public ResponseEntity<String> handleIOException(IOException e) {
if ("Broken pipe".equals(e.getMessage())) {
// 不记录日志或仅记录简要信息
return ResponseEntity.status(HttpStatus.NO_CONTENT).build();
}
// 记录其他I/O异常
log.error("Unhandled IOException: {}", e.getMessage(), e);
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(e.getMessage());
}
@ExceptionHandler(AsyncRequestNotUsableException.class)
public ResponseEntity<String> handleAsyncRequestNotUsableException(IOException e) {
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(e.getMessage());
}
@ExceptionHandler(AsyncRequestTimeoutException.class)
public ResponseEntity<String> handleAsyncRequestTimeoutException() {
return ResponseEntity.status(HttpStatus.NO_CONTENT).build();
}
@ExceptionHandler(ClientAbortException.class)
public ResponseEntity<String> handleClientAbortException() {
return ResponseEntity.status(HttpStatus.NO_CONTENT).build();
}
@ExceptionHandler(Exception.class)
@ResponseBody
public ResponseResult<Object> exceptionHandler(Exception e) {
return ResultUtil.error(ExceptionType.SYSTEM_ERROR.getCode(), e.getMessage());
}
@ExceptionHandler(CommonException.class)
@ResponseBody
public ResponseResult<Object> exceptionHandler(CommonException e) {
if (e.getCode() == 0) {
return ResultUtil.success(e.getData());
}
return ResultUtil.error(e.getCode(), e.getMsg());
}
@ExceptionHandler(MissingServletRequestParameterException.class)
@ResponseBody
public ResponseResult<Object> exceptionHandler(MissingServletRequestParameterException e) {
return ResultUtil.error(ExceptionType.MISSING_PARAMETERS.getCode(), "缺少参数%s".formatted(e.getParameterName()));
}
@ExceptionHandler(MethodArgumentNotValidException.class)
@ResponseBody
public ResponseResult<Object> exceptionHandler(MethodArgumentNotValidException e) {
return ResultUtil.error(ExceptionType.MISSING_PARAMETERS.getCode(), e.getBindingResult().getAllErrors().get(0).getDefaultMessage());
}
@ExceptionHandler(BindException.class)
@ResponseBody
public ResponseResult<Object> exceptionHandler(BindException e) {
return ResultUtil.error(ExceptionType.MISSING_PARAMETERS.getCode(), e.getBindingResult().getAllErrors().get(0).getDefaultMessage());
}
@ExceptionHandler(NoResourceFoundException.class)
@ResponseBody
@ResponseStatus(HttpStatus.NOT_FOUND)
public ErrorResponse handleNoResourceException() {
return new ErrorResponseException(HttpStatus.NOT_FOUND);
}
}

View File

@ -0,0 +1,70 @@
package com.jmal.clouddisk.exception;
import java.util.Optional;
import java.util.function.Function;
public class Either<L, R> {
private final L left;
private final R right;
private Either(L left, R right) {
this.left = left;
this.right = right;
}
public static <T,R> Function<T,R> wrap(CheckedFunction<T,R> checkedFunction) {
return t -> {
try {
return checkedFunction.apply(t);
} catch (Exception e) {
throw new RuntimeException(e);
}
};
}
public static <L, R> Either<L, R> Left(L value) {
return new Either<L, R>(value, null);
}
public static <L, R> Either<L, R> Right(R value) {
return new Either<L, R>(null, value);
}
public Optional<L> getLeft() {
return Optional.ofNullable(left);
}
public Optional<R> getRight() {
return Optional.ofNullable(right);
}
public boolean isLeft() {
return left != null;
}
public boolean isRight() {
return right != null;
}
public <T> Optional<T> mapLeft(Function<? super L, T> mapper) {
if (isLeft()) {
return Optional.of(mapper.apply(left));
}
return Optional.empty();
}
public <T> Optional<T> mapRight(Function<? super R, T> mapper) {
if (isRight()) {
return Optional.of(mapper.apply(right));
}
return Optional.empty();
}
@Override
public String toString() {
if (isLeft()) {
return "Left(" + left + ")";
}
return "Right(" + right + ")";
}
}

View File

@ -0,0 +1,142 @@
package com.jmal.clouddisk.exception;
/**
* @Description 异常枚举
* @Date 2019-08-16 14:20
* @author jmal
*/
public enum ExceptionType {
/***
* 警告
*/
WARNING(-2, ""),
/***
* 其他异常
*/
SYSTEM_ERROR(-1, "其他异常"),
/***
* 成功
*/
SYSTEM_SUCCESS(0, "true"),
/***
* 缺少参数
*/
MISSING_PARAMETERS(1, "缺少参数"),
/***
* 时间格式不正确
*/
INCORRECT_DATE(2, "时间格式不正确"),
/***
* 缺少Header
*/
MISSING_HEADERS(3, "缺少Header"),
/***
* 该资源已存在
*/
EXISTING_RESOURCES(4, "该资源已存在"),
/***
* 未登录或登录超时
*/
LOGIN_EXCEPTION(5, "未登录或登录超时"),
/***
* 分享码错误
*/
SHARE_CODE_EXCEPTION(6, "分享码错误"),
/***
* 未验证分享码
*/
SHARE_CODE_VALID(21, "未验证分享码"),
/***
* 文件不存在
*/
FILE_NOT_FIND(7, "文件不存在"),
/**
* 没有权限
*/
PERMISSION_DENIED(8, "没有权限"),
/**
* 自定义异常
*/
CUSTOM_EXCEPTION(9, "自定义异常"),
/**
* 参数值不对
*/
PARAMETERS_VALUE(10, "参数值不对"),
/**
* 参数值不对
*/
OFFLINE(11, "离线"),
/**
* 用户不存在
*/
USER_NOT_FIND(12, "用户不存在"),
/***
* 目录不存在
*/
DIR_NOT_FIND(13, "目录不存在"),
/***
* 解压失败
*/
FAIL_DECOMPRESS(14, "解压失败"),
/***
* 无法识别的文件
*/
UNRECOGNIZED_FILE(15, "无法识别的文件"),
/***
* 合并文件失败
*/
FAIL_MERGE_FILE(16, "合并文件失败"),
/***
* 删除文件失败
*/
FAIL_DELETE_FILE(17, "删除文件失败"),
/***
* 上传文件失败
*/
FAIL_UPLOAD_FILE(18, "上传文件失败"),
/***
* 禁止访问
*/
ACCESS_FORBIDDEN(19, "禁止访问"),
/***
* 空间已满
*/
SPACE_FULL(20, "空间已满"),
/***
* 资源被锁定
*/
LOCKED_RESOURCES(21, "该资源被锁定");
public int getCode() {
return code;
}
public String getMsg() {
return msg;
}
private final int code;
private final String msg;
ExceptionType(int code, String msg) {
this.code = code;
this.msg = msg;
}
}

View File

@ -0,0 +1,57 @@
package com.jmal.clouddisk.ftp;
import cn.hutool.extra.ftp.SimpleFtpServer;
import com.jmal.clouddisk.config.FileProperties;
import com.jmal.clouddisk.model.rbac.ConsumerDTO;
import com.jmal.clouddisk.service.IUserService;
import jakarta.annotation.PostConstruct;
import lombok.extern.slf4j.Slf4j;
import org.apache.ftpserver.usermanager.impl.BaseUser;
import org.springframework.stereotype.Component;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
/**
* @author jmal
* @Description FTPServer 配置
* @date 2023/2/21 17:17
*/
@Component
@Slf4j
public class FTPServerConfig {
private final FileProperties fileProperties;
private final IUserService userService;
private final MyPropertiesUserManager myPropertiesUserManager;
public FTPServerConfig(FileProperties fileProperties, IUserService userService, MyPropertiesUserManager myPropertiesUserManager) {
this.fileProperties = fileProperties;
this.userService = userService;
this.myPropertiesUserManager = myPropertiesUserManager;
}
@PostConstruct
public void startFTPServer() {
List<ConsumerDTO> userList = userService.userListAll();
SimpleFtpServer ftpServer = SimpleFtpServer.create();
ftpServer.setUserManager(myPropertiesUserManager);
ftpServer.setPort(fileProperties.getFtpServerPort());
userList.forEach(consumerDTO -> {
BaseUser user = new BaseUser();
user.setName(consumerDTO.getUsername());
Path path = Paths.get(fileProperties.getRootDir(), consumerDTO.getUsername());
user.setHomeDirectory(path.toString());
ftpServer.addUser(user);
});
ftpServer.start();
log.info("FTP server port: {}", fileProperties.getFtpServerPort());
}
}

View File

@ -0,0 +1,476 @@
package com.jmal.clouddisk.ftp;
import com.jmal.clouddisk.service.impl.UserServiceImpl;
import com.jmal.clouddisk.util.PasswordHash;
import lombok.extern.slf4j.Slf4j;
import org.apache.ftpserver.FtpServerConfigurationException;
import org.apache.ftpserver.ftplet.*;
import org.apache.ftpserver.usermanager.AnonymousAuthentication;
import org.apache.ftpserver.usermanager.Md5PasswordEncryptor;
import org.apache.ftpserver.usermanager.PropertiesUserManagerFactory;
import org.apache.ftpserver.usermanager.UsernamePasswordAuthentication;
import org.apache.ftpserver.usermanager.impl.*;
import org.apache.ftpserver.util.BaseProperties;
import org.apache.ftpserver.util.IoUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.io.*;
import java.net.URL;
import java.util.*;
/**
* <strong>Internal class, do not use directly.</strong>
*
* <p>Properties file based <code>UserManager</code> implementation. We use
* <code>user.properties</code> file to store user data.</p>
*
* </p>The file will use the following properties for storing users:</p>
* <table>
* <tr>
* <th>Property</th>
* <th>Documentation</th>
* </tr>
* <tr>
* <td>ftpserver.user.{username}.homedirectory</td>
* <td>Path to the home directory for the user, based on the file system implementation used</td>
* </tr>
* <tr>
* <td>ftpserver.user.{username}.userpassword</td>
* <td>The password for the user. Can be in clear text, MD5 hash or salted SHA hash based on the
* configuration on the user manager
* </td>
* </tr>
* <tr>
* <td>ftpserver.user.{username}.enableflag</td>
* <td>true if the user is enabled, false otherwise</td>
* </tr>
* <tr>
* <td>ftpserver.user.{username}.writepermission</td>
* <td>true if the user is allowed to upload files and create directories, false otherwise</td>
* </tr>
* <tr>
* <td>ftpserver.user.{username}.idletime</td>
* <td>The number of seconds the user is allowed to be idle before disconnected.
* 0 disables the idle timeout
* </td>
* </tr>
* <tr>
* <td>ftpserver.user.{username}.maxloginnumber</td>
* <td>The maximum number of concurrent logins by the user. 0 disables the check.</td>
* </tr>
* <tr>
* <td>ftpserver.user.{username}.maxloginperip</td>
* <td>The maximum number of concurrent logins from the same IP address by the user. 0 disables the check.</td>
* </tr>
* <tr>
* <td>ftpserver.user.{username}.uploadrate</td>
* <td>The maximum number of bytes per second the user is allowed to upload files. 0 disables the check.</td>
* </tr>
* <tr>
* <td>ftpserver.user.{username}.downloadrate</td>
* <td>The maximum number of bytes per second the user is allowed to download files. 0 disables the check.</td>
* </tr>
* </table>
*
* <p>Example:</p>
* <pre>
* ftpserver.user.admin.homedirectory=/ftproot
* ftpserver.user.admin.userpassword=admin
* ftpserver.user.admin.enableflag=true
* ftpserver.user.admin.writepermission=true
* ftpserver.user.admin.idletime=0
* ftpserver.user.admin.maxloginnumber=0
* ftpserver.user.admin.maxloginperip=0
* ftpserver.user.admin.uploadrate=0
* ftpserver.user.admin.downloadrate=0
* </pre>
*
* @author <a href="http://mina.apache.org">Apache MINA Project</a>
*/
@Component
@Slf4j
public class MyPropertiesUserManager extends AbstractUserManager {
private final static String PREFIX = "ftpserver.user.";
private BaseProperties userDataProp;
private File userDataFile;
private URL userUrl;
@Autowired
private UserServiceImpl userService;
/**
* Internal constructor, do not use directly. Use {@link PropertiesUserManagerFactory} instead.
*/
public MyPropertiesUserManager() {
super("admin", new Md5PasswordEncryptor());
userDataProp = new BaseProperties();
}
private void loadFromFile(File userDataFile) {
try {
userDataProp = new BaseProperties();
if (userDataFile != null) {
log.debug("File configured, will try loading");
if (userDataFile.exists()) {
this.userDataFile = userDataFile;
log.debug("File found on file system");
FileInputStream fis = null;
try {
fis = new FileInputStream(userDataFile);
userDataProp.load(fis);
} finally {
IoUtils.close(fis);
}
} else {
// try loading it from the classpath
log
.debug("File not found on file system, try loading from classpath");
InputStream is = getClass().getClassLoader()
.getResourceAsStream(userDataFile.getPath());
if (is != null) {
try {
userDataProp.load(is);
} finally {
IoUtils.close(is);
}
} else {
throw new FtpServerConfigurationException(
"User data file specified but could not be located, "
+ "neither on the file system or in the classpath: "
+ userDataFile.getPath());
}
}
}
} catch (IOException e) {
throw new FtpServerConfigurationException(
"Error loading user data file : " + userDataFile, e);
}
}
private void loadFromUrl(URL userDataPath) {
try {
userDataProp = new BaseProperties();
if (userDataPath != null) {
log.debug("URL configured, will try loading");
userUrl = userDataPath;
InputStream is = null;
is = userDataPath.openStream();
try {
userDataProp.load(is);
} finally {
IoUtils.close(is);
}
}
} catch (IOException e) {
throw new FtpServerConfigurationException(
"Error loading user data resource : " + userDataPath, e);
}
}
/**
* Reloads the contents of the user.properties file. This allows any manual modifications to the file to be recognised by the running server.
*/
public void refresh() {
synchronized (userDataProp) {
if (userDataFile != null) {
log.debug("Refreshing user manager using file: "
+ userDataFile.getAbsolutePath());
loadFromFile(userDataFile);
} else {
//file is null, must have been created using URL
log.debug("Refreshing user manager using URL: "
+ userUrl.toString());
loadFromUrl(userUrl);
}
}
}
/**
* Retrive the file backing this user manager
*
* @return The file
*/
public File getFile() {
return userDataFile;
}
/**
* Save user data. Store the properties.
*/
public synchronized void save(User usr) throws FtpException {
// null value check
if (usr.getName() == null) {
throw new NullPointerException("User name is null.");
}
String thisPrefix = PREFIX + usr.getName() + '.';
// set other properties
userDataProp.setProperty(thisPrefix + ATTR_PASSWORD, getPassword(usr));
String home = usr.getHomeDirectory();
if (home == null) {
home = "/";
}
userDataProp.setProperty(thisPrefix + ATTR_HOME, home);
userDataProp.setProperty(thisPrefix + ATTR_ENABLE, usr.getEnabled());
userDataProp.setProperty(thisPrefix + ATTR_WRITE_PERM, usr
.authorize(new WriteRequest()) != null);
userDataProp.setProperty(thisPrefix + ATTR_MAX_IDLE_TIME, usr
.getMaxIdleTime());
TransferRateRequest transferRateRequest = new TransferRateRequest();
transferRateRequest = (TransferRateRequest) usr
.authorize(transferRateRequest);
if (transferRateRequest != null) {
userDataProp.setProperty(thisPrefix + ATTR_MAX_UPLOAD_RATE,
transferRateRequest.getMaxUploadRate());
userDataProp.setProperty(thisPrefix + ATTR_MAX_DOWNLOAD_RATE,
transferRateRequest.getMaxDownloadRate());
} else {
userDataProp.remove(thisPrefix + ATTR_MAX_UPLOAD_RATE);
userDataProp.remove(thisPrefix + ATTR_MAX_DOWNLOAD_RATE);
}
// request that always will succeed
ConcurrentLoginRequest concurrentLoginRequest = new ConcurrentLoginRequest(
0, 0);
concurrentLoginRequest = (ConcurrentLoginRequest) usr
.authorize(concurrentLoginRequest);
if (concurrentLoginRequest != null) {
userDataProp.setProperty(thisPrefix + ATTR_MAX_LOGIN_NUMBER,
concurrentLoginRequest.getMaxConcurrentLogins());
userDataProp.setProperty(thisPrefix + ATTR_MAX_LOGIN_PER_IP,
concurrentLoginRequest.getMaxConcurrentLoginsPerIP());
} else {
userDataProp.remove(thisPrefix + ATTR_MAX_LOGIN_NUMBER);
userDataProp.remove(thisPrefix + ATTR_MAX_LOGIN_PER_IP);
}
saveUserData();
}
/**
* @throws FtpException
*/
private void saveUserData() throws FtpException {
if (userDataFile == null) {
return;
}
File dir = userDataFile.getAbsoluteFile().getParentFile();
if (dir != null && !dir.exists() && !dir.mkdirs()) {
String dirName = dir.getAbsolutePath();
throw new FtpServerConfigurationException(
"Cannot create directory for user data file : " + dirName);
}
// save user data
FileOutputStream fos = null;
try {
fos = new FileOutputStream(userDataFile);
userDataProp.store(fos, "Generated file - don't edit (please)");
} catch (IOException ex) {
log.error("Failed saving user data", ex);
throw new FtpException("Failed saving user data", ex);
} finally {
IoUtils.close(fos);
}
}
/**
* Delete an user. Removes all this user entries from the properties. After
* removing the corresponding from the properties, save the data.
*/
public void delete(String usrName) throws FtpException {
// remove entries from properties
String thisPrefix = PREFIX + usrName + '.';
Enumeration<?> propNames = userDataProp.propertyNames();
ArrayList<String> remKeys = new ArrayList<String>();
while (propNames.hasMoreElements()) {
String thisKey = propNames.nextElement().toString();
if (thisKey.startsWith(thisPrefix)) {
remKeys.add(thisKey);
}
}
Iterator<String> remKeysIt = remKeys.iterator();
while (remKeysIt.hasNext()) {
userDataProp.remove(remKeysIt.next());
}
saveUserData();
}
/**
* Get user password. Returns the encrypted value.
*
* <pre>
* If the password value is not null
* password = new password
* else
* if user does exist
* password = old password
* else
* password = &quot;&quot;
* </pre>
*/
private String getPassword(User usr) {
String name = usr.getName();
String password = usr.getPassword();
if (password != null) {
password = getPasswordEncryptor().encrypt(password);
} else {
String blankPassword = getPasswordEncryptor().encrypt("");
if (doesExist(name)) {
String key = PREFIX + name + '.' + ATTR_PASSWORD;
password = userDataProp.getProperty(key, blankPassword);
} else {
password = blankPassword;
}
}
return password;
}
/**
* Get all user names.
*/
public String[] getAllUserNames() {
// get all user names
String suffix = '.' + ATTR_HOME;
ArrayList<String> ulst = new ArrayList<String>();
Enumeration<?> allKeys = userDataProp.propertyNames();
int prefixlen = PREFIX.length();
int suffixlen = suffix.length();
while (allKeys.hasMoreElements()) {
String key = (String) allKeys.nextElement();
if (key.endsWith(suffix)) {
String name = key.substring(prefixlen);
int endIndex = name.length() - suffixlen;
name = name.substring(0, endIndex);
ulst.add(name);
}
}
Collections.sort(ulst);
return ulst.toArray(new String[0]);
}
/**
* Load user data.
*/
public User getUserByName(String userName) {
if (!doesExist(userName)) {
return null;
}
String baseKey = PREFIX + userName + '.';
BaseUser user = new BaseUser();
user.setName(userName);
user.setEnabled(userDataProp.getBoolean(baseKey + ATTR_ENABLE, true));
user.setHomeDirectory(userDataProp
.getProperty(baseKey + ATTR_HOME, "/"));
List<Authority> authorities = new ArrayList<Authority>();
if (userDataProp.getBoolean(baseKey + ATTR_WRITE_PERM, false)) {
authorities.add(new WritePermission());
}
int maxLogin = userDataProp.getInteger(baseKey + ATTR_MAX_LOGIN_NUMBER,
0);
int maxLoginPerIP = userDataProp.getInteger(baseKey
+ ATTR_MAX_LOGIN_PER_IP, 0);
authorities.add(new ConcurrentLoginPermission(maxLogin, maxLoginPerIP));
int uploadRate = userDataProp.getInteger(
baseKey + ATTR_MAX_UPLOAD_RATE, 0);
int downloadRate = userDataProp.getInteger(baseKey
+ ATTR_MAX_DOWNLOAD_RATE, 0);
authorities.add(new TransferRatePermission(downloadRate, uploadRate));
user.setAuthorities(authorities);
user.setMaxIdleTime(userDataProp.getInteger(baseKey
+ ATTR_MAX_IDLE_TIME, 0));
return user;
}
/**
* User existance check
*/
public boolean doesExist(String name) {
String key = PREFIX + name + '.' + ATTR_HOME;
return userDataProp.containsKey(key);
}
/**
* User authenticate method
*/
public User authenticate(Authentication authentication)
throws AuthenticationFailedException {
if (authentication instanceof UsernamePasswordAuthentication path) {
String user = path.getUsername();
String password = path.getPassword();
if (user == null) {
throw new AuthenticationFailedException("Authentication failed");
}
if (password == null) {
password = "";
}
String hashPassword = userService.getHashPasswordUserName(user);
if (PasswordHash.validatePassword(password, hashPassword)) {
return getUserByName(user);
} else {
throw new AuthenticationFailedException("Authentication failed");
}
} else if (authentication instanceof AnonymousAuthentication) {
if (doesExist("anonymous")) {
return getUserByName("anonymous");
} else {
throw new AuthenticationFailedException("Authentication failed");
}
} else {
throw new IllegalArgumentException(
"Authentication not supported by this user manager");
}
}
/**
* Close the user manager - remove existing entries.
*/
public synchronized void dispose() {
if (userDataProp != null) {
userDataProp.clear();
userDataProp = null;
}
}
}

View File

@ -0,0 +1,339 @@
package com.jmal.clouddisk.interceptor;
import cn.hutool.core.text.CharSequenceUtil;
import com.alibaba.fastjson2.JSON;
import com.jmal.clouddisk.exception.ExceptionType;
import com.jmal.clouddisk.model.UserAccessTokenDO;
import com.jmal.clouddisk.model.rbac.UserLoginContext;
import com.jmal.clouddisk.repository.IAuthDAO;
import com.jmal.clouddisk.service.IUserService;
import com.jmal.clouddisk.service.impl.UserServiceImpl;
import com.jmal.clouddisk.util.CaffeineUtil;
import com.jmal.clouddisk.util.ResponseResult;
import com.jmal.clouddisk.util.ResultUtil;
import com.jmal.clouddisk.util.TokenUtil;
import io.reactivex.rxjava3.core.Completable;
import io.reactivex.rxjava3.schedulers.Schedulers;
import jakarta.servlet.ServletOutputStream;
import jakarta.servlet.http.Cookie;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import lombok.extern.slf4j.Slf4j;
import org.jetbrains.annotations.NotNull;
import org.springframework.stereotype.Component;
import org.springframework.web.context.request.RequestContextHolder;
import org.springframework.web.context.request.ServletRequestAttributes;
import org.springframework.web.servlet.HandlerInterceptor;
import java.io.IOException;
import java.time.LocalDateTime;
import java.util.List;
/**
* @author jmal
* @Description 鉴权拦截器
* @Date 2020-01-31 22:04
*/
@Slf4j
@Component
public class AuthInterceptor implements HandlerInterceptor {
public static final String JMAL_TOKEN = "jmal-token";
public static final String REMEMBER_NAME = "rememberName";
public static final String NAME_HEADER = "name";
public static final String ACCESS_TOKEN = "access-token";
public static final String REFRESH_TOKEN = "refresh-token";
private static final int TWO_HOURS_IN_SECONDS = 2 * 60 * 60; // 7200
private static final int SECONDS_IN_DAY = 24 * 60 * 60; // 86400
private static final int NINETY_DAYS_IN_SECONDS = 90 * SECONDS_IN_DAY; // 7776000
private final IAuthDAO authDAO;
private final UserServiceImpl userService;
public AuthInterceptor(IAuthDAO authDAO, UserServiceImpl userService) {
this.authDAO = authDAO;
this.userService = userService;
}
@Override
public boolean preHandle(@NotNull HttpServletRequest request, @NotNull HttpServletResponse response, @NotNull Object handler) {
// 身份认证
String username = getUserNameByHeader(request, response);
if (!CharSequenceUtil.isBlank(username)) {
// jmal-token 身份认证通过, 设置该身份的权限
setAuthorities(username);
return true;
}
returnJson(response);
return false;
}
/***
* 根据header获取用户名
* @param request request
* @return 用户名
*/
public String getUserNameByHeader(HttpServletRequest request, HttpServletResponse response) {
String name = request.getHeader(NAME_HEADER);
String jmalToken = request.getHeader(JMAL_TOKEN);
if (CharSequenceUtil.isBlank(jmalToken)) {
jmalToken = request.getParameter(JMAL_TOKEN);
}
if (CharSequenceUtil.isBlank(name)) {
name = request.getParameter(NAME_HEADER);
}
if (CharSequenceUtil.isBlank(name)) {
name = request.getParameter(IUserService.USERNAME);
}
if (CharSequenceUtil.isBlank(jmalToken)) {
jmalToken = getCookie(request, JMAL_TOKEN);
}
if (CharSequenceUtil.isBlank(name)) {
name = getCookie(request, IUserService.USERNAME);
}
if (CharSequenceUtil.isBlank(jmalToken)) {
String refreshToken = getCookie(request, REFRESH_TOKEN);
if (CharSequenceUtil.isNotBlank(refreshToken)) {
// 自动续token
return renewJmalToken(request, response, refreshToken);
}
return getUserNameByAccessToken(request);
}
return getUserNameByJmalToken(request, response, jmalToken, name);
}
/***
* jmal-token 身份认证通过, 设置该身份的权限
* @param username username
*/
public void setAuthorities(String username) {
List<String> authorities = CaffeineUtil.getAuthoritiesCache(username);
if (authorities == null || authorities.isEmpty()) {
authorities = userService.getAuthorities(username);
CaffeineUtil.setAuthoritiesCache(username, authorities);
}
setAuthorities(username, authorities);
}
/***
* 设置用户登录信息
* access-token, jmal-token 通用
* @param username username
* @param authorities 权限标示列表
*/
private void setAuthorities(String username, List<String> authorities) {
ServletRequestAttributes requestAttributes = (ServletRequestAttributes) RequestContextHolder.getRequestAttributes();
if (requestAttributes != null) {
String userId = CaffeineUtil.getUserIdCache(username);
if (CharSequenceUtil.isBlank(userId)) {
userId = userService.getUserIdByUserName(username);
CaffeineUtil.setUserIdCache(username, userId);
}
UserLoginContext userLoginContext = new UserLoginContext();
userLoginContext.setAuthorities(authorities);
userLoginContext.setUserId(userId);
userLoginContext.setUsername(username);
requestAttributes.setAttribute("user", userLoginContext, 0);
}
}
/***
* 根据access-token获取用户名
* @param request request
* @return 用户名
*/
public String getUserNameByAccessToken(HttpServletRequest request) {
String token = request.getHeader(ACCESS_TOKEN);
if (CharSequenceUtil.isBlank(token)) {
token = request.getParameter(ACCESS_TOKEN);
}
if (CharSequenceUtil.isBlank(token)) {
return null;
}
UserAccessTokenDO userAccessTokenDO = authDAO.getUserNameByAccessToken(token);
if (userAccessTokenDO == null) {
return null;
}
String username = userAccessTokenDO.getUsername();
if (CharSequenceUtil.isBlank(username)) {
return null;
}
// access-token 认证通过 设置该身份的权限
Completable.fromAction(() -> authDAO.updateAccessToken(username, userAccessTokenDO.getAccessToken()))
.subscribeOn(Schedulers.io())
.subscribe();
setAuthorities(username);
return userAccessTokenDO.getUsername();
}
/**
* 根据jmal-token获取用户名
*
* @param request HttpServletRequest
* @param response HttpServletResponse
* @param jmalToken jmal-token
* @param name username
* @return username
*/
public String getUserNameByJmalToken(HttpServletRequest request, HttpServletResponse response, String jmalToken, String name) {
if (!CharSequenceUtil.isBlank(jmalToken) && !CharSequenceUtil.isBlank(name)) {
String hashPassword = userService.getHashPasswordUserName(name);
if (hashPassword == null) {
return null;
}
String username = TokenUtil.getTokenKey(jmalToken, hashPassword);
if (username == null && request != null) {
String refreshToken = getCookie(request, REFRESH_TOKEN);
if (CharSequenceUtil.isBlank(refreshToken)) {
return null;
}
// 自动续token
username = renewJmalToken(request, response, name, hashPassword, refreshToken);
}
if (name.equals(username)) {
return username;
}
return null;
}
return null;
}
/**
* 自动续token
*/
private String renewJmalToken(HttpServletRequest request, HttpServletResponse response, String name, String hashPassword, String refreshToken) {
String username = TokenUtil.getTokenKey(refreshToken, hashPassword);
if (name.equals(username)) {
boolean rememberMe = Boolean.parseBoolean(getCookie(request, REMEMBER_NAME));
String jmalToken = generateJmalToken(hashPassword, username);
setRefreshCookie(request, response, hashPassword, username, rememberMe, jmalToken);
return username;
}
return null;
}
private String renewJmalToken(HttpServletRequest request, HttpServletResponse response, String refreshToken) {
String username = TokenUtil.getUsername(refreshToken);
if (CharSequenceUtil.isBlank(username)) {
return null;
}
String hashPassword = userService.getHashPasswordUserName(username);
if (CharSequenceUtil.isBlank(hashPassword)) {
return null;
}
return renewJmalToken(request, response, username, hashPassword, refreshToken);
}
private static void setJmalTokenCookie(HttpServletRequest request, HttpServletResponse response, String username, String jmalToken) {
addHttpOnlyCookie(request, response, JMAL_TOKEN, jmalToken, TWO_HOURS_IN_SECONDS);
addHttpOnlyCookie(request, response, IUserService.USERNAME, username, TWO_HOURS_IN_SECONDS);
response.addHeader(JMAL_TOKEN, jmalToken);
}
/**
* 生成jmal-token
*
* @param hashPassword hashPassword
* @param username username
* @return jmal-token
*/
public static String generateJmalToken(String hashPassword, String username) {
LocalDateTime jmalTokenExpiration = LocalDateTime.now();
// jmal-token 期限为2小时
jmalTokenExpiration = jmalTokenExpiration.plusSeconds(TWO_HOURS_IN_SECONDS);
return TokenUtil.createToken(username, hashPassword, jmalTokenExpiration);
}
/**
* 设置 refreshToken
*
* @param request HttpServletRequest
* @param response HttpServletResponse
* @param hashPassword hashPassword
* @param username username
* @param rememberMe rememberMe
*/
public static void setRefreshCookie(HttpServletRequest request, HttpServletResponse response, String hashPassword, String username, boolean rememberMe, String jmalToken) {
LocalDateTime refreshTokenExpiration = LocalDateTime.now();
// 如果用户勾选了记住我, refreshToken期限为90天, 否则为1天
int refreshMaxAge = rememberMe ? NINETY_DAYS_IN_SECONDS : SECONDS_IN_DAY;
refreshTokenExpiration = refreshTokenExpiration.plusSeconds(refreshMaxAge);
addHttpOnlyCookie(request, response, REFRESH_TOKEN, TokenUtil.createToken(username, hashPassword, refreshTokenExpiration), refreshMaxAge);
addHttpOnlyCookie(request, response, REMEMBER_NAME, String.valueOf(rememberMe), refreshMaxAge);
setJmalTokenCookie(request, response, username, jmalToken);
}
private static void addHttpOnlyCookie(HttpServletRequest request, HttpServletResponse response, String name, String value, int maxAge) {
String scheme = request.getHeader("X-Forwarded-Proto");
if (CharSequenceUtil.isNotBlank(scheme)) {
scheme = scheme.split(",")[0].trim();
} else {
scheme = request.getScheme();
}
boolean isHttps = "https".equalsIgnoreCase(scheme);
Cookie cookie = new Cookie(name, value);
cookie.setMaxAge(maxAge);
cookie.setHttpOnly(true);
cookie.setPath("/");
cookie.setSecure(isHttps);
// SameSite
cookie.setAttribute("SameSite", "Strict");
response.addCookie(cookie);
}
public static void removeCookies(HttpServletResponse response, String ...keys) {
for (String key : keys) {
Cookie cookie = new Cookie(key, null);
cookie.setMaxAge(0);
cookie.setPath("/");
response.addCookie(cookie);
}
}
public static String getCookie(HttpServletRequest request, String name) {
Cookie[] cookies = request.getCookies();
if (cookies != null) {
for (Cookie cookie : cookies) {
if (cookie.getName().equals(name)) {
return cookie.getValue();
}
}
}
return null;
}
private void returnJson(HttpServletResponse response) {
response.setCharacterEncoding("UTF-8");
response.setContentType("application/json; charset=utf-8");
ServletOutputStream out = null;
try {
out = response.getOutputStream();
ResponseResult<Object> result = ResultUtil.error(ExceptionType.LOGIN_EXCEPTION.getCode(), ExceptionType.LOGIN_EXCEPTION.getMsg());
out.write(JSON.toJSONString(result).getBytes());
removeCookies(response, IUserService.USERNAME, AuthInterceptor.JMAL_TOKEN);
} catch (IOException e) {
log.error(e.getMessage(), e);
} finally {
try {
if (out != null) {
out.flush();
}
} catch (IOException e) {
log.error(e.getMessage(), e);
}
}
}
}

View File

@ -0,0 +1,74 @@
package com.jmal.clouddisk.interceptor;
import cn.hutool.core.text.CharSequenceUtil;
import cn.hutool.core.util.BooleanUtil;
import com.jmal.clouddisk.model.FileDocument;
import com.jmal.clouddisk.service.IUserService;
import com.jmal.clouddisk.service.impl.DirectLinkService;
import com.jmal.clouddisk.service.impl.FileServiceImpl;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.jetbrains.annotations.NotNull;
import org.springframework.stereotype.Component;
import org.springframework.web.servlet.HandlerInterceptor;
import java.util.List;
/**
* @author jmal
* @Description
*/
@Slf4j
@Component
@RequiredArgsConstructor
public class DirectFileInterceptor implements HandlerInterceptor {
private final DirectLinkService directLinkService;
private final PreFileInterceptor preFileInterceptor;
private final FileServiceImpl fileService;
private final IUserService userService;
@Override
public boolean preHandle(@NotNull HttpServletRequest request, @NotNull HttpServletResponse response, @NotNull Object handler) {
String uri = request.getRequestURI();
String[] pathSegments = uri.split("/");
// 验证路径格式
if (pathSegments.length != 4 || !"direct-file".equals(pathSegments[1])) {
response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
return false;
}
String mark = pathSegments[2];
String fileId = directLinkService.getFileIdByMark(mark);
if (CharSequenceUtil.isBlank(fileId)) {
response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
return false;
}
FileDocument fileDocument = preFileInterceptor.getFileDocument(response, fileId);
if (fileDocument == null) {
response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
return false;
}
if (BooleanUtil.isTrue(fileDocument.getIsFolder())) {
// 打包下载
String username = userService.getUserNameById(fileDocument.getUserId());
List<String> authorities = userService.getAuthorities(username);
if (!authorities.contains("cloud:file:packageDownload")) {
response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
return false;
}
fileService.packageDownload(request, response, List.of(fileId), username);
return false;
}
// 构造内部路径
preFileInterceptor.internalFilePath(request, response, fileDocument);
// 阻止原始请求继续处理
return false;
}
}

View File

@ -0,0 +1,481 @@
package com.jmal.clouddisk.interceptor;
import cn.hutool.core.convert.Convert;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.net.URLDecoder;
import cn.hutool.core.text.CharSequenceUtil;
import cn.hutool.core.util.BooleanUtil;
import cn.hutool.core.util.URLUtil;
import com.jmal.clouddisk.config.FileProperties;
import com.jmal.clouddisk.model.FileDocument;
import com.jmal.clouddisk.oss.web.WebOssService;
import com.jmal.clouddisk.service.IFileService;
import com.jmal.clouddisk.util.CaffeineUtil;
import com.jmal.clouddisk.util.FileContentTypeUtils;
import com.jmal.clouddisk.util.MyFileUtils;
import com.jmal.clouddisk.util.UrlEncodingChecker;
import com.luciad.imageio.webp.WebPWriteParam;
import jakarta.servlet.ServletOutputStream;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import net.coobird.thumbnailator.Thumbnails;
import net.coobird.thumbnailator.tasks.UnsupportedFormatException;
import org.jetbrains.annotations.NotNull;
import org.springframework.http.HttpHeaders;
import org.springframework.stereotype.Component;
import org.springframework.web.servlet.HandlerInterceptor;
import org.springframework.web.servlet.HandlerMapping;
import javax.imageio.*;
import javax.imageio.stream.FileCacheImageOutputStream;
import javax.imageio.stream.ImageInputStream;
import java.awt.image.BufferedImage;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Iterator;
/**
* @author jmal
* @Description 文件鉴权拦截器
* @Date 2020-01-31 22:04
*/
@Slf4j
@Component
@RequiredArgsConstructor
public class FileInterceptor implements HandlerInterceptor {
/***
* 下载操作
*/
private static final String DOWNLOAD = "download";
/**
* 预览
*/
private static final String PREVIEW = "preview";
/***
* 剪裁图片
*/
private static final String CROP = "crop";
/***
* 缩略图
*/
private static final String THUMBNAIL = "thumbnail";
/***
* webp
*/
private static final String WEBP = "webp";
/***
* 操作参数
*/
private static final String OPERATION = "o";
/***
* fileId参数
*/
private static final String SHARE_KEY = "shareKey";
/***
* 路径最小层级
*/
private static final int MIN_COUNT = 2;
private final FileProperties fileProperties;
private final IFileService fileService;
private final AuthInterceptor authInterceptor;
private final ShareFileInterceptor shareFileInterceptor;
private final WebOssService webOssService;
@Override
public boolean preHandle(@NotNull HttpServletRequest request, @NotNull HttpServletResponse response, @NotNull Object handler) {
if (internalValid(request, response)) return true;
if (fileAuthError(request, response)) {
response.setStatus(HttpServletResponse.SC_FORBIDDEN);
return false;
}
Path path = Paths.get(request.getRequestURI());
String filename = getDownloadFilename(request, path);
String operation = request.getParameter(OPERATION);
if (!CharSequenceUtil.isBlank(operation)) {
switch (operation) {
case DOWNLOAD -> {
response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=\"" + filename + "\"");
if (downloadOssFile(request, response, filename, path)) return false;
}
case PREVIEW -> {
if (previewOssFile(request, response, path)) return false;
}
case CROP -> handleCrop(request, response);
case THUMBNAIL -> thumbnail(request, response);
case WEBP -> webp(request, response);
default -> {
return true;
}
}
} else {
return !previewOssFile(request, response, path);
}
setHeader(request, response);
return true;
}
private void setHeader(@NotNull HttpServletRequest request, @NotNull HttpServletResponse response) {
FileDocument fileDocument = getFileDocument(Paths.get(URLDecoder.decode(request.getRequestURI(), StandardCharsets.UTF_8)));
String contentType = fileDocument.getContentType();
if (CharSequenceUtil.isNotBlank(contentType) && contentType.contains("/")) {
response.setHeader(HttpHeaders.CONTENT_TYPE, fileDocument.getContentType());
}
if (fileDocument.getEtag() != null) {
response.setHeader(HttpHeaders.ETAG, "\"" + fileDocument.getEtag() + "\"");
}
setCacheControl(request, response);
}
/**
* 获取下载文件名, 适配不同的浏览器
*
* @param request request
* @param path path
* @return filename
*/
private static String getDownloadFilename(HttpServletRequest request, Path path) {
String filename = String.valueOf(path.getFileName());
return getFilename(request, filename);
}
private static String getFilename(HttpServletRequest request, String filename) {
if (UrlEncodingChecker.isUrlEncoded(filename)) {
filename = URLUtil.decode(filename, StandardCharsets.UTF_8);
}
String gecko = "Gecko";
String webKit = "WebKit";
String userAgent = request.getHeader("User-Agent");
if (userAgent.contains(gecko) || userAgent.contains(webKit)) {
filename = new String(filename.getBytes(StandardCharsets.UTF_8), StandardCharsets.ISO_8859_1);
} else {
filename = URLUtil.encode(filename, StandardCharsets.UTF_8);
}
return filename;
}
public static void main(String[] args) {
System.out.println(URLUtil.encode("未命名文件 副本.txt", StandardCharsets.UTF_8));
}
private boolean internalValid(@NotNull HttpServletRequest request, @NotNull HttpServletResponse response) {
String requestId = (String) request.getAttribute("requestId");
String internalToken = (String) request.getAttribute("internalToken");
if (CharSequenceUtil.isNotBlank(requestId) && CharSequenceUtil.isNotBlank(internalToken)) {
if (!PreFileInterceptor.isValidInternalTokenCache(requestId, internalToken)) {
response.setStatus(HttpServletResponse.SC_FORBIDDEN);
}
Path path = Paths.get(request.getRequestURI());
String filename = getDownloadFilename(request, path);
if (downloadOssFile(request, response, filename, path)) return false;
setHeader(request, response);
return true;
}
return false;
}
private boolean downloadOssFile(@NotNull HttpServletRequest request, @NotNull HttpServletResponse response, String filename, Path path) {
Path prePth = path.subpath(1, path.getNameCount());
String ossPath = CaffeineUtil.getOssPath(prePth);
if (ossPath != null) {
webOssService.download(ossPath, prePth, request, response, filename);
return true;
}
return false;
}
private void setCacheControl(@NotNull HttpServletRequest request, @NotNull HttpServletResponse response) {
File file = getFileByRequest(request);
if (MyFileUtils.checkNoCacheFile(file)) {
response.setHeader(HttpHeaders.CACHE_CONTROL, "max-age=300");
} else {
response.setHeader(HttpHeaders.CACHE_CONTROL, "max-age=2592000");
}
}
/**
* 预览oss文件
*/
private boolean previewOssFile(@NotNull HttpServletRequest request, @NotNull HttpServletResponse response, Path path) {
Path prePth = path.subpath(1, path.getNameCount());
String ossPath = CaffeineUtil.getOssPath(prePth);
if (CharSequenceUtil.isNotBlank(ossPath)) {
webOssService.download(ossPath, prePth, request, response, null);
return true;
}
return false;
}
/***
* 文件鉴权是否失败
* 当有shareKey参数时说明是分享文件的访问
* shareKey 代表一个分享的文件或文件夹
* 判断当前uri所属的文件是否为该分享的文件或其子文件
* @return true 鉴权失败,false 鉴权成功
*/
private boolean fileAuthError(HttpServletRequest request, HttpServletResponse response) {
String path = (String) request.getAttribute(HandlerMapping.PATH_WITHIN_HANDLER_MAPPING_ATTRIBUTE);
request.setAttribute(HandlerMapping.PATH_WITHIN_HANDLER_MAPPING_ATTRIBUTE, URLDecoder.decode(path, StandardCharsets.UTF_8));
Path uriPath = Paths.get(URLDecoder.decode(request.getRequestURI(), StandardCharsets.UTF_8));
String shareKey = request.getParameter(SHARE_KEY);
if (!CharSequenceUtil.isBlank(shareKey)) {
return validShareFile(request, uriPath, shareKey);
} else {
String username = authInterceptor.getUserNameByHeader(request, response);
if (!CharSequenceUtil.isBlank(username)) {
authInterceptor.setAuthorities(username);
}
int nameCount = uriPath.getNameCount();
if (nameCount < MIN_COUNT) {
return true;
}
if (nameCount == MIN_COUNT && path.startsWith("logo")) {
return false;
}
if (!CharSequenceUtil.isBlank(username) && username.equals(uriPath.getName(1).toString())) {
return false;
}
return isNotAllowAccess(getFileDocument(uriPath), request);
}
}
private boolean validShareFile(HttpServletRequest request, Path uriPath, String shareKey) {
FileDocument fileDocument = fileService.getById(shareKey);
if (!isNotAllowAccess(fileDocument, request)) {
// 判断当前uri所属的文件是否为已分享的文件或其子文件
FileDocument thisFile = getFileDocument(uriPath);
if (thisFile.getPath().equals(fileDocument.getPath())) {
return false;
}
if (Boolean.TRUE.equals(fileDocument.getIsFolder())) {
String parentPath = fileDocument.getPath() + fileDocument.getName();
return !thisFile.getPath().startsWith(parentPath);
}
}
return true;
}
/***
* 判断文件是否不允许访问
* 如果为公共文件,或者分享有效期内的文件,则允许访问
* @return true 不允许访问,false 允许访问
*/
public boolean isNotAllowAccess(FileDocument fileDocument, HttpServletRequest request) {
if (fileDocument == null) {
return true;
}
if (BooleanUtil.isTrue(fileDocument.getIsPublic())) {
return false;
}
// 分享文件
if (BooleanUtil.isTrue(fileDocument.getIsShare())) {
return shareFileInterceptor.validShareFile(fileDocument, null, request);
}
return true;
}
private void webp(HttpServletRequest request, HttpServletResponse response) {
Path uriPath = Paths.get(URLDecoder.decode(request.getRequestURI(), StandardCharsets.UTF_8));
uriPath = uriPath.subpath(1, uriPath.getNameCount());
try {
File file = Paths.get(fileProperties.getRootDir(), uriPath.toString()).toFile();
// 从某处获取图像进行编码
BufferedImage image = ImageIO.read(file);
// 获取一个WebP ImageWriter实例
ImageWriter writer = ImageIO.getImageWritersByMIMEType("image/webp").next();
responseHeader(response, file.getName() + ".webp", new byte[1]);
// 配置编码参数
WebPWriteParam writeParam = new WebPWriteParam(writer.getLocale());
writeParam.setCompressionMode(ImageWriteParam.MODE_DEFAULT);
// 在ImageWriter上配置输出
FileCacheImageOutputStream output = new FileCacheImageOutputStream(response.getOutputStream(), null);
writer.setOutput(output);
// 编码
writer.write(null, new IIOImage(image, null, null), writeParam);
} catch (IOException e) {
log.error(e.getMessage(), e);
}
}
private void thumbnail(HttpServletRequest request, HttpServletResponse response) {
Path uriPath = Paths.get(URLDecoder.decode(request.getRequestURI(), StandardCharsets.UTF_8));
if (uriPath.getNameCount() < MIN_COUNT) {
return;
}
FileDocument fileDocument = getFileDocument(uriPath, false);
Path relativePath = uriPath.subpath(1, uriPath.getNameCount());
if (fileDocument != null) {
if (fileDocument.getContent() == null) {
File file = Paths.get(fileProperties.getRootDir(), relativePath.toString()).toFile();
if (file.exists()) {
fileDocument.setContent(FileUtil.readBytes(file));
}
}
responseWritImage(response, fileDocument.getName(), fileDocument.getContent());
}
}
private FileDocument getFileDocument(Path uriPath, boolean excludeContent) {
String username = uriPath.getName(1).toString();
String path = File.separator;
if (uriPath.getNameCount() > MIN_COUNT + 1) {
path = File.separator + uriPath.subpath(MIN_COUNT, uriPath.getNameCount()).getParent().toString() + File.separator;
}
String name = uriPath.getFileName().toString();
return fileService.getFileDocumentByPathAndName(path, name, username, excludeContent);
}
private FileDocument getFileDocument(Path uriPath) {
return getFileDocument(uriPath, true);
}
private void handleCrop(HttpServletRequest request, HttpServletResponse response) {
File file = getFileByRequest(request);
String q = request.getParameter("q");
String w = request.getParameter("w");
String h = request.getParameter("h");
byte[] img = imageCrop(file, q, w, h);
if (img.length > 0) {
responseWritImage(response, file.getName(), img);
}
}
private @NotNull File getFileByRequest(HttpServletRequest request) {
Path uriPath = Paths.get(URLDecoder.decode(request.getRequestURI(), StandardCharsets.UTF_8));
uriPath = uriPath.subpath(1, uriPath.getNameCount());
return Paths.get(fileProperties.getRootDir(), uriPath.toString()).toFile();
}
private void responseWritImage(HttpServletResponse response, String fileName, byte[] img) {
responseHeader(response, fileName, img);
try (ServletOutputStream outputStream = response.getOutputStream()) {
outputStream.write(img);
} catch (IOException e) {
log.error(e.getMessage(), e);
}
}
private void responseHeader(HttpServletResponse response, String fileName, byte[] img) {
if (!CharSequenceUtil.isBlank(fileName)) {
response.setHeader(HttpHeaders.CONTENT_TYPE, FileContentTypeUtils.getContentType(MyFileUtils.extName(fileName)));
}
if (img != null) {
response.setHeader(HttpHeaders.CONTENT_LENGTH, String.valueOf(img.length));
}
response.setHeader(HttpHeaders.CONNECTION, "close");
response.setHeader(HttpHeaders.CONTENT_ENCODING, "utf-8");
response.setHeader(HttpHeaders.CACHE_CONTROL, "max-age=2592000");
}
/**
* 剪裁图片
*
* @param srcFile 源文件
* @param q 剪裁后的质量
* @param w 剪裁后的宽度
* @param h 剪裁后的高度
* @return 剪裁后的文件
*/
public static byte[] imageCrop(File srcFile, String q, String w, String h) {
ByteArrayOutputStream out = null;
ImageInputStream iis = null;
try {
// 使用ImageInputStream来处理大文件
iis = ImageIO.createImageInputStream(srcFile);
Iterator<ImageReader> readers = ImageIO.getImageReaders(iis);
if (!readers.hasNext()) {
return new byte[0];
}
ImageReader reader = readers.next();
reader.setInput(iis, true);
// 获取图像元数据
BufferedImage bim = reader.read(0);
if (bim == null) {
return new byte[0];
}
int srcWidth = bim.getWidth();
int srcHeight = bim.getHeight();
// 处理质量参数
double quality = parseQuality(q);
int width = parseDimension(w);
int height = parseDimension(h);
Thumbnails.Builder<? extends BufferedImage> thumbnail = Thumbnails.of(bim)
.outputFormat("jpg") // 指定输出格式
.outputQuality(quality);
if (width > 0 && srcWidth > width) {
if (height <= 0 || srcHeight <= height) {
height = (int) (width / (double) srcWidth * srcHeight);
height = height == 0 ? width : height;
}
thumbnail.size(width, height);
} else {
// 宽高均小,指定原大小
thumbnail.size(srcWidth, srcHeight);
}
out = new ByteArrayOutputStream();
thumbnail.toOutputStream(out);
return out.toByteArray();
} catch (UnsupportedFormatException e) {
log.warn(e.getMessage(), e);
} catch (IOException e) {
log.error(e.getMessage(), e);
} finally {
if (out != null) {
try {
out.close();
} catch (IOException e) {
log.error("Error closing ByteArrayOutputStream", e);
}
}
if (iis != null) {
try {
iis.close();
} catch (IOException e) {
log.error("Error closing ImageInputStream", e);
}
}
}
return new byte[0];
}
private static double parseQuality(String q) {
try {
double quality = Convert.toDouble(q, 0.8);
return (quality >= 0 && quality <= 1) ? quality : 0.8;
} catch (NumberFormatException e) {
return 0.8;
}
}
private static int parseDimension(String dim) {
try {
return Convert.toInt(dim, -1);
} catch (NumberFormatException e) {
return -1;
}
}
}

View File

@ -0,0 +1,25 @@
package com.jmal.clouddisk.interceptor;
import cn.hutool.core.util.StrUtil;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import org.jetbrains.annotations.NotNull;
import org.springframework.context.i18n.LocaleContextHolder;
import org.springframework.web.servlet.i18n.LocaleChangeInterceptor;
import java.util.Locale;
public class HeaderLocaleChangeInterceptor extends LocaleChangeInterceptor {
@Override
public boolean preHandle(HttpServletRequest request, @NotNull HttpServletResponse response, @NotNull Object handler) {
String lang = request.getHeader("lang");
if (StrUtil.isNotBlank(lang) && lang.indexOf("_") > 0) {
Locale locale = Locale.forLanguageTag(lang.replace("_", "-"));
LocaleContextHolder.setLocale(locale);
} else {
LocaleContextHolder.setLocale(Locale.US);
}
return true;
}
}

View File

@ -0,0 +1,123 @@
package com.jmal.clouddisk.interceptor;
import cn.hutool.core.lang.UUID;
import cn.hutool.core.text.CharSequenceUtil;
import cn.hutool.crypto.digest.DigestUtil;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.jmal.clouddisk.model.FileDocument;
import com.jmal.clouddisk.service.IFileService;
import com.jmal.clouddisk.service.IUserService;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.jetbrains.annotations.NotNull;
import org.springframework.stereotype.Component;
import org.springframework.web.servlet.HandlerInterceptor;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
/**
* @author jmal
* @Description
*/
@Slf4j
@Component
@RequiredArgsConstructor
public class PreFileInterceptor implements HandlerInterceptor {
private final IFileService fileService;
private final IUserService userService;
private static final Cache<String, String> INTERNAL_TOKEN_CACHE = Caffeine.newBuilder().expireAfterWrite(5, TimeUnit.SECONDS).build();
private static void setInternalTokenCache(String requestId, String token) {
INTERNAL_TOKEN_CACHE.put(requestId, token);
}
public static boolean isValidInternalTokenCache(String requestId, String token) {
if (CharSequenceUtil.isBlank(token)) {
return false;
}
String cachedToken = INTERNAL_TOKEN_CACHE.getIfPresent(requestId);
boolean isValid = token.equals(cachedToken);
INTERNAL_TOKEN_CACHE.invalidate(requestId);
return isValid;
}
@Override
public boolean preHandle(@NotNull HttpServletRequest request, @NotNull HttpServletResponse response, @NotNull Object handler) {
String uri = request.getRequestURI();
String[] pathSegments = uri.split("/");
// 验证路径格式
if (pathSegments.length != 4 || !"pre-file".equals(pathSegments[1])) {
response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
return false;
}
String fileId = pathSegments[2];
FileDocument fileDocument = getFileDocument(response, fileId);
if (fileDocument == null) {
return false;
}
// 构造内部路径
internalFilePath(request, response, fileDocument);
// 阻止原始请求继续处理
return false;
}
public void internalFilePath(HttpServletRequest request, HttpServletResponse response, FileDocument fileDocument) {
setInternal(request, fileDocument.getId());
String internalFilePath = constructInternalFilePath(fileDocument);
try {
request.setAttribute("forward", true);
request.getRequestDispatcher(internalFilePath).forward(request, response);
} catch (Exception e) {
handleForwardException(response, internalFilePath, e);
}
}
private static void setInternal(HttpServletRequest request, String fileId) {
// 生成临时 token
String requestId = UUID.fastUUID().toString(true) + fileId;
String internalToken = DigestUtil.sha256Hex(requestId + System.currentTimeMillis());
request.setAttribute("requestId", requestId);
request.setAttribute("internalToken", internalToken);
setInternalTokenCache(requestId, internalToken);
}
public FileDocument getFileDocument(HttpServletResponse response, String fileId) {
if (CharSequenceUtil.isBlank(fileId)) {
response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
return null;
}
FileDocument fileDocument = fileService.getById(fileId);
if (fileDocument == null) {
response.setStatus(HttpServletResponse.SC_NOT_FOUND);
return null;
}
return fileDocument;
}
private String constructInternalFilePath(FileDocument fileDocument) {
return "/file/" + userService.getUserNameById(fileDocument.getUserId()) + fileDocument.getPath() + fileDocument.getName();
}
private static void handleForwardException(HttpServletResponse response, String internalFilePath, Exception e) {
log.error("Failed to forward request to {}", internalFilePath, e);
response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
try {
response.getWriter().write("Internal server error");
} catch (IOException ioException) {
log.error("Failed to write response", ioException);
}
}
}

View File

@ -0,0 +1,113 @@
package com.jmal.clouddisk.interceptor;
import cn.hutool.core.text.CharSequenceUtil;
import cn.hutool.core.util.BooleanUtil;
import com.jmal.clouddisk.model.FileDocument;
import com.jmal.clouddisk.model.ShareDO;
import com.jmal.clouddisk.service.Constants;
import com.jmal.clouddisk.service.IShareService;
import com.jmal.clouddisk.service.impl.UserLoginHolder;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.jetbrains.annotations.NotNull;
import org.springframework.stereotype.Component;
import org.springframework.web.servlet.HandlerInterceptor;
/**
* @author jmal
* @Description
*/
@Slf4j
@Component
@RequiredArgsConstructor
public class ShareFileInterceptor implements HandlerInterceptor {
private final IShareService shareService;
private final PreFileInterceptor preFileInterceptor;
private final UserLoginHolder userLoginHolder;
@Override
public boolean preHandle(@NotNull HttpServletRequest request, @NotNull HttpServletResponse response, @NotNull Object handler) {
String uri = request.getRequestURI();
String[] pathSegments = uri.split("/");
// 验证路径格式
if (pathSegments.length < 4 || !"share-file".equals(pathSegments[1])) {
response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
return false;
}
String shareToken = null;
if (pathSegments.length == 5) {
shareToken = pathSegments[3];
}
String fileId = pathSegments[2];
FileDocument fileDocument = preFileInterceptor.getFileDocument(response, fileId);
if (fileDocument == null) {
return false;
}
// 验证权限
if (isNotAllowAccess(fileDocument, shareToken, request)) {
return false;
}
// 构造内部路径
preFileInterceptor.internalFilePath(request, response, fileDocument);
// 阻止原始请求继续处理
return false;
}
private boolean isNotAllowAccess(FileDocument fileDocument, String shareToken, HttpServletRequest request) {
if (BooleanUtil.isTrue(fileDocument.getIsShare())) {
return validShareFile(fileDocument, shareToken, request);
}
return true;
}
public boolean validShareFile(FileDocument fileDocument, String shareToken, HttpServletRequest request) {
if (System.currentTimeMillis() >= fileDocument.getExpiresAt()) {
// 过期了
return true;
}
if (fileDocument.getShareId() == null) {
// 未分享
return true;
}
ShareDO shareDO = shareService.getShare(fileDocument.getShareId());
if (shareDO == null) {
// 分享不存在
return true;
}
if (BooleanUtil.isFalse(shareDO.getIsPrivacy())) {
return false;
}
if (request == null) {
// 未登录
return true;
}
if (CharSequenceUtil.isBlank(shareToken)) {
// 判断是否为挂载文件
String userId = userLoginHolder.getUserId();
if (CharSequenceUtil.isNotBlank(userId)) {
return !shareService.existsMountFile(shareDO.getFileId(), userId);
}
shareToken = request.getHeader(Constants.SHARE_TOKEN);
if (CharSequenceUtil.isBlank(shareToken)) {
shareToken = request.getParameter(Constants.SHARE_TOKEN);
}
}
if (CharSequenceUtil.isBlank(shareToken)) {
// 未携带share-token
return true;
}
shareService.validShare(shareToken, shareDO.getId());
return false;
}
}

View File

@ -0,0 +1,345 @@
package com.jmal.clouddisk.listener;
import cn.hutool.core.text.CharSequenceUtil;
import cn.hutool.core.thread.ThreadUtil;
import com.jmal.clouddisk.config.FileProperties;
import com.jmal.clouddisk.lucene.RebuildIndexTaskService;
import com.jmal.clouddisk.service.IFileService;
import io.methvin.watcher.DirectoryChangeEvent;
import io.methvin.watcher.DirectoryChangeListener;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.io.File;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
@Slf4j
@Service
@RequiredArgsConstructor
public class FileListener implements DirectoryChangeListener {
private final FileProperties fileProperties;
private final IFileService fileService;
private final RebuildIndexTaskService rebuildIndexTaskService;
/**
* 需要过滤掉的目录列表
*/
@Getter
private final Set<Path> filterDirSet = new CopyOnWriteArraySet<>();
// 使用ConcurrentHashMap存储最新事件,确保每个路径只有一个最新事件
private final Map<Path, DirectoryChangeEvent> eventMap = new ConcurrentHashMap<>();
// 处理队列 - 用于实际处理事件的工作队列
private final BlockingQueue<DirectoryChangeEvent> processingQueue = new LinkedBlockingQueue<>(100000);
// 负载检测
private final AtomicInteger eventBurstCounter = new AtomicInteger(0);
private final AtomicBoolean highLoadDetected = new AtomicBoolean(false);
private final AtomicBoolean scanningInProgress = new AtomicBoolean(false);
private volatile Instant lastBurstTime = Instant.now();
private static final int EVENT_BURST_THRESHOLD = 5000; // 单位时间内事件数阈值
private static final Duration BURST_WINDOW = Duration.ofSeconds(10); // 事件计数窗口
private static final Duration IDLE_THRESHOLD = Duration.ofSeconds(30); // 闲置时间阈值
private final ScheduledExecutorService scheduler = ThreadUtil.createScheduledExecutor(2);
private final ExecutorService processExecutor = ThreadUtil.newFixedExecutor(Runtime.getRuntime().availableProcessors(), 10000, "file-process-thread", true);
@PostConstruct
public void init() {
// 定期将事件从Map转移到处理队列
scheduler.scheduleAtFixedRate(this::transferEventsToQueue, 200, 200, TimeUnit.MILLISECONDS);
// 启动消费者线程
startConsumerThreads();
// 定期检查是否需要执行全盘扫描
scheduler.scheduleAtFixedRate(this::checkAndStartScan, 10, 10, TimeUnit.SECONDS);
// 定期重置事件计数器
scheduler.scheduleAtFixedRate(() -> {
eventBurstCounter.set(0);
// 如果高负载标志已设置且已经过了一段闲置期,则进行一次全盘扫描
if (highLoadDetected.get() &&
Duration.between(lastBurstTime, Instant.now()).compareTo(IDLE_THRESHOLD) > 0) {
checkAndStartScan();
}
}, BURST_WINDOW.toSeconds(), BURST_WINDOW.toSeconds(), TimeUnit.SECONDS);
}
private void checkAndStartScan() {
// 如果检测到高负载并且当前没有扫描正在进行中,且处理队列基本为空(系统闲置)
if (highLoadDetected.get() && !scanningInProgress.get() && processingQueue.size() < 10) {
highLoadDetected.set(false); // 重置高负载标志
startIncrementalScan();
}
}
private void startIncrementalScan() {
if (!scanningInProgress.compareAndSet(false, true)) {
return; // 已有扫描任务在运行
}
// 延时执行,确保当前处理队列中的事件都被处理完
rebuildIndexTaskService.doSync(null, null, false);
rebuildIndexTaskService.onSyncComplete(() -> {
log.info("执行增量文件扫描,确保100%处理...");
rebuildIndexTaskService.doSync(null, null, false);
rebuildIndexTaskService.onSyncComplete(() -> {
scanningInProgress.set(false);
log.info("增量文件扫描完成,已添加到处理队列。");
});
});
}
private void transferEventsToQueue() {
if (eventMap.isEmpty()) {
return;
}
List<DirectoryChangeEvent> eventsToProcess = new ArrayList<>();
// 原子地将事件从map中取出,避免竞态条件
for (Path key : eventMap.keySet()) {
DirectoryChangeEvent event = eventMap.remove(key);
if (event != null) {
eventsToProcess.add(event);
}
}
if (eventsToProcess.isEmpty()) {
return;
}
// 转移到处理队列
int transferCount = 0;
for (DirectoryChangeEvent event : eventsToProcess) {
boolean offered = processingQueue.offer(event);
if (!offered) {
log.error("处理队列已满,无法添加事件: {}, 路径: {}", event.eventType(), event.path());
// 放回原Map以便下次尝试
eventMap.put(event.path(), event);
} else {
transferCount++;
}
}
if (transferCount > 0) {
log.debug("转移了{}个事件到处理队列,当前队列大小: {}", transferCount, processingQueue.size());
}
}
private void startConsumerThreads() {
int processors = Runtime.getRuntime().availableProcessors();
for (int i = 0; i < processors; i++) {
processExecutor.submit(this::processQueueItems);
}
}
private void processQueueItems() {
while (!Thread.currentThread().isInterrupted()) {
try {
DirectoryChangeEvent event = processingQueue.poll(100, TimeUnit.MILLISECONDS);
if (event != null) {
try {
processEvent(event);
} catch (Exception e) {
log.error("处理事件失败: {}, 路径: {}", event.eventType(), event.path(), e);
}
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
}
}
}
public void addFilterDir(Path path) {
filterDirSet.add(path);
}
public void removeFilterDir(Path path) {
filterDirSet.remove(path);
}
public boolean containsFilterDir(Path path) {
return filterDirSet.contains(path);
}
@Override
public void onEvent(DirectoryChangeEvent directoryChangeEvent) {
Path eventPath = directoryChangeEvent.path();
DirectoryChangeEvent.EventType eventType = directoryChangeEvent.eventType();
// 增加事件计数器,检测高负载
int currentCount = eventBurstCounter.incrementAndGet();
lastBurstTime = Instant.now();
if (currentCount > EVENT_BURST_THRESHOLD) {
highLoadDetected.set(true);
log.warn("检测到高事件负载: {}事件/{}秒,标记为需要增量扫描", currentCount, BURST_WINDOW.getSeconds());
}
// 检查是否为忽略路径
if (filterDirSet.stream().anyMatch(eventPath::startsWith)) {
log.debug("忽略事件: {}, 路径: {}", eventType, eventPath);
return;
}
if (fileProperties.getMonitorIgnoreFilePrefix().stream().anyMatch(eventPath.getFileName()::startsWith)) {
log.debug("忽略文件:{}", eventPath.toFile().getAbsolutePath());
return;
}
log.debug("接收到事件: {}, 路径: {}", eventType, eventPath);
// 将事件放入Map
eventMap.put(eventPath, directoryChangeEvent);
}
private void processEvent(DirectoryChangeEvent directoryChangeEvent) {
Path eventPath = directoryChangeEvent.path();
DirectoryChangeEvent.EventType eventType = directoryChangeEvent.eventType();
File file = eventPath.toFile();
switch (eventType) {
case CREATE:
onFileCreate(file);
break;
case MODIFY:
onFileChange(file);
break;
case DELETE:
onFileDelete(file);
break;
default:
break;
}
}
/**
* 文件创建执行
*
* @param file 文件
*/
public void onFileCreate(File file) {
try {
String username = ownerOfChangeFile(file);
if (CharSequenceUtil.isBlank(username)) {
return;
}
fileService.createFile(username, file);
log.info("用户:{},新建文件:{}", username, file.getAbsolutePath());
} catch (Exception e) {
log.error("新建文件后续操作失败, {}", file.getAbsolutePath(), e);
}
}
/**
* 文件创建修改
*
* @param file 文件
*/
public void onFileChange(File file) {
try {
String username = ownerOfChangeFile(file);
if (CharSequenceUtil.isBlank(username)) {
return;
}
fileService.updateFile(username, file);
log.info("用户:{},修改文件:{}", username, file.getAbsolutePath());
} catch (Exception e) {
log.error("修改文件后续操作失败, fileAbsolutePath: {}", file.getAbsolutePath(), e);
}
}
/**
* 文件删除
*
* @param file 文件
*/
public void onFileDelete(File file) {
try {
String username = ownerOfChangeFile(file);
if (CharSequenceUtil.isBlank(username)) {
return;
}
fileService.deleteFile(username, file);
log.info("用户:{},删除文件:{}", username, file.getAbsolutePath());
} catch (Exception e) {
log.error("删除文件后续操作失败, fileAbsolutePath: {}", file.getAbsolutePath(), e);
}
}
/**
* 判断变化的文件属于哪个用户
*
* @return username
*/
private String ownerOfChangeFile(File file) {
String username = null;
try {
int rootPathCount = Paths.get(fileProperties.getRootDir()).getNameCount();
if (file.toPath().getNameCount() == rootPathCount + 1) {
return null;
}
username = file.toPath().subpath(rootPathCount, rootPathCount + 1).toString();
} catch (Exception e) {
log.error("解析路径失败, fileAbsolutePath: {}", file.getAbsolutePath(), e);
}
return username;
}
@Override
public boolean isWatching() {
return DirectoryChangeListener.super.isWatching();
}
@Override
public void onIdle(int i) {
DirectoryChangeListener.super.onIdle(i);
}
@Override
public void onException(Exception e) {
DirectoryChangeListener.super.onException(e);
}
@PreDestroy
public void shutdown() {
scheduler.shutdown();
processExecutor.shutdown();
try {
// 等待处理线程池在指定时间内完成任务
if (!processExecutor.awaitTermination(1, TimeUnit.SECONDS)) {
processExecutor.shutdownNow();
}
} catch (InterruptedException e) {
processExecutor.shutdownNow();
Thread.currentThread().interrupt();
}
try {
// 等待调度器在指定时间内完成任务
if (!scheduler.awaitTermination(1, TimeUnit.SECONDS)) {
scheduler.shutdownNow();
}
} catch (InterruptedException e) {
scheduler.shutdownNow();
Thread.currentThread().interrupt();
}
}
}

View File

@ -0,0 +1,227 @@
package com.jmal.clouddisk.listener;
import cn.hutool.core.date.DateUnit;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.io.file.PathUtil;
import cn.hutool.core.text.CharSequenceUtil;
import com.jmal.clouddisk.config.FileProperties;
import com.jmal.clouddisk.model.FileDocument;
import com.jmal.clouddisk.service.IFileService;
import com.jmal.clouddisk.util.SystemUtil;
import io.methvin.watcher.DirectoryWatcher;
import jakarta.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.beans.factory.config.BeanDefinition;
import org.springframework.context.annotation.ClassPathScanningCandidateComponentProvider;
import org.springframework.context.event.ContextRefreshedEvent;
import org.springframework.context.event.EventListener;
import org.springframework.core.type.filter.AnnotationTypeFilter;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.index.IndexOperations;
import org.springframework.data.mongodb.core.index.MongoPersistentEntityIndexResolver;
import org.springframework.data.mongodb.core.mapping.Document;
import org.springframework.data.mongodb.core.mapping.MongoMappingContext;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Objects;
/**
* @Description 启动时开启文件目录监控
* @Date 2020-02-21 14:54
* @author jmal
*/
@Service
@Slf4j
@RequiredArgsConstructor
public class FileMonitor {
final FileProperties fileProperties;
final MongoTemplate mongoTemplate;
final IFileService fileService;
final FileListener fileListener;
@Value("${version}")
private String version;
private String newVersion = null;
private DirectoryWatcher watcher;
@EventListener(ContextRefreshedEvent.class)
public void initIndicesAfterStartup() {
// 1. 获取映射上下文,这一步不变
MongoMappingContext mappingContext = (MongoMappingContext) mongoTemplate.getConverter().getMappingContext();
// 2. 创建索引解析器,这一步不变
MongoPersistentEntityIndexResolver resolver = new MongoPersistentEntityIndexResolver(mappingContext);
// 3. 使用 Spring 的类路径扫描器自动发现所有 @Document 注解的类
ClassPathScanningCandidateComponentProvider provider = new ClassPathScanningCandidateComponentProvider(false);
provider.addIncludeFilter(new AnnotationTypeFilter(Document.class));
// 替换为你的实体类所在的包名
String basePackage = "com.jmal.clouddisk.model";
for (BeanDefinition beanDefinition : provider.findCandidateComponents(basePackage)) {
try {
Class<?> entityClass = Class.forName(beanDefinition.getBeanClassName());
// 获取对应实体类的 IndexOperations
IndexOperations indexOps = mongoTemplate.indexOps(entityClass);
resolver.resolveIndexFor(entityClass).forEach(indexOps::createIndex);
} catch (ClassNotFoundException e) {
// 处理异常
log.error("Error loading class: {}", e.getMessage(), e);
}
}
}
@PostConstruct
public void init() throws IOException {
// 检测新版本
newVersion = SystemUtil.getNewVersion();
// 判断是否开启文件监控
if (Boolean.FALSE.equals(fileProperties.getMonitor())) {
return;
}
// 忽略目录
fileListener.addFilterDir(Paths.get(fileProperties.getRootDir(), fileProperties.getChunkFileDir()));
fileListener.addFilterDir(Paths.get(fileProperties.getRootDir(), fileProperties.getLuceneIndexDir()));
// 开启文件监控
newDirectoryWatcher();
}
private void newDirectoryWatcher() throws IOException {
Path rootDir = Paths.get(fileProperties.getRootDir());
PathUtil.mkdir(rootDir);
this.watcher = DirectoryWatcher.builder()
.path(rootDir)
.listener(fileListener)
.logger(log)
.fileHashing(false)
.build();
watcher.watchAsync();
log.info("\r\n文件监控服务已开启, 监控目录: {}, 忽略目录: {}", rootDir, fileListener.getFilterDirSet());
}
private void reloadDirectoryWatcher() {
try {
watcher.close();
newDirectoryWatcher();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
/**
* 在过滤器里添加路径
* @param path 需要过滤掉的路径
*/
public void addDirFilter(String path) {
Path filepath = Paths.get(fileProperties.getRootDir(), path);
fileListener.addFilterDir(filepath);
String username = Paths.get(path).getParent().getFileName().toString();
fileService.createFile(username, filepath.toFile());
reloadDirectoryWatcher();
}
/**
* 需要移除的路径
* @param path 需要移除的路径
*/
public void removeDirFilter(String path) {
Path filepath = Paths.get(fileProperties.getRootDir(), path);
if (fileListener.containsFilterDir(filepath)) {
fileListener.removeFilterDir(filepath);
String username = Paths.get(path).getParent().getFileName().toString();
fileService.deleteFile(username, Paths.get(fileProperties.getRootDir(), path).toFile());
reloadDirectoryWatcher();
}
}
/**
* <p>定时清理临时目录</p>
* 每天凌晨2点执行<br>
* 1.清理临时目录中3天前的文件<br>
* 2.清理视频转码缓存目录中的无效文件<br>
* 3.清理回收站中30天前的文件<br>
*/
@Scheduled(cron = "0 0 2 * * ?")
private void cleanTempDir() {
// 临时目录
Path tempPath = Paths.get(fileProperties.getRootDir(), fileProperties.getChunkFileDir());
for (File username : FileUtil.ls(tempPath.toString())) {
if (username.isDirectory()) {
for (File file : FileUtil.ls(username.getAbsolutePath())) {
clearUserCache(file);
}
}
}
}
/**
* 清理用户缓存文件
* @param file 需要清理的文件
*/
private void clearUserCache(File file) {
// 是否为三天前的文件
boolean sevenDayAgo = file.lastModified() < (System.currentTimeMillis() - DateUnit.DAY.getMillis() * 3);
// 是否为视频转码缓存目录
boolean videoCache = fileProperties.getVideoTranscodeCache().equals(file.getName()) || fileProperties.getLuceneIndexDir().equals(file.getParentFile().getName());
// 回收站
boolean trash = fileProperties.getJmalcloudTrashDir().equals(file.getName());
if (videoCache || trash) {
clearVideoCache(file, videoCache);
return;
}
if (sevenDayAgo) {
FileUtil.del(file);
}
}
/**
* 清理视频转码缓存目录
* @param file 需要清理的文件
* @param videoCache 是否为视频转码缓存目录
*/
private void clearVideoCache(File file, boolean videoCache) {
if (videoCache && file.listFiles() != null) {
for (File f : Objects.requireNonNull(file.listFiles())) {
FileDocument fileDocument = fileService.getById(f.getName());
if (fileDocument == null || f.isFile()) {
FileUtil.del(f);
log.info("删除视频转码缓存文件: {}", f.getAbsolutePath());
}
}
}
}
public String hasNewVersion() {
if (CharSequenceUtil.isBlank(newVersion)) {
return null;
}
// 判断是否有新版本, 比较newVersion和version
if (newVersion.compareTo("v" + version) > 0) {
return newVersion;
}
return null;
}
/**
* 每3小时检查一次版本
*/
@Scheduled(cron = "0 0 0/3 * * ?")
private void getNewVersion() {
newVersion = SystemUtil.getNewVersion();
}
}

View File

@ -0,0 +1,553 @@
package com.jmal.clouddisk.lucene;
import cn.hutool.core.convert.Convert;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.text.CharSequenceUtil;
import cn.hutool.core.thread.ThreadUtil;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.ReUtil;
import com.jmal.clouddisk.config.FileProperties;
import com.jmal.clouddisk.model.FileDocument;
import com.jmal.clouddisk.service.Constants;
import com.jmal.clouddisk.service.IUserService;
import com.jmal.clouddisk.service.impl.FileServiceImpl;
import com.jmal.clouddisk.util.HashUtil;
import com.mongodb.client.AggregateIterable;
import com.mongodb.client.result.UpdateResult;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.bson.Document;
import org.bson.conversions.Bson;
import org.springframework.dao.DataAccessException;
import org.springframework.data.domain.Sort;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;
import org.springframework.stereotype.Service;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicBoolean;
import static com.jmal.clouddisk.service.IUserService.USER_ID;
import static com.mongodb.client.model.Accumulators.sum;
import static com.mongodb.client.model.Aggregates.group;
import static com.mongodb.client.model.Aggregates.match;
import static com.mongodb.client.model.Filters.*;
/**
* EtagService
* 文件和文件夹的 ETag 更新逻辑
* 异步处理文件变化和删除的 ETag 更新
* 后台 Worker 处理标记为需要更新 ETag 的文件夹
* ETag 计算和更新的内部方法
*/
@Service
@RequiredArgsConstructor
@Slf4j
public class EtagService {
private final MongoTemplate mongoTemplate;
private final FileProperties fileProperties;
private final IUserService userService;
private static final String EMPTY_FOLDER_ETAG_BASE_STRING = "EMPTY_FOLDER_REPRESENTATION_MONGO_V2";
private static final int MAX_ETAG_UPDATE_ATTEMPTS = 5; // 最大失败重试次数
private final AtomicBoolean processingScheduled = new AtomicBoolean(false);
/**
* 处理文件夹Etag的线程池
*/
private ExecutorService executorMarkedFoldersService;
@PostConstruct
public void init() {
if (executorMarkedFoldersService == null) {
executorMarkedFoldersService = ThreadUtil.newFixedExecutor(1, 1, "EtagWorker-", false);
}
ThreadUtil.execute(() -> {
Query queryNoEtagQuery = new Query();
queryNoEtagQuery.addCriteria(Criteria.where(Constants.ETAG).exists(false).and(Constants.IS_FOLDER).is(true));
long countOfFoldersWithoutEtag = mongoTemplate.count(queryNoEtagQuery, FileDocument.class);
if (countOfFoldersWithoutEtag > 0) {
log.info("{} folders found without ETag. Marking them for ETag calculation.", countOfFoldersWithoutEtag);
Update update = new Update().set(Constants.NEEDS_ETAG_UPDATE_FIELD, true).currentDate(Constants.LAST_ETAG_UPDATE_REQUEST_AT_FIELD);
mongoTemplate.updateMulti(queryNoEtagQuery, update, FileDocument.class);
}
processRootFolderFiles();
log.info("Initial ETag setup finished. Ensuring ETag processing worker is active if needed.");
ensureProcessingMarkedFolders();
});
}
/**
* 统计文件夹的大小
*/
public long getFolderSize(String collectionName, String userId, String path) {
List<Bson> list = Arrays.asList(match(and(eq(USER_ID, userId), eq(Constants.IS_FOLDER, false), regex("path", "^" + ReUtil.escape(path)))), group(null, sum(Constants.TOTAL_SIZE, "$size")));
AggregateIterable<Document> result = mongoTemplate.getCollection(collectionName).aggregate(list);
long totalSize = 0;
Document doc = result.first();
if (doc != null) {
totalSize = Convert.toLong(doc.get(Constants.TOTAL_SIZE), 0L);
}
return totalSize;
}
/**
* 当文件内容发生变化(上传/修改/重命名/删除等)时调用。
* 立即更新文件的ETag,并异步标记其父文件夹需要更新ETag。
*
* @param file 发生变化的文件
*/
public void updateFileEtagAsync(String username, File file) {
if (!FileUtil.exist(file)) {
return;
}
if (FileUtil.isFile(file)) {
updateFileEtagAndMarkParentAsync(username, file);
}
}
/**
* 确保 ETag 处理任务已调度或正在运行。
* 如果当前没有任务在执行或计划执行,则提交一个新任务到执行器。
* 这个方法是幂等的,多次调用(如果任务已调度)不会产生副作用。
*/
private void ensureProcessingMarkedFolders() {
// 使用 compareAndSet 来原子性地检查并设置标志
// 如果 processingScheduled 为 false,则将其设置为 true 并执行提交逻辑
if (processingScheduled.compareAndSet(false, true)) {
log.debug("ETag processing task for marked folders is being scheduled.");
executorMarkedFoldersService.execute(() -> {
String workerId = "markedFoldersWorker-" + Thread.currentThread().getName();
try {
// 调用实际的循环处理逻辑
processMarkedFoldersLoop(workerId);
} finally {
// 当任务完成(无论是正常结束还是异常结束)后,
// 将标志重置为 false,允许后续的 ensureProcessingMarkedFolders 调用再次调度任务。
processingScheduled.set(false);
log.debug("[Worker {}] ETag processing task finished. Resetting schedule flag.", workerId);
// 关键:检查在本次处理运行期间是否有新的文件夹被标记
// 如果有,则再次尝试调度,确保它们得到处理
if (hasMoreMarkedFoldersInDb()) {
log.debug("[Worker {}] New folders were marked during or after the last processing run. Re-triggering ETag processing.", workerId);
ensureProcessingMarkedFolders();
} else {
log.debug("[Worker {}] No more marked folders found after processing run. ETag worker will rest until new marks.", workerId);
}
}
});
} else {
// 如果 processingScheduled 已经是 true,说明任务已在队列中或正在运行
log.debug("ETag processing task is already scheduled or running. This trigger call is a no-op.");
}
}
/**
* 检查数据库中是否还有需要更新ETag的文件夹。
*/
private boolean hasMoreMarkedFoldersInDb() {
Query query = Query.query(Criteria.where(Constants.NEEDS_ETAG_UPDATE_FIELD).is(true).and(Constants.IS_FOLDER).is(true));
query.limit(1); // 只需要知道是否存在,不需要完整计数
return mongoTemplate.exists(query, FileDocument.class);
}
/**
* 当文件内容发生变化(上传新版本/修改)时调用。
* 立即更新文件的ETag,并异步标记其父文件夹需要更新ETag。
*
* @param file 发生变化的文件
*/
private String updateFileEtagAndMarkParentAsync(String username, File file) {
try {
if (!FileUtil.exist(file) || !FileUtil.isFile(file)) {
return null;
}
String newEtag = HashUtil.sha256(file);
String fileName = file.getName();
String relativePath = getDbPath(username, file);
if (relativePath == null) {
return null;
}
String userId = userService.getUserIdByUserName(username);
Query query = getQueryByPath(userId, relativePath, fileName);
query.fields().include(Constants.ETAG);
FileDocument currentFileDoc = mongoTemplate.findOne(query, FileDocument.class);
String oldEtag = (currentFileDoc != null) ? currentFileDoc.getEtag() : null;
if (!ObjectUtil.equals(newEtag, oldEtag)) {
Update update = new Update().set(Constants.ETAG, newEtag);
// 如果有版本控制字段,这里也应该更新
mongoTemplate.updateFirst(query, update, FileDocument.class);
log.debug("File ETag updated for {}: {} -> {}", relativePath + fileName, oldEtag, newEtag);
// 标记父文件夹需要更新ETag
markFolderForEtagUpdate(userId, relativePath);
} else {
log.debug("File ETag for {} did not change or calculation failed.", file.getAbsoluteFile());
}
return newEtag;
} catch (Exception e) {
log.error("Error updating file ETag for {}: {}", file.getAbsoluteFile(), e.getMessage(), e);
}
return null;
}
/**
* 获取改文件在数据库中的路径(PATH_FIELD)字段的值
*
* @param username 用户名
* @param physicalFile 文件
*/
private String getDbPath(String username, File physicalFile) {
Path physicalPath = physicalFile.toPath();
Path userRootPath = Paths.get(fileProperties.getRootDir(), username);
if (!physicalPath.startsWith(userRootPath)) {
return null;
}
Path relativeNioPath = userRootPath.relativize(physicalPath);
if (relativeNioPath.toString().isEmpty()) {
return null;
}
if (relativeNioPath.getNameCount() <= 1 || relativeNioPath.getParent() == null) {
return "/";
}
return "/" + relativeNioPath.getParent().toString().replace(File.separatorChar, '/') + "/";
}
/**
* 当新文件夹创建或重命名时调用。
* 立即设置新文件夹的初始ETag,并异步标记其父文件夹需要更新ETag。
*
* @param username 用户名
* @param physicalNewFolder 新创建的文件夹的完整路径
*/
public void handleNewFolderCreationAsync(String username, File physicalNewFolder) {
if (!FileUtil.exist(physicalNewFolder) || !FileUtil.isDirectory(physicalNewFolder)) {
return;
}
String fileName = physicalNewFolder.getName();
String relativePath = getDbPath(username, physicalNewFolder);
if (relativePath == null) {
return;
}
try {
String userId = userService.getUserIdByUserName(username);
// 查询当前文件夹下是否有内容, 由于是异步处理,刚创建的文件夹可能还没来得及设置etag, 其下就有新文件了, 所有需要查询一次
String currentFolderNormalizedPath = relativePath + fileName + "/";
Query childrenQuery = Query.query(Criteria.where(Constants.PATH_FIELD).is(currentFolderNormalizedPath));
childrenQuery.addCriteria(Criteria.where(IUserService.USER_ID).is(userId));
boolean hasChildren = mongoTemplate.exists(childrenQuery, FileDocument.class);
if (!hasChildren) {
// 如果没有内容,则设置初始ETag
String initialEtag = HashUtil.sha256(EMPTY_FOLDER_ETAG_BASE_STRING);
Query query = getQueryByPath(userId, relativePath, fileName);
Update update = new Update().set(Constants.ETAG, initialEtag).set(Constants.NEEDS_ETAG_UPDATE_FIELD, false);
mongoTemplate.updateFirst(query, update, FileDocument.class);
log.info("Initial ETag set for new folder {}: {}", relativePath, initialEtag);
}
// 标记父文件夹需要更新ETag
markFolderForEtagUpdate(userService.getUserIdByUserName(username), relativePath);
} catch (DataAccessException e) {
log.error("Database error setting initial ETag for folder {}: {}", relativePath, e.getMessage(), e);
}
}
/**
* 通过 userId 和 dbPath 还有filename 可获取到唯一的文件/文件夹
*
* @param userId 用户id
* @param dbPath dbPath
* @param filename 文件名
*/
private static Query getQueryByPath(String userId, String dbPath, String filename) {
Query query = Query.query(Criteria.where(IUserService.USER_ID).is(userId));
query.addCriteria(Criteria.where(Constants.PATH_FIELD).is(dbPath));
query.addCriteria(Criteria.where(Constants.FILENAME_FIELD).is(filename));
return query;
}
/**
* 当文件或文件夹被删除时调用。
* 异步标记其原父文件夹需要更新ETag。
*
* @param username 用户名
* @param file 被删除的文件
*/
public void handleItemDeletionAsync(String username, File file) {
log.debug("Handling ETag update for parent of deleted item: {}", file.getAbsoluteFile());
String relativePath = getDbPath(username, file);
if (relativePath == null) {
return;
}
// 标记父文件夹需要更新ETag
markFolderForEtagUpdate(userService.getUserIdByUserName(username), relativePath);
}
/**
* 处理根目录下文件的ETag
*/
private void processRootFolderFiles() {
boolean run = true;
while (run) {
Query findQuery = Query.query(Criteria.where(Constants.ETAG).exists(false).and(Constants.IS_FOLDER).is(false).and(Constants.PATH_FIELD).is("/")).limit(16);
findQuery.fields().include(Constants.PATH_FIELD, Constants.FILENAME_FIELD, IUserService.USER_ID, Constants.ETAG);
long count = mongoTemplate.count(findQuery, FileDocument.class);
if (count == 0) {
run = false;
}
// 获取根目录下所有的文件
List<FileDocument> tasks = mongoTemplate.find(findQuery, FileDocument.class);
if (tasks.isEmpty()) {
continue;
}
log.debug("Found {} files marked for ETag update. Processing...", tasks.size());
for (FileDocument fileDoc : tasks) {
String username = userService.getUserNameById(fileDoc.getUserId());
Path path = Paths.get(fileProperties.getRootDir(), username, fileDoc.getPath(), fileDoc.getName());
if (Files.exists(path)) {
updateFileEtagAndMarkParentAsync(username, path.toFile());
}
}
}
}
/**
* 实际处理标记为需要更新ETag的文件夹的循环。
* 此方法由 ensureProcessingMarkedFolders 提交的任务在 EtagWorker 线程中执行。
*
* @param workerId 当前worker的唯一标识
*/
private void processMarkedFoldersLoop(String workerId) {
boolean run = true;
while (run && !Thread.currentThread().isInterrupted()) {
Query findQuery = Query.query(Criteria.where(Constants.NEEDS_ETAG_UPDATE_FIELD).is(true).and(Constants.IS_FOLDER).is(true)).with(Sort.by(Sort.Direction.ASC, Constants.LAST_ETAG_UPDATE_REQUEST_AT_FIELD)).limit(16);
findQuery.fields().include(Constants.PATH_FIELD, Constants.FILENAME_FIELD, IUserService.USER_ID, Constants.ETAG, "_id");
List<FileDocument> tasks = mongoTemplate.find(findQuery, FileDocument.class);
if (tasks.isEmpty()) {
run = false;
continue;
}
log.debug("[Worker {}] Found {} folders marked for ETag update. Processing...", workerId, tasks.size());
for (FileDocument folderDoc : tasks) {
if (Thread.currentThread().isInterrupted()) {
log.warn("[Worker {}] ETag processing loop was interrupted.", workerId);
run = false; // 尊重中断信号
break;
}
String folderPath = folderDoc.getPath();
String docId = folderDoc.getId();
String userId = folderDoc.getUserId();
log.debug("[Worker {}] Processing ETag for folder: {} (ID: {})", workerId, folderPath, docId);
try {
boolean etagActuallyChanged = calculateAndUpdateSingleFolderEtagInternal(folderDoc, workerId);
Query clearMarkQuery = Query.query(Criteria.where("_id").is(docId));
Update clearMarkUpdate = new Update().set(Constants.NEEDS_ETAG_UPDATE_FIELD, false).unset(Constants.ETAG_UPDATE_FAILED_ATTEMPTS_FIELD).unset(Constants.LAST_ETAG_UPDATE_ERROR_FIELD);
mongoTemplate.updateFirst(clearMarkQuery, clearMarkUpdate, FileDocument.class);
log.debug("[Worker {}] Cleared ETag update mark for folder: {}", workerId, folderPath + folderDoc.getName());
if (etagActuallyChanged) {
// 如果ETag变化了,标记其父文件夹
markFolderForEtagUpdate(userId, folderPath);
}
} catch (Exception e) {
log.error("[Worker {}] Critical error processing ETag for folder {}: {}", workerId, folderPath, e.getMessage(), e);
handleProcessingError(docId, folderPath, e, workerId);
}
}
log.debug("[Worker {}] Finished processing batch of {} folders.", workerId, tasks.size());
}
log.debug("[Worker {}] ETag processing loop finished or paused.", workerId);
}
/**
* 标记指定文件夹需要ETag更新
*
* @param userId 用户ID
* @param currentFolderPath 当前文件夹路径
*/
private void markFolderForEtagUpdate(String userId, String currentFolderPath) {
String parentFolderPath = getParentDbPath(currentFolderPath);
if (parentFolderPath == null) {
return;
}
String folderName = Paths.get(currentFolderPath).getFileName().toString();
Query query = getQueryByPath(userId, parentFolderPath, folderName);
Update update = new Update().set(Constants.NEEDS_ETAG_UPDATE_FIELD, true).currentDate(Constants.LAST_ETAG_UPDATE_REQUEST_AT_FIELD);
try {
UpdateResult result = mongoTemplate.updateFirst(query, update, FileDocument.class);
if (result.getModifiedCount() > 0 || result.getMatchedCount() > 0) { // 即使已经是true也更新时间戳
log.debug("Marked folder for ETag update: {}", currentFolderPath);
ensureProcessingMarkedFolders();
} else {
log.debug("Folder not found to mark for ETag update: {}", currentFolderPath);
}
} catch (DataAccessException e) {
log.error("DB error marking folder for ETag update {}: {}", currentFolderPath, e.getMessage(), e);
}
}
/**
* 内部方法:计算并更新单个文件夹的ETag。
* 被 processMarkedFolders 调用。
*
* @return true 如果ETag实际发生了变化并被更新
*/
private boolean calculateAndUpdateSingleFolderEtagInternal(FileDocument folderDoc, String workerId) {
String folderPath = folderDoc.getPath();
String oldEtag = folderDoc.getEtag();
String userId = folderDoc.getUserId();
String newCalculatedEtag;
String currentFolderNormalizedPath = folderDoc.getPath() + folderDoc.getName() + "/";
Query childrenQuery = Query.query(Criteria.where(Constants.PATH_FIELD).is(currentFolderNormalizedPath));
childrenQuery.addCriteria(Criteria.where(IUserService.USER_ID).is(userId));
childrenQuery.fields().include(Constants.FILENAME_FIELD).include(Constants.ETAG).include(Constants.IS_FOLDER).include(IUserService.USER_ID).include(Constants.PATH_FIELD);
List<FileDocument> children = mongoTemplate.find(childrenQuery, FileDocument.class);
long folderSize = 0;
if (children.isEmpty()) {
newCalculatedEtag = HashUtil.sha256(EMPTY_FOLDER_ETAG_BASE_STRING);
} else {
List<String> childRepresentations = children.stream().sorted(Comparator.comparing(FileDocument::getName)) // 按名称排序
.map(child -> formatChildRepresentation(workerId, child)).toList();
StringBuilder combinedRepresentation = new StringBuilder();
for (String rep : childRepresentations) {
if (rep == null) {
return false;
}
if (!rep.isBlank()) {
combinedRepresentation.append(rep).append(";");
}
}
newCalculatedEtag = HashUtil.sha256(combinedRepresentation.toString());
// 计算文件夹大小
folderSize = getFolderSize(FileServiceImpl.COLLECTION_NAME, userId, currentFolderNormalizedPath);
}
if (!newCalculatedEtag.equals(oldEtag)) {
Query updateQuery = Query.query(Criteria.where("_id").is(folderDoc.getId()));
// 如果使用乐观锁,这里也需要检查版本号
// query.addCriteria(Criteria.where("_version").is(folderDoc.getVersion()));
Update update = new Update().set(Constants.ETAG, newCalculatedEtag);
// 更新文件夹大小
update.set(Constants.SIZE, folderSize);
// update.inc("_version", 1);
UpdateResult result = mongoTemplate.updateFirst(updateQuery, update, FileDocument.class);
if (result.getModifiedCount() > 0) {
log.debug("[Worker {}] Folder ETag updated for {}: {} -> {}", workerId, folderPath + folderDoc.getName(), oldEtag, newCalculatedEtag);
return true;
} else {
// 可能在计算和写入之间,该文档被其他worker处理了(如果ETag已更新为相同值)或版本冲突
log.warn("[Worker {}] Folder ETag for {} was calculated as {} (old: {}), but DB update reported no modification. Possible race condition or already up-to-date.", workerId, folderPath, newCalculatedEtag, oldEtag);
return false; // 没有实际修改数据库
}
}
log.debug("[Worker {}] Folder ETag for {} did not change (calculated: {}, current: {}).", workerId, folderPath, newCalculatedEtag, oldEtag);
return false;
}
private void handleProcessingError(String docId, String folderPath, Exception e, String workerId) {
Query query = Query.query(Criteria.where("_id").is(docId));
FileDocument failedDoc = mongoTemplate.findOne(query, FileDocument.class);
int attempts = (failedDoc != null && failedDoc.getEtagUpdateFailedAttempts() != null) ? failedDoc.getEtagUpdateFailedAttempts() + 1 : 1;
Update update = new Update().set(Constants.LAST_ETAG_UPDATE_ERROR_FIELD, e.getMessage().substring(0, Math.min(e.getMessage().length(), 1000))) // 限制错误信息长度
.set(Constants.ETAG_UPDATE_FAILED_ATTEMPTS_FIELD, attempts);
if (attempts >= MAX_ETAG_UPDATE_ATTEMPTS) {
update.set(Constants.NEEDS_ETAG_UPDATE_FIELD, false); // 停止重试
log.error("[Worker {}] Folder {} ETag update failed after {} attempts. Giving up.", workerId, folderPath, attempts);
} else {
// 可以选择让它下次被重新拾取 (needsEtagUpdate 保持 true)
log.warn("[Worker {}] Folder {} ETag update failed (attempt {}). Will retry.", workerId, folderPath, attempts);
}
mongoTemplate.updateFirst(query, update, FileDocument.class);
}
private static String getParentDbPath(String currentFolderPath) {
if (CharSequenceUtil.isBlank(currentFolderPath) || currentFolderPath.equals("/")) {
return null; // 根目录没有父目录,或者已经是根
}
// 确保路径以 "/" 开头和结尾
if (!currentFolderPath.endsWith("/") || !currentFolderPath.startsWith("/")) {
return null;
}
// 移除尾部的斜杠以便找到倒数第二个斜杠
String pathWithoutTrailingSlash = currentFolderPath.substring(0, currentFolderPath.length() - 1);
int lastSlashIndex = pathWithoutTrailingSlash.lastIndexOf('/');
if (lastSlashIndex < 0) { // 例如 "folder/" 这种情况,父路径是 "/"
return "/";
}
if (lastSlashIndex == 0) { // 例如 "/folder/", 父路径是 "/"
return "/";
}
// 例如 "/A/B/", pathWithoutTrailingSlash = "/A/B", lastSlashIndex 在 A 后面
// substring(0, lastSlashIndex + 1) 会得到 "/A/"
return pathWithoutTrailingSlash.substring(0, lastSlashIndex + 1);
}
private String formatChildRepresentation(String workerId, FileDocument child) {
if (child.getEtag() == null) {
String username = userService.getUserNameById(child.getUserId());
Path path = Paths.get(fileProperties.getRootDir(), username, child.getPath(), child.getName());
if (Files.exists(path)) {
// 只对文件执行此操作
if (!child.getIsFolder()) {
return updateFileEtagAndMarkParentAsync(username, path.toFile());
} else {
// 子文件夹ETag为null,父文件夹本次计算中止
return null;
}
} else {
log.warn("[Worker {}] File {} does not exist. Skipping.", workerId, path);
return "";
}
}
return child.getName() + ":" + child.getEtag();
}
@PreDestroy
public void cleanup() {
if (executorMarkedFoldersService != null) {
executorMarkedFoldersService.shutdown();
}
}
}

View File

@ -0,0 +1,29 @@
package com.jmal.clouddisk.lucene;
import lombok.Getter;
/**
* 索引状态
*/
@Getter
public enum IndexStatus {
/**
* 未索引,待索引
*/
NOT_INDEX(0),
/**
* 正在进行索引
*/
INDEXING(1),
/**
* 已完成索引
*/
INDEXED(2);
private final int status;
IndexStatus(int status) {
this.status = status;
}
}

View File

@ -0,0 +1,192 @@
package com.jmal.clouddisk.lucene;
import com.jmal.clouddisk.config.FileProperties;
import jakarta.annotation.PreDestroy;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
import org.apache.lucene.analysis.ngram.NGramTokenFilter;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.search.ControlledRealTimeReopenThread;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.SearcherFactory;
import org.apache.lucene.search.SearcherManager;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;
/**
* @author jmal
* @Description Lucene Config
* @Date 2021/4/27 4:09 下午
*/
@Configuration
@RequiredArgsConstructor
@Slf4j
public class LuceneConfig {
private final FileProperties fileProperties;
private ControlledRealTimeReopenThread<IndexSearcher> controlledRealTimeReopenThread;
private IndexWriter indexWriterInstance; // 用于存储 IndexWriter Bean 的实例
private SearcherManager searcherManagerInstance; // 用于存储 SearcherManager Bean 的实例
public static final boolean IGNORE_CASE_FOR_NGRAM = true; // 是否忽略大小写 (可调)
/**
* 创建一个 Analyzer 实例
*/
@Bean
public Analyzer analyzer() {
return new SmartChineseAnalyzer();
}
private Path getIndexDir() {
return Paths.get(fileProperties.getRootDir(), fileProperties.getLuceneIndexDir());
}
/**
* 索引位置
*/
@Bean
public Directory luceneDirectory() throws IOException {
return FSDirectory.open(getIndexDir());
}
/**
* 创建一个用于精确子串匹配的 N-Gram Analyzer 实例
* 策略: KeywordTokenizer -> LowerCaseFilter (可选) -> NGramTokenFilter
*/
@Bean("ngramAnalyzer") // 给它一个不同的名字,以便注入
public Analyzer ngramAnalyzer() {
return new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
Tokenizer tokenizer = new KeywordTokenizer(); // 将整个代码段视为一个token
TokenStream stream = tokenizer;
if (IGNORE_CASE_FOR_NGRAM) {
stream = new LowerCaseFilter(stream); // 转换为小写
}
// 从这个 (可能已小写的) 大 Token 中生成 N-Grams
stream = new NGramTokenFilter(stream, fileProperties.getNgramMinSize(), fileProperties.getNgramMaxSize(), false);
return new TokenStreamComponents(tokenizer, stream);
}
};
}
/**
* 创建一个用于处理精确子串查询词的 Analyzer 实例
* 策略: KeywordTokenizer -> LowerCaseFilter (可选)
* 查询词不应该被N-Gram分解
*/
@Bean("keywordLowercaseAnalyzer")
public Analyzer keywordLowercaseAnalyzer() {
return new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
Tokenizer tokenizer = new KeywordTokenizer();
TokenStream stream = tokenizer;
if (IGNORE_CASE_FOR_NGRAM) { // 与 ngramAnalyzer 的大小写策略保持一致
stream = new LowerCaseFilter(stream);
}
return new TokenStreamComponents(tokenizer, stream);
}
};
}
@Bean
public IndexWriter indexWriter(Directory directory, Analyzer analyzer, // 注入默认的 SmartChineseAnalyzer (确保bean名为 "analyzer")
@Qualifier("ngramAnalyzer") Analyzer ngramAnalyzer // 注入 ngramAnalyzer
) throws IOException {
Map<String, Analyzer> analyzerPerField = new HashMap<>();
analyzerPerField.put(LuceneService.FIELD_CONTENT_NGRAM, ngramAnalyzer);
analyzerPerField.put(LuceneService.FIELD_FILENAME_NGRAM, ngramAnalyzer);
analyzerPerField.put(LuceneService.FIELD_TAG_NAME_NGRAM, ngramAnalyzer);
// 如果其他字段也需要特定分析器,在这里添加
// PerFieldAnalyzerWrapper 允许为特定字段指定分析器,其他字段使用默认分析器
PerFieldAnalyzerWrapper wrapperAnalyzer = new PerFieldAnalyzerWrapper(analyzer, analyzerPerField);
IndexWriterConfig indexWriterConfig = new IndexWriterConfig(wrapperAnalyzer);
indexWriterConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
this.indexWriterInstance = new IndexWriter(directory, indexWriterConfig);
return this.indexWriterInstance;
}
/**
* SearcherManager管理
*
* @return SearcherManager
*/
@Bean
public SearcherManager searcherManager(IndexWriter indexWriter) throws IOException {
this.searcherManagerInstance = new SearcherManager(indexWriter, false, false, new SearcherFactory());
controlledRealTimeReopenThread = new ControlledRealTimeReopenThread<>(indexWriter, searcherManagerInstance,
5.0, 0.025);
controlledRealTimeReopenThread.setDaemon(true);
//线程名称
controlledRealTimeReopenThread.setName("NRTReopenThread");
// 开启线程
controlledRealTimeReopenThread.start();
return searcherManagerInstance;
}
@PreDestroy
public void destroy() {
log.info("Shutting down Lucene components...");
// 1. 关闭 NRT Reopen 线程 (它会尝试关闭 SearcherManager)
if (this.controlledRealTimeReopenThread != null) {
try {
log.debug("Closing ControlledRealTimeReopenThread...");
this.controlledRealTimeReopenThread.close(); // 这个close()会等待线程结束
log.info("ControlledRealTimeReopenThread closed.");
} catch (Exception e) { // InterruptedException or IOException
log.error("Error closing ControlledRealTimeReopenThread: {}", e.getMessage(), e);
}
}
// 2. SearcherManager 可能已经被 cRTReopenThead 关闭,但作为保险再检查一次
// 实际上,SearcherManager.close() 也会关闭它内部的 IndexReader,
// 但通常不会关闭外部传入的 IndexWriter。
if (this.searcherManagerInstance != null) {
// SearcherManager 的 close() 通常由 cRTReopenThead.close() 间接触发,
// 但如果 cRTReopenThead 未成功初始化或启动,这里可以作为后备。
// 然而,重复关闭可能导致问题,所以最好依赖 cRTReopenThead 的关闭。
// 如果 cRTReopenThead.close() 确保了 SearcherManager 关闭,这里可以不显式调用。
log.debug("SearcherManager should have been closed by NRTReopenThread.");
}
// 3. 最重要:关闭 IndexWriter
// 这会提交任何挂起的更改,释放写锁,并关闭文件句柄。
if (this.indexWriterInstance != null && this.indexWriterInstance.isOpen()) {
try {
log.info("Closing IndexWriter...");
this.indexWriterInstance.close();
log.info("IndexWriter closed.");
} catch (IOException e) {
log.error("Error closing IndexWriter: {}", e.getMessage(), e);
}
}
log.info("Lucene components shutdown complete.");
}
}

View File

@ -0,0 +1,698 @@
package com.jmal.clouddisk.lucene;
import cn.hutool.core.io.CharsetDetector;
import cn.hutool.core.io.FileTypeUtil;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.text.CharSequenceUtil;
import cn.hutool.core.thread.ThreadUtil;
import com.jmal.clouddisk.config.FileProperties;
import com.jmal.clouddisk.model.FileIndex;
import com.jmal.clouddisk.model.FileIntroVO;
import com.jmal.clouddisk.model.Tag;
import com.jmal.clouddisk.service.Constants;
import com.jmal.clouddisk.service.IUserService;
import com.jmal.clouddisk.service.impl.CommonFileService;
import com.jmal.clouddisk.util.FileContentTypeUtils;
import com.jmal.clouddisk.util.HashUtil;
import com.jmal.clouddisk.util.MyFileUtils;
import com.mongodb.client.AggregateIterable;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.lucene.document.*;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.mozilla.universalchardet.UniversalDetector;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;
import org.springframework.stereotype.Service;
import java.io.*;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.charset.UnsupportedCharsetException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
import static com.jmal.clouddisk.service.impl.CommonFileService.COLLECTION_NAME;
/**
* @author jmal
* <p>
* 优化索引
* indexWriter.forceMerge(1);
* indexWriter.commit();
* @Description LuceneService
* @Date 2021/4/27 4:44 下午
*/
@Service
@Slf4j
@RequiredArgsConstructor
public class LuceneService {
private final FileProperties fileProperties;
private final MongoTemplate mongoTemplate;
private final IndexWriter indexWriter;
private final IUserService userService;
private final ReadContentService readContentService;
private final RebuildIndexTaskService rebuildIndexTaskService;
private final SearchFileService searchFileService;
private final EtagService esTagService;
public static final String MONGO_INDEX_FIELD = "index";
/**
* 创建索引线程池
*/
private ExecutorService executorCreateIndexService;
/**
* 更新索引内容线程池
*/
private ExecutorService executorUpdateContentIndexService;
/**
* 更新大文件索引内容线程池
*/
private ExecutorService executorUpdateBigContentIndexService;
/**
* 定时任务线程池
*/
ScheduledExecutorService scheduler;
/**
* 新建索引文件缓冲队列大小
*/
private static final int CREATE_INDEX_QUEUE_SIZE = 40960;
/**
* 新建索引文件缓冲队列
*/
private ArrayBlockingQueue<String> indexFileQueue;
/**
* 新建索引文件内容缓冲队列
*/
private ArrayBlockingQueue<String> indexFileContentQueue;
private final AtomicBoolean processingUnIndexedScheduled = new AtomicBoolean(false);
public static final String FIELD_CONTENT_NGRAM = "content_ngram";
public static final String FIELD_FILENAME_NGRAM = "filename_ngram";
public static final String FIELD_TAG_NAME_NGRAM = "tagName_ngram";
public static final String FIELD_CONTENT_FUZZY = "content";
public static final String FIELD_FILENAME_FUZZY = "filename";
public static final String FIELD_TAG_NAME_FUZZY = "tagName";
// 定义分段大小
private static final int CHUNK_SIZE_CHARS = 1024; // 例如,每 1KB 左右一个块,或者按行数
private static final int CHUNK_OVERLAP_CHARS = 7; // 段落间的重叠字符数,防止边界切割问题 (NGramTokenFilter本身可能处理部分边界,但显式重叠更保险)
public static final int BYTES_PER_MB = 1024 * 1024;
private static final long MEMORY_PER_SMALL_THREAD_MB = 500;
private static final long MEMORY_PER_BIG_THREAD_MB = 4096;
@PostConstruct
public void init() {
if (executorCreateIndexService == null) {
int processors = Runtime.getRuntime().availableProcessors() - 2;
executorCreateIndexService = ThreadUtil.newFixedExecutor(Math.max(processors, 3), 100, "createIndexFileTask", true);
}
// 获取jvm可用内存
long maxMemory = Runtime.getRuntime().maxMemory();
if (executorUpdateContentIndexService == null) {
// 获取可用处理器数量
int smallProcessors = Runtime.getRuntime().availableProcessors() - 3;
// 设置线程数, 假设每个线程占用内存为500M
int maxSmallProcessors = Math.toIntExact((maxMemory / BYTES_PER_MB) / MEMORY_PER_SMALL_THREAD_MB);
if (smallProcessors > maxSmallProcessors) {
smallProcessors = maxSmallProcessors;
}
smallProcessors = Math.max(smallProcessors, 1);
executorUpdateContentIndexService = ThreadUtil.newFixedExecutor(smallProcessors, 20, "updateContentIndexTask", true);
}
if (executorUpdateBigContentIndexService == null) {
// 设置线程数, 假设每个线程占用内存为4G
int bigProcessors = Math.toIntExact((maxMemory / BYTES_PER_MB) / MEMORY_PER_BIG_THREAD_MB);
bigProcessors = Math.max(bigProcessors, 1);
executorUpdateBigContentIndexService = ThreadUtil.newFixedExecutor(bigProcessors, 100, "updateBigContentIndexTask", true);
}
log.info("NGRAM_MAX_CONTENT_LENGTH_MB:{}, NGRAM_MIN_SIZE: {}, ngramMaxSize: {}", fileProperties.getNgramMaxContentLengthMB(), fileProperties.getNgramMinSize(), fileProperties.getNgramMaxSize());
}
/**
* 推送至新建索引文件缓存队列
*
* @param fileId fileId
*/
public void pushCreateIndexQueue(String fileId) {
if (CharSequenceUtil.isBlank(fileId)) {
return;
}
try {
getIndexFileQueue().put(fileId);
} catch (InterruptedException e) {
log.error("推送新建索引队列失败, fileId: {}, {}", fileId, e.getMessage(), e);
Thread.currentThread().interrupt();
}
}
/**
* 推送至新建索引文件缓存队列
*
* @param fileId fileId
*/
public void pushCreateIndexContentQueue(String fileId) {
if (CharSequenceUtil.isBlank(fileId)) {
return;
}
try {
getIndexFileContentQueue().put(fileId);
} catch (InterruptedException e) {
log.error("推送新建索引内容队列失败, fileId: {}, {}", fileId, e.getMessage(), e);
Thread.currentThread().interrupt();
}
}
private ArrayBlockingQueue<String> getIndexFileQueue() {
if (indexFileQueue == null) {
indexFileQueue = new ArrayBlockingQueue<>(CREATE_INDEX_QUEUE_SIZE);
if (scheduler == null) {
scheduler = new ScheduledThreadPoolExecutor(1, ThreadUtil.createThreadFactory("luceneScheduler"));
}
createIndexFileTask();
}
return indexFileQueue;
}
private ArrayBlockingQueue<String> getIndexFileContentQueue() {
if (indexFileContentQueue == null) {
indexFileContentQueue = new ArrayBlockingQueue<>(CREATE_INDEX_QUEUE_SIZE);
}
return indexFileContentQueue;
}
/**
* 新建索引文件任务
*/
private void createIndexFileTask() {
getIndexFileQueue();
getIndexFileContentQueue();
scheduler.scheduleAtFixedRate(() -> {
try {
List<String> fileIdList = new ArrayList<>(indexFileQueue.size());
List<String> toBeIndexedFileIdList = new ArrayList<>(indexFileContentQueue.size());
executorCreateIndexService.execute(() -> {
// 添加待索引标记
indexFileContentQueue.drainTo(toBeIndexedFileIdList);
addToBeIndexedFlagOfDoc(toBeIndexedFileIdList);
// 添加不带文件内容的索引
indexFileQueue.drainTo(fileIdList);
if (!fileIdList.isEmpty()) {
createIndexFiles(fileIdList);
}
});
// 更新任务进度
rebuildIndexTaskService.updateTaskProgress();
} catch (Exception e) {
log.error("创建索引失败", e);
}
}, 0, 1000, TimeUnit.MILLISECONDS);
}
/**
* 创建索引
*
* @param fileIdList fileIdList
*/
private void createIndexFiles(List<String> fileIdList) {
// 提取出fileIdList
List<FileIntroVO> fileIntroVOList = searchFileService.getFileIntroVOs(fileIdList);
for (FileIntroVO fileIntroVO : fileIntroVOList) {
rebuildIndexTaskService.incrementNotIndexTaskSize();
updateIndex(false, fileIntroVO);
}
rebuildIndexTaskService.removeDeletedFlag(fileIdList);
}
private void updateIndex(boolean readContent, FileIntroVO fileIntroVO) {
try {
File file = getFileByFileIntroVO(fileIntroVO);
boolean isContent = checkFileContent(file);
if (!readContent && isContent) {
pushCreateIndexContentQueue(fileIntroVO.getId());
}
if (!readContent && !isContent) {
rebuildIndexTaskService.incrementIndexedTaskSize();
}
FileIndex fileIndex = new FileIndex(file, fileIntroVO);
fileIndex.setTagName(getTagName(fileIntroVO));
setFileIndex(fileIndex);
if (readContent) {
String sha256 = HashUtil.sha256(file);
try (StringWriter contentWriter = new StringWriter()) {
readFileContent(file, fileIntroVO.getId(), contentWriter);
String fullContent = contentWriter.toString();
if (CharSequenceUtil.isBlank(fullContent)) {
updateIndexStatus(fileIntroVO, IndexStatus.INDEXED);
return;
}
// 调用改造后的 updateIndexDocument
updateIndexDocument(indexWriter, fileIndex, fullContent, fileIndex.getFileIndexHash(sha256));
startProcessFilesToBeIndexed();
}
} else {
updateIndexDocument(indexWriter, fileIndex, null, fileIndex.getFileIndexHash(null));
}
log.debug("添加索引, filepath: {}", file.getAbsoluteFile());
} catch (Exception e) {
log.warn("updateIndexError: {}", e.getMessage(), e);
} finally {
updateIndexStatus(fileIntroVO, IndexStatus.INDEXED);
if (readContent) {
rebuildIndexTaskService.incrementIndexedTaskSize();
}
}
}
/**
* 更新索引状态
*
* @param fileIntroVO fileIntroVO
* @param indexStatus IndexStatus
*/
private void updateIndexStatus(FileIntroVO fileIntroVO, IndexStatus indexStatus) {
Query query = new Query();
query.addCriteria(Criteria.where("_id").is(fileIntroVO.getId()));
Update update = new Update();
update.set(MONGO_INDEX_FIELD, indexStatus.getStatus());
mongoTemplate.updateFirst(query, update, COLLECTION_NAME);
// set etag
String username = userService.getUserNameById(fileIntroVO.getUserId());
esTagService.updateFileEtagAsync(username, getFileByFileIntroVO(fileIntroVO));
}
private File getFileByFileIntroVO(FileIntroVO fileIntroVO) {
String username = userService.getUserNameById(fileIntroVO.getUserId());
return Paths.get(fileProperties.getRootDir(), username, fileIntroVO.getPath(), fileIntroVO.getName()).toFile();
}
private String getTagName(FileIntroVO fileIntroVO) {
if (fileIntroVO != null && fileIntroVO.getTags() != null && !fileIntroVO.getTags().isEmpty()) {
return fileIntroVO.getTags().stream().map(Tag::getName).reduce((a, b) -> a + " " + b).orElse("");
}
return null;
}
/**
* 构建FileIndex
*
* @param fileIndex FileIndex
*/
public void setFileIndex(FileIndex fileIndex) {
File file = fileIndex.getFile();
if (!FileUtil.exist(fileIndex.getFile())) {
return;
}
fileIndex.setIsFolder(file.isDirectory());
fileIndex.setName(file.getName());
try {
fileIndex.setModified(Files.getLastModifiedTime(file.toPath()).toMillis());
} catch (IOException e) {
log.error("获取文件修改时间失败, file: {}, {}", file.getAbsolutePath(), e.getMessage(), e);
}
fileIndex.setSize(file.length());
setType(file, fileIndex);
}
private void setType(File file, FileIndex fileIndex) {
String fileName = file.getName();
String suffix = MyFileUtils.extName(fileName);
fileIndex.setType(Constants.OTHER);
if (CharSequenceUtil.isBlank(suffix)) {
fileIndex.setType(Constants.OTHER);
return;
}
String contentType = FileContentTypeUtils.getContentType(suffix);
if (CharSequenceUtil.isBlank(contentType)) {
fileIndex.setType(Constants.OTHER);
return;
}
if (contentType.startsWith(Constants.CONTENT_TYPE_IMAGE)) {
fileIndex.setType(Constants.CONTENT_TYPE_IMAGE);
}
if (contentType.startsWith(Constants.VIDEO)) {
fileIndex.setType(Constants.VIDEO);
}
if (contentType.startsWith(Constants.AUDIO)) {
fileIndex.setType(Constants.AUDIO);
}
if (fileProperties.getDocument().contains(suffix)) {
fileIndex.setType(Constants.DOCUMENT);
}
}
private void readFileContent(File file, String fileId, Writer writer) {
String charset = null;
try {
if (file == null || !file.isFile() || file.length() < 1) {
return;
}
String type = FileTypeUtil.getType(file).toLowerCase();
switch (type) {
case "pdf" -> readContentService.readPdfContent(file, fileId, writer);
case "dwg" -> readContentService.dwg2mxweb(file, fileId);
case "epub" -> readContentService.readEpubContent(file, fileId, writer);
case "ppt", "pptx" -> readContentService.readPPTContent(file, writer);
case "doc", "docx" -> readContentService.readWordContent(file, writer);
case "xls", "xlsx" -> readContentService.readExcelContent(file, writer);
default -> {
if (fileProperties.getSimText().contains(type)) {
charset = UniversalDetector.detectCharset(file);
if (CharSequenceUtil.isBlank(charset)) {
charset = String.valueOf(CharsetDetector.detect(file, StandardCharsets.UTF_8));
}
if (CharSequenceUtil.isBlank(charset)) {
return;
}
// 对于纯文本,可以流式读取
try (Reader reader = new InputStreamReader(new FileInputStream(file), Charset.forName(charset))) {
reader.transferTo(writer);
}
} else {
String contentType = CommonFileService.getContentType(file, FileContentTypeUtils.getContentType(file, MyFileUtils.extName(file.getName())));
if (contentType.contains("charset=utf-8")) {
try (Reader reader = new InputStreamReader(new FileInputStream(file), StandardCharsets.UTF_8)) {
reader.transferTo(writer);
}
}
}
}
}
} catch (UnsupportedCharsetException unsupportedCharsetException) {
log.warn("不支持的字符集, charset: {}, file: {}, {}", file.getAbsolutePath(), charset, unsupportedCharsetException.getMessage());
} catch (Exception e) {
log.error("读取文件内容失败, file: {}, {}", file.getAbsolutePath(), e.getMessage(), e);
}
}
private boolean checkFileContent(File file) {
try {
if (file == null) {
return false;
}
if (!file.isFile() || file.length() < 1) {
return false;
}
String type = FileTypeUtil.getType(file).toLowerCase();
if (MyFileUtils.hasContentFile(type)) {
return true;
}
if (fileProperties.getSimText().contains(type)) {
return true;
}
return MyFileUtils.hasCharset(file);
} catch (Exception e) {
return false;
}
}
public void deleteIndexDocuments(List<String> fileIds) {
try {
for (String fileId : fileIds) {
Term term = new Term("id", fileId);
indexWriter.deleteDocuments(term);
}
indexWriter.commit();
} catch (IOException e) {
log.error("删除索引失败, fileIds: {}, {}", fileIds, e.getMessage(), e);
}
}
/**
* 添加/更新索引
*
* @param indexWriter indexWriter
* @param fileIndex FileIndex
* @param fullContent fullContent
* @param fileIndexHash fileIndexHash 唯一索引标识
*/
public void updateIndexDocument(IndexWriter indexWriter, FileIndex fileIndex, String fullContent, String fileIndexHash) {
String fileId = fileIndex.getFileId();
try {
if (searchFileService.existsSha256(fileIndexHash)) {
return;
}
String fileName = (fileIndex.getName() == null ? "" : fileIndex.getName()) + " " + fileIndex.getRemark();
String tagName = fileIndex.getTagName();
Boolean isFolder = fileIndex.getIsFolder();
Boolean isFavorite = fileIndex.getIsFavorite();
String path = fileIndex.getPath();
Document newDocument = new Document();
newDocument.add(new StringField("id", fileId, Field.Store.YES));
newDocument.add(new StringField(IUserService.USER_ID, fileIndex.getUserId(), Field.Store.NO));
newDocument.add(new StringField(Constants.ETAG, fileIndexHash, Field.Store.NO));
if (fileIndex.getType() != null) {
newDocument.add(new StringField("type", fileIndex.getType(), Field.Store.NO));
}
if (CharSequenceUtil.isNotBlank(fileName)) {
newDocument.add(new Field(FIELD_FILENAME_NGRAM, fileName, TextField.TYPE_NOT_STORED));
newDocument.add(new TextField(FIELD_FILENAME_FUZZY, fileName, Field.Store.NO));
}
if (isFolder != null) {
newDocument.add(new IntPoint(Constants.IS_FOLDER, isFolder ? 1 : 0));
}
if (isFavorite != null) {
newDocument.add(new IntPoint(Constants.IS_FAVORITE, isFavorite ? 1 : 0));
}
if (path != null) {
newDocument.add(new StringField(Constants.PATH_FIELD, path, Field.Store.NO));
}
if (CharSequenceUtil.isNotBlank(tagName)) {
newDocument.add(new Field(FIELD_TAG_NAME_NGRAM, tagName, TextField.TYPE_NOT_STORED));
newDocument.add(new TextField(FIELD_TAG_NAME_FUZZY, tagName, Field.Store.NO));
}
if (CharSequenceUtil.isNotBlank(fullContent)) {
newDocument.add(new TextField(FIELD_CONTENT_FUZZY, fullContent, Field.Store.NO));
}
if (CharSequenceUtil.isNotBlank(fullContent) && fileProperties.getExactSearch()) {
String contentForNgram;
// 将完整内容字符串转换为内存中的输入流
try (InputStream contentStream = new ByteArrayInputStream(fullContent.getBytes(StandardCharsets.UTF_8))) {
// 调用内存高效的流式截断方法
contentForNgram = getContentForNgram(contentStream, fileIndex);
} catch (IOException e) {
log.error("Error creating ByteArrayInputStream: {}", e.getMessage(), e);
return;
}
if (CharSequenceUtil.isNotBlank(contentForNgram)) {
List<String> chunks = segmentContent(contentForNgram);
for (String chunk : chunks) {
if (CharSequenceUtil.isNotBlank(chunk)) {
newDocument.add(new Field(FIELD_CONTENT_NGRAM, chunk, TextField.TYPE_NOT_STORED));
}
}
}
}
if (fileIndex.getModified() != null) {
newDocument.add(new NumericDocValuesField("modified", fileIndex.getModified()));
}
if (fileIndex.getSize() != null) {
newDocument.add(new NumericDocValuesField("size", fileIndex.getSize()));
}
indexWriter.updateDocument(new Term("id", fileId), newDocument);
} catch (IOException e) {
log.error("更新索引失败, fileId: {}, {}", fileId, e.getMessage(), e);
}
}
/**
* 从输入流中高效地截取用于N-Gram的内容,避免OOM。
*
* @param contentStream 内容的输入流
* @param fileIndex 文件元数据,用于日志记录
* @return 截断后的UTF-8字符串
* @throws IOException 读取流异常
*/
private String getContentForNgram(InputStream contentStream, FileIndex fileIndex) throws IOException {
int maxContentLengthInBytes = fileProperties.getNgramMaxContentLength();
byte[] buffer = new byte[8192]; // 8KB的读取缓冲区
int bytesRead;
// 使用 ByteArrayOutputStream 来存储截取的结果
try (ByteArrayOutputStream baos = new ByteArrayOutputStream(maxContentLengthInBytes)) {
while ((bytesRead = contentStream.read(buffer)) != -1) {
int remainingCapacity = maxContentLengthInBytes - baos.size();
if (bytesRead > remainingCapacity) {
baos.write(buffer, 0, remainingCapacity);
log.warn("内容已截断以进行N-Gram索引(截断至 {} 字节), 文件大小: {}MB 文件: {}", maxContentLengthInBytes, fileIndex.getSize() / BYTES_PER_MB, Paths.get(fileIndex.getPath(), fileIndex.getName()));
break; // 已达到上限
}
baos.write(buffer, 0, bytesRead);
}
byte[] contentBytes = baos.toByteArray();
// 进行UTF-8字符边界检查,防止乱码
int effectiveLength = contentBytes.length;
while (effectiveLength > 0 && (contentBytes[effectiveLength - 1] & 0xC0) == 0x80) {
effectiveLength--;
}
if (effectiveLength == 0) {
return "";
}
return new String(contentBytes, 0, effectiveLength, StandardCharsets.UTF_8);
}
}
private List<String> segmentContent(String content) {
List<String> chunks = new ArrayList<>();
if (content == null || content.isEmpty()) {
return chunks;
}
int length = content.length();
if (length <= CHUNK_SIZE_CHARS) { // 如果内容本身小于或等于块大小,则作为一个块
chunks.add(content);
return chunks;
}
// 确保步进大于0
int step = CHUNK_SIZE_CHARS - CHUNK_OVERLAP_CHARS;
for (int i = 0; i < length; i += step) {
int end = Math.min(i + CHUNK_SIZE_CHARS, length);
chunks.add(content.substring(i, end));
if (end == length) {
break; // 已到达内容末尾
}
}
return chunks;
}
public FileIntroVO getFileIntroVO(String fileId) {
Query query = new Query();
query.addCriteria(Criteria.where("_id").is(fileId));
query.fields().include("id", "name", "userId", "path", "isFolder", "isFavorite", "remark", "tags", "etag", "size");
return mongoTemplate.findOne(query, FileIntroVO.class, COLLECTION_NAME);
}
/**
* 添加待索引标记
*/
private void addToBeIndexedFlagOfDoc(List<String> fielIdList) {
if (fielIdList.isEmpty()) {
return;
}
Query query = new Query();
query.addCriteria(Criteria.where("_id").in(fielIdList));
Update update = new Update();
update.set(MONGO_INDEX_FIELD, IndexStatus.NOT_INDEX.getStatus());
mongoTemplate.updateMulti(query, update, COLLECTION_NAME);
// 添加待索引标记
startProcessFilesToBeIndexed();
}
/**
* 处理待索引文件
*/
public void processFilesToBeIndexed() throws IOException {
boolean run = true;
log.debug("开始处理待索引文件");
while (run) {
if (!hasUnIndexFile()) {
log.debug("待索引文件处理完成");
rebuildIndexTaskService.rebuildingIndexCompleted();
indexWriter.commit();
run = false;
}
List<org.bson.Document> pipeline = Arrays.asList(new org.bson.Document("$match", new org.bson.Document(MONGO_INDEX_FIELD, IndexStatus.NOT_INDEX.getStatus())), new org.bson.Document("$project", new org.bson.Document("_id", 1)), new org.bson.Document("$limit", 8));
AggregateIterable<org.bson.Document> aggregateIterable = mongoTemplate.getCollection(COLLECTION_NAME).aggregate(pipeline);
for (org.bson.Document document : aggregateIterable) {
String fileId = document.getObjectId("_id").toHexString();
FileIntroVO fileIntroVO = getFileIntroVO(fileId);
if (fileIntroVO != null) {
// 处理待索引文件
updateIndexStatus(fileIntroVO, IndexStatus.INDEXING);
processFileThreaded(fileIntroVO);
}
}
}
}
/**
* 是否存在待索引文件
*/
private boolean hasUnIndexFile() {
Query query = new Query();
query.addCriteria(Criteria.where(MONGO_INDEX_FIELD).is(IndexStatus.NOT_INDEX.getStatus()));
long count = mongoTemplate.count(query, COLLECTION_NAME);
return count > 0;
}
private void processFileThreaded(FileIntroVO fileIntroVO) {
long size = fileIntroVO.getSize();
if (RebuildIndexTaskService.isSyncFile()) {
// 单线程处理
updateIndex(true, fileIntroVO);
} else {
// 根据文件大小选择多线程处理
if (size > 10 * 1024 * 1024) {
// 大文件,使用特定线程池处理
executorUpdateBigContentIndexService.execute(() -> updateIndex(true, fileIntroVO));
} else {
// 小文件,使用普通线程池处理
executorUpdateContentIndexService.execute(() -> updateIndex(true, fileIntroVO));
}
}
}
private void startProcessFilesToBeIndexed() {
if (processingUnIndexedScheduled.compareAndSet(false, true)) {
executorCreateIndexService.execute(() -> {
try {
processFilesToBeIndexed();
} catch (IOException e) {
log.error("处理待索引文件内容失败", e);
} finally {
processingUnIndexedScheduled.set(false);
}
});
}
}
@PreDestroy
public void destroy() {
if (executorCreateIndexService != null) {
executorCreateIndexService.shutdown();
}
if (executorUpdateContentIndexService != null) {
executorUpdateContentIndexService.shutdown();
}
if (executorUpdateBigContentIndexService != null) {
executorUpdateBigContentIndexService.shutdown();
}
if (scheduler != null) {
scheduler.shutdown();
}
}
}

View File

@ -0,0 +1,326 @@
package com.jmal.clouddisk.lucene;
import cn.hutool.core.text.CharSequenceUtil;
import com.jmal.clouddisk.media.VideoProcessService;
import com.jmal.clouddisk.ocr.OcrService;
import com.jmal.clouddisk.service.Constants;
import com.jmal.clouddisk.service.impl.CommonFileService;
import com.jmal.clouddisk.util.FileContentUtil;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import nl.siegmann.epublib.domain.Book;
import nl.siegmann.epublib.domain.Resource;
import nl.siegmann.epublib.domain.Spine;
import nl.siegmann.epublib.epub.EpubReader;
import org.apache.pdfbox.Loader;
import org.apache.pdfbox.cos.COSName;
import org.apache.pdfbox.io.RandomAccessReadBufferedFile;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.PDResources;
import org.apache.pdfbox.pdmodel.graphics.PDXObject;
import org.apache.pdfbox.pdmodel.graphics.image.PDImageXObject;
import org.apache.pdfbox.rendering.PDFRenderer;
import org.apache.pdfbox.text.PDFTextStripper;
import org.apache.poi.hslf.usermodel.HSLFShape;
import org.apache.poi.hslf.usermodel.HSLFSlide;
import org.apache.poi.hslf.usermodel.HSLFSlideShow;
import org.apache.poi.hslf.usermodel.HSLFTextShape;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.hwpf.HWPFDocument;
import org.apache.poi.hwpf.extractor.WordExtractor;
import org.apache.poi.openxml4j.exceptions.OLE2NotOfficeXmlFileException;
import org.apache.poi.poifs.filesystem.POIFSFileSystem;
import org.apache.poi.ss.usermodel.*;
import org.apache.poi.xslf.usermodel.XMLSlideShow;
import org.apache.poi.xslf.usermodel.XSLFShape;
import org.apache.poi.xslf.usermodel.XSLFSlide;
import org.apache.poi.xslf.usermodel.XSLFTextShape;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.apache.poi.xwpf.usermodel.*;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.springframework.stereotype.Service;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.regex.Pattern;
@Service
@RequiredArgsConstructor
@Slf4j
public class ReadContentService {
private final OcrService ocrService;
public final CommonFileService commonFileService;
public final VideoProcessService videoProcessService;
/**
* 将 DWG 文件转换为 MXWeb 文件
*
* @param file 文件
* @param fileId 文件 ID
*/
public void dwg2mxweb(File file, String fileId) {
String username = commonFileService.getUsernameByAbsolutePath(Path.of(file.getAbsolutePath()));
// 生成封面图像
if (CharSequenceUtil.isNotBlank(fileId)) {
String outputName = file.getName() + Constants.MXWEB_SUFFIX;
FileContentUtil.dwgConvert(file.getAbsolutePath(), videoProcessService.getVideoCacheDir(username, fileId), outputName);
}
}
public static boolean checkPageContent(PDDocument document, int pageIndex) throws IOException {
PDPage page = document.getPage(pageIndex); // 获取页面
if (page == null) {
return false;
}
// 检查是否含有图片
PDResources resources = page.getResources();
if (resources == null) {
return false;
}
for (COSName xObjectName : resources.getXObjectNames()) {
PDXObject xObject = resources.getXObject(xObjectName);
if (xObject instanceof PDImageXObject) {
// 如果找到至少一张图片,则可以提前退出
return true;
}
}
return false;
}
public void readPdfContent(File file, String fileId, Writer writer) {
try (PDDocument document = Loader.loadPDF(new RandomAccessReadBufferedFile(file))) {
String username = commonFileService.getUsernameByAbsolutePath(Path.of(file.getAbsolutePath()));
// 生成封面图像
if (CharSequenceUtil.isNotBlank(fileId)) {
File coverFile = FileContentUtil.pdfCoverImage(file, document, videoProcessService.getVideoCacheDir(username, fileId));
commonFileService.updateCoverFileDocument(fileId, coverFile);
}
PDFRenderer pdfRenderer = new PDFRenderer(document);
PDFTextStripper pdfStripper = new PDFTextStripper();
for (int pageIndex = 0; pageIndex < document.getNumberOfPages(); pageIndex++) { // 使用 0-based 索引
readPdfOfPage(file, pageIndex, pdfStripper, document, writer, pdfRenderer, username);
}
} catch (IOException e) {
FileContentUtil.readFailed(file, e);
}
}
private void readPdfOfPage(File file, int pageIndex, PDFTextStripper pdfStripper, PDDocument document, Writer writer, PDFRenderer pdfRenderer, String username) {
try {
int pageNumber = pageIndex + 1;
pdfStripper.setStartPage(pageNumber); // PDFTextStripper 使用 1-based 索引
pdfStripper.setEndPage(pageNumber);
String text = pdfStripper.getText(document).trim();
// 如果页面包含文字,添加提取的文字
if (!text.isEmpty()) {
writer.write(text);
}
// 如果页面包含图片或没有文字,则进行 OCR
if ((checkPageContent(document, pageIndex) || text.isEmpty()) && Boolean.TRUE.equals(ocrService.getOcrConfig().getEnable())) {
String ocrText = ocrService.extractPageWithOCR(file, pdfRenderer, pageIndex, document.getNumberOfPages(), username);
if (CharSequenceUtil.isNotBlank(ocrText)) {
writer.write(ocrText);
}
}
} catch (IOException e) {
log.error("提取文字失败, {}, 页数: {}", file.getName(), pageIndex, e);
} catch (Exception e) {
log.error("提取文字失败1, {}, 页数: {}", file.getName(), pageIndex, e);
}
}
public void readEpubContent(File file, String fileId, Writer writer) {
try (InputStream fileInputStream = new FileInputStream(file)) {
// 打开 EPUB 文件
EpubReader epubReader = new EpubReader();
Book book = epubReader.readEpub(fileInputStream);
// 生成封面图像
String username = commonFileService.getUsernameByAbsolutePath(Path.of(file.getAbsolutePath()));
if (CharSequenceUtil.isNotBlank(fileId)) {
File coverFile = FileContentUtil.epubCoverImage(book, videoProcessService.getVideoCacheDir(username, fileId));
commonFileService.updateCoverFileDocument(fileId, coverFile);
}
// 获取章节内容
Spine spine = book.getSpine();
for (int i = 0; i < spine.size(); i++) {
Resource resource = spine.getResource(i);
InputStream is = resource.getInputStream();
byte[] bytes = is.readAllBytes();
String htmlContent = new String(bytes, StandardCharsets.UTF_8);
// 使用 JSoup 解析 HTML 并提取纯文本
Document document = Jsoup.parse(htmlContent);
String textContent = document.text();
writer.write(textContent);
is.close();
}
} catch (IOException e) {
FileContentUtil.readFailed(file, e);
}
}
public void readPPTContent(File file, Writer writer) {
String fileName = file.getName().toLowerCase();
try (FileInputStream fis = new FileInputStream(file)) {
if (fileName.endsWith(".pptx")) {
// 读取 .pptx 文件
try (XMLSlideShow pptx = new XMLSlideShow(fis)) {
readSlides(pptx.getSlides(), writer);
}
} else if (fileName.endsWith(".ppt")) {
// 读取 .ppt 文件
try (HSLFSlideShow ppt = new HSLFSlideShow(fis)) {
readSlides(ppt.getSlides(), writer);
}
} else {
throw new IllegalArgumentException("Unsupported file format: " + fileName);
}
} catch (IOException e) {
FileContentUtil.readFailed(file, e);
}
}
// 通用方法读取幻灯片中的文本内容
private void readSlides(Iterable<?> slides, Writer writer) throws IOException {
for (Object slide : slides) {
if (slide instanceof XSLFSlide xslfSlide) {
for (XSLFShape shape : xslfSlide.getShapes()) {
if (shape instanceof XSLFTextShape textShape) {
writer.write(textShape.getText());
writer.write(" ");
}
}
} else if (slide instanceof HSLFSlide hslfSlide) {
for (HSLFShape shape : hslfSlide.getShapes()) {
if (shape instanceof HSLFTextShape textShape) {
writer.write(textShape.getText());
writer.write(" ");
}
}
}
}
}
public void readWordContent(File file, Writer writer) {
try (FileInputStream fis = new FileInputStream(file)) {
try {
// 读取 DOCX 文件
XWPFDocument document = new XWPFDocument(fis);
// 1. 读取段落
for (XWPFParagraph paragraph : document.getParagraphs()) {
writer.write(paragraph.getText());
writer.write("\n");
}
// 2. 读取表格
for (XWPFTable table : document.getTables()) {
for (XWPFTableRow row : table.getRows()) {
for (XWPFTableCell cell : row.getTableCells()) {
writer.write(cell.getText());
writer.write("\t");
}
writer.write("\n");
}
}
// 3. 读取页眉
for (XWPFHeader header : document.getHeaderList()) {
writer.write(header.getText());
writer.write("\n");
}
// 4. 读取页脚
for (XWPFFooter footer : document.getFooterList()) {
writer.write(footer.getText());
writer.write("\n");
}
} catch (OLE2NotOfficeXmlFileException e) {
// 读取 DOC 文件
try (FileInputStream fis2 = new FileInputStream(file);
POIFSFileSystem poifs = new POIFSFileSystem(fis2);
HWPFDocument doc = new HWPFDocument(poifs)) {
WordExtractor extractor = new WordExtractor(doc);
writer.write(extractor.getText());
}
}
} catch (IOException e) {
FileContentUtil.readFailed(file, e);
}
}
// 匹配包含至少一个中文或英文字符的字符串
private static final Pattern TEXT_PATTERN = Pattern.compile(".*[a-zA-Z一-龥]+.*");
public void readExcelContent(File file, Writer writer) {
try (FileInputStream fis = new FileInputStream(file)) {
try (Workbook workbook = createWorkbook(file, fis)) {
for (Sheet sheet : workbook) {
readSheetContent(sheet, writer);
}
}
} catch (IOException e) {
FileContentUtil.readFailed(file, e);
} catch (IllegalArgumentException e) {
log.warn("Unsupported file format: {}", file.getName());
}
}
private Workbook createWorkbook(File file, FileInputStream fis) throws IOException {
if (file.getName().endsWith(".xlsx")) {
return new XSSFWorkbook(fis);
} else if (file.getName().endsWith(".xls")) {
return new HSSFWorkbook(fis);
} else {
throw new IllegalArgumentException("Unsupported file format");
}
}
private void readSheetContent(Sheet sheet, Writer writer) throws IOException {
for (Row row : sheet) {
for (Cell cell : row) {
String cellValue = getCellValueAsString(cell);
// 包含至少一个中文或英文字符
if (TEXT_PATTERN.matcher(cellValue).matches()) {
writer.write(cellValue);
writer.write(" ");
}
}
}
}
private String getCellValueAsString(Cell cell) {
if (cell == null) {
return "";
}
switch (cell.getCellType()) {
case STRING:
return cell.getStringCellValue();
case BOOLEAN:
return Boolean.toString(cell.getBooleanCellValue());
case NUMERIC:
if (DateUtil.isCellDateFormatted(cell)) {
return cell.getDateCellValue().toString();
} else {
return Double.toString(cell.getNumericCellValue());
}
case FORMULA:
return cell.getCellFormula();
case BLANK:
default:
return "";
}
}
}

View File

@ -0,0 +1,649 @@
package com.jmal.clouddisk.lucene;
import cn.hutool.core.date.TimeInterval;
import cn.hutool.core.thread.ThreadUtil;
import cn.hutool.core.util.NumberUtil;
import cn.hutool.core.util.ReUtil;
import cn.hutool.core.util.StrUtil;
import com.jmal.clouddisk.config.FileProperties;
import com.jmal.clouddisk.model.FileDocument;
import com.jmal.clouddisk.ocr.OcrService;
import com.jmal.clouddisk.service.impl.CommonFileService;
import com.jmal.clouddisk.service.impl.MenuService;
import com.jmal.clouddisk.service.impl.RoleService;
import com.jmal.clouddisk.service.impl.UserServiceImpl;
import com.jmal.clouddisk.util.ResponseResult;
import com.jmal.clouddisk.util.ResultUtil;
import com.jmal.clouddisk.util.ThrottleExecutor;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.PrefixQuery;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.math.RoundingMode;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArraySet;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantLock;
/**
* <p>重建索引任务</p>
* 重建索引分为两个步骤<br>
* 1. 同步文件到数据库<br>
* 2. 创建文件内容索引<br>
*/
@Service
@Slf4j
@RequiredArgsConstructor
public class RebuildIndexTaskService {
private final IndexWriter indexWriter;
private final MenuService menuService;
private final RoleService roleService;
private final UserServiceImpl userService;
private final FileProperties fileProperties;
private final CommonFileService commonFileService;
private final MongoTemplate mongoTemplate;
private ExecutorService syncFileVisitorService;
private SyncFileVisitor syncFileVisitor;
private double totalCount;
private final OcrService ocrService;
/**
* 接收消息的用户
*/
private static final Set<String> RECIPIENT = new CopyOnWriteArraySet<>();
private static final String MSG_SYNCED = "synced";
/**
* 待索引任务总数
*/
private static final AtomicInteger NOT_INDEX_TASK_SIZE = new AtomicInteger(0);
/**
* 完成索引任务数
*/
private static final AtomicInteger INDEXED_TASK_SIZE = new AtomicInteger(0);
/**
* 同步文件操作锁, 防止重复操作
*/
private static final ReentrantLock SYNC_FILE_LOCK = new ReentrantLock();
private Runnable syncCompleteCallback;
/**
* 进度百分比
*/
private static final Map<String, Double> PERCENT_MAP = new ConcurrentHashMap<>(2);
/**
* 同步进度
*/
private static final String SYNC_PERCENT = "syncPercent";
/**
* 索引进度
*/
private static final String INDEXING_PERCENT = "indexingPercent";
private ThrottleExecutor throttleExecutor;
private Timer DELAY_DELETE_TAG_TIMER;
private final ReentrantLock deleteDocWithDeleteFlagLock = new ReentrantLock();
@PostConstruct
public void init() {
// 启动时检测是否存在菜单,不存在则初始化
if (!menuService.existsMenu()) {
menuService.initMenus();
roleService.initRoles();
}
// 启动时检测是否存在lucene索引,不存在则初始化
if (!checkIndexExists()) {
doSync(userService.getCreatorUsername(), null, true);
}
// 重置索引状态
resetIndexStatus();
}
private void getSyncFileVisitorService() {
int processors = Runtime.getRuntime().availableProcessors() - 4;
if (syncFileVisitorService == null || syncFileVisitorService.isShutdown()) {
syncFileVisitorService = ThreadUtil.newFixedExecutor(Math.max(processors, 2), 1, "syncFileVisitor", true);
}
}
public void doSync(String username, String path, boolean isDelIndex) {
if (isSyncFile() || isIndexing()) {
return;
}
if (StrUtil.isBlank(username)) {
username = userService.getCreatorUsername();
}
setPercentMap(0d, 0d);
String finalUsername = username;
ThreadUtil.execute(() -> {
if (!SYNC_FILE_LOCK.tryLock()) {
return;
}
try {
Path canPath;
if (StrUtil.isBlank(path)) {
canPath = Paths.get(fileProperties.getRootDir());
} else {
canPath = Paths.get(path);
}
if (!Files.exists(canPath)) {
return;
}
getSyncFileVisitorService();
rebuildingIndex(finalUsername, canPath, isDelIndex);
} finally {
SYNC_FILE_LOCK.unlock();
setPercentMap(100d, 100d);
pushMessage();
}
});
}
/**
* 重建索引
*
* @param recipient 接收消息的用户
* @param path 要扫描的路径
* @param isDelIndex 是否删除索引
*/
private void rebuildingIndex(String recipient, Path path, boolean isDelIndex) {
TimeInterval timeInterval = new TimeInterval();
try {
getRecipient(recipient);
restIndexedTasks();
if (DELAY_DELETE_TAG_TIMER != null) {
DELAY_DELETE_TAG_TIMER.cancel();
}
Set<FileVisitOption> fileVisitOptions = EnumSet.noneOf(FileVisitOption.class);
fileVisitOptions.add(FileVisitOption.FOLLOW_LINKS);
// 先移除删除标记, 以免因为扫描路径的不同导致删除标记未移除
removeDeletedFlag(null);
// 添加删除标记, 扫描完后如果标记还在则删除
addDeleteFlagOfDoc(path, isDelIndex);
// 重置索引状态
resetIndexStatus();
FileCountVisitor fileCountVisitor = new FileCountVisitor();
Files.walkFileTree(path, fileVisitOptions, Integer.MAX_VALUE, fileCountVisitor);
totalCount = fileCountVisitor.getCount();
log.info("path: {}, 开始同步, 文件数: {}", path, totalCount);
timeInterval.start();
if (syncFileVisitor == null) {
syncFileVisitor = new SyncFileVisitor(totalCount);
}
Files.walkFileTree(path, fileVisitOptions, Integer.MAX_VALUE, syncFileVisitor);
// 等待线程池里所有任务完成
waitTaskCompleted();
deleteDocWithDeleteFlag();
} catch (IOException e) {
log.error("{}{}", e.getMessage(), path, e);
} finally {
setPercentMap(100d, getIndexedPercentValue());
syncFileVisitor = null;
log.info("同步完成, 耗时: {}s", timeInterval.intervalSecond());
}
}
/**
* 删除path下的所有索引
* @param path path
*/
private void deleteAllIndex(String path) {
if (StrUtil.isBlank(path)) {
path = "/";
}
// 查询path下的所有索引
Term prefixTerm = new Term("path", path);
PrefixQuery prefixQuery = new PrefixQuery(prefixTerm);
try {
indexWriter.deleteDocuments(prefixQuery);
indexWriter.commit();
} catch (IOException e) {
log.error(e.getMessage(), e);
}
}
private void waitTaskCompleted() {
try {
log.info("等待同步文件完成");
syncFileVisitorService.shutdown();
// 等待线程池里所有任务完成
if (!syncFileVisitorService.awaitTermination(10, TimeUnit.MINUTES)) {
log.warn("同步文件超时, 尝试强制停止所有任务");
// 移除删除标记, 以免误删索引
removeDeletedFlag(null);
syncFileVisitorService.shutdownNow();
}
} catch (InterruptedException e) {
syncFileVisitorService.shutdownNow();
Thread.currentThread().interrupt();
}
}
private String getRecipient(String username) {
if (StrUtil.isNotBlank(username)) {
RECIPIENT.clear();
RECIPIENT.add(username);
}
return RECIPIENT.stream().findFirst().orElse(username);
}
private void deleteDocWithDeleteFlag() {
DELAY_DELETE_TAG_TIMER = new Timer();
DELAY_DELETE_TAG_TIMER.schedule(new TimerTask() {
@Override
public void run() {
if (!deleteDocWithDeleteFlagLock.tryLock()) {
return;
}
try {
commonFileService.deleteDocWithDeleteFlag();
} finally {
deleteDocWithDeleteFlagLock.unlock();
}
}
}, 10000);
}
public void onSyncComplete(Runnable callback) {
this.syncCompleteCallback = callback;
}
public void rebuildingIndexCompleted() {
if (!hasUnIndexedTasks() && NOT_INDEX_TASK_SIZE.get() > 0) {
setPercentMap(100d, 100d);
log.debug("重建索引完成, INDEXED_TASK_SIZE, {}, NOT_INDEX_TASK_SIZE: {}", INDEXED_TASK_SIZE, NOT_INDEX_TASK_SIZE);
restIndexedTasks();
pushMessage();
if (syncCompleteCallback != null) {
syncCompleteCallback.run();
}
}
}
private void restIndexedTasks() {
NOT_INDEX_TASK_SIZE.set(0);
INDEXED_TASK_SIZE.set(0);
totalCount = 0;
}
/**
* 增加待索引任务数
*/
public void incrementNotIndexTaskSize() {
NOT_INDEX_TASK_SIZE.incrementAndGet();
}
/**
* 增加已索引任务数
*/
public void incrementIndexedTaskSize() {
INDEXED_TASK_SIZE.incrementAndGet();
delayResetIndex();
}
public void delayResetIndex() {
synchronized (RebuildIndexTaskService.class) {
if (throttleExecutor == null) {
throttleExecutor = new ThrottleExecutor(10000);
}
}
throttleExecutor.schedule(this::rebuildingIndexCompleted);
}
private void updatePercent() {
setPercentMap(null, null);
pushMessage();
}
/**
* 是否正在同步文件
*/
public static boolean isSyncFile() {
return SYNC_FILE_LOCK.isLocked();
}
/**
* 是否正在创建索引
*/
public static boolean isIndexing() {
return NOT_INDEX_TASK_SIZE.get() > 0;
}
/**
* 把文件同步到数据库
* @param username 用户名
* @param path 文件相对路径
* @param isDelIndex 是否删除索引
*/
public ResponseResult<Object> sync(String username, String path, boolean isDelIndex) {
if (StrUtil.isNotBlank(path)) {
path = Paths.get(fileProperties.getRootDir(), username, path).toString();
}
doSync(username, path, isDelIndex);
return ResultUtil.success();
}
/**
* 是否正在同步中
*/
public Map<String, Double> isSync() {
if (PERCENT_MAP.isEmpty()) {
setPercentMap(100d, 100d);
}
return PERCENT_MAP;
}
/**
* 更新任务进度, 用于前端显示
*/
public void updateTaskProgress() {
if (!isSyncFile() && !isIndexing()) {
return;
}
if (getRecipient(null) == null) {
getRecipient(userService.getCreatorUsername());
}
// 更新进度
updatePercent();
}
/**
* 获取同步进度
*/
private double getSyncPercent() {
if (syncFileVisitor == null) {
return PERCENT_MAP.getOrDefault(SYNC_PERCENT, 100d);
}
return syncFileVisitor.getPercent();
}
/**
* 获取索引进度
*/
private double getIndexedPercent() {
if (NOT_INDEX_TASK_SIZE.get() == 0 || isSyncFile()) {
return 100;
}
return getIndexedPercentValue();
}
private double getIndexedPercentValue() {
if (totalCount == 0) {
return 0;
}
return NumberUtil.round((double) INDEXED_TASK_SIZE.get() / totalCount * 100, 2, RoundingMode.DOWN).doubleValue();
}
private class SyncFileVisitor extends SimpleFileVisitor<Path> {
@Getter
public final double totalCount;
private final AtomicInteger processCount;
public SyncFileVisitor(double totalCount) {
this.totalCount = totalCount;
this.processCount = new AtomicInteger(0);
}
public double getPercent() {
return NumberUtil.round(processCount.get() / totalCount * 100, 2).doubleValue();
}
@NotNull
@Override
public FileVisitResult visitFileFailed(@NotNull Path file, @NotNull IOException exc) throws IOException {
log.error(exc.getMessage(), exc);
return super.visitFileFailed(file, exc);
}
@NotNull
@Override
public FileVisitResult preVisitDirectory(@NotNull Path dir, @NotNull BasicFileAttributes attrs) throws IOException {
// 跳过临时文件目录
FileVisitResult skipTempDirectory = skipTempDirectory(dir);
if (skipTempDirectory != null) return skipTempDirectory;
String username = commonFileService.getUsernameByAbsolutePath(dir);
processCount.incrementAndGet();
if (StrUtil.isBlank(username)) {
return super.visitFile(dir, attrs);
}
processFile(dir, username);
return super.preVisitDirectory(dir, attrs);
}
@NotNull
@Override
public FileVisitResult visitFile(Path file, @NotNull BasicFileAttributes attrs) throws IOException {
// 判断文件名是否在monitorIgnoreFilePrefix中
if (fileProperties.getMonitorIgnoreFilePrefix().stream().anyMatch(file.getFileName()::startsWith)) {
log.debug("忽略文件:{}", file.getFileName());
return super.visitFile(file, attrs);
}
processCount.incrementAndGet();
String username = commonFileService.getUsernameByAbsolutePath(file);
if (StrUtil.isBlank(username)) {
return super.visitFile(file, attrs);
}
processFile(file, username);
return super.visitFile(file, attrs);
}
private void processFile(Path file, String username) {
// 使用 RxJava 执行异步文件创建
syncFileVisitorService.execute(() -> createFile(username, file));
}
private void createFile(String username, Path file) {
try {
commonFileService.createFile(username, file.toFile(), null, null);
} catch (Exception e) {
log.error("createFile error {}{}", e.getMessage(), file, e);
Query query = new Query();
query.fields().include("_id");
FileDocument fileDocument = commonFileService.getFileDocument(username, file.toFile().getAbsolutePath(), query);
if (fileDocument != null) {
// 需要移除删除标记
removeDeletedFlag(Collections.singletonList(fileDocument.getId()));
}
}
}
}
private void setPercentMap(Double syncPercent, Double indexingPercent) {
if (syncPercent == null) {
syncPercent = getSyncPercent();
}
if (indexingPercent == null) {
indexingPercent = getIndexedPercent();
}
PERCENT_MAP.put(SYNC_PERCENT, syncPercent);
PERCENT_MAP.put(INDEXING_PERCENT, indexingPercent);
if (syncPercent >= 100 && indexingPercent >= 100) {
ocrService.setMaxConcurrentRequests(ocrService.getOcrConfig().getMaxTasks());
}
}
private void pushMessage() {
commonFileService.pushMessage(getRecipient(null), PERCENT_MAP, RebuildIndexTaskService.MSG_SYNCED);
log.debug("索引进度: {}, isSyncFile: {}, INDEXED_TASK_SIZE, {}, NOT_INDEX_TASK_SIZE: {}", PERCENT_MAP, isSyncFile(), INDEXED_TASK_SIZE.get(), NOT_INDEX_TASK_SIZE.get());
}
private class FileCountVisitor extends SimpleFileVisitor<Path> {
private final AtomicLong count = new AtomicLong(0);
public long getCount() {
return count.get();
}
@NotNull
@Override
public FileVisitResult preVisitDirectory(@NotNull Path dir, @NotNull BasicFileAttributes attrs) throws IOException {
// 跳过临时文件目录
FileVisitResult skipTempDirectory = skipTempDirectory(dir);
if (skipTempDirectory != null) return skipTempDirectory;
String username = commonFileService.getUsernameByAbsolutePath(dir);
count.addAndGet(1);
if (StrUtil.isBlank(username)) {
return super.visitFile(dir, attrs);
}
return super.preVisitDirectory(dir, attrs);
}
@NotNull
@Override
public FileVisitResult visitFile(@NotNull Path file, @NotNull BasicFileAttributes attrs) throws IOException {
count.addAndGet(1);
return super.visitFile(file, attrs);
}
}
private @Nullable FileVisitResult skipTempDirectory(Path dir) {
// 跳过临时文件目录
if (dir.toFile().getName().equals(fileProperties.getChunkFileDir())) {
return FileVisitResult.SKIP_SUBTREE;
}
// 跳过lucene索引目录
if (dir.toFile().getName().equals(fileProperties.getLuceneIndexDir())) {
return FileVisitResult.SKIP_SUBTREE;
}
return null;
}
public boolean checkIndexExists() {
org.springframework.data.mongodb.core.query.Query query = new org.springframework.data.mongodb.core.query.Query();
query.addCriteria(Criteria.where("ossFolder").exists(true));
long count = mongoTemplate.count(query, CommonFileService.COLLECTION_NAME);
long indexCount = indexWriter.getDocStats().numDocs;
return indexCount > count;
}
/**
* 重置索引状态, 将正在索引的文件状态重置为未索引
*/
public void resetIndexStatus() {
org.springframework.data.mongodb.core.query.Query query = new org.springframework.data.mongodb.core.query.Query();
query.addCriteria(Criteria.where(LuceneService.MONGO_INDEX_FIELD).lte(IndexStatus.INDEXING.getStatus()));
Update update = new Update();
update.unset(LuceneService.MONGO_INDEX_FIELD);
mongoTemplate.updateMulti(query, update, CommonFileService.COLLECTION_NAME);
}
/**
* 检查是否有未索引或正在索引的任务
*/
public boolean hasUnIndexedTasks() {
org.springframework.data.mongodb.core.query.Query query = new org.springframework.data.mongodb.core.query.Query();
query.addCriteria(Criteria.where(LuceneService.MONGO_INDEX_FIELD).lte(IndexStatus.INDEXING.getStatus()));
long count = mongoTemplate.count(query, CommonFileService.COLLECTION_NAME);
return count > 0;
}
/**
* 添加删除标记
*/
private void addDeleteFlagOfDoc(Path filepath, boolean isDelIndex) {
if (filepath == null) {
return;
}
if (Files.notExists(filepath)) {
return;
}
if (filepath.toFile().isFile()) {
return;
}
String username = commonFileService.getUsernameByAbsolutePath(filepath);
String userId = null;
String path = null;
if (StrUtil.isNotBlank(username)) {
userId = userService.getUserIdByUserName(username);
Path relativePath = Paths.get(fileProperties.getRootDir(), username).relativize(filepath);
if (relativePath.getNameCount() > 0) {
path = "/" + relativePath + "/";
} else {
path = "/";
}
}
org.springframework.data.mongodb.core.query.Query query = new org.springframework.data.mongodb.core.query.Query();
query.addCriteria(Criteria.where("alonePage").exists(false));
query.addCriteria(Criteria.where("release").exists(false));
query.addCriteria(Criteria.where("mountFileId").exists(false));
if (StrUtil.isNotBlank(userId)) {
query.addCriteria(Criteria.where("userId").is(userId));
}
if (StrUtil.isNotBlank(path)) {
query.addCriteria(Criteria.where("path").regex("^" + ReUtil.escape(path)));
}
Update update = new Update();
// 添加删除标记用于在之后删除
update.set("delete", 1);
mongoTemplate.updateMulti(query, update, CommonFileService.COLLECTION_NAME);
if (isDelIndex) {
// 删除索引
deleteAllIndex(path);
}
}
/**
* 移除删除标记
*
* @param fileIdList fileIdList
*/
public void removeDeletedFlag(List<String> fileIdList) {
org.springframework.data.mongodb.core.query.Query query = new org.springframework.data.mongodb.core.query.Query();
if (fileIdList == null || fileIdList.isEmpty()) {
query.addCriteria(Criteria.where("delete").is(1));
} else {
query.addCriteria(Criteria.where("_id").in(fileIdList).and("delete").is(1));
}
Update update = new Update();
update.unset("delete");
mongoTemplate.updateMulti(query, update, CommonFileService.COLLECTION_NAME);
}
@PreDestroy
public void destroy() {
if (syncFileVisitorService != null) {
syncFileVisitorService.shutdown();
}
if (throttleExecutor != null) {
throttleExecutor.shutdown();
}
}
}

View File

@ -0,0 +1,589 @@
package com.jmal.clouddisk.lucene;
import cn.hutool.core.text.CharSequenceUtil;
import cn.hutool.core.util.BooleanUtil;
import com.jmal.clouddisk.config.FileProperties;
import com.jmal.clouddisk.exception.CommonException;
import com.jmal.clouddisk.exception.ExceptionType;
import com.jmal.clouddisk.model.FileDocument;
import com.jmal.clouddisk.model.FileIntroVO;
import com.jmal.clouddisk.model.TagDO;
import com.jmal.clouddisk.model.query.SearchDTO;
import com.jmal.clouddisk.model.query.SearchOptionHistoryDO;
import com.jmal.clouddisk.service.Constants;
import com.jmal.clouddisk.service.IUserService;
import com.jmal.clouddisk.service.impl.TagService;
import com.jmal.clouddisk.service.impl.UserLoginHolder;
import com.jmal.clouddisk.util.ResponseResult;
import com.jmal.clouddisk.util.ResultUtil;
import com.jmal.clouddisk.util.TimeUntils;
import com.mongodb.client.AggregateIterable;
import io.reactivex.rxjava3.core.Completable;
import io.reactivex.rxjava3.schedulers.Schedulers;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.index.StoredFields;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.*;
import org.bson.types.ObjectId;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.util.*;
import static com.jmal.clouddisk.lucene.LuceneService.FIELD_TAG_NAME_FUZZY;
import static com.jmal.clouddisk.service.impl.CommonFileService.COLLECTION_NAME;
@Service
@RequiredArgsConstructor
@Slf4j
public class SearchFileService {
private final Analyzer analyzer;
private final FileProperties fileProperties;
private final SearcherManager searcherManager;
private final UserLoginHolder userLoginHolder;
private final TagService tagService;
private final MongoTemplate mongoTemplate;
private final IUserService userService;
public boolean existsSha256(String fileIndexHash) {
IndexSearcher indexSearcher = null;
try {
searcherManager.maybeRefresh();
indexSearcher = searcherManager.acquire();
Term term = new Term(Constants.ETAG, fileIndexHash);
Query query = new TermQuery(term);
TopDocs topDocs = indexSearcher.search(query, 1);
return topDocs.totalHits.value() > 0;
} catch (IOException e) {
log.error("检查 {} 是否存在失败", Constants.ETAG, e);
} finally {
if (indexSearcher != null) {
try {
searcherManager.release(indexSearcher);
} catch (IOException e) {
log.error("释放搜索器失败", e);
}
}
}
return false;
}
public ResponseResult<List<FileIntroVO>> searchFile(SearchDTO searchDTO) {
String keyword = searchDTO.getKeyword();
if (keyword == null || keyword.trim().isEmpty() || searchDTO.getUserId() == null) {
return ResultUtil.success(Collections.emptyList());
}
ResponseResult<List<FileIntroVO>> result = ResultUtil.genResult();
IndexSearcher indexSearcher = null;
try {
beforeQuery(searchDTO);
if (!searchDTO.getUserId().equals(userLoginHolder.getUserId())) {
Map<String, Object> props = new HashMap<>();
props.put("fileUsername", userService.getUserNameById(searchDTO.getUserId()));
result.setProps(props);
}
int pageNum = searchDTO.getPage();
int pageSize = searchDTO.getPageSize();
searcherManager.maybeRefresh();
List<String> seenIds = new ArrayList<>();
indexSearcher = searcherManager.acquire();
Query query = getQuery(searchDTO);
Sort sort = getSort(searchDTO);
ScoreDoc lastScoreDoc = null;
if (pageNum > 1) {
int totalHitsToSkip = (pageNum - 1) * pageSize;
TopDocs topDocs = indexSearcher.search(query, totalHitsToSkip, sort);
if (topDocs.scoreDocs.length == totalHitsToSkip) {
lastScoreDoc = topDocs.scoreDocs[totalHitsToSkip - 1];
}
}
int count = indexSearcher.count(query);
TopDocs topDocs = indexSearcher.searchAfter(lastScoreDoc, query, pageSize, sort);
StoredFields storedFields = indexSearcher.storedFields();
for (ScoreDoc hit : topDocs.scoreDocs) {
Document doc = storedFields.document(hit.doc);
String id = doc.get("id");
seenIds.add(id);
}
List<FileIntroVO> fileIntroVOList = getFileIntroVOs(seenIds);
long now = System.currentTimeMillis();
fileIntroVOList.forEach(fileIntroVO -> {
long updateMilli = TimeUntils.getMilli(fileIntroVO.getUpdateDate());
fileIntroVO.setAgoTime(now - updateMilli);
});
result.setData(fileIntroVOList);
result.setCount(count);
String userId = userLoginHolder.getUserId();
// 添加搜索历史
Completable.fromAction(() -> addSearchHistory(userId, searchDTO)).subscribeOn(Schedulers.io()).subscribe();
return result;
} catch (IOException | ParseException | java.lang.IllegalArgumentException e) {
log.error("搜索失败", e);
return result.setData(Collections.emptyList()).setCount(0);
} finally {
if (indexSearcher != null) {
try {
searcherManager.release(indexSearcher);
} catch (IOException e) {
log.error("释放搜索器失败", e);
}
}
}
}
/**
* 添加搜索历史
*
* @param searchUserId searchUserId
* @param searchDTO searchDTO
*/
private void addSearchHistory(String searchUserId, SearchDTO searchDTO) {
if (CharSequenceUtil.isBlank(searchUserId) || searchDTO == null) {
return;
}
SearchOptionHistoryDO searchOptionHistoryDO = searchDTO.toSearchOptionDO();
searchOptionHistoryDO.setSearchTime(System.currentTimeMillis());
searchOptionHistoryDO.setSearchUserId(searchUserId);
org.springframework.data.mongodb.core.query.Query query = new org.springframework.data.mongodb.core.query.Query();
query.addCriteria(Criteria.where("keyword").is(searchDTO.getKeyword()));
query.addCriteria(Criteria.where("searchUserId").is(searchUserId));
// 删除重复的搜索记录
mongoTemplate.remove(query, SearchOptionHistoryDO.class);
// 删除一个月之前的搜索记录
org.springframework.data.mongodb.core.query.Query query1 = new org.springframework.data.mongodb.core.query.Query();
query1.addCriteria(Criteria.where("searchUserId").is(searchUserId));
query1.addCriteria(Criteria.where("searchTime").lt(System.currentTimeMillis() - 30 * 24 * 60 * 60 * 1000L));
mongoTemplate.remove(query1, SearchOptionHistoryDO.class);
// 插入搜索记录
mongoTemplate.insert(searchOptionHistoryDO);
}
public List<SearchDTO> recentlySearchHistory(String keyword) {
// 最近6条搜索记录
org.springframework.data.mongodb.core.query.Query query = new org.springframework.data.mongodb.core.query.Query();
query.addCriteria(Criteria.where("searchUserId").is(userLoginHolder.getUserId()));
if (CharSequenceUtil.isNotBlank(keyword)) {
query.addCriteria(Criteria.where("keyword").regex(keyword));
}
query.with(org.springframework.data.domain.Sort.by(org.springframework.data.domain.Sort.Direction.DESC, "searchTime"));
query.limit(6);
List<SearchOptionHistoryDO> searchOptionHistoryDOList = mongoTemplate.find(query, SearchOptionHistoryDO.class);
return searchOptionHistoryDOList.stream().map(SearchOptionHistoryDO::toSearchDTO).toList();
}
public void deleteSearchHistory(String id) {
if (CharSequenceUtil.isBlank(id)) {
throw new CommonException(ExceptionType.MISSING_PARAMETERS);
}
org.springframework.data.mongodb.core.query.Query query = new org.springframework.data.mongodb.core.query.Query();
query.addCriteria(Criteria.where("_id").is(id));
mongoTemplate.remove(query, SearchOptionHistoryDO.class);
}
public void deleteAllSearchHistory() {
String userId = userLoginHolder.getUserId();
if (CharSequenceUtil.isBlank(userId)) {
throw new CommonException(ExceptionType.MISSING_PARAMETERS);
}
org.springframework.data.mongodb.core.query.Query query = new org.springframework.data.mongodb.core.query.Query();
query.addCriteria(Criteria.where(IUserService.USER_ID).is(userId));
mongoTemplate.remove(query, SearchOptionHistoryDO.class);
}
/**
* 获取排序规则
*
* @param searchDTO searchDTO
* @return Sort
*/
private static Sort getSort(SearchDTO searchDTO) {
// 如果 searchDTO 为 null,返回默认相关度倒序排序
if (searchDTO == null) {
return Sort.RELEVANCE;
}
String sortProp = searchDTO.getSortProp();
String sortOrder = searchDTO.getSortOrder();
// 如果排序属性或顺序为空,默认按相关度倒序排序
if (CharSequenceUtil.isBlank(sortProp) || CharSequenceUtil.isBlank(sortOrder)) {
return Sort.RELEVANCE;
}
// 确定排序方向
boolean reverse = "descending".equalsIgnoreCase(sortOrder);
// 根据排序属性创建对应的 SortField
SortField sortField;
switch (sortProp.toLowerCase()) {
case "updatedate":
sortField = new SortField("modified", SortField.Type.LONG, reverse);
break;
case "size":
sortField = new SortField("size", SortField.Type.LONG, reverse);
break;
case "name": // 显式支持相关度排序
// 注意:FIELD_SCORE 本身是降序的,如果要求升序需要特殊处理
return reverse ? Sort.RELEVANCE : new Sort(new SortField(null, SortField.Type.SCORE, true));
default:
// 对于不支持的排序字段,默认按相关度倒序
return Sort.RELEVANCE;
}
return new Sort(sortField);
}
/**
* 构建查询器
*
* @param searchDTO searchDTO
* @return Query
* @throws ParseException ParseException
*/
private Query getQuery(SearchDTO searchDTO) throws ParseException {
// 精确搜索时,通常不希望对用户输入做过多改动
String exactKeyword = searchDTO.getKeyword().toLowerCase();
// 模糊搜索将关键字中的特殊字符转义
String fuzzyKeyword = QueryParser.escape(searchDTO.getKeyword().toLowerCase().trim());
// filename 和 tag Name 进行 NGram 匹配
BooleanQuery.Builder nameAndTagQueryBuilder = new BooleanQuery.Builder();
if (BooleanUtil.isTrue(searchDTO.getIncludeFileName())) {
BooleanQuery.Builder filenameBooleanQueryBuilder = getBooleanQueryFieldBuilder(LuceneService.FIELD_FILENAME_NGRAM, LuceneService.FIELD_FILENAME_FUZZY, exactKeyword, fuzzyKeyword, searchDTO);
nameAndTagQueryBuilder.add(filenameBooleanQueryBuilder.build(), BooleanClause.Occur.SHOULD);
}
if (BooleanUtil.isTrue(searchDTO.getIncludeTagName())) {
BooleanQuery.Builder tagNameBooleanQueryBuilder = getBooleanQueryFieldBuilder(LuceneService.FIELD_TAG_NAME_NGRAM, FIELD_TAG_NAME_FUZZY, exactKeyword, fuzzyKeyword, searchDTO);
nameAndTagQueryBuilder.add(tagNameBooleanQueryBuilder.build(), BooleanClause.Occur.SHOULD);
}
Query boostQuery = new BoostQuery(nameAndTagQueryBuilder.build(), 3.0f);
Query contentQuery = null;
if (BooleanUtil.isTrue(searchDTO.getIncludeFileContent())) {
BooleanQuery.Builder contentBooleanQueryBuilder = getBooleanQueryFieldBuilder(LuceneService.FIELD_CONTENT_NGRAM, LuceneService.FIELD_CONTENT_FUZZY, exactKeyword, fuzzyKeyword, searchDTO);
contentQuery = contentBooleanQueryBuilder.build();
}
// 如果keyword中有空格,将其拆分为多个关键字,这多个关键字之间是OR关系,在name或tag字段
Query nameAndTagQuery = null;
if (!BooleanUtil.isTrue(searchDTO.getExactSearch())) {
nameAndTagQuery = getMultipleKeywordsQuery(searchDTO, exactKeyword);
}
// 将正则表达式查询、短语查询和分词匹配查询组合成一个查询(OR关系)
BooleanQuery.Builder combinedQueryBuilder = new BooleanQuery.Builder().add(boostQuery, BooleanClause.Occur.SHOULD);
if (contentQuery != null) {
combinedQueryBuilder.add(contentQuery, BooleanClause.Occur.SHOULD);
}
if (nameAndTagQuery != null) {
combinedQueryBuilder.add(nameAndTagQuery, BooleanClause.Occur.SHOULD);
}
// 其他通用搜索选项
BooleanQuery otherQuery = getOtherOption(searchDTO);
BooleanQuery combinedQuery = combinedQueryBuilder.build();
// 创建最终查询(AND关系)
BooleanQuery.Builder fileQueryBuilder = new BooleanQuery.Builder().add(new TermQuery(new Term(IUserService.USER_ID, searchDTO.getSearchUserId())), BooleanClause.Occur.MUST);
if (!combinedQuery.clauses().isEmpty()) {
fileQueryBuilder.add(combinedQuery, BooleanClause.Occur.MUST);
}
if (!otherQuery.clauses().isEmpty()) {
fileQueryBuilder.add(otherQuery, BooleanClause.Occur.MUST);
}
// 添加其他不通用查询条件
otherQueryParams(searchDTO, fileQueryBuilder);
// or 挂载点查询
Query mountQuery = getMountQueryBuilder(searchDTO, combinedQuery, otherQuery);
// 构建最终查询
BooleanQuery.Builder finalQueryBuilder = new BooleanQuery.Builder();
// 是否仅搜索挂载文件
if (!searchDTO.onlySearchMount()) {
finalQueryBuilder.add(fileQueryBuilder.build(), BooleanClause.Occur.SHOULD);
}
// 是否搜索挂载文件
if (mountQuery != null) {
finalQueryBuilder.add(mountQuery, BooleanClause.Occur.SHOULD);
}
return finalQueryBuilder.build();
}
/**
* 创建字段查询
* 如果开启了精准搜索配置,且根据参数exactSearch为true,则创建精确查询;否则创建模糊查询
* 如果禁用精准搜索配置,则为“content”以外的字段(“filename”和“tagName”)创建模糊查询和精准查询, 两种查询使用OR组合
* 由于精确搜索配置导致前端没有exactSearch参数,默认使用NGram分词器索引“filename”和“tagName”或模糊查询组合查询将会有更好的效果”对于未来的维护者来说将非常有益
*
* @param fieldNameExact 精准查询字段名
* @param fieldNameFuzzy 模糊查询字段名
* @param exactKeyword 精准查询关键字
* @param fuzzyKeyword 模糊查询关键字
* @param searchDTO 查询参数
*/
private BooleanQuery.Builder getBooleanQueryFieldBuilder(String fieldNameExact, String fieldNameFuzzy, String exactKeyword, String fuzzyKeyword, SearchDTO searchDTO) throws ParseException {
BooleanQuery.Builder booleanQueryFieldBuilder = new BooleanQuery.Builder();
Query fuzzyQuery = null;
Query exactQuery = null;
if (fileProperties.getExactSearch()) {
exactQuery = getExactQuery(fieldNameExact, exactKeyword);
if (!BooleanUtil.isTrue(searchDTO.getExactSearch())) {
fuzzyQuery = getFuzzyQuery(fieldNameFuzzy, fuzzyKeyword);
}
} else {
if (LuceneService.FIELD_CONTENT_FUZZY.equals(fieldNameFuzzy)) {
fuzzyQuery = getFuzzyQuery(fieldNameFuzzy, fuzzyKeyword);
} else {
fuzzyQuery = getFuzzyQuery(fieldNameFuzzy, fuzzyKeyword);
exactQuery = getExactQuery(fieldNameExact, exactKeyword);
}
}
if (fuzzyQuery != null) {
booleanQueryFieldBuilder.add(fuzzyQuery, BooleanClause.Occur.SHOULD);
}
if (exactQuery != null) {
booleanQueryFieldBuilder.add(exactQuery, BooleanClause.Occur.SHOULD);
}
return booleanQueryFieldBuilder;
}
private Query getMultipleKeywordsQuery(SearchDTO searchDTO, String exactKeyword) {
if (exactKeyword.contains(" ") && (BooleanUtil.isTrue(searchDTO.getIncludeFileName()) || BooleanUtil.isTrue(searchDTO.getIncludeTagName()))) {
BooleanQuery.Builder nameAndTagMultipleWordsQueryBuilder = new BooleanQuery.Builder();
for (String key : exactKeyword.split(" ")) {
if (BooleanUtil.isTrue(searchDTO.getIncludeFileName())) {
Query filenameQuery = getExactQuery(LuceneService.FIELD_FILENAME_NGRAM, key);
nameAndTagMultipleWordsQueryBuilder.add(filenameQuery, BooleanClause.Occur.SHOULD);
}
if (BooleanUtil.isTrue(searchDTO.getIncludeTagName())) {
Query tagNameQuery = getExactQuery(LuceneService.FIELD_TAG_NAME_NGRAM, key);
nameAndTagMultipleWordsQueryBuilder.add(tagNameQuery, BooleanClause.Occur.SHOULD);
}
}
return nameAndTagMultipleWordsQueryBuilder.build();
}
return null;
}
private Query getFuzzyQuery(String fieldNameFuzzy, String fuzzyKeyword) throws ParseException {
Query contentQuery;
QueryParser contentParser = new QueryParser(fieldNameFuzzy, analyzer);
contentParser.setDefaultOperator(QueryParser.Operator.AND);
contentQuery = contentParser.parse(QueryParser.escape(fuzzyKeyword.trim()));
return contentQuery;
}
private Query getExactQuery(String fieldName, String exactSearchTerm) {
Query contentQuery;
if (CharSequenceUtil.isBlank(exactSearchTerm)) {
contentQuery = new MatchNoDocsQuery();
} else if (exactSearchTerm.length() <= fileProperties.getNgramMaxSize()) {
Term term = new Term(fieldName, exactSearchTerm);
contentQuery = new TermQuery(term);
} else {
// 如果用户输入长度大于 maxGramSize,则分解查询字符串
BooleanQuery.Builder decomposedQueryBuilder = new BooleanQuery.Builder();
boolean hasValidSubTerms = false;
// 生成重叠的、长度为 maxGram 的子串
// 滑动窗口:从索引 0 开始,每次取长度为 maxGram 的子串,然后窗口向右移动一个字符
for (int i = 0; i <= exactSearchTerm.length() - fileProperties.getNgramMaxSize(); i++) {
String subTerm = exactSearchTerm.substring(i, i + fileProperties.getNgramMaxSize());
if (CharSequenceUtil.isNotBlank(subTerm)) { // 理论上 subTerm 不会为空,但作为防御
decomposedQueryBuilder.add(new TermQuery(new Term(fieldName, subTerm)), BooleanClause.Occur.MUST);
hasValidSubTerms = true;
}
}
if (hasValidSubTerms) {
contentQuery = decomposedQueryBuilder.build();
log.debug("{} 精确子串搜索 (长查询分解后): {}", fieldName, contentQuery.toString());
} else {
// 如果分解后没有有效的子查询(例如,原始字符串非常特殊或maxGram设置不当导致无法分解)
// 虽然理论上 exactSearchTerm.length() > maxGram 应该总能分解出至少一个,但作为防御
contentQuery = new MatchNoDocsQuery();
}
}
return contentQuery;
}
private static BooleanQuery getOtherOption(SearchDTO searchDTO) {
BooleanQuery.Builder otherOptionBuilder = new BooleanQuery.Builder();
// 文件类型
if (CharSequenceUtil.isNotBlank(searchDTO.getType())) {
otherOptionBuilder.add(new TermQuery(new Term("type", searchDTO.getType())), BooleanClause.Occur.MUST);
}
// 是否文件夹
if (searchDTO.getIsFolder() != null) {
otherOptionBuilder.add(IntPoint.newExactQuery("isFolder", searchDTO.getIsFolder() ? 1 : 0), BooleanClause.Occur.MUST);
}
// 更新时间范围查询
if (searchDTO.getModifyStart() != null || searchDTO.getModifyEnd() != null) {
long start = searchDTO.getModifyStart() != null ? searchDTO.getModifyStart() : Long.MIN_VALUE;
long end = searchDTO.getModifyEnd() != null ? searchDTO.getModifyEnd() : Long.MAX_VALUE;
otherOptionBuilder.add(NumericDocValuesField.newSlowRangeQuery("modified", start, end), BooleanClause.Occur.MUST);
}
// 文件大小范围查询
if (searchDTO.getSizeMin() != null || searchDTO.getSizeMax() != null) {
long minSize = searchDTO.getSizeMin() != null ? searchDTO.getSizeMin() : Long.MIN_VALUE;
long maxSize = searchDTO.getSizeMax() != null ? searchDTO.getSizeMax() : Long.MAX_VALUE;
otherOptionBuilder.add(NumericDocValuesField.newSlowRangeQuery("size", minSize, maxSize), BooleanClause.Occur.MUST);
}
return otherOptionBuilder.build();
}
/**
* 获取挂载点查询
*
* @param searchDTO searchDTO
* @param combinedQuery 共用查询
* @return mountQueryBuilder
*/
private BooleanQuery getMountQueryBuilder(SearchDTO searchDTO, BooleanQuery combinedQuery, BooleanQuery otherQuery) {
BooleanQuery.Builder mountQueryBuilder;
if (BooleanUtil.isTrue(searchDTO.getSearchMount()) || searchDTO.onlySearchMount()) {
mountQueryBuilder = new BooleanQuery.Builder();
org.springframework.data.mongodb.core.query.Query query = new org.springframework.data.mongodb.core.query.Query();
query.addCriteria(Criteria.where("userId").is(searchDTO.getUserId()));
query.addCriteria(Criteria.where("mountFileId").exists(true));
query.fields().exclude("content");
query.fields().exclude("video");
query.fields().exclude("music");
query.fields().exclude("contentText");
query.fields().exclude("exif");
List<FileDocument> fileDocumentList = mongoTemplate.find(query, FileDocument.class);
fileDocumentList.parallelStream().forEach(fileDocument -> {
BooleanQuery.Builder mountQueryBuilderItem = new BooleanQuery.Builder();
org.springframework.data.mongodb.core.query.Query query1 = new org.springframework.data.mongodb.core.query.Query();
query1.fields().include("userId");
query1.fields().include("path");
query1.fields().include("name");
FileDocument fileBase = mongoTemplate.findById(fileDocument.getMountFileId(), FileDocument.class);
if (fileBase != null) {
// userId
mountQueryBuilderItem.add(new TermQuery(new Term(IUserService.USER_ID, fileBase.getUserId())), BooleanClause.Occur.MUST);
// path
mountQueryBuilderItem.add(getPathQueryBuilder(fileBase.getPath() + fileBase.getName()), BooleanClause.Occur.MUST);
// other
mountQueryBuilderItem.add(combinedQuery, BooleanClause.Occur.MUST);
if (!otherQuery.clauses().isEmpty()) {
mountQueryBuilderItem.add(otherQuery, BooleanClause.Occur.MUST);
}
}
mountQueryBuilder.add(mountQueryBuilderItem.build(), BooleanClause.Occur.SHOULD);
});
} else {
mountQueryBuilder = null;
}
return mountQueryBuilder == null ? null : mountQueryBuilder.build();
}
/**
* 查询前置处理
*
* @param searchDTO searchDTO
*/
private void beforeQuery(SearchDTO searchDTO) {
String folder = searchDTO.getFolder();
if (CharSequenceUtil.isNotBlank(folder)) {
// 挂载点查询
org.springframework.data.mongodb.core.query.Query query = new org.springframework.data.mongodb.core.query.Query();
query.addCriteria(Criteria.where("_id").is(folder));
FileDocument fileDocument = mongoTemplate.findOne(query, FileDocument.class);
if (fileDocument != null && BooleanUtil.isFalse(searchDTO.getSearchOverall())) {
searchDTO.setCurrentDirectory(fileDocument.getPath() + fileDocument.getName());
searchDTO.setMountUserId(fileDocument.getUserId());
}
}
}
private void otherQueryParams(SearchDTO searchDTO, BooleanQuery.Builder builder) {
boolean queryPath = CharSequenceUtil.isNotBlank(searchDTO.getCurrentDirectory()) && searchDTO.getCurrentDirectory().length() > 1;
if (queryPath) {
builder.add(getPathQueryBuilder(searchDTO.getCurrentDirectory()), BooleanClause.Occur.MUST);
}
// 标签
if (searchDTO.getTagId() != null) {
TagDO tagDO = tagService.getTagInfo(searchDTO.getTagId());
if (tagDO != null) {
builder.add(new TermQuery(new Term(FIELD_TAG_NAME_FUZZY, tagDO.getName())), BooleanClause.Occur.MUST);
}
}
// 是否收藏
if (searchDTO.getIsFavorite() != null && !queryPath) {
builder.add(IntPoint.newExactQuery(Constants.IS_FAVORITE, searchDTO.getIsFavorite() ? 1 : 0), BooleanClause.Occur.MUST);
}
}
private static BooleanQuery getPathQueryBuilder(String path) {
// 检查path最后一个字符是否有/, 如果没有,则添加
path = path.endsWith("/") ? path : path + "/";
BooleanQuery.Builder pathQueryBuilder = new BooleanQuery.Builder();
Term pathTerm = new Term("path", path);
PrefixQuery prefixQuery = new PrefixQuery(pathTerm);
pathQueryBuilder.add(new TermQuery(pathTerm), BooleanClause.Occur.SHOULD);
pathQueryBuilder.add(prefixQuery, BooleanClause.Occur.SHOULD);
return pathQueryBuilder.build();
}
public List<FileIntroVO> getFileIntroVOs(List<String> fileIdList) {
List<ObjectId> objectIds = fileIdList.stream()
.filter(ObjectId::isValid)
.map(ObjectId::new)
.toList();
List<org.bson.Document> pipeline = Arrays.asList(new org.bson.Document("$match",
new org.bson.Document("_id",
new org.bson.Document("$in", objectIds))),
new org.bson.Document("$project", new org.bson.Document("order", 0L).append("contentText", 0L).append("content", 0L)),
new org.bson.Document("$addFields",
new org.bson.Document("order",
new org.bson.Document("$indexOfArray", Arrays.asList(objectIds, "$_id")))),
new org.bson.Document("$sort",
new org.bson.Document("order", 1L))
);
AggregateIterable<org.bson.Document> aggregateIterable = mongoTemplate.getCollection(COLLECTION_NAME).aggregate(pipeline);
List<FileIntroVO> results = new ArrayList<>();
for (org.bson.Document document : aggregateIterable) {
FileIntroVO fileIntroVO = mongoTemplate.getConverter().read(FileIntroVO.class, document);
results.add(fileIntroVO);
}
return results;
}
}

View File

@ -0,0 +1,42 @@
package com.jmal.clouddisk.lucene;
import lombok.Data;
@Data
public class TaskProgress {
/**
* 任务id
*/
private String taskId;
/**
* 任务类型
*/
private TaskType taskType;
/**
* 任务类型 taskType.getType()
*/
private String type;
/**
* 任务名称
*/
private String name;
/**
* 文件路径
*/
private String path;
/**
* 任务状态
*/
private String progress;
private String username;
public TaskProgress(String taskId, String username, TaskType taskType, String name, String progress) {
this.taskId = taskId;
this.username = username;
this.taskType = taskType;
this.type = taskType.getType();
this.name = name;
this.progress = progress;
}
}

View File

@ -0,0 +1,117 @@
package com.jmal.clouddisk.lucene;
import cn.hutool.crypto.SecureUtil;
import com.jmal.clouddisk.service.impl.CommonFileService;
import com.jmal.clouddisk.service.impl.UserServiceImpl;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.io.File;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentSkipListMap;
@Service
@RequiredArgsConstructor
@Slf4j
public class TaskProgressService {
private final CommonFileService commonFileService;
private final UserServiceImpl userService;
private final static Map<String, TaskProgress> TASK_PROGRESS_MAP = new ConcurrentSkipListMap<>();
private final static String MSG_TASK_PROGRESS = "taskProgress";
private final static String MSG_TRANSCODE_STATUS = "transcodeStatus";
private String defaultUsername;
public List<TaskProgress> getTaskProgressList() {
return List.copyOf(TASK_PROGRESS_MAP.values());
}
/**
* 更新转码状态
* @param transcodeStatus 转码状态
*/
public void pushTranscodeStatus(Map<String, Integer> transcodeStatus) {
commonFileService.pushMessage(getDefaultUsername(), transcodeStatus, MSG_TRANSCODE_STATUS);
}
/**
* 添加任务进度
* @param file 文件
* @param taskType 任务类型
* @param progress 进度
*/
public void addTaskProgress(File file, TaskType taskType, String progress) {
getDefaultUsername();
String taskId = getTaskId(file);
TaskProgress taskProgress;
if (checkTaskProgress(taskId)) {
taskProgress = getTaskProgress(taskId);
taskProgress.setProgress(progress);
} else {
String username = commonFileService.getUsernameByAbsolutePath(file.toPath());
if (username == null) {
return;
}
taskProgress = new TaskProgress(taskId, username, taskType, file.getName(), progress);
taskProgress.setPath(commonFileService.getRelativePath(username, file.getAbsolutePath(), file.getName()));
}
addTaskProgress(taskProgress);
}
/**
* 移除任务进度
* @param file 文件
*/
public void removeTaskProgress(File file) {
String taskId = getTaskId(file);
removeTaskProgress(taskId);
}
private String getDefaultUsername() {
if (defaultUsername == null) {
defaultUsername = userService.getCreatorUsername();
}
return defaultUsername;
}
private static String getTaskId(File file) {
return SecureUtil.md5(file.getAbsoluteFile().toString());
}
private void removeTaskProgress(String taskId) {
if (checkTaskProgress(taskId)) {
TaskProgress taskProgress = getTaskProgress(taskId);
taskProgress.setProgress(null);
pushMessage(taskProgress);
TASK_PROGRESS_MAP.remove(taskId);
}
}
private void addTaskProgress(TaskProgress taskProgress) {
TASK_PROGRESS_MAP.put(taskProgress.getTaskId(), taskProgress);
pushMessage(taskProgress);
}
private boolean checkTaskProgress(String taskId) {
return TASK_PROGRESS_MAP.containsKey(taskId);
}
private TaskProgress getTaskProgress(String taskId) {
return TASK_PROGRESS_MAP.get(taskId);
}
private void pushMessage(TaskProgress taskProgress) {
if (!taskProgress.getUsername().equals(defaultUsername)) {
commonFileService.pushMessage(defaultUsername, taskProgress, MSG_TASK_PROGRESS);
}
commonFileService.pushMessage(taskProgress.getUsername(), taskProgress, MSG_TASK_PROGRESS);
}
}

View File

@ -0,0 +1,24 @@
package com.jmal.clouddisk.lucene;
import lombok.Getter;
/**
* 任务类型
*/
@Getter
public enum TaskType {
/**
* OCR识别
*/
OCR("OCR识别"),
/**
* 视频转码
*/
TRANSCODE_VIDEO("视频转码");
private final String type;
TaskType(String type) {
this.type = type;
}
}

View File

@ -0,0 +1,320 @@
package com.jmal.clouddisk.media;
import cn.hutool.core.convert.Convert;
import cn.hutool.core.util.NumberUtil;
import com.alibaba.fastjson2.JSON;
import com.alibaba.fastjson2.JSONArray;
import com.alibaba.fastjson2.JSONObject;
import com.jmal.clouddisk.service.Constants;
import lombok.extern.slf4j.Slf4j;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.stream.Collectors;
import static com.jmal.clouddisk.util.FFMPEGUtils.printErrorInfo;
@Slf4j
public class FFMPEGCommand {
/**
* 缩略图宽度
*/
static final int thumbnailWidth = 192;
/**
* 获取视频的分辨率和码率信息
*
* @param videoPath 视频路径
* @return 视频信息
*/
static VideoInfo getVideoInfo(String videoPath) {
try {
ProcessBuilder processBuilder = new ProcessBuilder(
"ffprobe", "-v", "error", "-select_streams", "v:0", "-show_format", "-show_streams", "-of", "json", videoPath);
Process process = processBuilder.start();
try (InputStream inputStream = process.getInputStream();
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) {
String json = reader.lines().collect(Collectors.joining());
JSONObject jsonObject = JSON.parseObject(json);
// 获取视频格式信息
JSONObject formatObject = jsonObject.getJSONObject("format");
String format = formatObject.getString("format_name");
// 获取视频时长
int duration = Convert.toInt(formatObject.get("duration"));
// 获取视频流信息
JSONArray streamsArray = jsonObject.getJSONArray("streams");
if (!streamsArray.isEmpty()) {
JSONObject streamObject = streamsArray.getJSONObject(0);
int width = streamObject.getIntValue("width");
int height = streamObject.getIntValue("height");
int rotation = 0;
if (streamObject.containsKey("side_data_list")) {
for (int i = 0; i < streamObject.getJSONArray("side_data_list").size(); i++) {
JSONObject sideData = streamObject.getJSONArray("side_data_list").getJSONObject(i);
if (sideData.containsKey("rotation")) {
rotation = sideData.getIntValue("rotation");
break;
}
}
}
// 转换rotation
// 获取视频帧率信息
String frameRateStr = streamObject.getString("r_frame_rate");
double frameRate = parseFrameRate(frameRateStr);
int bitrate = streamObject.getIntValue("bit_rate"); // bps
return new VideoInfo(videoPath, width, height, format, bitrate, duration, Math.abs(rotation), frameRate);
}
}
int exitCode = process.waitFor();
if (exitCode != 0) {
printErrorInfo(processBuilder, process);
}
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return new VideoInfo();
}
/**
* 解析帧率字符串
*
* @param frameRateStr 帧率字符串(形如 "30000/1001")
* @return 帧率
*/
static double parseFrameRate(String frameRateStr) {
if (frameRateStr != null && frameRateStr.contains("/")) {
String[] parts = frameRateStr.split("/");
if (parts.length == 2) {
try {
double numerator = Double.parseDouble(parts[0]);
double denominator = Double.parseDouble(parts[1]);
return NumberUtil.round(numerator / denominator, 2).doubleValue();
} catch (NumberFormatException e) {
log.error("Failed to parse frame rate: {}", frameRateStr, e);
}
}
}
return 0.0;
}
/**
* 合并缩略图
* @param inputPattern 输入文件名格式
* @param outputImage 输出图片
* @param columns 列数
* @param rows 行数
* @return ProcessBuilder
*/
static ProcessBuilder mergeVTT(String inputPattern, String outputImage, int columns, int rows) {
ProcessBuilder processBuilder = new ProcessBuilder(
Constants.FFMPEG,
"-y",
"-i", inputPattern,
"-filter_complex", String.format("tile=%dx%d", columns, rows),
outputImage
);
processBuilder.redirectErrorStream(true);
return processBuilder;
}
/**
* 获取视频封面
* @param videoPath 视频路径
* @param outputPath 输出路径
* @param videoDuration 视频时长
* @return ProcessBuilder
*/
static ProcessBuilder getVideoCoverProcessBuilder(String videoPath, String outputPath, int videoDuration) {
int targetTimestamp = (int) (videoDuration * 0.1);
String formattedTimestamp = VideoInfoUtil.formatTimestamp(targetTimestamp, false);
log.debug("\r\nvideoPath: {}, formattedTimestamp: {}", videoPath, formattedTimestamp);
ProcessBuilder processBuilder = new ProcessBuilder(
Constants.FFMPEG,
"-y",
"-ss", formattedTimestamp,
"-i", videoPath,
"-vf", String.format("scale=%s:-2", thumbnailWidth),
"-frames:v", "1",
outputPath
);
processBuilder.redirectErrorStream(true);
return processBuilder;
}
static ProcessBuilder useVideotoolbox(String fileId, Path fileAbsolutePath, int bitrate, int height, String videoCacheDir, String outputPath, int vttInterval, String thumbnailPattern, double frameRate) {
return new ProcessBuilder(
Constants.FFMPEG,
"-hwaccel", "videotoolbox",
"-i", fileAbsolutePath.toString(),
"-c:v", "h264_videotoolbox",
"-profile:v", "main",
"-pix_fmt", "yuv420p",
"-level", "4.0",
"-start_number", "0",
"-hls_time", "10",
"-hls_list_size", "0",
"-vf", "scale=-2:" + height,
"-b:v", Convert.toStr(bitrate),
"-preset", "medium",
"-g", "48",
"-sc_threshold", "0",
"-r", String.format("%.2f", frameRate),
"-f", "hls",
"-hls_segment_filename", Paths.get(videoCacheDir, fileId + "-%03d.ts").toString(),
outputPath,
"-vf", String.format("scale=%s:-2,fps=1/%d", thumbnailWidth, vttInterval),
thumbnailPattern
);
}
static ProcessBuilder cpuTranscoding(String fileId, Path fileAbsolutePath, int bitrate, int height, String videoCacheDir, String outputPath, int vttInterval, String thumbnailPattern, double frameRate) {
return new ProcessBuilder(
Constants.FFMPEG,
"-i", fileAbsolutePath.toString(),
"-profile:v", "main",
"-pix_fmt", "yuv420p",
"-level", "4.0",
"-start_number", "0",
"-hls_time", "10",
"-hls_list_size", "0",
"-vf", "scale=-2:" + height,
"-b:v", Convert.toStr(bitrate),
"-preset", "medium",
"-g", "48",
"-sc_threshold", "0",
"-r", String.format("%.2f", frameRate),
"-f", "hls",
"-hls_segment_filename", Paths.get(videoCacheDir, fileId + "-%03d.ts").toString(),
outputPath,
"-vf", String.format("scale=%s:-2,fps=1/%d", thumbnailWidth, vttInterval),
thumbnailPattern
);
}
static ProcessBuilder useNvencCuda(String fileId, Path fileAbsolutePath, int bitrate, int height, String videoCacheDir, String outputPath, double frameRate) {
// 使用CUDA硬件加速和NVENC编码器
return new ProcessBuilder(
Constants.FFMPEG,
"-init_hw_device", "cuda=cu:0",
"-filter_hw_device", "cu",
"-hwaccel", "cuda",
"-hwaccel_output_format", "cuda",
"-threads", "1",
"-autorotate", "0",
"-i", fileAbsolutePath.toString(),
"-autoscale", "0",
"-map_metadata", "-1",
"-map_chapters", "-1",
"-threads", "0",
"-map", "0:0",
"-map", "0:1",
"-map", "-0:s",
"-codec:v:0", "h264_nvenc",
"-preset", "p1",
"-b:v", Convert.toStr(bitrate),
"-maxrate", Convert.toStr(bitrate),
"-bufsize", "5643118",
"-g:v:0", "180",
"-keyint_min:v:0", "180",
"-vf", "setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709,scale_cuda=-2:" + height + ":format=yuv420p",
"-codec:a:0", "copy",
"-copyts",
"-avoid_negative_ts", "disabled",
"-max_muxing_queue_size", "2048",
"-r", String.format("%.2f", frameRate),
"-f", "hls",
"-max_delay", "5000000",
"-hls_time", "3",
"-hls_segment_type", "mpegts",
"-start_number", "0",
"-y",
"-hls_segment_filename", Paths.get(videoCacheDir, fileId + "-%03d.ts").toString(),
"-hls_playlist_type", "vod",
"-hls_list_size", "0",
outputPath
);
}
static ProcessBuilder useNvencCudaVtt(Path fileAbsolutePath, int vttInterval, String thumbnailPattern) {
// 使用CUDA硬件加速和NVENC编码器
return new ProcessBuilder(
Constants.FFMPEG,
"-init_hw_device", "cuda=cu:0",
"-filter_hw_device", "cu",
"-i", fileAbsolutePath.toString(),
"-y",
"-vf", String.format("scale=%s:-2,fps=1/%d", thumbnailWidth, vttInterval),
"-vsync", "vfr",
thumbnailPattern
);
}
/**
* 检测是否装有Mac Apple Silicon
*/
static boolean checkMacAppleSilicon() {
try {
Process process = Runtime.getRuntime().exec("ffmpeg -hwaccels");
BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
String line;
while ((line = reader.readLine()) != null) {
if (line.contains("videotoolbox")) {
return true;
}
}
} catch (IOException e) {
return false;
}
return false;
}
/**
* 检测是否装有NVIDIA显卡驱动
*/
static boolean checkNvidiaDrive() {
try {
Process process = Runtime.getRuntime().exec("nvidia-smi");
BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
String line;
while ((line = reader.readLine()) != null) {
if (line.contains("Version")) {
return true;
}
}
} catch (IOException e) {
return false;
}
return false;
}
/**
* 检查是否没有ffmpeg
* @return true: 没有ffmpeg
*/
public static boolean hasNoFFmpeg() {
try {
Process process = Runtime.getRuntime().exec("ffmpeg -version");
BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
String line;
while ((line = reader.readLine()) != null) {
if (line.contains("ffmpeg version")) {
return false;
}
}
return true;
} catch (IOException e) {
log.error(e.getMessage(), e);
}
return true;
}
}

View File

@ -0,0 +1,126 @@
package com.jmal.clouddisk.media;
import cn.hutool.core.util.BooleanUtil;
import cn.hutool.core.util.NumberUtil;
import lombok.extern.slf4j.Slf4j;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
@Slf4j
public class FFMPEGUtils {
/**
* h5播放器支持的视频格式
*/
private static final String[] WEB_SUPPORTED_FORMATS = {"mp4", "webm", "ogg", "flv", "hls", "mkv"};
/**
* 获取VTT文件间隔
*
* @param videoInfo 视频信息
* @param vttThumbnailCount 期望的缩略图数量
* @return VTT文件间隔
*/
static int getVttInterval(VideoInfo videoInfo, Integer vttThumbnailCount) {
// 期望的缩略图数量默认为60张
int videoDuration = videoInfo.getDuration();
// 计算缩略图间隔
if (videoDuration <= vttThumbnailCount) {
return 1;
}
return videoDuration / vttThumbnailCount;
}
/**
* 生成VTT文件
* @param videoInfo 视频信息
* @param interval 缩略图间隔
* @param vttFilePath VTT文件路径
* @param thumbnailImagePath 最终缩略图路径
*/
static void generateVTT(VideoInfo videoInfo, int interval, String vttFilePath, String thumbnailImagePath, int columns) {
try (BufferedWriter writer = new BufferedWriter(new FileWriter(vttFilePath))) {
writer.write("WEBVTT\n\n");
int expectedThumbnails = videoInfo.getDuration() / interval;
for (int i = 0; i < expectedThumbnails; i++) {
String startTime = VideoInfoUtil.formatTimestamp(i * interval, true);
String endTime = VideoInfoUtil.formatTimestamp((i + 1) * interval, true);
int column = i % columns;
int row = i / columns;
int thumbWidth = FFMPEGCommand.thumbnailWidth;
double tHeight = (double) videoInfo.getHeight() / ((double) videoInfo.getWidth() / FFMPEGCommand.thumbnailWidth);
int thumbHeight = (int) Math.floor(tHeight);
int x = column * thumbWidth;
int y = row * thumbHeight;
writer.write(String.format("%s --> %s\n", startTime, endTime));
writer.write(String.format("%s#xywh=%d,%d,%d,%d\n\n", thumbnailImagePath, x, y, thumbWidth, thumbHeight));
}
} catch (IOException e) {
log.warn(e.getMessage(), e);
}
}
/**
* 判断是否需要转码
*
* @param videoInfo 视频信息
* @return 是否需要转码
*/
static boolean needTranscode(VideoInfo videoInfo, TranscodeConfig transcodeConfig) {
if (BooleanUtil.isFalse(transcodeConfig.getEnable())) {
return false;
}
// 判断视频码率、高度、帧率是否满足转码条件
boolean bitrateCond = videoInfo.getBitrate() > 0 && videoInfo.getBitrate() <= transcodeConfig.getBitrateCond();
boolean heightCond = videoInfo.getHeight() > 0 && videoInfo.getHeight() <= transcodeConfig.getHeightCond();
boolean frameRateCond = videoInfo.getFrameRate() > 0 && videoInfo.getFrameRate() <= transcodeConfig.getFrameRateCond();
if (bitrateCond && heightCond && frameRateCond) {
// 都不满足, 则判断视频格式是否为HTML5 Video Player支持的格式, 不支持则需要转码
return !isSupportedFormat(videoInfo.getFormat());
}
return true;
}
/**
* 判断视频格式是否为HTML5 Video Player支持的格式
*
* @param format 视频格式
* @return 是否支持
*/
static boolean isSupportedFormat(String format) {
// HTML5 Video Player支持的视频格式
String[] formatList = format.split(",");
for (String f : formatList) {
for (String supportedFormat : WEB_SUPPORTED_FORMATS) {
if (f.trim().equalsIgnoreCase(supportedFormat)) {
return true;
}
}
}
return false;
}
/**
* 获取转码进度
* @param videoDuration 视频时长
* @param line 命令输出信息
* @return 转码进度
*/
static double getProgressStr(int videoDuration, String line) {
String[] parts = line.split("time=")[1].split(" ")[0].split(":");
int hours = Integer.parseInt(parts[0]);
int minutes = Integer.parseInt(parts[1]);
int seconds = Integer.parseInt(parts[2].split("\\.")[0]);
int totalSeconds = hours * 3600 + minutes * 60 + seconds;
// 计算转码进度百分比
double progress = (double) totalSeconds / videoDuration * 100;
if (progress > 100) {
progress = 100;
}
return NumberUtil.round(progress, 2).doubleValue();
}
}

View File

@ -0,0 +1,122 @@
package com.jmal.clouddisk.media;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.io.file.PathUtil;
import com.jmal.clouddisk.service.Constants;
import lombok.extern.slf4j.Slf4j;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static com.jmal.clouddisk.util.FFMPEGUtils.getWaitingForResults;
@Slf4j
public class HeifUtils {
/**
* 将heic转换为jpg
*/
public static String heifConvert(String inputPath, Path outputParentPath) {
if (!checkHeif()) {
return null;
}
Path filepath = Paths.get(inputPath + ".jpg");
String outputPath = Paths.get(outputParentPath.toString(), filepath.getFileName().toString()).toString();
if (FileUtil.exist(outputPath)) {
return outputPath;
}
try {
ProcessBuilder processBuilder = heifConvert(inputPath, outputPath);
Process process = processBuilder.start();
StringBuilder output = printProcessInfo(process);
outputPath = getWaitingForResults(outputPath, processBuilder, process);
delAuxiliaryImagePath(output);
// 打印命令 用空格连接
String command = String.join(" ", processBuilder.command());
log.info("heif-convert 执行成功, command: \r\n{}", command);
if (outputPath != null) {
PathUtil.move(Path.of(outputPath), filepath, true);
return filepath.toString();
}
} catch (InterruptedException e) {
log.error(e.getMessage(), e);
return null;
} catch (IOException e) {
log.error(e.getMessage(), e);
}
return null;
}
/**
* 删除辅助图像文件
* @param output process输出
*/
private static void delAuxiliaryImagePath(StringBuilder output) {
// 解析输出,查找辅助图像文件名, 并删除
String auxiliaryImagePath = null;
Pattern pattern = Pattern.compile("Auxiliary image written to (.+)");
Matcher matcher = pattern.matcher(output.toString());
if (matcher.find()) {
auxiliaryImagePath = matcher.group(1);
}
if (auxiliaryImagePath != null) {
FileUtil.del(auxiliaryImagePath);
}
}
/**
* 将heic转换为jpg
* @param filepath heic文件绝对路径
* @return ProcessBuilder
*/
private static ProcessBuilder heifConvert(String filepath, String outputPath) {
ProcessBuilder processBuilder = new ProcessBuilder(
Constants.HEIF_CONVERT,
filepath,
outputPath
);
processBuilder.redirectErrorStream(true);
return processBuilder;
}
public static StringBuilder printProcessInfo(Process process) throws IOException {
StringBuilder output = new StringBuilder();
try (InputStream inputStream = process.getInputStream();
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) {
String line;
while ((line = reader.readLine()) != null) {
output.append(line).append("\n");
}
}
return output;
}
/**
* 检查是否安装了heif-convert
*/
private static boolean checkHeif() {
try {
Process process = Runtime.getRuntime().exec(Constants.HEIF_CONVERT);
BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
String line;
while ((line = reader.readLine()) != null) {
if (line.contains("output")) {
return true;
}
}
return true;
} catch (IOException e) {
log.error(e.getMessage(), e);
}
return false;
}
}

View File

@ -0,0 +1,124 @@
package com.jmal.clouddisk.media;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.Valid;
import jakarta.validation.constraints.Max;
import jakarta.validation.constraints.Min;
import lombok.Data;
import org.springframework.data.mongodb.core.mapping.Document;
/**
* 转码配置
*/
@Data
@Document(collection = "transcodeConfig")
@Valid
@Schema
public class TranscodeConfig {
@Schema(description = "是否启用转码, 默认开启")
private Boolean enable;
@Max(value = 8, message = "最大任务数不能超过8")
@Min(value = 1, message = "最大任务数不能小于1")
@Schema(description = "最大任务数, 最多同时处理的转码任务数, 默认为1")
private Integer maxThreads;
@Max(value = 1000000, message = "码率不能超过 1000000 kbps")
@Min(value = 100, message = "码率不能低于 100 kbps")
@Schema(description = "转码条件: 视频码率(kbps), 默认 2500 kbps, 小于该值则不转码")
private Integer bitrateCond;
@Max(value = 10000, message = "高度不能超过 10000")
@Min(value = 100, message = "高度不能低于 100")
@Schema(description = "转码条件: 视频高度(视频宽度, 默认随高度等比例缩放), 默认 720, 小于该值则不转码")
private Integer heightCond;
@Max(value = 120, message = "目标帧率不能超过 120")
@Min(value = 1, message = "目标帧率不能低于 1")
@Schema(description = "转码条件: 视频帧率, 默认 30 fps, 小于该值则不转码")
private Integer frameRateCond;
@Max(value = 1000000, message = "码率不能超过 1000000 kbps")
@Min(value = 100, message = "码率不能低于 100 kbps")
@Schema(description = "转码后的视频码率(kbps), 默认 2500 kbps")
private Integer bitrate;
@Max(value = 10000, message = "高度不能超过 10000")
@Min(value = 100, message = "高度不能低于 100")
@Schema(description = "转码后的视频高度(视频宽度, 默认随高度等比例缩放), 默认 720")
private Integer height;
@Max(value = 120, message = "目标帧率不能超过 120")
@Min(value = 1, message = "目标帧率不能低于 1")
@Schema(description = "转码后的视频帧率, 默认 30 fps")
private Double frameRate;
@Max(value = 256, message = "缩略图数量超过 256")
@Min(value = 1, message = "缩略图数量不能低于 1")
@Schema(description = "vtt缩略图数量, 默认 60 张")
private Integer vttThumbnailCount;
@Schema(description = "是否重新转码, 转码参数变化后对已经转码过的视频重新转码, 默认开启")
private Boolean isReTranscode;
public Boolean getEnable() {
if (enable == null)
return true;
return enable;
}
public Integer getHeight() {
if (height == null)
return 720;
return height;
}
public Integer getBitrate() {
if (bitrate == null)
return 2500;
return bitrate;
}
public Double getFrameRate() {
if (frameRate == null)
return 30.0;
return frameRate;
}
public Integer getMaxThreads() {
if (maxThreads == null)
return 1;
return maxThreads;
}
public Integer getBitrateCond() {
if (bitrateCond == null)
return 2500;
return bitrateCond;
}
public Integer getHeightCond() {
if (heightCond == null)
return 720;
return heightCond;
}
public Integer getFrameRateCond() {
if (frameRateCond == null)
return 30;
return frameRateCond;
}
public Integer getVttThumbnailCount() {
if (vttThumbnailCount == null)
return 60;
return vttThumbnailCount;
}
public Boolean getIsReTranscode() {
if (isReTranscode == null)
return true;
return isReTranscode;
}
}

View File

@ -0,0 +1,29 @@
package com.jmal.clouddisk.media;
import lombok.Getter;
/**
* 转码状态
*/
@Getter
public enum TranscodeStatus {
/**
* 待转码
*/
NOT_TRANSCODE(0),
/**
* 正在进行转码
*/
TRANSCODING(1),
/**
* 已完成转码
*/
TRANSCENDED(2);
private final int status;
TranscodeStatus(int status) {
this.status = status;
}
}

View File

@ -0,0 +1,83 @@
package com.jmal.clouddisk.media;
import cn.hutool.core.util.StrUtil;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
@Data
@Slf4j
public class VideoInfo {
private Integer width;
private Integer height;
private Integer bitrate;
private String format;
private Double frameRate;
private Integer duration;
private String covertPath;
/**
* 视频旋转角度
*/
private Integer rotation;
/**
* 转码后的视频高度
*/
private Integer toHeight;
/**
* 转码后的视频码率
*/
private Integer toBitrate;
/**
* 转码后的视频帧率
*/
private Double toFrameRate;
public VideoInfo() {
this.width = 0;
this.height = 0;
this.frameRate = 0.0;
this.format = "";
this.bitrate = 0;
this.duration = 0;
}
public VideoInfo(String videoPath, int width, int height, String format, int bitrate, int duration, int rotation, double frameRate) {
if (rotation == 90 || rotation == 270) {
this.width = height;
this.height = width;
} else {
this.width = width;
this.height = height;
}
this.format = format;
this.bitrate = bitrate;
this.duration = duration;
this.frameRate = frameRate;
this.rotation = rotation;
log.debug("\r\nvideoPath: {}, width: {}, height: {}, format: {}, bitrate: {}, duration: {}", videoPath, width, height, format, bitrate, duration);
}
public VideoInfoDO toVideoInfoDO() {
VideoInfoDO videoInfoDO = new VideoInfoDO();
if (this.bitrate > 0) {
videoInfoDO.setBitrateNum(this.bitrate);
videoInfoDO.setBitrate(VideoInfoUtil.convertBitrateToReadableFormat(this.bitrate));
}
if (StrUtil.isNotBlank(this.format)) {
videoInfoDO.setFormat(this.format);
}
if (this.duration > 0) {
videoInfoDO.setDurationNum(this.duration);
videoInfoDO.setDuration(VideoInfoUtil.formatTimestamp(this.duration, false));
}
if (this.height > 0 && this.width > 0) {
videoInfoDO.setHeight(this.height);
videoInfoDO.setWidth(this.width);
}
if (this.frameRate > 0) {
videoInfoDO.setFrameRate(this.frameRate);
}
return videoInfoDO;
}
}

Some files were not shown because too many files have changed in this diff Show More