---

前言

springboot 斷點上傳、續(xù)傳、秒傳實現(xiàn)。
保存方式提供本地上傳（單機）和minio上傳（可集群）
本文主要是后端實現(xiàn)方案，數(shù)據(jù)庫持久化采用jpa

一、實現(xiàn)思路

1. 前端生成文件md5，根據(jù)md5檢查文件塊上傳進度或秒傳

2. 需要上傳分片的文件上傳分片文件

3. 分片合并后上傳服務(wù)器

二、數(shù)據(jù)庫表對象

說明：

1. AbstractDomainPd<String>為公共字段，如id，創(chuàng)建人，創(chuàng)建時間等，根據(jù)自己框架修改即可。
2. clientId 應(yīng)用id用于隔離不同應(yīng)用附件，非必須
   附件表：上傳成功的附件信息

@Entity
@Table(name = "gsdss_file", schema = "public")
@Data
public class AttachmentPO extends AbstractDomainPd<String> implements Serializable {/*** 相對路徑*/private String path;/*** 文件名*/private String fileName;/*** 文件大小*/private String size;/*** 文件MD5*/private String fileIdentifier;
}

分片信息表：記錄當前文件已上傳的分片數(shù)據(jù)

@Entity
@Table(name = "gsdss_file_chunk", schema = "public")
@Data
public class ChunkPO extends AbstractDomainPd<String> implements Serializable {/*** 應(yīng)用id*/private String clientId;/*** 文件塊編號，從1開始*/private Integer chunkNumber;/*** 文件標識MD5*/private String fileIdentifier;/*** 文件名*/private String fileName;/*** 相對路徑*/private String path;}

二、業(yè)務(wù)入?yún)ο?/h1>

檢查文件塊上傳進度或秒傳入?yún)ο?/p>

package com.gsafety.bg.gsdss.file.manage.model.req;import io.swagger.v3.oas.annotations.Hidden;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.web.multipart.MultipartFile;import javax.validation.constraints.NotNull;@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class ChunkReq {/*** 文件塊編號，從1開始*/@NotNullprivate Integer chunkNumber;/*** 文件標識MD5*/@NotNullprivate String fileIdentifier;/*** 相對路徑*/@NotNullprivate String path;/*** 塊內(nèi)容*/@Hiddenprivate MultipartFile file;/*** 應(yīng)用id*/@NotNullprivate String clientId;/*** 文件名*/@NotNullprivate String fileName;
}

上傳分片入?yún)?/p>

@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class CheckChunkReq {/*** 應(yīng)用id*/@NotNullprivate String clientId;/*** 文件名*/@NotNullprivate String fileName;/*** md5*/@NotNullprivate String fileIdentifier;
}

分片合并入?yún)?/p>

@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class FileReq {@Hiddenprivate MultipartFile file;/*** 文件名*/@NotNullprivate String fileName;/*** 文件大小*/@NotNullprivate Long fileSize;/*** eg:data/plan/*/@NotNullprivate String path;/*** md5*/@NotNullprivate String fileIdentifier;/*** 應(yīng)用id*/@NotNullprivate String clientId;
}

檢查文件塊上傳進度或秒傳返回結(jié)果

@Data
public class UploadResp implements Serializable {/*** 是否跳過上傳（已上傳的可以直接跳過，達到秒傳的效果）*/private boolean skipUpload = false;/*** 已經(jīng)上傳的文件塊編號，可以跳過，斷點續(xù)傳*/private List<Integer> uploadedChunks;/*** 文件信息*/private AttachmentResp fileInfo;}

三、本地上傳實現(xiàn)

    @Resourceprivate S3OssProperties properties;@Resourceprivate AttachmentService attachmentService;@Resourceprivate ChunkDao chunkDao;@Resourceprivate ChunkMapping chunkMapping;/*** 上傳分片文件** @param req*/@Overridepublic boolean uploadChunk(ChunkReq req) {BizPreconditions.checkArgumentNoStack(!req.getFile().isEmpty(), "上傳分片不能為空!");BizPreconditions.checkArgumentNoStack(req.getPath().endsWith("/"), "url參數(shù)必須是/結(jié)尾");//文件名-1String fileName = req.getFileName().concat("-").concat(req.getChunkNumber().toString());//分片文件上傳服務(wù)器的目錄地址 文件夾地址/chunks/文件md5String filePath = properties.getPath().concat(req.getClientId()).concat(File.separator).concat(req.getPath()).concat("chunks").concat(File.separator).concat(req.getFileIdentifier()).concat(File.separator);try {Path newPath = Paths.get(filePath);Files.createDirectories(newPath);//文件夾地址/md5/文件名-1newPath = Paths.get(filePath.concat(fileName));if (Files.notExists(newPath)) {Files.createFile(newPath);}Files.write(newPath, req.getFile().getBytes(), StandardOpenOption.CREATE);} catch (IOException e) {log.error(" 附件存儲失敗 ", e);throw new BusinessCheckException("附件存儲失敗");}// 存儲分片信息chunkDao.save(chunkMapping.req2PO(req));return true;}/*** 檢查文件塊*/@Overridepublic UploadResp checkChunk(CheckChunkReq req) {UploadResp result = new UploadResp();//查詢數(shù)據(jù)庫記錄//先判斷整個文件是否已經(jīng)上傳過了，如果是，則告訴前端跳過上傳，實現(xiàn)秒傳AttachmentResp resp = attachmentService.findByFileIdentifierAndClientId(req.getFileIdentifier(), req.getClientId());if (resp != null) {//當前文件信息另存AttachmentResp newResp = attachmentService.save(AttachmentReq.builder().fileName(req.getFileName()).origin(AttachmentConstants.TYPE.LOCAL_TYPE).clientId(req.getClientId()).path(resp.getPath()).size(resp.getSize()).fileIdentifier(req.getFileIdentifier()).build());result.setSkipUpload(true);result.setFileInfo(newResp);return result;}//如果完整文件不存在，則去數(shù)據(jù)庫判斷當前哪些文件塊已經(jīng)上傳過了，把結(jié)果告訴前端，跳過這些文件塊的上傳，實現(xiàn)斷點續(xù)傳List<ChunkPO> chunkList = chunkDao.findByFileIdentifierAndClientId(req.getFileIdentifier(), req.getClientId());//將已存在的塊的chunkNumber列表返回給前端,前端會規(guī)避掉這些塊if (!CollectionUtils.isEmpty(chunkList)) {List<Integer> collect = chunkList.stream().map(ChunkPO::getChunkNumber).collect(Collectors.toList());result.setUploadedChunks(collect);}return result;}/*** 分片合并** @param req*/@Overridepublic boolean mergeChunk(FileReq req) {String filename = req.getFileName();String date = DateUtil.localDateToString(LocalDate.now());//附件服務(wù)器存儲合并后的文件存放地址String file = properties.getPath().concat(req.getClientId()).concat(File.separator).concat(req.getPath()).concat(date).concat(File.separator).concat(filename);//服務(wù)器分片文件存放地址String folder = properties.getPath().concat(req.getClientId()).concat(File.separator).concat(req.getPath()).concat("chunks").concat(File.separator).concat(req.getFileIdentifier());//合并文件到本地目錄，并刪除分片文件boolean flag = mergeFile(file, folder, filename);if (!flag) {return false;}//保存文件記錄AttachmentResp resp = attachmentService.findByFileIdentifierAndClientId(req.getFileIdentifier(), req.getClientId());if (resp == null) {attachmentService.save(AttachmentReq.builder().fileName(filename).origin(AttachmentConstants.TYPE.LOCAL_TYPE).clientId(req.getClientId()).path(file).size(FileUtils.changeFileFormat(req.getFileSize())).fileIdentifier(req.getFileIdentifier()).build());}//插入文件記錄成功后,刪除chunk表中的對應(yīng)記錄,釋放空間chunkDao.deleteAllByFileIdentifierAndClientId(req.getFileIdentifier(), req.getClientId());return true;}/*** 文件合并** @param targetFile 要形成的文件地址* @param folder     分片文件存放地址* @param filename   文件的名稱*/private boolean mergeFile(String targetFile, String folder, String filename) {try {//先判斷文件是否存在if (FileUtils.fileExists(targetFile)) {//文件已存在return true;}Path newPath = Paths.get(StringUtils.substringBeforeLast(targetFile, File.separator));Files.createDirectories(newPath);Files.createFile(Paths.get(targetFile));Files.list(Paths.get(folder)).filter(path -> !path.getFileName().toString().equals(filename)).sorted((o1, o2) -> {String p1 = o1.getFileName().toString();String p2 = o2.getFileName().toString();int i1 = p1.lastIndexOf("-");int i2 = p2.lastIndexOf("-");return Integer.valueOf(p2.substring(i2)).compareTo(Integer.valueOf(p1.substring(i1)));}).forEach(path -> {try {//以追加的形式寫入文件Files.write(Paths.get(targetFile), Files.readAllBytes(path), StandardOpenOption.APPEND);//合并后刪除該塊Files.delete(path);} catch (IOException e) {log.error(e.getMessage(), e);throw new BusinessException("文件合并失敗");}});//刪除空文件夾FileUtils.delDir(folder);} catch (IOException e) {log.error("文件合并失敗: ", e);throw new BusinessException("文件合并失敗");}return true;}

三、minio上傳實現(xiàn)

    @Resourceprivate MinioTemplate minioTemplate;@Resourceprivate AttachmentService attachmentService;@Resourceprivate ChunkDao chunkDao;@Resourceprivate ChunkMapping chunkMapping;/*** 上傳分片文件*/@Overridepublic boolean uploadChunk(ChunkReq req) {String fileName = req.getFileName();BizPreconditions.checkArgumentNoStack(!req.getFile().isEmpty(), "上傳分片不能為空!");BizPreconditions.checkArgumentNoStack(req.getPath().endsWith(separator), "url參數(shù)必須是/結(jié)尾");String newFileName = req.getPath().concat("chunks").concat(separator).concat(req.getFileIdentifier()).concat(separator)+ fileName.concat("-").concat(req.getChunkNumber().toString());try {minioTemplate.putObject(req.getClientId(), newFileName, req.getFile());} catch (Exception e) {e.printStackTrace();throw new BusinessException("文件上傳失敗");}// 存儲分片信息chunkDao.save(chunkMapping.req2PO(req));return true;}/*** 檢查文件塊*/@Overridepublic UploadResp checkChunk(CheckChunkReq req) {UploadResp result = new UploadResp();//查詢數(shù)據(jù)庫記錄//先判斷整個文件是否已經(jīng)上傳過了，如果是，則告訴前端跳過上傳，實現(xiàn)秒傳AttachmentResp resp = attachmentService.findByFileIdentifierAndClientId(req.getFileIdentifier(), req.getClientId());if (resp != null) {//當前文件信息另存AttachmentResp newResp = attachmentService.save(AttachmentReq.builder().fileName(req.getFileName()).origin(AttachmentConstants.TYPE.MINIO_TYPE).clientId(req.getClientId()).path(resp.getPath()).size(resp.getSize()).fileIdentifier(req.getFileIdentifier()).build());result.setSkipUpload(true);result.setFileInfo(newResp);return result;}//如果完整文件不存在，則去數(shù)據(jù)庫判斷當前哪些文件塊已經(jīng)上傳過了，把結(jié)果告訴前端，跳過這些文件塊的上傳，實現(xiàn)斷點續(xù)傳List<ChunkPO> chunkList = chunkDao.findByFileIdentifierAndClientId(req.getFileIdentifier(), req.getClientId());//將已存在的塊的chunkNumber列表返回給前端,前端會規(guī)避掉這些塊if (!CollectionUtils.isEmpty(chunkList)) {List<Integer> collect = chunkList.stream().map(ChunkPO::getChunkNumber).collect(Collectors.toList());result.setUploadedChunks(collect);}return result;}/*** 分片合并** @param req*/@Overridepublic boolean mergeChunk(FileReq req) {String filename = req.getFileName();//合并文件到本地目錄String chunkPath = req.getPath().concat("chunks").concat(separator).concat(req.getFileIdentifier()).concat(separator);List<Item> chunkList = minioTemplate.getAllObjectsByPrefix(req.getClientId(), chunkPath, false);String fileHz = filename.substring(filename.lastIndexOf("."));String newFileName = req.getPath() + UUIDUtil.uuid() + fileHz;try {List<ComposeSource> sourceObjectList = chunkList.stream().sorted(Comparator.comparing(Item::size).reversed()).map(l -> ComposeSource.builder().bucket(req.getClientId()).object(l.objectName()).build()).collect(Collectors.toList());ObjectWriteResponse response = minioTemplate.composeObject(req.getClientId(), newFileName, sourceObjectList);//刪除分片bucket及文件minioTemplate.removeObjects(req.getClientId(), chunkPath);} catch (Exception e) {e.printStackTrace();throw new BusinessException("文件合并失敗");}//保存文件記錄AttachmentResp resp = attachmentService.findByFileIdentifierAndClientId(req.getFileIdentifier(), req.getClientId());if (resp == null) {attachmentService.save(AttachmentReq.builder().fileName(filename).origin(AttachmentConstants.TYPE.MINIO_TYPE).clientId(req.getClientId()).path(newFileName).size(FileUtils.changeFileFormat(req.getFileSize())).fileIdentifier(req.getFileIdentifier()).build());}//插入文件記錄成功后,刪除chunk表中的對應(yīng)記錄,釋放空間chunkDao.deleteAllByFileIdentifierAndClientId(req.getFileIdentifier(), req.getClientId());return true;}

MinioTemplate 參考

總結(jié)

1. 檢查文件塊上傳進度或秒傳

根據(jù)文件md5查詢附件信息表，如果存在，直接返回附件信息。
不存在查詢分片信息表，查詢當前文件分片上傳進度，返回已經(jīng)上傳過的分片編號

2. 上傳分片

分片文件上傳地址需要保證唯一性，可用文件MD5作為隔離
上傳后保存分片上傳信息
minio對合并分片文件有大小限制，除最后一個分片外，其他分片文件大小不得小于5MB，所以minio分片上傳需要分片大小最小為5MB，并且獲取分片需要按照分片文件大小排序，將最后一個分片放到最后進行合并

3. 分片合并

將分片文件合并為新文件到最終文件存放地址并刪除分片文件
保存最終文件信息到附件信息表
刪除對應(yīng)分片信息表數(shù)據(jù)