阿里云大文件分片上传

1:新建阿里云工具类

package com.bamboo.water_chivalry.project.file.controller;

import com.aliyun.oss.OSSClient;
import com.aliyun.oss.model.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.web.multipart.MultipartFile;

import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
/**
 * @PROJECT_NAME: water_chivalry
 * @AUTHOR: Hanson-Hsc
 * @DATE: 2020-07-15 11:16
 * @DESCRIPTION:
 * @VERSION:
 */
public class AliyunOSSUpload implements Runnable{
    private MultipartFile localFile;
    private long startPos;

    private long partSize;
    private int partNumber;
    private String uploadId;
    private static String key;
    private static String bucketName;

    // 新建一个List保存每个分块上传后的ETag和PartNumber
    protected static List<PartETag> partETags = Collections.synchronizedList(new ArrayList<PartETag>());

    private static Logger logger = LoggerFactory.getLogger(FileUploader.class);

    private static OSSClient client = null;

    /**
     * 创建构造方法
     *
     * @param localFile
     *            要上传的文件
     * @param startPos
     *            每个文件块的开始
     * @param partSize
     * @param partNumber
     * @param uploadId
     *            作为块的标识
     * @param key
     *            上传到OSS后的文件名
     */
    public AliyunOSSUpload(MultipartFile localFile, long startPos, long partSize, int partNumber, String uploadId, String key , String bucketName) {
        this.localFile = localFile;
        this.startPos = startPos;
        this.partSize = partSize;
        this.partNumber = partNumber;
        this.uploadId = uploadId;
        AliyunOSSUpload.key = key;
        AliyunOSSUpload.bucketName = bucketName;
    }

    /**
     * 分块上传核心方法(将文件分成按照每个5M分成N个块，并加入到一个list集合中)
     */
    @Override
    public void run() {
        InputStream instream = null;
        try {
            // 获取文件流
            instream = localFile.getInputStream();
            // 跳到每个分块的开头
            instream.skip(this.startPos);

            // 创建UploadPartRequest，上传分块
            UploadPartRequest uploadPartRequest = new UploadPartRequest();
            uploadPartRequest.setBucketName(bucketName);
            uploadPartRequest.setKey(key);
            uploadPartRequest.setUploadId(this.uploadId);
            uploadPartRequest.setInputStream(instream);
            uploadPartRequest.setPartSize(this.partSize);
            uploadPartRequest.setPartNumber(this.partNumber);

            UploadPartResult uploadPartResult = FileUploader.client.uploadPart(uploadPartRequest);
            logger.info("Part#" + this.partNumber + " done
");
            synchronized (partETags) {
                // 将返回的PartETag保存到List中。
                partETags.add(uploadPartResult.getPartETag());
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            if (instream != null) {
                try {
                    // 关闭文件流
                    instream.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }

    /**
     * 初始化分块上传事件并生成uploadID，用来作为区分分块上传事件的唯一标识
     *
     * @return
     */
    protected static String claimUploadId(String bucketName, String key) {
        InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(bucketName, key);
        InitiateMultipartUploadResult result = FileUploader.client.initiateMultipartUpload(request);
        logger.info(result.getUploadId());
        return result.getUploadId();
    }

    /**
     * 将文件分块进行升序排序并执行文件上传。
     *
     * @param uploadId
     */
    protected static void completeMultipartUpload(String uploadId) {
        // 将文件分块按照升序排序
        Collections.sort(partETags, new Comparator<PartETag>() {
            @Override
            public int compare(PartETag p1, PartETag p2) {
                return p1.getPartNumber() - p2.getPartNumber();
            }
        });

        CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(bucketName,
                key, uploadId, partETags);
        // 完成分块上传
        FileUploader.client.completeMultipartUpload(completeMultipartUploadRequest);
    }

    /**
     * 列出文件所有分块的清单
     *
     * @param uploadId
     */
    protected static void listAllParts(String uploadId) {
        ListPartsRequest listPartsRequest = new ListPartsRequest(bucketName, key, uploadId);
        // 获取上传的所有分块信息
        PartListing partListing = FileUploader.client.listParts(listPartsRequest);

        // 获取分块的大小
        int partCount = partListing.getParts().size();
        // 遍历所有分块
        for (int i = 0; i < partCount; i++) {
            PartSummary partSummary = partListing.getParts().get(i);
            logger.info("分块编号 " + partSummary.getPartNumber() + ", ETag=" + partSummary.getETag());
        }
    }
}

2:新建文件上传接口,此处只需要controller

package com.bamboo.water_chivalry.project.file.controller;

import com.alibaba.fastjson.JSONObject;
import com.aliyun.oss.OSSClient;
import com.bamboo.water_chivalry.common.enums.ResultEnum;
import com.bamboo.water_chivalry.common.exception.GlobalException;
import com.bamboo.water_chivalry.common.utils.JudgeVIFormat;
import com.bamboo.water_chivalry.common.utils.OSSUtil;
import com.bamboo.water_chivalry.common.utils.ResultVoUtil;
import com.bamboo.water_chivalry.common.vo.other.ResultVo;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.apache.shiro.authz.annotation.Logical;
import org.apache.shiro.authz.annotation.RequiresRoles;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;

import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

import static com.bamboo.water_chivalry.common.config.FileConfig.*;
import static com.bamboo.water_chivalry.common.constant.FileConstant.FILE_UPLOAD_SIZE;
import static com.bamboo.water_chivalry.common.enums.ResultEnum.IMAGE_FILES_CANNOT_BE_UPLOADED;
import static com.bamboo.water_chivalry.common.enums.ResultEnum.VIDEO_UPLOAD_ERROR;

/**
 * @PROJECT_NAME: water_chivalry
 * @AUTHOR: Hanson-Hsc
 * @DATE: 2020-07-15 11:18
 * @DESCRIPTION:
 * @VERSION:
 */
@RestController
@RequestMapping("/file")
@Api(tags = "大文件上传接口(多线程分片上传)")
public class FileUploader {

    protected static OSSClient client = null;

    private static Logger logger = LoggerFactory.getLogger(FileUploader.class);

    @PostMapping("/thread")
    @ApiOperation(value = "视频分片上传")public static ResultVo fileUpload(@RequestParam("file") MultipartFile file) {
        // 创建一个可重用固定线程数的线程池。若同一时间线程数大于10，则多余线程会放入队列中依次执行
        ExecutorService executorService = Executors.newFixedThreadPool(20);
        // 获取上传文件的名称，作为在OSS上的文件名
        String key = file.getOriginalFilename();
        String newFileName = UUID.randomUUID() + key.substring(key.lastIndexOf("."));
        // 创建OSSClient实例
        client = new OSSClient("oss-cn-shenzhen.aliyuncs.com", "你的accessKeyId", "你的access密钥");

 try {
            String uploadId = AliyunOSSUpload.claimUploadId(VIDEO_BUCKET_NAME, newFileName);
            // 设置每块为 5M(除最后一个分块以外，其他的分块大小都要大于5MB)
            final long partSize = 5 * 1024 * 1024L;
            //final long partSize = 1024 * 1024L;
            // 计算分块数目
            long fileLength = file.getSize();
            int partCount = (int) (fileLength / partSize);
            if (fileLength % partSize != 0) {
                partCount++;
            }

            // 分块 号码的范围是1~10000。如果超出这个范围，OSS将返回InvalidArgument的错误码。
            if (partCount > BLOCK_SCOPE) {
                throw new RuntimeException("文件过大(分块大小不能超过10000)");
            } else {
                logger.info("一共分了 " + partCount + " 块");
            }

            /**
             * 将分好的文件块加入到list集合中
             */
            for (int i = 0; i < partCount; i++) {
                long startPos = i * partSize;
                long curPartSize = (i + 1 == partCount) ? (fileLength - startPos) : partSize;

                // 线程执行。将分好的文件块加入到list集合中
                executorService.execute(new AliyunOSSUpload(file, startPos, curPartSize, i + 1, uploadId, newFileName, VIDEO_BUCKET_NAME));
            }

            /**
             * 等待所有分片完毕
             */
            // 关闭线程池（线程池不马上关闭），执行以前提交的任务，但不接受新任务。
            executorService.shutdown();
            // 如果关闭后所有任务都已完成，则返回 true。
            while (!executorService.isTerminated()) {
                try {
                    // 用于等待子线程结束，再继续执行下面的代码
                    executorService.awaitTermination(5, TimeUnit.SECONDS);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            }

            /**
             * partETags(上传块的ETag与块编号（PartNumber）的组合) 如果校验与之前计算的分块大小不同，则抛出异常
             */
            System.out.println(AliyunOSSUpload.partETags.size() + " -----   " + partCount);
            if (AliyunOSSUpload.partETags.size() != partCount) {
                throw new IllegalStateException("OSS分块大小与文件所计算的分块大小不一致");
            } else {
                logger.info("将要上传的文件名  " + key + "
");
            }

            /*
             * 列出文件所有的分块清单并打印到日志中，该方法仅仅作为输出使用
             */
            AliyunOSSUpload.listAllParts(uploadId);

            /*
             * 完成分块上传
             */
            AliyunOSSUpload.completeMultipartUpload(uploadId);

            JSONObject result = new JSONObject();
            result.put("url", HTTPS + VIDEO_BUCKET_NAME + "." + END_POINT + "/" + client.getObject(VIDEO_BUCKET_NAME, newFileName).getKey());
            // 返回上传文件的URL地址
            return ResultVoUtil.success(result);

        } catch (Exception e) {
            logger.error(VIDEO_UPLOAD_ERROR.getMsg(), e);
            return ResultVoUtil.error(ResultEnum.VIDEO_UPLOAD_ERROR);
        } finally {
            AliyunOSSUpload.partETags.clear();
            if (client != null) {
                client.shutdown();
            }
        }
    }
}