异步处理百万数据导出

这是一个项目(guijiu-oms)案例直接拷过来的

　　我会把Controller + service + ExcelUtils工具类贴出来

前置准备

　　1.异步任务运行表

　　2.poi依赖

　　3.ftp服务器,相应的上传下载工具类

Controller

    /**
     * 订单列表导出*/
    @ApiOperation("导出")
    @PostMapping("/export")
    public Result export(@RequestBody QueryArgs args, HttpServletRequest req, HttpServletResponse res) throws Exception {
        Long taskId = SEQUtil.getUUID();
        SoExportTaskSchedulePO po = new SoExportTaskSchedulePO();
        po.setId(taskId);
        po.setTaskActionType(SoConstant.TASK_ACTION_TYPE_EXPORT);
        po.setTaskType(SoConstant.TASK_TYPE_SO);
        po.setTaskStatus(SoConstant.TASK_STATUS_NEW);
        UserInfo user = SessionHelper.getUser();
        po.setUserId(user.getUserId());
        po.setUserName(user.getUsername());
        po.setFilterRecord(JSON.toJSONString(args));
        po.setCreateUserid(user.getUserId());
        po.setCreateUsername(user.getUsername());
        po.setCreateTime(new Date());
        po.setCompanyId(SessionHelper.getCompanyId());
        po.setStartTime(new Date());
        Long addWithTx = soExportTaskScheduleService.addWithTx(po);
        SessionCopy sessionCopy = SessionHelper.createSessionCopy();
        soAsyncService.exportSync(args, addWithTx, 0, sessionCopy);
        return ObjectResult.ok(taskId);
    }

service 

  /**
     * 订单导出（订单以及订单行）
     *
     * @param args
     * @param taskId
     */
    @Async("orderTaskExecutor")
    public void exportSync(QueryArgs args, Long taskId, Integer flag, SessionCopy sessionCopy) throws Exception {
        try {
            SessionHelper.copySession(sessionCopy);

　　　　　　　　// 任务状态
　　　　　　　　//public final static Integer TASK_STATUS_NEW = 1;
　　　　　　　　//public final static Integer TASK_STATUS_RUNNING = 2;
　　　　　　　　//public final static Integer TASK_STATUS_FAILURE = 3;
　　　　　　　　//public final static Integer TASK_STATUS_FINISH = 4;
　　　　　　　　//public final static Integer TASK_STATUS_PART = 6;

            Integer taskStatus = SoConstant.TASK_STATUS_RUNNING;//任务状态
            soExportTaskScheduleService.updateFieldsByParamWithTx(new UF("taskStatus", taskStatus).eq("id", taskId));

            List<? extends SoVO> listVO;
            ExcelExportConfig cfg = new ExcelExportConfig();
            cfg.setAutoSizeColumn(false);
            String fileName = "";

            if (flag == 0) {
                listVO = service.exportListSoByPage(args);
                mappingExportFields(cfg, SessionHelper.getPlatformId());
                fileName = "订单列表.xlsx";
            } else {
                listVO = service.getSoOrderLineDetail(args);
                mappingExportOrderLineFields(cfg);
                fileName = "订单行列表.xlsx";
            }

            // 返回url
            String uploadFile = null;
            try {
                // 返回url
                uploadFile = uploadService.uploadFile(fileName, ExcelUtils.getDoExportInputStream(fileName, listVO, cfg));
                taskStatus = SoConstant.TASK_STATUS_FINISH;
            } catch (Exception ex) {
                logger.info("上传文件失败！", ex);
                taskStatus = SoConstant.TASK_STATUS_FAILURE;
            }
            UpdateFieldParam uf = new UF("task_status", taskStatus, "file_path", uploadFile)
                    .update("finish_time", new Date())
                    .update("totalCount",listVO.size());
            if(SoConstant.TASK_STATUS_FINISH.equals(taskStatus)){
                uf.update("successCount",listVO.size());
                uf.update("failedCount",0);
            }else if(SoConstant.TASK_STATUS_FINISH.equals(taskStatus)){
                uf.update("successCount",0);
                uf.update("failedCount",listVO.size());
                uf.update("failedReason","保存文件失败");
            }

            soExportTaskScheduleService.updateFieldsByParamWithTx(uf.eq("id", taskId));
        } finally {
            SessionHelper.clearSessionCopy();
        }
    }

ExcelUtils工具类

public class ExcelUtils {
    
    private ExcelUtils() {}
    
    public static int getMaxExportMaxRows() {
        return OmsEnv.getInstance().getMaxExportRows();
    }
    
    public static int getMaxImportRows() {
        return OmsEnv.getInstance().getMaxImportRows();
    }
    
    public static void doExport(String name, List<?> list, ExcelExportConfig cfg, HttpServletRequest req,
                            HttpServletResponse res) throws Exception {
        ExcelExporter exporter = Excels.newExcelExporterForBigData(getMaxExportMaxRows());

        OutputStream os = res.getOutputStream();
        try {
            WebUtils.setXlsxAttachmentHeader(req, res, name + ".xlsx");

            exporter.setSheetData(list, cfg).export(os);
        } finally {
            Closer.close(os);
        }
    }

    /**
     * @param <T>
     *            要导出的POJO类型，在其字段上使用@{@link ExcelField}注解
     * @param cfg
     *            可以为null表示用默认配置，也可以传用户自定义的部分（仍然会自动增加列的映射关系）
     * @param nameConvertor
     *            nameConvertor @{@link ExcelField}注解中列名转换，常用于多语言，null为不转换
     */
    public static <T> void doExport(String name, List<? extends T> list, Class<T> klass, ExcelExportConfig cfg,
                                ColumnNameConvertor nameConvertor, HttpServletRequest req, HttpServletResponse res) throws Exception {
        try (OutputStream os = res.getOutputStream()) {
            WebUtils.setXlsxAttachmentHeader(req, res, name + ".xlsx");
            ExcelUtil.doExport(list, klass, os, cfg, nameConvertor);
        }
    }

    /**
     * @param <T>
     *            要导入的POJO类型，在其字段上使用@{@link ExcelField}注解
     * @param cfg
     *            可以为null表示用默认配置，也可以传用户自定义的部分（仍然会自动增加列的映射关系）
     * @param nameConvertor
     *            nameConvertor @{@link ExcelField}注解中列名转换，常用于多语言，null为不转换
     */
    public static <T> List<T> doImport(Class<T> resultClass, ExcelParseConfig cfg, ColumnNameConvertor nameConvertor,
                                   MultipartHttpServletRequest request) throws Exception {
        MultipartFile multipartFile = request.getFile("fileData");

        try (InputStream stream = multipartFile.getInputStream()) {
            return doImport(stream, multipartFile.getOriginalFilename(), resultClass, cfg, nameConvertor);
        }
    }
    
    public static <T> List<T> doImport(InputStream stream, String filename, Class<T> resultClass, 
        ExcelParseConfig cfg, ColumnNameConvertor nameConvertor, Integer maxRow)
            throws Exception {
        ExcelParser parser = Excels.newExcelParser(stream, ExcelFileType.getFileType(filename));

        int dataRows = getDataRows(parser, cfg);
        if (dataRows > maxRow) {
            throw new I18nException(CodeEnum.IMPORT_MAX_ROW);
        }
        
        return doImport(parser, resultClass, cfg, nameConvertor);
    }

    public static <T> List<T> doImport(InputStream stream, String filename, Class<T> resultClass, ExcelParseConfig cfg, ColumnNameConvertor nameConvertor)
            throws Exception {
        ExcelParser parser = Excels.newExcelParser(stream, ExcelFileType.getFileType(filename));

        int dataRows = getDataRows(parser, cfg);
        if (dataRows > getMaxImportRows()) {
            throw new I18nException(CodeEnum.IMPORT_MAX_ROW);
        }
        
        return doImport(parser, resultClass, cfg, nameConvertor);
    }
    
    @SuppressWarnings("unchecked")
    public static <T> List<T> doImport(ExcelParser parser, Class<T> resultClass, ExcelParseConfig cfg, ColumnNameConvertor nameConvertor)
            throws Exception {
        try {
            if (! cfg.isAutoMapColName() && cfg.getColNameMap().isEmpty()) {
                cfg = ExcelUtil.makeImportConfig(resultClass, cfg, nameConvertor);
                cfg.setSuppressTypeCastError(true);
            }
            
            if (resultClass.isArray()) {
                return (List<T>) parser.parse2Array(cfg);
            } else if (Map.class.isAssignableFrom(resultClass)) {
                return (List<T>) parser.parse2Map(cfg);
            }
            return (List<T>) parser.parse2Bean(cfg, resultClass);
        } catch (ExcelParseException epe) {
            String errmsg = "格式错误，请检查。";
            Throwable cause = ExceptionUtils.getOriginal(epe);
            if (cause != null && cause instanceof VisibleException) {
                errmsg = ((VisibleException) cause).getMessage();
            }
            String columnTitle = StringUtils.isEmpty(epe.getColName()) ? String.format("第 %d", epe.getColumn() + 1)
                    : cfg.getColNameMap().entrySet().stream()
                    .collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey, (a, b) -> b))
                    .get(epe.getColName());
            throw new VisibleException(String.format("第 %d 行，%s列%s", epe.getRow() + 1, columnTitle, errmsg), epe);
        }
    }
    
    public static int getDataRows(ExcelParser parser, ExcelParseConfig cfg) {
        int dataRows = 0;
        for (Integer index : cfg.getSheetIndexes()) {
            dataRows += parser.getWorkbook().getSheetAt(index).getPhysicalNumberOfRows() - cfg.getDataStartRow();
        }
        return dataRows;
    }
    
    public static InputStream getDoExportInputStream(String name, List<?> list, ExcelExportConfig cfg) throws Exception  {
        // 大部分拷贝了doExport的内容
        // 方法是获取了workBook对象，将其输入到字节流中，再转为inputstream
        ExcelExporter exporter = Excels.newExcelExporterForBigData(getMaxExportMaxRows());
        
        //WebUtils.setXlsxAttachmentHeader(req, res, name + ".xlsx");
    
        exporter.setSheetData(list, cfg);
        ByteArrayOutputStream os = new ByteArrayOutputStream();
        Workbook workbook = exporter.getWorkbook();
        InputStream fs = null;
        try {
            workbook.write(os);
            byte[] byteArray = os.toByteArray();
            fs = new ByteArrayInputStream(byteArray);
        }finally {
            os.close();
        }
        
        return fs;
    }
    
    /**
     * 将Excel中每单元格的错误以逗号分隔，红色加粗字体写到每行的最后一列
     * @param parseConfig
     * @param excelErrColl
     * @param errColHeader
     * @param os
     * @throws IOException
     */
    public static void writeErrorExcel(ExcelParseConfig parseConfig, ExcelErrorCollector excelErrColl, String errColHeader, OutputStream os)
    throws IOException {
        if (! excelErrColl.hasAnyError()) {
            return;
        }

        Set<String> sheetNameSet = new HashSet<String>();
        
        CellStyle style = excelErrColl.getWorkbook().createCellStyle();
        Font font = excelErrColl.getWorkbook().createFont();
        font.setColor(HSSFFont.COLOR_RED);
        font.setBold(true);
        style.setFont(font);
        int titleRow = parseConfig.getTitleRow();
        
        for (ExcelRowError error : excelErrColl.getRowIterator()) {
            Row row = error.getRow();
            
            if (! sheetNameSet.contains(row.getSheet().getSheetName())) {
                sheetNameSet.add(row.getSheet().getSheetName());
                
                Row firstRow = row.getSheet().getRow(titleRow);
                int col = firstRow.getLastCellNum() + 1;
                row.getSheet().autoSizeColumn(col);
                Cell cell = firstRow.createCell(col, CellType.STRING);
                cell.setCellValue(errColHeader);
                cell.setCellStyle(style);
            }
            
            int col = row.getLastCellNum() + 1;
            Cell cell = row.createCell(col, CellType.STRING);
            cell.setCellValue(ArrayUtils.join(error.getErrors(), ","));
            cell.setCellStyle(style);
        }
        
        excelErrColl.getWorkbook().write(os);
    }

    public static void writeErrorExcel(Workbook workbook, ExcelParseConfig parseConfig, 
            List<LineMsg> errors, OutputStream os) throws IOException {
        CellStyle style = workbook.createCellStyle();
        Font font = workbook.createFont();
        font.setColor(HSSFFont.COLOR_RED);
        font.setBold(true);
        style.setFont(font);
        int titleRow = parseConfig.getTitleRow();
        
        Map<Pair<Integer, Integer>, Cell> cellCache = Maps.newHashMap();
        Sheet sheet = workbook.getSheetAt(0);
        Row firstRow = sheet.getRow(titleRow);
        int col = firstRow.getLastCellNum() + 1;
        sheet.autoSizeColumn(col);
        
        for (LineMsg line : errors) {
            Row row = sheet.getRow(line.getLineNo());
            
            Cell cell = cellCache.get(Pair.of(line.getLineNo(), col));
            if (cell == null) {
                cell = row.createCell(col, CellType.STRING);    
                cell.setCellStyle(style);
                cell.setCellValue(line.getMsg());
                cellCache.put(Pair.of(line.getLineNo(), col), cell);
            } else {
                cell.setCellValue(cell.getStringCellValue() + ";" + line.getMsg());
            }
        }
        
        workbook.write(os);
    }
}

ExcelExportConfig类

public class ExcelExportConfig {
    int sheetIndex;
    String sheetName;
    
    boolean autoMapColName = false;
    
    boolean writeHeader = true;
    int startRow;
    int maxDataRows;
    
    String dateFormat = "yyyy-MM-dd HH:mm:ss";
    
    /** field -> column */
    Map<String, String> colNameMap = new LinkedHashMap<String, String>();
    /** field -> columnIdx */
    Map<String, Integer> colIndexMap = new HashMap<String, Integer>();
    
    String[] ignoreProperties;
    
    ExcelExportSheetCallback sheetCallback;
    ExcelExportRowCallback rowCallback;
    ExcelExportCellCallback cellCallback;

    boolean defaultStringType = true;
    Map<String, Class<?>> colTypeMap = new HashMap<String, Class<?>>();
    Map<String, BiConsumer<CellStyle, Cell>> colStyleMap = new HashMap<>();
    Map<String, Integer> colWidthMap = new HashMap<>();
    
    int sheetRandomAccessWindowSize;
    
    boolean autoSizeColumn = true;
    boolean autoBreak = false;
    /** 列表分隔符 */
    String delimiter = ",";
    
    public ExcelExportConfig setSheetCallback(ExcelExportSheetCallback sheetCallback) {
        this.sheetCallback = sheetCallback;
        return this;
    }

    public ExcelExportConfig setRowCallback(ExcelExportRowCallback rowCallback) {
        this.rowCallback = rowCallback;
        return this;
    }

    public ExcelExportConfig setCellCallback(ExcelExportCellCallback cellCallback) {
        this.cellCallback = cellCallback;
        return this;
    }
    
    public ExcelExportConfig withSheetRandomAccessWindowSize(int sheetRandomAccessWindowSize) {
        this.sheetRandomAccessWindowSize = sheetRandomAccessWindowSize;
        return this;
    }
    /**
     * 设置Cell的类型是否为文本类型（默认为true）
     * @param defaultStringType
     * @return
     */
    public ExcelExportConfig setDefaultStringType(boolean defaultStringType) {
        this.defaultStringType = defaultStringType;
        return this;
    }
    
    public ExcelExportConfig mapColType(String col, Class<?> valueType) {
        colTypeMap.put(col, valueType);
        return this;
    }
    public ExcelExportConfig mapColStyle(String col, BiConsumer<CellStyle, Cell> style) {
        colStyleMap.put(col, style);
        return this;
    }
    public ExcelExportConfig mapColWidth(String col, Integer width) {
        colWidthMap.put(col, width);
        return this;
    }
    /**
     * 设置数据从第几行开始，包括Header(如果有)，如果是自定义Header，则需要设置此值
     * @param startRow
     * @return
     */
    public ExcelExportConfig setStartRow(int startRow) {
        this.startRow = startRow;
        return this;
    }
    public ExcelExportConfig setMaxDataRows(int maxDataRows) {
        this.maxDataRows = maxDataRows;
        return this;
    }
    public ExcelExportConfig setSheetIndex(int sheetIndex) {
        this.sheetIndex = sheetIndex;
        return this;
    }
    public ExcelExportConfig setSheetName(String sheetName) {
        this.sheetName = sheetName;
        return this;
    }
    /**
     * 是否写数据头（默认为true）
     * @param writeHeader
     * @return
     */
    public ExcelExportConfig setWriteHeader(boolean writeHeader) {
        this.writeHeader = writeHeader;
        return this;
    }

    public ExcelExportConfig setDateFormat(String dateFormat) {
        this.dateFormat = dateFormat;
        return this;
    }

    public ExcelExportConfig setIgnoreProperties(String... ignoreProperties) {
        this.ignoreProperties = ignoreProperties;
        return this;
    }
    /**
     * 设置自动映射列名（默认为true）
     * @param autoMapColName
     * @return
     */
    public ExcelExportConfig setAutoMapColName(boolean autoMapColName) {
        this.autoMapColName = autoMapColName;
        return this;
    }
    /**
     * 设置是否自动调整列宽（默认为true）
     * @param autoSizeColumn
     * @return
     */
    public ExcelExportConfig setAutoSizeColumn(boolean autoSizeColumn) {
        this.autoSizeColumn = autoSizeColumn;
        return this;
    }
    /**
     * 设置是否自动换行（默认为false）
     * @param autoBreak
     * @return
     */
    public ExcelExportConfig setAutoBreak(boolean autoBreak) {
        this.autoBreak = autoBreak;
        return this;
    }
    /**
     * 设定分隔符（默认为半角逗号）
     * @param delimiter
     * @return
     */
    public ExcelExportConfig setDelimiter(String delimiter){
        this.delimiter = delimiter;
        return this;
    }
    
    /**
     * 映射Excel中列的标题，默认标题为key
     * @param key
     * @param column
     * @return
     */
    public ExcelExportConfig mapColName(String key, String column) {
        colNameMap.put(key, column);
        colIndexMap.put(key, colIndexMap.size());
        return this;
    }
    /**
     * 映射Excel中列的索引，默认按mapColName(..)的添加顺序，如果未调用mapColIndex(..)，则可能为乱序
     * @param key
     * @param column
     * @return
     */
    public ExcelExportConfig mapColIndex(String key, int column) {
        if (column < colIndexMap.size()) { 
            // 如果column小于colIndexMap的大小，则一定会与现有的列顺序冲突，需要作移位处理
            String oldKey = null;
            for (String k : colIndexMap.keySet()) {
                if (colIndexMap.get(k) == column) {
                    oldKey = k;
                    break;
                }
            }
            colIndexMap.remove(oldKey);
            colIndexMap.put(oldKey, colIndexMap.size());
        }
        colIndexMap.put(key, column);
        return this;
    }
    
    /*------------------------------------------------------------------------------*/

    public String getSheetName() {
        return sheetName;
    }

    public boolean isWriteHeader() {
        return writeHeader;
    }

    public int getStartRow() {
        return startRow;
    }
    
    public int getMaxDataRows() {
        return maxDataRows;
    }

    public String getDateFormat() {
        return dateFormat;
    }

    public Map<String, String> getColNameMap() {
        return colNameMap;
    }

    public Map<String, Integer> getColIndexMap() {
        return colIndexMap;
    }

    public boolean isAutoMapColName() {
        return autoMapColName;
    }

    public String[] getIgnoreProperties() {
        return ignoreProperties;
    }

    public ExcelExportSheetCallback getSheetCallback() {
        return sheetCallback;
    }

    public ExcelExportRowCallback getRowCallback() {
        return rowCallback;
    }

    public ExcelExportCellCallback getCellCallback() {
        return cellCallback;
    }

    public boolean isDefaultStringType() {
        return defaultStringType;
    }

    public Map<String, Class<?>> getColTypeMap() {
        return colTypeMap;
    }
    public Map<String, BiConsumer<CellStyle, Cell>> getColStyleMap() {
        return colStyleMap;
    }
    public Map<String, Integer> getColWidthMap() {
        return colWidthMap;
    }

    public String getDelimiter() {
        return delimiter;
    }
    
}