diff --git a/epmet-user/epmet-user-server/src/main/java/com/epmet/excel/handler/DynamicEasyExcelListener.java b/epmet-user/epmet-user-server/src/main/java/com/epmet/excel/handler/DynamicEasyExcelListener.java index 4c12594a45..bf65c3bbf2 100644 --- a/epmet-user/epmet-user-server/src/main/java/com/epmet/excel/handler/DynamicEasyExcelListener.java +++ b/epmet-user/epmet-user-server/src/main/java/com/epmet/excel/handler/DynamicEasyExcelListener.java @@ -2,12 +2,16 @@ package com.epmet.excel.handler; import com.alibaba.excel.context.AnalysisContext; import com.alibaba.excel.event.AnalysisEventListener; -import com.alibaba.fastjson.JSON; +import com.epmet.dto.result.FormItemResult; +import com.epmet.service.impl.IcResiUserImportServiceImpl; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * 创建一个监听器 @@ -15,6 +19,64 @@ import java.util.Map; @Slf4j public class DynamicEasyExcelListener extends AnalysisEventListener> { + /** + * 一次性导入的条数 + */ + private static final Integer ONCE_BATCH_IMPORT_ITEMS_SIZE = 2000; + + /** + * 客户id + */ + private String customerId; + + /** + * 当前操作人的AgencyId + */ + private String currUserAgencyId; + + /** + * 当前用户ID + */ + private String currentUserId; + + /** + * 当前用户agencyId + */ + private String currUserAgencyPids; + + /** + * 数据库表名 + */ + private String tableName; + + private IcResiUserImportServiceImpl icResiUserImportService; + + /** + * item列表 + */ + private List formItemList; + + /** + * 是否是基础信息表 + */ + private Boolean isPrimary; + + /** + * 有几行是表头 + */ + private Integer headRowNumber; + + /** + * key:itemId + * value:ColumnWrapper 列封装信息，列基础信息和列值 + */ + private Map itemIdAndColumnWrapper; + + /** + * 被丢弃的header，原因：checkbox的情况下，选项在表格中是多列，但是在item中不存在 + */ + private Map abandonedHeaders; + /** * 表头数据（存储所有的表头数据） */ @@ -26,6 +88,23 @@ public class DynamicEasyExcelListener extends AnalysisEventListener> dataList = new ArrayList<>(); // Map dataList = new HashMap<>(); + + public DynamicEasyExcelListener(IcResiUserImportServiceImpl icResiUserImportService, String customerId, + String currentUserId, String currUserAgencyId, + String currUserAgencyPids, Boolean isPrimary, String tableName, + List formItemList, Integer headRowNumber) { + + this.customerId = customerId; + this.icResiUserImportService = icResiUserImportService; + this.formItemList = formItemList; + this.isPrimary = isPrimary; + this.currentUserId = currentUserId; + this.currUserAgencyId = currUserAgencyId; + this.currUserAgencyPids = currUserAgencyPids; + this.tableName = tableName; + this.headRowNumber = headRowNumber; + } + /** * 这里会一行行的返回头 * @@ -34,9 +113,35 @@ public class DynamicEasyExcelListener extends AnalysisEventListener headMap, AnalysisContext context) { - //log.info("解析到一条头数据:{}", JSON.toJSONString(headMap)); - //存储全部表头数据 headList.add(headMap); + + if (headList.size() < headRowNumber) { + return; + } + + // 合并多级表头到一个list中，key为列序号 + Map> headers = icResiUserImportService.mergeMultiLevelHeadLabels(headList); + + // 清洗表头数据,通过items剔除，并且得到options + abandonedHeaders = icResiUserImportService.removeAndGetOptionsFromHeaders(headers, formItemList); + + // 交换表头信息，以label连起来的string作为key，列号的列表作为value + HashMap> combinedHeaders = icResiUserImportService.exchangeKeyAndValueOfHeaders(headers); + + // 得到客户配置item数据。<"兴趣爱好:兴趣特长", item对象> + Map customizedLabelCompbinedItemsMap = formItemList.stream().collect( + Collectors.toMap(formItem -> { + String groupLabel = formItem.getGroupLabel(); + String label = formItem.getLabel(); + if (StringUtils.isNotBlank(groupLabel)) { + return groupLabel.concat(":").concat(label); + } else { + return label; + } + }, formItem -> formItem) + ); + + itemIdAndColumnWrapper = icResiUserImportService.convertExcelHeaders2DBColumnWrappers(customizedLabelCompbinedItemsMap, combinedHeaders); } /** @@ -48,26 +153,37 @@ public class DynamicEasyExcelListener extends AnalysisEventListener data, AnalysisContext context) { - //log.info("解析到一条数据:{}", JSON.toJSONString(data)); + // 每2000条数据处理一次 dataList.add(data); + + // 达到了批量导入的阈值，执行一次持久化 + if (dataList.size() >= ONCE_BATCH_IMPORT_ITEMS_SIZE) { + execPersistant(); + } } /** - * 所有数据解析完成了 都会来调用 - * + * 所有数据解析完成了会调用 + * 此处也要判断，然后执行持久化，因为最后一部分数据，达不到批量阈值，不能漏掉，在最后结束的时候给他执行进去 * @param context */ @Override public void doAfterAllAnalysed(AnalysisContext context) { - // 这里也要保存数据，确保最后遗留的数据也存储到数据库 - //log.info("所有数据解析完成！"); - } - - public List> getHeadList() { - return headList; + if (dataList.size() != 0) { + execPersistant(); + } } - public List> getDataList() { - return dataList; + /** + * 执行持久化 + */ + private void execPersistant() { + // 持久化 + if (isPrimary) { + icResiUserImportService.persistIcResiBaseInfo(itemIdAndColumnWrapper, dataList, currUserAgencyId, abandonedHeaders, currUserAgencyPids, currentUserId, tableName); + } else { + icResiUserImportService.persistIcResiExtraInfo(itemIdAndColumnWrapper, dataList, currUserAgencyId, abandonedHeaders, currentUserId, tableName, customerId); + } + dataList.clear(); } } \ No newline at end of file diff --git a/epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/IcResiUserImportServiceImpl.java b/epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/IcResiUserImportServiceImpl.java index 71e0e0d9ca..d236c92004 100644 --- a/epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/IcResiUserImportServiceImpl.java +++ b/epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/IcResiUserImportServiceImpl.java @@ -232,7 +232,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res // 上传主表信息 importIcResiBaseInfoFromExcel(formItemList,excelPathName, IcResiUserTableEnum.IC_RESI_USER.getSheetNo(), IcResiUserTableEnum.IC_RESI_USER.getHeadRowNo(), - currUserAgencyId, agencyInfo.getPids(), loginUserId, IcResiUserTableEnum.IC_RESI_USER.getTableName()); + currUserAgencyId, agencyInfo.getPids(), loginUserId, IcResiUserTableEnum.IC_RESI_USER.getTableName(), customerId); // 上传附表信息 for (IcResiUserTableEnum sheet : IcResiUserTableEnum.values()) { @@ -320,43 +320,12 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res * @param currentUserId * @return */ - private Object importIcResiBaseInfoFromExcel(List formItemList, String excelPathName, int sheetNo, int headRowNumber, String currUserAgencyId, String currUserAgencyPids, String currentUserId, - String tableName) { - DynamicEasyExcelListener readListener = new DynamicEasyExcelListener(); - //EasyExcelFactory.read(new File(excelPathName)).registerReadListener(readListener).headRowNumber(headRowNumber).sheet(sheetNo).doRead(); - EasyExcel.read(new File(excelPathName)).registerReadListener(readListener).headRowNumber(headRowNumber).sheet(sheetNo).doRead(); - - List> headList = readListener.getHeadList(); - List> dataList = readListener.getDataList(); - - // 合并多级表头到一个list中，key为列序号 - Map> headers = mergeMultiLevelHeadLabels(headList); - - // 清洗表头数据,通过items剔除，并且得到options - Map abandonedHeaders = removeAndGetOptionsFromHeaders(headers, formItemList); - - // 交换表头信息，以label连起来的string作为key，列号的列表作为value - HashMap> combinedHeaders = exchangeKeyAndValueOfHeaders(headers); - - // 得到客户配置item数据。<"兴趣爱好:兴趣特长", item对象> - Map customizedLabelCompbinedItemsMap = formItemList.stream().collect( - Collectors.toMap(formItem -> { - String groupLabel = formItem.getGroupLabel(); - String label = formItem.getLabel(); - if (StringUtils.isNotBlank(groupLabel)) { - return groupLabel.concat(":").concat(label); - } else { - return label; - } - }, formItem -> formItem) - ); - - Map itemIdAndColumnWrapper = convertExcelHeaders2DBColumnWrappers(customizedLabelCompbinedItemsMap, combinedHeaders); - - // 持久化 - persistIcResiBaseInfo(itemIdAndColumnWrapper, dataList, currUserAgencyId, abandonedHeaders, currUserAgencyPids, currentUserId, tableName); + private void importIcResiBaseInfoFromExcel(List formItemList, String excelPathName, int sheetNo, int headRowNumber, String currUserAgencyId, String currUserAgencyPids, String currentUserId, + String tableName, String customerId) { - return headers; + DynamicEasyExcelListener readListener = new DynamicEasyExcelListener(this, customerId, currentUserId, currUserAgencyId, currUserAgencyPids, + true, tableName, formItemList, headRowNumber); + EasyExcel.read(new File(excelPathName)).registerReadListener(readListener).headRowNumber(headRowNumber).sheet(sheetNo).doRead(); } /** @@ -371,34 +340,12 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res * @param targetTableName 要插入哪一个表 * @return */ - private Object importIcResiExtraInfoFromExcel(List formItemList, String excelPathName, int sheetNo, int headRowNumber, String currUserAgencyId, String currentUserId, + private void importIcResiExtraInfoFromExcel(List formItemList, String excelPathName, int sheetNo, int headRowNumber, String currUserAgencyId, String currentUserId, String targetTableName, String customerId) { - DynamicEasyExcelListener readListener = new DynamicEasyExcelListener(); - EasyExcelFactory.read(new File(excelPathName)).registerReadListener(readListener).headRowNumber(headRowNumber).sheet(sheetNo).doRead(); - - List> headList = readListener.getHeadList(); - List> dataList = readListener.getDataList(); - - Map> headers = mergeMultiLevelHeadLabels(headList); - - Map abandonedHeaders = removeAndGetOptionsFromHeaders(headers, formItemList); - HashMap> combinedHeaders = exchangeKeyAndValueOfHeaders(headers); - - Map formItemMap = formItemList.stream().collect( - Collectors.toMap(formItem -> { - String groupLabel = formItem.getGroupLabel(); - String label = formItem.getLabel(); - if (StringUtils.isNotBlank(groupLabel)) { - return groupLabel.concat(":").concat(label); - } else { - return label; - } - }, formItem -> formItem) - ); - Map headerColumnWrapper = convertExcelHeaders2DBColumnWrappers(formItemMap, combinedHeaders); - persistIcResiExtraInfo(headerColumnWrapper, dataList, currUserAgencyId, abandonedHeaders, currentUserId, targetTableName, customerId); - return headerColumnWrapper; + DynamicEasyExcelListener readListener = new DynamicEasyExcelListener(this, customerId, currentUserId, currUserAgencyId, null, + false, targetTableName, formItemList, headRowNumber); + EasyExcelFactory.read(new File(excelPathName)).registerReadListener(readListener).headRowNumber(headRowNumber).sheet(sheetNo).doRead(); } /** @@ -408,7 +355,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res * @param currUserAgencyId 当前用户的组织id * @param checkBoxOptionColumnIdxAndLabel 复选框的列号&label中文 */ - private void persistIcResiBaseInfo(Map itemIdAndColumnWrapper, List> dataRows, + public void persistIcResiBaseInfo(Map itemIdAndColumnWrapper, List> dataRows, String currUserAgencyId, Map checkBoxOptionColumnIdxAndLabel, String currUserAgencyPids, String currentUserId, String tableName) { @@ -517,7 +464,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res * @param checkBoxOptionColumnIdxAndLabel 复选框的列号&label中文 * @param targetTableName 要插入到哪一个表 */ - private void persistIcResiExtraInfo(Map headerColumnWrapper, List> dataRows, + public void persistIcResiExtraInfo(Map headerColumnWrapper, List> dataRows, String currUserAgencyId, Map checkBoxOptionColumnIdxAndLabel, String currentUserId, String targetTableName, String customerId) { @@ -709,7 +656,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res * @author wxz * @date 2021.10.28 21:27:18 */ - private HashMap> exchangeKeyAndValueOfHeaders(Map> headers) { + public HashMap> exchangeKeyAndValueOfHeaders(Map> headers) { HashMap> itemAndColIndexs = new LinkedHashMap<>(); headers.forEach((k, v) -> { @@ -734,7 +681,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res * @author wxz * @date 2021.10.28 21:07:12 */ - private Map removeAndGetOptionsFromHeaders(Map> headers, List items) { + public Map removeAndGetOptionsFromHeaders(Map> headers, List items) { List itemLabels = items.stream().map(i -> i.getLabel()).collect(Collectors.toList()); Map abandonedOptions = new HashMap<>(); for (Map.Entry> entry:headers.entrySet()) { @@ -765,7 +712,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res * value：列号组成的列表(例如[1,2,3]) * @return key:itemId；value：columnWrapper */ - private Map convertExcelHeaders2DBColumnWrappers(Map customizedLabelCompbinedItemsMap, Map> combinedHeaders) { + public Map convertExcelHeaders2DBColumnWrappers(Map customizedLabelCompbinedItemsMap, Map> combinedHeaders) { // HashMap> tables = new HashMap<>(); Map columns = new LinkedHashMap<>(combinedHeaders.size()); @@ -835,7 +782,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res * @author wxz * @date 2021.10.27 16:17:34 */ - private Map> mergeMultiLevelHeadLabels(List> headList) { + public Map> mergeMultiLevelHeadLabels(List> headList) { Map lastNotNullHeads = new LinkedHashMap<>();