elink-star
/
ShibeiWorkPlatform
12
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

修改： 

1.居民信息导入：将数据分多批次导入，减少内存占用
dev
wangxianzhang 4 years ago
parent
ad96489c99
commit
4922ee5707
2 changed files with 146 additions and 83 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 144
      epmet-user/epmet-user-server/src/main/java/com/epmet/excel/handler/DynamicEasyExcelListener.java
  2. 85
      epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/IcResiUserImportServiceImpl.java

144
epmet-user/epmet-user-server/src/main/java/com/epmet/excel/handler/DynamicEasyExcelListener.java
View File

@ -2,12 +2,16 @@ package com.epmet.excel.handler;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener;
import com.alibaba.fastjson.JSON;
import com.epmet.dto.result.FormItemResult;
import com.epmet.service.impl.IcResiUserImportServiceImpl;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* 创建一个监听器
@ -15,6 +19,64 @@ import java.util.Map;
@Slf4j
public class DynamicEasyExcelListener extends AnalysisEventListener<Map<Integer, String>> {
/**
* 一次性导入的条数
*/
private static final Integer ONCE_BATCH_IMPORT_ITEMS_SIZE = 2000;
/**
* 客户id
*/
private String customerId;
/**
* 当前操作人的AgencyId
*/
private String currUserAgencyId;
/**
* 当前用户ID
*/
private String currentUserId;
/**
* 当前用户agencyId
*/
private String currUserAgencyPids;
/**
* 数据库表名
*/
private String tableName;
private IcResiUserImportServiceImpl icResiUserImportService;
/**
* item列表
*/
private List<FormItemResult> formItemList;
/**
* 是否是基础信息表
*/
private Boolean isPrimary;
/**
* 有几行是表头
*/
private Integer headRowNumber;
/**
* key:itemId
* value:ColumnWrapper 列封装信息，列基础信息和列值
*/
private Map<String, IcResiUserImportServiceImpl.ColumnWrapper> itemIdAndColumnWrapper;
/**
* 被丢弃的header，原因：checkbox的情况下，选项在表格中是多列，但是在item中不存在
*/
private Map<Integer, String> abandonedHeaders;
/**
* 表头数据（存储所有的表头数据）
*/
@ -26,6 +88,23 @@ public class DynamicEasyExcelListener extends AnalysisEventListener<Map<Integer,
private List<Map<Integer, String>> dataList = new ArrayList<>();
// Map<Integer, String> dataList = new HashMap<>();
public DynamicEasyExcelListener(IcResiUserImportServiceImpl icResiUserImportService, String customerId,
String currentUserId, String currUserAgencyId,
String currUserAgencyPids, Boolean isPrimary, String tableName,
List<FormItemResult> formItemList, Integer headRowNumber) {
this.customerId = customerId;
this.icResiUserImportService = icResiUserImportService;
this.formItemList = formItemList;
this.isPrimary = isPrimary;
this.currentUserId = currentUserId;
this.currUserAgencyId = currUserAgencyId;
this.currUserAgencyPids = currUserAgencyPids;
this.tableName = tableName;
this.headRowNumber = headRowNumber;
}
/**
* 这里会一行行的返回头
*
@ -34,9 +113,35 @@ public class DynamicEasyExcelListener extends AnalysisEventListener<Map<Integer,
*/
@Override
public void invokeHeadMap(Map<Integer, String> headMap, AnalysisContext context) {
//log.info("解析到一条头数据:{}", JSON.toJSONString(headMap));
//存储全部表头数据
headList.add(headMap);
if (headList.size() < headRowNumber) {
return;
}
// 合并多级表头到一个list中，key为列序号
Map<Integer, List<String>> headers = icResiUserImportService.mergeMultiLevelHeadLabels(headList);
// 清洗表头数据,通过items剔除，并且得到options
abandonedHeaders = icResiUserImportService.removeAndGetOptionsFromHeaders(headers, formItemList);
// 交换表头信息，以label连起来的string作为key，列号的列表作为value
HashMap<String, List<Integer>> combinedHeaders = icResiUserImportService.exchangeKeyAndValueOfHeaders(headers);
// 得到客户配置item数据。<"兴趣爱好:兴趣特长", item对象>
Map<String, FormItemResult> customizedLabelCompbinedItemsMap = formItemList.stream().collect(
Collectors.toMap(formItem -> {
String groupLabel = formItem.getGroupLabel();
String label = formItem.getLabel();
if (StringUtils.isNotBlank(groupLabel)) {
return groupLabel.concat(":").concat(label);
} else {
return label;
}
}, formItem -> formItem)
);
itemIdAndColumnWrapper = icResiUserImportService.convertExcelHeaders2DBColumnWrappers(customizedLabelCompbinedItemsMap, combinedHeaders);
}
/**
@ -48,26 +153,37 @@ public class DynamicEasyExcelListener extends AnalysisEventListener<Map<Integer,
*/
@Override
public void invoke(Map<Integer, String> data, AnalysisContext context) {
//log.info("解析到一条数据:{}", JSON.toJSONString(data));
// 每2000条数据处理一次
dataList.add(data);
// 达到了批量导入的阈值，执行一次持久化
if (dataList.size() >= ONCE_BATCH_IMPORT_ITEMS_SIZE) {
execPersistant();
}
}
/**
* 所有数据解析完成了 都会来调用
*
* 所有数据解析完成了会调用
* 此处也要判断，然后执行持久化，因为最后一部分数据，达不到批量阈值，不能漏掉，在最后结束的时候给他执行进去
* @param context
*/
@Override
public void doAfterAllAnalysed(AnalysisContext context) {
// 这里也要保存数据，确保最后遗留的数据也存储到数据库
//log.info("所有数据解析完成！");
}
public List<Map<Integer, String>> getHeadList() {
return headList;
if (dataList.size() != 0) {
execPersistant();
}
}
public List<Map<Integer, String>> getDataList() {
return dataList;
/**
* 执行持久化
*/
private void execPersistant() {
// 持久化
if (isPrimary) {
icResiUserImportService.persistIcResiBaseInfo(itemIdAndColumnWrapper, dataList, currUserAgencyId, abandonedHeaders, currUserAgencyPids, currentUserId, tableName);
} else {
icResiUserImportService.persistIcResiExtraInfo(itemIdAndColumnWrapper, dataList, currUserAgencyId, abandonedHeaders, currentUserId, tableName, customerId);
}
dataList.clear();
}
}

85
epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/IcResiUserImportServiceImpl.java
View File

@ -232,7 +232,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res
// 上传主表信息
importIcResiBaseInfoFromExcel(formItemList,excelPathName, IcResiUserTableEnum.IC_RESI_USER.getSheetNo(), IcResiUserTableEnum.IC_RESI_USER.getHeadRowNo(),
currUserAgencyId, agencyInfo.getPids(), loginUserId, IcResiUserTableEnum.IC_RESI_USER.getTableName());
currUserAgencyId, agencyInfo.getPids(), loginUserId, IcResiUserTableEnum.IC_RESI_USER.getTableName(), customerId);
// 上传附表信息
for (IcResiUserTableEnum sheet : IcResiUserTableEnum.values()) {
@ -320,43 +320,12 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res
* @param currentUserId
* @return
*/
private Object importIcResiBaseInfoFromExcel(List<FormItemResult> formItemList, String excelPathName, int sheetNo, int headRowNumber, String currUserAgencyId, String currUserAgencyPids, String currentUserId,
String tableName) {
DynamicEasyExcelListener readListener = new DynamicEasyExcelListener();
//EasyExcelFactory.read(new File(excelPathName)).registerReadListener(readListener).headRowNumber(headRowNumber).sheet(sheetNo).doRead();
EasyExcel.read(new File(excelPathName)).registerReadListener(readListener).headRowNumber(headRowNumber).sheet(sheetNo).doRead();
List<Map<Integer, String>> headList = readListener.getHeadList();
List<Map<Integer, String>> dataList = readListener.getDataList();
// 合并多级表头到一个list中，key为列序号
Map<Integer, List<String>> headers = mergeMultiLevelHeadLabels(headList);
// 清洗表头数据,通过items剔除，并且得到options
Map<Integer, String> abandonedHeaders = removeAndGetOptionsFromHeaders(headers, formItemList);
// 交换表头信息，以label连起来的string作为key，列号的列表作为value
HashMap<String, List<Integer>> combinedHeaders = exchangeKeyAndValueOfHeaders(headers);
// 得到客户配置item数据。<"兴趣爱好:兴趣特长", item对象>
Map<String, FormItemResult> customizedLabelCompbinedItemsMap = formItemList.stream().collect(
Collectors.toMap(formItem -> {
String groupLabel = formItem.getGroupLabel();
String label = formItem.getLabel();
if (StringUtils.isNotBlank(groupLabel)) {
return groupLabel.concat(":").concat(label);
} else {
return label;
}
}, formItem -> formItem)
);
Map<String, ColumnWrapper> itemIdAndColumnWrapper = convertExcelHeaders2DBColumnWrappers(customizedLabelCompbinedItemsMap, combinedHeaders);
// 持久化
persistIcResiBaseInfo(itemIdAndColumnWrapper, dataList, currUserAgencyId, abandonedHeaders, currUserAgencyPids, currentUserId, tableName);
private void importIcResiBaseInfoFromExcel(List<FormItemResult> formItemList, String excelPathName, int sheetNo, int headRowNumber, String currUserAgencyId, String currUserAgencyPids, String currentUserId,
String tableName, String customerId) {
return headers;
DynamicEasyExcelListener readListener = new DynamicEasyExcelListener(this, customerId, currentUserId, currUserAgencyId, currUserAgencyPids,
true, tableName, formItemList, headRowNumber);
EasyExcel.read(new File(excelPathName)).registerReadListener(readListener).headRowNumber(headRowNumber).sheet(sheetNo).doRead();
}
/**
@ -371,34 +340,12 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res
* @param targetTableName 要插入哪一个表
* @return
*/
private Object importIcResiExtraInfoFromExcel(List<FormItemResult> formItemList, String excelPathName, int sheetNo, int headRowNumber, String currUserAgencyId, String currentUserId,
private void importIcResiExtraInfoFromExcel(List<FormItemResult> formItemList, String excelPathName, int sheetNo, int headRowNumber, String currUserAgencyId, String currentUserId,
String targetTableName, String customerId) {
DynamicEasyExcelListener readListener = new DynamicEasyExcelListener();
EasyExcelFactory.read(new File(excelPathName)).registerReadListener(readListener).headRowNumber(headRowNumber).sheet(sheetNo).doRead();
List<Map<Integer, String>> headList = readListener.getHeadList();
List<Map<Integer, String>> dataList = readListener.getDataList();
Map<Integer, List<String>> headers = mergeMultiLevelHeadLabels(headList);
Map<Integer, String> abandonedHeaders = removeAndGetOptionsFromHeaders(headers, formItemList);
HashMap<String, List<Integer>> combinedHeaders = exchangeKeyAndValueOfHeaders(headers);
Map<String, FormItemResult> formItemMap = formItemList.stream().collect(
Collectors.toMap(formItem -> {
String groupLabel = formItem.getGroupLabel();
String label = formItem.getLabel();
if (StringUtils.isNotBlank(groupLabel)) {
return groupLabel.concat(":").concat(label);
} else {
return label;
}
}, formItem -> formItem)
);
Map<String, ColumnWrapper> headerColumnWrapper = convertExcelHeaders2DBColumnWrappers(formItemMap, combinedHeaders);
persistIcResiExtraInfo(headerColumnWrapper, dataList, currUserAgencyId, abandonedHeaders, currentUserId, targetTableName, customerId);
return headerColumnWrapper;
DynamicEasyExcelListener readListener = new DynamicEasyExcelListener(this, customerId, currentUserId, currUserAgencyId, null,
false, targetTableName, formItemList, headRowNumber);
EasyExcelFactory.read(new File(excelPathName)).registerReadListener(readListener).headRowNumber(headRowNumber).sheet(sheetNo).doRead();
}
/**
@ -408,7 +355,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res
* @param currUserAgencyId 当前用户的组织id
* @param checkBoxOptionColumnIdxAndLabel 复选框的列号&label中文
*/
private void persistIcResiBaseInfo(Map<String, ColumnWrapper> itemIdAndColumnWrapper, List<Map<Integer, String>> dataRows,
public void persistIcResiBaseInfo(Map<String, ColumnWrapper> itemIdAndColumnWrapper, List<Map<Integer, String>> dataRows,
String currUserAgencyId, Map<Integer, String> checkBoxOptionColumnIdxAndLabel,
String currUserAgencyPids, String currentUserId, String tableName) {
@ -517,7 +464,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res
* @param checkBoxOptionColumnIdxAndLabel 复选框的列号&label中文
* @param targetTableName 要插入到哪一个表
*/
private void persistIcResiExtraInfo(Map<String, ColumnWrapper> headerColumnWrapper, List<Map<Integer, String>> dataRows,
public void persistIcResiExtraInfo(Map<String, ColumnWrapper> headerColumnWrapper, List<Map<Integer, String>> dataRows,
String currUserAgencyId, Map<Integer, String> checkBoxOptionColumnIdxAndLabel,
String currentUserId, String targetTableName, String customerId) {
@ -709,7 +656,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res
* @author wxz
* @date 2021.10.28 21:27:18
*/
private HashMap<String, List<Integer>> exchangeKeyAndValueOfHeaders(Map<Integer, List<String>> headers) {
public HashMap<String, List<Integer>> exchangeKeyAndValueOfHeaders(Map<Integer, List<String>> headers) {
HashMap<String, List<Integer>> itemAndColIndexs = new LinkedHashMap<>();
headers.forEach((k, v) -> {
@ -734,7 +681,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res
* @author wxz
* @date 2021.10.28 21:07:12
*/
private Map<Integer, String> removeAndGetOptionsFromHeaders(Map<Integer, List<String>> headers, List<FormItemResult> items) {
public Map<Integer, String> removeAndGetOptionsFromHeaders(Map<Integer, List<String>> headers, List<FormItemResult> items) {
List<String> itemLabels = items.stream().map(i -> i.getLabel()).collect(Collectors.toList());
Map<Integer, String> abandonedOptions = new HashMap<>();
for (Map.Entry<Integer, List<String>> entry:headers.entrySet()) {
@ -765,7 +712,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res
* value：列号组成的列表(例如[1,2,3])
* @return key:itemId；value：columnWrapper
*/
private Map<String, ColumnWrapper> convertExcelHeaders2DBColumnWrappers(Map<String, FormItemResult> customizedLabelCompbinedItemsMap, Map<String, List<Integer>> combinedHeaders) {
public Map<String, ColumnWrapper> convertExcelHeaders2DBColumnWrappers(Map<String, FormItemResult> customizedLabelCompbinedItemsMap, Map<String, List<Integer>> combinedHeaders) {
// HashMap<String, List<ColumnWrapper>> tables = new HashMap<>();
Map<String, ColumnWrapper> columns = new LinkedHashMap<>(combinedHeaders.size());
@ -835,7 +782,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res
* @author wxz
* @date 2021.10.27 16:17:34
*/
private Map<Integer, List<String>> mergeMultiLevelHeadLabels(List<Map<Integer, String>> headList) {
public Map<Integer, List<String>> mergeMultiLevelHeadLabels(List<Map<Integer, String>> headList) {
Map<Integer, String> lastNotNullHeads = new LinkedHashMap<>();

Write Preview
Loading…
Cancel
Save