From 4922ee570734e7323a5c39e83a0971077e6f21d1 Mon Sep 17 00:00:00 2001 From: wangxianzhang Date: Sun, 13 Feb 2022 15:41:44 +0800 Subject: [PATCH 1/5] =?UTF-8?q?=E4=BF=AE=E6=94=B9=EF=BC=9A=201.=E5=B1=85?= =?UTF-8?q?=E6=B0=91=E4=BF=A1=E6=81=AF=E5=AF=BC=E5=85=A5=EF=BC=9A=E5=B0=86?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=88=86=E5=A4=9A=E6=89=B9=E6=AC=A1=E5=AF=BC?= =?UTF-8?q?=E5=85=A5=EF=BC=8C=E5=87=8F=E5=B0=91=E5=86=85=E5=AD=98=E5=8D=A0?= =?UTF-8?q?=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../handler/DynamicEasyExcelListener.java | 144 ++++++++++++++++-- .../impl/IcResiUserImportServiceImpl.java | 85 ++--------- 2 files changed, 146 insertions(+), 83 deletions(-) diff --git a/epmet-user/epmet-user-server/src/main/java/com/epmet/excel/handler/DynamicEasyExcelListener.java b/epmet-user/epmet-user-server/src/main/java/com/epmet/excel/handler/DynamicEasyExcelListener.java index 4c12594a45..bf65c3bbf2 100644 --- a/epmet-user/epmet-user-server/src/main/java/com/epmet/excel/handler/DynamicEasyExcelListener.java +++ b/epmet-user/epmet-user-server/src/main/java/com/epmet/excel/handler/DynamicEasyExcelListener.java @@ -2,12 +2,16 @@ package com.epmet.excel.handler; import com.alibaba.excel.context.AnalysisContext; import com.alibaba.excel.event.AnalysisEventListener; -import com.alibaba.fastjson.JSON; +import com.epmet.dto.result.FormItemResult; +import com.epmet.service.impl.IcResiUserImportServiceImpl; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * 创建一个监听器 @@ -15,6 +19,64 @@ import java.util.Map; @Slf4j public class DynamicEasyExcelListener extends AnalysisEventListener> { + /** + * 一次性导入的条数 + */ + private static final Integer ONCE_BATCH_IMPORT_ITEMS_SIZE = 2000; + + /** + * 客户id + */ + private String customerId; + + /** + * 当前操作人的AgencyId + */ + private String currUserAgencyId; + + /** + * 当前用户ID + */ + private String currentUserId; + + /** + * 当前用户agencyId + */ + private String currUserAgencyPids; + + /** + * 数据库表名 + */ + private String tableName; + + private IcResiUserImportServiceImpl icResiUserImportService; + + /** + * item列表 + */ + private List formItemList; + + /** + * 是否是基础信息表 + */ + private Boolean isPrimary; + + /** + * 有几行是表头 + */ + private Integer headRowNumber; + + /** + * key:itemId + * value:ColumnWrapper 列封装信息，列基础信息和列值 + */ + private Map itemIdAndColumnWrapper; + + /** + * 被丢弃的header，原因：checkbox的情况下，选项在表格中是多列，但是在item中不存在 + */ + private Map abandonedHeaders; + /** * 表头数据（存储所有的表头数据） */ @@ -26,6 +88,23 @@ public class DynamicEasyExcelListener extends AnalysisEventListener> dataList = new ArrayList<>(); // Map dataList = new HashMap<>(); + + public DynamicEasyExcelListener(IcResiUserImportServiceImpl icResiUserImportService, String customerId, + String currentUserId, String currUserAgencyId, + String currUserAgencyPids, Boolean isPrimary, String tableName, + List formItemList, Integer headRowNumber) { + + this.customerId = customerId; + this.icResiUserImportService = icResiUserImportService; + this.formItemList = formItemList; + this.isPrimary = isPrimary; + this.currentUserId = currentUserId; + this.currUserAgencyId = currUserAgencyId; + this.currUserAgencyPids = currUserAgencyPids; + this.tableName = tableName; + this.headRowNumber = headRowNumber; + } + /** * 这里会一行行的返回头 * @@ -34,9 +113,35 @@ public class DynamicEasyExcelListener extends AnalysisEventListener headMap, AnalysisContext context) { - //log.info("解析到一条头数据:{}", JSON.toJSONString(headMap)); - //存储全部表头数据 headList.add(headMap); + + if (headList.size() < headRowNumber) { + return; + } + + // 合并多级表头到一个list中，key为列序号 + Map> headers = icResiUserImportService.mergeMultiLevelHeadLabels(headList); + + // 清洗表头数据,通过items剔除，并且得到options + abandonedHeaders = icResiUserImportService.removeAndGetOptionsFromHeaders(headers, formItemList); + + // 交换表头信息，以label连起来的string作为key，列号的列表作为value + HashMap> combinedHeaders = icResiUserImportService.exchangeKeyAndValueOfHeaders(headers); + + // 得到客户配置item数据。<"兴趣爱好:兴趣特长", item对象> + Map customizedLabelCompbinedItemsMap = formItemList.stream().collect( + Collectors.toMap(formItem -> { + String groupLabel = formItem.getGroupLabel(); + String label = formItem.getLabel(); + if (StringUtils.isNotBlank(groupLabel)) { + return groupLabel.concat(":").concat(label); + } else { + return label; + } + }, formItem -> formItem) + ); + + itemIdAndColumnWrapper = icResiUserImportService.convertExcelHeaders2DBColumnWrappers(customizedLabelCompbinedItemsMap, combinedHeaders); } /** @@ -48,26 +153,37 @@ public class DynamicEasyExcelListener extends AnalysisEventListener data, AnalysisContext context) { - //log.info("解析到一条数据:{}", JSON.toJSONString(data)); + // 每2000条数据处理一次 dataList.add(data); + + // 达到了批量导入的阈值，执行一次持久化 + if (dataList.size() >= ONCE_BATCH_IMPORT_ITEMS_SIZE) { + execPersistant(); + } } /** - * 所有数据解析完成了 都会来调用 - * + * 所有数据解析完成了会调用 + * 此处也要判断，然后执行持久化，因为最后一部分数据，达不到批量阈值，不能漏掉，在最后结束的时候给他执行进去 * @param context */ @Override public void doAfterAllAnalysed(AnalysisContext context) { - // 这里也要保存数据，确保最后遗留的数据也存储到数据库 - //log.info("所有数据解析完成！"); - } - - public List> getHeadList() { - return headList; + if (dataList.size() != 0) { + execPersistant(); + } } - public List> getDataList() { - return dataList; + /** + * 执行持久化 + */ + private void execPersistant() { + // 持久化 + if (isPrimary) { + icResiUserImportService.persistIcResiBaseInfo(itemIdAndColumnWrapper, dataList, currUserAgencyId, abandonedHeaders, currUserAgencyPids, currentUserId, tableName); + } else { + icResiUserImportService.persistIcResiExtraInfo(itemIdAndColumnWrapper, dataList, currUserAgencyId, abandonedHeaders, currentUserId, tableName, customerId); + } + dataList.clear(); } } \ No newline at end of file diff --git a/epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/IcResiUserImportServiceImpl.java b/epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/IcResiUserImportServiceImpl.java index 71e0e0d9ca..d236c92004 100644 --- a/epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/IcResiUserImportServiceImpl.java +++ b/epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/IcResiUserImportServiceImpl.java @@ -232,7 +232,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res // 上传主表信息 importIcResiBaseInfoFromExcel(formItemList,excelPathName, IcResiUserTableEnum.IC_RESI_USER.getSheetNo(), IcResiUserTableEnum.IC_RESI_USER.getHeadRowNo(), - currUserAgencyId, agencyInfo.getPids(), loginUserId, IcResiUserTableEnum.IC_RESI_USER.getTableName()); + currUserAgencyId, agencyInfo.getPids(), loginUserId, IcResiUserTableEnum.IC_RESI_USER.getTableName(), customerId); // 上传附表信息 for (IcResiUserTableEnum sheet : IcResiUserTableEnum.values()) { @@ -320,43 +320,12 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res * @param currentUserId * @return */ - private Object importIcResiBaseInfoFromExcel(List formItemList, String excelPathName, int sheetNo, int headRowNumber, String currUserAgencyId, String currUserAgencyPids, String currentUserId, - String tableName) { - DynamicEasyExcelListener readListener = new DynamicEasyExcelListener(); - //EasyExcelFactory.read(new File(excelPathName)).registerReadListener(readListener).headRowNumber(headRowNumber).sheet(sheetNo).doRead(); - EasyExcel.read(new File(excelPathName)).registerReadListener(readListener).headRowNumber(headRowNumber).sheet(sheetNo).doRead(); - - List> headList = readListener.getHeadList(); - List> dataList = readListener.getDataList(); - - // 合并多级表头到一个list中，key为列序号 - Map> headers = mergeMultiLevelHeadLabels(headList); - - // 清洗表头数据,通过items剔除，并且得到options - Map abandonedHeaders = removeAndGetOptionsFromHeaders(headers, formItemList); - - // 交换表头信息，以label连起来的string作为key，列号的列表作为value - HashMap> combinedHeaders = exchangeKeyAndValueOfHeaders(headers); - - // 得到客户配置item数据。<"兴趣爱好:兴趣特长", item对象> - Map customizedLabelCompbinedItemsMap = formItemList.stream().collect( - Collectors.toMap(formItem -> { - String groupLabel = formItem.getGroupLabel(); - String label = formItem.getLabel(); - if (StringUtils.isNotBlank(groupLabel)) { - return groupLabel.concat(":").concat(label); - } else { - return label; - } - }, formItem -> formItem) - ); - - Map itemIdAndColumnWrapper = convertExcelHeaders2DBColumnWrappers(customizedLabelCompbinedItemsMap, combinedHeaders); - - // 持久化 - persistIcResiBaseInfo(itemIdAndColumnWrapper, dataList, currUserAgencyId, abandonedHeaders, currUserAgencyPids, currentUserId, tableName); + private void importIcResiBaseInfoFromExcel(List formItemList, String excelPathName, int sheetNo, int headRowNumber, String currUserAgencyId, String currUserAgencyPids, String currentUserId, + String tableName, String customerId) { - return headers; + DynamicEasyExcelListener readListener = new DynamicEasyExcelListener(this, customerId, currentUserId, currUserAgencyId, currUserAgencyPids, + true, tableName, formItemList, headRowNumber); + EasyExcel.read(new File(excelPathName)).registerReadListener(readListener).headRowNumber(headRowNumber).sheet(sheetNo).doRead(); } /** @@ -371,34 +340,12 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res * @param targetTableName 要插入哪一个表 * @return */ - private Object importIcResiExtraInfoFromExcel(List formItemList, String excelPathName, int sheetNo, int headRowNumber, String currUserAgencyId, String currentUserId, + private void importIcResiExtraInfoFromExcel(List formItemList, String excelPathName, int sheetNo, int headRowNumber, String currUserAgencyId, String currentUserId, String targetTableName, String customerId) { - DynamicEasyExcelListener readListener = new DynamicEasyExcelListener(); - EasyExcelFactory.read(new File(excelPathName)).registerReadListener(readListener).headRowNumber(headRowNumber).sheet(sheetNo).doRead(); - - List> headList = readListener.getHeadList(); - List> dataList = readListener.getDataList(); - - Map> headers = mergeMultiLevelHeadLabels(headList); - - Map abandonedHeaders = removeAndGetOptionsFromHeaders(headers, formItemList); - HashMap> combinedHeaders = exchangeKeyAndValueOfHeaders(headers); - - Map formItemMap = formItemList.stream().collect( - Collectors.toMap(formItem -> { - String groupLabel = formItem.getGroupLabel(); - String label = formItem.getLabel(); - if (StringUtils.isNotBlank(groupLabel)) { - return groupLabel.concat(":").concat(label); - } else { - return label; - } - }, formItem -> formItem) - ); - Map headerColumnWrapper = convertExcelHeaders2DBColumnWrappers(formItemMap, combinedHeaders); - persistIcResiExtraInfo(headerColumnWrapper, dataList, currUserAgencyId, abandonedHeaders, currentUserId, targetTableName, customerId); - return headerColumnWrapper; + DynamicEasyExcelListener readListener = new DynamicEasyExcelListener(this, customerId, currentUserId, currUserAgencyId, null, + false, targetTableName, formItemList, headRowNumber); + EasyExcelFactory.read(new File(excelPathName)).registerReadListener(readListener).headRowNumber(headRowNumber).sheet(sheetNo).doRead(); } /** @@ -408,7 +355,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res * @param currUserAgencyId 当前用户的组织id * @param checkBoxOptionColumnIdxAndLabel 复选框的列号&label中文 */ - private void persistIcResiBaseInfo(Map itemIdAndColumnWrapper, List> dataRows, + public void persistIcResiBaseInfo(Map itemIdAndColumnWrapper, List> dataRows, String currUserAgencyId, Map checkBoxOptionColumnIdxAndLabel, String currUserAgencyPids, String currentUserId, String tableName) { @@ -517,7 +464,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res * @param checkBoxOptionColumnIdxAndLabel 复选框的列号&label中文 * @param targetTableName 要插入到哪一个表 */ - private void persistIcResiExtraInfo(Map headerColumnWrapper, List> dataRows, + public void persistIcResiExtraInfo(Map headerColumnWrapper, List> dataRows, String currUserAgencyId, Map checkBoxOptionColumnIdxAndLabel, String currentUserId, String targetTableName, String customerId) { @@ -709,7 +656,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res * @author wxz * @date 2021.10.28 21:27:18 */ - private HashMap> exchangeKeyAndValueOfHeaders(Map> headers) { + public HashMap> exchangeKeyAndValueOfHeaders(Map> headers) { HashMap> itemAndColIndexs = new LinkedHashMap<>(); headers.forEach((k, v) -> { @@ -734,7 +681,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res * @author wxz * @date 2021.10.28 21:07:12 */ - private Map removeAndGetOptionsFromHeaders(Map> headers, List items) { + public Map removeAndGetOptionsFromHeaders(Map> headers, List items) { List itemLabels = items.stream().map(i -> i.getLabel()).collect(Collectors.toList()); Map abandonedOptions = new HashMap<>(); for (Map.Entry> entry:headers.entrySet()) { @@ -765,7 +712,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res * value：列号组成的列表(例如[1,2,3]) * @return key:itemId；value：columnWrapper */ - private Map convertExcelHeaders2DBColumnWrappers(Map customizedLabelCompbinedItemsMap, Map> combinedHeaders) { + public Map convertExcelHeaders2DBColumnWrappers(Map customizedLabelCompbinedItemsMap, Map> combinedHeaders) { // HashMap> tables = new HashMap<>(); Map columns = new LinkedHashMap<>(combinedHeaders.size()); @@ -835,7 +782,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res * @author wxz * @date 2021.10.27 16:17:34 */ - private Map> mergeMultiLevelHeadLabels(List> headList) { + public Map> mergeMultiLevelHeadLabels(List> headList) { Map lastNotNullHeads = new LinkedHashMap<>(); From f7c5e8ceb6bad19b21bf90af3f50ff59e462b0bb Mon Sep 17 00:00:00 2001 From: wangxianzhang Date: Mon, 14 Feb 2022 16:35:00 +0800 Subject: [PATCH 2/5] =?UTF-8?q?=E4=BF=AE=E6=94=B9=EF=BC=9A=201.=E5=B1=85?= =?UTF-8?q?=E6=B0=91=E4=BF=A1=E6=81=AF=E5=AF=BC=E5=85=A5=EF=BC=9A=E5=B0=86?= =?UTF-8?q?=E7=B1=BB=E5=88=AB=E5=8F=98=E6=9B=B4=E6=95=B0=E6=8D=AE=E6=94=BE?= =?UTF-8?q?=E5=88=B0redis=E4=B8=AD=EF=BC=8C=E5=87=8F=E5=B0=91=E5=86=85?= =?UTF-8?q?=E5=AD=98=E5=8D=A0=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../epmet/commons/tools/redis/RedisKeys.java | 33 ++++++ .../epmet/bean/ResiImportCategoryData.java | 13 +- .../ResiImportResiCategoryChangedCache.java | 7 +- ... => IcResiImportDynamicExcelListener.java} | 10 +- .../impl/IcResiUserImportServiceImpl.java | 111 +++++++++++++----- 5 files changed, 126 insertions(+), 48 deletions(-) rename epmet-user/epmet-user-server/src/main/java/com/epmet/excel/handler/{DynamicEasyExcelListener.java => IcResiImportDynamicExcelListener.java} (91%) diff --git a/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/redis/RedisKeys.java b/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/redis/RedisKeys.java index dc3fb232bc..53c50c5c2e 100644 --- a/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/redis/RedisKeys.java +++ b/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/redis/RedisKeys.java @@ -640,4 +640,37 @@ public class RedisKeys { public static String getHouseInfoCacheKey(String houseId,String customerId){ return rootPrefix.concat("house:").concat(customerId).concat(":").concat(houseId); } + + /** + * 居民导入-缓存目录key + * @param importTag + * @return + */ + public static String icResiImportBaseKey(String importTag) { + return rootPrefix.concat("resi:").concat("import:").concat(importTag); + } + + /** + * 居民导入-数据分类的key + * add：新增居民 + * category：类别变更 + * transfer：调动 + * @param importTag + * @param type + * @return + */ + public static String icResiImportTypeKey(String importTag, String type) { + return icResiImportBaseKey(importTag).concat(":").concat(type); + } + + /** + * 居民导入-详细信息key + * @param importTag + * @param type + * @param resiId + * @return + */ + public static String icResiImportResiCategoryKey(String importTag, String type, String resiId) { + return icResiImportTypeKey(importTag, type).concat(":").concat(resiId); + } } diff --git a/epmet-user/epmet-user-server/src/main/java/com/epmet/bean/ResiImportCategoryData.java b/epmet-user/epmet-user-server/src/main/java/com/epmet/bean/ResiImportCategoryData.java index 436be52b29..1526038d21 100644 --- a/epmet-user/epmet-user-server/src/main/java/com/epmet/bean/ResiImportCategoryData.java +++ b/epmet-user/epmet-user-server/src/main/java/com/epmet/bean/ResiImportCategoryData.java @@ -14,6 +14,7 @@ import java.util.Map; @Data public class ResiImportCategoryData { + private String resiId; private String agencyId; private String gridId; private String villageId; @@ -30,7 +31,8 @@ public class ResiImportCategoryData { public ResiImportCategoryData() { } - public ResiImportCategoryData(String agencyId, String gridId, String villageId, String buildId, String unitId, String homeId, Map categories) { + public ResiImportCategoryData(String resiId, String agencyId, String gridId, String villageId, String buildId, String unitId, String homeId, Map categories) { + this.resiId = resiId; this.agencyId = agencyId; this.gridId = gridId; this.villageId = villageId; @@ -39,13 +41,4 @@ public class ResiImportCategoryData { this.homeId = homeId; this.categories = categories; } - - public ResiImportCategoryData(String agencyId, String gridId, String villageId, String buildId, String unitId, String homeId) { - this.agencyId = agencyId; - this.gridId = gridId; - this.villageId = villageId; - this.buildId = buildId; - this.unitId = unitId; - this.homeId = homeId; - } } diff --git a/epmet-user/epmet-user-server/src/main/java/com/epmet/bean/ResiImportResiCategoryChangedCache.java b/epmet-user/epmet-user-server/src/main/java/com/epmet/bean/ResiImportResiCategoryChangedCache.java index 6203c917a9..29470e875c 100644 --- a/epmet-user/epmet-user-server/src/main/java/com/epmet/bean/ResiImportResiCategoryChangedCache.java +++ b/epmet-user/epmet-user-server/src/main/java/com/epmet/bean/ResiImportResiCategoryChangedCache.java @@ -11,11 +11,16 @@ import java.util.Map; @Data public class ResiImportResiCategoryChangedCache { + /** + * 导入的tag，用来标记唯一一次导入操作。 + */ + private String importTag; + /** * 新增居民 * Map> */ - private Map newResis = new HashMap<>(); +// private Map newResis = new HashMap<>(); /** * 调动的居民 diff --git a/epmet-user/epmet-user-server/src/main/java/com/epmet/excel/handler/DynamicEasyExcelListener.java b/epmet-user/epmet-user-server/src/main/java/com/epmet/excel/handler/IcResiImportDynamicExcelListener.java similarity index 91% rename from epmet-user/epmet-user-server/src/main/java/com/epmet/excel/handler/DynamicEasyExcelListener.java rename to epmet-user/epmet-user-server/src/main/java/com/epmet/excel/handler/IcResiImportDynamicExcelListener.java index bf65c3bbf2..427e70f0a3 100644 --- a/epmet-user/epmet-user-server/src/main/java/com/epmet/excel/handler/DynamicEasyExcelListener.java +++ b/epmet-user/epmet-user-server/src/main/java/com/epmet/excel/handler/IcResiImportDynamicExcelListener.java @@ -17,7 +17,7 @@ import java.util.stream.Collectors; * 创建一个监听器 */ @Slf4j -public class DynamicEasyExcelListener extends AnalysisEventListener> { +public class IcResiImportDynamicExcelListener extends AnalysisEventListener> { /** * 一次性导入的条数 @@ -89,10 +89,10 @@ public class DynamicEasyExcelListener extends AnalysisEventListener dataList = new HashMap<>(); - public DynamicEasyExcelListener(IcResiUserImportServiceImpl icResiUserImportService, String customerId, - String currentUserId, String currUserAgencyId, - String currUserAgencyPids, Boolean isPrimary, String tableName, - List formItemList, Integer headRowNumber) { + public IcResiImportDynamicExcelListener(IcResiUserImportServiceImpl icResiUserImportService, String customerId, + String currentUserId, String currUserAgencyId, + String currUserAgencyPids, Boolean isPrimary, String tableName, + List formItemList, Integer headRowNumber) { this.customerId = customerId; this.icResiUserImportService = icResiUserImportService; diff --git a/epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/IcResiUserImportServiceImpl.java b/epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/IcResiUserImportServiceImpl.java index d236c92004..23248b0c83 100644 --- a/epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/IcResiUserImportServiceImpl.java +++ b/epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/IcResiUserImportServiceImpl.java @@ -3,8 +3,10 @@ package com.epmet.service.impl; import cn.afterturn.easypoi.excel.ExcelExportUtil; import cn.afterturn.easypoi.excel.annotation.Excel; import cn.afterturn.easypoi.excel.entity.ExportParams; +import cn.hutool.core.bean.BeanUtil; import com.alibaba.excel.EasyExcel; import com.alibaba.excel.EasyExcelFactory; +import com.baomidou.mybatisplus.core.toolkit.IdWorker; import com.epmet.bean.ResiImportCategoryData; import com.epmet.bean.ResiImportResiCategoryChangedCache; import com.epmet.bean.ResiImportChangedData; @@ -16,8 +18,11 @@ import com.epmet.commons.tools.exception.EpmetErrorCode; import com.epmet.commons.tools.exception.ExceptionUtils; import com.epmet.commons.tools.exception.RenException; import com.epmet.commons.tools.feign.ResultDataResolver; +import com.epmet.commons.tools.redis.RedisKeys; +import com.epmet.commons.tools.redis.RedisUtils; import com.epmet.commons.tools.redis.common.CustomerStaffRedis; import com.epmet.commons.tools.security.user.LoginUserUtil; +import com.epmet.commons.tools.utils.ConvertUtils; import com.epmet.commons.tools.utils.Result; import com.epmet.dao.IcResiUserDao; import com.epmet.dao.IcUserChangeDetailedDao; @@ -31,7 +36,7 @@ import com.epmet.entity.IcUserChangeDetailedEntity; import com.epmet.entity.IcUserChangeRecordEntity; import com.epmet.entity.IcUserTransferRecordEntity; import com.epmet.enums.IcResiUserTableEnum; -import com.epmet.excel.handler.DynamicEasyExcelListener; +import com.epmet.excel.handler.IcResiImportDynamicExcelListener; import com.epmet.feign.EpmetAdminOpenFeignClient; import com.epmet.feign.EpmetUserOpenFeignClient; import com.epmet.feign.GovOrgOpenFeignClient; @@ -118,6 +123,9 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res @Autowired private UserService userService; + @Autowired + private RedisUtils redisUtils; + /** * 字表中不需要的列 @@ -253,12 +261,12 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res } // 执行人员类别变更记录 - Map newResis = newlyOrChangedResi.get().getNewResis(); +// Map newResis = newlyOrChangedResi.get().getNewResis(); Map categoryChangedResis = newlyOrChangedResi.get().getCategoryChangedResis(); Map transferedResis = newlyOrChangedResi.get().getTransferedResis(); //保存调动或者变更记录 - saveNewResiCategoryRecord(newResis); + saveNewResiCategoryRecord(); saveResiCategoryChangedRecord(categoryChangedResis); saveTransferedResiRecord(transferedResis); @@ -268,6 +276,16 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res log.error("【导入IC居民附加信息】下载导入结果信息失败:{}", ExceptionUtils.getErrorStackTrace(e)); } } finally { + // 清空rediskey + try { + String importTag = newlyOrChangedResi.get().getImportTag(); + redisUtils.deleteByPattern(RedisKeys.icResiImportBaseKey(importTag).concat("*")); + } catch (Exception e) { + String errorMsg = ExceptionUtils.getErrorStackTrace(e); + log.error("【导入IC居民附加信息】清空redis 类别缓存出错:{}", errorMsg); + } + + // 清空线程变量 skipedRows.remove(); errorRows.remove(); newlyOrChangedResi.remove(); @@ -280,6 +298,10 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res * threadLocal初始化 */ private void initImportThreadLocal(String customerId) { + // 生成importTag，用于标记唯一一次导入操作，导入完成之后，用来删除redis里面临时的key + String operatorId = loginUserUtil.getLoginUserId(); + String importTag = operatorId + System.currentTimeMillis(); + // 跳过的,不导入的行 Map> skipedRowsMap = new LinkedHashMap<>(); for (IcResiUserTableEnum e : IcResiUserTableEnum.values()) { @@ -296,6 +318,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res // 居民类别信息 ResiImportResiCategoryChangedCache c = new ResiImportResiCategoryChangedCache(); + c.setImportTag(importTag); newlyOrChangedResi.set(c); List resiCategoryItems = getResultDataOrThrowsException(operCustomizeOpenFeignClient.listResiCategoryItems(customerId), @@ -323,7 +346,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res private void importIcResiBaseInfoFromExcel(List formItemList, String excelPathName, int sheetNo, int headRowNumber, String currUserAgencyId, String currUserAgencyPids, String currentUserId, String tableName, String customerId) { - DynamicEasyExcelListener readListener = new DynamicEasyExcelListener(this, customerId, currentUserId, currUserAgencyId, currUserAgencyPids, + IcResiImportDynamicExcelListener readListener = new IcResiImportDynamicExcelListener(this, customerId, currentUserId, currUserAgencyId, currUserAgencyPids, true, tableName, formItemList, headRowNumber); EasyExcel.read(new File(excelPathName)).registerReadListener(readListener).headRowNumber(headRowNumber).sheet(sheetNo).doRead(); } @@ -343,7 +366,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res private void importIcResiExtraInfoFromExcel(List formItemList, String excelPathName, int sheetNo, int headRowNumber, String currUserAgencyId, String currentUserId, String targetTableName, String customerId) { - DynamicEasyExcelListener readListener = new DynamicEasyExcelListener(this, customerId, currentUserId, currUserAgencyId, null, + IcResiImportDynamicExcelListener readListener = new IcResiImportDynamicExcelListener(this, customerId, currentUserId, currUserAgencyId, null, false, targetTableName, formItemList, headRowNumber); EasyExcelFactory.read(new File(excelPathName)).registerReadListener(readListener).headRowNumber(headRowNumber).sheet(sheetNo).doRead(); } @@ -415,8 +438,10 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res } } else { // 新增居民 + String resiId = IdWorker.getIdStr(); + columnAndValues.put("CREATED_BY", currentUserId); - columnAndValues.put("ID", UUID.randomUUID().toString().replace("-", "")); + columnAndValues.put("ID", resiId); icResiUserDao.add(tableName, columnAndValues); // 过滤出本居民含有哪些类别 @@ -425,14 +450,26 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res .filter((categoryColumnName) -> "1".equals(columnAndValues.get(categoryColumnName))) .collect(Collectors.toMap((k) -> k, (k) -> columnAndValues.get(k))); - newlyOrChangedResi.get().getNewResis().put(columnAndValues.get("ID"), - new ResiImportCategoryData(columnAndValues.get("AGENCY_ID"), - columnAndValues.get("GRID_ID"), - columnAndValues.get("VILLAGE_ID"), - columnAndValues.get("BUILD_ID"), - columnAndValues.get("UNIT_ID"), - columnAndValues.get("HOME_ID"), - resiCategories)); +// newlyOrChangedResi.get().getNewResis().put(resiId, +// new ResiImportCategoryData(columnAndValues.get("AGENCY_ID"), +// columnAndValues.get("GRID_ID"), +// columnAndValues.get("VILLAGE_ID"), +// columnAndValues.get("BUILD_ID"), +// columnAndValues.get("UNIT_ID"), +// columnAndValues.get("HOME_ID"), +// resiCategories)); + + ResiImportCategoryData categoryData = new ResiImportCategoryData(resiId, + columnAndValues.get("AGENCY_ID"), + columnAndValues.get("GRID_ID"), + columnAndValues.get("VILLAGE_ID"), + columnAndValues.get("BUILD_ID"), + columnAndValues.get("UNIT_ID"), + columnAndValues.get("HOME_ID"), + resiCategories); + + redisUtils.hMSet(RedisKeys.icResiImportResiCategoryKey(newlyOrChangedResi.get().getImportTag(), "add", resiId), BeanUtil.beanToMap(categoryData)); + } } catch (Exception e) { @@ -1040,6 +1077,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res } ResiImportCategoryData oldOne = new ResiImportCategoryData( + existingResiMap.get("ID"), existingResiMap.get("AGENCY_ID"), existingResiMap.get("GRID_ID"), existingResiMap.get("VILLAGE_ID"), @@ -1049,7 +1087,8 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res existingResiCategories); ResiImportCategoryData newOne = new ResiImportCategoryData( - existingResiMap.get("AGENCY_ID"), + newResiMap.get("ID"), + newResiMap.get("AGENCY_ID"), newResiMap.get("GRID_ID"), newResiMap.get("VILLAGE_ID"), newResiMap.get("BUILD_ID"), @@ -1100,6 +1139,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res } ResiImportCategoryData oldOne = new ResiImportCategoryData( + existingResiMap.get("ID"), existingResiMap.get("AGENCY_ID"), existingResiMap.get("GRID_ID"), existingResiMap.get("VILLAGE_ID"), @@ -1108,7 +1148,9 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res existingResiMap.get("HOME_ID"), oldCategories); - ResiImportCategoryData newOne = new ResiImportCategoryData(existingResiMap.get("AGENCY_ID"), + ResiImportCategoryData newOne = new ResiImportCategoryData( + newResiMap.get("ID"), + newResiMap.get("AGENCY_ID"), newResiMap.get("GRID_ID"), newResiMap.get("VILLAGE_ID"), newResiMap.get("BUILD_ID"), @@ -1127,14 +1169,19 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res private void saveSubTableInfoToCategoryChangedResiCache(String icResiId, String columnName, Map resiInfoMap) { ResiImportResiCategoryChangedCache cc = newlyOrChangedResi.get(); - Map newResis = cc.getNewResis(); +// Map newResis = cc.getNewResis(); Map categoryChangedResis = cc.getCategoryChangedResis(); Map transferedResis = cc.getTransferedResis(); - ResiImportCategoryData newResi = newResis.get(icResiId); - if (newResi != null) { +// ResiImportCategoryData newResi = newResis.get(icResiId); + + String userCateRedisKey = RedisKeys.icResiImportResiCategoryKey(newlyOrChangedResi.get().getImportTag(), "add", icResiId); + Map < String, Object > addUserMap = redisUtils.hGetAll(userCateRedisKey); + if (addUserMap != null) { //说明是新增居民 - newResi.getCategories().put(columnName, "1"); + ResiImportCategoryData newResiCateData = ConvertUtils.mapToEntity(addUserMap, ResiImportCategoryData.class); + newResiCateData.getCategories().put(columnName, "1"); + redisUtils.hMSet(userCateRedisKey, BeanUtil.beanToMap(newResiCateData)); return; } @@ -1286,35 +1333,35 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res /** * 保存新增居民类别记录 - * @param newResis */ - public void saveNewResiCategoryRecord(Map newResis) { + public void saveNewResiCategoryRecord() { Date now = new Date(); + Set newResiKeys = redisUtils.keys(RedisKeys.icResiImportTypeKey(newlyOrChangedResi.get().getImportTag(), "add").concat("*")); + CustomerStaffInfoCacheResult operator = CustomerStaffRedis.getStaffInfo(loginUserUtil.getLoginUserCustomerId(), loginUserUtil.getLoginUserId()); - for (Map.Entry resi : newResis.entrySet()) { - String resiId = resi.getKey(); - ResiImportCategoryData data = resi.getValue(); + for (String resiKey : newResiKeys) { + Map newResiInfoMap = redisUtils.hGetAll(resiKey); + ResiImportCategoryData newResiInfoObj = ConvertUtils.mapToEntity(newResiInfoMap, ResiImportCategoryData.class); - IcResiUserEntity resiInfo = icResiUserDao.selectById(resiId); + IcResiUserEntity resiInfo = icResiUserDao.selectById(newResiInfoObj.getResiId()); // 插入changeRecord - IcUserChangeRecordEntity changeRecord = fillChangeRecord(loginUserUtil.getLoginUserCustomerId(), resiId, resiInfo.getName(), null, + IcUserChangeRecordEntity changeRecord = fillChangeRecord(loginUserUtil.getLoginUserCustomerId(), newResiInfoObj.getResiId(), resiInfo.getName(), null, loginUserUtil.getLoginUserId(), operator.getRealName(), "-", "-", "add", "新增", "", now); icUserChangeRecordDao.insert(changeRecord); // 插入changeDetail - for (Map.Entry column : data.getCategories().entrySet()) { - IcUserChangeDetailedEntity changedetail = fillChangeDetail(loginUserUtil.getLoginUserCustomerId(), resiId, changeRecord.getId(), data.getAgencyId(), - data.getGridId(), data.getVillageId(), data.getBuildId(), data.getUnitId(), - data.getHomeId(), "add", "新增", column.getKey(), 1, resiInfo.getPids()); + for (Map.Entry column : newResiInfoObj.getCategories().entrySet()) { + IcUserChangeDetailedEntity changedetail = fillChangeDetail(loginUserUtil.getLoginUserCustomerId(), newResiInfoObj.getResiId(), changeRecord.getId(), newResiInfoObj.getAgencyId(), + newResiInfoObj.getGridId(), newResiInfoObj.getVillageId(), newResiInfoObj.getBuildId(), newResiInfoObj.getUnitId(), + newResiInfoObj.getHomeId(), "add", "新增", column.getKey(), 1, resiInfo.getPids()); icUserChangeDetailedDao.insert(changedetail); } } - } /** From e41cf5a5069a4db811e346d29217dbe58272a00d Mon Sep 17 00:00:00 2001 From: wangxianzhang Date: Tue, 15 Feb 2022 14:04:09 +0800 Subject: [PATCH 3/5] =?UTF-8?q?-=20=E4=BC=98=E5=8C=96=E5=B1=85=E6=B0=91?= =?UTF-8?q?=E5=AF=BC=E5=85=A5=20-=20=E4=BC=98=E5=8C=96=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E5=88=87=E9=9D=A2=E6=89=93=E5=8D=B0=EF=BC=8C=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E5=AF=B9MultipartFile=E7=B1=BB=E5=9E=8B=E7=9A=84=E8=BF=87?= =?UTF-8?q?=E6=BB=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../commons/tools/aspect/BaseRequestLogAspect.java | 2 ++ .../service/impl/IcResiUserImportServiceImpl.java | 10 +++++++++- .../epmet-user-server/src/main/resources/bootstrap.yml | 5 +++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/aspect/BaseRequestLogAspect.java b/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/aspect/BaseRequestLogAspect.java index 055d174c2e..3321091b4f 100644 --- a/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/aspect/BaseRequestLogAspect.java +++ b/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/aspect/BaseRequestLogAspect.java @@ -10,6 +10,7 @@ import org.aspectj.lang.ProceedingJoinPoint; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.dao.DuplicateKeyException; +import org.springframework.web.multipart.MultipartFile; import javax.servlet.ServletRequest; import javax.servlet.ServletResponse; @@ -286,6 +287,7 @@ public abstract class BaseRequestLogAspect { if (object != null && !(object instanceof ServletRequest) && !(object instanceof ServletResponse) + && !(object instanceof MultipartFile) ) { try { // 尝试作为json解析 diff --git a/epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/IcResiUserImportServiceImpl.java b/epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/IcResiUserImportServiceImpl.java index 23248b0c83..32314bff9b 100644 --- a/epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/IcResiUserImportServiceImpl.java +++ b/epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/IcResiUserImportServiceImpl.java @@ -265,6 +265,9 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res Map categoryChangedResis = newlyOrChangedResi.get().getCategoryChangedResis(); Map transferedResis = newlyOrChangedResi.get().getTransferedResis(); + log.info("类别变动居民数:{}", categoryChangedResis.size()); + log.info("调动居民数:{}", transferedResis.size()); + //保存调动或者变更记录 saveNewResiCategoryRecord(); saveResiCategoryChangedRecord(categoryChangedResis); @@ -469,7 +472,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res resiCategories); redisUtils.hMSet(RedisKeys.icResiImportResiCategoryKey(newlyOrChangedResi.get().getImportTag(), "add", resiId), BeanUtil.beanToMap(categoryData)); - + categoryData = null; } } catch (Exception e) { @@ -1182,6 +1185,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res ResiImportCategoryData newResiCateData = ConvertUtils.mapToEntity(addUserMap, ResiImportCategoryData.class); newResiCateData.getCategories().put(columnName, "1"); redisUtils.hMSet(userCateRedisKey, BeanUtil.beanToMap(newResiCateData)); + newResiCateData = null; return; } @@ -1360,7 +1364,11 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res newResiInfoObj.getHomeId(), "add", "新增", column.getKey(), 1, resiInfo.getPids()); icUserChangeDetailedDao.insert(changedetail); + changedetail = null; } + + changeRecord = null; + newResiInfoObj = null; } } diff --git a/epmet-user/epmet-user-server/src/main/resources/bootstrap.yml b/epmet-user/epmet-user-server/src/main/resources/bootstrap.yml index ac5d97de01..cce9a52831 100644 --- a/epmet-user/epmet-user-server/src/main/resources/bootstrap.yml +++ b/epmet-user/epmet-user-server/src/main/resources/bootstrap.yml @@ -79,6 +79,11 @@ spring: namespace: @nacos.config.namespace@ group: @nacos.config.group@ file-extension: yaml + servlet: + multipart: + max-file-size: 100MB + max-request-size: 100MB + management: endpoints: web: From ec1eee8d70de340a3ccac417a7e890011d4a4f70 Mon Sep 17 00:00:00 2001 From: wangxianzhang Date: Wed, 16 Feb 2022 17:43:40 +0800 Subject: [PATCH 4/5] =?UTF-8?q?=E4=BF=AE=E6=94=B9=EF=BC=9A=20-=20=E5=B1=85?= =?UTF-8?q?=E6=B0=91=E5=AF=BC=E5=85=A5=EF=BC=9A=E5=A2=9E=E5=8A=A0=E5=AF=BC?= =?UTF-8?q?=E5=85=A5=E4=BB=BB=E5=8A=A1=E8=AE=B0=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../epmet/constant/ImportTaskConstants.java | 26 ++++++++ .../controller/IcResiUserController.java | 30 ++++++++- .../java/com/epmet/dao/ImportTaskDao.java | 31 +++++++++ .../com/epmet/entity/ImportTaskEntity.java | 48 ++++++++++++++ .../service/IcResiUserImportService.java | 2 +- .../com/epmet/service/ImportTaskService.java | 35 +++++++++++ .../impl/IcResiUserImportServiceImpl.java | 63 ++++++++++++++----- .../service/impl/ImportTaskServiceImpl.java | 57 +++++++++++++++++ .../main/resources/mapper/ImportTaskDao.xml | 33 ++++++++++ 9 files changed, 304 insertions(+), 21 deletions(-) create mode 100644 epmet-user/epmet-user-client/src/main/java/com/epmet/constant/ImportTaskConstants.java create mode 100644 epmet-user/epmet-user-server/src/main/java/com/epmet/dao/ImportTaskDao.java create mode 100644 epmet-user/epmet-user-server/src/main/java/com/epmet/entity/ImportTaskEntity.java create mode 100644 epmet-user/epmet-user-server/src/main/java/com/epmet/service/ImportTaskService.java create mode 100644 epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/ImportTaskServiceImpl.java create mode 100644 epmet-user/epmet-user-server/src/main/resources/mapper/ImportTaskDao.xml diff --git a/epmet-user/epmet-user-client/src/main/java/com/epmet/constant/ImportTaskConstants.java b/epmet-user/epmet-user-client/src/main/java/com/epmet/constant/ImportTaskConstants.java new file mode 100644 index 0000000000..685e80a39a --- /dev/null +++ b/epmet-user/epmet-user-client/src/main/java/com/epmet/constant/ImportTaskConstants.java @@ -0,0 +1,26 @@ +package com.epmet.constant; + +/** + * 导入任务的业务类型常量 + */ +public interface ImportTaskConstants { + /** + * 居民 + */ + String BIZ_TYPE_RESI = "resi"; + + /** + * 处理状态：处理中 + */ + String PROCESS_STATUS_PROCESSING = "processing"; + + /** + * 处理状态：成功 + */ + String PROCESS_STATUS_FINISHED_SUCCESS = "finished_success"; + + /** + * 处理状态：完成，但未完全成功 + */ + String PROCESS_STATUS_FINISHED_FAIL = "finished_fail"; +} diff --git a/epmet-user/epmet-user-server/src/main/java/com/epmet/controller/IcResiUserController.java b/epmet-user/epmet-user-server/src/main/java/com/epmet/controller/IcResiUserController.java index 1350ce159f..7624198dfb 100644 --- a/epmet-user/epmet-user-server/src/main/java/com/epmet/controller/IcResiUserController.java +++ b/epmet-user/epmet-user-server/src/main/java/com/epmet/controller/IcResiUserController.java @@ -38,11 +38,13 @@ import com.epmet.commons.tools.redis.RedisKeys; import com.epmet.commons.tools.redis.RedisUtils; import com.epmet.commons.tools.redis.common.CustomerStaffRedis; import com.epmet.commons.tools.security.dto.TokenDto; +import com.epmet.commons.tools.security.user.LoginUserUtil; import com.epmet.commons.tools.utils.ExcelUtils; import com.epmet.commons.tools.utils.HttpClientManager; import com.epmet.commons.tools.utils.IpUtils; import com.epmet.commons.tools.utils.Result; import com.epmet.commons.tools.validator.ValidatorUtils; +import com.epmet.constant.ImportTaskConstants; import com.epmet.constant.SystemMessageType; import com.epmet.dto.IcResiUserDTO; import com.epmet.dto.form.*; @@ -54,6 +56,7 @@ import com.epmet.feign.OperCustomizeOpenFeignClient; import com.epmet.feign.OssFeignClient; import com.epmet.service.IcResiUserImportService; import com.epmet.service.IcResiUserService; +import com.epmet.service.ImportTaskService; import jodd.io.FileUtil; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; @@ -118,6 +121,11 @@ public class IcResiUserController { private RedisUtils redisUtils; @Autowired private EpmetMessageOpenFeignClient epmetMessageOpenFeignClient; + @Autowired + private LoginUserUtil loginUserUtil; + @Autowired + private ImportTaskService importTaskService; + /** * 模板枚举 @@ -431,6 +439,18 @@ public class IcResiUserController { throw new RenException("文件类型不匹配"); } + String operatorId = loginUserUtil.getLoginUserId(); + String importTaskId; + + // 记录导入任务 + if (importTaskService.existsProcessingTask(operatorId, ImportTaskConstants.BIZ_TYPE_RESI)) { + throw new EpmetException(EpmetErrorCode.EPMET_COMMON_OPERATION_FAIL.getCode(), + "已存在执行中的导入任务，请等待执行完成。", + "已存在执行中的导入任务，请等待执行完成。"); + } else { + importTaskId = importTaskService.createProcessTask(operatorId, ImportTaskConstants.BIZ_TYPE_RESI); + } + Path savePath = null; try { String fileName = UUID.randomUUID().toString().concat(".").concat(extension); @@ -438,10 +458,14 @@ public class IcResiUserController { IOUtils.copy(file.getInputStream(), new FileOutputStream(savePath.toString())); List formItemList = icResiUserService.listFormItems(customerId,IcFormCodeEnum.RESI_BASE_INFO.getCode()); - icResiUserImportService.importIcResiInfoFromExcel(formItemList, savePath.toString(), response); - } catch (IOException e) { - String errorMsg = ExceptionUtils.getErrorStackTrace(e); + + icResiUserImportService.importIcResiInfoFromExcel(importTaskId, formItemList, savePath.toString(), response); + } catch (Throwable e) { + String errorMsg = ExceptionUtils.getThrowableErrorStackTrace(e); log.error("【导入居民信息失败】导入失败：{}", errorMsg); + + // 要将导入任务状态设置为结束但不成功 + importTaskService.finish(importTaskId, ImportTaskConstants.PROCESS_STATUS_FINISHED_FAIL, operatorId, null, e.getMessage()); throw new RenException(EpmetErrorCode.SERVER_ERROR.getCode()); } finally { try { diff --git a/epmet-user/epmet-user-server/src/main/java/com/epmet/dao/ImportTaskDao.java b/epmet-user/epmet-user-server/src/main/java/com/epmet/dao/ImportTaskDao.java new file mode 100644 index 0000000000..1306b7e3cd --- /dev/null +++ b/epmet-user/epmet-user-server/src/main/java/com/epmet/dao/ImportTaskDao.java @@ -0,0 +1,31 @@ +package com.epmet.dao; + +import com.epmet.commons.mybatis.dao.BaseDao; +import com.epmet.entity.ImportTaskEntity; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; + +/** + * + * + * @author generator generator@elink-cn.com + * @since v1.0.0 2022-02-15 + */ +@Mapper +public interface ImportTaskDao extends BaseDao { + + /** + * 完成导入 + * @param taskId 任务id + * @param processStatus 处理状态 + * @param operatorId 操作者id + * @param resultDesc 结果文字描述 + * @param resultDescFile 结果描述文件 + * @return + */ + int finish(@Param("taskId") String taskId, + @Param("process_status") String processStatus, + @Param("operatorId") String operatorId, + @Param("resultDesc") String resultDesc, + @Param("resultDescFile") String resultDescFile); +} \ No newline at end of file diff --git a/epmet-user/epmet-user-server/src/main/java/com/epmet/entity/ImportTaskEntity.java b/epmet-user/epmet-user-server/src/main/java/com/epmet/entity/ImportTaskEntity.java new file mode 100644 index 0000000000..742c6cc93e --- /dev/null +++ b/epmet-user/epmet-user-server/src/main/java/com/epmet/entity/ImportTaskEntity.java @@ -0,0 +1,48 @@ +package com.epmet.entity; + +import com.baomidou.mybatisplus.annotation.TableName; + +import com.epmet.commons.mybatis.entity.BaseEpmetEntity; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.util.Date; + +/** + * + * + * @author generator generator@elink-cn.com + * @since v1.0.0 2022-02-15 + */ +@Data +@EqualsAndHashCode(callSuper=false) +@TableName("import_task") +public class ImportTaskEntity extends BaseEpmetEntity { + + private static final long serialVersionUID = 1L; + + /** + * 业务类型。resi:居民；楼栋:building；房屋:house。依次补充 + */ + private String bizType; + + /** + * 处理状态。processing:处理中；finished:完成； + */ + private String processStatus; + + /** + * 谁导入的 + */ + private String operatorId; + + /** + * 开始导入的时间 + */ + private Date startTime; + + private String resultDescFile; + + private String resultDesc; + +} diff --git a/epmet-user/epmet-user-server/src/main/java/com/epmet/service/IcResiUserImportService.java b/epmet-user/epmet-user-server/src/main/java/com/epmet/service/IcResiUserImportService.java index 7f3d0bdca0..35748c9ddc 100644 --- a/epmet-user/epmet-user-server/src/main/java/com/epmet/service/IcResiUserImportService.java +++ b/epmet-user/epmet-user-server/src/main/java/com/epmet/service/IcResiUserImportService.java @@ -12,5 +12,5 @@ import java.util.List; */ public interface IcResiUserImportService { - void importIcResiInfoFromExcel(List formItemList, String excelPathName, HttpServletResponse response); + void importIcResiInfoFromExcel(String importTaskId, List formItemList, String excelPathName, HttpServletResponse response); } diff --git a/epmet-user/epmet-user-server/src/main/java/com/epmet/service/ImportTaskService.java b/epmet-user/epmet-user-server/src/main/java/com/epmet/service/ImportTaskService.java new file mode 100644 index 0000000000..65245abd98 --- /dev/null +++ b/epmet-user/epmet-user-server/src/main/java/com/epmet/service/ImportTaskService.java @@ -0,0 +1,35 @@ +package com.epmet.service; + + +/** + * + * + * @author generator generator@elink-cn.com + * @since v1.0.0 2022-02-15 + */ +public interface ImportTaskService { + + /** + * 检查指定类型该用户是否存在处理中的导入任务 + * @param operatorId 操作者ID + * @param bizType 业务类型。resi：居民 + * @return + */ + boolean existsProcessingTask(String operatorId, String bizType); + + /** + * 创建处理任务 + * @param operatorId + * @param bizType + */ + String createProcessTask(String operatorId, String bizType); + + /** + * 结束导入 + * @param taskId 任务id + * @param processStatus 处理状态 + * @param resultDescFile 结果描述文件 + * @param resultDesc 结果描述文本 + */ + Boolean finish(String taskId, String processStatus, String operatorId, String resultDescFile, String resultDesc); +} \ No newline at end of file diff --git a/epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/IcResiUserImportServiceImpl.java b/epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/IcResiUserImportServiceImpl.java index 32314bff9b..4ff0a8abaa 100644 --- a/epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/IcResiUserImportServiceImpl.java +++ b/epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/IcResiUserImportServiceImpl.java @@ -24,6 +24,7 @@ import com.epmet.commons.tools.redis.common.CustomerStaffRedis; import com.epmet.commons.tools.security.user.LoginUserUtil; import com.epmet.commons.tools.utils.ConvertUtils; import com.epmet.commons.tools.utils.Result; +import com.epmet.constant.ImportTaskConstants; import com.epmet.dao.IcResiUserDao; import com.epmet.dao.IcUserChangeDetailedDao; import com.epmet.dao.IcUserChangeRecordDao; @@ -72,7 +73,6 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res // 错误和跳过excel行暂存 public static final ThreadLocal>> errorRows = new ThreadLocal<>(); - public static final ThreadLocal>> skipedRows = new ThreadLocal<>(); /** * 导入的居民中，新增或者变动的居民 @@ -126,6 +126,9 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res @Autowired private RedisUtils redisUtils; + @Autowired + private ImportTaskService importTaskService; + /** * 字表中不需要的列 @@ -212,11 +215,13 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res /** * 导入居民信息 - * 导入主表和所有子表信息 - * @return + * @param importTaskId 导入任务id + * @param formItemList item列表 + * @param excelPathName excel缓存路径 + * @param response 响应对象 */ @Override - public void importIcResiInfoFromExcel(List formItemList, String excelPathName, HttpServletResponse response) { + public void importIcResiInfoFromExcel(String importTaskId, List formItemList, String excelPathName, HttpServletResponse response) { String loginUserId = loginUserUtil.getLoginUserId(); String loginUserApp = loginUserUtil.getLoginUserApp(); @@ -235,6 +240,8 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res CustomerAgencyDTO agencyInfo = getResultDataOrThrowsException(govOrgOpenFeignClient.getAgencyById(currUserAgencyId), ServiceConstant.GOV_ORG_SERVER, EpmetErrorCode.SERVER_ERROR.getCode(), null, null); String customerId = agencyInfo.getCustomerId(); + boolean hasErrorRows = false; + try { initImportThreadLocal(customerId); @@ -273,11 +280,27 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res saveResiCategoryChangedRecord(categoryChangedResis); saveTransferedResiRecord(transferedResis); + hasErrorRows = hasErrorRows(); + try { - downLoadResults(response); + // todo 做了导入记录之后，这里就要判断，没有错误就不生成文件了 + downLoadResults(hasErrorRows, response); } catch (IOException e) { log.error("【导入IC居民附加信息】下载导入结果信息失败:{}", ExceptionUtils.getErrorStackTrace(e)); } + + // 更新上传记录 + if (hasErrorRows) { + importTaskService.finish(importTaskId, ImportTaskConstants.PROCESS_STATUS_FINISHED_FAIL, loginUserId, + null, null); + } else { + importTaskService.finish(importTaskId, ImportTaskConstants.PROCESS_STATUS_FINISHED_SUCCESS, loginUserId, + null, null); + } + + } catch (Exception e) { + // 抛出，让外层捕获处理，记录 + throw e; } finally { // 清空rediskey try { @@ -289,7 +312,6 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res } // 清空线程变量 - skipedRows.remove(); errorRows.remove(); newlyOrChangedResi.remove(); resiCategoryColumnNameAndLabel.remove(); @@ -297,6 +319,22 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res } } + /** + * 是否有错误行 + * @return + */ + private Boolean hasErrorRows() { + boolean hasError = false; + Map> tableAndErrorRows = errorRows.get(); + for (Map.Entry> entry:tableAndErrorRows.entrySet()) { + if (entry.getValue().size() != 0) { + hasError = true; + } + } + + return hasError; + } + /** * threadLocal初始化 */ @@ -310,7 +348,6 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res for (IcResiUserTableEnum e : IcResiUserTableEnum.values()) { skipedRowsMap.put(e.getTableName(), new LinkedList<>()); } - skipedRows.set(skipedRowsMap); // 错误信息 Map> errorRowsMap = new LinkedHashMap<>(); @@ -1000,19 +1037,11 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res * @param response * @throws IOException */ - public void downLoadResults(HttpServletResponse response) throws IOException { + public void downLoadResults(Boolean hasErrorRows, HttpServletResponse response) throws IOException { String fileName; // 判断是否有错误信息，以确定文件名 - boolean hasError = false; - Map> tableAndErrorRows = errorRows.get(); - for (Map.Entry> entry:tableAndErrorRows.entrySet()) { - if (entry.getValue().size() != 0) { - hasError = true; - } - } - - if (hasError) { + if (hasErrorRows) { fileName = "导入失败条目清单.xls"; } else { fileName = "导入成功.xls"; diff --git a/epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/ImportTaskServiceImpl.java b/epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/ImportTaskServiceImpl.java new file mode 100644 index 0000000000..d80ee5d9a6 --- /dev/null +++ b/epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/ImportTaskServiceImpl.java @@ -0,0 +1,57 @@ +package com.epmet.service.impl; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.epmet.constant.ImportTaskConstants; +import com.epmet.dao.ImportTaskDao; +import com.epmet.entity.ImportTaskEntity; +import com.epmet.service.ImportTaskService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.Date; + +/** + * + * + * @author generator generator@elink-cn.com + * @since v1.0.0 2022-02-15 + */ +@Service +public class ImportTaskServiceImpl implements ImportTaskService { + + @Autowired + private ImportTaskDao importRecordDao; + + /** + * 该用户，该业务类型，是否有正在处理的导入任务 + * @param operatorId 操作者ID + * @param bizType 业务类型。resi：居民 + * @return + */ + @Override + public boolean existsProcessingTask(String operatorId, String bizType) { + LambdaQueryWrapper query = new LambdaQueryWrapper<>(); + query.eq(ImportTaskEntity::getOperatorId, operatorId); + query.eq(ImportTaskEntity::getBizType, bizType); + query.eq(ImportTaskEntity::getProcessStatus, ImportTaskConstants.PROCESS_STATUS_PROCESSING); + + return importRecordDao.selectCount(query) > 0; + } + + @Override + public String createProcessTask(String operatorId, String bizType) { + ImportTaskEntity importRecord = new ImportTaskEntity(); + importRecord.setProcessStatus(ImportTaskConstants.PROCESS_STATUS_PROCESSING); + importRecord.setOperatorId(operatorId); + importRecord.setBizType(bizType); + importRecord.setStartTime(new Date()); + + importRecordDao.insert(importRecord); + return importRecord.getId(); + } + + @Override + public Boolean finish(String taskId, String processStatus, String operatorId, String resultDescFile, String resultDesc) { + return importRecordDao.finish(taskId, processStatus, operatorId, resultDesc, resultDescFile) > 0; + } +} \ No newline at end of file diff --git a/epmet-user/epmet-user-server/src/main/resources/mapper/ImportTaskDao.xml b/epmet-user/epmet-user-server/src/main/resources/mapper/ImportTaskDao.xml new file mode 100644 index 0000000000..edc56fa5c7 --- /dev/null +++ b/epmet-user/epmet-user-server/src/main/resources/mapper/ImportTaskDao.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + update import_task + set PROCESS_STATUS = #{process_status}, + RESULT_DESC = #{resultDesc}, + RESULT_DESC_FILE = #{resultDescFile}, + UPDATED_BY=#{operatorId}, + UPDATED_TIME=NOW() + where ID=#{taskId} + and PROCESS_STATUS = 'processing' + + \ No newline at end of file From cb3839b821bfd2d346b2540ae7cf32cf753516f5 Mon Sep 17 00:00:00 2001 From: wangxianzhang Date: Wed, 16 Feb 2022 20:11:04 +0800 Subject: [PATCH 5/5] =?UTF-8?q?=E4=BF=AE=E6=94=B9=EF=BC=9A=20-=20=E5=B1=85?= =?UTF-8?q?=E6=B0=91=E5=AF=BC=E5=85=A5=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/epmet/service/impl/IcResiUserImportServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/IcResiUserImportServiceImpl.java b/epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/IcResiUserImportServiceImpl.java index 4ff0a8abaa..18cd3cd6ec 100644 --- a/epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/IcResiUserImportServiceImpl.java +++ b/epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/IcResiUserImportServiceImpl.java @@ -285,7 +285,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res try { // todo 做了导入记录之后，这里就要判断，没有错误就不生成文件了 downLoadResults(hasErrorRows, response); - } catch (IOException e) { + } catch (Exception e) { log.error("【导入IC居民附加信息】下载导入结果信息失败:{}", ExceptionUtils.getErrorStackTrace(e)); }