elink-star
/
ShibeiWorkPlatform
12
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

修改： 

1.居民信息导入：将类别变更数据放到redis中，减少内存占用
dev
wangxianzhang 4 years ago
parent
4922ee5707
commit
f7c5e8ceb6
5 changed files with 126 additions and 48 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 33
      epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/redis/RedisKeys.java
  2. 13
      epmet-user/epmet-user-server/src/main/java/com/epmet/bean/ResiImportCategoryData.java
  3. 7
      epmet-user/epmet-user-server/src/main/java/com/epmet/bean/ResiImportResiCategoryChangedCache.java
  4. 4
      epmet-user/epmet-user-server/src/main/java/com/epmet/excel/handler/IcResiImportDynamicExcelListener.java
  5. 101
      epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/IcResiUserImportServiceImpl.java

33
epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/redis/RedisKeys.java
View File

@ -640,4 +640,37 @@ public class RedisKeys {
public static String getHouseInfoCacheKey(String houseId,String customerId){
return rootPrefix.concat("house:").concat(customerId).concat(":").concat(houseId);
}
/**
* 居民导入-缓存目录key
* @param importTag
* @return
*/
public static String icResiImportBaseKey(String importTag) {
return rootPrefix.concat("resi:").concat("import:").concat(importTag);
}
/**
* 居民导入-数据分类的key
* add：新增居民
* category：类别变更
* transfer：调动
* @param importTag
* @param type
* @return
*/
public static String icResiImportTypeKey(String importTag, String type) {
return icResiImportBaseKey(importTag).concat(":").concat(type);
}
/**
* 居民导入-详细信息key
* @param importTag
* @param type
* @param resiId
* @return
*/
public static String icResiImportResiCategoryKey(String importTag, String type, String resiId) {
return icResiImportTypeKey(importTag, type).concat(":").concat(resiId);
}
}

13
epmet-user/epmet-user-server/src/main/java/com/epmet/bean/ResiImportCategoryData.java
View File

@ -14,6 +14,7 @@ import java.util.Map;
@Data
public class ResiImportCategoryData {
private String resiId;
private String agencyId;
private String gridId;
private String villageId;
@ -30,7 +31,8 @@ public class ResiImportCategoryData {
public ResiImportCategoryData() {
}
public ResiImportCategoryData(String agencyId, String gridId, String villageId, String buildId, String unitId, String homeId, Map<String, String> categories) {
public ResiImportCategoryData(String resiId, String agencyId, String gridId, String villageId, String buildId, String unitId, String homeId, Map<String, String> categories) {
this.resiId = resiId;
this.agencyId = agencyId;
this.gridId = gridId;
this.villageId = villageId;
@ -39,13 +41,4 @@ public class ResiImportCategoryData {
this.homeId = homeId;
this.categories = categories;
}
public ResiImportCategoryData(String agencyId, String gridId, String villageId, String buildId, String unitId, String homeId) {
this.agencyId = agencyId;
this.gridId = gridId;
this.villageId = villageId;
this.buildId = buildId;
this.unitId = unitId;
this.homeId = homeId;
}
}

7
epmet-user/epmet-user-server/src/main/java/com/epmet/bean/ResiImportResiCategoryChangedCache.java
View File

@ -11,11 +11,16 @@ import java.util.Map;
@Data
public class ResiImportResiCategoryChangedCache {
/**
* 导入的tag，用来标记唯一一次导入操作。
*/
private String importTag;
/**
* 新增居民
* Map<resiId:<ResiImportCategoryData>>
*/
private Map<String, ResiImportCategoryData> newResis = new HashMap<>();
// private Map<String, ResiImportCategoryData> newResis = new HashMap<>();
/**
* 调动的居民

4
epmet-user/epmet-user-server/src/main/java/com/epmet/excel/handler/DynamicEasyExcelListener.java → epmet-user/epmet-user-server/src/main/java/com/epmet/excel/handler/IcResiImportDynamicExcelListener.java
View File

@ -17,7 +17,7 @@ import java.util.stream.Collectors;
* 创建一个监听器
*/
@Slf4j
public class DynamicEasyExcelListener extends AnalysisEventListener<Map<Integer, String>> {
public class IcResiImportDynamicExcelListener extends AnalysisEventListener<Map<Integer, String>> {
/**
* 一次性导入的条数
@ -89,7 +89,7 @@ public class DynamicEasyExcelListener extends AnalysisEventListener<Map<Integer,
// Map<Integer, String> dataList = new HashMap<>();
public DynamicEasyExcelListener(IcResiUserImportServiceImpl icResiUserImportService, String customerId,
public IcResiImportDynamicExcelListener(IcResiUserImportServiceImpl icResiUserImportService, String customerId,
String currentUserId, String currUserAgencyId,
String currUserAgencyPids, Boolean isPrimary, String tableName,
List<FormItemResult> formItemList, Integer headRowNumber) {

101
epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/IcResiUserImportServiceImpl.java
View File

@ -3,8 +3,10 @@ package com.epmet.service.impl;
import cn.afterturn.easypoi.excel.ExcelExportUtil;
import cn.afterturn.easypoi.excel.annotation.Excel;
import cn.afterturn.easypoi.excel.entity.ExportParams;
import cn.hutool.core.bean.BeanUtil;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.EasyExcelFactory;
import com.baomidou.mybatisplus.core.toolkit.IdWorker;
import com.epmet.bean.ResiImportCategoryData;
import com.epmet.bean.ResiImportResiCategoryChangedCache;
import com.epmet.bean.ResiImportChangedData;
@ -16,8 +18,11 @@ import com.epmet.commons.tools.exception.EpmetErrorCode;
import com.epmet.commons.tools.exception.ExceptionUtils;
import com.epmet.commons.tools.exception.RenException;
import com.epmet.commons.tools.feign.ResultDataResolver;
import com.epmet.commons.tools.redis.RedisKeys;
import com.epmet.commons.tools.redis.RedisUtils;
import com.epmet.commons.tools.redis.common.CustomerStaffRedis;
import com.epmet.commons.tools.security.user.LoginUserUtil;
import com.epmet.commons.tools.utils.ConvertUtils;
import com.epmet.commons.tools.utils.Result;
import com.epmet.dao.IcResiUserDao;
import com.epmet.dao.IcUserChangeDetailedDao;
@ -31,7 +36,7 @@ import com.epmet.entity.IcUserChangeDetailedEntity;
import com.epmet.entity.IcUserChangeRecordEntity;
import com.epmet.entity.IcUserTransferRecordEntity;
import com.epmet.enums.IcResiUserTableEnum;
import com.epmet.excel.handler.DynamicEasyExcelListener;
import com.epmet.excel.handler.IcResiImportDynamicExcelListener;
import com.epmet.feign.EpmetAdminOpenFeignClient;
import com.epmet.feign.EpmetUserOpenFeignClient;
import com.epmet.feign.GovOrgOpenFeignClient;
@ -118,6 +123,9 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res
@Autowired
private UserService userService;
@Autowired
private RedisUtils redisUtils;
/**
* 字表中不需要的列
@ -253,12 +261,12 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res
}
// 执行人员类别变更记录
Map<String, ResiImportCategoryData> newResis = newlyOrChangedResi.get().getNewResis();
// Map<String, ResiImportCategoryData> newResis = newlyOrChangedResi.get().getNewResis();
Map<String, ResiImportChangedData> categoryChangedResis = newlyOrChangedResi.get().getCategoryChangedResis();
Map<String, ResiImportChangedData> transferedResis = newlyOrChangedResi.get().getTransferedResis();
//保存调动或者变更记录
saveNewResiCategoryRecord(newResis);
saveNewResiCategoryRecord();
saveResiCategoryChangedRecord(categoryChangedResis);
saveTransferedResiRecord(transferedResis);
@ -268,6 +276,16 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res
log.error("【导入IC居民附加信息】下载导入结果信息失败:{}", ExceptionUtils.getErrorStackTrace(e));
}
} finally {
// 清空rediskey
try {
String importTag = newlyOrChangedResi.get().getImportTag();
redisUtils.deleteByPattern(RedisKeys.icResiImportBaseKey(importTag).concat("*"));
} catch (Exception e) {
String errorMsg = ExceptionUtils.getErrorStackTrace(e);
log.error("【导入IC居民附加信息】清空redis 类别缓存出错:{}", errorMsg);
}
// 清空线程变量
skipedRows.remove();
errorRows.remove();
newlyOrChangedResi.remove();
@ -280,6 +298,10 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res
* threadLocal初始化
*/
private void initImportThreadLocal(String customerId) {
// 生成importTag，用于标记唯一一次导入操作，导入完成之后，用来删除redis里面临时的key
String operatorId = loginUserUtil.getLoginUserId();
String importTag = operatorId + System.currentTimeMillis();
// 跳过的,不导入的行
Map<String, List<SkipedRow>> skipedRowsMap = new LinkedHashMap<>();
for (IcResiUserTableEnum e : IcResiUserTableEnum.values()) {
@ -296,6 +318,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res
// 居民类别信息
ResiImportResiCategoryChangedCache c = new ResiImportResiCategoryChangedCache();
c.setImportTag(importTag);
newlyOrChangedResi.set(c);
List<ResiCategoryItemResultDTO> resiCategoryItems = getResultDataOrThrowsException(operCustomizeOpenFeignClient.listResiCategoryItems(customerId),
@ -323,7 +346,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res
private void importIcResiBaseInfoFromExcel(List<FormItemResult> formItemList, String excelPathName, int sheetNo, int headRowNumber, String currUserAgencyId, String currUserAgencyPids, String currentUserId,
String tableName, String customerId) {
DynamicEasyExcelListener readListener = new DynamicEasyExcelListener(this, customerId, currentUserId, currUserAgencyId, currUserAgencyPids,
IcResiImportDynamicExcelListener readListener = new IcResiImportDynamicExcelListener(this, customerId, currentUserId, currUserAgencyId, currUserAgencyPids,
true, tableName, formItemList, headRowNumber);
EasyExcel.read(new File(excelPathName)).registerReadListener(readListener).headRowNumber(headRowNumber).sheet(sheetNo).doRead();
}
@ -343,7 +366,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res
private void importIcResiExtraInfoFromExcel(List<FormItemResult> formItemList, String excelPathName, int sheetNo, int headRowNumber, String currUserAgencyId, String currentUserId,
String targetTableName, String customerId) {
DynamicEasyExcelListener readListener = new DynamicEasyExcelListener(this, customerId, currentUserId, currUserAgencyId, null,
IcResiImportDynamicExcelListener readListener = new IcResiImportDynamicExcelListener(this, customerId, currentUserId, currUserAgencyId, null,
false, targetTableName, formItemList, headRowNumber);
EasyExcelFactory.read(new File(excelPathName)).registerReadListener(readListener).headRowNumber(headRowNumber).sheet(sheetNo).doRead();
}
@ -415,8 +438,10 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res
}
} else {
// 新增居民
String resiId = IdWorker.getIdStr();
columnAndValues.put("CREATED_BY", currentUserId);
columnAndValues.put("ID", UUID.randomUUID().toString().replace("-", ""));
columnAndValues.put("ID", resiId);
icResiUserDao.add(tableName, columnAndValues);
// 过滤出本居民含有哪些类别
@ -425,14 +450,26 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res
.filter((categoryColumnName) -> "1".equals(columnAndValues.get(categoryColumnName)))
.collect(Collectors.toMap((k) -> k, (k) -> columnAndValues.get(k)));
newlyOrChangedResi.get().getNewResis().put(columnAndValues.get("ID"),
new ResiImportCategoryData(columnAndValues.get("AGENCY_ID"),
// newlyOrChangedResi.get().getNewResis().put(resiId,
// new ResiImportCategoryData(columnAndValues.get("AGENCY_ID"),
// columnAndValues.get("GRID_ID"),
// columnAndValues.get("VILLAGE_ID"),
// columnAndValues.get("BUILD_ID"),
// columnAndValues.get("UNIT_ID"),
// columnAndValues.get("HOME_ID"),
// resiCategories));
ResiImportCategoryData categoryData = new ResiImportCategoryData(resiId,
columnAndValues.get("AGENCY_ID"),
columnAndValues.get("GRID_ID"),
columnAndValues.get("VILLAGE_ID"),
columnAndValues.get("BUILD_ID"),
columnAndValues.get("UNIT_ID"),
columnAndValues.get("HOME_ID"),
resiCategories));
resiCategories);
redisUtils.hMSet(RedisKeys.icResiImportResiCategoryKey(newlyOrChangedResi.get().getImportTag(), "add", resiId), BeanUtil.beanToMap(categoryData));
}
} catch (Exception e) {
@ -1040,6 +1077,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res
}
ResiImportCategoryData oldOne = new ResiImportCategoryData(
existingResiMap.get("ID"),
existingResiMap.get("AGENCY_ID"),
existingResiMap.get("GRID_ID"),
existingResiMap.get("VILLAGE_ID"),
@ -1049,7 +1087,8 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res
existingResiCategories);
ResiImportCategoryData newOne = new ResiImportCategoryData(
existingResiMap.get("AGENCY_ID"),
newResiMap.get("ID"),
newResiMap.get("AGENCY_ID"),
newResiMap.get("GRID_ID"),
newResiMap.get("VILLAGE_ID"),
newResiMap.get("BUILD_ID"),
@ -1100,6 +1139,7 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res
}
ResiImportCategoryData oldOne = new ResiImportCategoryData(
existingResiMap.get("ID"),
existingResiMap.get("AGENCY_ID"),
existingResiMap.get("GRID_ID"),
existingResiMap.get("VILLAGE_ID"),
@ -1108,7 +1148,9 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res
existingResiMap.get("HOME_ID"),
oldCategories);
ResiImportCategoryData newOne = new ResiImportCategoryData(existingResiMap.get("AGENCY_ID"),
ResiImportCategoryData newOne = new ResiImportCategoryData(
newResiMap.get("ID"),
newResiMap.get("AGENCY_ID"),
newResiMap.get("GRID_ID"),
newResiMap.get("VILLAGE_ID"),
newResiMap.get("BUILD_ID"),
@ -1127,14 +1169,19 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res
private void saveSubTableInfoToCategoryChangedResiCache(String icResiId, String columnName, Map<String, String> resiInfoMap) {
ResiImportResiCategoryChangedCache cc = newlyOrChangedResi.get();
Map<String, ResiImportCategoryData> newResis = cc.getNewResis();
// Map<String, ResiImportCategoryData> newResis = cc.getNewResis();
Map<String, ResiImportChangedData> categoryChangedResis = cc.getCategoryChangedResis();
Map<String, ResiImportChangedData> transferedResis = cc.getTransferedResis();
ResiImportCategoryData newResi = newResis.get(icResiId);
if (newResi != null) {
// ResiImportCategoryData newResi = newResis.get(icResiId);
String userCateRedisKey = RedisKeys.icResiImportResiCategoryKey(newlyOrChangedResi.get().getImportTag(), "add", icResiId);
Map < String, Object > addUserMap = redisUtils.hGetAll(userCateRedisKey);
if (addUserMap != null) {
//说明是新增居民
newResi.getCategories().put(columnName, "1");
ResiImportCategoryData newResiCateData = ConvertUtils.mapToEntity(addUserMap, ResiImportCategoryData.class);
newResiCateData.getCategories().put(columnName, "1");
redisUtils.hMSet(userCateRedisKey, BeanUtil.beanToMap(newResiCateData));
return;
}
@ -1286,35 +1333,35 @@ public class IcResiUserImportServiceImpl implements IcResiUserImportService, Res
/**
* 保存新增居民类别记录
* @param newResis
*/
public void saveNewResiCategoryRecord(Map<String, ResiImportCategoryData> newResis) {
public void saveNewResiCategoryRecord() {
Date now = new Date();
Set<String> newResiKeys = redisUtils.keys(RedisKeys.icResiImportTypeKey(newlyOrChangedResi.get().getImportTag(), "add").concat("*"));
CustomerStaffInfoCacheResult operator = CustomerStaffRedis.getStaffInfo(loginUserUtil.getLoginUserCustomerId(), loginUserUtil.getLoginUserId());
for (Map.Entry<String, ResiImportCategoryData> resi : newResis.entrySet()) {
String resiId = resi.getKey();
ResiImportCategoryData data = resi.getValue();
for (String resiKey : newResiKeys) {
Map<String, Object> newResiInfoMap = redisUtils.hGetAll(resiKey);
ResiImportCategoryData newResiInfoObj = ConvertUtils.mapToEntity(newResiInfoMap, ResiImportCategoryData.class);
IcResiUserEntity resiInfo = icResiUserDao.selectById(resiId);
IcResiUserEntity resiInfo = icResiUserDao.selectById(newResiInfoObj.getResiId());
// 插入changeRecord
IcUserChangeRecordEntity changeRecord = fillChangeRecord(loginUserUtil.getLoginUserCustomerId(), resiId, resiInfo.getName(), null,
IcUserChangeRecordEntity changeRecord = fillChangeRecord(loginUserUtil.getLoginUserCustomerId(), newResiInfoObj.getResiId(), resiInfo.getName(), null,
loginUserUtil.getLoginUserId(), operator.getRealName(), "-", "-",
"add", "新增", "", now);
icUserChangeRecordDao.insert(changeRecord);
// 插入changeDetail
for (Map.Entry<String, String> column : data.getCategories().entrySet()) {
IcUserChangeDetailedEntity changedetail = fillChangeDetail(loginUserUtil.getLoginUserCustomerId(), resiId, changeRecord.getId(), data.getAgencyId(),
data.getGridId(), data.getVillageId(), data.getBuildId(), data.getUnitId(),
data.getHomeId(), "add", "新增", column.getKey(), 1, resiInfo.getPids());
for (Map.Entry<String, String> column : newResiInfoObj.getCategories().entrySet()) {
IcUserChangeDetailedEntity changedetail = fillChangeDetail(loginUserUtil.getLoginUserCustomerId(), newResiInfoObj.getResiId(), changeRecord.getId(), newResiInfoObj.getAgencyId(),
newResiInfoObj.getGridId(), newResiInfoObj.getVillageId(), newResiInfoObj.getBuildId(), newResiInfoObj.getUnitId(),
newResiInfoObj.getHomeId(), "add", "新增", column.getKey(), 1, resiInfo.getPids());
icUserChangeDetailedDao.insert(changedetail);
}
}
}
/**

Write Preview
Loading…
Cancel
Save