diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/evaluationindex/extract/FactOriginGroupMainDailyDao.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/evaluationindex/extract/FactOriginGroupMainDailyDao.java index 5bd0c89103..12e6151f3f 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/evaluationindex/extract/FactOriginGroupMainDailyDao.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/evaluationindex/extract/FactOriginGroupMainDailyDao.java @@ -61,6 +61,15 @@ public interface FactOriginGroupMainDailyDao extends BaseDao + * @author wangc + * @date 2020.11.17 09:56 + */ + List selectAllGroupByCustomerId(@Param("customerId") String customerId); + /** * @param customerId * @return diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/evaluationindex/screen/ScreenDifficultyDataDao.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/evaluationindex/screen/ScreenDifficultyDataDao.java index 2b66e785bb..3d90291cba 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/evaluationindex/screen/ScreenDifficultyDataDao.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/evaluationindex/screen/ScreenDifficultyDataDao.java @@ -91,4 +91,13 @@ public interface ScreenDifficultyDataDao extends BaseDao list); int insertBatchImg(@Param("list") List list); + + /** + * @Description 根据客户Id查出全部的难点赌点的项目Id和网格Id,因为需要比对全部的信息然后删除因为删除网格而导致的垃圾数据 + * @param customerId + * @return java.util.List + * @author wangc + * @date 2020.11.17 09:33 + */ + List selectAllDifficultyByCustomerId(@Param("customerId")String customerId); } diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/todata/impl/GroupExtractServiceImpl.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/todata/impl/GroupExtractServiceImpl.java index cd04bc123e..dedb5bc3a3 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/todata/impl/GroupExtractServiceImpl.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/todata/impl/GroupExtractServiceImpl.java @@ -66,7 +66,7 @@ public class GroupExtractServiceImpl implements GroupExtractService { /** * @param param * @return - * @Description 业务抽取 - 小组祥光 + * @Description 业务抽取 - 小组 * @author wangc * @date 2020.09.18 21:07 **/ @@ -88,22 +88,31 @@ public class GroupExtractServiceImpl implements GroupExtractService { List memberList = new LinkedList<>(); if (!CollectionUtils.isEmpty(originGroupData)) { List gridList = dimGridService.getGridAttributes(param.getCustomerId(),originGroupData.stream().map(FactOriginGroupMainDailyDTO::getGridId).distinct().collect(Collectors.toList())); - List missingGroups = new LinkedList<>(); + if (!CollectionUtils.isEmpty(gridList)) { Map gridMap = gridList.stream().collect(Collectors.toMap(GridAttributesResultDTO::getGridId, Function.identity(), (key1, key2) -> key2)); - + //剔除本次要更新数据中的垃圾数据 for(Iterator iter = originGroupData.iterator(); + iter.hasNext();){ + FactOriginGroupMainDailyDTO pointer = iter.next(); + if(null == gridMap.get(pointer.getGridId()))iter.remove(); + + } + //记录全量中的垃圾数 + List missingGroups = new LinkedList<>(); + List population = factOriginGroupMainDailyDao.selectAllGroupByCustomerId(param.getCustomerId()); + if(!CollectionUtils.isEmpty(population)){ + for(Iterator iter = population.iterator(); iter.hasNext();){ FactOriginGroupMainDailyDTO pointer = iter.next(); if(null == gridMap.get(pointer.getGridId())){ missingGroups.add(pointer.getId()); - iter.remove(); } } - - originGroupData.forEach(group -> { + } + originGroupData.forEach(group -> { GridAttributesResultDTO attr = gridMap.get(group.getGridId()); if (null != attr) { group.setAgencyId(attr.getAgencyId()); diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenGrassrootsGovernDataAbsorptionServiceImpl.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenGrassrootsGovernDataAbsorptionServiceImpl.java index 8a60c6076b..f81c2e7259 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenGrassrootsGovernDataAbsorptionServiceImpl.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenGrassrootsGovernDataAbsorptionServiceImpl.java @@ -126,22 +126,7 @@ public class ScreenGrassrootsGovernDataAbsorptionServiceImpl implements ScreenGr }); } - - //6.存入数据库 - //不按照时间删除,每次插入之前将该客户下的所有历史数据清空 - //过滤 统计维度中没有的数据 - Set gridIdSet = new HashSet<>(); - List newInsertList = registeredUsers.stream() - .filter(o -> { - if (StringUtils.isBlank(o.getGridName())) { - gridIdSet.add(o.getGridId()); - return false; - } else { - return true; - } - }).collect(Collectors.toList()); - log.warn("userScoreDataHub grids:{} not in DimGrid", JSON.toJSONString(gridIdSet)); - screenPartyUserRankDataService.dataClean(newInsertList, param.getCustomerId()); + screenPartyUserRankDataService.dataClean(registeredUsers, param.getCustomerId()); } /** @@ -160,43 +145,51 @@ public class ScreenGrassrootsGovernDataAbsorptionServiceImpl implements ScreenGr //查询数据 List difficulties = factOriginProjectMainDailyService.getDifficultyBaseInfo(param.getCustomerId(),projectService.getOvertimeProjectByParameter(param.getCustomerId(),existed)); if(CollectionUtils.isEmpty(difficulties)) return; - //剔除垃圾数据 + //2.查询出客户下网格的相关信息 List gridList = customerGridService.queryGridInfoList(param.getCustomerId()); Map gridMap = gridList.stream().collect(Collectors.toMap(GridInfoDTO :: getGridId,a -> a,(o, n) -> o)); - List missing = new LinkedList<>(); + + //剔除垃圾数据(本次需要更新的数据集) for(Iterator iter = difficulties.iterator(); iter.hasNext();){ ScreenDifficultyDataEntity pointer = iter.next(); if(null == gridMap.get(pointer.getOrgId())){ - missing.add(pointer.getEventId()); iter.remove(); } } + //记录垃圾数(全量) + List missing = new LinkedList<>(); + List population = screenDifficultyDataService.getAllDifficultyByCustomerId(param.getCustomerId()); + if(!CollectionUtils.isEmpty(population)){ + for(Iterator iter = population.iterator(); iter.hasNext();){ + ScreenDifficultyDataEntity pointer = iter.next(); + if(null == gridMap.get(pointer.getOrgId())){ + missing.add(pointer.getEventId()); + } + } + } List projectIds = difficulties.stream().map(ScreenDifficultyDataEntity :: getEventId).distinct().collect(Collectors.toList()); //最近一次操作 Map latestOperationMap = projectProcessService.getLatestOperation(projectIds,param.getCustomerId()); - - boolean isOperationNull = CollectionUtils.isEmpty(latestOperationMap) ? true : false; - //图片 List projectSourceMap = factOriginProjectMainDailyService.getNewProject(param.getCustomerId(),projectIds); Map> imgMap = topicService.getTopicImgs(projectSourceMap); - boolean isImgNull = CollectionUtils.isEmpty(imgMap) ? true : false; + Map contentMap = topicService.getTopicContent(projectSourceMap); - boolean isContentNull = CollectionUtils.isEmpty(contentMap) ? true : false; + difficulties.forEach( diff -> { - if(!isImgNull) { + if(!CollectionUtils.isEmpty(imgMap)) { List figureList = imgMap.get(diff.getEventId()); diff.setEventImgUrl(CollectionUtils.isEmpty(figureList) ? "" : figureList.get(NumConstant.ZERO).getEventImgUrl()); } - if(!isContentNull) { + if(!CollectionUtils.isEmpty(contentMap)) { diff.setEventContent(contentMap.get(diff.getEventId())); } - if(!isOperationNull){ + if(! CollectionUtils.isEmpty(latestOperationMap)){ ProjectLatestOperationResultDTO oper = latestOperationMap.get(diff.getEventId()); if(null != oper){ diff.setLatestOperateDesc(oper.getOperationName()); diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/screen/ScreenDifficultyDataService.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/screen/ScreenDifficultyDataService.java index c133c88c84..5fc1af8dcd 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/screen/ScreenDifficultyDataService.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/screen/ScreenDifficultyDataService.java @@ -51,4 +51,12 @@ public interface ScreenDifficultyDataService extends BaseService difficulties, List imgs,List missing); + /** + * @Description 根据客户Id查出全部的难点赌点的项目Id和网格Id,因为需要比对全部的信息然后删除因为删除网格而导致的垃圾数据 + * @param customerId + * @return java.util.List + * @author wangc + * @date 2020.11.17 09:31 + */ + List getAllDifficultyByCustomerId(String customerId); } \ No newline at end of file diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/screen/ScreenPartyUserRankDataService.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/screen/ScreenPartyUserRankDataService.java index 99ce30dc0c..ea844971a9 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/screen/ScreenPartyUserRankDataService.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/screen/ScreenPartyUserRankDataService.java @@ -40,5 +40,4 @@ public interface ScreenPartyUserRankDataService extends BaseService dataList,String customerId); - } \ No newline at end of file diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/screen/impl/ScreenDifficultyDataServiceImpl.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/screen/impl/ScreenDifficultyDataServiceImpl.java index a371230024..89673d1e26 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/screen/impl/ScreenDifficultyDataServiceImpl.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/screen/impl/ScreenDifficultyDataServiceImpl.java @@ -68,4 +68,16 @@ public class ScreenDifficultyDataServiceImpl extends BaseServiceImpl + * @author wangc + * @date 2020.11.17 09:31 + */ + @Override + public List getAllDifficultyByCustomerId(String customerId) { + return baseDao.selectAllDifficultyByCustomerId(customerId); + } } \ No newline at end of file diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/evaluationindex/extract/FactOriginGroupMainDailyDao.xml b/epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/evaluationindex/extract/FactOriginGroupMainDailyDao.xml index c2c75be60f..2717a8cb5e 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/evaluationindex/extract/FactOriginGroupMainDailyDao.xml +++ b/epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/evaluationindex/extract/FactOriginGroupMainDailyDao.xml @@ -38,6 +38,18 @@ CUSTOMER_ID = #{customerId} + + + DELETE FROM fact_origin_group_main_daily diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/evaluationindex/screen/ScreenDifficultyDataDao.xml b/epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/evaluationindex/screen/ScreenDifficultyDataDao.xml index c56a4e85ab..100a2c29dd 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/evaluationindex/screen/ScreenDifficultyDataDao.xml +++ b/epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/evaluationindex/screen/ScreenDifficultyDataDao.xml @@ -219,4 +219,14 @@ +