抽取到大屏-难点赌点、热心市民，小组原始数据，兼容因为删除网格而造成垃圾数据的情况

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/evaluationindex/extract/FactOriginGroupMainDailyDao.java

@@ -61,6 +61,15 @@ public interface FactOriginGroupMainDailyDao extends BaseDao<FactOriginGroupMain
      **/
     int selectIfExist(@Param("customerId") String customerId);
 
+    /**
+     * @Description 根据用户Id查询所有组原始数据
+     * @param customerId
+     * @return java.util.List<com.epmet.dto.extract.FactOriginGroupMainDailyDTO>
+     * @author wangc
+     * @date 2020.11.17 09:56
+    */
+    List<FactOriginGroupMainDailyDTO> selectAllGroupByCustomerId(@Param("customerId") String customerId);
+
     /**
      * @param customerId
      * @return

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/evaluationindex/screen/ScreenDifficultyDataDao.java

@@ -91,4 +91,13 @@ public interface ScreenDifficultyDataDao extends BaseDao<ScreenDifficultyDataEnt
     int insertBatch(@Param("list") List<ScreenDifficultyDataEntity> list);
 
     int insertBatchImg(@Param("list") List<ScreenDifficultyImgDataEntity> list);
+
+    /**
+     * @Description 根据客户Id查出全部的难点赌点的项目Id和网格Id，因为需要比对全部的信息然后删除因为删除网格而导致的垃圾数据
+     * @param customerId
+     * @return java.util.List<com.epmet.entity.evaluationindex.screen.ScreenDifficultyDataEntity>
+     * @author wangc
+     * @date 2020.11.17 09:33
+     */
+    List<ScreenDifficultyDataEntity> selectAllDifficultyByCustomerId(@Param("customerId")String customerId);
 }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/todata/impl/GroupExtractServiceImpl.java

@@ -66,7 +66,7 @@ public class GroupExtractServiceImpl implements GroupExtractService {
     /**
      * @param param
      * @return
-     * @Description 业务抽取 - 小组祥光
+     * @Description 业务抽取 - 小组
      * @author wangc
      * @date 2020.09.18 21:07
      **/
@@ -88,22 +88,31 @@ public class GroupExtractServiceImpl implements GroupExtractService {
         List<ExtractGroupMemberActionRecordResultDTO> memberList = new LinkedList<>();
         if (!CollectionUtils.isEmpty(originGroupData)) {
         List<GridAttributesResultDTO> gridList = dimGridService.getGridAttributes(param.getCustomerId(),originGroupData.stream().map(FactOriginGroupMainDailyDTO::getGridId).distinct().collect(Collectors.toList()));
-        List<String> missingGroups = new LinkedList<>();
+
         if (!CollectionUtils.isEmpty(gridList)) {
             Map<String, GridAttributesResultDTO> gridMap =
                     gridList.stream().collect(Collectors.toMap(GridAttributesResultDTO::getGridId, Function.identity(), (key1, key2) -> key2));
 
-
+                //剔除本次要更新数据中的垃圾数据
                 for(Iterator<FactOriginGroupMainDailyDTO> iter = originGroupData.iterator();
+                    iter.hasNext();){
+                    FactOriginGroupMainDailyDTO pointer = iter.next();
+                    if(null == gridMap.get(pointer.getGridId()))iter.remove();
+
+                }
+            //记录全量中的垃圾数
+            List<String> missingGroups = new LinkedList<>();
+            List<FactOriginGroupMainDailyDTO> population = factOriginGroupMainDailyDao.selectAllGroupByCustomerId(param.getCustomerId());
+            if(!CollectionUtils.isEmpty(population)){
+                for(Iterator<FactOriginGroupMainDailyDTO> iter = population.iterator();
                     iter.hasNext();){
                     FactOriginGroupMainDailyDTO pointer = iter.next();
                     if(null == gridMap.get(pointer.getGridId())){
                         missingGroups.add(pointer.getId());
-                        iter.remove();
                     }
                 }
-
+            }
-                originGroupData.forEach(group -> {
+            originGroupData.forEach(group -> {
                     GridAttributesResultDTO attr = gridMap.get(group.getGridId());
                     if (null != attr) {
                         group.setAgencyId(attr.getAgencyId());

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenGrassrootsGovernDataAbsorptionServiceImpl.java

@@ -126,22 +126,7 @@ public class ScreenGrassrootsGovernDataAbsorptionServiceImpl implements ScreenGr
             });
 
         }
-
+        screenPartyUserRankDataService.dataClean(registeredUsers, param.getCustomerId());
-        //6.存入数据库
-        //不按照时间删除，每次插入之前将该客户下的所有历史数据清空
-        //过滤 统计维度中没有的数据
-        Set<String> gridIdSet = new HashSet<>();
-        List<ScreenPartyUserRankDataEntity> newInsertList = registeredUsers.stream()
-                .filter(o -> {
-                    if (StringUtils.isBlank(o.getGridName())) {
-                        gridIdSet.add(o.getGridId());
-                        return false;
-                    } else {
-                        return true;
-                    }
-                }).collect(Collectors.toList());
-        log.warn("userScoreDataHub grids:{} not in DimGrid", JSON.toJSONString(gridIdSet));
-        screenPartyUserRankDataService.dataClean(newInsertList, param.getCustomerId());
     }
 
     /**
@@ -160,43 +145,51 @@ public class ScreenGrassrootsGovernDataAbsorptionServiceImpl implements ScreenGr
         //查询数据
         List<ScreenDifficultyDataEntity> difficulties = factOriginProjectMainDailyService.getDifficultyBaseInfo(param.getCustomerId(),projectService.getOvertimeProjectByParameter(param.getCustomerId(),existed));
         if(CollectionUtils.isEmpty(difficulties)) return;
-        //剔除垃圾数据
+
         //2.查询出客户下网格的相关信息
         List<GridInfoDTO> gridList = customerGridService.queryGridInfoList(param.getCustomerId());
         Map<String,GridInfoDTO> gridMap = gridList.stream().collect(Collectors.toMap(GridInfoDTO :: getGridId,a -> a,(o, n) -> o));
-        List<String> missing = new LinkedList<>();
+
+        //剔除垃圾数据(本次需要更新的数据集)
         for(Iterator<ScreenDifficultyDataEntity> iter = difficulties.iterator(); iter.hasNext();){
             ScreenDifficultyDataEntity pointer = iter.next();
             if(null == gridMap.get(pointer.getOrgId())){
-                missing.add(pointer.getEventId());
                 iter.remove();
             }
         }
+        //记录垃圾数(全量)
+        List<String> missing = new LinkedList<>();
+        List<ScreenDifficultyDataEntity> population = screenDifficultyDataService.getAllDifficultyByCustomerId(param.getCustomerId());
+        if(!CollectionUtils.isEmpty(population)){
+            for(Iterator<ScreenDifficultyDataEntity> iter = population.iterator(); iter.hasNext();){
+                ScreenDifficultyDataEntity pointer = iter.next();
+                if(null == gridMap.get(pointer.getOrgId())){
+                    missing.add(pointer.getEventId());
+                }
+            }
+        }
 
 
         List<String> projectIds = difficulties.stream().map(ScreenDifficultyDataEntity :: getEventId).distinct().collect(Collectors.toList());
         //最近一次操作
         Map<String, ProjectLatestOperationResultDTO> latestOperationMap = projectProcessService.getLatestOperation(projectIds,param.getCustomerId());
-
-        boolean isOperationNull = CollectionUtils.isEmpty(latestOperationMap) ? true : false;
-
         //图片
         List<ProjectSourceMapFormDTO> projectSourceMap = factOriginProjectMainDailyService.getNewProject(param.getCustomerId(),projectIds);
 
         Map<String,List<ScreenDifficultyImgDataEntity>> imgMap =
                 topicService.getTopicImgs(projectSourceMap);
-        boolean isImgNull = CollectionUtils.isEmpty(imgMap) ? true : false;
+
         Map<String,String> contentMap = topicService.getTopicContent(projectSourceMap);
-        boolean isContentNull = CollectionUtils.isEmpty(contentMap) ? true : false;
+
         difficulties.forEach( diff -> {
-            if(!isImgNull) {
+            if(!CollectionUtils.isEmpty(imgMap)) {
                 List<ScreenDifficultyImgDataEntity> figureList = imgMap.get(diff.getEventId());
                 diff.setEventImgUrl(CollectionUtils.isEmpty(figureList) ? "" : figureList.get(NumConstant.ZERO).getEventImgUrl());
             }
-            if(!isContentNull) {
+            if(!CollectionUtils.isEmpty(contentMap)) {
                 diff.setEventContent(contentMap.get(diff.getEventId()));
             }
-            if(!isOperationNull){
+            if(! CollectionUtils.isEmpty(latestOperationMap)){
                 ProjectLatestOperationResultDTO oper = latestOperationMap.get(diff.getEventId());
                 if(null != oper){
                     diff.setLatestOperateDesc(oper.getOperationName());

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/screen/ScreenDifficultyDataService.java

@@ -51,4 +51,12 @@ public interface ScreenDifficultyDataService extends BaseService<ScreenDifficult
     */
     void dataClean(String customerId,List<ScreenDifficultyDataEntity> difficulties, List<ScreenDifficultyImgDataEntity> imgs,List<String> missing);
 
+    /**
+     * @Description 根据客户Id查出全部的难点赌点的项目Id和网格Id，因为需要比对全部的信息然后删除因为删除网格而导致的垃圾数据
+     * @param customerId
+     * @return java.util.List<com.epmet.entity.evaluationindex.screen.ScreenDifficultyDataEntity>
+     * @author wangc
+     * @date 2020.11.17 09:31
+     */
+    List<ScreenDifficultyDataEntity> getAllDifficultyByCustomerId(String customerId);
 }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/screen/ScreenPartyUserRankDataService.java

@@ -40,5 +40,4 @@ public interface ScreenPartyUserRankDataService extends BaseService<ScreenPartyU
      * @date 2020.09.27 09:44
     **/
     void dataClean(List<ScreenPartyUserRankDataEntity> dataList,String customerId);
-
 }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/screen/impl/ScreenDifficultyDataServiceImpl.java

@@ -68,4 +68,16 @@ public class ScreenDifficultyDataServiceImpl extends BaseServiceImpl<ScreenDiffi
         }
         baseDao.updateTime(customerId, DateUtils.getBeforeNDay(NumConstant.ONE));
     }
+
+    /**
+     * @Description 根据客户Id查出全部的难点赌点的项目Id和网格Id，因为需要比对全部的信息然后删除因为删除网格而导致的垃圾数据
+     * @param customerId
+     * @return java.util.List<com.epmet.entity.evaluationindex.screen.ScreenDifficultyDataEntity>
+     * @author wangc
+     * @date 2020.11.17 09:31
+     */
+    @Override
+    public List<ScreenDifficultyDataEntity> getAllDifficultyByCustomerId(String customerId) {
+        return baseDao.selectAllDifficultyByCustomerId(customerId);
+    }
 }

epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/evaluationindex/extract/FactOriginGroupMainDailyDao.xml

@@ -38,6 +38,18 @@
         CUSTOMER_ID = #{customerId}
     </select>
 
+    <!-- 根据用户Id查询所有组原始数据 -->
+    <select id="selectAllGroupByCustomerId" resultType="com.epmet.dto.extract.FactOriginGroupMainDailyDTO">
+        SELECT
+            id,grid_id
+        FROM
+            fact_origin_group_main_daily
+        WHERE
+            del_flag = '0'
+        AND
+            customer_id = #{customerId}
+    </select>
+
     <delete id="deleteBatchByCustomerId" >
         DELETE FROM
         fact_origin_group_main_daily

epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/evaluationindex/screen/ScreenDifficultyDataDao.xml

@@ -219,4 +219,14 @@
         </foreach>
     </insert>
 
+    <select id="selectAllDifficultyByCustomerId" resultType="com.epmet.entity.evaluationindex.screen.ScreenDifficultyDataEntity">
+        SELECT
+            org_id,
+            event_id
+        FROM
+            screen_difficulty_data
+        WHERE
+            del_flag = '0'
+            AND customer_id = #{customerId}
+    </select>
 </mapper>