难点堵点 优化步骤1

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/DemoController.java

@@ -997,11 +997,6 @@ public class DemoController {
     @Autowired
     private ScreenGrassrootsGovernDataAbsorptionService ndddYhjfService;
 
-    @PostMapping("diffOld")
-    public Result diff(@RequestBody ScreenCentralZoneDataFormDTO param){
-        ndddYhjfService.difficultyDataHub(param);
-        return new Result();
-    }
     @PostMapping("testDiffNew")
     public Result testDiff(@RequestBody ScreenCentralZoneDataFormDTO param){
         return new Result().ok(ndddYhjfService.difficultyDataExtract(param));

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/evaluationindex/extract/FactOriginProjectMainDailyDao.java

@@ -26,10 +26,7 @@ import com.epmet.dto.indexcollect.result.CpcIndexCommonDTO;
 import com.epmet.dto.pingyin.result.*;
 import com.epmet.dto.screen.ScreenProjectDataDTO;
 import com.epmet.dto.screen.form.ProjectSourceMapFormDTO;
-import com.epmet.entity.evaluationindex.extract.FactOriginProjectMainDailyEntity;
+import com.epmet.entity.evaluationindex.extract.*;
-import com.epmet.entity.evaluationindex.extract.FactOriginProjectOrgPeriodDailyEntity;
-import com.epmet.entity.evaluationindex.extract.GovernGridTotalCommonDTO;
-import com.epmet.entity.evaluationindex.extract.GovernProjectInfoDTO;
 import com.epmet.entity.evaluationindex.screen.ScreenDifficultyDataEntity;
 import org.apache.ibatis.annotations.Mapper;
 import org.apache.ibatis.annotations.Param;
@@ -405,11 +402,11 @@ public interface FactOriginProjectMainDailyDao extends BaseDao<FactOriginProject
 	 *
 	 * @param customerId
 	 * @param dateId
-	 * @return java.util.List<com.epmet.entity.evaluationindex.extract.FactOriginProjectMainDailyEntity>
+	 * @return java.util.List<com.epmet.entity.evaluationindex.extract.FactOriginProjectMainAndPeriodDTO>
 	 * @author LiuJanJun
 	 * @date 2021/4/12 1:35 下午
 	 */
-	List<FactOriginProjectMainDailyEntity> getProjectForDiff(@Param("customerId") String customerId, @Param("dateId") String dateId);
+	List<FactOriginProjectMainAndPeriodDTO> getProjectForDiff(@Param("customerId") String customerId, @Param("offset") int offset, @Param("pageSize") int pageSize);
 
 	/**
 	 * desc: 获取难点堵点项目数据耗时
@@ -420,7 +417,7 @@ public interface FactOriginProjectMainDailyDao extends BaseDao<FactOriginProject
 	 * @author LiuJanJun
 	 * @date 2021/4/12 1:35 下午
 	 */
-	List<FactOriginProjectOrgPeriodDailyEntity> getProjectPeriodForDiff(@Param("customerId") String customerId, @Param("dateId") String dateId);
+	List<FactOriginProjectOrgPeriodDailyEntity> getProjectPeriodForDiff(@Param("customerId") String customerId, @Param("projectId") String projectId);
 
 	List<ScreenProjectDataDTO> selectResponseCount(@Param("customerId") String customerId);
 
@@ -551,4 +548,5 @@ public interface FactOriginProjectMainDailyDao extends BaseDao<FactOriginProject
 	 * @date 2021/7/5 14:49
 	 */
 	List<OrgStatisticsResultDTO> getMemberProjectCount(@Param("customerId") String customerId, @Param("dateId") String dateId);
+
 }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/evaluationindex/screen/ScreenDifficultyDataDao.java

@@ -19,7 +19,6 @@ package com.epmet.dao.evaluationindex.screen;
 
 import com.epmet.commons.mybatis.dao.BaseDao;
 import com.epmet.dto.screencoll.form.DifficultyDataDetailFormDTO;
-import com.epmet.dto.screencoll.form.DifficultyDataFormDTO;
 import com.epmet.entity.evaluationindex.screen.ScreenDifficultyDataEntity;
 import com.epmet.entity.evaluationindex.screen.ScreenDifficultyImgDataEntity;
 import org.apache.ibatis.annotations.Mapper;
@@ -56,7 +55,7 @@ public interface ScreenDifficultyDataDao extends BaseDao<ScreenDifficultyDataEnt
      **/
     void batchInsertDifficultyData(@Param("list") List<DifficultyDataDetailFormDTO> list, @Param("customerId")String customerId);
 
-    
+
     /**
      * @Description 根据customerId查询是否有难点赌点的数据
      * @param customerId
@@ -92,12 +91,4 @@ public interface ScreenDifficultyDataDao extends BaseDao<ScreenDifficultyDataEnt
 
     int insertBatchImg(@Param("list") List<ScreenDifficultyImgDataEntity> list);
 
-    /**
-     * @Description 根据客户Id查出全部的难点赌点的项目Id和网格Id，因为需要比对全部的信息然后删除因为删除网格而导致的垃圾数据
-     * @param customerId
-     * @return java.util.List<com.epmet.entity.evaluationindex.screen.ScreenDifficultyDataEntity>
-     * @author wangc
-     * @date 2020.11.17 09:33
-     */
-    List<ScreenDifficultyDataEntity> selectAllDifficultyByCustomerId(@Param("customerId")String customerId);
 }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/entity/evaluationindex/extract/FactOriginProjectMainAndPeriodDTO.java

@@ -0,0 +1,156 @@
+/**
+ * Copyright 2018 人人开源 https://www.renren.io
+ * <p>
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ * <p>
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * <p>
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package com.epmet.entity.evaluationindex.extract;
+
+import com.epmet.commons.mybatis.entity.BaseEpmetEntity;
+import lombok.Data;
+
+import java.util.List;
+
+/**
+ * 项目主表_日统计
+ *
+ * @author generator generator@elink-cn.com
+ * @since v1.0.0 2020-09-16
+ */
+@Data
+public class FactOriginProjectMainAndPeriodDTO extends BaseEpmetEntity {
+
+	private static final long serialVersionUID = 1L;
+
+    /**
+     * 项目的发布日期yyyyMMdd
+     */
+	private String dateId;
+
+    /**
+     * 周ID
+     */
+	private String weekId;
+
+    /**
+     * 月份ID
+     */
+	private String monthId;
+
+    /**
+     * 季度ID
+     */
+	private String quarterId;
+
+    /**
+     * 年度ID
+     */
+	private String yearId;
+
+    /**
+     * 客户ID
+     */
+	private String customerId;
+
+    /**
+     * 转议题用户ID
+     */
+	private String issueCreatorId;
+
+    /**
+     * 转为项目的议题ID
+     */
+	private String issueId;
+
+    /**
+     * 转为议题的话题ID
+     */
+	private String topicId;
+
+    /**
+     * 网格ID
+     */
+	private String gridId;
+
+    /**
+     * 上级ID
+     */
+	private String pid;
+
+	/**
+	 * 所有上级Id集合  (项目来源)
+	 * */
+	private String pids;
+
+    /**
+     * 组织ID（网格所属组织ID）
+     */
+	private String agencyId;
+
+	/**
+	 * 来源：议题issue
+	 */
+	private String origin;
+
+	/**
+	 * 来源ID
+	 */
+	private String originId;
+
+    /**
+     * 状态
+     */
+	private String projectStatus;
+
+	/**
+	 * resolved   unresolved
+	 * */
+	private String isResolved;
+
+    /**
+     * 创建话题用户的ID
+     */
+	private String topicCreatorId;
+
+    /**
+     * 是否是党员 1:是
+     */
+	private String isParty;
+
+    /**
+     * 是否超期 1:是
+     */
+	private String isOverdue;
+
+	/**
+	 * 是否满意 1:是
+	 */
+	private Integer isSatisfied;
+
+    /**
+     * 办结组织Ids（:隔开，有可能是社区id可能是网格id，无需区分级别，在统计时模糊查询）
+     */
+	private String finishOrgIds;
+
+	/**
+	 * 项目创建人（议题转项目或立项人）
+	 */
+	private String projectCreator;
+
+	/**
+	 * 项目处理时间节点
+	 */
+	private List<FactOriginProjectOrgPeriodDailyEntity> periodList;
+
+}

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/todata/FactOriginProjectMainDailyService.java

@@ -27,10 +27,7 @@ import com.epmet.dto.pingyin.result.*;
 import com.epmet.dto.screen.ScreenProjectDataDTO;
 import com.epmet.dto.screen.form.ProjectSourceMapFormDTO;
 import com.epmet.dto.screen.form.ScreenCentralZoneDataFormDTO;
-import com.epmet.entity.evaluationindex.extract.FactOriginProjectMainDailyEntity;
+import com.epmet.entity.evaluationindex.extract.*;
-import com.epmet.entity.evaluationindex.extract.FactOriginProjectOrgPeriodDailyEntity;
-import com.epmet.entity.evaluationindex.extract.GovernGridTotalCommonDTO;
-import com.epmet.entity.evaluationindex.extract.GovernProjectInfoDTO;
 import com.epmet.entity.evaluationindex.screen.ScreenDifficultyDataEntity;
 
 import java.util.List;
@@ -377,11 +374,13 @@ public interface FactOriginProjectMainDailyService extends BaseService<FactOrigi
      * desc: 条件获取项目数据
      *
      * @param param
+     * @param offset
+     * @param pageSize
      * @return java.util.List<com.epmet.dto.extract.FactOriginProjectMainDailyEntity>
      * @author LiuJanJun
      * @date 2021/4/12 11:07 上午
      */
-    List<FactOriginProjectMainDailyEntity> getProjectForDiff(ScreenCentralZoneDataFormDTO param);
+    List<FactOriginProjectMainAndPeriodDTO> getProjectAndPeriodForDiff(ScreenCentralZoneDataFormDTO param, int offset, int pageSize);
 
     /**
      * desc: 根据项目数据 获取相关连的项目耗时

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/todata/impl/FactOriginProjectMainDailyServiceImpl.java

@@ -36,10 +36,7 @@ import com.epmet.dto.result.CostDayResultDTO;
 import com.epmet.dto.screen.ScreenProjectDataDTO;
 import com.epmet.dto.screen.form.ProjectSourceMapFormDTO;
 import com.epmet.dto.screen.form.ScreenCentralZoneDataFormDTO;
-import com.epmet.entity.evaluationindex.extract.FactOriginProjectMainDailyEntity;
+import com.epmet.entity.evaluationindex.extract.*;
-import com.epmet.entity.evaluationindex.extract.FactOriginProjectOrgPeriodDailyEntity;
-import com.epmet.entity.evaluationindex.extract.GovernGridTotalCommonDTO;
-import com.epmet.entity.evaluationindex.extract.GovernProjectInfoDTO;
 import com.epmet.entity.evaluationindex.screen.ScreenDifficultyDataEntity;
 import com.epmet.feign.EpmetCommonServiceOpenFeignClient;
 import com.epmet.service.evaluationindex.extract.todata.FactOriginProjectMainDailyService;
@@ -527,8 +524,8 @@ public class FactOriginProjectMainDailyServiceImpl extends BaseServiceImpl<FactO
     }
 
     @Override
-    public List<FactOriginProjectMainDailyEntity> getProjectForDiff(ScreenCentralZoneDataFormDTO param) {
+    public List<FactOriginProjectMainAndPeriodDTO> getProjectAndPeriodForDiff(ScreenCentralZoneDataFormDTO param, int offset, int pageSize) {
-        return baseDao.getProjectForDiff(param.getCustomerId(),param.getDateId());
+        return baseDao.getProjectForDiff(param.getCustomerId(),offset,pageSize);
     }
 
     @Override

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/ScreenGrassrootsGovernDataAbsorptionService.java

@@ -18,15 +18,6 @@ public interface ScreenGrassrootsGovernDataAbsorptionService {
     **/
     void userScoreDataHub(ScreenCentralZoneDataFormDTO param);
 
-    /**
-     * @Description 难点赌点数据中转站
-     * @param param
-     * @return
-     * @author wangc
-     * @date 2020.09.25 10:00
-    **/
-    void difficultyDataHub(ScreenCentralZoneDataFormDTO param);
-
     /**
      * desc: 新的难点堵点抽取
      *

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenGrassrootsGovernDataAbsorptionServiceImpl.java

@@ -8,12 +8,10 @@ import com.epmet.constant.ProjectConstant;
 import com.epmet.dto.ProjectCategoryDTO;
 import com.epmet.dto.ProjectDTO;
 import com.epmet.dto.org.GridInfoDTO;
-import com.epmet.dto.project.result.ProjectLatestOperationResultDTO;
-import com.epmet.dto.screen.form.ProjectSourceMapFormDTO;
 import com.epmet.dto.screen.form.ScreenCentralZoneDataFormDTO;
 import com.epmet.dto.topic.TopicAttachmentDTO;
 import com.epmet.dto.topic.result.ResiTopicAndImgResultDTO;
-import com.epmet.entity.evaluationindex.extract.FactOriginProjectMainDailyEntity;
+import com.epmet.entity.evaluationindex.extract.FactOriginProjectMainAndPeriodDTO;
 import com.epmet.entity.evaluationindex.extract.FactOriginProjectOrgPeriodDailyEntity;
 import com.epmet.entity.evaluationindex.screen.ScreenDifficultyDataEntity;
 import com.epmet.entity.evaluationindex.screen.ScreenDifficultyImgDataEntity;
@@ -155,101 +153,8 @@ public class ScreenGrassrootsGovernDataAbsorptionServiceImpl implements ScreenGr
         screenPartyUserRankDataService.dataClean(registeredUsers, param.getCustomerId());
     }
 
-    /**
-     * @param param
-     * @return
-     * @Description 难点赌点数据中转站
-     * @author wangc
-     * @date 2020.09.25 10:00
-     **/
-    @Override
-    public void difficultyDataHub(ScreenCentralZoneDataFormDTO param) {
-        //查询数据
-        List<ScreenDifficultyDataEntity> difficulties = factOriginProjectMainDailyService.getDifficultyBaseInfo(param.getCustomerId(), projectService.getOvertimeProjectByParameter(param.getCustomerId()));
-        if (CollectionUtils.isEmpty(difficulties)) {
-            return;
-        }
-
-        //2.查询出客户下网格的相关信息
-        List<GridInfoDTO> gridList = customerGridService.queryGridInfoList(param.getCustomerId());
-        Map<String, GridInfoDTO> gridMap = gridList.stream().collect(Collectors.toMap(GridInfoDTO::getGridId, a -> a, (o, n) -> o));
-
-        //剔除垃圾数据(本次需要更新的数据集)
-        //for(Iterator<ScreenDifficultyDataEntity> iter = difficulties.iterator(); iter.hasNext();){
-        //    ScreenDifficultyDataEntity pointer = iter.next();
-        //    if(null == gridMap.get(pointer.getOrgId())){
-        //        iter.remove();
-        //    }
-        //}
-        difficulties.removeIf(diff -> null == gridMap.get(diff.getOrgId()));
-
-        List<String> projectIds = difficulties.stream().map(ScreenDifficultyDataEntity::getEventId).distinct().collect(Collectors.toList());
-        //最近一次操作
-        Map<String, ProjectLatestOperationResultDTO> latestOperationMap = projectProcessService.getLatestOperation(projectIds, param.getCustomerId());
-        //图片
-        List<ProjectSourceMapFormDTO> projectSourceMap = difficulties.stream().map(diff -> {
-            ProjectSourceMapFormDTO map = new ProjectSourceMapFormDTO();
-            map.setProjectId(diff.getEventId());
-            map.setSourceId(diff.getEventImgUrl());
-            return map;
-        }).collect(Collectors.toList());
-        //factOriginProjectMainDailyService.getNewProject(param.getCustomerId(),projectIds);
-
-        Map<String, List<ScreenDifficultyImgDataEntity>> imgMap = topicService.getTopicImgs(projectSourceMap);
-
-        Map<String, String> contentMap = topicService.getTopicContent(projectSourceMap);
-
-        difficulties.forEach(diff -> {
-
-            List<ScreenDifficultyImgDataEntity> figureList = CollectionUtils.isEmpty(imgMap) ? null : imgMap.get(diff.getEventId());
-            diff.setEventImgUrl(CollectionUtils.isEmpty(figureList) ? "" : figureList.get(NumConstant.ZERO).getEventImgUrl());
-
-
-            if (!CollectionUtils.isEmpty(contentMap)) {
-                diff.setEventContent(contentMap.get(diff.getEventId()));
-            }
-            if (!CollectionUtils.isEmpty(latestOperationMap)) {
-                ProjectLatestOperationResultDTO oper = latestOperationMap.get(diff.getEventId());
-                if (null != oper) {
-                    diff.setLatestOperateDesc(oper.getOperationName());
-                    diff.setEventTitle(oper.getTitle());
-                    diff.setEventCostTime(oper.getCostTime());
-                }
-            }
-
-            diff.setDataEndTime(DateUtils.getBeforeNDay(NumConstant.ONE));
-        });
-        List<ScreenDifficultyImgDataEntity> imgList = new LinkedList<>();
-        imgMap.values().forEach(imgList::addAll);
-
-        //立案后会有 一个process 如果没有则说明是垃圾数据 如果有其他方式立项的项目则需要考虑下兼容
-        difficulties.removeIf(diff -> StringUtils.isBlank(diff.getLatestOperateDesc()));
-
-        screenDifficultyDataService.dataClean(param.getCustomerId(), difficulties, imgList);
-        log.info("【大屏数据抽取-难点赌点执行完毕】 客户Id{} 难点赌点数据{}", param.getCustomerId(), JSON.toJSONString(difficulties));
-        difficulties.removeIf( diff -> StringUtils.isBlank(diff.getLatestOperateDesc()));
-        imgList.forEach(item -> {
-            item.setCustomerId(param.getCustomerId());
-        });
-        screenDifficultyDataService.dataClean(param.getCustomerId(),difficulties,imgList);
-        log.info("【大屏数据抽取-难点赌点执行完毕】 客户Id{} 难点赌点数据{}",param.getCustomerId(),JSON.toJSONString(difficulties));
-    }
-
     @Override
     public boolean difficultyDataExtract(ScreenCentralZoneDataFormDTO param) {
-        //1.根据客户Id 获取所有项目数据
-        List<FactOriginProjectMainDailyEntity> difficultyBaseList = factOriginProjectMainDailyService.getProjectForDiff(param);
-        if (CollectionUtils.isEmpty(difficultyBaseList)) {
-            log.warn("difficultyDataExtract customerId:{} have any project", param.getCustomerId());
-            return false;
-        }
-        //获取业务库项目标题
-        List<ProjectDTO> bizProjectInfoList = projectService.getProjectInfo(param.getCustomerId(), null, null);
-        if (CollectionUtils.isEmpty(bizProjectInfoList)) {
-            log.warn("difficultyDataExtract customerId:{} have any project", param.getCustomerId());
-            return false;
-        }
-        Map<String, ProjectDTO> bizProjectInfoMap = bizProjectInfoList.stream().collect(Collectors.toMap(ProjectDTO::getId, o -> o, (o1, o2) -> o1));
         //获取agency维度
         List<DimAgencyEntity> agencyList = dimAgencyService.getAgencyListByCustomerId(param.getCustomerId());
         Map<String, DimAgencyEntity> agencyMap = agencyList.stream().collect(Collectors.toMap(DimAgencyEntity::getId, o -> o, (o1, o2) -> o1));
@@ -266,74 +171,97 @@ public class ScreenGrassrootsGovernDataAbsorptionServiceImpl implements ScreenGr
         Map<String, Set<String>> projectCategoryMap = projectCategoryData.stream()
                 .collect(Collectors.groupingBy(ProjectCategoryDTO::getProjectId, Collectors.mapping(ProjectCategoryDTO::getCategoryCode, Collectors.toSet())));
 
-        //2.获取项目的所有节点耗时数据
+        //获取业务库项目标题
-        List<FactOriginProjectOrgPeriodDailyEntity> projectPeriodList = factOriginProjectMainDailyService.getProjectPeriod(param);
+        List<ProjectDTO> bizProjectInfoList = projectService.getProjectInfo(param.getCustomerId(), null, null);
-        //处理部门数去重（只要流转到就算）
+        if (CollectionUtils.isEmpty(bizProjectInfoList)) {
-        Map<String, Set<String>> orgCountMap = new HashMap<>();
+            log.warn("difficultyDataExtract customerId:{} have any project", param.getCustomerId());
-        //被处理次数
+            return false;
-        Map<String, Integer> handleCountMap = new HashMap<>();
+        }
-        Set<String> valideProjectIdSet = fillAnyCountAndValidProject(param, projectPeriodList, orgCountMap, handleCountMap);
+        Map<String, ProjectDTO> bizProjectInfoMap = bizProjectInfoList.stream().collect(Collectors.toMap(ProjectDTO::getId, o -> o, (o1, o2) -> o1));
-
+
-        List<ScreenDifficultyDataEntity> diffList = new ArrayList<>();
+        int pageNo = NumConstant.ONE;
-        List<ScreenDifficultyImgDataEntity> imgDataEntities = new ArrayList<>();
+        int pageSize = NumConstant.FIVE_HUNDRED;
-        for (FactOriginProjectMainDailyEntity project : difficultyBaseList) {
+        List<FactOriginProjectMainAndPeriodDTO> difficultyBaseList;
-            if (agencyMap.get(project.getAgencyId()) == null || bizProjectInfoMap.get(project.getId()) == null) {
+        do {
-                log.warn("未获取到相关的项目信息或者项目的所属组织信息,agencyId:{},projectId:{}",project.getAgencyId(),project.getId());
+            //1.根据客户Id 获取所有项目数据
-                continue;
+            int offset = (pageNo++ - NumConstant.ONE) * pageSize;
-            }
+            difficultyBaseList = factOriginProjectMainDailyService.getProjectAndPeriodForDiff(param, offset,pageSize);
-            //判断是否符合难点堵点条件
+            if (CollectionUtils.isEmpty(difficultyBaseList)) {
-            if (!valideProjectIdSet.contains(project.getId())) {
+                log.warn("difficultyDataExtract customerId:{} have any project", param.getCustomerId());
-                continue;
+                return false;
-            }
-            ScreenDifficultyDataEntity diffEntity = buildBaseDiffEntity(project, agencyMap, gridMap, bizProjectInfoMap);
-            Set<String> regCount = orgCountMap.get(project.getId());
-            if (regCount != null) {
-                diffEntity.setEventReOrg(regCount.size());
-            } else {
-                diffEntity.setEventReOrg(0);
-            }
-            Integer handleOrgCount = handleCountMap.get(project.getId());
-            if (handleOrgCount != null) {
-                diffEntity.setEventHandledCount(handleOrgCount);
-            } else {
-                diffEntity.setEventHandledCount(0);
             }
 
-            diffEntity.setDataEndTime(StringUtils.isBlank(param.getDateId())?DateUtils.getBeforeNDay(NumConstant.ONE):param.getDateId());
+            //2.获取项目的所有节点耗时数据
-            setCategoryInfo(diffEntity, project.getId(), categoryList, projectCategoryMap);
+            List<FactOriginProjectOrgPeriodDailyEntity> projectPeriodList = difficultyBaseList.stream()
-
+                    .filter(o->o.getPeriodList() != null)
-
+                    .flatMap(o->o.getPeriodList().stream())
-            if (StringUtils.isNotBlank(project.getTopicId())) {
+                    .collect(Collectors.toList());
-                //获取图片及话题内容
+            //处理部门数去重（只要流转到就算）
-                ResiTopicAndImgResultDTO topicInfo = topicService.getTopicAndImgs(project.getTopicId(), "image");
+            Map<String, Set<String>> orgCountMap = new HashMap<>();
-                if (topicInfo != null) {
+            //被处理次数
-                    List<TopicAttachmentDTO> topicImgList = topicInfo.getTopicImgList();
+            Map<String, Integer> handleCountMap = new HashMap<>();
-                    if (!CollectionUtils.isEmpty(topicImgList)){
+            Set<String> valideProjectIdSet = fillAnyCountAndValidProject(param, projectPeriodList, orgCountMap, handleCountMap);
-                        diffEntity.setEventImgUrl(topicImgList.get(NumConstant.ZERO).getAttachmentUrl());
+
-                        topicImgList.forEach(img -> {
+            List<ScreenDifficultyDataEntity> diffList = new ArrayList<>();
-                            ScreenDifficultyImgDataEntity imgData = new ScreenDifficultyImgDataEntity();
+            List<ScreenDifficultyImgDataEntity> imgDataEntities = new ArrayList<>();
-                            imgData.setEventId(project.getId());
+            for (FactOriginProjectMainAndPeriodDTO project : difficultyBaseList) {
-                            imgData.setEventImgUrl(img.getAttachmentUrl());
+                if (agencyMap.get(project.getAgencyId()) == null || bizProjectInfoMap.get(project.getId()) == null) {
-                            imgData.setSort(img.getSort());
+                    log.warn("未获取到相关的项目信息或者项目的所属组织信息,agencyId:{},projectId:{}",project.getAgencyId(),project.getId());
-                            imgDataEntities.add(imgData);
+                    continue;
-                            imgData.setCustomerId(param.getCustomerId());
+                }
-                        });
+                //判断是否符合难点堵点条件
+                if (!valideProjectIdSet.contains(project.getId())) {
+                    continue;
+                }
+                ScreenDifficultyDataEntity diffEntity = buildBaseDiffEntity(project, agencyMap, gridMap, bizProjectInfoMap);
+                Set<String> regCount = orgCountMap.get(project.getId());
+                if (regCount != null) {
+                    diffEntity.setEventReOrg(regCount.size());
+                } else {
+                    diffEntity.setEventReOrg(0);
+                }
+                Integer handleOrgCount = handleCountMap.get(project.getId());
+                if (handleOrgCount != null) {
+                    diffEntity.setEventHandledCount(handleOrgCount);
+                } else {
+                    diffEntity.setEventHandledCount(0);
+                }
+
+                diffEntity.setDataEndTime(StringUtils.isBlank(param.getDateId())?DateUtils.getBeforeNDay(NumConstant.ONE):param.getDateId());
+                setCategoryInfo(diffEntity, project.getId(), categoryList, projectCategoryMap);
+
+
+                if (StringUtils.isNotBlank(project.getTopicId())) {
+                    //获取图片及话题内容
+                    ResiTopicAndImgResultDTO topicInfo = topicService.getTopicAndImgs(project.getTopicId(), "image");
+                    if (topicInfo != null) {
+                        List<TopicAttachmentDTO> topicImgList = topicInfo.getTopicImgList();
+                        if (!CollectionUtils.isEmpty(topicImgList)){
+                            diffEntity.setEventImgUrl(topicImgList.get(NumConstant.ZERO).getAttachmentUrl());
+                            topicImgList.forEach(img -> {
+                                ScreenDifficultyImgDataEntity imgData = new ScreenDifficultyImgDataEntity();
+                                imgData.setEventId(project.getId());
+                                imgData.setEventImgUrl(img.getAttachmentUrl());
+                                imgData.setSort(img.getSort());
+                                imgDataEntities.add(imgData);
+                                imgData.setCustomerId(param.getCustomerId());
+                            });
+                        }
+                        diffEntity.setEventContent(topicInfo.getTopicContent());
+
+                        log.info("projectId:{} imgs:{}", project.getId(), imgDataEntities.stream().filter(o -> o.getEventId().equals(project.getId())).count());
                     }
-                    diffEntity.setEventContent(topicInfo.getTopicContent());
 
-                    log.info("projectId:{} imgs:{}", project.getId(), imgDataEntities.stream().filter(o -> o.getEventId().equals(project.getId())).count());
                 }
-
+                diffList.add(diffEntity);
             }
-            diffList.add(diffEntity);
+            log.info("========:" + JSON.toJSONString(diffList));
-        }
+            log.info("========:" + JSON.toJSONString(imgDataEntities));
-        List<ScreenDifficultyDataEntity> collect = diffList.stream().filter(o -> o.getEventCostTime() < 5 * 24 * 60).collect(Collectors.toList());
+
-        log.info("========:" + JSON.toJSONString(collect));
+            screenDifficultyDataService.dataClean(param.getCustomerId(),diffList,imgDataEntities);
-        log.info("========:" + JSON.toJSONString(diffList));
+        } while (!CollectionUtils.isEmpty(difficultyBaseList)&&difficultyBaseList.size()==pageSize);
-        log.info("========:" + JSON.toJSONString(imgDataEntities));
 
-        screenDifficultyDataService.dataClean(param.getCustomerId(),diffList,imgDataEntities);
+        //log.info("【大屏数据抽取-难点赌点执行完毕】 客户Id{} 难点赌点数据{}", param.getCustomerId(), JSON.toJSONString(diffList));
-        log.info("【大屏数据抽取-难点赌点执行完毕】 客户Id{} 难点赌点数据{}", param.getCustomerId(), JSON.toJSONString(diffList));
         //3.获取项目的最后操作记录
         return true;
     }
@@ -422,7 +350,7 @@ public class ScreenGrassrootsGovernDataAbsorptionServiceImpl implements ScreenGr
      * @param bizProjectInfoMap
      * @return
      */
-    private ScreenDifficultyDataEntity buildBaseDiffEntity(FactOriginProjectMainDailyEntity project, Map<String, DimAgencyEntity> agencyMap,
+    private ScreenDifficultyDataEntity buildBaseDiffEntity(FactOriginProjectMainAndPeriodDTO project, Map<String, DimAgencyEntity> agencyMap,
                                                            Map<String, GridInfoDTO> gridMap, Map<String, ProjectDTO> bizProjectInfoMap) {
         ProjectDTO projectDTO = bizProjectInfoMap.get(project.getId());
         ScreenDifficultyDataEntity diff = new ScreenDifficultyDataEntity();

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/screen/ScreenDifficultyDataService.java

@@ -34,7 +34,7 @@ public interface ScreenDifficultyDataService extends BaseService<ScreenDifficult
     /**
      * @Description 查询数据库中是否有该客户下的难点赌点信息
      * @param customerId
-     * @return com.epmet.dto.screen.result.DifficultyIfExistedResultDTO   
+     * @return com.epmet.dto.screen.result.DifficultyIfExistedResultDTO
      * @author wangc
      * @date 2020.09.28 10:53
     */
@@ -50,13 +50,4 @@ public interface ScreenDifficultyDataService extends BaseService<ScreenDifficult
      * @date 2020.09.28 11:04
     */
     void dataClean(String customerId,List<ScreenDifficultyDataEntity> difficulties, List<ScreenDifficultyImgDataEntity> imgs);
-
+}
-    /**
-     * @Description 根据客户Id查出全部的难点赌点的项目Id和网格Id，因为需要比对全部的信息然后删除因为删除网格而导致的垃圾数据
-     * @param customerId
-     * @return java.util.List<com.epmet.entity.evaluationindex.screen.ScreenDifficultyDataEntity>
-     * @author wangc
-     * @date 2020.11.17 09:31
-     */
-    List<ScreenDifficultyDataEntity> getAllDifficultyByCustomerId(String customerId);
-}

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/screen/impl/ScreenDifficultyDataServiceImpl.java

@@ -21,7 +21,6 @@ package com.epmet.service.evaluationindex.screen.impl;
 import com.epmet.commons.dynamic.datasource.annotation.DataSource;
 import com.epmet.commons.mybatis.service.impl.BaseServiceImpl;
 import com.epmet.commons.tools.constant.NumConstant;
-import com.epmet.commons.tools.utils.DateUtils;
 import com.epmet.constant.DataSourceConstant;
 import com.epmet.dao.evaluationindex.screen.ScreenDifficultyDataDao;
 import com.epmet.entity.evaluationindex.screen.ScreenDifficultyDataEntity;
@@ -54,30 +53,15 @@ public class ScreenDifficultyDataServiceImpl extends BaseServiceImpl<ScreenDiffi
     @Override
     @Transactional(rollbackFor = Exception.class)
     public void dataClean(String customerId, List<ScreenDifficultyDataEntity> difficulties, List<ScreenDifficultyImgDataEntity> imgs) {
-        List<ScreenDifficultyDataEntity> orient = baseDao.selectAllDifficultyByCustomerId(customerId);
+        List<String> projectId = difficulties.stream().map(ScreenDifficultyDataEntity::getEventId).distinct().collect(Collectors.toList());
-        if(!CollectionUtils.isEmpty(orient)){
+        baseDao.deleteBatchDifficultyImg(projectId);
-            baseDao.deleteBatchDifficultyImg(orient.stream().map(ScreenDifficultyDataEntity::getEventId).distinct().collect(Collectors.toList()));
+        baseDao.deleteBatchDifficulty(customerId,projectId);
-        }
-        baseDao.deleteBatchDifficulty(customerId,null);
-
         if (!CollectionUtils.isEmpty(difficulties)) {
             baseDao.insertBatch(difficulties);
         }
         if (!CollectionUtils.isEmpty(imgs)) {
             baseDao.insertBatchImg(imgs);
         }
-        //baseDao.updateTime(customerId, DateUtils.getBeforeNDay(NumConstant.ONE));
     }
 
-    /**
+}
-     * @Description 根据客户Id查出全部的难点赌点的项目Id和网格Id，因为需要比对全部的信息然后删除因为删除网格而导致的垃圾数据
-     * @param customerId
-     * @return java.util.List<com.epmet.entity.evaluationindex.screen.ScreenDifficultyDataEntity>
-     * @author wangc
-     * @date 2020.11.17 09:31
-     */
-    @Override
-    public List<ScreenDifficultyDataEntity> getAllDifficultyByCustomerId(String customerId) {
-        return baseDao.selectAllDifficultyByCustomerId(customerId);
-    }
-}

epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/evaluationindex/extract/FactOriginProjectMainDailyDao.xml

@@ -3,32 +3,7 @@
 
 <mapper namespace="com.epmet.dao.evaluationindex.extract.FactOriginProjectMainDailyDao">
 
-    <resultMap type="com.epmet.entity.evaluationindex.extract.FactOriginProjectMainDailyEntity" id="factOriginProjectMainDailyMap">
+
-        <result property="id" column="ID"/>
-        <result property="dateId" column="DATE_ID"/>
-        <result property="weekId" column="WEEK_ID"/>
-        <result property="monthId" column="MONTH_ID"/>
-        <result property="quarterId" column="QUARTER_ID"/>
-        <result property="yearId" column="YEAR_ID"/>
-        <result property="customerId" column="CUSTOMER_ID"/>
-        <result property="issueCreatorId" column="ISSUE_CREATOR_ID"/>
-        <result property="issueId" column="ISSUE_ID"/>
-        <result property="topicId" column="TOPIC_ID"/>
-        <result property="gridId" column="GRID_ID"/>
-        <result property="pid" column="PID"/>
-        <result property="agencyId" column="AGENCY_ID"/>
-        <result property="projectStatus" column="PROJECT_STATUS"/>
-        <result property="topicCreatorId" column="TOPIC_CREATOR_ID"/>
-        <result property="isParty" column="IS_PARTY"/>
-        <result property="isOverdue" column="IS_OVERDUE"/>
-        <result property="finishOrgIds" column="FINISH_ORG_IDS"/>
-        <result property="delFlag" column="DEL_FLAG"/>
-        <result property="revision" column="REVISION"/>
-        <result property="createdBy" column="CREATED_BY"/>
-        <result property="createdTime" column="CREATED_TIME"/>
-        <result property="updatedBy" column="UPDATED_BY"/>
-        <result property="updatedTime" column="UPDATED_TIME"/>
-    </resultMap>
     <select id="getProjectByCustomer" resultType="com.epmet.dto.extract.FactOriginProjectMainDailyDTO">
         SELECT
             ID,
@@ -663,23 +638,47 @@
             </otherwise>
         </choose>
     </select>
+     <resultMap type="com.epmet.entity.evaluationindex.extract.FactOriginProjectMainAndPeriodDTO" id="factOriginProjectMainPeriodMap">
+        <result property="id" column="ID"/>
+        <result property="dateId" column="DATE_ID"/>
+        <result property="weekId" column="WEEK_ID"/>
+        <result property="monthId" column="MONTH_ID"/>
+        <result property="quarterId" column="QUARTER_ID"/>
+        <result property="yearId" column="YEAR_ID"/>
+        <result property="customerId" column="CUSTOMER_ID"/>
+        <result property="issueCreatorId" column="ISSUE_CREATOR_ID"/>
+        <result property="issueId" column="ISSUE_ID"/>
+        <result property="topicId" column="TOPIC_ID"/>
+        <result property="gridId" column="GRID_ID"/>
+        <result property="pid" column="PID"/>
+        <result property="pids" column="PIDS"/>
+        <result property="agencyId" column="AGENCY_ID"/>
+        <result property="projectStatus" column="PROJECT_STATUS"/>
+        <result property="topicCreatorId" column="TOPIC_CREATOR_ID"/>
+        <result property="isParty" column="IS_PARTY"/>
+        <result property="isOverdue" column="IS_OVERDUE"/>
+        <result property="finishOrgIds" column="FINISH_ORG_IDS"/>
+
+        <collection property="periodList" ofType="com.epmet.entity.evaluationindex.extract.FactOriginProjectOrgPeriodDailyEntity"
+        select="getProjectPeriodForDiff" column="id"/>
+    </resultMap>
     <!--获取难点堵点项目数据-->
-    <select id="getProjectForDiff" resultType="com.epmet.entity.evaluationindex.extract.FactOriginProjectMainDailyEntity">
+    <select id="getProjectForDiff" resultMap = "factOriginProjectMainPeriodMap">
         SELECT
-            id, date_id, customer_id, issue_id, topic_id, grid_id, pid, pids, agency_id, project_status, is_resolved,
+            m.id, m.date_id dateId, m.customer_id, m.issue_id, m.topic_id, m.grid_id,
-            topic_creator_id, is_party, is_overdue, finish_org_ids
+            m.pid, m.pids, m.agency_id, m.project_status, m.is_resolved,
-        FROM fact_origin_project_main_daily
+            m.topic_creator_id, m.is_party, m.is_overdue, m.finish_org_ids
+        FROM fact_origin_project_main_daily m
         WHERE
-            CUSTOMER_ID = #{customerId}
+            m.CUSTOMER_ID = #{customerId}
-            <if test="dateId != null and dateId != ''">
+            AND m.DEL_FLAG = '0'
-                AND DATE_ID = #{dateId,jdbcType=VARCHAR}
+            LIMIT #{offset} ,#{pageSize}
-            </if>
     </select>
     <select id="getProjectPeriodForDiff" resultType="com.epmet.entity.evaluationindex.extract.FactOriginProjectOrgPeriodDailyEntity">
         SELECT
             t.ID, t.CUSTOMER_ID, t.PROJECT_ID, t.ORG_ID, t.ORG_TYPE, t.PID,t. PIDS, t.INFORMED_DATE, t.HANDLED_DATE, t.TOTAL_PERIOD, t.PERIOD_TILL_REPLY_FIRSTLY, t.OPERATION, t.IS_RESOLVED
         FROM fact_origin_project_org_period_daily t
-        WHERE CUSTOMER_ID = #{customerId}
+        WHERE PROJECT_ID = #{projectId}
     </select>
 
     <!-- 组织查询项目总数 -->

epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/evaluationindex/screen/ScreenDifficultyDataDao.xml

@@ -221,14 +221,4 @@
         </foreach>
     </insert>
 
-    <select id="selectAllDifficultyByCustomerId" resultType="com.epmet.entity.evaluationindex.screen.ScreenDifficultyDataEntity">
-        SELECT
-            org_id,
-            event_id
-        FROM
-            screen_difficulty_data
-        WHERE
-            del_flag = '0'
-            AND customer_id = #{customerId}
-    </select>
 </mapper>