diff --git a/epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/dto/screen/form/ScreenCentralZoneDataFormDTO.java b/epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/dto/screen/form/ScreenCentralZoneDataFormDTO.java index 5a5d0af839..21c1a0f414 100644 --- a/epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/dto/screen/form/ScreenCentralZoneDataFormDTO.java +++ b/epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/dto/screen/form/ScreenCentralZoneDataFormDTO.java @@ -29,4 +29,14 @@ public class ScreenCentralZoneDataFormDTO implements Serializable { private String endDate; private String projectId; + + /** + * 首次的话 全量拉取 + */ + private Boolean isFirst; + + /** + * 时间间隔天数 14 就是查询14天内的数据 + */ + private String internalDay; } diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/evaluationindex/extract/FactOriginProjectMainDailyDao.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/evaluationindex/extract/FactOriginProjectMainDailyDao.java index 8826ac6f9a..7d1c6903ab 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/evaluationindex/extract/FactOriginProjectMainDailyDao.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/evaluationindex/extract/FactOriginProjectMainDailyDao.java @@ -26,6 +26,7 @@ import com.epmet.dto.indexcollect.result.CpcIndexCommonDTO; import com.epmet.dto.pingyin.result.*; import com.epmet.dto.screen.ScreenProjectDataDTO; import com.epmet.dto.screen.form.ProjectSourceMapFormDTO; +import com.epmet.dto.screen.form.ScreenCentralZoneDataFormDTO; import com.epmet.entity.evaluationindex.extract.*; import com.epmet.entity.evaluationindex.screen.ScreenDifficultyDataEntity; import org.apache.ibatis.annotations.Mapper; @@ -400,13 +401,13 @@ public interface FactOriginProjectMainDailyDao extends BaseDao * @author LiuJanJun * @date 2021/4/12 1:35 下午 */ - List getProjectForDiff(@Param("customerId") String customerId, @Param("offset") int offset, @Param("pageSize") int pageSize); + List getProjectForDiff(@Param("param") ScreenCentralZoneDataFormDTO param, @Param("offset") int offset, @Param("pageSize") int pageSize); /** * desc: 获取难点堵点项目数据耗时 diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/todata/impl/FactOriginProjectMainDailyServiceImpl.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/todata/impl/FactOriginProjectMainDailyServiceImpl.java index 8ad4183718..a307aaf171 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/todata/impl/FactOriginProjectMainDailyServiceImpl.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/todata/impl/FactOriginProjectMainDailyServiceImpl.java @@ -525,7 +525,7 @@ public class FactOriginProjectMainDailyServiceImpl extends BaseServiceImpl getProjectAndPeriodForDiff(ScreenCentralZoneDataFormDTO param, int offset, int pageSize) { - return baseDao.getProjectForDiff(param.getCustomerId(),offset,pageSize); + return baseDao.getProjectForDiff(param,offset,pageSize); } @Override diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenGrassrootsGovernDataAbsorptionServiceImpl.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenGrassrootsGovernDataAbsorptionServiceImpl.java index d8d971fda1..6bacc7a64c 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenGrassrootsGovernDataAbsorptionServiceImpl.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenGrassrootsGovernDataAbsorptionServiceImpl.java @@ -155,115 +155,120 @@ public class ScreenGrassrootsGovernDataAbsorptionServiceImpl implements ScreenGr @Override public boolean difficultyDataExtract(ScreenCentralZoneDataFormDTO param) { - //获取agency维度 - List agencyList = dimAgencyService.getAgencyListByCustomerId(param.getCustomerId()); - Map agencyMap = agencyList.stream().collect(Collectors.toMap(DimAgencyEntity::getId, o -> o, (o1, o2) -> o1)); - //获取grid维度 - List gridList = customerGridService.queryGridInfoList(param.getCustomerId()); - Map gridMap = gridList.stream().collect(Collectors.toMap(GridInfoDTO::getGridId, o -> o, (o1, o2) -> o1)); - - //获取客户下所有项目分类 - List categoryList = issueProjectCategoryDictService.getAllByCId(param.getCustomerId()); - - //项目分类数据 todo 暂时查询所有 - List projectCategoryData = projectService.getProjectCategoryData(param.getCustomerId(), null); - //key projectId:categoryCode Set - Map> projectCategoryMap = projectCategoryData.stream() - .collect(Collectors.groupingBy(ProjectCategoryDTO::getProjectId, Collectors.mapping(ProjectCategoryDTO::getCategoryCode, Collectors.toSet()))); - - //获取业务库项目标题 - List bizProjectInfoList = projectService.getProjectInfo(param.getCustomerId(), null, null); - if (CollectionUtils.isEmpty(bizProjectInfoList)) { - log.warn("difficultyDataExtract customerId:{} have any project", param.getCustomerId()); - return false; - } - Map bizProjectInfoMap = bizProjectInfoList.stream().collect(Collectors.toMap(ProjectDTO::getId, o -> o, (o1, o2) -> o1)); - - int pageNo = NumConstant.ONE; - int pageSize = NumConstant.FIVE_HUNDRED; - List difficultyBaseList; - do { - //1.根据客户Id 获取所有项目数据 - int offset = (pageNo++ - NumConstant.ONE) * pageSize; - difficultyBaseList = factOriginProjectMainDailyService.getProjectAndPeriodForDiff(param, offset,pageSize); - if (CollectionUtils.isEmpty(difficultyBaseList)) { + try { + //默认查询10天内的数据 + param.setInternalDay(DateUtils.getBeforeNDay(NumConstant.TEN)); + //获取agency维度 + List agencyList = dimAgencyService.getAgencyListByCustomerId(param.getCustomerId()); + Map agencyMap = agencyList.stream().collect(Collectors.toMap(DimAgencyEntity::getId, o -> o, (o1, o2) -> o1)); + //获取grid维度 + List gridList = customerGridService.queryGridInfoList(param.getCustomerId()); + Map gridMap = gridList.stream().collect(Collectors.toMap(GridInfoDTO::getGridId, o -> o, (o1, o2) -> o1)); + + //获取客户下所有项目分类 + List categoryList = issueProjectCategoryDictService.getAllByCId(param.getCustomerId()); + + //项目分类数据 todo 暂时查询所有 + List projectCategoryData = projectService.getProjectCategoryData(param.getCustomerId(), null); + //key projectId:categoryCode Set + Map> projectCategoryMap = projectCategoryData.stream() + .collect(Collectors.groupingBy(ProjectCategoryDTO::getProjectId, Collectors.mapping(ProjectCategoryDTO::getCategoryCode, Collectors.toSet()))); + + //获取业务库项目标题 + List bizProjectInfoList = projectService.getProjectInfo(param.getCustomerId(), null, null); + if (CollectionUtils.isEmpty(bizProjectInfoList)) { log.warn("difficultyDataExtract customerId:{} have any project", param.getCustomerId()); return false; } - - //2.获取项目的所有节点耗时数据 - List projectPeriodList = difficultyBaseList.stream() - .filter(o->o.getPeriodList() != null) - .flatMap(o->o.getPeriodList().stream()) - .collect(Collectors.toList()); - //处理部门数去重（只要流转到就算） - Map> orgCountMap = new HashMap<>(); - //被处理次数 - Map handleCountMap = new HashMap<>(); - Set valideProjectIdSet = fillAnyCountAndValidProject(param, projectPeriodList, orgCountMap, handleCountMap); - - List diffList = new ArrayList<>(); - List imgDataEntities = new ArrayList<>(); - for (FactOriginProjectMainAndPeriodDTO project : difficultyBaseList) { - if (agencyMap.get(project.getAgencyId()) == null || bizProjectInfoMap.get(project.getId()) == null) { - log.warn("未获取到相关的项目信息或者项目的所属组织信息,agencyId:{},projectId:{}",project.getAgencyId(),project.getId()); - continue; - } - //判断是否符合难点堵点条件 - if (!valideProjectIdSet.contains(project.getId())) { - continue; - } - ScreenDifficultyDataEntity diffEntity = buildBaseDiffEntity(project, agencyMap, gridMap, bizProjectInfoMap); - Set regCount = orgCountMap.get(project.getId()); - if (regCount != null) { - diffEntity.setEventReOrg(regCount.size()); - } else { - diffEntity.setEventReOrg(0); - } - Integer handleOrgCount = handleCountMap.get(project.getId()); - if (handleOrgCount != null) { - diffEntity.setEventHandledCount(handleOrgCount); - } else { - diffEntity.setEventHandledCount(0); + Map bizProjectInfoMap = bizProjectInfoList.stream().collect(Collectors.toMap(ProjectDTO::getId, o -> o, (o1, o2) -> o1)); + + int pageNo = NumConstant.ONE; + int pageSize = NumConstant.ONE_THOUSAND; + List difficultyBaseList; + do { + //1.根据客户Id 获取所有项目数据 + int offset = (pageNo++ - NumConstant.ONE) * pageSize; + difficultyBaseList = factOriginProjectMainDailyService.getProjectAndPeriodForDiff(param, offset,pageSize); + if (CollectionUtils.isEmpty(difficultyBaseList)) { + log.warn("difficultyDataExtract customerId:{} have any project", param.getCustomerId()); + return false; } - diffEntity.setDataEndTime(StringUtils.isBlank(param.getDateId())?DateUtils.getBeforeNDay(NumConstant.ONE):param.getDateId()); - setCategoryInfo(diffEntity, project.getId(), categoryList, projectCategoryMap); - - - if (StringUtils.isNotBlank(project.getTopicId())) { - //获取图片及话题内容 - ResiTopicAndImgResultDTO topicInfo = topicService.getTopicAndImgs(project.getTopicId(), "image"); - if (topicInfo != null) { - List topicImgList = topicInfo.getTopicImgList(); - if (!CollectionUtils.isEmpty(topicImgList)){ - diffEntity.setEventImgUrl(topicImgList.get(NumConstant.ZERO).getAttachmentUrl()); - topicImgList.forEach(img -> { - ScreenDifficultyImgDataEntity imgData = new ScreenDifficultyImgDataEntity(); - imgData.setEventId(project.getId()); - imgData.setEventImgUrl(img.getAttachmentUrl()); - imgData.setSort(img.getSort()); - imgDataEntities.add(imgData); - imgData.setCustomerId(param.getCustomerId()); - }); + //2.获取项目的所有节点耗时数据 + List projectPeriodList = difficultyBaseList.stream() + .filter(o->o.getPeriodList() != null) + .flatMap(o->o.getPeriodList().stream()) + .collect(Collectors.toList()); + //处理部门数去重（只要流转到就算） + Map> orgCountMap = new HashMap<>(); + //被处理次数 + Map handleCountMap = new HashMap<>(); + Set valideProjectIdSet = fillAnyCountAndValidProject(param, projectPeriodList, orgCountMap, handleCountMap); + + List diffList = new ArrayList<>(); + List imgDataEntities = new ArrayList<>(); + for (FactOriginProjectMainAndPeriodDTO project : difficultyBaseList) { + if (agencyMap.get(project.getAgencyId()) == null || bizProjectInfoMap.get(project.getId()) == null) { + log.warn("未获取到相关的项目信息或者项目的所属组织信息,agencyId:{},projectId:{}",project.getAgencyId(),project.getId()); + continue; + } + //判断是否符合难点堵点条件 + if (!valideProjectIdSet.contains(project.getId())) { + continue; + } + ScreenDifficultyDataEntity diffEntity = buildBaseDiffEntity(project, agencyMap, gridMap, bizProjectInfoMap); + Set regCount = orgCountMap.get(project.getId()); + if (regCount != null) { + diffEntity.setEventReOrg(regCount.size()); + } else { + diffEntity.setEventReOrg(0); + } + Integer handleOrgCount = handleCountMap.get(project.getId()); + if (handleOrgCount != null) { + diffEntity.setEventHandledCount(handleOrgCount); + } else { + diffEntity.setEventHandledCount(0); + } + + diffEntity.setDataEndTime(StringUtils.isBlank(param.getDateId())?DateUtils.getBeforeNDay(NumConstant.ONE):param.getDateId()); + setCategoryInfo(diffEntity, project.getId(), categoryList, projectCategoryMap); + + + if (StringUtils.isNotBlank(project.getTopicId())) { + //获取图片及话题内容 + ResiTopicAndImgResultDTO topicInfo = topicService.getTopicAndImgs(project.getTopicId(), "image"); + if (topicInfo != null) { + List topicImgList = topicInfo.getTopicImgList(); + if (!CollectionUtils.isEmpty(topicImgList)){ + diffEntity.setEventImgUrl(topicImgList.get(NumConstant.ZERO).getAttachmentUrl()); + topicImgList.forEach(img -> { + ScreenDifficultyImgDataEntity imgData = new ScreenDifficultyImgDataEntity(); + imgData.setEventId(project.getId()); + imgData.setEventImgUrl(img.getAttachmentUrl()); + imgData.setSort(img.getSort()); + imgDataEntities.add(imgData); + imgData.setCustomerId(param.getCustomerId()); + }); + } + diffEntity.setEventContent(topicInfo.getTopicContent()); + + log.info("projectId:{} imgs:{}", project.getId(), imgDataEntities.stream().filter(o -> o.getEventId().equals(project.getId())).count()); } - diffEntity.setEventContent(topicInfo.getTopicContent()); - log.info("projectId:{} imgs:{}", project.getId(), imgDataEntities.stream().filter(o -> o.getEventId().equals(project.getId())).count()); } - + diffList.add(diffEntity); } - diffList.add(diffEntity); - } - log.info("========:" + JSON.toJSONString(diffList)); - log.info("========:" + JSON.toJSONString(imgDataEntities)); - - screenDifficultyDataService.dataClean(param.getCustomerId(),diffList,imgDataEntities); - } while (!CollectionUtils.isEmpty(difficultyBaseList)&&difficultyBaseList.size()==pageSize); - - //log.info("【大屏数据抽取-难点赌点执行完毕】 客户Id{} 难点赌点数据{}", param.getCustomerId(), JSON.toJSONString(diffList)); - //3.获取项目的最后操作记录 - return true; + log.info("========:" + JSON.toJSONString(diffList)); + log.info("========:" + JSON.toJSONString(imgDataEntities)); + screenDifficultyDataService.dataClean(param.getCustomerId(),diffList,imgDataEntities); + log.info("difficultyDataExtract excute pageNo:{}",pageNo); + } while (!CollectionUtils.isEmpty(difficultyBaseList)&&difficultyBaseList.size()==pageSize); + + //3.获取项目的最后操作记录 + return true; + } finally { + log.info("【大屏数据抽取-难点赌点执行完毕】 客户Id{}", param.getCustomerId()); + } } /** diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/evaluationindex/extract/FactOriginProjectMainDailyDao.xml b/epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/evaluationindex/extract/FactOriginProjectMainDailyDao.xml index 1d34dfb47b..a6ba0e7a8b 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/evaluationindex/extract/FactOriginProjectMainDailyDao.xml +++ b/epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/evaluationindex/extract/FactOriginProjectMainDailyDao.xml @@ -668,10 +668,19 @@ m.id, m.date_id dateId, m.customer_id, m.issue_id, m.topic_id, m.grid_id, m.pid, m.pids, m.agency_id, m.project_status, m.is_resolved, m.topic_creator_id, m.is_party, m.is_overdue, m.finish_org_ids + + , IFNULL(b.DATE_ID, DATE_FORMAT( now(), '%Y%m%d' )) as updateDate + FROM fact_origin_project_main_daily m + + LEFT JOIN fact_origin_project_log_daily b ON m.ID = b.PROJECT_ID AND b.ACTION_CODE = 'close' + WHERE - m.CUSTOMER_ID = #{customerId} + m.CUSTOMER_ID = #{param.customerId} AND m.DEL_FLAG = '0' + + HAVING updateDate>= #{param.internalDay} + LIMIT #{offset} ,#{pageSize}