From 8b3c365abcd703fe96b2d65df4d9f39bbab48f64 Mon Sep 17 00:00:00 2001 From: jianjun Date: Mon, 10 Jan 2022 16:37:30 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E9=9A=BE=E7=82=B9=E5=A0=B5=E7=82=B91000?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../impl/ScreenGrassrootsGovernDataAbsorptionServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenGrassrootsGovernDataAbsorptionServiceImpl.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenGrassrootsGovernDataAbsorptionServiceImpl.java index d8d971fda1..f1a13c0f12 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenGrassrootsGovernDataAbsorptionServiceImpl.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenGrassrootsGovernDataAbsorptionServiceImpl.java @@ -180,7 +180,7 @@ public class ScreenGrassrootsGovernDataAbsorptionServiceImpl implements ScreenGr Map bizProjectInfoMap = bizProjectInfoList.stream().collect(Collectors.toMap(ProjectDTO::getId, o -> o, (o1, o2) -> o1)); int pageNo = NumConstant.ONE; - int pageSize = NumConstant.FIVE_HUNDRED; + int pageSize = NumConstant.ONE_THOUSAND; List difficultyBaseList; do { //1.根据客户Id 获取所有项目数据 From 78c493239cc6b5a5da6c13347490e7e31675f2c9 Mon Sep 17 00:00:00 2001 From: jianjun Date: Mon, 10 Jan 2022 18:21:25 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E9=9A=BE=E7=82=B9=E5=A0=B5=E7=82=B9=20?= =?UTF-8?q?=E5=8F=AA=E5=B7=AE=E6=9F=A5=E8=BF=9110=E5=A4=A9=E7=9A=84?= =?UTF-8?q?=E7=BB=93=E6=A1=88=E6=95=B0=E6=8D=AE=E5=92=8C=E6=9C=AA=E7=BB=93?= =?UTF-8?q?=E6=A1=88=E7=9A=84=E6=89=80=E6=9C=89=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../form/ScreenCentralZoneDataFormDTO.java | 10 + .../FactOriginProjectMainDailyDao.java | 7 +- ...FactOriginProjectMainDailyServiceImpl.java | 2 +- ...srootsGovernDataAbsorptionServiceImpl.java | 205 +++++++++--------- .../extract/FactOriginProjectMainDailyDao.xml | 11 +- 5 files changed, 130 insertions(+), 105 deletions(-) diff --git a/epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/dto/screen/form/ScreenCentralZoneDataFormDTO.java b/epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/dto/screen/form/ScreenCentralZoneDataFormDTO.java index 5a5d0af839..21c1a0f414 100644 --- a/epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/dto/screen/form/ScreenCentralZoneDataFormDTO.java +++ b/epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/dto/screen/form/ScreenCentralZoneDataFormDTO.java @@ -29,4 +29,14 @@ public class ScreenCentralZoneDataFormDTO implements Serializable { private String endDate; private String projectId; + + /** + * 首次的话 全量拉取 + */ + private Boolean isFirst; + + /** + * 时间间隔天数 14 就是查询14天内的数据 + */ + private String internalDay; } diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/evaluationindex/extract/FactOriginProjectMainDailyDao.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/evaluationindex/extract/FactOriginProjectMainDailyDao.java index 8826ac6f9a..7d1c6903ab 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/evaluationindex/extract/FactOriginProjectMainDailyDao.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/evaluationindex/extract/FactOriginProjectMainDailyDao.java @@ -26,6 +26,7 @@ import com.epmet.dto.indexcollect.result.CpcIndexCommonDTO; import com.epmet.dto.pingyin.result.*; import com.epmet.dto.screen.ScreenProjectDataDTO; import com.epmet.dto.screen.form.ProjectSourceMapFormDTO; +import com.epmet.dto.screen.form.ScreenCentralZoneDataFormDTO; import com.epmet.entity.evaluationindex.extract.*; import com.epmet.entity.evaluationindex.screen.ScreenDifficultyDataEntity; import org.apache.ibatis.annotations.Mapper; @@ -400,13 +401,13 @@ public interface FactOriginProjectMainDailyDao extends BaseDao * @author LiuJanJun * @date 2021/4/12 1:35 下午 */ - List getProjectForDiff(@Param("customerId") String customerId, @Param("offset") int offset, @Param("pageSize") int pageSize); + List getProjectForDiff(@Param("param") ScreenCentralZoneDataFormDTO param, @Param("offset") int offset, @Param("pageSize") int pageSize); /** * desc: 获取难点堵点项目数据耗时 diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/todata/impl/FactOriginProjectMainDailyServiceImpl.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/todata/impl/FactOriginProjectMainDailyServiceImpl.java index 8ad4183718..a307aaf171 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/todata/impl/FactOriginProjectMainDailyServiceImpl.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/todata/impl/FactOriginProjectMainDailyServiceImpl.java @@ -525,7 +525,7 @@ public class FactOriginProjectMainDailyServiceImpl extends BaseServiceImpl getProjectAndPeriodForDiff(ScreenCentralZoneDataFormDTO param, int offset, int pageSize) { - return baseDao.getProjectForDiff(param.getCustomerId(),offset,pageSize); + return baseDao.getProjectForDiff(param,offset,pageSize); } @Override diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenGrassrootsGovernDataAbsorptionServiceImpl.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenGrassrootsGovernDataAbsorptionServiceImpl.java index f1a13c0f12..6bacc7a64c 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenGrassrootsGovernDataAbsorptionServiceImpl.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenGrassrootsGovernDataAbsorptionServiceImpl.java @@ -155,115 +155,120 @@ public class ScreenGrassrootsGovernDataAbsorptionServiceImpl implements ScreenGr @Override public boolean difficultyDataExtract(ScreenCentralZoneDataFormDTO param) { - //获取agency维度 - List agencyList = dimAgencyService.getAgencyListByCustomerId(param.getCustomerId()); - Map agencyMap = agencyList.stream().collect(Collectors.toMap(DimAgencyEntity::getId, o -> o, (o1, o2) -> o1)); - //获取grid维度 - List gridList = customerGridService.queryGridInfoList(param.getCustomerId()); - Map gridMap = gridList.stream().collect(Collectors.toMap(GridInfoDTO::getGridId, o -> o, (o1, o2) -> o1)); - - //获取客户下所有项目分类 - List categoryList = issueProjectCategoryDictService.getAllByCId(param.getCustomerId()); - - //项目分类数据 todo 暂时查询所有 - List projectCategoryData = projectService.getProjectCategoryData(param.getCustomerId(), null); - //key projectId:categoryCode Set - Map> projectCategoryMap = projectCategoryData.stream() - .collect(Collectors.groupingBy(ProjectCategoryDTO::getProjectId, Collectors.mapping(ProjectCategoryDTO::getCategoryCode, Collectors.toSet()))); - - //获取业务库项目标题 - List bizProjectInfoList = projectService.getProjectInfo(param.getCustomerId(), null, null); - if (CollectionUtils.isEmpty(bizProjectInfoList)) { - log.warn("difficultyDataExtract customerId:{} have any project", param.getCustomerId()); - return false; - } - Map bizProjectInfoMap = bizProjectInfoList.stream().collect(Collectors.toMap(ProjectDTO::getId, o -> o, (o1, o2) -> o1)); - - int pageNo = NumConstant.ONE; - int pageSize = NumConstant.ONE_THOUSAND; - List difficultyBaseList; - do { - //1.根据客户Id 获取所有项目数据 - int offset = (pageNo++ - NumConstant.ONE) * pageSize; - difficultyBaseList = factOriginProjectMainDailyService.getProjectAndPeriodForDiff(param, offset,pageSize); - if (CollectionUtils.isEmpty(difficultyBaseList)) { + try { + //默认查询10天内的数据 + param.setInternalDay(DateUtils.getBeforeNDay(NumConstant.TEN)); + //获取agency维度 + List agencyList = dimAgencyService.getAgencyListByCustomerId(param.getCustomerId()); + Map agencyMap = agencyList.stream().collect(Collectors.toMap(DimAgencyEntity::getId, o -> o, (o1, o2) -> o1)); + //获取grid维度 + List gridList = customerGridService.queryGridInfoList(param.getCustomerId()); + Map gridMap = gridList.stream().collect(Collectors.toMap(GridInfoDTO::getGridId, o -> o, (o1, o2) -> o1)); + + //获取客户下所有项目分类 + List categoryList = issueProjectCategoryDictService.getAllByCId(param.getCustomerId()); + + //项目分类数据 todo 暂时查询所有 + List projectCategoryData = projectService.getProjectCategoryData(param.getCustomerId(), null); + //key projectId:categoryCode Set + Map> projectCategoryMap = projectCategoryData.stream() + .collect(Collectors.groupingBy(ProjectCategoryDTO::getProjectId, Collectors.mapping(ProjectCategoryDTO::getCategoryCode, Collectors.toSet()))); + + //获取业务库项目标题 + List bizProjectInfoList = projectService.getProjectInfo(param.getCustomerId(), null, null); + if (CollectionUtils.isEmpty(bizProjectInfoList)) { log.warn("difficultyDataExtract customerId:{} have any project", param.getCustomerId()); return false; } - - //2.获取项目的所有节点耗时数据 - List projectPeriodList = difficultyBaseList.stream() - .filter(o->o.getPeriodList() != null) - .flatMap(o->o.getPeriodList().stream()) - .collect(Collectors.toList()); - //处理部门数去重（只要流转到就算） - Map> orgCountMap = new HashMap<>(); - //被处理次数 - Map handleCountMap = new HashMap<>(); - Set valideProjectIdSet = fillAnyCountAndValidProject(param, projectPeriodList, orgCountMap, handleCountMap); - - List diffList = new ArrayList<>(); - List imgDataEntities = new ArrayList<>(); - for (FactOriginProjectMainAndPeriodDTO project : difficultyBaseList) { - if (agencyMap.get(project.getAgencyId()) == null || bizProjectInfoMap.get(project.getId()) == null) { - log.warn("未获取到相关的项目信息或者项目的所属组织信息,agencyId:{},projectId:{}",project.getAgencyId(),project.getId()); - continue; - } - //判断是否符合难点堵点条件 - if (!valideProjectIdSet.contains(project.getId())) { - continue; - } - ScreenDifficultyDataEntity diffEntity = buildBaseDiffEntity(project, agencyMap, gridMap, bizProjectInfoMap); - Set regCount = orgCountMap.get(project.getId()); - if (regCount != null) { - diffEntity.setEventReOrg(regCount.size()); - } else { - diffEntity.setEventReOrg(0); - } - Integer handleOrgCount = handleCountMap.get(project.getId()); - if (handleOrgCount != null) { - diffEntity.setEventHandledCount(handleOrgCount); - } else { - diffEntity.setEventHandledCount(0); + Map bizProjectInfoMap = bizProjectInfoList.stream().collect(Collectors.toMap(ProjectDTO::getId, o -> o, (o1, o2) -> o1)); + + int pageNo = NumConstant.ONE; + int pageSize = NumConstant.ONE_THOUSAND; + List difficultyBaseList; + do { + //1.根据客户Id 获取所有项目数据 + int offset = (pageNo++ - NumConstant.ONE) * pageSize; + difficultyBaseList = factOriginProjectMainDailyService.getProjectAndPeriodForDiff(param, offset,pageSize); + if (CollectionUtils.isEmpty(difficultyBaseList)) { + log.warn("difficultyDataExtract customerId:{} have any project", param.getCustomerId()); + return false; } - diffEntity.setDataEndTime(StringUtils.isBlank(param.getDateId())?DateUtils.getBeforeNDay(NumConstant.ONE):param.getDateId()); - setCategoryInfo(diffEntity, project.getId(), categoryList, projectCategoryMap); - - - if (StringUtils.isNotBlank(project.getTopicId())) { - //获取图片及话题内容 - ResiTopicAndImgResultDTO topicInfo = topicService.getTopicAndImgs(project.getTopicId(), "image"); - if (topicInfo != null) { - List topicImgList = topicInfo.getTopicImgList(); - if (!CollectionUtils.isEmpty(topicImgList)){ - diffEntity.setEventImgUrl(topicImgList.get(NumConstant.ZERO).getAttachmentUrl()); - topicImgList.forEach(img -> { - ScreenDifficultyImgDataEntity imgData = new ScreenDifficultyImgDataEntity(); - imgData.setEventId(project.getId()); - imgData.setEventImgUrl(img.getAttachmentUrl()); - imgData.setSort(img.getSort()); - imgDataEntities.add(imgData); - imgData.setCustomerId(param.getCustomerId()); - }); + //2.获取项目的所有节点耗时数据 + List projectPeriodList = difficultyBaseList.stream() + .filter(o->o.getPeriodList() != null) + .flatMap(o->o.getPeriodList().stream()) + .collect(Collectors.toList()); + //处理部门数去重（只要流转到就算） + Map> orgCountMap = new HashMap<>(); + //被处理次数 + Map handleCountMap = new HashMap<>(); + Set valideProjectIdSet = fillAnyCountAndValidProject(param, projectPeriodList, orgCountMap, handleCountMap); + + List diffList = new ArrayList<>(); + List imgDataEntities = new ArrayList<>(); + for (FactOriginProjectMainAndPeriodDTO project : difficultyBaseList) { + if (agencyMap.get(project.getAgencyId()) == null || bizProjectInfoMap.get(project.getId()) == null) { + log.warn("未获取到相关的项目信息或者项目的所属组织信息,agencyId:{},projectId:{}",project.getAgencyId(),project.getId()); + continue; + } + //判断是否符合难点堵点条件 + if (!valideProjectIdSet.contains(project.getId())) { + continue; + } + ScreenDifficultyDataEntity diffEntity = buildBaseDiffEntity(project, agencyMap, gridMap, bizProjectInfoMap); + Set regCount = orgCountMap.get(project.getId()); + if (regCount != null) { + diffEntity.setEventReOrg(regCount.size()); + } else { + diffEntity.setEventReOrg(0); + } + Integer handleOrgCount = handleCountMap.get(project.getId()); + if (handleOrgCount != null) { + diffEntity.setEventHandledCount(handleOrgCount); + } else { + diffEntity.setEventHandledCount(0); + } + + diffEntity.setDataEndTime(StringUtils.isBlank(param.getDateId())?DateUtils.getBeforeNDay(NumConstant.ONE):param.getDateId()); + setCategoryInfo(diffEntity, project.getId(), categoryList, projectCategoryMap); + + + if (StringUtils.isNotBlank(project.getTopicId())) { + //获取图片及话题内容 + ResiTopicAndImgResultDTO topicInfo = topicService.getTopicAndImgs(project.getTopicId(), "image"); + if (topicInfo != null) { + List topicImgList = topicInfo.getTopicImgList(); + if (!CollectionUtils.isEmpty(topicImgList)){ + diffEntity.setEventImgUrl(topicImgList.get(NumConstant.ZERO).getAttachmentUrl()); + topicImgList.forEach(img -> { + ScreenDifficultyImgDataEntity imgData = new ScreenDifficultyImgDataEntity(); + imgData.setEventId(project.getId()); + imgData.setEventImgUrl(img.getAttachmentUrl()); + imgData.setSort(img.getSort()); + imgDataEntities.add(imgData); + imgData.setCustomerId(param.getCustomerId()); + }); + } + diffEntity.setEventContent(topicInfo.getTopicContent()); + + log.info("projectId:{} imgs:{}", project.getId(), imgDataEntities.stream().filter(o -> o.getEventId().equals(project.getId())).count()); } - diffEntity.setEventContent(topicInfo.getTopicContent()); - log.info("projectId:{} imgs:{}", project.getId(), imgDataEntities.stream().filter(o -> o.getEventId().equals(project.getId())).count()); } - + diffList.add(diffEntity); } - diffList.add(diffEntity); - } - log.info("========:" + JSON.toJSONString(diffList)); - log.info("========:" + JSON.toJSONString(imgDataEntities)); - - screenDifficultyDataService.dataClean(param.getCustomerId(),diffList,imgDataEntities); - } while (!CollectionUtils.isEmpty(difficultyBaseList)&&difficultyBaseList.size()==pageSize); - - //log.info("【大屏数据抽取-难点赌点执行完毕】 客户Id{} 难点赌点数据{}", param.getCustomerId(), JSON.toJSONString(diffList)); - //3.获取项目的最后操作记录 - return true; + log.info("========:" + JSON.toJSONString(diffList)); + log.info("========:" + JSON.toJSONString(imgDataEntities)); + screenDifficultyDataService.dataClean(param.getCustomerId(),diffList,imgDataEntities); + log.info("difficultyDataExtract excute pageNo:{}",pageNo); + } while (!CollectionUtils.isEmpty(difficultyBaseList)&&difficultyBaseList.size()==pageSize); + + //3.获取项目的最后操作记录 + return true; + } finally { + log.info("【大屏数据抽取-难点赌点执行完毕】 客户Id{}", param.getCustomerId()); + } } /** diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/evaluationindex/extract/FactOriginProjectMainDailyDao.xml b/epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/evaluationindex/extract/FactOriginProjectMainDailyDao.xml index 1d34dfb47b..a6ba0e7a8b 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/evaluationindex/extract/FactOriginProjectMainDailyDao.xml +++ b/epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/evaluationindex/extract/FactOriginProjectMainDailyDao.xml @@ -668,10 +668,19 @@ m.id, m.date_id dateId, m.customer_id, m.issue_id, m.topic_id, m.grid_id, m.pid, m.pids, m.agency_id, m.project_status, m.is_resolved, m.topic_creator_id, m.is_party, m.is_overdue, m.finish_org_ids + + , IFNULL(b.DATE_ID, DATE_FORMAT( now(), '%Y%m%d' )) as updateDate + FROM fact_origin_project_main_daily m + + LEFT JOIN fact_origin_project_log_daily b ON m.ID = b.PROJECT_ID AND b.ACTION_CODE = 'close' + WHERE - m.CUSTOMER_ID = #{customerId} + m.CUSTOMER_ID = #{param.customerId} AND m.DEL_FLAG = '0' + + HAVING updateDate>= #{param.internalDay} + LIMIT #{offset} ,#{pageSize}