From fd417f7101dbc242804f45a9e75f8c8ffa06f297 Mon Sep 17 00:00:00 2001 From: jianjun Date: Tue, 14 Dec 2021 12:59:00 +0800 Subject: [PATCH] =?UTF-8?q?=E7=BB=9F=E4=B8=80=E6=8A=BD=E5=8F=96=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=20=E6=B7=BB=E5=8A=A0=E5=8F=82=E6=95=B0=20=E6=96=B9?= =?UTF-8?q?=E4=BE=BF=E9=83=A8=E5=88=86=E6=8A=BD=E5=8F=96=E4=BB=A5=E5=8F=8A?= =?UTF-8?q?=E9=9A=BE=E7=82=B9=E5=A0=B5=E7=82=B9bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../commons/tools/enums/BizTypeEnum.java | 22 +++- .../form/AggregationExtractFormDTO.java | 7 ++ .../FactAggregationExtractController.java | 102 +++++++++++++++--- .../service/impl/StatsProjectServiceImpl.java | 35 +----- .../impl/StatsPublicityServiceImpl.java | 2 +- .../main/resources/mapper/topic/TopicDao.xml | 5 +- .../BizPointUserTotalDetailServiceImpl.java | 5 - 7 files changed, 118 insertions(+), 60 deletions(-) diff --git a/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/enums/BizTypeEnum.java b/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/enums/BizTypeEnum.java index 1dec273299..8edc31e7e2 100644 --- a/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/enums/BizTypeEnum.java +++ b/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/enums/BizTypeEnum.java @@ -11,10 +11,24 @@ import lombok.Getter; @Getter @AllArgsConstructor public enum BizTypeEnum { - //枚举类型 + //枚举类型 user,group,topic,issue,project,article + USER("user", "用户"), GROUP("group", "小组"), - ACTIVITY("activity", "活动"), - AGENCY("agency", "组织"); + TOPIC("topic", "话题"), + ISSUE("issue", "议题"), + PROJECT("project", "项目"), + ARTICLE("article", "文章（党建声音）"), + ; + + public static BizTypeEnum getEnum(String code) { + BizTypeEnum[] values = BizTypeEnum.values(); + for (BizTypeEnum value : values) { + if (value.getType().equals(code)) { + return value; + } + } + return null; + } /** * 类型 @@ -23,6 +37,6 @@ public enum BizTypeEnum { /** * 描述 */ - private String typeDesc; + private String desc; } diff --git a/epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/dto/extract/form/AggregationExtractFormDTO.java b/epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/dto/extract/form/AggregationExtractFormDTO.java index e307587313..1dd051f139 100644 --- a/epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/dto/extract/form/AggregationExtractFormDTO.java +++ b/epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/dto/extract/form/AggregationExtractFormDTO.java @@ -3,6 +3,7 @@ package com.epmet.dto.extract.form; import lombok.Data; import java.io.Serializable; +import java.util.Set; /** * desc: 从业务数据抽取到统计库Form DTO @@ -30,4 +31,10 @@ public class AggregationExtractFormDTO implements Serializable { * 抽取类型 monthly，daily */ private String type; + + /** + * 要抽取的具体业务类型 不传默认所有 + * @see com.epmet.commons.tools.enums.BizTypeEnum + */ + private Set bizType; } diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/FactAggregationExtractController.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/FactAggregationExtractController.java index 9d833335c5..c05e3d4b7b 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/FactAggregationExtractController.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/FactAggregationExtractController.java @@ -1,7 +1,12 @@ package com.epmet.controller; +import cn.hutool.core.thread.ThreadUtil; import com.alibaba.fastjson.JSON; +import com.epmet.commons.tools.constant.NumConstant; import com.epmet.commons.tools.constant.StrConstant; +import com.epmet.commons.tools.enums.BizTypeEnum; +import com.epmet.commons.tools.exception.EpmetErrorCode; +import com.epmet.commons.tools.exception.EpmetException; import com.epmet.commons.tools.utils.DateUtils; import com.epmet.commons.tools.utils.Result; import com.epmet.dto.StatsFormDTO; @@ -15,8 +20,10 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import java.util.Date; import java.util.List; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; /** * @author liujianjun @@ -50,42 +57,101 @@ public class FactAggregationExtractController { * @date 2020/11/10 3:14 下午 */ @RequestMapping("extractDailyOrMonthly") - public Result extractDaily(@RequestBody AggregationExtractFormDTO formDTO) { + public Result extractDailyOrMonthly(@RequestBody AggregationExtractFormDTO formDTO) { if (StringUtils.isBlank(formDTO.getStartDate()) && StringUtils.isBlank(formDTO.getEndDate())) { log.error("抽取日期范围不能为空"); return new Result().error(); } - if (formDTO.getStartDate().indexOf(StrConstant.HYPHEN) < 0 || formDTO.getEndDate().indexOf(StrConstant.HYPHEN) < 0) { + if (!formDTO.getStartDate().contains(StrConstant.HYPHEN) || !formDTO.getEndDate().contains(StrConstant.HYPHEN)) { log.error("抽取日期范围格式不正确,yyyy-MM-dd"); return new Result().error(); } + + BizTypeEnum bizTypeEnum = null; + + Set bizType = formDTO.getBizType(); + for (String item : bizType) { + bizTypeEnum = BizTypeEnum.getEnum(item); + if (bizTypeEnum == null) { + log.error("抽取业务类型不正确,BizTypeEnum"); + throw new EpmetException(EpmetErrorCode.SERVER_ERROR.getCode()); + } + } + + List daysBetween = DateUtils.getDaysBetween(formDTO.getStartDate(), formDTO.getEndDate(), DateUtils.DATE_PATTERN, DateUtils.DATE_PATTERN); - daysBetween.forEach(dateId -> { - Date date = DateUtils.parse(dateId, DateUtils.DATE_PATTERN); + for (String dateId : daysBetween) { StatsFormDTO param = new StatsFormDTO(); param.setCustomerId(formDTO.getCustomerId()); param.setDate(dateId); - switch (formDTO.getType()) { - case "monthly": + if ("monthly".equals(formDTO.getType())) { + if (bizType.isEmpty() || bizType.contains(BizTypeEnum.GROUP.getType())) { extractGroupDataMonthly(param); + } + if (bizType.isEmpty() || bizType.contains(BizTypeEnum.ARTICLE.getType())) { extractArticleDataMonthlyAndQuarterly(param); + } + } else { + //todo 如果要加方法 记得这个枚举里要加 + int threadCount = bizType.isEmpty() ? BizTypeEnum.values().length : bizType.size(); + System.out.println(threadCount); + CountDownLatch countDownLatch = ThreadUtil.newCountDownLatch(threadCount); + if (bizType.isEmpty() || bizType.contains(BizTypeEnum.USER.getType())) { + ThreadUtil.execute(() -> { + extractUserDataDaily(param); + countDownLatch.countDown(); + }); + } + if (bizType.isEmpty() || bizType.contains(BizTypeEnum.GROUP.getType())) { + ThreadUtil.execute(() -> { + extractGroupDataDaily(param); + countDownLatch.countDown(); + }); + } + if (bizType.isEmpty() || bizType.contains(BizTypeEnum.TOPIC.getType())) { + ThreadUtil.execute(() -> { + extractTopicDataDaily(param); + countDownLatch.countDown(); + }); + } + if (bizType.isEmpty() || bizType.contains(BizTypeEnum.ISSUE.getType())) { + ThreadUtil.execute(() -> { + extractIssueDataDaily(param); + countDownLatch.countDown(); + }); + } + if (bizType.isEmpty() || bizType.contains(BizTypeEnum.PROJECT.getType())) { + ThreadUtil.execute(() -> { + extractProjectDataDaily(param); + countDownLatch.countDown(); + }); + } + if (bizType.isEmpty() || bizType.contains(BizTypeEnum.ARTICLE.getType())) { + ThreadUtil.execute(() -> { + extractAtricleDataDaily(param); + countDownLatch.countDown(); + }); + } + try { + //如果半小时还没执行完 则中断 说明执行方法有问题 要优化 + countDownLatch.await(NumConstant.THIRTY, TimeUnit.MINUTES); + } catch (InterruptedException e) { + log.error("extractDailyOrMonthly dateId:{} 始终没有结束 中断操作 InterruptedException", dateId); break; - default: - extractUserDataDaily(param); - extractGroupDataDaily(param); - extractTopicDataDaily(param); - extractIssueDataDaily(param); - extractProjectDataDaily(param); - extractAtricleDataDaily(param); + } } - }); + + log.info("extractDailyOrMonthly dateId:{} run end", dateId); + } + return new Result(); } private void extractUserDataDaily(StatsFormDTO formDTO) { try { statsUserService.partition(formDTO); + log.info("extractUserDataDaily end param:{}", JSON.toJSONString(formDTO)); } catch (Exception e) { log.error("抽取【用户】数据失败", e); } @@ -103,6 +169,7 @@ public class FactAggregationExtractController { private void extractIssueDataDaily(StatsFormDTO formDTO) { try { statsIssueService.agencyGridIssueStats(formDTO); + log.info("extractIssueDataDaily end param:{}", JSON.toJSONString(formDTO)); } catch (Exception e) { log.error("抽取【议题】数据失败", e); } @@ -112,6 +179,7 @@ public class FactAggregationExtractController { try { statsProjectService.gridProjectStats(formDTO); statsProjectService.agencyProjectStats(formDTO); + log.info("extractProjectDataDaily end param:{}", JSON.toJSONString(formDTO)); } catch (Exception e) { log.error("抽取【项目】数据失败", e); } @@ -122,6 +190,7 @@ public class FactAggregationExtractController { statsPublicityService.articleSummaryDailyStatsjob(formDTO); statsPublicityService.tagUsedDailyStatsjob(formDTO); statsPublicityService.tagViewedDailyStatsjob(formDTO); + log.info("extractAtricleDataDaily end param:{}", JSON.toJSONString(formDTO)); } catch (Exception e) { log.error("抽取【文章】数据失败", e); } @@ -135,6 +204,7 @@ public class FactAggregationExtractController { statsGroupService.groupGridDaily(groupStatsFormDTO); statsGroupService.groupAgencyDaily(groupStatsFormDTO); + log.info("extractGroupDataDaily end param:{}", JSON.toJSONString(formDTO)); } catch (Exception e) { log.error("抽取【小组】数据失败", e); } @@ -147,6 +217,7 @@ public class FactAggregationExtractController { groupStatsFormDTO.setCustomerId(formDTO.getCustomerId()); statsGroupService.groupAgencyMonthly(groupStatsFormDTO); + log.info("extractGroupDataMonthly end param:{}", JSON.toJSONString(formDTO)); } catch (Exception e) { log.error("抽取【小组】数据失败", e); } @@ -158,6 +229,7 @@ public class FactAggregationExtractController { statsPublicityService.tagUsedQuarterlyStatsjob(formDTO); statsPublicityService.tagViewedMonthlyStatsjob(formDTO); statsPublicityService.tagViewedQuarterlyStatsjob(formDTO); + log.info("extractArticleDataMonthlyAndQuarterly end param:{}", JSON.toJSONString(formDTO)); } catch (Exception e) { log.error("抽取【文章】数据失败", e); } diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsProjectServiceImpl.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsProjectServiceImpl.java index ca28b9fd3e..3b1d1c45c8 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsProjectServiceImpl.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsProjectServiceImpl.java @@ -77,7 +77,6 @@ public class StatsProjectServiceImpl implements StatsProjectService { } //入参有客户Id的则按客户Id执行，没有的则全部客户都执行 if (null != formDTO && StringUtils.isNotBlank(formDTO.getCustomerId())) { - log.info("单独统计客户机关-项目-日月数据，当前统计的客户Id:" + formDTO.getCustomerId()); customerAgencyStats(formDTO.getCustomerId(), date); } else { int pageNo = 1; @@ -88,7 +87,6 @@ public class StatsProjectServiceImpl implements StatsProjectService { if (!CollectionUtils.isEmpty(customerIdList)) { for (String customerId : customerIdList) { try { - log.info("for循环统计机关-项目-日月数据，当前统计的客户Id:" + customerId); //遍历统计每一个客户数据 customerAgencyStats(customerId, date); } catch (Exception e) { @@ -110,13 +108,11 @@ public class StatsProjectServiceImpl implements StatsProjectService { DimIdGenerator.DimIdBean dimId = DimIdGenerator.getDimIdBean(date); //2:根据客户Id查询机关维度表数据 - log.info("StatsProjectServiceImpl.customerAgencyStats-根据客户Id查询机关维度数据，对应客户Id:" + customerId); DimAgencyDTO dimAgencyDTO = new DimAgencyDTO(); dimAgencyDTO.setCustomerId(customerId); List dimAgencyList = dimAgencyService.getDimAgencyList(dimAgencyDTO); //3:根据客户Id查询项目业务表已结案数据(查询传入日期及之前的数据) - log.info("StatsProjectServiceImpl.customerAgencyStats-根据客户Id查询项目业务表已结案数据，对应客户Id:" + customerId); ProjectEntity projectEntity = new ProjectEntity(); projectEntity.setCustomerId(customerId); projectEntity.setCreatedTime(date); @@ -124,7 +120,6 @@ public class StatsProjectServiceImpl implements StatsProjectService { List projectList = projectService.getProjectList(projectEntity); //4:查询项目处理进展表中有效数据(创建日期截取yyyy-mm-dd格式字段值)(查询传入日期及之前的数据) - log.info("StatsProjectServiceImpl.customerAgencyStats-根据客户Id查询项目进展表业务数据，对应客户Id:" + customerId); List processList = projectProcessService.getProcessList(projectEntity); //20210721 sun 业务逻辑调整，网格删除，组织没有删除情况，相应的组织层级数据统计应舍弃以删除网格数据 start @@ -134,21 +129,9 @@ public class StatsProjectServiceImpl implements StatsProjectService { if (list.size() > NumConstant.ZERO) { Map map = list.stream().collect(Collectors.toMap(String::toString, v -> v, (c1, c2) -> c1)); //4-2.遍历删除项目主表查询的无效数据 - Iterator proiter = projectList.iterator(); - while (proiter.hasNext()) { - ProjectEntity next = proiter.next(); - if (map.containsKey(next.getId())) { - proiter.remove(); - } - } + projectList.removeIf(next -> map.containsKey(next.getId())); //4-3.遍历删除项目节点表查询的无效数据 - Iterator iterator = processList.iterator(); - while (iterator.hasNext()) { - ProjectProcessEntity next = iterator.next(); - if (map.containsKey(next.getProjectId())) { - iterator.remove(); - } - } + processList.removeIf(next -> map.containsKey(next.getProjectId())); } //20210721 sun end @@ -156,21 +139,17 @@ public class StatsProjectServiceImpl implements StatsProjectService { if (null != dimAgencyList && dimAgencyList.size() > NumConstant.ZERO) { //5.1:执行机关日数据统计 try { - log.info("StatsProjectServiceImpl.customerAgencyStats-开始执行机关日统计方法，方法名:agencyDateProjectStats,客户Id:" + customerId); agencyDateProjectStats(customerId, dimId, date, dimAgencyList, projectList, processList); } catch (Exception e) { log.error(String.format(ProjectConstant.STATS_FAILED_PREFIX, "agencyDateProjectStats", customerId, dimId.getDateId()), e); } //5.2:执行机关月数据统计 - //if (Calendar.getInstance().get(Calendar.DATE) == 1) { try { - log.info("StatsProjectServiceImpl.customerAgencyStats-开始执行机关月统计方法，方法名:agencyMonthProjectStats,客户Id:" + customerId); agencyMonthProjectStats(customerId, dimId, dimAgencyList); } catch (Exception e) { log.error(String.format(ProjectConstant.STATS_FAILED_PREFIX, "agencyMonthProjectStats", customerId, dimId.getMonthId()), e); } - //} } } @@ -404,7 +383,6 @@ public class StatsProjectServiceImpl implements StatsProjectService { } //入参有客户Id的则按客户Id执行，没有的则全部客户都执行 if (null != formDTO && StringUtils.isNotBlank(formDTO.getCustomerId())) { - log.info("单独统计客户网格-项目-日月数据，当前统计的客户Id:" + formDTO.getCustomerId()); customerGridStats(formDTO.getCustomerId(), date); } else { int pageNo = 1; @@ -415,11 +393,9 @@ public class StatsProjectServiceImpl implements StatsProjectService { if (!CollectionUtils.isEmpty(customerIdList)) { for (String customerId : customerIdList) { try { - log.info("for循环统计网格-项目-日月数据，当前统计的客户Id:" + customerId); //遍历统计每一个客户数据 customerGridStats(customerId, date); } catch (Exception e) { - log.error("项目-网格-数据统计程序错误，对应客户Id:" + customerId, e); log.error("Error creating model JSON", e); } } @@ -437,11 +413,9 @@ public class StatsProjectServiceImpl implements StatsProjectService { DimIdGenerator.DimIdBean dimId = DimIdGenerator.getDimIdBean(date); //2:根据客户Id查询网格维度表数据 - log.info("StatsProjectServiceImpl.customerGridStats-根据客户Id查询网格维度数据，对应客户Id:" + customerId); List dimGridList = dimGridService.getGridListByCustomerId(customerId); //3:根据客户Id查询项目业务表已结案数据(查询传入日期及之前的数据) - log.info("StatsProjectServiceImpl.customerGridStats-根据客户Id查询项目业务表已结案数据，对应客户Id:" + customerId); ProjectEntity projectEntity = new ProjectEntity(); projectEntity.setCustomerId(customerId); projectEntity.setCreatedTime(date); @@ -449,14 +423,12 @@ public class StatsProjectServiceImpl implements StatsProjectService { List projectList = projectService.getProjectList(projectEntity); //4:查询项目处理进展表中有效数据(创建日期截取yyyy-mm-dd格式字段值)(查询传入日期及之前的数据) - log.info("StatsProjectServiceImpl.customerGridStats-根据客户Id查询项目进展表业务数据，对应客户Id:" + customerId); List processList = projectProcessService.getProcessList(projectEntity); //5:网格层级日月统计 if (null != dimGridList && dimGridList.size() > NumConstant.ZERO) { //5.1:执行网格日数据统计 try { - log.info("StatsProjectServiceImpl.customerGridStats-开始执行机关日统计方法，方法名:gridDateProjectStats,客户Id:" + customerId); gridDateProjectStats(customerId, dimId, date, dimGridList, projectList, processList); } catch (Exception e) { log.error(String.format(ProjectConstant.STATS_FAILED_PREFIX, "gridDateProjectStats", customerId, dimId.getDateId()), e); @@ -464,7 +436,6 @@ public class StatsProjectServiceImpl implements StatsProjectService { //5.2:执行网格月数据统计 try { - log.info("StatsProjectServiceImpl.customerGridStats-开始执行机关月统计方法，方法名:gridMonthProjectStats,客户Id:" + customerId); gridMonthProjectStats(customerId, dimId, dimGridList); } catch (Exception e) { log.error(String.format(ProjectConstant.STATS_FAILED_PREFIX, "gridMonthProjectStats", customerId, dimId.getMonthId()), e); @@ -711,4 +682,4 @@ public class StatsProjectServiceImpl implements StatsProjectService { return date; } -} \ No newline at end of file +} diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsPublicityServiceImpl.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsPublicityServiceImpl.java index 397fcefa7a..2d77962df5 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsPublicityServiceImpl.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsPublicityServiceImpl.java @@ -404,7 +404,7 @@ public class StatsPublicityServiceImpl implements StatsPublicityService { for (ArticleGridPublishedSummaryDTO summaryDTO : publishedArticleCount) { FactArticlePublishedDepartmentDailyEntity gridDailyEntities = departmentDailyEntityMap.get(summaryDTO.getPublisherId()); if (gridDailyEntities == null) { - log.error("publicitySummary bizData departmentId:{} not exist in dimDepartment", summaryDTO.getGridId()); + log.warn("publicitySummary bizData departmentId:{} not exist in dimDepartment", summaryDTO.getGridId()); continue; } gridDailyEntities.setArticleTotalCount(summaryDTO.getArticleTotalCount()); diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/topic/TopicDao.xml b/epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/topic/TopicDao.xml index 09754ca373..3bda2d3cc5 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/topic/TopicDao.xml +++ b/epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/topic/TopicDao.xml @@ -290,11 +290,10 @@ a.ATTACHMENT_URL, a.SORT from resi_topic t - LEFT JOIN resi_topic_attachment a ON t.ID = a.TOPIC_ID + LEFT JOIN resi_topic_attachment a ON t.ID = a.TOPIC_ID AND ATTACHMENT_TYPE = 'image' WHERE t.del_flag = '0' - AND ATTACHMENT_TYPE = 'image' - and TOPIC_ID = #{topicId} + and t.ID = #{topicId}