统一抽取接口 添加参数 方便部分抽取以及难点堵点bug

epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/enums/BizTypeEnum.java

@@ -11,10 +11,24 @@ import lombok.Getter;
 @Getter
 @AllArgsConstructor
 public enum BizTypeEnum {
-	//枚举类型
+	//枚举类型 user,group,topic,issue,project,article
+	USER("user", "用户"),
 	GROUP("group", "小组"),
-	ACTIVITY("activity", "活动"),
+	TOPIC("topic", "话题"),
-	AGENCY("agency", "组织");
+	ISSUE("issue", "议题"),
+	PROJECT("project", "项目"),
+	ARTICLE("article", "文章（党建声音）"),
+	;
+
+	public static BizTypeEnum getEnum(String code) {
+		BizTypeEnum[] values = BizTypeEnum.values();
+		for (BizTypeEnum value : values) {
+			if (value.getType().equals(code)) {
+				return value;
+			}
+		}
+		return null;
+	}
 
 	/**
 	 * 类型
@@ -23,6 +37,6 @@ public enum BizTypeEnum {
 	/**
 	 * 描述
 	 */
-	private String typeDesc;
+	private String desc;
 
 }

epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/dto/extract/form/AggregationExtractFormDTO.java

@@ -3,6 +3,7 @@ package com.epmet.dto.extract.form;
 import lombok.Data;
 
 import java.io.Serializable;
+import java.util.Set;
 
 /**
  * desc: 从业务数据抽取到统计库Form DTO
@@ -30,4 +31,10 @@ public class AggregationExtractFormDTO implements Serializable {
      * 抽取类型 monthly，daily
      */
     private String type;
+
+    /**
+     * 要抽取的具体业务类型 不传默认所有
+     * @see com.epmet.commons.tools.enums.BizTypeEnum
+     */
+    private Set<String> bizType;
 }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/FactAggregationExtractController.java

@@ -1,7 +1,12 @@
 package com.epmet.controller;
 
+import cn.hutool.core.thread.ThreadUtil;
 import com.alibaba.fastjson.JSON;
+import com.epmet.commons.tools.constant.NumConstant;
 import com.epmet.commons.tools.constant.StrConstant;
+import com.epmet.commons.tools.enums.BizTypeEnum;
+import com.epmet.commons.tools.exception.EpmetErrorCode;
+import com.epmet.commons.tools.exception.EpmetException;
 import com.epmet.commons.tools.utils.DateUtils;
 import com.epmet.commons.tools.utils.Result;
 import com.epmet.dto.StatsFormDTO;
@@ -15,8 +20,10 @@ import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
 
-import java.util.Date;
 import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 
 /**
  * @author liujianjun
@@ -50,42 +57,101 @@ public class FactAggregationExtractController {
      * @date 2020/11/10 3:14 下午
      */
     @RequestMapping("extractDailyOrMonthly")
-    public Result extractDaily(@RequestBody AggregationExtractFormDTO formDTO) {
+    public Result extractDailyOrMonthly(@RequestBody AggregationExtractFormDTO formDTO) {
         if (StringUtils.isBlank(formDTO.getStartDate()) && StringUtils.isBlank(formDTO.getEndDate())) {
             log.error("抽取日期范围不能为空");
             return new Result().error();
         }
-        if (formDTO.getStartDate().indexOf(StrConstant.HYPHEN) < 0 || formDTO.getEndDate().indexOf(StrConstant.HYPHEN) < 0) {
+        if (!formDTO.getStartDate().contains(StrConstant.HYPHEN) || !formDTO.getEndDate().contains(StrConstant.HYPHEN)) {
             log.error("抽取日期范围格式不正确,yyyy-MM-dd");
             return new Result().error();
         }
+
+        BizTypeEnum bizTypeEnum = null;
+
+        Set<String> bizType = formDTO.getBizType();
+        for (String item : bizType) {
+            bizTypeEnum = BizTypeEnum.getEnum(item);
+            if (bizTypeEnum == null) {
+                log.error("抽取业务类型不正确,BizTypeEnum");
+                throw new EpmetException(EpmetErrorCode.SERVER_ERROR.getCode());
+            }
+        }
+
+
         List<String> daysBetween = DateUtils.getDaysBetween(formDTO.getStartDate(), formDTO.getEndDate(), DateUtils.DATE_PATTERN, DateUtils.DATE_PATTERN);
-        daysBetween.forEach(dateId -> {
+        for (String dateId : daysBetween) {
-            Date date = DateUtils.parse(dateId, DateUtils.DATE_PATTERN);
             StatsFormDTO param = new StatsFormDTO();
             param.setCustomerId(formDTO.getCustomerId());
             param.setDate(dateId);
 
-            switch (formDTO.getType()) {
+            if ("monthly".equals(formDTO.getType())) {
-                case "monthly":
+                if (bizType.isEmpty() || bizType.contains(BizTypeEnum.GROUP.getType())) {
                     extractGroupDataMonthly(param);
+                }
+                if (bizType.isEmpty() || bizType.contains(BizTypeEnum.ARTICLE.getType())) {
                     extractArticleDataMonthlyAndQuarterly(param);
+                }
+            } else {
+                //todo 如果要加方法 记得这个枚举里要加
+                int threadCount = bizType.isEmpty() ? BizTypeEnum.values().length : bizType.size();
+                System.out.println(threadCount);
+                CountDownLatch countDownLatch = ThreadUtil.newCountDownLatch(threadCount);
+                if (bizType.isEmpty() || bizType.contains(BizTypeEnum.USER.getType())) {
+                    ThreadUtil.execute(() -> {
+                        extractUserDataDaily(param);
+                        countDownLatch.countDown();
+                    });
+                }
+                if (bizType.isEmpty() || bizType.contains(BizTypeEnum.GROUP.getType())) {
+                    ThreadUtil.execute(() -> {
+                        extractGroupDataDaily(param);
+                        countDownLatch.countDown();
+                    });
+                }
+                if (bizType.isEmpty() || bizType.contains(BizTypeEnum.TOPIC.getType())) {
+                    ThreadUtil.execute(() -> {
+                        extractTopicDataDaily(param);
+                        countDownLatch.countDown();
+                    });
+                }
+                if (bizType.isEmpty() || bizType.contains(BizTypeEnum.ISSUE.getType())) {
+                    ThreadUtil.execute(() -> {
+                        extractIssueDataDaily(param);
+                        countDownLatch.countDown();
+                    });
+                }
+                if (bizType.isEmpty() || bizType.contains(BizTypeEnum.PROJECT.getType())) {
+                    ThreadUtil.execute(() -> {
+                        extractProjectDataDaily(param);
+                        countDownLatch.countDown();
+                    });
+                }
+                if (bizType.isEmpty() || bizType.contains(BizTypeEnum.ARTICLE.getType())) {
+                    ThreadUtil.execute(() -> {
+                        extractAtricleDataDaily(param);
+                        countDownLatch.countDown();
+                    });
+                }
+                try {
+                    //如果半小时还没执行完 则中断 说明执行方法有问题 要优化
+                    countDownLatch.await(NumConstant.THIRTY, TimeUnit.MINUTES);
+                } catch (InterruptedException e) {
+                    log.error("extractDailyOrMonthly dateId:{} 始终没有结束 中断操作 InterruptedException", dateId);
                     break;
-                default:
+                }
-                    extractUserDataDaily(param);
-                    extractGroupDataDaily(param);
-                    extractTopicDataDaily(param);
-                    extractIssueDataDaily(param);
-                    extractProjectDataDaily(param);
-                    extractAtricleDataDaily(param);
             }
-        });
+
+            log.info("extractDailyOrMonthly dateId:{} run end", dateId);
+        }
+
         return new Result();
     }
 
     private void extractUserDataDaily(StatsFormDTO formDTO) {
         try {
             statsUserService.partition(formDTO);
+            log.info("extractUserDataDaily end param:{}", JSON.toJSONString(formDTO));
         } catch (Exception e) {
             log.error("抽取【用户】数据失败", e);
         }
@@ -103,6 +169,7 @@ public class FactAggregationExtractController {
     private void extractIssueDataDaily(StatsFormDTO formDTO) {
         try {
             statsIssueService.agencyGridIssueStats(formDTO);
+            log.info("extractIssueDataDaily end param:{}", JSON.toJSONString(formDTO));
         } catch (Exception e) {
             log.error("抽取【议题】数据失败", e);
         }
@@ -112,6 +179,7 @@ public class FactAggregationExtractController {
         try {
             statsProjectService.gridProjectStats(formDTO);
             statsProjectService.agencyProjectStats(formDTO);
+            log.info("extractProjectDataDaily end param:{}", JSON.toJSONString(formDTO));
         } catch (Exception e) {
             log.error("抽取【项目】数据失败", e);
         }
@@ -122,6 +190,7 @@ public class FactAggregationExtractController {
             statsPublicityService.articleSummaryDailyStatsjob(formDTO);
             statsPublicityService.tagUsedDailyStatsjob(formDTO);
             statsPublicityService.tagViewedDailyStatsjob(formDTO);
+            log.info("extractAtricleDataDaily end param:{}", JSON.toJSONString(formDTO));
         } catch (Exception e) {
             log.error("抽取【文章】数据失败", e);
         }
@@ -135,6 +204,7 @@ public class FactAggregationExtractController {
 
             statsGroupService.groupGridDaily(groupStatsFormDTO);
             statsGroupService.groupAgencyDaily(groupStatsFormDTO);
+            log.info("extractGroupDataDaily end param:{}", JSON.toJSONString(formDTO));
         } catch (Exception e) {
             log.error("抽取【小组】数据失败", e);
         }
@@ -147,6 +217,7 @@ public class FactAggregationExtractController {
             groupStatsFormDTO.setCustomerId(formDTO.getCustomerId());
 
             statsGroupService.groupAgencyMonthly(groupStatsFormDTO);
+            log.info("extractGroupDataMonthly end param:{}", JSON.toJSONString(formDTO));
         } catch (Exception e) {
             log.error("抽取【小组】数据失败", e);
         }
@@ -158,6 +229,7 @@ public class FactAggregationExtractController {
             statsPublicityService.tagUsedQuarterlyStatsjob(formDTO);
             statsPublicityService.tagViewedMonthlyStatsjob(formDTO);
             statsPublicityService.tagViewedQuarterlyStatsjob(formDTO);
+            log.info("extractArticleDataMonthlyAndQuarterly end param:{}", JSON.toJSONString(formDTO));
         } catch (Exception e) {
             log.error("抽取【文章】数据失败", e);
         }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsProjectServiceImpl.java

@@ -77,7 +77,6 @@ public class StatsProjectServiceImpl implements StatsProjectService {
         }
         //入参有客户Id的则按客户Id执行，没有的则全部客户都执行
         if (null != formDTO && StringUtils.isNotBlank(formDTO.getCustomerId())) {
-            log.info("单独统计客户机关-项目-日月数据，当前统计的客户Id:" + formDTO.getCustomerId());
             customerAgencyStats(formDTO.getCustomerId(), date);
         } else {
             int pageNo = 1;
@@ -88,7 +87,6 @@ public class StatsProjectServiceImpl implements StatsProjectService {
                 if (!CollectionUtils.isEmpty(customerIdList)) {
                     for (String customerId : customerIdList) {
                         try {
-                            log.info("for循环统计机关-项目-日月数据，当前统计的客户Id:" + customerId);
                             //遍历统计每一个客户数据
                             customerAgencyStats(customerId, date);
                         } catch (Exception e) {
@@ -110,13 +108,11 @@ public class StatsProjectServiceImpl implements StatsProjectService {
         DimIdGenerator.DimIdBean dimId = DimIdGenerator.getDimIdBean(date);
 
         //2:根据客户Id查询机关维度表数据
-        log.info("StatsProjectServiceImpl.customerAgencyStats-根据客户Id查询机关维度数据，对应客户Id:" + customerId);
         DimAgencyDTO dimAgencyDTO = new DimAgencyDTO();
         dimAgencyDTO.setCustomerId(customerId);
         List<DimAgencyDTO> dimAgencyList = dimAgencyService.getDimAgencyList(dimAgencyDTO);
 
         //3:根据客户Id查询项目业务表已结案数据(查询传入日期及之前的数据)
-        log.info("StatsProjectServiceImpl.customerAgencyStats-根据客户Id查询项目业务表已结案数据，对应客户Id:" + customerId);
         ProjectEntity projectEntity = new ProjectEntity();
         projectEntity.setCustomerId(customerId);
         projectEntity.setCreatedTime(date);
@@ -124,7 +120,6 @@ public class StatsProjectServiceImpl implements StatsProjectService {
         List<ProjectEntity> projectList = projectService.getProjectList(projectEntity);
 
         //4:查询项目处理进展表中有效数据(创建日期截取yyyy-mm-dd格式字段值)(查询传入日期及之前的数据)
-        log.info("StatsProjectServiceImpl.customerAgencyStats-根据客户Id查询项目进展表业务数据，对应客户Id:" + customerId);
         List<ProjectProcessEntity> processList = projectProcessService.getProcessList(projectEntity);
 
         //20210721 sun 业务逻辑调整，网格删除，组织没有删除情况，相应的组织层级数据统计应舍弃以删除网格数据  start
@@ -134,21 +129,9 @@ public class StatsProjectServiceImpl implements StatsProjectService {
         if (list.size() > NumConstant.ZERO) {
             Map<String, String> map = list.stream().collect(Collectors.toMap(String::toString, v -> v, (c1, c2) -> c1));
             //4-2.遍历删除项目主表查询的无效数据
-            Iterator<ProjectEntity> proiter = projectList.iterator();
+            projectList.removeIf(next -> map.containsKey(next.getId()));
-            while (proiter.hasNext()) {
-                ProjectEntity next = proiter.next();
-                if (map.containsKey(next.getId())) {
-                    proiter.remove();
-                }
-            }
             //4-3.遍历删除项目节点表查询的无效数据
-            Iterator<ProjectProcessEntity> iterator = processList.iterator();
+            processList.removeIf(next -> map.containsKey(next.getProjectId()));
-            while (iterator.hasNext()) {
-                ProjectProcessEntity next = iterator.next();
-                if (map.containsKey(next.getProjectId())) {
-                    iterator.remove();
-                }
-            }
         }
         //20210721 sun  end
 
@@ -156,21 +139,17 @@ public class StatsProjectServiceImpl implements StatsProjectService {
         if (null != dimAgencyList && dimAgencyList.size() > NumConstant.ZERO) {
             //5.1:执行机关日数据统计
             try {
-                log.info("StatsProjectServiceImpl.customerAgencyStats-开始执行机关日统计方法，方法名:agencyDateProjectStats,客户Id:" + customerId);
                 agencyDateProjectStats(customerId, dimId, date, dimAgencyList, projectList, processList);
             } catch (Exception e) {
                 log.error(String.format(ProjectConstant.STATS_FAILED_PREFIX, "agencyDateProjectStats", customerId, dimId.getDateId()), e);
             }
 
             //5.2:执行机关月数据统计
-            //if (Calendar.getInstance().get(Calendar.DATE) == 1) {
             try {
-                log.info("StatsProjectServiceImpl.customerAgencyStats-开始执行机关月统计方法，方法名:agencyMonthProjectStats,客户Id:" + customerId);
                 agencyMonthProjectStats(customerId, dimId, dimAgencyList);
             } catch (Exception e) {
                 log.error(String.format(ProjectConstant.STATS_FAILED_PREFIX, "agencyMonthProjectStats", customerId, dimId.getMonthId()), e);
             }
-            //}
         }
 
     }
@@ -404,7 +383,6 @@ public class StatsProjectServiceImpl implements StatsProjectService {
         }
         //入参有客户Id的则按客户Id执行，没有的则全部客户都执行
         if (null != formDTO && StringUtils.isNotBlank(formDTO.getCustomerId())) {
-            log.info("单独统计客户网格-项目-日月数据，当前统计的客户Id:" + formDTO.getCustomerId());
             customerGridStats(formDTO.getCustomerId(), date);
         } else {
             int pageNo = 1;
@@ -415,11 +393,9 @@ public class StatsProjectServiceImpl implements StatsProjectService {
                 if (!CollectionUtils.isEmpty(customerIdList)) {
                     for (String customerId : customerIdList) {
                         try {
-                            log.info("for循环统计网格-项目-日月数据，当前统计的客户Id:" + customerId);
                             //遍历统计每一个客户数据
                             customerGridStats(customerId, date);
                         } catch (Exception e) {
-                            log.error("项目-网格-数据统计程序错误，对应客户Id:" + customerId, e);
                             log.error("Error creating model JSON", e);
                         }
                     }
@@ -437,11 +413,9 @@ public class StatsProjectServiceImpl implements StatsProjectService {
         DimIdGenerator.DimIdBean dimId = DimIdGenerator.getDimIdBean(date);
 
         //2:根据客户Id查询网格维度表数据
-        log.info("StatsProjectServiceImpl.customerGridStats-根据客户Id查询网格维度数据，对应客户Id:" + customerId);
         List<DimGridEntity> dimGridList = dimGridService.getGridListByCustomerId(customerId);
 
         //3:根据客户Id查询项目业务表已结案数据(查询传入日期及之前的数据)
-        log.info("StatsProjectServiceImpl.customerGridStats-根据客户Id查询项目业务表已结案数据，对应客户Id:" + customerId);
         ProjectEntity projectEntity = new ProjectEntity();
         projectEntity.setCustomerId(customerId);
         projectEntity.setCreatedTime(date);
@@ -449,14 +423,12 @@ public class StatsProjectServiceImpl implements StatsProjectService {
         List<ProjectEntity> projectList = projectService.getProjectList(projectEntity);
 
         //4:查询项目处理进展表中有效数据(创建日期截取yyyy-mm-dd格式字段值)(查询传入日期及之前的数据)
-        log.info("StatsProjectServiceImpl.customerGridStats-根据客户Id查询项目进展表业务数据，对应客户Id:" + customerId);
         List<ProjectProcessEntity> processList = projectProcessService.getProcessList(projectEntity);
 
         //5:网格层级日月统计
         if (null != dimGridList && dimGridList.size() > NumConstant.ZERO) {
             //5.1:执行网格日数据统计
             try {
-                log.info("StatsProjectServiceImpl.customerGridStats-开始执行机关日统计方法，方法名:gridDateProjectStats,客户Id:" + customerId);
                 gridDateProjectStats(customerId, dimId, date, dimGridList, projectList, processList);
             } catch (Exception e) {
                 log.error(String.format(ProjectConstant.STATS_FAILED_PREFIX, "gridDateProjectStats", customerId, dimId.getDateId()), e);
@@ -464,7 +436,6 @@ public class StatsProjectServiceImpl implements StatsProjectService {
 
             //5.2:执行网格月数据统计
             try {
-                log.info("StatsProjectServiceImpl.customerGridStats-开始执行机关月统计方法，方法名:gridMonthProjectStats,客户Id:" + customerId);
                 gridMonthProjectStats(customerId, dimId, dimGridList);
             } catch (Exception e) {
                 log.error(String.format(ProjectConstant.STATS_FAILED_PREFIX, "gridMonthProjectStats", customerId, dimId.getMonthId()), e);
@@ -711,4 +682,4 @@ public class StatsProjectServiceImpl implements StatsProjectService {
         return date;
     }
 
-}
+}

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsPublicityServiceImpl.java

@@ -404,7 +404,7 @@ public class StatsPublicityServiceImpl implements StatsPublicityService {
 				for (ArticleGridPublishedSummaryDTO summaryDTO : publishedArticleCount) {
 					FactArticlePublishedDepartmentDailyEntity gridDailyEntities = departmentDailyEntityMap.get(summaryDTO.getPublisherId());
 					if (gridDailyEntities == null) {
-						log.error("publicitySummary bizData departmentId:{} not exist in dimDepartment", summaryDTO.getGridId());
+						log.warn("publicitySummary bizData departmentId:{} not exist in dimDepartment", summaryDTO.getGridId());
 						continue;
 					}
 					gridDailyEntities.setArticleTotalCount(summaryDTO.getArticleTotalCount());

epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/topic/TopicDao.xml

@@ -290,11 +290,10 @@
             a.ATTACHMENT_URL,
             a.SORT
         from resi_topic t
-        LEFT JOIN resi_topic_attachment a ON t.ID = a.TOPIC_ID
+        LEFT JOIN resi_topic_attachment a ON t.ID = a.TOPIC_ID  AND ATTACHMENT_TYPE = 'image'
         WHERE
             t.del_flag = '0'
-            AND ATTACHMENT_TYPE = 'image'
+            and t.ID = #{topicId}
-            and TOPIC_ID = #{topicId}
     </select>
 
     <select id="selectTopicList" resultType="com.epmet.dto.stats.topic.result.TopicListResultDTO">

epmet-module/epmet-point/epmet-point-server/src/main/java/com/epmet/service/impl/BizPointUserTotalDetailServiceImpl.java

@@ -23,7 +23,6 @@ import com.epmet.commons.mybatis.service.impl.BaseServiceImpl;
 import com.epmet.commons.tools.constant.FieldConstant;
 import com.epmet.commons.tools.constant.NumConstant;
 import com.epmet.commons.tools.constant.StrConstant;
-import com.epmet.commons.tools.enums.BizTypeEnum;
 import com.epmet.commons.tools.exception.RenException;
 import com.epmet.commons.tools.page.PageData;
 import com.epmet.commons.tools.security.dto.TokenDto;
@@ -160,10 +159,6 @@ public class BizPointUserTotalDetailServiceImpl extends BaseServiceImpl<BizPoint
         return entity.getTotalPoint();
     }
 
-    public static void main(String[] args) {
-        System.out.println(BizTypeEnum.ACTIVITY.getType());
-    }
-
     /**
      * 小组积分贡献榜
      *