业务数据统计抽取

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/DemoController.java

@@ -15,6 +15,7 @@ import com.epmet.dao.evaluationindex.screen.ScreenCustomerGridDao;
 import com.epmet.dao.stats.DimCustomerDao;
 import com.epmet.dao.stats.DimDateDao;
 import com.epmet.dto.AgencySubTreeDto;
+import com.epmet.dto.extract.form.ExtractFormDTO;
 import com.epmet.dto.indexcal.CalculateCommonFormDTO;
 import com.epmet.dto.stats.form.CustomerIdAndDateIdFormDTO;
 import com.epmet.entity.evaluationindex.indexcoll.FactIndexGovrnAblityGridMonthlyEntity;
@@ -503,15 +504,21 @@ public class DemoController {
     @PostMapping("inserttopicorigin")
     public Result topicDataCleaning(@RequestParam("customerId") String customerId, @RequestParam("dateId")String dateId) {
         if (StringUtils.isNotBlank(customerId) && StringUtils.isNotBlank(dateId)) {
-            factOriginTopicMainDailyService.topicCleaning(customerId,dateId);
+            ExtractFormDTO param = new ExtractFormDTO();
+            param.setDateId(dateId);
+            param.setCustomerId(customerId);
+            Boolean aBoolean = factOriginTopicMainDailyService.topicCleaning(param);
         }else{
             QueryWrapper<DimCustomerEntity> customerEntityQueryWrapper = new QueryWrapper<>();
             List<DimCustomerEntity> customerEntityList=dimCustomerDao.selectList(customerEntityQueryWrapper);
             QueryWrapper<DimDateEntity> wrapper = new QueryWrapper<>();
             List<DimDateEntity> dimDateEntityList= dimDateDao.selectList(wrapper);
             for(DimCustomerEntity customerEntity:customerEntityList){
-                for(DimDateEntity dateEntity:dimDateEntityList){
+                for(DimDateEntity dateEntity:dimDateEntityList) {
-                    factOriginTopicMainDailyService.topicCleaning(customerEntity.getId(),dateEntity.getId());
+                    ExtractFormDTO param = new ExtractFormDTO();
+                    param.setDateId(dateEntity.getId());
+                    param.setCustomerId(customerEntity.getId());
+                    factOriginTopicMainDailyService.topicCleaning(param);
                 }
             }
         }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/FactOriginController.java

@@ -3,7 +3,7 @@ package com.epmet.controller;
 import com.epmet.commons.tools.utils.Result;
 import com.epmet.commons.tools.validator.ValidatorUtils;
 import com.epmet.dto.extract.form.ExtractFormDTO;
-import com.epmet.dto.extract.form.ProjectFormDTO;
+import com.epmet.service.evaluationindex.extract.FactOriginExtractService;
 import com.epmet.service.evaluationindex.extract.FactOriginTopicMainDailyService;
 import com.epmet.service.evaluationindex.extract.IssueExtractService;
 import com.epmet.service.evaluationindex.extract.ProjectExtractService;
@@ -24,7 +24,8 @@ import org.springframework.web.bind.annotation.RestController;
 @RestController
 @RequestMapping("factorigin")
 public class FactOriginController {
-
+    @Autowired
+    private FactOriginExtractService factOriginExtractService;
     @Autowired
     private IssueExtractService issueExtractService;
     @Autowired
@@ -33,13 +34,26 @@ public class FactOriginController {
     private ProjectExtractService projectExtractService;
 
     /**
-     * @Description  议题抽取(main)
+     * desc:
+     *
      * @param extractFormDTO
+     * @return
+     */
+    @PostMapping("extractall")
+    public Result extractAll(@RequestBody ExtractFormDTO extractFormDTO) {
+        factOriginExtractService.extractAll(extractFormDTO);
+        return new Result();
+    }
+
+
+    /**
+     * @param extractFormDTO
+     * @Description 议题抽取(main)
      * @author zxc
      * @date 2020/9/15 2:02 下午
      */
     @PostMapping("issueextractmain")
-    public Result issueExtractMain(@RequestBody ExtractFormDTO extractFormDTO){
+    public Result issueExtractMain(@RequestBody ExtractFormDTO extractFormDTO) {
         ValidatorUtils.validateEntity(extractFormDTO, ExtractFormDTO.ExtractForm.class);
         issueExtractService.issueExtractMain(extractFormDTO);
         return new Result();
@@ -59,6 +73,7 @@ public class FactOriginController {
     }
 
     /**
+     * @param extractFormDTO
      * @return com.epmet.commons.tools.utils.Result
      * @author yinzuomei
      * @description 话题 (fact_origin_topic_main_daily	话题主表_日统计  fact_origin_topic_log_daily 话题明细_日统计)
@@ -73,8 +88,8 @@ public class FactOriginController {
     }
 
     @PostMapping("project")
-    public Result projectData(@RequestBody ProjectFormDTO formDTO) {
+    public Result projectData(@RequestBody ExtractFormDTO extractFormDTO) {
-        projectExtractService.saveOriginProjectDaily(formDTO.getCustomerId(), formDTO.getDate());
+        projectExtractService.saveOriginProjectDaily(extractFormDTO);
         return new Result();
     }
 }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/FactOriginExtractService.java

@@ -0,0 +1,18 @@
+package com.epmet.service.evaluationindex.extract;
+
+import com.epmet.dto.extract.form.ExtractFormDTO;
+
+/**
+ * @author zhaoqifeng
+ * @dscription
+ * @date 2020/9/15 14:00
+ */
+public interface FactOriginExtractService {
+
+    /**
+     * desc:抽取所有业务数据到统计库
+     *
+     * @param extractFormDTO
+     */
+    void extractAll(ExtractFormDTO extractFormDTO);
+}

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/ProjectExtractService.java

@@ -1,6 +1,6 @@
 package com.epmet.service.evaluationindex.extract;
 
-import java.util.Date;
+import com.epmet.dto.extract.form.ExtractFormDTO;
 
 /**
  * @author zhaoqifeng
@@ -12,11 +12,11 @@ public interface ProjectExtractService {
 
     /**
      * 项目主表、明细日统计
+     *
+     * @param extractFormDTO
+     * @return
      * @author zhaoqifeng
      * @date 2020/9/15 14:38
-	 * @param customerId
-	 * @param date
-	 * @return
      */
-	void saveOriginProjectDaily(String customerId, Date date);
+    void saveOriginProjectDaily(ExtractFormDTO extractFormDTO);
 }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/impl/FactOriginExtractServiceImpl.java

@@ -0,0 +1,100 @@
+package com.epmet.service.evaluationindex.extract.impl;
+
+import com.alibaba.fastjson.JSON;
+import com.epmet.commons.tools.constant.NumConstant;
+import com.epmet.commons.tools.utils.DateUtils;
+import com.epmet.dto.extract.form.ExtractFormDTO;
+import com.epmet.service.evaluationindex.extract.FactOriginExtractService;
+import com.epmet.service.evaluationindex.extract.FactOriginTopicMainDailyService;
+import com.epmet.service.evaluationindex.extract.IssueExtractService;
+import com.epmet.service.evaluationindex.extract.ProjectExtractService;
+import com.epmet.service.stats.DimCustomerService;
+import com.epmet.util.DimIdGenerator;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import org.springframework.util.CollectionUtils;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.*;
+
+/**
+ * desc：抽取业务数据 到 统计库  汇聚类
+ */
+@Slf4j
+@Service
+public class FactOriginExtractServiceImpl implements FactOriginExtractService {
+    ThreadFactory namedThreadFactory = new ThreadFactoryBuilder()
+            .setNameFormat("factOriginExtract-pool-%d").build();
+    ExecutorService threadPool = new ThreadPoolExecutor(4, 8,
+            10L, TimeUnit.MINUTES,
+            new LinkedBlockingQueue<>(100), namedThreadFactory, new ThreadPoolExecutor.AbortPolicy());
+
+    @Autowired
+    private IssueExtractService issueExtractService;
+    @Autowired
+    private FactOriginTopicMainDailyService factOriginTopicMainDailyService;
+    @Autowired
+    private ProjectExtractService projectExtractService;
+    @Autowired
+    private DimCustomerService dimCustomerService;
+
+
+    @Override
+    public void extractAll(ExtractFormDTO extractFormDTO) {
+        String dateId = extractFormDTO.getDateId();
+        String customerId = extractFormDTO.getCustomerId();
+        if (StringUtils.isBlank(dateId)) {
+            dateId = DimIdGenerator.getDateDimId(DateUtils.addDateDays(new Date(), -1));
+        }
+        List<String> customerIds = new ArrayList<>();
+        if (StringUtils.isNotBlank(customerId)) {
+            customerIds.add(customerId);
+        }
+        int pageNo = NumConstant.ONE;
+        int pageSize = NumConstant.ONE_HUNDRED;
+        customerIds = dimCustomerService.selectCustomerIdPage(pageNo, pageSize);
+        if (CollectionUtils.isEmpty(customerIds)) {
+            log.error("extractAll 获取客户Id为空");
+            return;
+        }
+
+        String finalDateId = dateId;
+        customerIds.forEach(cId -> {
+            ExtractFormDTO param = new ExtractFormDTO();
+            param.setCustomerId(cId);
+            param.setDateId(finalDateId);
+            submitJob(param);
+        });
+
+    }
+
+    private void submitJob(ExtractFormDTO param) {
+        threadPool.submit(() -> {
+            try {
+                factOriginTopicMainDailyService.topicCleaning(param);
+            } catch (Exception e) {
+                log.error("抽取【话题数据】发生异常,参数:" + JSON.toJSONString(param), e);
+            }
+        });
+        threadPool.submit(() -> {
+            try {
+                issueExtractService.issueExtractMain(param);
+                issueExtractService.issueExtractLog(param);
+            } catch (Exception e) {
+                log.error("抽取【议题数据】发生异常,参数:" + JSON.toJSONString(param), e);
+            }
+        });
+        threadPool.submit(() -> {
+            try {
+                projectExtractService.saveOriginProjectDaily(param);
+            } catch (Exception e) {
+                log.error("抽取【项目数据】发生异常,参数:" + JSON.toJSONString(param), e);
+            }
+        });
+    }
+}

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/impl/IssueExtractServiceImpl.java

@@ -25,7 +25,6 @@ import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 import org.springframework.util.CollectionUtils;
 
-import javax.validation.constraints.NotBlank;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -66,7 +65,8 @@ public class IssueExtractServiceImpl implements IssueExtractService {
         List<IssueInfoResultDTO> listResult = issueService.selectIssueInfo(customerId, dateId);
         List<IssueMainDailyFormDTO> result = new ArrayList<>();
         if (CollectionUtils.isEmpty(listResult)){
-            throw new RenException(ExtractConstant.ISSUE_INFO);
+            log.warn("issueExtractMain selectIssueInfo return empty,customerId:{},dateId:{}", customerId, dateId);
+            return true;
         }
         listResult.forEach(issue -> {
             IssueMainDailyFormDTO issueMainDailyFormDTO = ConvertUtils.sourceToTarget(issue, IssueMainDailyFormDTO.class);

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/impl/ProjectExtractServiceImpl.java

@@ -5,6 +5,7 @@ import com.epmet.commons.tools.utils.DateUtils;
 import com.epmet.commons.tools.utils.Result;
 import com.epmet.dto.ProjectDTO;
 import com.epmet.dto.extract.FactOriginProjectMainDailyDTO;
+import com.epmet.dto.extract.form.ExtractFormDTO;
 import com.epmet.dto.form.WorkDayFormDTO;
 import com.epmet.dto.issue.IssueDTO;
 import com.epmet.dto.party.PartyMemberDTO;
@@ -61,11 +62,10 @@ public class ProjectExtractServiceImpl implements ProjectExtractService {
 
 
 	@Override
-	public void saveOriginProjectDaily(String customerId, Date date) {
+	public void saveOriginProjectDaily(ExtractFormDTO extractFormDTO) {
-		String dateString = null;
+		String dateString = extractFormDTO.getDateId();
-		if (null != date) {
+		String customerId = extractFormDTO.getCustomerId();
-			dateString = DateUtils.format(date);
+
-		}
 		//获取已关闭项目列表
 		List<ProjectProcessEntity> closedList = projectProcessService.getClosedProjectList(customerId, dateString);
 		List<FactOriginProjectMainDailyDTO> pendingList = factOriginProjectMainDailyService.getPendingList(customerId);