统一抽取

epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/dto/extract/form/ExtractFormDTO.java

@@ -15,11 +15,21 @@ public class ExtractFormDTO implements Serializable {
 
     private static final long serialVersionUID = -6180252151765854242L;
 
-    public interface ExtractForm extends CustomerClientShowGroup{}
+    public interface ExtractForm extends CustomerClientShowGroup {
+    }
 
     @NotBlank(message = "客户ID不能为空", groups = ExtractForm.class)
     private String customerId;
 
     @NotBlank(message = "dateId不能为空", groups = ExtractForm.class)
     private String dateId;
+
+    /**
+     * 开始时间
+     */
+    private String startDate;
+    /**
+     * 结束时间
+     */
+    private String endDate;
 }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/FactOriginController.java

@@ -1,5 +1,6 @@
 package com.epmet.controller;
 
+import com.epmet.commons.tools.utils.DateUtils;
 import com.epmet.commons.tools.utils.Result;
 import com.epmet.commons.tools.validator.ValidatorUtils;
 import com.epmet.dto.extract.form.ExtractFormDTO;
@@ -7,6 +8,7 @@ import com.epmet.service.evaluationindex.extract.FactOriginExtractService;
 import com.epmet.service.evaluationindex.extract.FactOriginTopicMainDailyService;
 import com.epmet.service.evaluationindex.extract.IssueExtractService;
 import com.epmet.service.evaluationindex.extract.ProjectExtractService;
+import com.epmet.util.DimIdGenerator;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.PostMapping;
@@ -14,6 +16,8 @@ import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
 
+import java.util.Date;
+
 
 /**
  * 原始数据清洗
@@ -41,7 +45,18 @@ public class FactOriginController {
      */
     @PostMapping("extractall")
     public Result extractAll(@RequestBody ExtractFormDTO extractFormDTO) {
+        if (StringUtils.isNotBlank(extractFormDTO.getStartDate()) && StringUtils.isNotBlank(extractFormDTO.getEndDate())) {
+            Date startDate = DateUtils.stringToDate(extractFormDTO.getStartDate(), "yyyyMMdd");
+            Date endDate = DateUtils.stringToDate(extractFormDTO.getEndDate(), "yyyyMMdd");
+            do {
+                String dateDimId = DimIdGenerator.getDateDimId(startDate);
+                extractFormDTO.setDateId(dateDimId);
+                factOriginExtractService.extractAll(extractFormDTO);
+                startDate = DateUtils.addDateDays(DateUtils.stringToDate(extractFormDTO.getStartDate(), "yyyyMMdd"), 1);
+            } while (endDate.compareTo(startDate) >= 0);
+        } else {
             factOriginExtractService.extractAll(extractFormDTO);
+        }
         return new Result();
     }
 

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/impl/FactOriginExtractServiceImpl.java

@@ -32,7 +32,7 @@ public class FactOriginExtractServiceImpl implements FactOriginExtractService {
             .setNameFormat("factOriginExtract-pool-%d").build();
     ExecutorService threadPool = new ThreadPoolExecutor(4, 8,
             10L, TimeUnit.MINUTES,
-            new LinkedBlockingQueue<>(100), namedThreadFactory, new ThreadPoolExecutor.AbortPolicy());
+            new LinkedBlockingQueue<>(500), namedThreadFactory, new ThreadPoolExecutor.CallerRunsPolicy());
 
     @Autowired
     private IssueExtractService issueExtractService;

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/impl/IssueExtractServiceImpl.java

@@ -161,7 +161,8 @@ public class IssueExtractServiceImpl implements IssueExtractService {
         // 1. 查询议题process
         List<IssueProcessInfoResultDTO> listResult = issueService.selectIssueProcessInfo(customerId, dateId);
         if (CollectionUtils.isEmpty(listResult)){
-            throw new RenException(ExtractConstant.ISSUE_PROCESS);
+            log.warn("issueExtractMain issueExtractLog return empty,customerId:{},dateId:{}", customerId, dateId);
+            return true;
         }
         listResult.forEach(issueProcess -> {
             IssueLogDailyFormDTO issueLogDailyFormDTO = ConvertUtils.sourceToTarget(issueProcess, IssueLogDailyFormDTO.class);

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/impl/ProjectExtractServiceImpl.java

@@ -65,7 +65,6 @@ public class ProjectExtractServiceImpl implements ProjectExtractService {
 	public void saveOriginProjectDaily(ExtractFormDTO extractFormDTO) {
 		String dateString = extractFormDTO.getDateId();
 		String customerId = extractFormDTO.getCustomerId();
-
 		//获取已关闭项目列表
 		List<ProjectProcessEntity> closedList = projectProcessService.getClosedProjectList(customerId, dateString);
 		List<FactOriginProjectMainDailyDTO> pendingList = factOriginProjectMainDailyService.getPendingList(customerId);

epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/project/ProjectDao.xml

@@ -93,7 +93,7 @@
           AND ORIGIN = 'issue'
           AND CUSTOMER_ID = #{customerId}
           <if test="date != null and date.trim() != ''">
-            AND DATE_FORMAT(CREATED_TIME, '%Y-%m-%d') = #{date}
+            AND DATE_FORMAT(CREATED_TIME, '%Y%m%d') = #{date}
           </if>
     </select>
 

epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/project/ProjectProcessDao.xml

@@ -30,7 +30,7 @@
         INNER JOIN project p ON pp.PROJECT_ID = p.ID
         WHERE p.CUSTOMER_ID = #{customerId}
         <if test="date != null and date.trim() != ''">
-            AND DATE_FORMAT(pp.CREATED_TIME, '%Y-%m-%d') = #{date}
+            AND DATE_FORMAT(pp.CREATED_TIME, '%Y%m%d') = #{date}
         </if>
     </select>
     <select id="selectProcessStaffByCustomer" resultType="com.epmet.dto.project.ProcessInfoDTO">
@@ -47,7 +47,7 @@
         INNER JOIN project p ON pp.PROJECT_ID = p.ID
         WHERE p.CUSTOMER_ID = #{customerId}
         <if test="date != null and date.trim() != ''">
-            AND DATE_FORMAT(pp.CREATED_TIME, '%Y-%m-%d') = #{date}
+            AND DATE_FORMAT(pp.CREATED_TIME, '%Y%m%d') = #{date}
         </if>
     </select>
     <select id="selectSatisfactionRecordByCustomer" resultType="com.epmet.dto.project.ProcessInfoDTO">
@@ -61,7 +61,7 @@
         INNER JOIN project p ON psd.PROJECT_ID = p.ID
         WHERE p.CUSTOMER_ID = #{customerId}
         <if test="date != null and date.trim() != ''">
-            AND DATE_FORMAT(psd.CREATED_TIME, '%Y-%m-%d') = #{date}
+            AND DATE_FORMAT(psd.CREATED_TIME, '%Y%m%d') = #{date}
         </if>
     </select>
     <select id="selectClosedProjectList" resultType="com.epmet.entity.project.ProjectProcessEntity">
@@ -73,7 +73,7 @@
         WHERE p.CUSTOMER_ID = #{customerId}
           AND pp.OPERATION = 'close'
         <if test="date != null and date.trim() != ''">
-            AND DATE_FORMAT(pp.CREATED_TIME, '%Y-%m-%d') = #{date}
+            AND DATE_FORMAT(pp.CREATED_TIME, '%Y%m%d') = #{date}
         </if>
     </select>