From a95534223541af4216fef556b05f0213faa43744 Mon Sep 17 00:00:00 2001 From: jianjun Date: Fri, 18 Sep 2020 15:55:33 +0800 Subject: [PATCH] =?UTF-8?q?=E4=B8=9A=E5=8A=A1=E6=95=B0=E6=8D=AE=E7=BB=9F?= =?UTF-8?q?=E8=AE=A1=E6=8A=BD=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/epmet/controller/DemoController.java | 13 ++- .../controller/FactOriginController.java | 27 +++-- .../extract/FactOriginExtractService.java | 18 ++++ .../extract/ProjectExtractService.java | 20 ++-- .../impl/FactOriginExtractServiceImpl.java | 100 ++++++++++++++++++ .../extract/impl/IssueExtractServiceImpl.java | 4 +- .../impl/ProjectExtractServiceImpl.java | 10 +- 7 files changed, 166 insertions(+), 26 deletions(-) create mode 100644 epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/FactOriginExtractService.java create mode 100644 epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/impl/FactOriginExtractServiceImpl.java diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/DemoController.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/DemoController.java index 6db7fd4245..47122de6ad 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/DemoController.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/DemoController.java @@ -15,6 +15,7 @@ import com.epmet.dao.evaluationindex.screen.ScreenCustomerGridDao; import com.epmet.dao.stats.DimCustomerDao; import com.epmet.dao.stats.DimDateDao; import com.epmet.dto.AgencySubTreeDto; +import com.epmet.dto.extract.form.ExtractFormDTO; import com.epmet.dto.indexcal.CalculateCommonFormDTO; import com.epmet.dto.stats.form.CustomerIdAndDateIdFormDTO; import com.epmet.entity.evaluationindex.indexcoll.FactIndexGovrnAblityGridMonthlyEntity; @@ -503,15 +504,21 @@ public class DemoController { @PostMapping("inserttopicorigin") public Result topicDataCleaning(@RequestParam("customerId") String customerId, @RequestParam("dateId")String dateId) { if (StringUtils.isNotBlank(customerId) && StringUtils.isNotBlank(dateId)) { - factOriginTopicMainDailyService.topicCleaning(customerId,dateId); + ExtractFormDTO param = new ExtractFormDTO(); + param.setDateId(dateId); + param.setCustomerId(customerId); + Boolean aBoolean = factOriginTopicMainDailyService.topicCleaning(param); }else{ QueryWrapper customerEntityQueryWrapper = new QueryWrapper<>(); List customerEntityList=dimCustomerDao.selectList(customerEntityQueryWrapper); QueryWrapper wrapper = new QueryWrapper<>(); List dimDateEntityList= dimDateDao.selectList(wrapper); for(DimCustomerEntity customerEntity:customerEntityList){ - for(DimDateEntity dateEntity:dimDateEntityList){ - factOriginTopicMainDailyService.topicCleaning(customerEntity.getId(),dateEntity.getId()); + for(DimDateEntity dateEntity:dimDateEntityList) { + ExtractFormDTO param = new ExtractFormDTO(); + param.setDateId(dateEntity.getId()); + param.setCustomerId(customerEntity.getId()); + factOriginTopicMainDailyService.topicCleaning(param); } } } diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/FactOriginController.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/FactOriginController.java index 8ddfc6a872..eaabd2fa48 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/FactOriginController.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/FactOriginController.java @@ -3,7 +3,7 @@ package com.epmet.controller; import com.epmet.commons.tools.utils.Result; import com.epmet.commons.tools.validator.ValidatorUtils; import com.epmet.dto.extract.form.ExtractFormDTO; -import com.epmet.dto.extract.form.ProjectFormDTO; +import com.epmet.service.evaluationindex.extract.FactOriginExtractService; import com.epmet.service.evaluationindex.extract.FactOriginTopicMainDailyService; import com.epmet.service.evaluationindex.extract.IssueExtractService; import com.epmet.service.evaluationindex.extract.ProjectExtractService; @@ -24,7 +24,8 @@ import org.springframework.web.bind.annotation.RestController; @RestController @RequestMapping("factorigin") public class FactOriginController { - + @Autowired + private FactOriginExtractService factOriginExtractService; @Autowired private IssueExtractService issueExtractService; @Autowired @@ -33,13 +34,26 @@ public class FactOriginController { private ProjectExtractService projectExtractService; /** - * @Description 议题抽取(main) + * desc: + * + * @param extractFormDTO + * @return + */ + @PostMapping("extractall") + public Result extractAll(@RequestBody ExtractFormDTO extractFormDTO) { + factOriginExtractService.extractAll(extractFormDTO); + return new Result(); + } + + + /** * @param extractFormDTO + * @Description 议题抽取(main) * @author zxc * @date 2020/9/15 2:02 下午 */ @PostMapping("issueextractmain") - public Result issueExtractMain(@RequestBody ExtractFormDTO extractFormDTO){ + public Result issueExtractMain(@RequestBody ExtractFormDTO extractFormDTO) { ValidatorUtils.validateEntity(extractFormDTO, ExtractFormDTO.ExtractForm.class); issueExtractService.issueExtractMain(extractFormDTO); return new Result(); @@ -59,6 +73,7 @@ public class FactOriginController { } /** + * @param extractFormDTO * @return com.epmet.commons.tools.utils.Result * @author yinzuomei * @description 话题 (fact_origin_topic_main_daily 话题主表_日统计 fact_origin_topic_log_daily 话题明细_日统计) @@ -73,8 +88,8 @@ public class FactOriginController { } @PostMapping("project") - public Result projectData(@RequestBody ProjectFormDTO formDTO) { - projectExtractService.saveOriginProjectDaily(formDTO.getCustomerId(), formDTO.getDate()); + public Result projectData(@RequestBody ExtractFormDTO extractFormDTO) { + projectExtractService.saveOriginProjectDaily(extractFormDTO); return new Result(); } } diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/FactOriginExtractService.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/FactOriginExtractService.java new file mode 100644 index 0000000000..886a7daf3c --- /dev/null +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/FactOriginExtractService.java @@ -0,0 +1,18 @@ +package com.epmet.service.evaluationindex.extract; + +import com.epmet.dto.extract.form.ExtractFormDTO; + +/** + * @author zhaoqifeng + * @dscription + * @date 2020/9/15 14:00 + */ +public interface FactOriginExtractService { + + /** + * desc:抽取所有业务数据到统计库 + * + * @param extractFormDTO + */ + void extractAll(ExtractFormDTO extractFormDTO); +} diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/ProjectExtractService.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/ProjectExtractService.java index 88efc32884..d6103f71a5 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/ProjectExtractService.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/ProjectExtractService.java @@ -1,6 +1,6 @@ package com.epmet.service.evaluationindex.extract; -import java.util.Date; +import com.epmet.dto.extract.form.ExtractFormDTO; /** * @author zhaoqifeng @@ -10,13 +10,13 @@ import java.util.Date; public interface ProjectExtractService { - /** - * 项目主表、明细日统计 - * @author zhaoqifeng - * @date 2020/9/15 14:38 - * @param customerId - * @param date - * @return - */ - void saveOriginProjectDaily(String customerId, Date date); + /** + * 项目主表、明细日统计 + * + * @param extractFormDTO + * @return + * @author zhaoqifeng + * @date 2020/9/15 14:38 + */ + void saveOriginProjectDaily(ExtractFormDTO extractFormDTO); } diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/impl/FactOriginExtractServiceImpl.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/impl/FactOriginExtractServiceImpl.java new file mode 100644 index 0000000000..7c6275929d --- /dev/null +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/impl/FactOriginExtractServiceImpl.java @@ -0,0 +1,100 @@ +package com.epmet.service.evaluationindex.extract.impl; + +import com.alibaba.fastjson.JSON; +import com.epmet.commons.tools.constant.NumConstant; +import com.epmet.commons.tools.utils.DateUtils; +import com.epmet.dto.extract.form.ExtractFormDTO; +import com.epmet.service.evaluationindex.extract.FactOriginExtractService; +import com.epmet.service.evaluationindex.extract.FactOriginTopicMainDailyService; +import com.epmet.service.evaluationindex.extract.IssueExtractService; +import com.epmet.service.evaluationindex.extract.ProjectExtractService; +import com.epmet.service.stats.DimCustomerService; +import com.epmet.util.DimIdGenerator; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.concurrent.*; + +/** + * desc：抽取业务数据 到 统计库 汇聚类 + */ +@Slf4j +@Service +public class FactOriginExtractServiceImpl implements FactOriginExtractService { + ThreadFactory namedThreadFactory = new ThreadFactoryBuilder() + .setNameFormat("factOriginExtract-pool-%d").build(); + ExecutorService threadPool = new ThreadPoolExecutor(4, 8, + 10L, TimeUnit.MINUTES, + new LinkedBlockingQueue<>(100), namedThreadFactory, new ThreadPoolExecutor.AbortPolicy()); + + @Autowired + private IssueExtractService issueExtractService; + @Autowired + private FactOriginTopicMainDailyService factOriginTopicMainDailyService; + @Autowired + private ProjectExtractService projectExtractService; + @Autowired + private DimCustomerService dimCustomerService; + + + @Override + public void extractAll(ExtractFormDTO extractFormDTO) { + String dateId = extractFormDTO.getDateId(); + String customerId = extractFormDTO.getCustomerId(); + if (StringUtils.isBlank(dateId)) { + dateId = DimIdGenerator.getDateDimId(DateUtils.addDateDays(new Date(), -1)); + } + List customerIds = new ArrayList<>(); + if (StringUtils.isNotBlank(customerId)) { + customerIds.add(customerId); + } + int pageNo = NumConstant.ONE; + int pageSize = NumConstant.ONE_HUNDRED; + customerIds = dimCustomerService.selectCustomerIdPage(pageNo, pageSize); + if (CollectionUtils.isEmpty(customerIds)) { + log.error("extractAll 获取客户Id为空"); + return; + } + + String finalDateId = dateId; + customerIds.forEach(cId -> { + ExtractFormDTO param = new ExtractFormDTO(); + param.setCustomerId(cId); + param.setDateId(finalDateId); + submitJob(param); + }); + + } + + private void submitJob(ExtractFormDTO param) { + threadPool.submit(() -> { + try { + factOriginTopicMainDailyService.topicCleaning(param); + } catch (Exception e) { + log.error("抽取【话题数据】发生异常,参数:" + JSON.toJSONString(param), e); + } + }); + threadPool.submit(() -> { + try { + issueExtractService.issueExtractMain(param); + issueExtractService.issueExtractLog(param); + } catch (Exception e) { + log.error("抽取【议题数据】发生异常,参数:" + JSON.toJSONString(param), e); + } + }); + threadPool.submit(() -> { + try { + projectExtractService.saveOriginProjectDaily(param); + } catch (Exception e) { + log.error("抽取【项目数据】发生异常,参数:" + JSON.toJSONString(param), e); + } + }); + } +} diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/impl/IssueExtractServiceImpl.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/impl/IssueExtractServiceImpl.java index 702487dad9..cc57e3d000 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/impl/IssueExtractServiceImpl.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/impl/IssueExtractServiceImpl.java @@ -25,7 +25,6 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.CollectionUtils; -import javax.validation.constraints.NotBlank; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -66,7 +65,8 @@ public class IssueExtractServiceImpl implements IssueExtractService { List listResult = issueService.selectIssueInfo(customerId, dateId); List result = new ArrayList<>(); if (CollectionUtils.isEmpty(listResult)){ - throw new RenException(ExtractConstant.ISSUE_INFO); + log.warn("issueExtractMain selectIssueInfo return empty,customerId:{},dateId:{}", customerId, dateId); + return true; } listResult.forEach(issue -> { IssueMainDailyFormDTO issueMainDailyFormDTO = ConvertUtils.sourceToTarget(issue, IssueMainDailyFormDTO.class); diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/impl/ProjectExtractServiceImpl.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/impl/ProjectExtractServiceImpl.java index ca63c8c9aa..cd8802aa8b 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/impl/ProjectExtractServiceImpl.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/impl/ProjectExtractServiceImpl.java @@ -5,6 +5,7 @@ import com.epmet.commons.tools.utils.DateUtils; import com.epmet.commons.tools.utils.Result; import com.epmet.dto.ProjectDTO; import com.epmet.dto.extract.FactOriginProjectMainDailyDTO; +import com.epmet.dto.extract.form.ExtractFormDTO; import com.epmet.dto.form.WorkDayFormDTO; import com.epmet.dto.issue.IssueDTO; import com.epmet.dto.party.PartyMemberDTO; @@ -61,11 +62,10 @@ public class ProjectExtractServiceImpl implements ProjectExtractService { @Override - public void saveOriginProjectDaily(String customerId, Date date) { - String dateString = null; - if (null != date) { - dateString = DateUtils.format(date); - } + public void saveOriginProjectDaily(ExtractFormDTO extractFormDTO) { + String dateString = extractFormDTO.getDateId(); + String customerId = extractFormDTO.getCustomerId(); + //获取已关闭项目列表 List closedList = projectProcessService.getClosedProjectList(customerId, dateString); List pendingList = factOriginProjectMainDailyService.getPendingList(customerId);