From 80a6ca48511c1b63997efa578d89fc897a682749 Mon Sep 17 00:00:00 2001 From: jianjun Date: Mon, 5 Jul 2021 10:24:21 +0800 Subject: [PATCH] =?UTF-8?q?=E7=BD=91=E6=A0=BC=E5=91=98=E5=B7=A1=E6=9F=A5?= =?UTF-8?q?=E7=BB=9F=E8=AE=A1=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1=E7=BB=9F?= =?UTF-8?q?=E4=B8=80=E8=B5=B7=E6=9D=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dto/extract/form/BizDataFormDTO.java | 41 +++++++++++ .../feign/DataStatisticalOpenFeignClient.java | 11 ++- ...ataStatisticalOpenFeignClientFallBack.java | 9 +-- .../controller/BizDataStatsController.java | 11 +++ .../extract/biz/BizDataStatsService.java | 8 +++ .../biz/impl/BizDataStatsServiceImpl.java | 69 +++++++++++++++++++ .../epmet/service/BizDataStatsService.java | 4 +- .../service/impl/BizDataStatsServiceImpl.java | 6 +- .../com/epmet/task/StaffPatrolStatsTask.java | 6 +- 9 files changed, 144 insertions(+), 21 deletions(-) create mode 100644 epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/dto/extract/form/BizDataFormDTO.java diff --git a/epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/dto/extract/form/BizDataFormDTO.java b/epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/dto/extract/form/BizDataFormDTO.java new file mode 100644 index 0000000000..23f164afe3 --- /dev/null +++ b/epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/dto/extract/form/BizDataFormDTO.java @@ -0,0 +1,41 @@ +package com.epmet.dto.extract.form; + +import com.epmet.commons.tools.validator.group.CustomerClientShowGroup; +import lombok.Data; + +import javax.validation.constraints.NotBlank; +import java.io.Serializable; + +/** + * desc: 从业务数据抽取到统计库 dto + * + * @Author zxc + * @DateTime 2020/9/16 6:01 下午 + */ +@Data +public class BizDataFormDTO implements Serializable { + + + private static final long serialVersionUID = 2860395703825549825L; + + public interface ExtractForm extends CustomerClientShowGroup { + } + + @NotBlank(message = "客户ID不能为空", groups = ExtractForm.class) + private String customerId; + + @NotBlank(message = "dateId不能为空", groups = ExtractForm.class) + private String dateId; + + private String userId; + private String gridId; + + /** + * 开始时间 + */ + private String startDate; + /** + * 结束时间 + */ + private String endDate; +} diff --git a/epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/feign/DataStatisticalOpenFeignClient.java b/epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/feign/DataStatisticalOpenFeignClient.java index fdf97a670f..ee5daf2a47 100644 --- a/epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/feign/DataStatisticalOpenFeignClient.java +++ b/epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/feign/DataStatisticalOpenFeignClient.java @@ -3,10 +3,7 @@ package com.epmet.feign; import com.epmet.commons.tools.constant.ServiceConstant; import com.epmet.commons.tools.utils.Result; import com.epmet.dto.StatsFormDTO; -import com.epmet.dto.extract.form.ExtractIndexFormDTO; -import com.epmet.dto.extract.form.ExtractOriginFormDTO; -import com.epmet.dto.extract.form.ExtractScreenFormDTO; -import com.epmet.dto.extract.form.StaffPatrolStatsFormDTO; +import com.epmet.dto.extract.form.*; import com.epmet.dto.group.form.GroupStatsFormDTO; import com.epmet.dto.screen.form.InitCustomerIndexForm; import com.epmet.dto.stats.form.CustomerIdAndDateIdFormDTO; @@ -258,13 +255,13 @@ public interface DataStatisticalOpenFeignClient { Result gridVanguardStats(@RequestBody StatsFormDTO formDTO); /** - * desc: 网格员巡查统计初始化及校正 + * desc: 业务库按天统计 统一入库 * * @param formDTO * @return com.epmet.commons.tools.utils.Result * @author LiuJanJun * @date 2021/7/2 3:04 下午 */ - @PostMapping("/data/stats/bizData/stats/patrol") - Result staffPatrolStats(@RequestBody StaffPatrolStatsFormDTO formDTO); + @PostMapping("/data/stats/bizData/stats/daily") + Result exeStatsDaily(@RequestBody BizDataFormDTO formDTO); } diff --git a/epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/feign/impl/DataStatisticalOpenFeignClientFallBack.java b/epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/feign/impl/DataStatisticalOpenFeignClientFallBack.java index ecd3c7faf2..4846a3e557 100644 --- a/epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/feign/impl/DataStatisticalOpenFeignClientFallBack.java +++ b/epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/feign/impl/DataStatisticalOpenFeignClientFallBack.java @@ -4,10 +4,7 @@ import com.epmet.commons.tools.constant.ServiceConstant; import com.epmet.commons.tools.utils.ModuleUtils; import com.epmet.commons.tools.utils.Result; import com.epmet.dto.StatsFormDTO; -import com.epmet.dto.extract.form.ExtractIndexFormDTO; -import com.epmet.dto.extract.form.ExtractOriginFormDTO; -import com.epmet.dto.extract.form.ExtractScreenFormDTO; -import com.epmet.dto.extract.form.StaffPatrolStatsFormDTO; +import com.epmet.dto.extract.form.*; import com.epmet.dto.group.form.GroupStatsFormDTO; import com.epmet.dto.screen.form.InitCustomerIndexForm; import com.epmet.dto.stats.form.CustomerIdAndDateIdFormDTO; @@ -259,7 +256,7 @@ public class DataStatisticalOpenFeignClientFallBack implements DataStatisticalOp } @Override - public Result staffPatrolStats(StaffPatrolStatsFormDTO formDTO) { - return ModuleUtils.feignConError(ServiceConstant.DATA_STATISTICAL_SERVER, "staffPatrolStats", formDTO); + public Result exeStatsDaily(BizDataFormDTO formDTO) { + return ModuleUtils.feignConError(ServiceConstant.DATA_STATISTICAL_SERVER, "exeStatsDaily", formDTO); } } diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/BizDataStatsController.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/BizDataStatsController.java index 5910f13386..7fe1add999 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/BizDataStatsController.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/BizDataStatsController.java @@ -1,6 +1,7 @@ package com.epmet.controller; import com.epmet.commons.tools.utils.Result; +import com.epmet.dto.extract.form.BizDataFormDTO; import com.epmet.dto.extract.form.StaffPatrolStatsFormDTO; import com.epmet.service.evaluationindex.extract.biz.BizDataStatsService; import org.springframework.beans.factory.annotation.Autowired; @@ -22,6 +23,16 @@ public class BizDataStatsController { private BizDataStatsService bizDataStatsService; /** + * @return com.epmet.commons.tools.utils.Result + * @param formDTO + * @description 业务库统计数据 统一入库 + * @Date 2021/3/26 13:27 + **/ + @RequestMapping("daily") + public Result execute(@RequestBody BizDataFormDTO formDTO) { + bizDataStatsService.exeDailyAll(formDTO); + return new Result(); + } /** * @return com.epmet.commons.tools.utils.Result * @param formDTO * @description 工作端数据一期，用户分析：参与用户、注册用户分析 diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/biz/BizDataStatsService.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/biz/BizDataStatsService.java index 8bff5118b7..062bd2c49c 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/biz/BizDataStatsService.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/biz/BizDataStatsService.java @@ -17,6 +17,7 @@ package com.epmet.service.evaluationindex.extract.biz; +import com.epmet.dto.extract.form.BizDataFormDTO; import com.epmet.dto.extract.form.StaffPatrolStatsFormDTO; /** @@ -27,6 +28,13 @@ import com.epmet.dto.extract.form.StaffPatrolStatsFormDTO; */ public interface BizDataStatsService { + /** + * desc:抽取所有业务数据到统计库 + * + * @param dataFormDTO + */ + void exeDailyAll(BizDataFormDTO dataFormDTO); + void executeStaffPatrolStats(StaffPatrolStatsFormDTO formDTO); } diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/biz/impl/BizDataStatsServiceImpl.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/biz/impl/BizDataStatsServiceImpl.java index 2db1363f5e..01cb232071 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/biz/impl/BizDataStatsServiceImpl.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/biz/impl/BizDataStatsServiceImpl.java @@ -6,6 +6,7 @@ import com.epmet.commons.tools.utils.ConvertUtils; import com.epmet.commons.tools.utils.DateUtils; import com.epmet.constant.ProjectConstant; import com.epmet.dto.ProjectDTO; +import com.epmet.dto.extract.form.BizDataFormDTO; import com.epmet.dto.extract.form.StaffPatrolStatsFormDTO; import com.epmet.dto.user.result.CustomerGridStaffDTO; import com.epmet.dto.user.result.CustomerStaffDTO; @@ -14,11 +15,14 @@ import com.epmet.dto.user.result.StatsStaffPatrolRecordDailyDTO; import com.epmet.service.evaluationindex.extract.biz.BizDataStatsService; import com.epmet.service.org.CustomerStaffService; import com.epmet.service.project.ProjectService; +import com.epmet.service.stats.DimCustomerService; import com.epmet.service.user.StatsStaffPatrolService; import com.epmet.service.user.UserService; import com.epmet.util.DimIdGenerator; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.springframework.beans.factory.annotation.Autowired; @@ -28,6 +32,7 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Map; +import java.util.concurrent.*; import java.util.stream.Collectors; /** @@ -40,6 +45,11 @@ import java.util.stream.Collectors; @Slf4j @Service public class BizDataStatsServiceImpl implements BizDataStatsService { + ThreadFactory namedThreadFactory = new ThreadFactoryBuilder() + .setNameFormat("bizDataStats-pool-%d").build(); + ExecutorService threadPool = new ThreadPoolExecutor(3, 6, + 10L, TimeUnit.MINUTES, + new LinkedBlockingQueue<>(500), namedThreadFactory, new ThreadPoolExecutor.CallerRunsPolicy()); @Autowired private CustomerStaffService customerStaffService; @Autowired @@ -48,6 +58,36 @@ public class BizDataStatsServiceImpl implements BizDataStatsService { private ProjectService projectService; @Autowired private StatsStaffPatrolService statsStaffPatrolService; + @Autowired + private DimCustomerService dimCustomerService; + + @Override + public void exeDailyAll(BizDataFormDTO dataFormDTO) { + String customerId = dataFormDTO.getCustomerId(); + if (StringUtils.isBlank(dataFormDTO.getDateId()) && (StringUtils.isBlank(dataFormDTO.getStartDate()) && StringUtils.isBlank(dataFormDTO.getEndDate()))) { + //如果没有设置开始日期、结束日期,默认查询昨天 + dataFormDTO.setDateId(DimIdGenerator.getDateDimId(DateUtils.addDateDays(new Date(), -1))); + } + List customerIds = new ArrayList<>(); + if (StringUtils.isNotBlank(customerId)) { + //指定某个客户 + customerIds.add(customerId); + } else { + //查询全部客户 + int pageNo = NumConstant.ONE; + int pageSize = NumConstant.ONE_HUNDRED; + customerIds = dimCustomerService.selectCustomerIdPage(pageNo, pageSize); + if (org.springframework.util.CollectionUtils.isEmpty(customerIds)) { + log.error("exeDailyAll 获取客户Id为空"); + return; + } + } + customerIds.forEach(cId -> { + + log.info("exeDailyAll param:{}", JSON.toJSONString(dataFormDTO)); + submitJob(dataFormDTO); + }); + } @Override public void executeStaffPatrolStats(StaffPatrolStatsFormDTO formDTO) { @@ -216,4 +256,33 @@ public class BizDataStatsServiceImpl implements BizDataStatsService { log.debug("getAllGridMembers result:{},param:{}", JSON.toJSONString(insertList), JSON.toJSONString(formDTO)); return insertList; } + + private void submitJob(BizDataFormDTO param) { + boolean isRange = StringUtils.isBlank(param.getDateId()); + List daysBetween = null; + if (isRange) { + daysBetween = DateUtils.getDaysBetween(param.getStartDate(), param.getEndDate()); + } + List finalDaysBetween = daysBetween; + threadPool.submit(() -> { + StaffPatrolStatsFormDTO formDTO = ConvertUtils.sourceToTarget(param, StaffPatrolStatsFormDTO.class); + if (!isRange) { + try { + formDTO.setDateId(param.getDateId()); + this.executeStaffPatrolStats(formDTO); + } catch (Exception e) { + log.error("【网格员巡查数据统计】发生异常,参数:" + JSON.toJSONString(formDTO), e); + } + } else { + try { + for (String dateDimId : finalDaysBetween) { + formDTO.setDateId(dateDimId); + this.executeStaffPatrolStats(formDTO); + } + } catch (Exception e) { + log.error("【网格员巡查数据统计】发生异常,参数:" + JSON.toJSONString(param), e); + } + } + }); + } } diff --git a/epmet-module/epmet-job/epmet-job-server/src/main/java/com/epmet/service/BizDataStatsService.java b/epmet-module/epmet-job/epmet-job-server/src/main/java/com/epmet/service/BizDataStatsService.java index d610bf6790..56254e637f 100644 --- a/epmet-module/epmet-job/epmet-job-server/src/main/java/com/epmet/service/BizDataStatsService.java +++ b/epmet-module/epmet-job/epmet-job-server/src/main/java/com/epmet/service/BizDataStatsService.java @@ -1,10 +1,10 @@ package com.epmet.service; import com.epmet.commons.tools.utils.Result; -import com.epmet.dto.extract.form.StaffPatrolStatsFormDTO; +import com.epmet.dto.extract.form.BizDataFormDTO; public interface BizDataStatsService { - Result exeStaffPatrolStats(StaffPatrolStatsFormDTO formDTO); + Result exeStatsDaily(BizDataFormDTO formDTO); } diff --git a/epmet-module/epmet-job/epmet-job-server/src/main/java/com/epmet/service/impl/BizDataStatsServiceImpl.java b/epmet-module/epmet-job/epmet-job-server/src/main/java/com/epmet/service/impl/BizDataStatsServiceImpl.java index 0e7a7b53e9..5616de5bbe 100644 --- a/epmet-module/epmet-job/epmet-job-server/src/main/java/com/epmet/service/impl/BizDataStatsServiceImpl.java +++ b/epmet-module/epmet-job/epmet-job-server/src/main/java/com/epmet/service/impl/BizDataStatsServiceImpl.java @@ -1,7 +1,7 @@ package com.epmet.service.impl; import com.epmet.commons.tools.utils.Result; -import com.epmet.dto.extract.form.StaffPatrolStatsFormDTO; +import com.epmet.dto.extract.form.BizDataFormDTO; import com.epmet.feign.DataStatisticalOpenFeignClient; import com.epmet.service.BizDataStatsService; import org.springframework.beans.factory.annotation.Autowired; @@ -19,7 +19,7 @@ public class BizDataStatsServiceImpl implements BizDataStatsService { @Autowired private DataStatisticalOpenFeignClient feignClient; @Override - public Result exeStaffPatrolStats(StaffPatrolStatsFormDTO formDTO) { - return feignClient.staffPatrolStats(formDTO); + public Result exeStatsDaily(BizDataFormDTO formDTO) { + return feignClient.exeStatsDaily(formDTO); } } diff --git a/epmet-module/epmet-job/epmet-job-server/src/main/java/com/epmet/task/StaffPatrolStatsTask.java b/epmet-module/epmet-job/epmet-job-server/src/main/java/com/epmet/task/StaffPatrolStatsTask.java index d07f233108..affb4a2e9d 100644 --- a/epmet-module/epmet-job/epmet-job-server/src/main/java/com/epmet/task/StaffPatrolStatsTask.java +++ b/epmet-module/epmet-job/epmet-job-server/src/main/java/com/epmet/task/StaffPatrolStatsTask.java @@ -2,7 +2,7 @@ package com.epmet.task; import com.alibaba.fastjson.JSON; import com.epmet.commons.tools.utils.Result; -import com.epmet.dto.extract.form.StaffPatrolStatsFormDTO; +import com.epmet.dto.extract.form.BizDataFormDTO; import com.epmet.service.BizDataStatsService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -23,8 +23,8 @@ public class StaffPatrolStatsTask implements ITask { @Override public void run(String params) { log.debug("staffPatrolStatsTask定时任务正在执行，参数为：{}", params); - StaffPatrolStatsFormDTO formDTO = JSON.parseObject(params, StaffPatrolStatsFormDTO.class); - Result result = bizDataStatsService.exeStaffPatrolStats(formDTO); + BizDataFormDTO formDTO = JSON.parseObject(params, BizDataFormDTO.class); + Result result = bizDataStatsService.exeStatsDaily(formDTO); if (result.success()){ log.debug("staffPatrolStatsTask定时任务正在执行定时任务执行成功"); }else {