Browse Source

Merge remote-tracking branch 'remotes/origin/dev_patrol_stats' into develop

dev_shibei_match
jianjun 4 years ago
parent
commit
e902cbb98e
  1. 41
      epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/dto/extract/form/BizDataFormDTO.java
  2. 11
      epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/feign/DataStatisticalOpenFeignClient.java
  3. 9
      epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/feign/impl/DataStatisticalOpenFeignClientFallBack.java
  4. 11
      epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/BizDataStatsController.java
  5. 8
      epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/biz/BizDataStatsService.java
  6. 69
      epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/biz/impl/BizDataStatsServiceImpl.java
  7. 4
      epmet-module/epmet-job/epmet-job-server/src/main/java/com/epmet/service/BizDataStatsService.java
  8. 6
      epmet-module/epmet-job/epmet-job-server/src/main/java/com/epmet/service/impl/BizDataStatsServiceImpl.java
  9. 6
      epmet-module/epmet-job/epmet-job-server/src/main/java/com/epmet/task/StaffPatrolStatsTask.java

41
epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/dto/extract/form/BizDataFormDTO.java

@ -0,0 +1,41 @@
package com.epmet.dto.extract.form;
import com.epmet.commons.tools.validator.group.CustomerClientShowGroup;
import lombok.Data;
import javax.validation.constraints.NotBlank;
import java.io.Serializable;
/**
* desc: 从业务数据抽取到统计库 dto
*
* @Author zxc
* @DateTime 2020/9/16 6:01 下午
*/
@Data
public class BizDataFormDTO implements Serializable {
private static final long serialVersionUID = 2860395703825549825L;
public interface ExtractForm extends CustomerClientShowGroup {
}
@NotBlank(message = "客户ID不能为空", groups = ExtractForm.class)
private String customerId;
@NotBlank(message = "dateId不能为空", groups = ExtractForm.class)
private String dateId;
private String userId;
private String gridId;
/**
* 开始时间
*/
private String startDate;
/**
* 结束时间
*/
private String endDate;
}

11
epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/feign/DataStatisticalOpenFeignClient.java

@ -3,10 +3,7 @@ package com.epmet.feign;
import com.epmet.commons.tools.constant.ServiceConstant;
import com.epmet.commons.tools.utils.Result;
import com.epmet.dto.StatsFormDTO;
import com.epmet.dto.extract.form.ExtractIndexFormDTO;
import com.epmet.dto.extract.form.ExtractOriginFormDTO;
import com.epmet.dto.extract.form.ExtractScreenFormDTO;
import com.epmet.dto.extract.form.StaffPatrolStatsFormDTO;
import com.epmet.dto.extract.form.*;
import com.epmet.dto.group.form.GroupStatsFormDTO;
import com.epmet.dto.screen.form.InitCustomerIndexForm;
import com.epmet.dto.stats.form.CustomerIdAndDateIdFormDTO;
@ -258,13 +255,13 @@ public interface DataStatisticalOpenFeignClient {
Result gridVanguardStats(@RequestBody StatsFormDTO formDTO);
/**
* desc: 网格员巡查统计初始化及校正
* desc: 业务库按天统计 统一入库
*
* @param formDTO
* @return com.epmet.commons.tools.utils.Result
* @author LiuJanJun
* @date 2021/7/2 3:04 下午
*/
@PostMapping("/data/stats/bizData/stats/patrol")
Result staffPatrolStats(@RequestBody StaffPatrolStatsFormDTO formDTO);
@PostMapping("/data/stats/bizData/stats/daily")
Result exeStatsDaily(@RequestBody BizDataFormDTO formDTO);
}

9
epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/feign/impl/DataStatisticalOpenFeignClientFallBack.java

@ -4,10 +4,7 @@ import com.epmet.commons.tools.constant.ServiceConstant;
import com.epmet.commons.tools.utils.ModuleUtils;
import com.epmet.commons.tools.utils.Result;
import com.epmet.dto.StatsFormDTO;
import com.epmet.dto.extract.form.ExtractIndexFormDTO;
import com.epmet.dto.extract.form.ExtractOriginFormDTO;
import com.epmet.dto.extract.form.ExtractScreenFormDTO;
import com.epmet.dto.extract.form.StaffPatrolStatsFormDTO;
import com.epmet.dto.extract.form.*;
import com.epmet.dto.group.form.GroupStatsFormDTO;
import com.epmet.dto.screen.form.InitCustomerIndexForm;
import com.epmet.dto.stats.form.CustomerIdAndDateIdFormDTO;
@ -259,7 +256,7 @@ public class DataStatisticalOpenFeignClientFallBack implements DataStatisticalOp
}
@Override
public Result staffPatrolStats(StaffPatrolStatsFormDTO formDTO) {
return ModuleUtils.feignConError(ServiceConstant.DATA_STATISTICAL_SERVER, "staffPatrolStats", formDTO);
public Result exeStatsDaily(BizDataFormDTO formDTO) {
return ModuleUtils.feignConError(ServiceConstant.DATA_STATISTICAL_SERVER, "exeStatsDaily", formDTO);
}
}

11
epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/BizDataStatsController.java

@ -1,6 +1,7 @@
package com.epmet.controller;
import com.epmet.commons.tools.utils.Result;
import com.epmet.dto.extract.form.BizDataFormDTO;
import com.epmet.dto.extract.form.StaffPatrolStatsFormDTO;
import com.epmet.service.evaluationindex.extract.biz.BizDataStatsService;
import org.springframework.beans.factory.annotation.Autowired;
@ -22,6 +23,16 @@ public class BizDataStatsController {
private BizDataStatsService bizDataStatsService;
/**
* @return com.epmet.commons.tools.utils.Result
* @param formDTO
* @description 业务库统计数据 统一入库
* @Date 2021/3/26 13:27
**/
@RequestMapping("daily")
public Result execute(@RequestBody BizDataFormDTO formDTO) {
bizDataStatsService.exeDailyAll(formDTO);
return new Result();
} /**
* @return com.epmet.commons.tools.utils.Result
* @param formDTO
* @description 工作端数据一期用户分析参与用户注册用户分析

8
epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/biz/BizDataStatsService.java

@ -17,6 +17,7 @@
package com.epmet.service.evaluationindex.extract.biz;
import com.epmet.dto.extract.form.BizDataFormDTO;
import com.epmet.dto.extract.form.StaffPatrolStatsFormDTO;
/**
@ -27,6 +28,13 @@ import com.epmet.dto.extract.form.StaffPatrolStatsFormDTO;
*/
public interface BizDataStatsService {
/**
* desc:抽取所有业务数据到统计库
*
* @param dataFormDTO
*/
void exeDailyAll(BizDataFormDTO dataFormDTO);
void executeStaffPatrolStats(StaffPatrolStatsFormDTO formDTO);
}

69
epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/biz/impl/BizDataStatsServiceImpl.java

@ -6,6 +6,7 @@ import com.epmet.commons.tools.utils.ConvertUtils;
import com.epmet.commons.tools.utils.DateUtils;
import com.epmet.constant.ProjectConstant;
import com.epmet.dto.ProjectDTO;
import com.epmet.dto.extract.form.BizDataFormDTO;
import com.epmet.dto.extract.form.StaffPatrolStatsFormDTO;
import com.epmet.dto.user.result.CustomerGridStaffDTO;
import com.epmet.dto.user.result.CustomerStaffDTO;
@ -14,11 +15,14 @@ import com.epmet.dto.user.result.StatsStaffPatrolRecordDailyDTO;
import com.epmet.service.evaluationindex.extract.biz.BizDataStatsService;
import com.epmet.service.org.CustomerStaffService;
import com.epmet.service.project.ProjectService;
import com.epmet.service.stats.DimCustomerService;
import com.epmet.service.user.StatsStaffPatrolService;
import com.epmet.service.user.UserService;
import com.epmet.util.DimIdGenerator;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.springframework.beans.factory.annotation.Autowired;
@ -28,6 +32,7 @@ import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.concurrent.*;
import java.util.stream.Collectors;
/**
@ -40,6 +45,11 @@ import java.util.stream.Collectors;
@Slf4j
@Service
public class BizDataStatsServiceImpl implements BizDataStatsService {
ThreadFactory namedThreadFactory = new ThreadFactoryBuilder()
.setNameFormat("bizDataStats-pool-%d").build();
ExecutorService threadPool = new ThreadPoolExecutor(3, 6,
10L, TimeUnit.MINUTES,
new LinkedBlockingQueue<>(500), namedThreadFactory, new ThreadPoolExecutor.CallerRunsPolicy());
@Autowired
private CustomerStaffService customerStaffService;
@Autowired
@ -48,6 +58,36 @@ public class BizDataStatsServiceImpl implements BizDataStatsService {
private ProjectService projectService;
@Autowired
private StatsStaffPatrolService statsStaffPatrolService;
@Autowired
private DimCustomerService dimCustomerService;
@Override
public void exeDailyAll(BizDataFormDTO dataFormDTO) {
String customerId = dataFormDTO.getCustomerId();
if (StringUtils.isBlank(dataFormDTO.getDateId()) && (StringUtils.isBlank(dataFormDTO.getStartDate()) && StringUtils.isBlank(dataFormDTO.getEndDate()))) {
//如果没有设置开始日期、结束日期,默认查询昨天
dataFormDTO.setDateId(DimIdGenerator.getDateDimId(DateUtils.addDateDays(new Date(), -1)));
}
List<String> customerIds = new ArrayList<>();
if (StringUtils.isNotBlank(customerId)) {
//指定某个客户
customerIds.add(customerId);
} else {
//查询全部客户
int pageNo = NumConstant.ONE;
int pageSize = NumConstant.ONE_HUNDRED;
customerIds = dimCustomerService.selectCustomerIdPage(pageNo, pageSize);
if (org.springframework.util.CollectionUtils.isEmpty(customerIds)) {
log.error("exeDailyAll 获取客户Id为空");
return;
}
}
customerIds.forEach(cId -> {
log.info("exeDailyAll param:{}", JSON.toJSONString(dataFormDTO));
submitJob(dataFormDTO);
});
}
@Override
public void executeStaffPatrolStats(StaffPatrolStatsFormDTO formDTO) {
@ -216,4 +256,33 @@ public class BizDataStatsServiceImpl implements BizDataStatsService {
log.debug("getAllGridMembers result:{},param:{}", JSON.toJSONString(insertList), JSON.toJSONString(formDTO));
return insertList;
}
private void submitJob(BizDataFormDTO param) {
boolean isRange = StringUtils.isBlank(param.getDateId());
List<String> daysBetween = null;
if (isRange) {
daysBetween = DateUtils.getDaysBetween(param.getStartDate(), param.getEndDate());
}
List<String> finalDaysBetween = daysBetween;
threadPool.submit(() -> {
StaffPatrolStatsFormDTO formDTO = ConvertUtils.sourceToTarget(param, StaffPatrolStatsFormDTO.class);
if (!isRange) {
try {
formDTO.setDateId(param.getDateId());
this.executeStaffPatrolStats(formDTO);
} catch (Exception e) {
log.error("【网格员巡查数据统计】发生异常,参数:" + JSON.toJSONString(formDTO), e);
}
} else {
try {
for (String dateDimId : finalDaysBetween) {
formDTO.setDateId(dateDimId);
this.executeStaffPatrolStats(formDTO);
}
} catch (Exception e) {
log.error("【网格员巡查数据统计】发生异常,参数:" + JSON.toJSONString(param), e);
}
}
});
}
}

4
epmet-module/epmet-job/epmet-job-server/src/main/java/com/epmet/service/BizDataStatsService.java

@ -1,10 +1,10 @@
package com.epmet.service;
import com.epmet.commons.tools.utils.Result;
import com.epmet.dto.extract.form.StaffPatrolStatsFormDTO;
import com.epmet.dto.extract.form.BizDataFormDTO;
public interface BizDataStatsService {
Result exeStaffPatrolStats(StaffPatrolStatsFormDTO formDTO);
Result exeStatsDaily(BizDataFormDTO formDTO);
}

6
epmet-module/epmet-job/epmet-job-server/src/main/java/com/epmet/service/impl/BizDataStatsServiceImpl.java

@ -1,7 +1,7 @@
package com.epmet.service.impl;
import com.epmet.commons.tools.utils.Result;
import com.epmet.dto.extract.form.StaffPatrolStatsFormDTO;
import com.epmet.dto.extract.form.BizDataFormDTO;
import com.epmet.feign.DataStatisticalOpenFeignClient;
import com.epmet.service.BizDataStatsService;
import org.springframework.beans.factory.annotation.Autowired;
@ -19,7 +19,7 @@ public class BizDataStatsServiceImpl implements BizDataStatsService {
@Autowired
private DataStatisticalOpenFeignClient feignClient;
@Override
public Result exeStaffPatrolStats(StaffPatrolStatsFormDTO formDTO) {
return feignClient.staffPatrolStats(formDTO);
public Result exeStatsDaily(BizDataFormDTO formDTO) {
return feignClient.exeStatsDaily(formDTO);
}
}

6
epmet-module/epmet-job/epmet-job-server/src/main/java/com/epmet/task/StaffPatrolStatsTask.java

@ -2,7 +2,7 @@ package com.epmet.task;
import com.alibaba.fastjson.JSON;
import com.epmet.commons.tools.utils.Result;
import com.epmet.dto.extract.form.StaffPatrolStatsFormDTO;
import com.epmet.dto.extract.form.BizDataFormDTO;
import com.epmet.service.BizDataStatsService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
@ -23,8 +23,8 @@ public class StaffPatrolStatsTask implements ITask {
@Override
public void run(String params) {
log.debug("staffPatrolStatsTask定时任务正在执行,参数为:{}", params);
StaffPatrolStatsFormDTO formDTO = JSON.parseObject(params, StaffPatrolStatsFormDTO.class);
Result result = bizDataStatsService.exeStaffPatrolStats(formDTO);
BizDataFormDTO formDTO = JSON.parseObject(params, BizDataFormDTO.class);
Result result = bizDataStatsService.exeStatsDaily(formDTO);
if (result.success()){
log.debug("staffPatrolStatsTask定时任务正在执行定时任务执行成功");
}else {

Loading…
Cancel
Save