elink-star
/
ShibeiWorkPlatform
12
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Merge remote-tracking branch 'origin/dev' into dev

dev
wangxianzhang 4 years ago
parent
36f7f9471d 5acdfa9463
commit
ab25971c09
2 changed files with 63 additions and 53 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 102
      epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/dataToIndex/impl/IndexOriginExtractServiceImpl.java
  2. 14
      epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenExtractServiceImpl.java

102
epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/dataToIndex/impl/IndexOriginExtractServiceImpl.java
View File

@ -4,11 +4,14 @@ import cn.hutool.core.collection.CollectionUtil;
import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSON;
import com.epmet.commons.tools.constant.CustomerIdConstant; import com.epmet.commons.tools.constant.CustomerIdConstant;
import com.epmet.commons.tools.constant.NumConstant; import com.epmet.commons.tools.constant.NumConstant;
import com.epmet.commons.tools.enums.EnvEnum;
import com.epmet.commons.tools.utils.HttpClientManager;
import com.epmet.commons.tools.validator.ValidatorUtils; import com.epmet.commons.tools.validator.ValidatorUtils;
import com.epmet.dto.extract.form.ExtractIndexFormDTO; import com.epmet.dto.extract.form.ExtractIndexFormDTO;
import com.epmet.dto.indexcal.CalculateCommonFormDTO;
import com.epmet.service.evaluationindex.extract.dataToIndex.*; import com.epmet.service.evaluationindex.extract.dataToIndex.*;
import com.epmet.service.evaluationindex.indexcal.IndexCalculateService;
import com.epmet.service.stats.DimCustomerService; import com.epmet.service.stats.DimCustomerService;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
@ -18,7 +21,8 @@ import org.springframework.util.CollectionUtils;
import java.time.LocalDate; import java.time.LocalDate;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.concurrent.*; import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
/** /**
* desc：指标原始数据抽取服务实现类 * desc：指标原始数据抽取服务实现类
@ -26,12 +30,13 @@ import java.util.concurrent.*;
@Slf4j @Slf4j
@Service @Service
public class IndexOriginExtractServiceImpl implements IndexOriginExtractService { public class IndexOriginExtractServiceImpl implements IndexOriginExtractService {
ThreadFactory namedThreadFactory = new ThreadFactoryBuilder() /*ThreadFactory namedThreadFactory = new ThreadFactoryBuilder()
.setNameFormat("indexOriginExtract-pool-%d").build(); .setNameFormat("indexOriginExtractAndCal-pool-%d").build();
ExecutorService threadPool = new ThreadPoolExecutor(1, 1, ExecutorService threadPool = new ThreadPoolExecutor(1, 1,
10L, TimeUnit.MINUTES, 10L, TimeUnit.MINUTES,
new LinkedBlockingQueue<>(500), namedThreadFactory, new ThreadPoolExecutor.CallerRunsPolicy()); new LinkedBlockingQueue<>(500), namedThreadFactory, new ThreadPoolExecutor.CallerRunsPolicy());*/
@Autowired
private ExecutorService executorService;
@Autowired @Autowired
private CalCpcIndexService calCpcIndexService; private CalCpcIndexService calCpcIndexService;
@Autowired @Autowired
@ -45,7 +50,10 @@ public class IndexOriginExtractServiceImpl implements IndexOriginExtractService
@Autowired @Autowired
private IndexCollDistrictService indexCollDistrictService; private IndexCollDistrictService indexCollDistrictService;
@Autowired @Autowired
private IndexCollDistrictDepartmentService indexCollDistrictDepartmentService; private IndexCollDistrictDepartmentService indexCollDistrictDepartmentService;
@Autowired
private IndexCalculateService indexCalculateService;
/** /**
* desc：从统计库对象抽取指标数据 * desc：从统计库对象抽取指标数据
* *
@ -56,7 +64,7 @@ public class IndexOriginExtractServiceImpl implements IndexOriginExtractService
String monthId = formDTO.getMonthId(); String monthId = formDTO.getMonthId();
String customerId = formDTO.getCustomerId(); String customerId = formDTO.getCustomerId();
if (StringUtils.isBlank(monthId)) { if (StringUtils.isBlank(monthId)) {
monthId = LocalDate.now().minusMonths(NumConstant.ONE).toString().replace("-","").substring(NumConstant.ZERO,NumConstant.SIX); monthId = LocalDate.now().minusMonths(NumConstant.ONE).toString().replace("-", "").substring(NumConstant.ZERO, NumConstant.SIX);
} }
List<String> customerIds = new ArrayList<>(); List<String> customerIds = new ArrayList<>();
if (StringUtils.isNotBlank(customerId)) { if (StringUtils.isNotBlank(customerId)) {
@ -67,7 +75,7 @@ public class IndexOriginExtractServiceImpl implements IndexOriginExtractService
List<String> customerIdList = null; List<String> customerIdList = null;
do { do {
customerIdList = dimCustomerService.selectCustomerIdPage(pageNo++, pageSize); customerIdList = dimCustomerService.selectCustomerIdPage(pageNo++, pageSize);
if (!CollectionUtils.isEmpty(customerIdList)){ if (!CollectionUtils.isEmpty(customerIdList)) {
customerIds.addAll(customerIdList); customerIds.addAll(customerIdList);
} }
} while (!CollectionUtil.isEmpty(customerIdList) && customerIdList.size() == pageSize); } while (!CollectionUtil.isEmpty(customerIdList) && customerIdList.size() == pageSize);
@ -91,88 +99,104 @@ public class IndexOriginExtractServiceImpl implements IndexOriginExtractService
private void submitJob(ExtractIndexFormDTO param) { private void submitJob(ExtractIndexFormDTO param) {
CountDownLatch countDownLatch = new CountDownLatch(2); CountDownLatch countDownLatch = new CountDownLatch(NumConstant.SIX);
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
final String customerId = param.getCustomerId(); final String customerId = param.getCustomerId();
final String monthId = param.getMonthId(); final String monthId = param.getMonthId();
threadPool.submit(() -> { executorService.submit(() -> {
try { try {
long startCpc = System.currentTimeMillis(); long startCpc = System.currentTimeMillis();
calCpcIndexService.calCpcPartyAbility(customerId, monthId); calCpcIndexService.calCpcPartyAbility(customerId, monthId);
log.error("党员相关-党建能力执行完毕======总耗时:{}ms，customerId:{}",System.currentTimeMillis()-startCpc, param.getCustomerId()); log.error("党员相关-党建能力执行完毕======总耗时:{}ms，customerId:{}", System.currentTimeMillis() - startCpc, param.getCustomerId());
} catch (Exception e) { } catch (Exception e) {
log.error("抽取【党员相关数据】发生异常,参数:" + JSON.toJSONString(param), e); log.error("抽取【党员相关数据】发生异常,参数:" + JSON.toJSONString(param), e);
}finally { } finally {
countDownLatch.countDown(); countDownLatch.countDown();
} }
}); });
threadPool.submit(() -> { executorService.submit(() -> {
try { try {
long startGridGovern = System.currentTimeMillis(); long startGridGovern = System.currentTimeMillis();
calGridIndexService.calGridIndexGovernAbility(customerId, monthId); calGridIndexService.calGridIndexGovernAbility(customerId, monthId);
log.error("网格相关-治理能力执行完毕======总耗时:{}ms，customerId:{}",System.currentTimeMillis()-startGridGovern, param.getCustomerId()); log.error("网格相关-治理能力执行完毕======总耗时:{}ms，customerId:{}", System.currentTimeMillis() - startGridGovern, param.getCustomerId());
} catch (Exception e) { } catch (Exception e) {
log.error("抽取【网格治理能力数据】发生异常,参数:" + JSON.toJSONString(param), e); log.error("抽取【网格治理能力数据】发生异常,参数:" + JSON.toJSONString(param), e);
} }
try { try {
long startGridParty = System.currentTimeMillis(); long startGridParty = System.currentTimeMillis();
calGridIndexService.calGridIndexPartyAbility(customerId, monthId); calGridIndexService.calGridIndexPartyAbility(customerId, monthId);
log.error("网格相关-党建能力执行完毕======总耗时:{}ms，customerId:{}",System.currentTimeMillis()-startGridParty, param.getCustomerId()); log.error("网格相关-党建能力执行完毕======总耗时:{}ms，customerId:{}", System.currentTimeMillis() - startGridParty, param.getCustomerId());
} catch (Exception e) { } catch (Exception e) {
log.error("抽取【网格党建能力数据】发生异常,参数:" + JSON.toJSONString(param), e); log.error("抽取【网格党建能力数据】发生异常,参数:" + JSON.toJSONString(param), e);
} }
try { try {
long startGridService = System.currentTimeMillis(); long startGridService = System.currentTimeMillis();
calGridIndexService.calGridIndexServiceAbility(customerId, monthId); calGridIndexService.calGridIndexServiceAbility(customerId, monthId);
log.error("网格相关-服务能力执行完毕======总耗时:{}ms，customerId:{}",System.currentTimeMillis()-startGridService, param.getCustomerId()); log.error("网格相关-服务能力执行完毕======总耗时:{}ms，customerId:{}", System.currentTimeMillis() - startGridService, param.getCustomerId());
} catch (Exception e) { } catch (Exception e) {
log.error("抽取【网格服务能力数据】发生异常,参数:" + JSON.toJSONString(param), e); log.error("抽取【网格服务能力数据】发生异常,参数:" + JSON.toJSONString(param), e);
} }
countDownLatch.countDown();
try{ });
//dimAgency executorService.submit(() -> {
try {
long startCommunity = System.currentTimeMillis(); long startCommunity = System.currentTimeMillis();
indexCollCommunityService.saveCommunityAbility(customerId, monthId); indexCollCommunityService.saveCommunityAbility(customerId, monthId);
log.error("社区相关-三大能力执行完毕======总耗时:{}ms，customerId:{}",System.currentTimeMillis()-startCommunity, param.getCustomerId()); log.error("社区相关-三大能力执行完毕======总耗时:{}ms，customerId:{}", System.currentTimeMillis() - startCommunity, param.getCustomerId());
}catch (Exception e){ } catch (Exception e) {
log.error("抽取【社区治理能力-社区党建能力-服务能力】发生异常,参数:" + JSON.toJSONString(param), e); log.error("抽取【社区治理能力-社区党建能力-服务能力】发生异常,参数:" + JSON.toJSONString(param), e);
} }
try{ countDownLatch.countDown();
//dimAgency });
executorService.submit(() -> {
try {
long startStreet = System.currentTimeMillis(); long startStreet = System.currentTimeMillis();
indexCollStreetService.saveStreetAbility(customerId, monthId); indexCollStreetService.saveStreetAbility(customerId, monthId);
log.error("街道相关-三大能力执行完毕======总耗时:{}ms，customerId:{}",System.currentTimeMillis()-startStreet, param.getCustomerId()); log.error("街道相关-三大能力执行完毕======总耗时:{}ms，customerId:{}", System.currentTimeMillis() - startStreet, param.getCustomerId());
}catch (Exception e){ } catch (Exception e) {
log.error("抽取【街道治理能力-街道党建能力-服务能力】发生异常,参数:" + JSON.toJSONString(param), e); log.error("抽取【街道治理能力-街道党建能力-服务能力】发生异常,参数:" + JSON.toJSONString(param), e);
} }
countDownLatch.countDown();
});
try{ executorService.submit(() -> {
//dimAgency try {
long startDept = System.currentTimeMillis(); long startDept = System.currentTimeMillis();
indexCollDistrictDepartmentService.saveDepartmentAbility(customerId, monthId); indexCollDistrictDepartmentService.saveDepartmentAbility(customerId, monthId);
log.error("区直部门相关-治理能力执行完毕======总耗时:{}ms，customerId:{}",System.currentTimeMillis()-startDept, param.getCustomerId()); log.error("区直部门相关-治理能力执行完毕======总耗时:{}ms，customerId:{}", System.currentTimeMillis() - startDept, param.getCustomerId());
}catch (Exception e){ } catch (Exception e) {
log.error("抽取【区直部门治理能力】发生异常,参数:" + JSON.toJSONString(param), e); log.error("抽取【区直部门治理能力】发生异常,参数:" + JSON.toJSONString(param), e);
} }
countDownLatch.countDown();
});
try{ executorService.submit(() -> {
//dimAgency try {
long startDistrict = System.currentTimeMillis(); long startDistrict = System.currentTimeMillis();
indexCollDistrictService.saveDistrictAbility(customerId, monthId); indexCollDistrictService.saveDistrictAbility(customerId, monthId);
log.error("全区相关-三大能力执行完毕======总耗时:{}ms，customerId:{}",System.currentTimeMillis()-startDistrict, param.getCustomerId()); log.error("全区相关-三大能力执行完毕======总耗时:{}ms，customerId:{}", System.currentTimeMillis() - startDistrict, param.getCustomerId());
}catch (Exception e){ } catch (Exception e) {
log.error("抽取【全区治理能力-全区党建能力-服务能力】发生异常,参数:" + JSON.toJSONString(param), e); log.error("抽取【全区治理能力-全区党建能力-服务能力】发生异常,参数:" + JSON.toJSONString(param), e);
} }
countDownLatch.countDown(); countDownLatch.countDown();
}); });
try { try {
//等前面都执行完 再执行计算指标得分
countDownLatch.await(); countDownLatch.await();
executorService.submit(() -> {
long startT = System.currentTimeMillis();
CalculateCommonFormDTO formDTO1 = new CalculateCommonFormDTO();
try {
formDTO1.setMonthId(monthId);
formDTO1.setCustomerId(customerId);
Boolean aBoolean = indexCalculateService.indexCalculate(formDTO1);
HttpClientManager.getInstance().sendAlarmMsg(EnvEnum.getCurrentEnv().getName() + "客户Id：" + customerId + ";monthId：" + monthId + ",calculateAll全部指标计算完成,是否成功：" + aBoolean + "，总耗时：" + (System.currentTimeMillis() - startT) / 1000 + "秒");
} catch (Exception e) {
log.error("extractMonthly 计算分数异常，参数:{}", JSON.toJSONString(formDTO1));
}
});
} catch (InterruptedException e) { } catch (InterruptedException e) {
log.error("indexOriginExtractAll countDownLatch exception", e); log.error("indexOriginExtractAll countDownLatch exception", e);
} }
log.error("indexOriginExtractAll执行完毕======总耗时:{}ms，customerId:{}",System.currentTimeMillis()-start, param.getCustomerId()); log.info("indexOriginExtractAll及指标计算执行完毕======总耗时:{}ms，customerId:{}", System.currentTimeMillis() - start, param.getCustomerId());
} }
} }

14
epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenExtractServiceImpl.java
View File

@ -474,20 +474,6 @@ public class ScreenExtractServiceImpl implements ScreenExtractService {
log.info("===== extractMonthly method end not contains shi bei:{}======", customerId); log.info("===== extractMonthly method end not contains shi bei:{}======", customerId);
return; return;
} }
//此方法保持在最后即可 计算指标分数 todo 优化 手动创建线程池 控制任务数量
ExecutorService pool = Executors.newSingleThreadExecutor();
pool.submit(() -> {
long start = System.currentTimeMillis();
CalculateCommonFormDTO formDTO1 = new CalculateCommonFormDTO();
try {
formDTO1.setMonthId(monthId);
formDTO1.setCustomerId(customerId);
Boolean aBoolean = indexCalculateService.indexCalculate(formDTO1);
HttpClientManager.getInstance().sendAlarmMsg(EnvEnum.getCurrentEnv().getName() + "客户Id：" + formDTO.getCustomerId() + ";monthId：" + formDTO1.getMonthId() + ",calculateAll全部指标计算完成,是否成功：" + aBoolean + "，总耗时：" + (System.currentTimeMillis() - start) / 1000 + "秒");
} catch (Exception e) {
log.error("extractMonthly 计算分数异常，参数:{}", JSON.toJSONString(formDTO1));
}
});
log.info("===== extractMonthly method end ======"); log.info("===== extractMonthly method end ======");
} }
} }

Write Preview
Loading…
Cancel
Save