From 01c54ce593eaa4d5dfb97ff2786c17f4746603fc Mon Sep 17 00:00:00 2001 From: yinzuomei <576302893@qq.com> Date: Mon, 11 Oct 2021 10:47:03 +0800 Subject: [PATCH 1/3] =?UTF-8?q?146,=E5=B7=A5=E4=BD=9C=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E9=87=87=E9=9B=86=EF=BC=8Curl=E5=BC=80?= =?UTF-8?q?=E6=96=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- epmet-gateway/src/main/resources/bootstrap.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/epmet-gateway/src/main/resources/bootstrap.yml b/epmet-gateway/src/main/resources/bootstrap.yml index a5276aa300..c0308758b9 100644 --- a/epmet-gateway/src/main/resources/bootstrap.yml +++ b/epmet-gateway/src/main/resources/bootstrap.yml @@ -507,6 +507,8 @@ epmet: - /data/stats/basereport/** - /data/stats/governance/** - /third/private-epmet/push-component-access-token + - /data/stats/plugins/ofs/** + - /data/stats/plugins/workrecord/** # 对外开放接口认证白名单 From e66a8eeaeb4db7160306043946a0ebe3b591b6a8 Mon Sep 17 00:00:00 2001 From: jianjun Date: Mon, 11 Oct 2021 10:48:18 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E7=94=A8=E6=88=B7=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=E6=8A=A5=E9=94=99=20=E6=B7=BB=E5=8A=A0=E7=BA=BF=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../FactAggregationExtractController.java | 10 +- .../service/impl/StatsUserServiceImpl.java | 6 +- .../user/impl/UserStatisticalServiceImpl.java | 221 ++++++++++++++---- .../src/main/resources/bootstrap.yml | 2 +- 4 files changed, 181 insertions(+), 58 deletions(-) diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/FactAggregationExtractController.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/FactAggregationExtractController.java index c0618c9250..a0c26bcd6b 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/FactAggregationExtractController.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/FactAggregationExtractController.java @@ -72,11 +72,11 @@ public class FactAggregationExtractController { break; default: extractUserDataDaily(param); - extractGroupDataDaily(param); - extractTopicDataDaily(param); - extractIssueDataDaily(param); - extractProjectDataDaily(param); - extractAtricleDataDaily(param); +// extractGroupDataDaily(param); +// extractTopicDataDaily(param); +// extractIssueDataDaily(param); +// extractProjectDataDaily(param); +// extractAtricleDataDaily(param); } }); return new Result(); diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsUserServiceImpl.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsUserServiceImpl.java index dc082b44ca..bdc1999ff7 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsUserServiceImpl.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsUserServiceImpl.java @@ -3,6 +3,7 @@ package com.epmet.service.impl; import com.alibaba.fastjson.JSON; import com.epmet.commons.tools.constant.NumConstant; +import com.epmet.commons.tools.exception.ExceptionUtils; import com.epmet.commons.tools.exception.RenException; import com.epmet.commons.tools.utils.DateUtils; import com.epmet.constant.ProjectConstant; @@ -29,7 +30,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; import org.springframework.util.CollectionUtils; import java.util.Calendar; @@ -138,7 +138,7 @@ public class StatsUserServiceImpl implements StatsUserService { UserStatisticalData agencyData = userService.traverseAgencyUser(agencies, date, timeDimension); userStatisticalService.insertUniquely(agencyData); } catch (Exception e) { - log.error(String.format(ProjectConstant.STATS_FAILED_PREFIX, ModuleConstant.EXCEPTION_DING_INTERCEPTOR_PREFIX_AGENCY_USER_STATISTICAL, customerId, new Date().toString(), e.getMessage())); + log.error(String.format(ProjectConstant.STATS_FAILED_PREFIX, ModuleConstant.EXCEPTION_DING_INTERCEPTOR_PREFIX_AGENCY_USER_STATISTICAL, customerId, new Date().toString(), ExceptionUtils.getErrorStackTrace(e))); } //5.计算网格统计数据、生成唯一性统计数据 @@ -146,7 +146,7 @@ public class StatsUserServiceImpl implements StatsUserService { UserStatisticalData gridData = userService.traverseGridUser(agencies, date, timeDimension); userStatisticalService.insertUniquely(gridData); } catch (Exception e) { - log.error(String.format(ProjectConstant.STATS_FAILED_PREFIX, ModuleConstant.EXCEPTION_DING_INTERCEPTOR_PREFIX_GRID_USER_STATISTICAL, customerId, new Date().toString(), e.getMessage())); + log.error(String.format(ProjectConstant.STATS_FAILED_PREFIX, ModuleConstant.EXCEPTION_DING_INTERCEPTOR_PREFIX_GRID_USER_STATISTICAL, customerId, new Date().toString(), ExceptionUtils.getErrorStackTrace(e))); } } diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/stats/user/impl/UserStatisticalServiceImpl.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/stats/user/impl/UserStatisticalServiceImpl.java index 50b3ade54f..92cbe678ba 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/stats/user/impl/UserStatisticalServiceImpl.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/stats/user/impl/UserStatisticalServiceImpl.java @@ -6,15 +6,17 @@ import com.epmet.dao.stats.user.*; import com.epmet.dto.stats.user.*; import com.epmet.dto.stats.user.result.UserStatisticalData; import com.epmet.service.stats.user.UserStatisticalService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.ListUtils; import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; /** * @Description @@ -22,10 +24,10 @@ import java.util.List; * @Auth wangc * @Date 2020-06-23 14:27 */ +@Slf4j @Service public class UserStatisticalServiceImpl implements UserStatisticalService { - private static final Logger logger = LoggerFactory.getLogger(UserStatisticalServiceImpl.class); @Autowired private FactParticipationUserAgencyDailyDao participationUserAgencyDailyDao; @@ -51,104 +53,225 @@ public class UserStatisticalServiceImpl implements UserStatisticalService { @Autowired private FactRegUserGridMonthlyDao regUserGridMonthlyDao; + @Autowired + private ExecutorService executorService; + @Override - @Transactional(rollbackFor = Exception.class) public void insertUniquely(UserStatisticalData data) { - if (null == data) return ; + if (null == data) { + return; + } - if(StringUtils.isBlank(data.getDateId()) || StringUtils.isBlank(data.getMonthId())){ - logger.warn("缺失重要参数:dateId或monthId"); - return ; + if (StringUtils.isBlank(data.getDateId()) || StringUtils.isBlank(data.getMonthId())) { + log.error("缺失重要参数:dateId或monthId"); + return; } + CountDownLatch countDownLatch = new CountDownLatch(8); + executorService.submit(() -> { + try { + insertParticipationUserAgencyDaily(data); + } catch (Exception e) { + log.error("insertParticipationUserAgencyDaily exception", e); + } finally { + countDownLatch.countDown(); + } + }); + + executorService.submit(() -> { + try { + insertParticipationUserAgencyMonthly(data); + } catch (Exception e) { + log.error("insertParticipationUserAgencyMonthly exception", e); + } finally { + countDownLatch.countDown(); + } + }); + + executorService.submit(() -> { + try { + insertParticipationUserGridMonthly(data); + } catch (Exception e) { + log.error("insertParticipationUserGridMonthly exception", e); + } finally { + countDownLatch.countDown(); + } + }); + + executorService.submit(() -> { + try { + insertParticipationUserGridDaily(data); + } catch (Exception e) { + log.error("insertParticipationUserGridDaily exception", e); + } finally { + countDownLatch.countDown(); + } + }); + + executorService.submit(() -> { + try { + insertRegUserAgencyDaily(data); + } catch (Exception e) { + log.error("insertRegUserAgencyDaily exception", e); + } finally { + countDownLatch.countDown(); + } + }); + + executorService.submit(() -> { + try { + insertRegUserAgencyMonthly(data); + } catch (Exception e) { + log.error("insertRegUserAgencyMonthly exception", e); + } finally { + countDownLatch.countDown(); + } + }); + + executorService.submit(() -> { + try { + insertRegUserGridDaily(data); + } catch (Exception e) { + log.error("insertRegUserGridDaily exception", e); + } finally { + countDownLatch.countDown(); + } + }); + + executorService.submit(() -> { + try { + insertRegUserGridMonthly(data); + } catch (Exception e) { + log.error("insertRegUserGridMonthly exception", e); + } finally { + countDownLatch.countDown(); + } + }); + + + countDownLatch.getCount(); + + /* insertParticipationUserAgencyDaily(data); + insertParticipationUserAgencyMonthly(data); + insertParticipationUserGridMonthly(data); + insertParticipationUserGridDaily(data); + insertRegUserAgencyDaily(data); + insertRegUserAgencyMonthly(data); + insertRegUserGridDaily(data); + insertRegUserGridMonthly(data);*/ + + log.info("执行完毕======"); + } - if(null != data.getPartiAgencyDailyList() && data.getPartiAgencyDailyList().size() > NumConstant.ZERO){ + @Transactional(rollbackFor = Exception.class) + public void insertRegUserGridMonthly(UserStatisticalData data) { + if (CollectionUtils.isNotEmpty(data.getRegGridMonthlyList())) { Integer delNum; do { - delNum = participationUserAgencyDailyDao.deleteByParams(data.getDateId(),data.getCustomerId()); - }while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND); - List> partition = ListUtils.partition(data.getPartiAgencyDailyList(), NumConstant.ONE_HUNDRED); + delNum = regUserGridMonthlyDao.deleteByParams(data.getMonthId(), data.getCustomerId()); + } while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND); + List> partition = ListUtils.partition(data.getRegGridMonthlyList(), NumConstant.ONE_HUNDRED); partition.forEach(p -> { - participationUserAgencyDailyDao.insertBatch(p); + regUserGridMonthlyDao.insertBatch(p); }); } + } - if(null != data.getPartiAgencyMonthlyList() && data.getPartiAgencyMonthlyList().size() > NumConstant.ZERO){ + @Transactional(rollbackFor = Exception.class) + public void insertRegUserGridDaily(UserStatisticalData data) { + if (CollectionUtils.isNotEmpty(data.getRegGridDailyList())) { Integer delNum; do { - delNum = participationUserAgencyMonthlyDao.deleteByParams(data.getMonthId(),data.getCustomerId()); - }while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND); - List> partition = ListUtils.partition(data.getPartiAgencyMonthlyList(), NumConstant.ONE_HUNDRED); + delNum = regUserGridDailyDao.deleteByParams(data.getDateId(), data.getCustomerId()); + } while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND); + List> partition = ListUtils.partition(data.getRegGridDailyList(), NumConstant.ONE_HUNDRED); partition.forEach(p -> { - participationUserAgencyMonthlyDao.insertBatch(p); + regUserGridDailyDao.insertBatch(p); }); } + } - if(null != data.getPartiGridMonthlyList() && data.getPartiGridMonthlyList().size() > NumConstant.ZERO){ + @Transactional(rollbackFor = Exception.class) + public void insertRegUserAgencyMonthly(UserStatisticalData data) { + if (CollectionUtils.isNotEmpty(data.getRegAgencyMonthlyList())) { Integer delNum; do { - delNum = participationUserGridMonthlyDao.deleteByParams(data.getMonthId(),data.getCustomerId()); - }while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND); - List> partition = ListUtils.partition(data.getPartiGridMonthlyList(), NumConstant.ONE_HUNDRED); + delNum = regUserAgencyMonthlyDao.deleteByParams(data.getMonthId(), data.getCustomerId()); + } while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND); + List> partition = ListUtils.partition(data.getRegAgencyMonthlyList(), NumConstant.ONE_HUNDRED); partition.forEach(p -> { - participationUserGridMonthlyDao.insertBatch(p); + regUserAgencyMonthlyDao.insertBatch(p); }); } + } - if(null != data.getPartiGridDailyList() && data.getPartiGridDailyList().size() > NumConstant.ZERO){ + @Transactional(rollbackFor = Exception.class) + public void insertRegUserAgencyDaily(UserStatisticalData data) { + if (CollectionUtils.isNotEmpty(data.getRegAgencyDailyList())) { Integer delNum; do { - delNum = participationUserGridDailyDao.deleteByParams(data.getDateId(),data.getCustomerId()); - }while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND); - List> partition = ListUtils.partition(data.getPartiGridDailyList(), NumConstant.ONE_HUNDRED); + delNum = regUserAgencyDailyDao.deleteByParams(data.getDateId(), data.getCustomerId()); + } while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND); + List> partition = ListUtils.partition(data.getRegAgencyDailyList(), NumConstant.ONE_HUNDRED); partition.forEach(p -> { - participationUserGridDailyDao.insertBatch(p); + regUserAgencyDailyDao.insertBatch(p); }); } + } - if(null != data.getRegAgencyDailyList() && data.getRegAgencyDailyList().size() > NumConstant.ZERO){ + @Transactional(rollbackFor = Exception.class) + public void insertParticipationUserGridDaily(UserStatisticalData data) { + if (CollectionUtils.isNotEmpty(data.getPartiGridDailyList())) { Integer delNum; do { - delNum = regUserAgencyDailyDao.deleteByParams(data.getDateId(),data.getCustomerId()); - }while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND); - List> partition = ListUtils.partition(data.getRegAgencyDailyList(), NumConstant.ONE_HUNDRED); + delNum = participationUserGridDailyDao.deleteByParams(data.getDateId(), data.getCustomerId()); + } while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND); + List> partition = ListUtils.partition(data.getPartiGridDailyList(), NumConstant.ONE_HUNDRED); partition.forEach(p -> { - regUserAgencyDailyDao.insertBatch(p); + participationUserGridDailyDao.insertBatch(p); }); } + } - if(null != data.getRegAgencyMonthlyList() && data.getRegAgencyMonthlyList().size() > NumConstant.ZERO){ + @Transactional(rollbackFor = Exception.class) + public void insertParticipationUserGridMonthly(UserStatisticalData data) { + if (CollectionUtils.isNotEmpty(data.getPartiGridMonthlyList())) { Integer delNum; do { - delNum = regUserAgencyMonthlyDao.deleteByParams(data.getMonthId(),data.getCustomerId()); - }while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND); - List> partition = ListUtils.partition(data.getRegAgencyMonthlyList(), NumConstant.ONE_HUNDRED); + delNum = participationUserGridMonthlyDao.deleteByParams(data.getMonthId(), data.getCustomerId()); + } while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND); + List> partition = ListUtils.partition(data.getPartiGridMonthlyList(), NumConstant.ONE_HUNDRED); partition.forEach(p -> { - regUserAgencyMonthlyDao.insertBatch(p); + participationUserGridMonthlyDao.insertBatch(p); }); } + } - if(null != data.getRegGridDailyList() && data.getRegGridDailyList().size() > NumConstant.ZERO){ + @Transactional(rollbackFor = Exception.class) + public void insertParticipationUserAgencyMonthly(UserStatisticalData data) { + if (CollectionUtils.isNotEmpty(data.getPartiAgencyMonthlyList())) { Integer delNum; do { - delNum = regUserGridDailyDao.deleteByParams(data.getDateId(),data.getCustomerId()); - }while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND); - List> partition = ListUtils.partition(data.getRegGridDailyList(), NumConstant.ONE_HUNDRED); + delNum = participationUserAgencyMonthlyDao.deleteByParams(data.getMonthId(), data.getCustomerId()); + } while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND); + List> partition = ListUtils.partition(data.getPartiAgencyMonthlyList(), NumConstant.ONE_HUNDRED); partition.forEach(p -> { - regUserGridDailyDao.insertBatch(p); + participationUserAgencyMonthlyDao.insertBatch(p); }); } + } - if(null != data.getRegGridMonthlyList() && data.getRegGridMonthlyList().size() > NumConstant.ZERO){ + @Transactional(rollbackFor = Exception.class) + public void insertParticipationUserAgencyDaily(UserStatisticalData data) { + if (CollectionUtils.isNotEmpty(data.getPartiAgencyDailyList())) { Integer delNum; do { - delNum = regUserGridMonthlyDao.deleteByParams(data.getMonthId(),data.getCustomerId()); - }while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND); - List> partition = ListUtils.partition(data.getRegGridMonthlyList(), NumConstant.ONE_HUNDRED); + delNum = participationUserAgencyDailyDao.deleteByParams(data.getDateId(), data.getCustomerId()); + } while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND); + List> partition = ListUtils.partition(data.getPartiAgencyDailyList(), NumConstant.ONE_HUNDRED); partition.forEach(p -> { - regUserGridMonthlyDao.insertBatch(p); + participationUserAgencyDailyDao.insertBatch(p); }); } - } } diff --git a/epmet-module/epmet-job/epmet-job-server/src/main/resources/bootstrap.yml b/epmet-module/epmet-job/epmet-job-server/src/main/resources/bootstrap.yml index 48ed6f28e9..e0cbb99af4 100644 --- a/epmet-module/epmet-job/epmet-job-server/src/main/resources/bootstrap.yml +++ b/epmet-module/epmet-job/epmet-job-server/src/main/resources/bootstrap.yml @@ -119,7 +119,7 @@ hystrix: timeoutInMilliseconds: 600000 #缺省为1000 ribbon: - ReadTimeout: 300000 + ReadTimeout: 600000 ConnectTimeout: 300000 dingTalk: From 991e1dd8730390617cff340d19598aa9c650d398 Mon Sep 17 00:00:00 2001 From: jianjun Date: Mon, 11 Oct 2021 10:49:28 +0800 Subject: [PATCH 3/3] =?UTF-8?q?=E7=94=A8=E6=88=B7=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=E6=8A=A5=E9=94=99=20=E6=B7=BB=E5=8A=A0=E7=BA=BF=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/stats/user/impl/UserStatisticalServiceImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/stats/user/impl/UserStatisticalServiceImpl.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/stats/user/impl/UserStatisticalServiceImpl.java index 92cbe678ba..8e1eb8179d 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/stats/user/impl/UserStatisticalServiceImpl.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/stats/user/impl/UserStatisticalServiceImpl.java @@ -62,7 +62,7 @@ public class UserStatisticalServiceImpl implements UserStatisticalService { if (null == data) { return; } - + long start = System.currentTimeMillis(); if (StringUtils.isBlank(data.getDateId()) || StringUtils.isBlank(data.getMonthId())) { log.error("缺失重要参数:dateId或monthId"); return; @@ -160,7 +160,7 @@ public class UserStatisticalServiceImpl implements UserStatisticalService { insertRegUserGridDaily(data); insertRegUserGridMonthly(data);*/ - log.info("执行完毕======"); + log.warn("执行完毕======总耗时:{}ms",System.currentTimeMillis()-start); } @Transactional(rollbackFor = Exception.class)