elink-star
/
ShibeiWorkPlatform
12
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Merge branch 'dev' of http://git.elinkit.com.cn:7070/r/epmet-cloud

dev_shibei_match
jianjun 4 years ago
parent
0142f6fcc5 b7294651aa
commit
c08d74d557
5 changed files with 184 additions and 59 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      epmet-gateway/src/main/resources/bootstrap.yml
  2. 10
      epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/FactAggregationExtractController.java
  3. 6
      epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsUserServiceImpl.java
  4. 223
      epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/stats/user/impl/UserStatisticalServiceImpl.java
  5. 2
      epmet-module/epmet-job/epmet-job-server/src/main/resources/bootstrap.yml

2
epmet-gateway/src/main/resources/bootstrap.yml
View File

@ -507,6 +507,8 @@ epmet:
- /data/stats/basereport/** - /data/stats/basereport/**
- /data/stats/governance/** - /data/stats/governance/**
- /third/private-epmet/push-component-access-token - /third/private-epmet/push-component-access-token
- /data/stats/plugins/ofs/**
- /data/stats/plugins/workrecord/**
# 对外开放接口认证白名单 # 对外开放接口认证白名单

10
epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/FactAggregationExtractController.java
View File

@ -72,11 +72,11 @@ public class FactAggregationExtractController {
break; break;
default: default:
extractUserDataDaily(param); extractUserDataDaily(param);
extractGroupDataDaily(param); // extractGroupDataDaily(param);
extractTopicDataDaily(param); // extractTopicDataDaily(param);
extractIssueDataDaily(param); // extractIssueDataDaily(param);
extractProjectDataDaily(param); // extractProjectDataDaily(param);
extractAtricleDataDaily(param); // extractAtricleDataDaily(param);
} }
}); });
return new Result(); return new Result();

6
epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsUserServiceImpl.java
View File

@ -3,6 +3,7 @@ package com.epmet.service.impl;
import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSON;
import com.epmet.commons.tools.constant.NumConstant; import com.epmet.commons.tools.constant.NumConstant;
import com.epmet.commons.tools.exception.ExceptionUtils;
import com.epmet.commons.tools.exception.RenException; import com.epmet.commons.tools.exception.RenException;
import com.epmet.commons.tools.utils.DateUtils; import com.epmet.commons.tools.utils.DateUtils;
import com.epmet.constant.ProjectConstant; import com.epmet.constant.ProjectConstant;
@ -29,7 +30,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import java.util.Calendar; import java.util.Calendar;
@ -138,7 +138,7 @@ public class StatsUserServiceImpl implements StatsUserService {
UserStatisticalData agencyData = userService.traverseAgencyUser(agencies, date, timeDimension); UserStatisticalData agencyData = userService.traverseAgencyUser(agencies, date, timeDimension);
userStatisticalService.insertUniquely(agencyData); userStatisticalService.insertUniquely(agencyData);
} catch (Exception e) { } catch (Exception e) {
log.error(String.format(ProjectConstant.STATS_FAILED_PREFIX, ModuleConstant.EXCEPTION_DING_INTERCEPTOR_PREFIX_AGENCY_USER_STATISTICAL, customerId, new Date().toString(), e.getMessage())); log.error(String.format(ProjectConstant.STATS_FAILED_PREFIX, ModuleConstant.EXCEPTION_DING_INTERCEPTOR_PREFIX_AGENCY_USER_STATISTICAL, customerId, new Date().toString(), ExceptionUtils.getErrorStackTrace(e)));
} }
//5.计算网格统计数据、生成唯一性统计数据 //5.计算网格统计数据、生成唯一性统计数据
@ -146,7 +146,7 @@ public class StatsUserServiceImpl implements StatsUserService {
UserStatisticalData gridData = userService.traverseGridUser(agencies, date, timeDimension); UserStatisticalData gridData = userService.traverseGridUser(agencies, date, timeDimension);
userStatisticalService.insertUniquely(gridData); userStatisticalService.insertUniquely(gridData);
} catch (Exception e) { } catch (Exception e) {
log.error(String.format(ProjectConstant.STATS_FAILED_PREFIX, ModuleConstant.EXCEPTION_DING_INTERCEPTOR_PREFIX_GRID_USER_STATISTICAL, customerId, new Date().toString(), e.getMessage())); log.error(String.format(ProjectConstant.STATS_FAILED_PREFIX, ModuleConstant.EXCEPTION_DING_INTERCEPTOR_PREFIX_GRID_USER_STATISTICAL, customerId, new Date().toString(), ExceptionUtils.getErrorStackTrace(e)));
} }
} }

223
epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/stats/user/impl/UserStatisticalServiceImpl.java
View File

@ -6,15 +6,17 @@ import com.epmet.dao.stats.user.*;
import com.epmet.dto.stats.user.*; import com.epmet.dto.stats.user.*;
import com.epmet.dto.stats.user.result.UserStatisticalData; import com.epmet.dto.stats.user.result.UserStatisticalData;
import com.epmet.service.stats.user.UserStatisticalService; import com.epmet.service.stats.user.UserStatisticalService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.ListUtils; import org.apache.commons.collections4.ListUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import java.util.List; import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
/** /**
* @Description * @Description
@ -22,10 +24,10 @@ import java.util.List;
* @Auth wangc * @Auth wangc
* @Date 2020-06-23 14:27 * @Date 2020-06-23 14:27
*/ */
@Slf4j
@Service @Service
public class UserStatisticalServiceImpl implements UserStatisticalService { public class UserStatisticalServiceImpl implements UserStatisticalService {
private static final Logger logger = LoggerFactory.getLogger(UserStatisticalServiceImpl.class);
@Autowired @Autowired
private FactParticipationUserAgencyDailyDao participationUserAgencyDailyDao; private FactParticipationUserAgencyDailyDao participationUserAgencyDailyDao;
@ -51,104 +53,225 @@ public class UserStatisticalServiceImpl implements UserStatisticalService {
@Autowired @Autowired
private FactRegUserGridMonthlyDao regUserGridMonthlyDao; private FactRegUserGridMonthlyDao regUserGridMonthlyDao;
@Autowired
private ExecutorService executorService;
@Override @Override
@Transactional(rollbackFor = Exception.class)
public void insertUniquely(UserStatisticalData data) { public void insertUniquely(UserStatisticalData data) {
if (null == data) return ; if (null == data) {
return;
if(StringUtils.isBlank(data.getDateId()) || StringUtils.isBlank(data.getMonthId())){ }
logger.warn("缺失重要参数:dateId或monthId"); long start = System.currentTimeMillis();
return ; if (StringUtils.isBlank(data.getDateId()) || StringUtils.isBlank(data.getMonthId())) {
log.error("缺失重要参数:dateId或monthId");
return;
} }
CountDownLatch countDownLatch = new CountDownLatch(8);
executorService.submit(() -> {
try {
insertParticipationUserAgencyDaily(data);
} catch (Exception e) {
log.error("insertParticipationUserAgencyDaily exception", e);
} finally {
countDownLatch.countDown();
}
});
executorService.submit(() -> {
try {
insertParticipationUserAgencyMonthly(data);
} catch (Exception e) {
log.error("insertParticipationUserAgencyMonthly exception", e);
} finally {
countDownLatch.countDown();
}
});
executorService.submit(() -> {
try {
insertParticipationUserGridMonthly(data);
} catch (Exception e) {
log.error("insertParticipationUserGridMonthly exception", e);
} finally {
countDownLatch.countDown();
}
});
executorService.submit(() -> {
try {
insertParticipationUserGridDaily(data);
} catch (Exception e) {
log.error("insertParticipationUserGridDaily exception", e);
} finally {
countDownLatch.countDown();
}
});
executorService.submit(() -> {
try {
insertRegUserAgencyDaily(data);
} catch (Exception e) {
log.error("insertRegUserAgencyDaily exception", e);
} finally {
countDownLatch.countDown();
}
});
executorService.submit(() -> {
try {
insertRegUserAgencyMonthly(data);
} catch (Exception e) {
log.error("insertRegUserAgencyMonthly exception", e);
} finally {
countDownLatch.countDown();
}
});
executorService.submit(() -> {
try {
insertRegUserGridDaily(data);
} catch (Exception e) {
log.error("insertRegUserGridDaily exception", e);
} finally {
countDownLatch.countDown();
}
});
executorService.submit(() -> {
try {
insertRegUserGridMonthly(data);
} catch (Exception e) {
log.error("insertRegUserGridMonthly exception", e);
} finally {
countDownLatch.countDown();
}
});
countDownLatch.getCount();
/* insertParticipationUserAgencyDaily(data);
insertParticipationUserAgencyMonthly(data);
insertParticipationUserGridMonthly(data);
insertParticipationUserGridDaily(data);
insertRegUserAgencyDaily(data);
insertRegUserAgencyMonthly(data);
insertRegUserGridDaily(data);
insertRegUserGridMonthly(data);*/
log.warn("执行完毕======总耗时:{}ms",System.currentTimeMillis()-start);
}
if(null != data.getPartiAgencyDailyList() && data.getPartiAgencyDailyList().size() > NumConstant.ZERO){ @Transactional(rollbackFor = Exception.class)
public void insertRegUserGridMonthly(UserStatisticalData data) {
if (CollectionUtils.isNotEmpty(data.getRegGridMonthlyList())) {
Integer delNum; Integer delNum;
do { do {
delNum = participationUserAgencyDailyDao.deleteByParams(data.getDateId(),data.getCustomerId()); delNum = regUserGridMonthlyDao.deleteByParams(data.getMonthId(), data.getCustomerId());
}while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND); } while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND);
List<List<FactParticipationUserAgencyDailyDTO>> partition = ListUtils.partition(data.getPartiAgencyDailyList(), NumConstant.ONE_HUNDRED); List<List<FactRegUserGridMonthlyDTO>> partition = ListUtils.partition(data.getRegGridMonthlyList(), NumConstant.ONE_HUNDRED);
partition.forEach(p -> { partition.forEach(p -> {
participationUserAgencyDailyDao.insertBatch(p); regUserGridMonthlyDao.insertBatch(p);
}); });
} }
}
if(null != data.getPartiAgencyMonthlyList() && data.getPartiAgencyMonthlyList().size() > NumConstant.ZERO){ @Transactional(rollbackFor = Exception.class)
public void insertRegUserGridDaily(UserStatisticalData data) {
if (CollectionUtils.isNotEmpty(data.getRegGridDailyList())) {
Integer delNum; Integer delNum;
do { do {
delNum = participationUserAgencyMonthlyDao.deleteByParams(data.getMonthId(),data.getCustomerId()); delNum = regUserGridDailyDao.deleteByParams(data.getDateId(), data.getCustomerId());
}while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND); } while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND);
List<List<FactParticipationUserAgencyMonthlyDTO>> partition = ListUtils.partition(data.getPartiAgencyMonthlyList(), NumConstant.ONE_HUNDRED); List<List<FactRegUserGridDailyDTO>> partition = ListUtils.partition(data.getRegGridDailyList(), NumConstant.ONE_HUNDRED);
partition.forEach(p -> { partition.forEach(p -> {
participationUserAgencyMonthlyDao.insertBatch(p); regUserGridDailyDao.insertBatch(p);
}); });
} }
}
if(null != data.getPartiGridMonthlyList() && data.getPartiGridMonthlyList().size() > NumConstant.ZERO){ @Transactional(rollbackFor = Exception.class)
public void insertRegUserAgencyMonthly(UserStatisticalData data) {
if (CollectionUtils.isNotEmpty(data.getRegAgencyMonthlyList())) {
Integer delNum; Integer delNum;
do { do {
delNum = participationUserGridMonthlyDao.deleteByParams(data.getMonthId(),data.getCustomerId()); delNum = regUserAgencyMonthlyDao.deleteByParams(data.getMonthId(), data.getCustomerId());
}while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND); } while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND);
List<List<FactParticipationUserGridMonthlyDTO>> partition = ListUtils.partition(data.getPartiGridMonthlyList(), NumConstant.ONE_HUNDRED); List<List<FactRegUserAgencyMonthlyDTO>> partition = ListUtils.partition(data.getRegAgencyMonthlyList(), NumConstant.ONE_HUNDRED);
partition.forEach(p -> { partition.forEach(p -> {
participationUserGridMonthlyDao.insertBatch(p); regUserAgencyMonthlyDao.insertBatch(p);
}); });
} }
}
if(null != data.getPartiGridDailyList() && data.getPartiGridDailyList().size() > NumConstant.ZERO){ @Transactional(rollbackFor = Exception.class)
public void insertRegUserAgencyDaily(UserStatisticalData data) {
if (CollectionUtils.isNotEmpty(data.getRegAgencyDailyList())) {
Integer delNum; Integer delNum;
do { do {
delNum = participationUserGridDailyDao.deleteByParams(data.getDateId(),data.getCustomerId()); delNum = regUserAgencyDailyDao.deleteByParams(data.getDateId(), data.getCustomerId());
}while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND); } while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND);
List<List<FactParticipationUserGridDailyDTO>> partition = ListUtils.partition(data.getPartiGridDailyList(), NumConstant.ONE_HUNDRED); List<List<FactRegUserAgencyDailyDTO>> partition = ListUtils.partition(data.getRegAgencyDailyList(), NumConstant.ONE_HUNDRED);
partition.forEach(p -> { partition.forEach(p -> {
participationUserGridDailyDao.insertBatch(p); regUserAgencyDailyDao.insertBatch(p);
}); });
} }
}
if(null != data.getRegAgencyDailyList() && data.getRegAgencyDailyList().size() > NumConstant.ZERO){ @Transactional(rollbackFor = Exception.class)
public void insertParticipationUserGridDaily(UserStatisticalData data) {
if (CollectionUtils.isNotEmpty(data.getPartiGridDailyList())) {
Integer delNum; Integer delNum;
do { do {
delNum = regUserAgencyDailyDao.deleteByParams(data.getDateId(),data.getCustomerId()); delNum = participationUserGridDailyDao.deleteByParams(data.getDateId(), data.getCustomerId());
}while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND); } while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND);
List<List<FactRegUserAgencyDailyDTO>> partition = ListUtils.partition(data.getRegAgencyDailyList(), NumConstant.ONE_HUNDRED); List<List<FactParticipationUserGridDailyDTO>> partition = ListUtils.partition(data.getPartiGridDailyList(), NumConstant.ONE_HUNDRED);
partition.forEach(p -> { partition.forEach(p -> {
regUserAgencyDailyDao.insertBatch(p); participationUserGridDailyDao.insertBatch(p);
}); });
} }
}
if(null != data.getRegAgencyMonthlyList() && data.getRegAgencyMonthlyList().size() > NumConstant.ZERO){ @Transactional(rollbackFor = Exception.class)
public void insertParticipationUserGridMonthly(UserStatisticalData data) {
if (CollectionUtils.isNotEmpty(data.getPartiGridMonthlyList())) {
Integer delNum; Integer delNum;
do { do {
delNum = regUserAgencyMonthlyDao.deleteByParams(data.getMonthId(),data.getCustomerId()); delNum = participationUserGridMonthlyDao.deleteByParams(data.getMonthId(), data.getCustomerId());
}while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND); } while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND);
List<List<FactRegUserAgencyMonthlyDTO>> partition = ListUtils.partition(data.getRegAgencyMonthlyList(), NumConstant.ONE_HUNDRED); List<List<FactParticipationUserGridMonthlyDTO>> partition = ListUtils.partition(data.getPartiGridMonthlyList(), NumConstant.ONE_HUNDRED);
partition.forEach(p -> { partition.forEach(p -> {
regUserAgencyMonthlyDao.insertBatch(p); participationUserGridMonthlyDao.insertBatch(p);
}); });
} }
}
if(null != data.getRegGridDailyList() && data.getRegGridDailyList().size() > NumConstant.ZERO){ @Transactional(rollbackFor = Exception.class)
public void insertParticipationUserAgencyMonthly(UserStatisticalData data) {
if (CollectionUtils.isNotEmpty(data.getPartiAgencyMonthlyList())) {
Integer delNum; Integer delNum;
do { do {
delNum = regUserGridDailyDao.deleteByParams(data.getDateId(),data.getCustomerId()); delNum = participationUserAgencyMonthlyDao.deleteByParams(data.getMonthId(), data.getCustomerId());
}while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND); } while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND);
List<List<FactRegUserGridDailyDTO>> partition = ListUtils.partition(data.getRegGridDailyList(), NumConstant.ONE_HUNDRED); List<List<FactParticipationUserAgencyMonthlyDTO>> partition = ListUtils.partition(data.getPartiAgencyMonthlyList(), NumConstant.ONE_HUNDRED);
partition.forEach(p -> { partition.forEach(p -> {
regUserGridDailyDao.insertBatch(p); participationUserAgencyMonthlyDao.insertBatch(p);
}); });
} }
}
if(null != data.getRegGridMonthlyList() && data.getRegGridMonthlyList().size() > NumConstant.ZERO){ @Transactional(rollbackFor = Exception.class)
public void insertParticipationUserAgencyDaily(UserStatisticalData data) {
if (CollectionUtils.isNotEmpty(data.getPartiAgencyDailyList())) {
Integer delNum; Integer delNum;
do { do {
delNum = regUserGridMonthlyDao.deleteByParams(data.getMonthId(),data.getCustomerId()); delNum = participationUserAgencyDailyDao.deleteByParams(data.getDateId(), data.getCustomerId());
}while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND); } while (delNum > NumConstant.ZERO && delNum == NumConstant.ONE_THOUSAND);
List<List<FactRegUserGridMonthlyDTO>> partition = ListUtils.partition(data.getRegGridMonthlyList(), NumConstant.ONE_HUNDRED); List<List<FactParticipationUserAgencyDailyDTO>> partition = ListUtils.partition(data.getPartiAgencyDailyList(), NumConstant.ONE_HUNDRED);
partition.forEach(p -> { partition.forEach(p -> {
regUserGridMonthlyDao.insertBatch(p); participationUserAgencyDailyDao.insertBatch(p);
}); });
} }
} }
} }

2
epmet-module/epmet-job/epmet-job-server/src/main/resources/bootstrap.yml
View File

@ -119,7 +119,7 @@ hystrix:
timeoutInMilliseconds: 600000 #缺省为1000 timeoutInMilliseconds: 600000 #缺省为1000
ribbon: ribbon:
ReadTimeout: 300000 ReadTimeout: 600000
ConnectTimeout: 300000 ConnectTimeout: 300000
dingTalk: dingTalk:

Write Preview
Loading…
Cancel
Save