Browse Source

Merge remote-tracking branch 'remotes/origin/dev'

dev_shibei_match
jianjun 5 years ago
parent
commit
c5ee56a92a
  1. 7
      epmet-commons/epmet-commons-tools/pom.xml
  2. 56
      epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/config/RedissonConfig.java
  3. 1
      epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/constant/NumConstant.java
  4. 8
      epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/redis/RedisKeys.java
  5. 3
      epmet-module/data-statistical/data-statistical-server/Dockerfile
  6. 1
      epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/IndexCalculateController.java
  7. 11
      epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/indexcal/impl/DeptScoreServiceImpl.java
  8. 9
      epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/indexcal/impl/GridCorreLationServiceImpl.java
  9. 10
      epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/indexcal/impl/IndexCalculateCommunityServiceImpl.java
  10. 9
      epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/indexcal/impl/IndexCalculateDistrictServiceImpl.java
  11. 10
      epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/indexcal/impl/IndexCalculateStreetServiceImpl.java
  12. 81
      epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/indexcoll/impl/FactIndexCollectServiceImpl.java
  13. 2
      epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/support/normalizing/ScoreConstants.java
  14. 2
      epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/support/normalizing/batch/BatchScoreCalculator.java
  15. 10
      epmet-module/data-statistical/data-statistical-server/src/main/resources/logback-spring.xml
  16. 61
      epmet-module/data-statistical/data-statistical-server/src/test/java/com/epmet/stats/test/RedissonTest.java
  17. 7
      epmet-module/data-statistical/data-statistical-server/src/test/java/resources/数据值略小时可能的情况.txt

7
epmet-commons/epmet-commons-tools/pom.xml

@ -49,6 +49,13 @@
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-redis</artifactId> <artifactId>spring-boot-starter-data-redis</artifactId>
</dependency> </dependency>
<!-- https://mvnrepository.com/artifact/org.redisson/redisson -->
<dependency>
<groupId>org.redisson</groupId>
<artifactId>redisson</artifactId>
<version>3.12.5</version>
</dependency>
<dependency> <dependency>
<groupId>com.fasterxml.jackson.core</groupId> <groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId> <artifactId>jackson-databind</artifactId>

56
epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/config/RedissonConfig.java

@ -0,0 +1,56 @@
package com.epmet.commons.tools.config;
import com.epmet.commons.tools.constant.NumConstant;
import com.epmet.commons.tools.constant.StrConstant;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.redisson.Redisson;
import org.redisson.api.RedissonClient;
import org.redisson.codec.JsonJacksonCodec;
import org.redisson.config.Config;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
* redisson 配置类
*/
@Slf4j
@Configuration
public class RedissonConfig {
@Value("${spring.redis.host}")
private String host;
@Value("${spring.redis.port}")
private String port;
@Value("${spring.redis.password}")
private String password;
@Bean
public RedissonClient getRedisson() {
if (StringUtils.isBlank(host)) {
log.warn("getRedisson redis param is null,don't need to init redissonClient");
return null;
}
try {
Config config = new Config();
config.setCodec(new JsonJacksonCodec());
config.setThreads(NumConstant.FOUR);
config.setNettyThreads(NumConstant.FOUR);
//redis://ip:port
//redis的部署方式有单节点部署、主从方式部署、哨兵方式部署、集群方式部署
config.useSingleServer().setAddress("redis://".concat(host).concat(StrConstant.COLON).concat(port));
config.useSingleServer().setPassword(password);
config.useSingleServer().setConnectTimeout(NumConstant.ONE_THOUSAND * NumConstant.FIVE);
config.useSingleServer().setDatabase(NumConstant.TEN);
return Redisson.create(config);
} catch (Exception e) {
log.error("初始化redisson失败", e);
return null;
}
}
}

1
epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/constant/NumConstant.java

@ -39,6 +39,7 @@ public interface NumConstant {
int SIXTY = 60; int SIXTY = 60;
int ONE_HUNDRED = 100; int ONE_HUNDRED = 100;
BigDecimal ONE_HUNDRED_DECIMAL = new BigDecimal(100); BigDecimal ONE_HUNDRED_DECIMAL = new BigDecimal(100);
BigDecimal ZERO_DECIMAL = new BigDecimal(0);
int ONE_THOUSAND = 1000; int ONE_THOUSAND = 1000;
int MAX = 99999999; int MAX = 99999999;
int EIGHTY_EIGHT = 88; int EIGHTY_EIGHT = 88;

8
epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/redis/RedisKeys.java

@ -330,9 +330,17 @@ public class RedisKeys {
/** /**
* 获取计算标记的key前缀 * 获取计算标记的key前缀
*
* @return * @return
*/ */
public static String getCustomerStatsCalKeyPrefix() { public static String getCustomerStatsCalKeyPrefix() {
return rootPrefix.concat("stats:calflag"); return rootPrefix.concat("stats:calflag");
} }
/**
* 插入大屏指标数据分布式锁 key
*/
public static String getScreenIndexDataLockKey() {
return rootPrefix.concat("stats:indexcal:lock");
}
} }

3
epmet-module/data-statistical/data-statistical-server/Dockerfile

@ -8,4 +8,5 @@ COPY ./target/*.jar ./data-stats.jar
EXPOSE 8108 EXPOSE 8108
ENTRYPOINT ["sh", "-c", "$RUN_INSTRUCT"] #ENTRYPOINT ["sh", "-c", "$RUN_INSTRUCT"]
ENTRYPOINT ["sh","-c","exec $RUN_INSTRUCT"]

1
epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/IndexCalculateController.java

@ -60,6 +60,7 @@ public class IndexCalculateController {
@PreDestroy @PreDestroy
public void saveCalStatus() { public void saveCalStatus() {
log.info("data-statical-server服务被关闭,执行保存计算状态的动作");
// 实例销毁之前,将正在本实例中执行计算的客户列表的计算状态修改为pendding,等待其他实例重新计算 // 实例销毁之前,将正在本实例中执行计算的客户列表的计算状态修改为pendding,等待其他实例重新计算
futureMap.forEach((customerId, future) -> { futureMap.forEach((customerId, future) -> {
CalculateFlagModel flag = (CalculateFlagModel) redisUtils.get(RedisKeys.getCustomerStatsCalFlag(customerId)); CalculateFlagModel flag = (CalculateFlagModel) redisUtils.get(RedisKeys.getCustomerStatsCalFlag(customerId));

11
epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/indexcal/impl/DeptScoreServiceImpl.java

@ -140,12 +140,15 @@ public class DeptScoreServiceImpl extends BaseServiceImpl<DeptScoreDao, DeptScor
Map<String, Set<String>> selfSubParentMap = new HashMap<>(); Map<String, Set<String>> selfSubParentMap = new HashMap<>();
selfSubParentMap.put(ProjectConstant.ZI_SHEN, new HashSet<>()); selfSubParentMap.put(ProjectConstant.ZI_SHEN, new HashSet<>());
selfSubParentMap.put(ProjectConstant.XIA_JI, new HashSet<>()); selfSubParentMap.put(ProjectConstant.XIA_JI, new HashSet<>());
Map<String, BigDecimal> weightMap = new HashMap<>();
selfSubIndexList.forEach(o -> { selfSubIndexList.forEach(o -> {
//找出自身 和下级的指标 //找出自身 和下级的指标
if (o.getAllIndexCodePath().indexOf(ProjectConstant.XIA_JI) > -1) { if (o.getAllIndexCodePath().indexOf(ProjectConstant.XIA_JI) > -1) {
selfSubParentMap.get(ProjectConstant.XIA_JI).add(o.getIndexCode()); selfSubParentMap.get(ProjectConstant.XIA_JI).add(o.getIndexCode());
weightMap.put(ProjectConstant.XIA_JI, weightMap.getOrDefault(ProjectConstant.XIA_JI, new BigDecimal(0)).add(o.getWeight()));
} else { } else {
selfSubParentMap.get(ProjectConstant.ZI_SHEN).add(o.getIndexCode()); selfSubParentMap.get(ProjectConstant.ZI_SHEN).add(o.getIndexCode());
weightMap.put(ProjectConstant.ZI_SHEN, weightMap.getOrDefault(ProjectConstant.ZI_SHEN, new BigDecimal(0)).add(o.getWeight()));
} }
}); });
Map<String, DeptSelfSubScoreEntity> insertMap = new HashMap<>(); Map<String, DeptSelfSubScoreEntity> insertMap = new HashMap<>();
@ -158,16 +161,16 @@ public class DeptScoreServiceImpl extends BaseServiceImpl<DeptScoreDao, DeptScor
scoreEntity.setSelfScore(new BigDecimal(0)); scoreEntity.setSelfScore(new BigDecimal(0));
scoreEntity.setSubScore(new BigDecimal(0)); scoreEntity.setSubScore(new BigDecimal(0));
scoreEntity.setParentIndexCode(index.getIndexCode()); scoreEntity.setParentIndexCode(index.getIndexCode());
scoreEntity.setSelfWeight(new BigDecimal(0)); scoreEntity.setSelfWeight(weightMap.getOrDefault(score.getIndexCode(), new BigDecimal(0)));
scoreEntity.setSubWeight(new BigDecimal(0)); scoreEntity.setSubWeight(weightMap.getOrDefault(score.getIndexCode(), new BigDecimal(0)));
} }
BigDecimal partScore = score.getScore().multiply(score.getWeight()); BigDecimal partScore = score.getScore().multiply(score.getWeight());
if (selfSubParentMap.get(ProjectConstant.XIA_JI).contains(score.getIndexCode())) { if (selfSubParentMap.get(ProjectConstant.XIA_JI).contains(score.getIndexCode())) {
scoreEntity.setSubScore(scoreEntity.getSubScore().add(partScore)); scoreEntity.setSubScore(scoreEntity.getSubScore().add(partScore));
scoreEntity.setSubWeight(scoreEntity.getSubWeight().add(score.getWeight())); scoreEntity.setSelfWeight(weightMap.get(ProjectConstant.XIA_JI));
} else { } else {
scoreEntity.setSelfScore(scoreEntity.getSelfScore().add(partScore)); scoreEntity.setSelfScore(scoreEntity.getSelfScore().add(partScore));
scoreEntity.setSelfWeight(scoreEntity.getSelfWeight().add(score.getWeight())); scoreEntity.setSelfWeight(weightMap.get(ProjectConstant.ZI_SHEN));
} }
log.debug("=====key" + key + ",grid:{},originScore:{},weight:{},finalScore:{},total", score.getAgencyId(), score.getScore(), score.getWeight(), partScore); log.debug("=====key" + key + ",grid:{},originScore:{},weight:{},finalScore:{},total", score.getAgencyId(), score.getScore(), score.getWeight(), partScore);
}); });

9
epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/indexcal/impl/GridCorreLationServiceImpl.java

@ -129,12 +129,15 @@ public class GridCorreLationServiceImpl implements GridCorreLationService {
Map<String, Set<String>> selfSubParentMap = new HashMap<>(); Map<String, Set<String>> selfSubParentMap = new HashMap<>();
selfSubParentMap.put(ProjectConstant.ZI_SHEN, new HashSet<>()); selfSubParentMap.put(ProjectConstant.ZI_SHEN, new HashSet<>());
selfSubParentMap.put(ProjectConstant.XIA_JI, new HashSet<>()); selfSubParentMap.put(ProjectConstant.XIA_JI, new HashSet<>());
Map<String, BigDecimal> weightMap = new HashMap<>();
selfSubIndexList.forEach(o -> { selfSubIndexList.forEach(o -> {
//找出自身 和下级的指标 //找出自身 和下级的指标
if (o.getAllIndexCodePath().indexOf(ProjectConstant.XIA_JI) > -1) { if (o.getAllIndexCodePath().indexOf(ProjectConstant.XIA_JI) > -1) {
selfSubParentMap.get(ProjectConstant.XIA_JI).add(o.getIndexCode()); selfSubParentMap.get(ProjectConstant.XIA_JI).add(o.getIndexCode());
weightMap.put(ProjectConstant.XIA_JI, weightMap.getOrDefault(ProjectConstant.XIA_JI, new BigDecimal(0)).add(o.getWeight()));
} else { } else {
selfSubParentMap.get(ProjectConstant.ZI_SHEN).add(o.getIndexCode()); selfSubParentMap.get(ProjectConstant.ZI_SHEN).add(o.getIndexCode());
weightMap.put(ProjectConstant.ZI_SHEN, weightMap.getOrDefault(ProjectConstant.ZI_SHEN, new BigDecimal(0)).add(o.getWeight()));
} }
}); });
Map<String, GridSelfSubScoreEntity> insertMap = new HashMap<>(); Map<String, GridSelfSubScoreEntity> insertMap = new HashMap<>();
@ -148,16 +151,14 @@ public class GridCorreLationServiceImpl implements GridCorreLationService {
scoreEntity.setSelfScore(new BigDecimal(0)); scoreEntity.setSelfScore(new BigDecimal(0));
scoreEntity.setSubScore(new BigDecimal(0)); scoreEntity.setSubScore(new BigDecimal(0));
scoreEntity.setParentIndexCode(index.getIndexCode()); scoreEntity.setParentIndexCode(index.getIndexCode());
scoreEntity.setSelfWeight(new BigDecimal(0)); scoreEntity.setSubWeight(weightMap.getOrDefault(ProjectConstant.XIA_JI, new BigDecimal(0)));
scoreEntity.setSubWeight(new BigDecimal(0)); scoreEntity.setSelfWeight(weightMap.getOrDefault(ProjectConstant.ZI_SHEN, new BigDecimal(0)));
} }
BigDecimal partScore = score.getScore().multiply(score.getWeight()); BigDecimal partScore = score.getScore().multiply(score.getWeight());
if (selfSubParentMap.get(ProjectConstant.XIA_JI).contains(score.getIndexCode())) { if (selfSubParentMap.get(ProjectConstant.XIA_JI).contains(score.getIndexCode())) {
scoreEntity.setSubScore(scoreEntity.getSubScore().add(partScore)); scoreEntity.setSubScore(scoreEntity.getSubScore().add(partScore));
scoreEntity.setSubWeight(scoreEntity.getSubWeight().add(score.getWeight()));
} else { } else {
scoreEntity.setSelfScore(scoreEntity.getSelfScore().add(partScore)); scoreEntity.setSelfScore(scoreEntity.getSelfScore().add(partScore));
scoreEntity.setSelfWeight(scoreEntity.getSelfWeight().add(score.getWeight()));
} }
log.debug("=====key" + key + ",grid:{},originScore:{},weight:{},finalScore:{},total", score.getGridId(), score.getScore(), score.getWeight(), partScore); log.debug("=====key" + key + ",grid:{},originScore:{},weight:{},finalScore:{},total", score.getGridId(), score.getScore(), score.getWeight(), partScore);
}); });

10
epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/indexcal/impl/IndexCalculateCommunityServiceImpl.java

@ -138,12 +138,15 @@ public class IndexCalculateCommunityServiceImpl implements IndexCalculateCommuni
Map<String, Set<String>> selfSubParentMap = new HashMap<>(); Map<String, Set<String>> selfSubParentMap = new HashMap<>();
selfSubParentMap.put(ProjectConstant.ZI_SHEN, new HashSet<>()); selfSubParentMap.put(ProjectConstant.ZI_SHEN, new HashSet<>());
selfSubParentMap.put(ProjectConstant.XIA_JI, new HashSet<>()); selfSubParentMap.put(ProjectConstant.XIA_JI, new HashSet<>());
Map<String, BigDecimal> weightMap = new HashMap<>();
selfSubIndexList.forEach(o -> { selfSubIndexList.forEach(o -> {
//找出自身 和下级的指标 //找出自身 和下级的指标
if (o.getAllIndexCodePath().indexOf(ProjectConstant.XIA_JI) > -1) { if (o.getAllIndexCodePath().indexOf(ProjectConstant.XIA_JI) > -1) {
selfSubParentMap.get(ProjectConstant.XIA_JI).add(o.getIndexCode()); selfSubParentMap.get(ProjectConstant.XIA_JI).add(o.getIndexCode());
weightMap.put(ProjectConstant.XIA_JI, weightMap.getOrDefault(ProjectConstant.XIA_JI, new BigDecimal(0)).add(o.getWeight()));
} else { } else {
selfSubParentMap.get(ProjectConstant.ZI_SHEN).add(o.getIndexCode()); selfSubParentMap.get(ProjectConstant.ZI_SHEN).add(o.getIndexCode());
weightMap.put(ProjectConstant.ZI_SHEN, weightMap.getOrDefault(ProjectConstant.ZI_SHEN, new BigDecimal(0)).add(o.getWeight()));
} }
}); });
Map<String, CommunitySelfSubScoreEntity> insertMap = new HashMap<>(); Map<String, CommunitySelfSubScoreEntity> insertMap = new HashMap<>();
@ -156,16 +159,15 @@ public class IndexCalculateCommunityServiceImpl implements IndexCalculateCommuni
scoreEntity.setSelfScore(new BigDecimal(0)); scoreEntity.setSelfScore(new BigDecimal(0));
scoreEntity.setSubScore(new BigDecimal(0)); scoreEntity.setSubScore(new BigDecimal(0));
scoreEntity.setParentIndexCode(index.getIndexCode()); scoreEntity.setParentIndexCode(index.getIndexCode());
scoreEntity.setSelfWeight(new BigDecimal(0)); scoreEntity.setSubWeight(weightMap.get(ProjectConstant.XIA_JI));
scoreEntity.setSubWeight(new BigDecimal(0)); scoreEntity.setSelfWeight(weightMap.get(ProjectConstant.ZI_SHEN));
} }
BigDecimal partScore = score.getScore().multiply(score.getWeight()); BigDecimal partScore = score.getScore().multiply(score.getWeight());
if (selfSubParentMap.get(ProjectConstant.XIA_JI).contains(score.getIndexCode())) { if (selfSubParentMap.get(ProjectConstant.XIA_JI).contains(score.getIndexCode())) {
scoreEntity.setSubScore(scoreEntity.getSubScore().add(partScore)); scoreEntity.setSubScore(scoreEntity.getSubScore().add(partScore));
scoreEntity.setSubWeight(scoreEntity.getSubWeight().add(score.getWeight()));
} else { } else {
scoreEntity.setSelfScore(scoreEntity.getSelfScore().add(partScore)); scoreEntity.setSelfScore(scoreEntity.getSelfScore().add(partScore));
scoreEntity.setSelfWeight(scoreEntity.getSelfWeight().add(score.getWeight()));
} }
log.debug("=====key" + key + ",grid:{},originScore:{},weight:{},finalScore:{},total", score.getAgencyId(), score.getScore(), score.getWeight(), partScore); log.debug("=====key" + key + ",grid:{},originScore:{},weight:{},finalScore:{},total", score.getAgencyId(), score.getScore(), score.getWeight(), partScore);
}); });

9
epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/indexcal/impl/IndexCalculateDistrictServiceImpl.java

@ -136,12 +136,15 @@ public class IndexCalculateDistrictServiceImpl implements IndexCalculateDistrict
Map<String, Set<String>> selfSubParentMap = new HashMap<>(); Map<String, Set<String>> selfSubParentMap = new HashMap<>();
selfSubParentMap.put(ProjectConstant.ZI_SHEN, new HashSet<>()); selfSubParentMap.put(ProjectConstant.ZI_SHEN, new HashSet<>());
selfSubParentMap.put(ProjectConstant.XIA_JI, new HashSet<>()); selfSubParentMap.put(ProjectConstant.XIA_JI, new HashSet<>());
Map<String, BigDecimal> weightMap = new HashMap<>();
selfSubIndexList.forEach(o -> { selfSubIndexList.forEach(o -> {
//找出自身 和下级的指标 //找出自身 和下级的指标
if (o.getAllIndexCodePath().indexOf(ProjectConstant.XIA_JI) > -1) { if (o.getAllIndexCodePath().indexOf(ProjectConstant.XIA_JI) > -1) {
selfSubParentMap.get(ProjectConstant.XIA_JI).add(o.getIndexCode()); selfSubParentMap.get(ProjectConstant.XIA_JI).add(o.getIndexCode());
weightMap.put(ProjectConstant.XIA_JI, weightMap.getOrDefault(ProjectConstant.XIA_JI, new BigDecimal(0)).add(o.getWeight()));
} else { } else {
selfSubParentMap.get(ProjectConstant.ZI_SHEN).add(o.getIndexCode()); selfSubParentMap.get(ProjectConstant.ZI_SHEN).add(o.getIndexCode());
weightMap.put(ProjectConstant.ZI_SHEN, weightMap.getOrDefault(ProjectConstant.ZI_SHEN, new BigDecimal(0)).add(o.getWeight()));
} }
}); });
Map<String, AgencySelfSubScoreEntity> insertMap = new HashMap<>(); Map<String, AgencySelfSubScoreEntity> insertMap = new HashMap<>();
@ -154,16 +157,14 @@ public class IndexCalculateDistrictServiceImpl implements IndexCalculateDistrict
scoreEntity.setSelfScore(new BigDecimal(0)); scoreEntity.setSelfScore(new BigDecimal(0));
scoreEntity.setSubScore(new BigDecimal(0)); scoreEntity.setSubScore(new BigDecimal(0));
scoreEntity.setParentIndexCode(index.getIndexCode()); scoreEntity.setParentIndexCode(index.getIndexCode());
scoreEntity.setSelfWeight(new BigDecimal(0)); scoreEntity.setSubWeight(weightMap.getOrDefault(ProjectConstant.XIA_JI, new BigDecimal(0)));
scoreEntity.setSubWeight(new BigDecimal(0)); scoreEntity.setSelfWeight(weightMap.getOrDefault(ProjectConstant.ZI_SHEN, new BigDecimal(0)));
} }
BigDecimal partScore = score.getScore().multiply(score.getWeight()); BigDecimal partScore = score.getScore().multiply(score.getWeight());
if (selfSubParentMap.get(ProjectConstant.XIA_JI).contains(score.getIndexCode())) { if (selfSubParentMap.get(ProjectConstant.XIA_JI).contains(score.getIndexCode())) {
scoreEntity.setSubScore(scoreEntity.getSubScore().add(partScore)); scoreEntity.setSubScore(scoreEntity.getSubScore().add(partScore));
scoreEntity.setSubWeight(scoreEntity.getSubWeight().add(score.getWeight()));
} else { } else {
scoreEntity.setSelfScore(scoreEntity.getSelfScore().add(partScore)); scoreEntity.setSelfScore(scoreEntity.getSelfScore().add(partScore));
scoreEntity.setSelfWeight(scoreEntity.getSelfWeight().add(score.getWeight()));
} }
log.debug("=====key" + key + ",grid:{},originScore:{},weight:{},finalScore:{},total", score.getAgencyId(), score.getScore(), score.getWeight(), partScore); log.debug("=====key" + key + ",grid:{},originScore:{},weight:{},finalScore:{},total", score.getAgencyId(), score.getScore(), score.getWeight(), partScore);
}); });

10
epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/indexcal/impl/IndexCalculateStreetServiceImpl.java

@ -139,13 +139,17 @@ public class IndexCalculateStreetServiceImpl implements IndexCalculateStreetServ
Map<String, Set<String>> selfSubParentMap = new HashMap<>(); Map<String, Set<String>> selfSubParentMap = new HashMap<>();
selfSubParentMap.put(ProjectConstant.ZI_SHEN, new HashSet<>()); selfSubParentMap.put(ProjectConstant.ZI_SHEN, new HashSet<>());
selfSubParentMap.put(ProjectConstant.XIA_JI, new HashSet<>()); selfSubParentMap.put(ProjectConstant.XIA_JI, new HashSet<>());
Map<String, BigDecimal> weightMap = new HashMap<>();
selfSubIndexList.forEach(o -> { selfSubIndexList.forEach(o -> {
//找出自身 和下级的指标 //找出自身 和下级的指标
if (o.getAllIndexCodePath().indexOf(ProjectConstant.XIA_JI) > -1) { if (o.getAllIndexCodePath().indexOf(ProjectConstant.XIA_JI) > -1) {
selfSubParentMap.get(ProjectConstant.XIA_JI).add(o.getIndexCode()); selfSubParentMap.get(ProjectConstant.XIA_JI).add(o.getIndexCode());
weightMap.put(ProjectConstant.XIA_JI, weightMap.getOrDefault(ProjectConstant.XIA_JI, new BigDecimal(0)).add(o.getWeight()));
} else { } else {
selfSubParentMap.get(ProjectConstant.ZI_SHEN).add(o.getIndexCode()); selfSubParentMap.get(ProjectConstant.ZI_SHEN).add(o.getIndexCode());
weightMap.put(ProjectConstant.ZI_SHEN, weightMap.getOrDefault(ProjectConstant.ZI_SHEN, new BigDecimal(0)).add(o.getWeight()));
} }
}); });
Map<String, AgencySelfSubScoreEntity> insertMap = new HashMap<>(); Map<String, AgencySelfSubScoreEntity> insertMap = new HashMap<>();
subScore.forEach(score -> { subScore.forEach(score -> {
@ -157,16 +161,14 @@ public class IndexCalculateStreetServiceImpl implements IndexCalculateStreetServ
scoreEntity.setSelfScore(new BigDecimal(0)); scoreEntity.setSelfScore(new BigDecimal(0));
scoreEntity.setSubScore(new BigDecimal(0)); scoreEntity.setSubScore(new BigDecimal(0));
scoreEntity.setParentIndexCode(index.getIndexCode()); scoreEntity.setParentIndexCode(index.getIndexCode());
scoreEntity.setSelfWeight(new BigDecimal(0)); scoreEntity.setSubWeight(weightMap.getOrDefault(ProjectConstant.XIA_JI, new BigDecimal(0)));
scoreEntity.setSubWeight(new BigDecimal(0)); scoreEntity.setSelfWeight(weightMap.getOrDefault(ProjectConstant.ZI_SHEN, new BigDecimal(0)));
} }
BigDecimal partScore = score.getScore().multiply(score.getWeight()); BigDecimal partScore = score.getScore().multiply(score.getWeight());
if (selfSubParentMap.get(ProjectConstant.XIA_JI).contains(score.getIndexCode())) { if (selfSubParentMap.get(ProjectConstant.XIA_JI).contains(score.getIndexCode())) {
scoreEntity.setSubScore(scoreEntity.getSubScore().add(partScore)); scoreEntity.setSubScore(scoreEntity.getSubScore().add(partScore));
scoreEntity.setSubWeight(scoreEntity.getSubWeight().add(score.getWeight()));
} else { } else {
scoreEntity.setSelfScore(scoreEntity.getSelfScore().add(partScore)); scoreEntity.setSelfScore(scoreEntity.getSelfScore().add(partScore));
scoreEntity.setSelfWeight(scoreEntity.getSelfWeight().add(score.getWeight()));
} }
log.debug("=====key" + key + ",grid:{},originScore:{},weight:{},finalScore:{},total", score.getAgencyId(), score.getScore(), score.getWeight(), partScore); log.debug("=====key" + key + ",grid:{},originScore:{},weight:{},finalScore:{},total", score.getAgencyId(), score.getScore(), score.getWeight(), partScore);
}); });

81
epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/indexcoll/impl/FactIndexCollectServiceImpl.java

@ -3,6 +3,7 @@ package com.epmet.service.evaluationindex.indexcoll.impl;
import com.epmet.commons.dynamic.datasource.annotation.DataSource; import com.epmet.commons.dynamic.datasource.annotation.DataSource;
import com.epmet.commons.tools.constant.NumConstant; import com.epmet.commons.tools.constant.NumConstant;
import com.epmet.commons.tools.constant.StrConstant; import com.epmet.commons.tools.constant.StrConstant;
import com.epmet.commons.tools.redis.RedisKeys;
import com.epmet.commons.tools.utils.ConvertUtils; import com.epmet.commons.tools.utils.ConvertUtils;
import com.epmet.constant.DataSourceConstant; import com.epmet.constant.DataSourceConstant;
import com.epmet.constant.OrgTypeConstant; import com.epmet.constant.OrgTypeConstant;
@ -27,6 +28,8 @@ import com.epmet.service.evaluationindex.indexcoll.FactIndexCollectService;
import com.google.common.cache.Cache; import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheBuilder;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.redisson.api.RLock;
import org.redisson.api.RedissonClient;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
@ -38,6 +41,7 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors; import java.util.stream.Collectors;
/** /**
@ -84,6 +88,8 @@ public class FactIndexCollectServiceImpl implements FactIndexCollectService {
private ScreenCustomerDeptDao screenCustomerDeptDao; private ScreenCustomerDeptDao screenCustomerDeptDao;
@Autowired @Autowired
private ScreenCustomerAgencyDao screenCustomerAgencyDao; private ScreenCustomerAgencyDao screenCustomerAgencyDao;
@Autowired
private RedissonClient redissonClient;
@Override @Override
@Transactional(rollbackFor = Exception.class) @Transactional(rollbackFor = Exception.class)
@ -198,35 +204,60 @@ public class FactIndexCollectServiceImpl implements FactIndexCollectService {
} }
} }
/**
* @Description 目前只支持到区县级别区县级以上的未实现
* @Param monthId
* @Param customerId
* @author zxc
* @date 2020/10/20 2:59 下午
*/
@Override @Override
@Transactional(rollbackFor = Exception.class)
public void insertScreenIndexDataMonthlyAndYearly(String monthId, String customerId) { public void insertScreenIndexDataMonthlyAndYearly(String monthId, String customerId) {
if (NumConstant.SIX != monthId.length()){ if (NumConstant.SIX != monthId.length()) {
throw new RuntimeException("入参monthId格式不正确:monthId =" + monthId); throw new RuntimeException("入参monthId格式不正确:monthId =" + monthId);
} }
allParentIds.cleanUp(); RLock lock = redissonClient.getLock(RedisKeys.getScreenIndexDataLockKey());
// 根据网格类型,删除 指数-指数数据(每月数值)表中 匹配的数据 try {
this.batchDelIndexDataMonthly(customerId, monthId, OrgTypeConstant.GRID); AtomicInteger tryTimes = new AtomicInteger(0);
// 开始处理 网格相关分值表 boolean lockFlag = false;
this.startHandleIndexGridScore(monthId, customerId); do {
lockFlag = lock.tryLock(1, 10, TimeUnit.MINUTES);
// 根据组织类型,删除 指数-指数数据(每月数值)表中 匹配的数据 } while (!lockFlag && tryTimes.addAndGet(1) < 3);
this.batchDelIndexDataMonthly(customerId, monthId, OrgTypeConstant.AGENCY); if (!lockFlag) {
// 开始处理 社区相关分值表 log.error("插入大屏指数方法获取锁失败");
this.startHandleIndexCommunityScore(monthId, customerId); return;
}
// 根据部门类型,删除 指数-指数数据(每月数值)表中 匹配的数据 allParentIds.cleanUp();
this.batchDelIndexDataMonthly(customerId, monthId, OrgTypeConstant.DEPARTMENT); // 根据网格类型,删除 指数-指数数据(每月数值)表中 匹配的数据
// 开始处理 区直部门分值表 this.batchDelIndexDataMonthly(customerId, monthId, OrgTypeConstant.GRID);
this.startHandleIndexDeptScore(monthId, customerId); // 开始处理 网格相关分值表
this.startHandleIndexGridScore(monthId, customerId);
// 同样是 组织类型,所以不再重复删除了
// 开始处理 区/街道相关分数表 // 根据组织类型,删除 指数-指数数据(每月数值)表中 匹配的数据
this.startHandleIndexAgencyScore(monthId, customerId); this.batchDelIndexDataMonthly(customerId, monthId, OrgTypeConstant.AGENCY);
// 开始处理 社区相关分值表
// 根据年,汇总今年各项得到,计算平均值后 插入年表 screen_index_data_yearly this.startHandleIndexCommunityScore(monthId, customerId);
this.insertIndexDataYear(monthId, customerId);
allParentIds.invalidateAll(); // 根据部门类型,删除 指数-指数数据(每月数值)表中 匹配的数据
this.batchDelIndexDataMonthly(customerId, monthId, OrgTypeConstant.DEPARTMENT);
// 开始处理 区直部门分值表
this.startHandleIndexDeptScore(monthId, customerId);
// 同样是 组织类型,所以不再重复删除了
// 开始处理 区/街道相关分数表
this.startHandleIndexAgencyScore(monthId, customerId);
// 根据年,汇总今年各项得到,计算平均值后 插入年表 screen_index_data_yearly
this.insertIndexDataYear(monthId, customerId);
allParentIds.invalidateAll();
} catch (InterruptedException inter) {
log.error("插入计算指标数据异常", inter);
} finally {
//判断是否是当前线程 持有锁
if (lock.isHeldByCurrentThread()) {
lock.unlock();
}
}
} }
/** /**

2
epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/support/normalizing/ScoreConstants.java

@ -3,7 +3,7 @@ package com.epmet.support.normalizing;
import java.math.BigDecimal; import java.math.BigDecimal;
/** /**
* 分值常量 * 分值常量 归一区间
*/ */
public class ScoreConstants { public class ScoreConstants {

2
epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/support/normalizing/batch/BatchScoreCalculator.java

@ -1,6 +1,7 @@
package com.epmet.support.normalizing.batch; package com.epmet.support.normalizing.batch;
import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSON;
import com.epmet.commons.tools.constant.NumConstant;
import com.epmet.support.normalizing.BigDecimalScoreCalculator; import com.epmet.support.normalizing.BigDecimalScoreCalculator;
import com.epmet.support.normalizing.Correlation; import com.epmet.support.normalizing.Correlation;
import com.epmet.support.normalizing.ScoreCalculator; import com.epmet.support.normalizing.ScoreCalculator;
@ -79,6 +80,7 @@ public class BatchScoreCalculator {
}else { }else {
//如果不需要归一 则 直接value*权重 //如果不需要归一 则 直接value*权重
if (idx.isScore()) { if (idx.isScore()) {
vo.setSampleValue(new BigDecimal(vo.getSampleValue().toString()).compareTo(NumConstant.ZERO_DECIMAL) == NumConstant.ZERO ? ScoreConstants.MIN_SCORE : new BigDecimal(vo.getSampleValue().toString()));
normalizeValue = getFinalSampleValue(vo.getSampleValue(), threshold); normalizeValue = getFinalSampleValue(vo.getSampleValue(), threshold);
} else { } else {
normalizeValue = scoreCalculator.normalize(getFinalSampleValue(vo.getSampleValue(), threshold)); normalizeValue = scoreCalculator.normalize(getFinalSampleValue(vo.getSampleValue(), threshold));

10
epmet-module/data-statistical/data-statistical-server/src/main/resources/logback-spring.xml

@ -149,6 +149,16 @@
<appender-ref ref="WARN_FILE"/> <appender-ref ref="WARN_FILE"/>
<appender-ref ref="ERROR_FILE"/> <appender-ref ref="ERROR_FILE"/>
</root> </root>
<!-- 屏蔽logger -->
<logger name="org.springframework" level="INFO"/>
<logger name="org.apache" level="WARN"/>
<logger name="org.mybatis.spring" level="WARN"/>
<logger name="io.netty" level="WARN"/>
<logger name="org.redisson.connection" level="WARN"/>
<logger name="org.hibernate.validator" level="WARN"/>
<!-- 屏蔽jdk日志 -->
<logger name="java" level="WARN"/>
<!-- 屏蔽logger 结束-->
</springProfile> </springProfile>
<!-- 生产环境 --> <!-- 生产环境 -->

61
epmet-module/data-statistical/data-statistical-server/src/test/java/com/epmet/stats/test/RedissonTest.java

@ -0,0 +1,61 @@
package com.epmet.stats.test;
import com.epmet.DataStatsApplication;
import lombok.extern.slf4j.Slf4j;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.redisson.api.RLock;
import org.redisson.api.RedissonClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import java.util.concurrent.TimeUnit;
/**
* descredisson测试类
*/
@Slf4j
@RunWith(value = SpringRunner.class)
@SpringBootTest(classes = {DataStatsApplication.class})
public class RedissonTest {
@Autowired
private RedissonClient redissonClient;
@Test
public void lockTest() {
//获取一个名为 lockName 的锁实例
RLock lock = redissonClient.getLock("lockName");
try {
// 尝试加锁(推荐使用)
// 参数1 等待时长 waitTime:5 等待时长是5秒 如果5秒内还获取不到锁 则返回false,
// 参数2 持有时长 leaseTime:5 持有锁时长超过5秒 就释放锁 此时如果继续lock.unlock()会抛出异常
// 参数3 时间单位
boolean bs = lock.tryLock(5, 6, TimeUnit.SECONDS);
//如果获取不到锁 会一直阻塞到着 直至获取到锁 不推荐使用
//lock.lock();
//异步方式
/* RFuture<Void> future = lock.lockAsync();
if (future.isSuccess()){
//todo something
}*/
if (bs) {
// 业务代码
System.out.println("进入业务代码: " + 123);
} else {
Thread.sleep(300);
}
} catch (Exception e) {
log.error("lockTest exception", e);
} finally {
//判断是否是当前线程 持有锁
if (lock.isHeldByCurrentThread()) {
lock.unlock();
}
}
}
}

7
epmet-module/data-statistical/data-statistical-server/src/test/java/resources/数据值略小时可能的情况.txt

@ -0,0 +1,7 @@
1. 当数据偏小时,可能社区下没有网格,例如 网格发文数量得分就为0;
2. screen_index_data_monthly中的 数据条数 与客户下(组织数 + 直属部门数 + 网格数)的和不统一,
原因:数据插入的时候,只算到了区县级,区县级以上的没有计算,少了 市级和省级agency
文档地址:
1. 大屏指标项文档说明:https://www.kdocs.cn/view/l/svtSfaUyzNYZ?f=130
2. 大屏或手机端:https://www.kdocs.cn/view/l/suilmk0Ziss1?f=130
Loading…
Cancel
Save