luyan
/
epmet-cloud-rizhao0920
forked from rongchao/epmet-cloud-rizhao
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Merge remote-tracking branch 'origin/develop' into release_temp

dev
zxc 4 years ago
parent
f380835ccd 5d27d6cf53
commit
dc2f01a126
6 changed files with 151 additions and 131 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/constant/NumConstant.java
  2. 2
      epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/distributedlock/LockConstants.java
  3. 2
      epmet-module/data-report/data-report-server/src/main/java/com/epmet/datareport/dao/evaluationindex/screen/ScreenProjectDataDao.java
  4. 6
      epmet-module/data-report/data-report-server/src/main/java/com/epmet/datareport/service/evaluationindex/screen/impl/ScreenProjectServiceImpl.java
  5. 1
      epmet-module/data-report/data-report-server/src/main/resources/mapper/screen/ScreenProjectDataDao.xml
  6. 269
      epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenExtractServiceImpl.java

2
epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/constant/NumConstant.java
View File

@ -64,6 +64,8 @@ public interface NumConstant {
long THREE_L = 3L; long THREE_L = 3L;
long FOUR_L = 4L; long FOUR_L = 4L;
long MINUS_ONE_L = -1L; long MINUS_ONE_L = -1L;
long TEN_L = 10L;
long SIX_HUNDRED_L = 600L;
String ZERO_STR = "0"; String ZERO_STR = "0";
String ONE_STR = "1"; String ONE_STR = "1";

2
epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/distributedlock/LockConstants.java
View File

@ -9,4 +9,6 @@ public interface LockConstants {
String TEST_LOCK_NAME = "testLock"; String TEST_LOCK_NAME = "testLock";
String STATS_LOCK_NAME = "stats"; String STATS_LOCK_NAME = "stats";
String SCREEN_DAILY = "screenDaily";
} }

2
epmet-module/data-report/data-report-server/src/main/java/com/epmet/datareport/dao/evaluationindex/screen/ScreenProjectDataDao.java
View File

@ -54,7 +54,7 @@ public interface ScreenProjectDataDao {
* @author zxc * @author zxc
* @date 2021/11/5 1:40 下午 * @date 2021/11/5 1:40 下午
*/ */
List<CategoryProjectListResultDTO> selectCategoryProjectList(@Param("customerId") String customerId,@Param("orgId") String orgId, List<CategoryProjectListResultDTO> selectCategoryProjectList(@Param("orgId") String orgId,
@Param("categoryCode")String categoryCode,@Param("status")String status, @Param("categoryCode")String categoryCode,@Param("status")String status,
@Param("startTime")String startTime,@Param("endTime")String endTime); @Param("startTime")String startTime,@Param("endTime")String endTime);

6
epmet-module/data-report/data-report-server/src/main/java/com/epmet/datareport/service/evaluationindex/screen/impl/ScreenProjectServiceImpl.java
View File

@ -11,6 +11,7 @@ import com.epmet.commons.tools.redis.common.bean.AgencyInfoCache;
import com.epmet.commons.tools.redis.common.bean.GridInfoCache; import com.epmet.commons.tools.redis.common.bean.GridInfoCache;
import com.epmet.commons.tools.security.dto.TokenDto; import com.epmet.commons.tools.security.dto.TokenDto;
import com.epmet.commons.tools.utils.ConvertUtils; import com.epmet.commons.tools.utils.ConvertUtils;
import com.epmet.commons.tools.utils.DateUtils;
import com.epmet.commons.tools.utils.Result; import com.epmet.commons.tools.utils.Result;
import com.epmet.constant.DataSourceConstant; import com.epmet.constant.DataSourceConstant;
import com.epmet.datareport.constant.FactConstant; import com.epmet.datareport.constant.FactConstant;
@ -253,6 +254,9 @@ public class ScreenProjectServiceImpl implements ScreenProjectService {
*/ */
@Override @Override
public List<ProjectCategoryResultDTO> selectProjectCategory(ProjectCategoryFormDTO formDTO, TokenDto tokenDto) { public List<ProjectCategoryResultDTO> selectProjectCategory(ProjectCategoryFormDTO formDTO, TokenDto tokenDto) {
if (StringUtils.isNotBlank(formDTO.getStartTime())){
formDTO.setStartTime(DateUtils.getBeforeNDay(formDTO.getStartTime(),NumConstant.ONE));
}
if (StringUtils.isBlank(formDTO.getOrgId())){ if (StringUtils.isBlank(formDTO.getOrgId())){
CustomerStaffInfoCacheResult staffInfo = CustomerStaffRedis.getStaffInfo(tokenDto.getCustomerId(), tokenDto.getUserId()); CustomerStaffInfoCacheResult staffInfo = CustomerStaffRedis.getStaffInfo(tokenDto.getCustomerId(), tokenDto.getUserId());
if (null == staffInfo){ if (null == staffInfo){
@ -382,7 +386,7 @@ public class ScreenProjectServiceImpl implements ScreenProjectService {
PageCategoryProjectListResultDTO result = new PageCategoryProjectListResultDTO(); PageCategoryProjectListResultDTO result = new PageCategoryProjectListResultDTO();
List<CategoryProjectListResultDTO> list; List<CategoryProjectListResultDTO> list;
PageInfo<CategoryProjectListResultDTO> objectPageInfo = PageHelper.startPage(formDTO.getPageNo(), formDTO.getPageSize(), formDTO.getIsPage()) PageInfo<CategoryProjectListResultDTO> objectPageInfo = PageHelper.startPage(formDTO.getPageNo(), formDTO.getPageSize(), formDTO.getIsPage())
.doSelectPageInfo(() -> screenProjectDataDao.selectCategoryProjectList(tokenDto.getCustomerId(), formDTO.getOrgId(), formDTO.getCategoryCode(), formDTO.getStatus(),formDTO.getStartTime(),formDTO.getEndTime())); .doSelectPageInfo(() -> screenProjectDataDao.selectCategoryProjectList(formDTO.getOrgId(), formDTO.getCategoryCode(), formDTO.getStatus(),formDTO.getStartTime(),formDTO.getEndTime()));
list = objectPageInfo.getList(); list = objectPageInfo.getList();
result.setTotal(Integer.valueOf(String.valueOf(objectPageInfo.getTotal()))); result.setTotal(Integer.valueOf(String.valueOf(objectPageInfo.getTotal())));

1
epmet-module/data-report/data-report-server/src/main/resources/mapper/screen/ScreenProjectDataDao.xml
View File

@ -180,7 +180,6 @@
</if> </if>
AND CONCAT(pd.ALL_PARENT_IDS,':',pd.ORG_ID) LIKE CONCAT('%',#{orgId},'%') AND CONCAT(pd.ALL_PARENT_IDS,':',pd.ORG_ID) LIKE CONCAT('%',#{orgId},'%')
AND pc.CATEGORY_CODE LIKE CONCAT(#{categoryCode},'%') AND pc.CATEGORY_CODE LIKE CONCAT(#{categoryCode},'%')
AND pd.CUSTOMER_ID = #{customerId}
AND DATE_FORMAT(pd.PROJECT_CREATE_TIME,'%Y%m%d') <![CDATA[ <= ]]> #{endTime} AND DATE_FORMAT(pd.PROJECT_CREATE_TIME,'%Y%m%d') <![CDATA[ <= ]]> #{endTime}
<if test="null != startTime and startTime != '' "> <if test="null != startTime and startTime != '' ">
AND DATE_FORMAT(pd.PROJECT_CREATE_TIME,'%Y%m%d') >= #{startTime} AND DATE_FORMAT(pd.PROJECT_CREATE_TIME,'%Y%m%d') >= #{startTime}

269
epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenExtractServiceImpl.java
View File

@ -2,6 +2,8 @@ package com.epmet.service.evaluationindex.extract.toscreen.impl;
import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSON;
import com.epmet.commons.tools.constant.NumConstant; import com.epmet.commons.tools.constant.NumConstant;
import com.epmet.commons.tools.distributedlock.DistributedLock;
import com.epmet.commons.tools.distributedlock.LockConstants;
import com.epmet.commons.tools.enums.EnvEnum; import com.epmet.commons.tools.enums.EnvEnum;
import com.epmet.commons.tools.utils.DateUtils; import com.epmet.commons.tools.utils.DateUtils;
import com.epmet.commons.tools.utils.HttpClientManager; import com.epmet.commons.tools.utils.HttpClientManager;
@ -20,6 +22,7 @@ import com.epmet.service.stats.DimCustomerService;
import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.common.util.concurrent.ThreadFactoryBuilder;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.redisson.api.RLock;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
@ -86,6 +89,8 @@ public class ScreenExtractServiceImpl implements ScreenExtractService {
private FactGridMemberStatisticsDailyService factGridMemberStatisticsDailyService; private FactGridMemberStatisticsDailyService factGridMemberStatisticsDailyService;
@Autowired @Autowired
private ScreenProjectCategoryGridAndOrgDailyService screenProjectCategoryGridAndOrgDailyService; private ScreenProjectCategoryGridAndOrgDailyService screenProjectCategoryGridAndOrgDailyService;
@Autowired
private DistributedLock distributedLock;
/** /**
* @param extractOriginFormDTO * @param extractOriginFormDTO
@ -163,155 +168,163 @@ public class ScreenExtractServiceImpl implements ScreenExtractService {
* @date 2020/9/24 10:16 上午 * @date 2020/9/24 10:16 上午
*/ */
public void extractDaily(String customerId, String dateId, boolean isLast) { public void extractDaily(String customerId, String dateId, boolean isLast) {
//等待3个线程执行完毕后再 继续执行下一个客户的 避免死锁 RLock lock = null;
final CountDownLatch latch = new CountDownLatch(NumConstant.FOUR); try {
threadPool.submit(() -> { // 锁持有10分钟，等待10s
//党员基本情况screen_cpc_base_data lock = distributedLock.getLock(LockConstants.SCREEN_DAILY, NumConstant.SIX_HUNDRED_L, NumConstant.TEN_L, TimeUnit.SECONDS);
try { //等待3个线程执行完毕后再 继续执行下一个客户的 避免死锁
partyBaseInfoService.statsPartyMemberBaseInfoToScreen(customerId, dateId); final CountDownLatch latch = new CountDownLatch(NumConstant.FOUR);
} catch (Exception e) { threadPool.submit(() -> {
log.error("党员基本情况抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); //党员基本情况screen_cpc_base_data
}
//先锋模范screen_pioneer_data
if (isLast) {
try { try {
pioneerDataExtractService.extractGridPioneerData(customerId, dateId); partyBaseInfoService.statsPartyMemberBaseInfoToScreen(customerId, dateId);
} catch (Exception e) { } catch (Exception e) {
log.error("先锋模范【网格】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); log.error("党员基本情况抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e);
}
//先锋模范screen_pioneer_data
if (isLast) {
try {
pioneerDataExtractService.extractGridPioneerData(customerId, dateId);
} catch (Exception e) {
log.error("先锋模范【网格】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e);
}
try {
pioneerDataExtractService.extractCommunityPioneerData(customerId, dateId);
} catch (Exception e) {
log.error("先锋模范【社区】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e);
}
try {
pioneerDataExtractService.extractExceptCommunityPioneerData(customerId, dateId);
} catch (Exception e) {
log.error("先锋模范【extractExceptCommunityPioneerData】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e);
}
} }
latch.countDown();
log.info("extractDaily 1 thread run end ========= dateId:{},customerId:{}", dateId, customerId);
});
threadPool.submit(() -> {
//公众参与排行（注册人数、参与人数、话题数、议题数、项目数）screen_public_party_total_data
try { try {
pioneerDataExtractService.extractCommunityPioneerData(customerId, dateId); publicPartiTotalDataExtractService.extractPublicPartiTotalData(customerId, dateId);
} catch (Exception e) { } catch (Exception e) {
log.error("先锋模范【社区】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); log.error("公众参与排行抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e);
}
if (isLast) {
//基层治理- 难点赌点 screen_difficulty_data | screen_difficulty_img_data
try {
ScreenCentralZoneDataFormDTO param2 = new ScreenCentralZoneDataFormDTO();
param2.setCustomerId(customerId);
param2.setDateId(null);
log.info("【难点赌点数据上报开始------】 当前客户Id{}", param2.getCustomerId());
//screenGrassrootsGovernDataAbsorptionService.difficultyDataHub(param);
screenGrassrootsGovernDataAbsorptionService.difficultyDataExtract(param2);
log.info("【难点赌点数据上报结束------】 当前客户Id{}", param2.getCustomerId());
} catch (Exception e) {
log.error("基层治理-难点赌点抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e);
}
} }
latch.countDown();
log.info("extractDaily 2 thread run end ========= dateId:{},customerId:{}", dateId, customerId);
});
threadPool.submit(() -> {
ScreenCentralZoneDataFormDTO param = new ScreenCentralZoneDataFormDTO();
param.setCustomerId(customerId);
param.setDateId(dateId);
//中央区 screen_user_total_data
try { try {
pioneerDataExtractService.extractExceptCommunityPioneerData(customerId, dateId); screenCentralZoneDataAbsorptionService.centralZoneDataHub(param);
} catch (Exception e) { } catch (Exception e) {
log.error("先锋模范【extractExceptCommunityPioneerData】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); log.error("中央区抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e);
} }
}
latch.countDown();
log.info("extractDaily 1 thread run end ========= dateId:{},customerId:{}", dateId, customerId);
});
threadPool.submit(() -> {
//公众参与排行（注册人数、参与人数、话题数、议题数、项目数）screen_public_party_total_data
try {
publicPartiTotalDataExtractService.extractPublicPartiTotalData(customerId, dateId);
} catch (Exception e) {
log.error("公众参与排行抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e);
}
if (isLast) {
//基层治理- 难点赌点 screen_difficulty_data | screen_difficulty_img_data
try { try {
ScreenCentralZoneDataFormDTO param2 = new ScreenCentralZoneDataFormDTO(); // 项目(事件)分析按网格_按天统计
param2.setCustomerId(customerId); screenProjectGridDailyService.extractionProjectGridDaily(customerId, dateId);
param2.setDateId(null);
log.info("【难点赌点数据上报开始------】 当前客户Id{}", param2.getCustomerId());
//screenGrassrootsGovernDataAbsorptionService.difficultyDataHub(param);
screenGrassrootsGovernDataAbsorptionService.difficultyDataExtract(param2);
log.info("【难点赌点数据上报结束------】 当前客户Id{}", param2.getCustomerId());
} catch (Exception e) { } catch (Exception e) {
log.error("基层治理-难点赌点抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); log.error("项目(事件)分析按网格_按天统计失败，customerId为：" + customerId + "dateId为：" + dateId, e);
}
try {
// 项目(事件)分析按组织_按天统计
screenProjectOrgDailyService.extractionProjectOrgDaily(customerId, dateId);
} catch (Exception e) {
log.error("项目(事件)分析按组织_按天统计失败，customerId为：" + customerId + "dateId为：" + dateId, e);
} }
}
latch.countDown();
log.info("extractDaily 2 thread run end ========= dateId:{},customerId:{}", dateId, customerId);
});
threadPool.submit(() -> {
ScreenCentralZoneDataFormDTO param = new ScreenCentralZoneDataFormDTO();
param.setCustomerId(customerId);
param.setDateId(dateId);
//中央区 screen_user_total_data
try {
screenCentralZoneDataAbsorptionService.centralZoneDataHub(param);
} catch (Exception e) {
log.error("中央区抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e);
}
try {
// 项目(事件)分析按网格_按天统计
screenProjectGridDailyService.extractionProjectGridDaily(customerId, dateId);
} catch (Exception e) {
log.error("项目(事件)分析按网格_按天统计失败，customerId为：" + customerId + "dateId为：" + dateId, e);
}
try {
// 项目(事件)分析按组织_按天统计
screenProjectOrgDailyService.extractionProjectOrgDaily(customerId, dateId);
} catch (Exception e) {
log.error("项目(事件)分析按组织_按天统计失败，customerId为：" + customerId + "dateId为：" + dateId, e);
}
//按天统计：网格内各个分类下的项目总数 //按天统计：网格内各个分类下的项目总数
try { try {
screenProjectCategoryGridAndOrgDailyService.extractCategoryProjectGridData(customerId, dateId); screenProjectCategoryGridAndOrgDailyService.extractCategoryProjectGridData(customerId, dateId);
} catch (Exception e) { } catch (Exception e) {
log.error("按天统计：网格内各个分类下的项目总数，customerId为：" + customerId + "dateId为：" + dateId, e); log.error("按天统计：网格内各个分类下的项目总数，customerId为：" + customerId + "dateId为：" + dateId, e);
} }
// 按天统计：组织内各个分类下的项目总数 // 按天统计：组织内各个分类下的项目总数
try { try {
screenProjectCategoryGridAndOrgDailyService.extractCategoryProjectOrgData(customerId, dateId); screenProjectCategoryGridAndOrgDailyService.extractCategoryProjectOrgData(customerId, dateId);
} catch (Exception e) { } catch (Exception e) {
log.error("按天统计：组织内各个分类下的项目总数，customerId为：" + customerId + "dateId为：" + dateId, e); log.error("按天统计：组织内各个分类下的项目总数，customerId为：" + customerId + "dateId为：" + dateId, e);
} }
latch.countDown(); latch.countDown();
log.info("extractDaily 3 thread run end ========= dateId:{},customerId:{}", dateId, customerId); log.info("extractDaily 3 thread run end ========= dateId:{},customerId:{}", dateId, customerId);
}); });
threadPool.submit(() -> { threadPool.submit(() -> {
//治理能力排行screen_govern_rank_data //治理能力排行screen_govern_rank_data
try { try {
governRankDataExtractService.extractGridDataDaily(customerId, dateId); governRankDataExtractService.extractGridDataDaily(customerId, dateId);
} catch (Exception e) { } catch (Exception e) {
log.error("治理能力排行【网格】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); log.error("治理能力排行【网格】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e);
} }
try { try {
governRankDataExtractService.extractCommunityDataDaily(customerId, dateId); governRankDataExtractService.extractCommunityDataDaily(customerId, dateId);
} catch (Exception e) { } catch (Exception e) {
log.error("治理能力排行【社区】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); log.error("治理能力排行【社区】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e);
} }
try { try {
governRankDataExtractService.extractStreetDataDaily(customerId, dateId); governRankDataExtractService.extractStreetDataDaily(customerId, dateId);
} catch (Exception e) { } catch (Exception e) {
log.error("治理能力排行【街道】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); log.error("治理能力排行【街道】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e);
} }
try { try {
governRankDataExtractService.extractDistrictDataDaily(customerId, dateId); governRankDataExtractService.extractDistrictDataDaily(customerId, dateId);
} catch (Exception e) { } catch (Exception e) {
log.error("治理能力排行【全区】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); log.error("治理能力排行【全区】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e);
} }
try { try {
ExtractFactGridGovernDailyFromDTO extractFactGridGovernDailyFromDTO = new ExtractFactGridGovernDailyFromDTO(); ExtractFactGridGovernDailyFromDTO extractFactGridGovernDailyFromDTO = new ExtractFactGridGovernDailyFromDTO();
extractFactGridGovernDailyFromDTO.setCustomerId(customerId); extractFactGridGovernDailyFromDTO.setCustomerId(customerId);
extractFactGridGovernDailyFromDTO.setDateId(dateId); extractFactGridGovernDailyFromDTO.setDateId(dateId);
factGridGovernDailyService.extractFactGridGovernDaily(extractFactGridGovernDailyFromDTO); factGridGovernDailyService.extractFactGridGovernDaily(extractFactGridGovernDailyFromDTO);
} catch (Exception e) { } catch (Exception e) {
log.error("治理指数-网格fact_grid_govern_daily抽取失败，customerId为：" + customerId + "dateId为：" + dateId, e); log.error("治理指数-网格fact_grid_govern_daily抽取失败，customerId为：" + customerId + "dateId为：" + dateId, e);
} }
try { try {
factAgencyGovernDailyService.extractFactAgencyGovernDaily(customerId, dateId); factAgencyGovernDailyService.extractFactAgencyGovernDaily(customerId, dateId);
} catch (Exception e) { } catch (Exception e) {
log.error("治理指数-组织fact_agency_govern_daily抽取失败，customerId为：" + customerId + "dateId为：" + dateId, e); log.error("治理指数-组织fact_agency_govern_daily抽取失败，customerId为：" + customerId + "dateId为：" + dateId, e);
} }
try {
factGridMemberStatisticsDailyService.extractGridMemberStatisticsDaily(customerId, dateId);
} catch (Exception e) {
log.error("网格员数据统计fact_grid_member_statistics_daily抽取失败，customerId为：" + customerId + "dateId为：" + dateId, e);
}
extractPartData(customerId, dateId);
latch.countDown();
log.info("extractDaily 4 thread run end ========= dateId:{},customerId:{}", dateId, customerId);
});
try { try {
factGridMemberStatisticsDailyService.extractGridMemberStatisticsDaily(customerId, dateId); latch.await();
} catch (Exception e) { } catch (InterruptedException e) {
log.error("网格员数据统计fact_grid_member_statistics_daily抽取失败，customerId为：" + customerId + "dateId为：" + dateId, e); log.error("extractDaily run exception", e);
} }
extractPartData(customerId, dateId); log.info("===== extractDaily method end customerId:{}======",customerId);
latch.countDown(); }finally {
log.info("extractDaily 4 thread run end ========= dateId:{},customerId:{}", dateId, customerId); lock.unlock();
});
try {
latch.await();
} catch (InterruptedException e) {
log.error("extractDaily run exception", e);
} }
log.info("===== extractDaily method end customerId:{}======",customerId);
} }
@Override @Override

Write Preview
Loading…
Cancel
Save