diff --git a/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/constant/NumConstant.java b/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/constant/NumConstant.java index 1281b269f4..9ba8e7a96a 100644 --- a/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/constant/NumConstant.java +++ b/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/constant/NumConstant.java @@ -64,6 +64,8 @@ public interface NumConstant { long THREE_L = 3L; long FOUR_L = 4L; long MINUS_ONE_L = -1L; + long TEN_L = 10L; + long SIX_HUNDRED_L = 600L; String ZERO_STR = "0"; String ONE_STR = "1"; diff --git a/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/distributedlock/LockConstants.java b/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/distributedlock/LockConstants.java index 3174a43a8e..a823816952 100644 --- a/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/distributedlock/LockConstants.java +++ b/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/distributedlock/LockConstants.java @@ -9,4 +9,6 @@ public interface LockConstants { String TEST_LOCK_NAME = "testLock"; String STATS_LOCK_NAME = "stats"; + + String SCREEN_DAILY = "screenDaily"; } diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenExtractServiceImpl.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenExtractServiceImpl.java index e5a348fbbf..f76adbe421 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenExtractServiceImpl.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenExtractServiceImpl.java @@ -2,6 +2,8 @@ package com.epmet.service.evaluationindex.extract.toscreen.impl; import com.alibaba.fastjson.JSON; import com.epmet.commons.tools.constant.NumConstant; +import com.epmet.commons.tools.distributedlock.DistributedLock; +import com.epmet.commons.tools.distributedlock.LockConstants; import com.epmet.commons.tools.enums.EnvEnum; import com.epmet.commons.tools.utils.DateUtils; import com.epmet.commons.tools.utils.HttpClientManager; @@ -20,6 +22,7 @@ import com.epmet.service.stats.DimCustomerService; import com.google.common.util.concurrent.ThreadFactoryBuilder; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.redisson.api.RLock; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; @@ -86,6 +89,8 @@ public class ScreenExtractServiceImpl implements ScreenExtractService { private FactGridMemberStatisticsDailyService factGridMemberStatisticsDailyService; @Autowired private ScreenProjectCategoryGridAndOrgDailyService screenProjectCategoryGridAndOrgDailyService; + @Autowired + private DistributedLock distributedLock; /** * @param extractOriginFormDTO @@ -163,155 +168,163 @@ public class ScreenExtractServiceImpl implements ScreenExtractService { * @date 2020/9/24 10:16 上午 */ public void extractDaily(String customerId, String dateId, boolean isLast) { - //等待3个线程执行完毕后再 继续执行下一个客户的 避免死锁 - final CountDownLatch latch = new CountDownLatch(NumConstant.FOUR); - threadPool.submit(() -> { - //党员基本情况screen_cpc_base_data - try { - partyBaseInfoService.statsPartyMemberBaseInfoToScreen(customerId, dateId); - } catch (Exception e) { - log.error("党员基本情况抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); - } - //先锋模范screen_pioneer_data - if (isLast) { + RLock lock = null; + try { + // 锁持有10分钟，等待10s + lock = distributedLock.getLock(LockConstants.SCREEN_DAILY, NumConstant.SIX_HUNDRED_L, NumConstant.TEN_L, TimeUnit.SECONDS); + //等待3个线程执行完毕后再 继续执行下一个客户的 避免死锁 + final CountDownLatch latch = new CountDownLatch(NumConstant.FOUR); + threadPool.submit(() -> { + //党员基本情况screen_cpc_base_data try { - pioneerDataExtractService.extractGridPioneerData(customerId, dateId); + partyBaseInfoService.statsPartyMemberBaseInfoToScreen(customerId, dateId); } catch (Exception e) { - log.error("先锋模范【网格】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); + log.error("党员基本情况抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); + } + //先锋模范screen_pioneer_data + if (isLast) { + try { + pioneerDataExtractService.extractGridPioneerData(customerId, dateId); + } catch (Exception e) { + log.error("先锋模范【网格】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); + } + try { + pioneerDataExtractService.extractCommunityPioneerData(customerId, dateId); + } catch (Exception e) { + log.error("先锋模范【社区】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); + } + try { + pioneerDataExtractService.extractExceptCommunityPioneerData(customerId, dateId); + } catch (Exception e) { + log.error("先锋模范【extractExceptCommunityPioneerData】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); + } } + + latch.countDown(); + log.info("extractDaily 1 thread run end ========= dateId:{},customerId:{}", dateId, customerId); + }); + threadPool.submit(() -> { + //公众参与排行（注册人数、参与人数、话题数、议题数、项目数）screen_public_party_total_data try { - pioneerDataExtractService.extractCommunityPioneerData(customerId, dateId); + publicPartiTotalDataExtractService.extractPublicPartiTotalData(customerId, dateId); } catch (Exception e) { - log.error("先锋模范【社区】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); + log.error("公众参与排行抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); + } + if (isLast) { + //基层治理- 难点赌点 screen_difficulty_data | screen_difficulty_img_data + try { + ScreenCentralZoneDataFormDTO param2 = new ScreenCentralZoneDataFormDTO(); + param2.setCustomerId(customerId); + param2.setDateId(null); + log.info("【难点赌点数据上报开始------】 当前客户Id{}", param2.getCustomerId()); + //screenGrassrootsGovernDataAbsorptionService.difficultyDataHub(param); + + screenGrassrootsGovernDataAbsorptionService.difficultyDataExtract(param2); + log.info("【难点赌点数据上报结束------】 当前客户Id{}", param2.getCustomerId()); + } catch (Exception e) { + log.error("基层治理-难点赌点抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); + } } + latch.countDown(); + log.info("extractDaily 2 thread run end ========= dateId:{},customerId:{}", dateId, customerId); + }); + threadPool.submit(() -> { + ScreenCentralZoneDataFormDTO param = new ScreenCentralZoneDataFormDTO(); + param.setCustomerId(customerId); + param.setDateId(dateId); + //中央区 screen_user_total_data try { - pioneerDataExtractService.extractExceptCommunityPioneerData(customerId, dateId); + screenCentralZoneDataAbsorptionService.centralZoneDataHub(param); } catch (Exception e) { - log.error("先锋模范【extractExceptCommunityPioneerData】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); + log.error("中央区抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); } - } - latch.countDown(); - log.info("extractDaily 1 thread run end ========= dateId:{},customerId:{}", dateId, customerId); - }); - threadPool.submit(() -> { - //公众参与排行（注册人数、参与人数、话题数、议题数、项目数）screen_public_party_total_data - try { - publicPartiTotalDataExtractService.extractPublicPartiTotalData(customerId, dateId); - } catch (Exception e) { - log.error("公众参与排行抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); - } - if (isLast) { - //基层治理- 难点赌点 screen_difficulty_data | screen_difficulty_img_data try { - ScreenCentralZoneDataFormDTO param2 = new ScreenCentralZoneDataFormDTO(); - param2.setCustomerId(customerId); - param2.setDateId(null); - log.info("【难点赌点数据上报开始------】 当前客户Id{}", param2.getCustomerId()); - //screenGrassrootsGovernDataAbsorptionService.difficultyDataHub(param); - - screenGrassrootsGovernDataAbsorptionService.difficultyDataExtract(param2); - log.info("【难点赌点数据上报结束------】 当前客户Id{}", param2.getCustomerId()); + // 项目(事件)分析按网格_按天统计 + screenProjectGridDailyService.extractionProjectGridDaily(customerId, dateId); } catch (Exception e) { - log.error("基层治理-难点赌点抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); + log.error("项目(事件)分析按网格_按天统计失败，customerId为：" + customerId + "dateId为：" + dateId, e); + } + try { + // 项目(事件)分析按组织_按天统计 + screenProjectOrgDailyService.extractionProjectOrgDaily(customerId, dateId); + } catch (Exception e) { + log.error("项目(事件)分析按组织_按天统计失败，customerId为：" + customerId + "dateId为：" + dateId, e); } - } - latch.countDown(); - log.info("extractDaily 2 thread run end ========= dateId:{},customerId:{}", dateId, customerId); - }); - threadPool.submit(() -> { - ScreenCentralZoneDataFormDTO param = new ScreenCentralZoneDataFormDTO(); - param.setCustomerId(customerId); - param.setDateId(dateId); - //中央区 screen_user_total_data - try { - screenCentralZoneDataAbsorptionService.centralZoneDataHub(param); - } catch (Exception e) { - log.error("中央区抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); - } - - try { - // 项目(事件)分析按网格_按天统计 - screenProjectGridDailyService.extractionProjectGridDaily(customerId, dateId); - } catch (Exception e) { - log.error("项目(事件)分析按网格_按天统计失败，customerId为：" + customerId + "dateId为：" + dateId, e); - } - try { - // 项目(事件)分析按组织_按天统计 - screenProjectOrgDailyService.extractionProjectOrgDaily(customerId, dateId); - } catch (Exception e) { - log.error("项目(事件)分析按组织_按天统计失败，customerId为：" + customerId + "dateId为：" + dateId, e); - } - //按天统计：网格内各个分类下的项目总数 - try { - screenProjectCategoryGridAndOrgDailyService.extractCategoryProjectGridData(customerId, dateId); - } catch (Exception e) { - log.error("按天统计：网格内各个分类下的项目总数，customerId为：" + customerId + "dateId为：" + dateId, e); - } + //按天统计：网格内各个分类下的项目总数 + try { + screenProjectCategoryGridAndOrgDailyService.extractCategoryProjectGridData(customerId, dateId); + } catch (Exception e) { + log.error("按天统计：网格内各个分类下的项目总数，customerId为：" + customerId + "dateId为：" + dateId, e); + } - // 按天统计：组织内各个分类下的项目总数 - try { - screenProjectCategoryGridAndOrgDailyService.extractCategoryProjectOrgData(customerId, dateId); - } catch (Exception e) { - log.error("按天统计：组织内各个分类下的项目总数，customerId为：" + customerId + "dateId为：" + dateId, e); - } - latch.countDown(); - log.info("extractDaily 3 thread run end ========= dateId:{},customerId:{}", dateId, customerId); - }); - threadPool.submit(() -> { - //治理能力排行screen_govern_rank_data - try { - governRankDataExtractService.extractGridDataDaily(customerId, dateId); - } catch (Exception e) { - log.error("治理能力排行【网格】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); - } - try { - governRankDataExtractService.extractCommunityDataDaily(customerId, dateId); - } catch (Exception e) { - log.error("治理能力排行【社区】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); - } - try { - governRankDataExtractService.extractStreetDataDaily(customerId, dateId); - } catch (Exception e) { - log.error("治理能力排行【街道】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); - } - try { - governRankDataExtractService.extractDistrictDataDaily(customerId, dateId); - } catch (Exception e) { - log.error("治理能力排行【全区】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); - } + // 按天统计：组织内各个分类下的项目总数 + try { + screenProjectCategoryGridAndOrgDailyService.extractCategoryProjectOrgData(customerId, dateId); + } catch (Exception e) { + log.error("按天统计：组织内各个分类下的项目总数，customerId为：" + customerId + "dateId为：" + dateId, e); + } + latch.countDown(); + log.info("extractDaily 3 thread run end ========= dateId:{},customerId:{}", dateId, customerId); + }); + threadPool.submit(() -> { + //治理能力排行screen_govern_rank_data + try { + governRankDataExtractService.extractGridDataDaily(customerId, dateId); + } catch (Exception e) { + log.error("治理能力排行【网格】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); + } + try { + governRankDataExtractService.extractCommunityDataDaily(customerId, dateId); + } catch (Exception e) { + log.error("治理能力排行【社区】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); + } + try { + governRankDataExtractService.extractStreetDataDaily(customerId, dateId); + } catch (Exception e) { + log.error("治理能力排行【街道】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); + } + try { + governRankDataExtractService.extractDistrictDataDaily(customerId, dateId); + } catch (Exception e) { + log.error("治理能力排行【全区】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); + } - try { - ExtractFactGridGovernDailyFromDTO extractFactGridGovernDailyFromDTO = new ExtractFactGridGovernDailyFromDTO(); - extractFactGridGovernDailyFromDTO.setCustomerId(customerId); - extractFactGridGovernDailyFromDTO.setDateId(dateId); - factGridGovernDailyService.extractFactGridGovernDaily(extractFactGridGovernDailyFromDTO); - } catch (Exception e) { - log.error("治理指数-网格fact_grid_govern_daily抽取失败，customerId为：" + customerId + "dateId为：" + dateId, e); - } + try { + ExtractFactGridGovernDailyFromDTO extractFactGridGovernDailyFromDTO = new ExtractFactGridGovernDailyFromDTO(); + extractFactGridGovernDailyFromDTO.setCustomerId(customerId); + extractFactGridGovernDailyFromDTO.setDateId(dateId); + factGridGovernDailyService.extractFactGridGovernDaily(extractFactGridGovernDailyFromDTO); + } catch (Exception e) { + log.error("治理指数-网格fact_grid_govern_daily抽取失败，customerId为：" + customerId + "dateId为：" + dateId, e); + } - try { - factAgencyGovernDailyService.extractFactAgencyGovernDaily(customerId, dateId); - } catch (Exception e) { - log.error("治理指数-组织fact_agency_govern_daily抽取失败，customerId为：" + customerId + "dateId为：" + dateId, e); - } + try { + factAgencyGovernDailyService.extractFactAgencyGovernDaily(customerId, dateId); + } catch (Exception e) { + log.error("治理指数-组织fact_agency_govern_daily抽取失败，customerId为：" + customerId + "dateId为：" + dateId, e); + } + try { + factGridMemberStatisticsDailyService.extractGridMemberStatisticsDaily(customerId, dateId); + } catch (Exception e) { + log.error("网格员数据统计fact_grid_member_statistics_daily抽取失败，customerId为：" + customerId + "dateId为：" + dateId, e); + } + extractPartData(customerId, dateId); + latch.countDown(); + log.info("extractDaily 4 thread run end ========= dateId:{},customerId:{}", dateId, customerId); + }); try { - factGridMemberStatisticsDailyService.extractGridMemberStatisticsDaily(customerId, dateId); - } catch (Exception e) { - log.error("网格员数据统计fact_grid_member_statistics_daily抽取失败，customerId为：" + customerId + "dateId为：" + dateId, e); + latch.await(); + } catch (InterruptedException e) { + log.error("extractDaily run exception", e); } - extractPartData(customerId, dateId); - latch.countDown(); - log.info("extractDaily 4 thread run end ========= dateId:{},customerId:{}", dateId, customerId); - }); - try { - latch.await(); - } catch (InterruptedException e) { - log.error("extractDaily run exception", e); + log.info("===== extractDaily method end customerId:{}======",customerId); + }finally { + lock.unlock(); } - log.info("===== extractDaily method end customerId:{}======",customerId); + } @Override