diff --git a/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/constant/NumConstant.java b/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/constant/NumConstant.java index 1281b269f4..9ba8e7a96a 100644 --- a/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/constant/NumConstant.java +++ b/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/constant/NumConstant.java @@ -64,6 +64,8 @@ public interface NumConstant { long THREE_L = 3L; long FOUR_L = 4L; long MINUS_ONE_L = -1L; + long TEN_L = 10L; + long SIX_HUNDRED_L = 600L; String ZERO_STR = "0"; String ONE_STR = "1"; diff --git a/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/distributedlock/LockConstants.java b/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/distributedlock/LockConstants.java index 3174a43a8e..a823816952 100644 --- a/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/distributedlock/LockConstants.java +++ b/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/distributedlock/LockConstants.java @@ -9,4 +9,6 @@ public interface LockConstants { String TEST_LOCK_NAME = "testLock"; String STATS_LOCK_NAME = "stats"; + + String SCREEN_DAILY = "screenDaily"; } diff --git a/epmet-module/data-report/data-report-server/src/main/java/com/epmet/datareport/dao/evaluationindex/screen/ScreenProjectDataDao.java b/epmet-module/data-report/data-report-server/src/main/java/com/epmet/datareport/dao/evaluationindex/screen/ScreenProjectDataDao.java index 8c7ce5e97d..d20a3f1183 100644 --- a/epmet-module/data-report/data-report-server/src/main/java/com/epmet/datareport/dao/evaluationindex/screen/ScreenProjectDataDao.java +++ b/epmet-module/data-report/data-report-server/src/main/java/com/epmet/datareport/dao/evaluationindex/screen/ScreenProjectDataDao.java @@ -54,7 +54,7 @@ public interface ScreenProjectDataDao { * @author zxc * @date 2021/11/5 1:40 下午 */ - List selectCategoryProjectList(@Param("customerId") String customerId,@Param("orgId") String orgId, + List selectCategoryProjectList(@Param("orgId") String orgId, @Param("categoryCode")String categoryCode,@Param("status")String status, @Param("startTime")String startTime,@Param("endTime")String endTime); diff --git a/epmet-module/data-report/data-report-server/src/main/java/com/epmet/datareport/service/evaluationindex/screen/impl/ScreenProjectServiceImpl.java b/epmet-module/data-report/data-report-server/src/main/java/com/epmet/datareport/service/evaluationindex/screen/impl/ScreenProjectServiceImpl.java index f29d7b7ccd..ed99731870 100644 --- a/epmet-module/data-report/data-report-server/src/main/java/com/epmet/datareport/service/evaluationindex/screen/impl/ScreenProjectServiceImpl.java +++ b/epmet-module/data-report/data-report-server/src/main/java/com/epmet/datareport/service/evaluationindex/screen/impl/ScreenProjectServiceImpl.java @@ -11,6 +11,7 @@ import com.epmet.commons.tools.redis.common.bean.AgencyInfoCache; import com.epmet.commons.tools.redis.common.bean.GridInfoCache; import com.epmet.commons.tools.security.dto.TokenDto; import com.epmet.commons.tools.utils.ConvertUtils; +import com.epmet.commons.tools.utils.DateUtils; import com.epmet.commons.tools.utils.Result; import com.epmet.constant.DataSourceConstant; import com.epmet.datareport.constant.FactConstant; @@ -253,6 +254,9 @@ public class ScreenProjectServiceImpl implements ScreenProjectService { */ @Override public List selectProjectCategory(ProjectCategoryFormDTO formDTO, TokenDto tokenDto) { + if (StringUtils.isNotBlank(formDTO.getStartTime())){ + formDTO.setStartTime(DateUtils.getBeforeNDay(formDTO.getStartTime(),NumConstant.ONE)); + } if (StringUtils.isBlank(formDTO.getOrgId())){ CustomerStaffInfoCacheResult staffInfo = CustomerStaffRedis.getStaffInfo(tokenDto.getCustomerId(), tokenDto.getUserId()); if (null == staffInfo){ @@ -382,7 +386,7 @@ public class ScreenProjectServiceImpl implements ScreenProjectService { PageCategoryProjectListResultDTO result = new PageCategoryProjectListResultDTO(); List list; PageInfo objectPageInfo = PageHelper.startPage(formDTO.getPageNo(), formDTO.getPageSize(), formDTO.getIsPage()) - .doSelectPageInfo(() -> screenProjectDataDao.selectCategoryProjectList(tokenDto.getCustomerId(), formDTO.getOrgId(), formDTO.getCategoryCode(), formDTO.getStatus(),formDTO.getStartTime(),formDTO.getEndTime())); + .doSelectPageInfo(() -> screenProjectDataDao.selectCategoryProjectList(formDTO.getOrgId(), formDTO.getCategoryCode(), formDTO.getStatus(),formDTO.getStartTime(),formDTO.getEndTime())); list = objectPageInfo.getList(); result.setTotal(Integer.valueOf(String.valueOf(objectPageInfo.getTotal()))); diff --git a/epmet-module/data-report/data-report-server/src/main/resources/mapper/screen/ScreenProjectDataDao.xml b/epmet-module/data-report/data-report-server/src/main/resources/mapper/screen/ScreenProjectDataDao.xml index df49c65900..28b422ab4d 100644 --- a/epmet-module/data-report/data-report-server/src/main/resources/mapper/screen/ScreenProjectDataDao.xml +++ b/epmet-module/data-report/data-report-server/src/main/resources/mapper/screen/ScreenProjectDataDao.xml @@ -180,7 +180,6 @@ AND CONCAT(pd.ALL_PARENT_IDS,':',pd.ORG_ID) LIKE CONCAT('%',#{orgId},'%') AND pc.CATEGORY_CODE LIKE CONCAT(#{categoryCode},'%') - AND pd.CUSTOMER_ID = #{customerId} AND DATE_FORMAT(pd.PROJECT_CREATE_TIME,'%Y%m%d') #{endTime} AND DATE_FORMAT(pd.PROJECT_CREATE_TIME,'%Y%m%d') >= #{startTime} diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenExtractServiceImpl.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenExtractServiceImpl.java index e5a348fbbf..f76adbe421 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenExtractServiceImpl.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenExtractServiceImpl.java @@ -2,6 +2,8 @@ package com.epmet.service.evaluationindex.extract.toscreen.impl; import com.alibaba.fastjson.JSON; import com.epmet.commons.tools.constant.NumConstant; +import com.epmet.commons.tools.distributedlock.DistributedLock; +import com.epmet.commons.tools.distributedlock.LockConstants; import com.epmet.commons.tools.enums.EnvEnum; import com.epmet.commons.tools.utils.DateUtils; import com.epmet.commons.tools.utils.HttpClientManager; @@ -20,6 +22,7 @@ import com.epmet.service.stats.DimCustomerService; import com.google.common.util.concurrent.ThreadFactoryBuilder; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.redisson.api.RLock; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; @@ -86,6 +89,8 @@ public class ScreenExtractServiceImpl implements ScreenExtractService { private FactGridMemberStatisticsDailyService factGridMemberStatisticsDailyService; @Autowired private ScreenProjectCategoryGridAndOrgDailyService screenProjectCategoryGridAndOrgDailyService; + @Autowired + private DistributedLock distributedLock; /** * @param extractOriginFormDTO @@ -163,155 +168,163 @@ public class ScreenExtractServiceImpl implements ScreenExtractService { * @date 2020/9/24 10:16 上午 */ public void extractDaily(String customerId, String dateId, boolean isLast) { - //等待3个线程执行完毕后再 继续执行下一个客户的 避免死锁 - final CountDownLatch latch = new CountDownLatch(NumConstant.FOUR); - threadPool.submit(() -> { - //党员基本情况screen_cpc_base_data - try { - partyBaseInfoService.statsPartyMemberBaseInfoToScreen(customerId, dateId); - } catch (Exception e) { - log.error("党员基本情况抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); - } - //先锋模范screen_pioneer_data - if (isLast) { + RLock lock = null; + try { + // 锁持有10分钟，等待10s + lock = distributedLock.getLock(LockConstants.SCREEN_DAILY, NumConstant.SIX_HUNDRED_L, NumConstant.TEN_L, TimeUnit.SECONDS); + //等待3个线程执行完毕后再 继续执行下一个客户的 避免死锁 + final CountDownLatch latch = new CountDownLatch(NumConstant.FOUR); + threadPool.submit(() -> { + //党员基本情况screen_cpc_base_data try { - pioneerDataExtractService.extractGridPioneerData(customerId, dateId); + partyBaseInfoService.statsPartyMemberBaseInfoToScreen(customerId, dateId); } catch (Exception e) { - log.error("先锋模范【网格】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); + log.error("党员基本情况抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); + } + //先锋模范screen_pioneer_data + if (isLast) { + try { + pioneerDataExtractService.extractGridPioneerData(customerId, dateId); + } catch (Exception e) { + log.error("先锋模范【网格】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); + } + try { + pioneerDataExtractService.extractCommunityPioneerData(customerId, dateId); + } catch (Exception e) { + log.error("先锋模范【社区】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); + } + try { + pioneerDataExtractService.extractExceptCommunityPioneerData(customerId, dateId); + } catch (Exception e) { + log.error("先锋模范【extractExceptCommunityPioneerData】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); + } } + + latch.countDown(); + log.info("extractDaily 1 thread run end ========= dateId:{},customerId:{}", dateId, customerId); + }); + threadPool.submit(() -> { + //公众参与排行（注册人数、参与人数、话题数、议题数、项目数）screen_public_party_total_data try { - pioneerDataExtractService.extractCommunityPioneerData(customerId, dateId); + publicPartiTotalDataExtractService.extractPublicPartiTotalData(customerId, dateId); } catch (Exception e) { - log.error("先锋模范【社区】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); + log.error("公众参与排行抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); + } + if (isLast) { + //基层治理- 难点赌点 screen_difficulty_data | screen_difficulty_img_data + try { + ScreenCentralZoneDataFormDTO param2 = new ScreenCentralZoneDataFormDTO(); + param2.setCustomerId(customerId); + param2.setDateId(null); + log.info("【难点赌点数据上报开始------】 当前客户Id{}", param2.getCustomerId()); + //screenGrassrootsGovernDataAbsorptionService.difficultyDataHub(param); + + screenGrassrootsGovernDataAbsorptionService.difficultyDataExtract(param2); + log.info("【难点赌点数据上报结束------】 当前客户Id{}", param2.getCustomerId()); + } catch (Exception e) { + log.error("基层治理-难点赌点抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); + } } + latch.countDown(); + log.info("extractDaily 2 thread run end ========= dateId:{},customerId:{}", dateId, customerId); + }); + threadPool.submit(() -> { + ScreenCentralZoneDataFormDTO param = new ScreenCentralZoneDataFormDTO(); + param.setCustomerId(customerId); + param.setDateId(dateId); + //中央区 screen_user_total_data try { - pioneerDataExtractService.extractExceptCommunityPioneerData(customerId, dateId); + screenCentralZoneDataAbsorptionService.centralZoneDataHub(param); } catch (Exception e) { - log.error("先锋模范【extractExceptCommunityPioneerData】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); + log.error("中央区抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); } - } - latch.countDown(); - log.info("extractDaily 1 thread run end ========= dateId:{},customerId:{}", dateId, customerId); - }); - threadPool.submit(() -> { - //公众参与排行（注册人数、参与人数、话题数、议题数、项目数）screen_public_party_total_data - try { - publicPartiTotalDataExtractService.extractPublicPartiTotalData(customerId, dateId); - } catch (Exception e) { - log.error("公众参与排行抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); - } - if (isLast) { - //基层治理- 难点赌点 screen_difficulty_data | screen_difficulty_img_data try { - ScreenCentralZoneDataFormDTO param2 = new ScreenCentralZoneDataFormDTO(); - param2.setCustomerId(customerId); - param2.setDateId(null); - log.info("【难点赌点数据上报开始------】 当前客户Id{}", param2.getCustomerId()); - //screenGrassrootsGovernDataAbsorptionService.difficultyDataHub(param); - - screenGrassrootsGovernDataAbsorptionService.difficultyDataExtract(param2); - log.info("【难点赌点数据上报结束------】 当前客户Id{}", param2.getCustomerId()); + // 项目(事件)分析按网格_按天统计 + screenProjectGridDailyService.extractionProjectGridDaily(customerId, dateId); } catch (Exception e) { - log.error("基层治理-难点赌点抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); + log.error("项目(事件)分析按网格_按天统计失败，customerId为：" + customerId + "dateId为：" + dateId, e); + } + try { + // 项目(事件)分析按组织_按天统计 + screenProjectOrgDailyService.extractionProjectOrgDaily(customerId, dateId); + } catch (Exception e) { + log.error("项目(事件)分析按组织_按天统计失败，customerId为：" + customerId + "dateId为：" + dateId, e); } - } - latch.countDown(); - log.info("extractDaily 2 thread run end ========= dateId:{},customerId:{}", dateId, customerId); - }); - threadPool.submit(() -> { - ScreenCentralZoneDataFormDTO param = new ScreenCentralZoneDataFormDTO(); - param.setCustomerId(customerId); - param.setDateId(dateId); - //中央区 screen_user_total_data - try { - screenCentralZoneDataAbsorptionService.centralZoneDataHub(param); - } catch (Exception e) { - log.error("中央区抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); - } - - try { - // 项目(事件)分析按网格_按天统计 - screenProjectGridDailyService.extractionProjectGridDaily(customerId, dateId); - } catch (Exception e) { - log.error("项目(事件)分析按网格_按天统计失败，customerId为：" + customerId + "dateId为：" + dateId, e); - } - try { - // 项目(事件)分析按组织_按天统计 - screenProjectOrgDailyService.extractionProjectOrgDaily(customerId, dateId); - } catch (Exception e) { - log.error("项目(事件)分析按组织_按天统计失败，customerId为：" + customerId + "dateId为：" + dateId, e); - } - //按天统计：网格内各个分类下的项目总数 - try { - screenProjectCategoryGridAndOrgDailyService.extractCategoryProjectGridData(customerId, dateId); - } catch (Exception e) { - log.error("按天统计：网格内各个分类下的项目总数，customerId为：" + customerId + "dateId为：" + dateId, e); - } + //按天统计：网格内各个分类下的项目总数 + try { + screenProjectCategoryGridAndOrgDailyService.extractCategoryProjectGridData(customerId, dateId); + } catch (Exception e) { + log.error("按天统计：网格内各个分类下的项目总数，customerId为：" + customerId + "dateId为：" + dateId, e); + } - // 按天统计：组织内各个分类下的项目总数 - try { - screenProjectCategoryGridAndOrgDailyService.extractCategoryProjectOrgData(customerId, dateId); - } catch (Exception e) { - log.error("按天统计：组织内各个分类下的项目总数，customerId为：" + customerId + "dateId为：" + dateId, e); - } - latch.countDown(); - log.info("extractDaily 3 thread run end ========= dateId:{},customerId:{}", dateId, customerId); - }); - threadPool.submit(() -> { - //治理能力排行screen_govern_rank_data - try { - governRankDataExtractService.extractGridDataDaily(customerId, dateId); - } catch (Exception e) { - log.error("治理能力排行【网格】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); - } - try { - governRankDataExtractService.extractCommunityDataDaily(customerId, dateId); - } catch (Exception e) { - log.error("治理能力排行【社区】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); - } - try { - governRankDataExtractService.extractStreetDataDaily(customerId, dateId); - } catch (Exception e) { - log.error("治理能力排行【街道】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); - } - try { - governRankDataExtractService.extractDistrictDataDaily(customerId, dateId); - } catch (Exception e) { - log.error("治理能力排行【全区】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); - } + // 按天统计：组织内各个分类下的项目总数 + try { + screenProjectCategoryGridAndOrgDailyService.extractCategoryProjectOrgData(customerId, dateId); + } catch (Exception e) { + log.error("按天统计：组织内各个分类下的项目总数，customerId为：" + customerId + "dateId为：" + dateId, e); + } + latch.countDown(); + log.info("extractDaily 3 thread run end ========= dateId:{},customerId:{}", dateId, customerId); + }); + threadPool.submit(() -> { + //治理能力排行screen_govern_rank_data + try { + governRankDataExtractService.extractGridDataDaily(customerId, dateId); + } catch (Exception e) { + log.error("治理能力排行【网格】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); + } + try { + governRankDataExtractService.extractCommunityDataDaily(customerId, dateId); + } catch (Exception e) { + log.error("治理能力排行【社区】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); + } + try { + governRankDataExtractService.extractStreetDataDaily(customerId, dateId); + } catch (Exception e) { + log.error("治理能力排行【街道】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); + } + try { + governRankDataExtractService.extractDistrictDataDaily(customerId, dateId); + } catch (Exception e) { + log.error("治理能力排行【全区】抽取到大屏失败，customerId为：" + customerId + "dateId为：" + dateId, e); + } - try { - ExtractFactGridGovernDailyFromDTO extractFactGridGovernDailyFromDTO = new ExtractFactGridGovernDailyFromDTO(); - extractFactGridGovernDailyFromDTO.setCustomerId(customerId); - extractFactGridGovernDailyFromDTO.setDateId(dateId); - factGridGovernDailyService.extractFactGridGovernDaily(extractFactGridGovernDailyFromDTO); - } catch (Exception e) { - log.error("治理指数-网格fact_grid_govern_daily抽取失败，customerId为：" + customerId + "dateId为：" + dateId, e); - } + try { + ExtractFactGridGovernDailyFromDTO extractFactGridGovernDailyFromDTO = new ExtractFactGridGovernDailyFromDTO(); + extractFactGridGovernDailyFromDTO.setCustomerId(customerId); + extractFactGridGovernDailyFromDTO.setDateId(dateId); + factGridGovernDailyService.extractFactGridGovernDaily(extractFactGridGovernDailyFromDTO); + } catch (Exception e) { + log.error("治理指数-网格fact_grid_govern_daily抽取失败，customerId为：" + customerId + "dateId为：" + dateId, e); + } - try { - factAgencyGovernDailyService.extractFactAgencyGovernDaily(customerId, dateId); - } catch (Exception e) { - log.error("治理指数-组织fact_agency_govern_daily抽取失败，customerId为：" + customerId + "dateId为：" + dateId, e); - } + try { + factAgencyGovernDailyService.extractFactAgencyGovernDaily(customerId, dateId); + } catch (Exception e) { + log.error("治理指数-组织fact_agency_govern_daily抽取失败，customerId为：" + customerId + "dateId为：" + dateId, e); + } + try { + factGridMemberStatisticsDailyService.extractGridMemberStatisticsDaily(customerId, dateId); + } catch (Exception e) { + log.error("网格员数据统计fact_grid_member_statistics_daily抽取失败，customerId为：" + customerId + "dateId为：" + dateId, e); + } + extractPartData(customerId, dateId); + latch.countDown(); + log.info("extractDaily 4 thread run end ========= dateId:{},customerId:{}", dateId, customerId); + }); try { - factGridMemberStatisticsDailyService.extractGridMemberStatisticsDaily(customerId, dateId); - } catch (Exception e) { - log.error("网格员数据统计fact_grid_member_statistics_daily抽取失败，customerId为：" + customerId + "dateId为：" + dateId, e); + latch.await(); + } catch (InterruptedException e) { + log.error("extractDaily run exception", e); } - extractPartData(customerId, dateId); - latch.countDown(); - log.info("extractDaily 4 thread run end ========= dateId:{},customerId:{}", dateId, customerId); - }); - try { - latch.await(); - } catch (InterruptedException e) { - log.error("extractDaily run exception", e); + log.info("===== extractDaily method end customerId:{}======",customerId); + }finally { + lock.unlock(); } - log.info("===== extractDaily method end customerId:{}======",customerId); + } @Override