diff --git a/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/exception/RenException.java b/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/exception/RenException.java index 8d6285a26c..73ae4bf6b5 100644 --- a/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/exception/RenException.java +++ b/epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/exception/RenException.java @@ -19,6 +19,12 @@ import org.apache.commons.lang3.StringUtils; * @since 1.0.0 */ public class RenException extends RuntimeException { + + public enum MessageMode { + //code_内部信息_外部信息 + CODE_INTERNAL_EXTERNAL + } + private static final long serialVersionUID = 1L; private int code; /** @@ -47,7 +53,7 @@ public class RenException extends RuntimeException { } } - public RenException(int code, String internalMsg, String msg) { + public RenException(int code, String internalMsg, String msg, MessageMode mode) { this(code, internalMsg); this.msg = msg; } diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/ScreenExtractDailyController.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/ScreenExtractDailyController.java index 133dc09f8c..07919aa425 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/ScreenExtractDailyController.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/ScreenExtractDailyController.java @@ -1,23 +1,33 @@ package com.epmet.controller; +import com.alibaba.fastjson.JSON; import com.epmet.commons.tools.utils.Result; import com.epmet.dto.extract.form.ExtractOriginFormDTO; import com.epmet.dto.extract.form.ExtractScreenFormDTO; import com.epmet.service.evaluationindex.extract.toscreen.ScreenExtractService; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import java.util.concurrent.*; + /** * @Author zxc * @DateTime 2020/9/24 9:42 上午 */ +@Slf4j @RestController @RequestMapping("screenextract") public class ScreenExtractDailyController { - + ThreadFactory namedThreadFactory = new ThreadFactoryBuilder() + .setNameFormat("ScreenExtractDailyController-pool-%d").build(); + ExecutorService threadPool = new ThreadPoolExecutor(1, 1, + 10L, TimeUnit.MINUTES, + new LinkedBlockingQueue<>(500), namedThreadFactory, new ThreadPoolExecutor.CallerRunsPolicy()); @Autowired private ScreenExtractService screenExtractService; @@ -29,7 +39,15 @@ public class ScreenExtractDailyController { */ @PostMapping("extractdailyall") public Result screenExtractDaily(@RequestBody ExtractOriginFormDTO extractOriginFormDTO) { - screenExtractService.extractDailyAll(extractOriginFormDTO); + threadPool.submit(() -> { + log.info("screenExtractDaily start,param:{}", JSON.toJSONString(extractOriginFormDTO)); + try { + screenExtractService.extractDailyAll(extractOriginFormDTO); + log.info("screenExtractDaily end"); + } catch (Exception e) { + log.error("screenExtractDaily exception", e); + } + }); return new Result(); } diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/evaluationindex/extract/FactOriginGroupMainDailyDao.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/evaluationindex/extract/FactOriginGroupMainDailyDao.java index e5e603a40f..5bd0c89103 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/evaluationindex/extract/FactOriginGroupMainDailyDao.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/evaluationindex/extract/FactOriginGroupMainDailyDao.java @@ -79,6 +79,16 @@ public interface FactOriginGroupMainDailyDao extends BaseDao groups); + /** * @param customerId * @return diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/todata/FactOriginGroupMainDailyService.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/todata/FactOriginGroupMainDailyService.java index f6fc9f2dc6..c9a74e31b8 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/todata/FactOriginGroupMainDailyService.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/todata/FactOriginGroupMainDailyService.java @@ -64,7 +64,7 @@ public interface FactOriginGroupMainDailyService extends BaseService originGroupData, List memberList); + boolean insertExtractedData(boolean isFirst, String customerId, String dateId, List originGroupData, List memberList, List missing); /** * @param customerId diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/todata/impl/FactOriginGroupMainDailyServiceImpl.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/todata/impl/FactOriginGroupMainDailyServiceImpl.java index 093905835b..ab3efce8b4 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/todata/impl/FactOriginGroupMainDailyServiceImpl.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/todata/impl/FactOriginGroupMainDailyServiceImpl.java @@ -84,7 +84,7 @@ public class FactOriginGroupMainDailyServiceImpl extends BaseServiceImpl originGroupData, List memberList) { + public boolean insertExtractedData(boolean isFirst, String customerId, String dateId,List originGroupData, List memberList,List missing) { if (isFirst) { //isFirst baseDao.deleteBatchMemberByCustomerId(customerId,null,null); @@ -95,8 +95,13 @@ public class FactOriginGroupMainDailyServiceImpl extends BaseServiceImpl memberList = new LinkedList<>(); if (!CollectionUtils.isEmpty(originGroupData)) { List gridList = dimGridService.getGridAttributes(param.getCustomerId(),originGroupData.stream().map(FactOriginGroupMainDailyDTO::getGridId).distinct().collect(Collectors.toList())); - + List missingGroups = new LinkedList<>(); if (!CollectionUtils.isEmpty(gridList)) { Map gridMap = gridList.stream().collect(Collectors.toMap(GridAttributesResultDTO::getGridId, Function.identity(), (key1, key2) -> key2)); - if (!CollectionUtils.isEmpty(originGroupData)) { + + + for(Iterator iter = originGroupData.iterator(); + iter.hasNext();){ + FactOriginGroupMainDailyDTO pointer = iter.next(); + if(null == gridMap.get(pointer.getGridId())){ + missingGroups.add(pointer.getId()); + iter.remove(); + } + } + originGroupData.forEach(group -> { GridAttributesResultDTO attr = gridMap.get(group.getGridId()); if (null != attr) { @@ -141,14 +148,15 @@ public class GroupExtractServiceImpl implements GroupExtractService { } }); - } + factOriginGroupMainDailyService.insertExtractedData( count <= NumConstant.ZERO ? true : false, param.getCustomerId(), param.getDateId(), originGroupData, - memberList + memberList, + missingGroups ); diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/PublicPartExtractServiceImpl.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/PublicPartExtractServiceImpl.java index 3c58609ada..f26101c3dd 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/PublicPartExtractServiceImpl.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/PublicPartExtractServiceImpl.java @@ -112,14 +112,18 @@ public class PublicPartExtractServiceImpl implements PublicPartExtractService { return; } Map userCountMap = userCountList.stream().collect(Collectors.toMap(GridUserCountResultDTO::getOrgId, o -> o)); - issueTotal.forEach(issue -> { + for (FactIssueGridMonthlyEntity issue : issueTotal) { String gridId = issue.getGridId(); ScreenUserJoinEntity entity = insertMap.get(gridId); + if (entity == null) { + log.warn("gridId:{} is not exist in dimGrid,monthId:{},gridId:{}", formDTO.getCustomerId(), formDTO.getMonthId(), gridId); + continue; + } entity.setJoinTotal(issue.getIssueIncr()); GridUserCountResultDTO user = userCountMap.get(gridId); if (user == null) { log.warn("gridId:{} is not exist in issue,customerId:{},monthId:{},gridId:{}", JSON.toJSONString(issue), formDTO.getCustomerId(), formDTO.getMonthId(), gridId); - return; + continue; } //百人人均议题：统计周期内总的议题数/（注册用户数/100） log.debug("issue:{}", JSON.toJSONString(issue)); @@ -129,7 +133,7 @@ public class PublicPartExtractServiceImpl implements PublicPartExtractService { .divide(new BigDecimal(NumConstant.ONE_HUNDRED)), 4, BigDecimal.ROUND_HALF_UP); entity.setAvgIssue(avgIssueCount); } - }); + } //获取该月 表决的人数 @@ -153,6 +157,10 @@ public class PublicPartExtractServiceImpl implements PublicPartExtractService { for (Map.Entry> entry : voteMap.entrySet()) { String gridId = entry.getKey(); ScreenUserJoinEntity entity = insertMap.get(gridId); + if (entity == null) { + log.warn("gridId:{} is not exist in dimGrid,monthId:{},gridId:{}", formDTO.getCustomerId(), formDTO.getMonthId(), gridId); + continue; + } List issueList = entry.getValue(); BigDecimal bigDecimal = new BigDecimal(0); BigDecimal votedByIssueCount = new BigDecimal(issueList.get(0).getIssueCount()); diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenExtractServiceImpl.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenExtractServiceImpl.java index 1ad6a429fc..2ca72cacc5 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenExtractServiceImpl.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenExtractServiceImpl.java @@ -243,7 +243,7 @@ public class ScreenExtractServiceImpl implements ScreenExtractService { param.setDateId(monthId); screenGrassrootsGovernDataAbsorptionService.userScoreDataHub(param); }catch(Exception e){ - log.error("大屏热心市民/党员得分数据写入失败，参数为：{}",JSON.toJSONString(formDTO)); + log.error("大屏热心市民/党员得分数据写入失败，参数为：{}" + JSON.toJSONString(formDTO), e); } //此方法保持在最后即可 计算指标分数 todo 优化 手动创建线程池 控制任务数量 ExecutorService pool = Executors.newSingleThreadExecutor(); diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenGrassrootsGovernDataAbsorptionServiceImpl.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenGrassrootsGovernDataAbsorptionServiceImpl.java index d791fb239e..53aebc29cb 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenGrassrootsGovernDataAbsorptionServiceImpl.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenGrassrootsGovernDataAbsorptionServiceImpl.java @@ -1,5 +1,6 @@ package com.epmet.service.evaluationindex.extract.toscreen.impl; +import com.alibaba.fastjson.JSON; import com.epmet.commons.tools.constant.NumConstant; import com.epmet.commons.tools.constant.StrConstant; import com.epmet.commons.tools.utils.DateUtils; @@ -21,16 +22,13 @@ import com.epmet.service.project.ProjectService; import com.epmet.service.topic.TopicService; import com.epmet.service.user.UserService; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; -import org.springframework.util.StringUtils; import java.math.BigDecimal; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; @@ -91,10 +89,10 @@ public class ScreenGrassrootsGovernDataAbsorptionServiceImpl implements ScreenGr dateId = DateUtils.getBeforeNMonth(NumConstant.ONE); }else{ String dateType = DateUtils.identifyTimeDimension(dateId); - if(StringUtils.isEmpty(dateType) || !org.apache.commons.lang3.StringUtils.equalsAny(dateType,"date","month")){ + if (StringUtils.isEmpty(dateType) || !StringUtils.equalsAny(dateType, "date", "month")) { dateId = DateUtils.getBeforeNMonth(NumConstant.ONE); - }else if(org.apache.commons.lang3.StringUtils.equals(dateType,"date")){ - dateId = dateId.substring(NumConstant.ZERO,dateId.length() - NumConstant.TWO); + } else if (StringUtils.equals(dateType, "date")) { + dateId = dateId.substring(NumConstant.ZERO, dateId.length() - NumConstant.TWO); } } final String finalDateId = dateId; @@ -125,7 +123,19 @@ public class ScreenGrassrootsGovernDataAbsorptionServiceImpl implements ScreenGr //6.存入数据库 //不按照时间删除，每次插入之前将该客户下的所有历史数据清空 - screenPartyUserRankDataService.dataClean(registeredUsers,param.getCustomerId()); + //过滤 统计维度中没有的数据 + Set gridIdSet = new HashSet<>(); + List newInsertList = registeredUsers.stream() + .filter(o -> { + if (StringUtils.isBlank(o.getGridName())) { + gridIdSet.add(o.getGridId()); + return false; + } else { + return true; + } + }).collect(Collectors.toList()); + log.warn("userScoreDataHub grids:{} not in DimGrid", JSON.toJSONString(gridIdSet)); + screenPartyUserRankDataService.dataClean(newInsertList, param.getCustomerId()); } /** diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/evaluationindex/extract/FactOriginGroupMainDailyDao.xml b/epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/evaluationindex/extract/FactOriginGroupMainDailyDao.xml index 4710c224fa..c2c75be60f 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/evaluationindex/extract/FactOriginGroupMainDailyDao.xml +++ b/epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/evaluationindex/extract/FactOriginGroupMainDailyDao.xml @@ -64,6 +64,17 @@ + + + DELETE FROM + fact_origin_group_member_daily + WHERE + CUSTOMER_ID = #{customerId} + + GROUP_ID = #{groupId} + + + DELETE FROM fact_origin_group_main_daily @@ -74,6 +85,16 @@ + + + DELETE FROM + fact_origin_group_main_daily + + CUSTOMER_ID = #{customerId} + AND + DATE_ID = + + UPDATE fact_origin_group_main_daily diff --git a/epmet-module/gov-issue/gov-issue-client/src/main/java/com/epmet/dto/form/IssueShiftedFromTopicFormDTO.java b/epmet-module/gov-issue/gov-issue-client/src/main/java/com/epmet/dto/form/IssueShiftedFromTopicFormDTO.java index 16ac17dca2..399062ece8 100644 --- a/epmet-module/gov-issue/gov-issue-client/src/main/java/com/epmet/dto/form/IssueShiftedFromTopicFormDTO.java +++ b/epmet-module/gov-issue/gov-issue-client/src/main/java/com/epmet/dto/form/IssueShiftedFromTopicFormDTO.java @@ -1,5 +1,6 @@ package com.epmet.dto.form; +import com.fasterxml.jackson.annotation.JsonFormat; import lombok.Data; import javax.validation.constraints.NotBlank; @@ -73,6 +74,7 @@ public class IssueShiftedFromTopicFormDTO implements Serializable { /** * 议题创建时间 * */ + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") private Date createdTime; /** diff --git a/epmet-module/oper-crm/oper-crm-server/src/main/java/com/epmet/feign/OperCustomizeFeignClient.java b/epmet-module/oper-crm/oper-crm-server/src/main/java/com/epmet/feign/OperCustomizeFeignClient.java index 015b0d3684..b59a5c7a20 100644 --- a/epmet-module/oper-crm/oper-crm-server/src/main/java/com/epmet/feign/OperCustomizeFeignClient.java +++ b/epmet-module/oper-crm/oper-crm-server/src/main/java/com/epmet/feign/OperCustomizeFeignClient.java @@ -9,7 +9,7 @@ import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; //@FeignClient(name = ServiceConstant.OPER_CUSTOMIZE_SERVER, fallback = OperCustomizeFeignClientFallBack.class, url = "localhost:8089") -@FeignClient(name = ServiceConstant.OPER_CUSTOMIZE_SERVER, fallback = OperCustomizeFeignClientFallBack.class,url="localhost:8089") +@FeignClient(name = ServiceConstant.OPER_CUSTOMIZE_SERVER, fallback = OperCustomizeFeignClientFallBack.class) public interface OperCustomizeFeignClient { /** diff --git a/epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/CustomerStaffServiceImpl.java b/epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/CustomerStaffServiceImpl.java index be351746dc..cc2b99e8ae 100644 --- a/epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/CustomerStaffServiceImpl.java +++ b/epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/CustomerStaffServiceImpl.java @@ -594,14 +594,15 @@ public class CustomerStaffServiceImpl extends BaseServiceImpl govOrgResult = govOrgOpenFeignClient.staffInfoExt(result); if(govOrgResult.success() && null != govOrgResult.getData()){ return govOrgResult.getData(); } } + log.warn("com.epmet.service.impl.CustomerStaffServiceImpl.extStaffInfo，根据传入的用户Id未找到对应的工作人员，传参：{}",JSON.toJSONString(staffParam)); return null; }