Merge branch 'dev'

epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/exception/RenException.java

@@ -19,6 +19,12 @@ import org.apache.commons.lang3.StringUtils;
  * @since 1.0.0
  */
 public class RenException extends RuntimeException {
+
+    public enum MessageMode {
+        //code_内部信息_外部信息
+        CODE_INTERNAL_EXTERNAL
+    }
+
     private static final long serialVersionUID = 1L;
     private int code;
     /**
@@ -47,7 +53,7 @@ public class RenException extends RuntimeException {
         }
     }
 
-    public RenException(int code, String internalMsg, String msg) {
+    public RenException(int code, String internalMsg, String msg, MessageMode mode) {
         this(code, internalMsg);
         this.msg = msg;
     }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/ScreenExtractDailyController.java

@@ -1,23 +1,33 @@
 package com.epmet.controller;
 
+import com.alibaba.fastjson.JSON;
 import com.epmet.commons.tools.utils.Result;
 import com.epmet.dto.extract.form.ExtractOriginFormDTO;
 import com.epmet.dto.extract.form.ExtractScreenFormDTO;
 import com.epmet.service.evaluationindex.extract.toscreen.ScreenExtractService;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
 
+import java.util.concurrent.*;
+
 /**
  * @Author zxc
  * @DateTime 2020/9/24 9:42 上午
  */
+@Slf4j
 @RestController
 @RequestMapping("screenextract")
 public class ScreenExtractDailyController {
-
+    ThreadFactory namedThreadFactory = new ThreadFactoryBuilder()
+            .setNameFormat("ScreenExtractDailyController-pool-%d").build();
+    ExecutorService threadPool = new ThreadPoolExecutor(1, 1,
+            10L, TimeUnit.MINUTES,
+            new LinkedBlockingQueue<>(500), namedThreadFactory, new ThreadPoolExecutor.CallerRunsPolicy());
     @Autowired
     private ScreenExtractService screenExtractService;
 
@@ -29,7 +39,15 @@ public class ScreenExtractDailyController {
      */
     @PostMapping("extractdailyall")
     public Result screenExtractDaily(@RequestBody ExtractOriginFormDTO extractOriginFormDTO) {
-        screenExtractService.extractDailyAll(extractOriginFormDTO);
+        threadPool.submit(() -> {
+            log.info("screenExtractDaily start,param:{}", JSON.toJSONString(extractOriginFormDTO));
+            try {
+                screenExtractService.extractDailyAll(extractOriginFormDTO);
+                log.info("screenExtractDaily end");
+            } catch (Exception e) {
+                log.error("screenExtractDaily exception", e);
+            }
+        });
         return new Result();
     }
 

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/evaluationindex/extract/FactOriginGroupMainDailyDao.java

@@ -79,6 +79,16 @@ public interface FactOriginGroupMainDailyDao extends BaseDao<FactOriginGroupMain
      **/
     void deleteBatchMemberByCustomerId(@Param("customerId") String customerId, @Param("dimId") String dimId, @Param("dateType") String dateType);
 
+    /**
+     * @Description 删除指定组Id的成员
+     * @param customerId
+     * @param groups
+     * @return void
+     * @author wangc
+     * @date 2020.11.16 11:15
+    */
+    void deleteMemberByGroupList(@Param("customerId") String customerId,@Param("groups") List<String> groups);
+
     /**
      * @param customerId
      * @return

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/todata/FactOriginGroupMainDailyService.java

@@ -64,7 +64,7 @@ public interface FactOriginGroupMainDailyService extends BaseService<FactOriginG
      * @author wangc
      * @date 2020.09.23 16:30
     **/
-    boolean insertExtractedData(boolean isFirst, String customerId, String dateId, List<FactOriginGroupMainDailyDTO> originGroupData, List<ExtractGroupMemberActionRecordResultDTO> memberList);
+    boolean insertExtractedData(boolean isFirst, String customerId, String dateId, List<FactOriginGroupMainDailyDTO> originGroupData, List<ExtractGroupMemberActionRecordResultDTO> memberList, List<String> missing);
 
     /**
      * @param customerId

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/todata/impl/FactOriginGroupMainDailyServiceImpl.java

@@ -84,7 +84,7 @@ public class FactOriginGroupMainDailyServiceImpl extends BaseServiceImpl<FactOri
      **/
     @Override
     @Transactional(rollbackFor = Exception.class)
-    public boolean insertExtractedData(boolean isFirst, String customerId, String dateId,List<FactOriginGroupMainDailyDTO> originGroupData, List<ExtractGroupMemberActionRecordResultDTO> memberList) {
+    public boolean insertExtractedData(boolean isFirst, String customerId, String dateId,List<FactOriginGroupMainDailyDTO> originGroupData, List<ExtractGroupMemberActionRecordResultDTO> memberList,List<String> missing) {
         if (isFirst) {
             //isFirst
             baseDao.deleteBatchMemberByCustomerId(customerId,null,null);
@@ -95,8 +95,13 @@ public class FactOriginGroupMainDailyServiceImpl extends BaseServiceImpl<FactOri
         }
         //删除要插入的组主表数据
         baseDao.deleteBatchByGroupId(customerId,originGroupData.stream().map(FactOriginGroupMainDailyDTO :: getId).distinct().collect(Collectors.toList()));
+
         baseDao.insertBatchMain(originGroupData);
         baseDao.insertBatchMembers(memberList);
+        if(!CollectionUtils.isEmpty(missing)){
+            baseDao.deleteBatchByGroupId(customerId,missing);
+            baseDao.deleteMemberByGroupList(customerId,missing);
+        }
         return true;
     }
 

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/todata/impl/GroupExtractServiceImpl.java

@@ -23,10 +23,7 @@ import org.springframework.stereotype.Service;
 import org.springframework.util.CollectionUtils;
 
 import java.text.SimpleDateFormat;
-import java.util.Date;
+import java.util.*;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -91,11 +88,21 @@ public class GroupExtractServiceImpl implements GroupExtractService {
         List<ExtractGroupMemberActionRecordResultDTO> memberList = new LinkedList<>();
         if (!CollectionUtils.isEmpty(originGroupData)) {
         List<GridAttributesResultDTO> gridList = dimGridService.getGridAttributes(param.getCustomerId(),originGroupData.stream().map(FactOriginGroupMainDailyDTO::getGridId).distinct().collect(Collectors.toList()));
-
+        List<String> missingGroups = new LinkedList<>();
         if (!CollectionUtils.isEmpty(gridList)) {
             Map<String, GridAttributesResultDTO> gridMap =
                     gridList.stream().collect(Collectors.toMap(GridAttributesResultDTO::getGridId, Function.identity(), (key1, key2) -> key2));
-            if (!CollectionUtils.isEmpty(originGroupData)) {
+
+
+                for(Iterator<FactOriginGroupMainDailyDTO> iter = originGroupData.iterator();
+                    iter.hasNext();){
+                    FactOriginGroupMainDailyDTO pointer = iter.next();
+                    if(null == gridMap.get(pointer.getGridId())){
+                        missingGroups.add(pointer.getId());
+                        iter.remove();
+                    }
+                }
+
                 originGroupData.forEach(group -> {
                     GridAttributesResultDTO attr = gridMap.get(group.getGridId());
                     if (null != attr) {
@@ -141,14 +148,15 @@ public class GroupExtractServiceImpl implements GroupExtractService {
                     }
 
                 });
-            }
+
 
             factOriginGroupMainDailyService.insertExtractedData(
                     count <= NumConstant.ZERO ? true : false,
                     param.getCustomerId(),
                     param.getDateId(),
                     originGroupData,
-                    memberList
+                    memberList,
+                    missingGroups
             );
 
 

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/PublicPartExtractServiceImpl.java

@@ -112,14 +112,18 @@ public class PublicPartExtractServiceImpl implements PublicPartExtractService {
             return;
         }
         Map<String, GridUserCountResultDTO> userCountMap = userCountList.stream().collect(Collectors.toMap(GridUserCountResultDTO::getOrgId, o -> o));
-        issueTotal.forEach(issue -> {
+        for (FactIssueGridMonthlyEntity issue : issueTotal) {
             String gridId = issue.getGridId();
             ScreenUserJoinEntity entity = insertMap.get(gridId);
+            if (entity == null) {
+                log.warn("gridId:{} is not exist in dimGrid,monthId:{},gridId:{}", formDTO.getCustomerId(), formDTO.getMonthId(), gridId);
+                continue;
+            }
             entity.setJoinTotal(issue.getIssueIncr());
             GridUserCountResultDTO user = userCountMap.get(gridId);
             if (user == null) {
                 log.warn("gridId:{} is not exist in issue,customerId:{},monthId:{},gridId:{}", JSON.toJSONString(issue), formDTO.getCustomerId(), formDTO.getMonthId(), gridId);
-                return;
+                continue;
             }
             //百人人均议题：统计周期内总的议题数/（注册用户数/100）
             log.debug("issue:{}", JSON.toJSONString(issue));
@@ -129,7 +133,7 @@ public class PublicPartExtractServiceImpl implements PublicPartExtractService {
                         .divide(new BigDecimal(NumConstant.ONE_HUNDRED)), 4, BigDecimal.ROUND_HALF_UP);
                 entity.setAvgIssue(avgIssueCount);
             }
-        });
+        }
 
 
         //获取该月 表决的人数
@@ -153,6 +157,10 @@ public class PublicPartExtractServiceImpl implements PublicPartExtractService {
         for (Map.Entry<String, List<IssueVoteUserCountResultDTO>> entry : voteMap.entrySet()) {
             String gridId = entry.getKey();
             ScreenUserJoinEntity entity = insertMap.get(gridId);
+            if (entity == null) {
+                log.warn("gridId:{} is not exist in dimGrid,monthId:{},gridId:{}", formDTO.getCustomerId(), formDTO.getMonthId(), gridId);
+                continue;
+            }
             List<IssueVoteUserCountResultDTO> issueList = entry.getValue();
             BigDecimal bigDecimal = new BigDecimal(0);
             BigDecimal votedByIssueCount = new BigDecimal(issueList.get(0).getIssueCount());

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenExtractServiceImpl.java

@@ -243,7 +243,7 @@ public class ScreenExtractServiceImpl implements ScreenExtractService {
             param.setDateId(monthId);
             screenGrassrootsGovernDataAbsorptionService.userScoreDataHub(param);
         }catch(Exception e){
-            log.error("大屏热心市民/党员得分数据写入失败，参数为：{}",JSON.toJSONString(formDTO));
+            log.error("大屏热心市民/党员得分数据写入失败，参数为：{}" + JSON.toJSONString(formDTO), e);
         }
         //此方法保持在最后即可   计算指标分数 todo 优化 手动创建线程池 控制任务数量
         ExecutorService pool = Executors.newSingleThreadExecutor();

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/ScreenGrassrootsGovernDataAbsorptionServiceImpl.java

@@ -1,5 +1,6 @@
 package com.epmet.service.evaluationindex.extract.toscreen.impl;
 
+import com.alibaba.fastjson.JSON;
 import com.epmet.commons.tools.constant.NumConstant;
 import com.epmet.commons.tools.constant.StrConstant;
 import com.epmet.commons.tools.utils.DateUtils;
@@ -21,16 +22,13 @@ import com.epmet.service.project.ProjectService;
 import com.epmet.service.topic.TopicService;
 import com.epmet.service.user.UserService;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.util.CollectionUtils;
-import org.springframework.util.StringUtils;
 
 import java.math.BigDecimal;
-import java.util.HashMap;
+import java.util.*;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
 import java.util.stream.Collectors;
 
 
@@ -91,10 +89,10 @@ public class ScreenGrassrootsGovernDataAbsorptionServiceImpl implements ScreenGr
             dateId = DateUtils.getBeforeNMonth(NumConstant.ONE);
         }else{
             String dateType = DateUtils.identifyTimeDimension(dateId);
-            if(StringUtils.isEmpty(dateType) || !org.apache.commons.lang3.StringUtils.equalsAny(dateType,"date","month")){
+            if (StringUtils.isEmpty(dateType) || !StringUtils.equalsAny(dateType, "date", "month")) {
                 dateId = DateUtils.getBeforeNMonth(NumConstant.ONE);
-            }else if(org.apache.commons.lang3.StringUtils.equals(dateType,"date")){
+            } else if (StringUtils.equals(dateType, "date")) {
-                dateId = dateId.substring(NumConstant.ZERO,dateId.length() - NumConstant.TWO);
+                dateId = dateId.substring(NumConstant.ZERO, dateId.length() - NumConstant.TWO);
             }
         }
         final String finalDateId = dateId;
@@ -125,7 +123,19 @@ public class ScreenGrassrootsGovernDataAbsorptionServiceImpl implements ScreenGr
 
         //6.存入数据库
         //不按照时间删除，每次插入之前将该客户下的所有历史数据清空
-        screenPartyUserRankDataService.dataClean(registeredUsers,param.getCustomerId());
+        //过滤 统计维度中没有的数据
+        Set<String> gridIdSet = new HashSet<>();
+        List<ScreenPartyUserRankDataEntity> newInsertList = registeredUsers.stream()
+                .filter(o -> {
+                    if (StringUtils.isBlank(o.getGridName())) {
+                        gridIdSet.add(o.getGridId());
+                        return false;
+                    } else {
+                        return true;
+                    }
+                }).collect(Collectors.toList());
+        log.warn("userScoreDataHub grids:{} not in DimGrid", JSON.toJSONString(gridIdSet));
+        screenPartyUserRankDataService.dataClean(newInsertList, param.getCustomerId());
     }
 
     /**

epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/evaluationindex/extract/FactOriginGroupMainDailyDao.xml

@@ -64,6 +64,17 @@
         </if>
     </delete>
 
+    <!-- 删除指定组Id的成员 -->
+    <delete id="deleteMemberByGroupList">
+        DELETE FROM
+        fact_origin_group_member_daily
+        WHERE
+        CUSTOMER_ID = #{customerId}
+        <foreach collection="groups" item="groupId" open="AND (" separator=" OR "  close=" )" >
+            GROUP_ID = #{groupId}
+        </foreach>
+    </delete>
+
     <delete id="deleteBatchByGroupId" parameterType="java.util.List">
         DELETE FROM
         fact_origin_group_main_daily
@@ -74,6 +85,16 @@
         </foreach>
         </where>
     </delete>
+    
+    <delete id="deleteSpecifiedDateData">
+        DELETE FROM
+        fact_origin_group_main_daily
+        <where>
+        CUSTOMER_ID = #{customerId}
+        AND
+        DATE_ID =
+        </where>
+    </delete>
 
     <update id="updatePartyFlag" parameterType="java.util.List">
         UPDATE fact_origin_group_main_daily

epmet-module/gov-issue/gov-issue-client/src/main/java/com/epmet/dto/form/IssueShiftedFromTopicFormDTO.java

@@ -1,5 +1,6 @@
 package com.epmet.dto.form;
 
+import com.fasterxml.jackson.annotation.JsonFormat;
 import lombok.Data;
 
 import javax.validation.constraints.NotBlank;
@@ -73,6 +74,7 @@ public class IssueShiftedFromTopicFormDTO implements Serializable {
     /**
      * 议题创建时间
      * */
+    @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8")
     private Date createdTime;
 
     /**

epmet-module/oper-crm/oper-crm-server/src/main/java/com/epmet/feign/OperCustomizeFeignClient.java

@@ -9,7 +9,7 @@ import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 
 //@FeignClient(name = ServiceConstant.OPER_CUSTOMIZE_SERVER, fallback = OperCustomizeFeignClientFallBack.class, url = "localhost:8089")
-@FeignClient(name = ServiceConstant.OPER_CUSTOMIZE_SERVER, fallback = OperCustomizeFeignClientFallBack.class,url="localhost:8089")
+@FeignClient(name = ServiceConstant.OPER_CUSTOMIZE_SERVER, fallback = OperCustomizeFeignClientFallBack.class)
 public interface OperCustomizeFeignClient {
 
     /**

epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/CustomerStaffServiceImpl.java

@@ -594,14 +594,15 @@ public class CustomerStaffServiceImpl extends BaseServiceImpl<CustomerStaffDao,
         //查找工作人员信息
         //最近一次登陆的而网格、人员信息、权限
         ExtStaffInfoResultDTO result = baseDao.selectStaffInfoExt(staffParam.getStaffId());
-        result.setUserId(staffParam.getStaffId());
         if(null != result) {
+			result.setUserId(staffParam.getStaffId());
             Result<ExtStaffInfoResultDTO> govOrgResult =
                 govOrgOpenFeignClient.staffInfoExt(result);
             if(govOrgResult.success() && null != govOrgResult.getData()){
                 return govOrgResult.getData();
             }
         }
+		log.warn("com.epmet.service.impl.CustomerStaffServiceImpl.extStaffInfo，根据传入的用户Id未找到对应的工作人员，传参：{}",JSON.toJSONString(staffParam));
         return null;
     }