Merge remote-tracking branch 'origin/dev_yzm' into dev_temp

epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/utils/DateUtils.java

@@ -424,6 +424,13 @@ public class DateUtils {
         return DateUtils.format(date,DateUtils.DATE_PATTERN_YYYYMMDD);
     }
 
+    public static String getBeforeNDay(int beforDay,String format){
+        Calendar c = Calendar.getInstance();
+        c.add(Calendar.DATE, - beforDay);
+        Date date = c.getTime();
+        return DateUtils.format(date,format);
+    }
+
     /**
      * @return java.lang.String
      * @param beforMonth

epmet-module/data-statistical/data-statistical-client/src/main/java/com/epmet/dto/stats/user/result/UserStatisticalData.java

@@ -17,25 +17,58 @@ import java.util.List;
 public class UserStatisticalData implements Serializable {
     private static final long serialVersionUID = 7423427555123585566L;
 
+    /**
+     * 机关下(按日)参与用户数分析
+     */
     private List<FactParticipationUserAgencyDailyDTO> partiAgencyDailyList;
 
+    /**
+     * 网格下(按日)参与用户数分析
+     */
     private List<FactParticipationUserGridDailyDTO> partiGridDailyList;
 
+    /**
+     * 机关下(按月)参与用户数分析
+     */
     private List<FactParticipationUserAgencyMonthlyDTO> partiAgencyMonthlyList;
 
+    /**
+     * 网格下(月)参与用户数分析
+     */
     private List<FactParticipationUserGridMonthlyDTO> partiGridMonthlyList;
 
+    /**
+     * 机关(按日)注册用户数分析
+     */
     private List<FactRegUserAgencyDailyDTO> regAgencyDailyList;
 
+    /**
+     * 网格(按日)注册用户数分析
+     */
     private List<FactRegUserGridDailyDTO> regGridDailyList;
 
+    /**
+     * 机关(按月)注册用户数分析
+     */
     private List<FactRegUserAgencyMonthlyDTO> regAgencyMonthlyList;
 
+    /**
+     * 网格(月)注册用户数分析
+     */
     private List<FactRegUserGridMonthlyDTO> regGridMonthlyList;
 
+    /**
+     * 当前正在计算的客户id
+     */
     private String customerId;
 
+    /**
+     * yyyyMMdd
+     */
     private String dateId;
 
+    /**
+     * yyyyMM
+     */
     private String monthId;
 }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/StatsUserController.java

@@ -21,6 +21,12 @@ public class StatsUserController {
     @Autowired
     private StatsUserService statsUserService;
 
+    /**
+     * @return com.epmet.commons.tools.utils.Result
+     * @param formDTO
+     * @description 工作端数据一期，用户分析：参与用户、注册用户分析
+     * @Date 2021/3/26 13:27
+     **/
     @RequestMapping("execute")
     public Result execute(@RequestBody StatsFormDTO formDTO) {
         statsUserService.partition(formDTO);

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsUserServiceImpl.java

@@ -1,6 +1,7 @@
 package com.epmet.service.impl;
 
 
+import com.alibaba.fastjson.JSON;
 import com.epmet.commons.tools.constant.NumConstant;
 import com.epmet.commons.tools.utils.DateUtils;
 import com.epmet.constant.ProjectConstant;
@@ -58,17 +59,25 @@ public class StatsUserServiceImpl implements StatsUserService {
      **/
     @Override
     public void partition(StatsFormDTO formDTO) {
+        if(StringUtils.isBlank(formDTO.getDate())){
+            //如果定时任务没有指定参数，默认数据更新至t-1,包含t-1这一天内的数据
+            formDTO.setDate(DateUtils.getBeforeNDay(1,DateUtils.DATE_PATTERN));
+        }
+
         int pageNo = NumConstant.ONE;
         int pageSize = NumConstant.ONE_HUNDRED;
         List<String> customerIdList = null;
         Date date = null;
+
         if (StringUtils.isNotBlank(formDTO.getDate())) {
+            //如果指定了参数,转化为yyyy-MM-dd格式
             date = DateUtils.stringToDate(formDTO.getDate(), DateUtils.DATE_PATTERN);
         }
         if (StringUtils.isNotBlank(formDTO.getCustomerId())) {
             generate(formDTO.getCustomerId(), date);
         } else {
             do {
+                //每100个客户一组
                 customerIdList = dimCustomerService.selectCustomerIdPage(pageNo++, pageSize);
                 if (!CollectionUtils.isEmpty(customerIdList)) {
                     for (String customerId : customerIdList) {
@@ -90,6 +99,7 @@ public class StatsUserServiceImpl implements StatsUserService {
      * @date 2020.06.28 14:40
     **/
     void generate(String customerId,Date date){
+        log.info("customerId:"+customerId+";date:"+DateUtils.format(date,DateUtils.DATE_TIME_PATTERN));
         //1.初始化时间参数
         Calendar calendar = Calendar.getInstance();
         calendar.setTime(new Date());
@@ -99,13 +109,11 @@ public class StatsUserServiceImpl implements StatsUserService {
         calendar.set(Calendar.MINUTE, NumConstant.ZERO);
         calendar.set(Calendar.SECOND, NumConstant.ZERO);
 
-        //2.初始化时间维度
+        //2.初始化时间维度{"dateId":"20210325","monthId":"202103","quarterId":"2021Q1","weekId":"2021W13","yearId":"2021"}
         DimIdGenerator.DimIdBean timeDimension = DimIdGenerator.getDimIdBean(null == date ? calendar.getTime() : date);
-
+        log.info("timeDimension:"+ JSON.toJSONString(timeDimension));
         //3.初始化机关维度
         List<AgencySubTreeDto> agencies =  dimAgencyService.getAllAgency(customerId);
-        //List<AgencySubTreeDto> topAgencies = dimAgencyService.getTopAgency(customerId);
-
         //4.计算机关统计数据、生成唯一性统计数据
         try {
             UserStatisticalData agencyData = userService.traverseAgencyUser(agencies, date, timeDimension);

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/user/impl/UserServiceImpl.java

@@ -62,8 +62,8 @@ public class UserServiceImpl implements UserService {
         dataPacket.setDateId(timeDimension.getDateId());
         dataPacket.setMonthId(timeDimension.getMonthId());
         //自上向下检索
-
         Map<String, List<AgencySubTreeDto>> agencyMap = agencies.stream().collect(Collectors.groupingBy(AgencySubTreeDto::getAgencyId));
+        //subGridOfAgency key:agencyId ,value:当前组织下的所有网格（即直属网格+下级组织下的所有网格）
         Map<String,Set<String>> subGridOfAgency = new HashMap<>();
         agencies.forEach(agency -> {
             initAgencyGridMap(agency.getAgencyId(),agency,subGridOfAgency);
@@ -71,12 +71,13 @@ public class UserServiceImpl implements UserService {
 
         //对每一个机关进行数据统计
         if(subGridOfAgency.size() > NumConstant.ZERO){
-            subGridOfAgency.forEach((k,v) -> {
+            //如果当前客户存在组织
+            subGridOfAgency.forEach((agencyId,subGridIds) -> {
                 queryUserData(ModuleConstant.DIM_SUB_AGENCY,
-                        k,
-                        (null == agencyMap.get(k) || agencyMap.get(k).isEmpty()) ? NumConstant.ZERO_STR : agencyMap.get(k).get(NumConstant.ZERO).getPid(),
-                        (null == agencyMap.get(k) || agencyMap.get(k).isEmpty()) ? null : agencyMap.get(k).get(NumConstant.ZERO).getCustomerId(),
-                        v,
+                        agencyId,
+                        // (null == agencyMap.get(agencyId) || agencyMap.get(agencyId).isEmpty()) ? NumConstant.ZERO_STR : agencyMap.get(agencyId).get(NumConstant.ZERO).getPid(),
+                        (null == agencyMap.get(agencyId) || agencyMap.get(agencyId).isEmpty()) ? null : agencyMap.get(agencyId).get(NumConstant.ZERO).getCustomerId(),
+                        subGridIds,
                         targetDate,
                         dataPacket,
                         timeDimension);
@@ -107,11 +108,11 @@ public class UserServiceImpl implements UserService {
         //自上向下检索
         Map<String, List<AgencySubTreeDto>> agencyMap = agencies.stream().collect(Collectors.groupingBy(AgencySubTreeDto::getAgencyId));
         if(null != agencyMap && agencyMap.size() > NumConstant.ZERO){
-            agencyMap.forEach((k,v) -> {
+            agencyMap.forEach((agencyId,v) -> {
                 if(null != v && v.size() > NumConstant.ZERO){
                     queryUserData(ModuleConstant.DIM_BELONGING_GRID,
-                            k,
-                            v.get(NumConstant.ZERO).getPid(),
+                            agencyId,
+                            // v.get(NumConstant.ZERO).getPid(),
                             v.get(NumConstant.ZERO).getCustomerId(),
                             new HashSet<>(v.get(NumConstant.ZERO).getGridIds()),
                             targetDate,
@@ -181,22 +182,22 @@ public class UserServiceImpl implements UserService {
 
     /**
      * @Description 初始化机关-所有下级网格Map
-     * @param pid - 固定一个机关Id
+     * @param agencyId - 当前组织id
      * @param agency - AgencySubTreeDto
      * @param subGridOfAgency - Map<String,Set<String>>
      * @return
      * @author wangc
      * @date 2020.06.18 15:54
      **/
-    void initAgencyGridMap(String pid, AgencySubTreeDto agency, Map<String,Set<String>> subGridOfAgency){
+    void initAgencyGridMap(String agencyId, AgencySubTreeDto agency, Map<String,Set<String>> subGridOfAgency){
 
         //向map中放入数据
-        if(subGridOfAgency.containsKey(pid)){
+        if(subGridOfAgency.containsKey(agencyId)){
             //包含key
-            Set<String> grids = subGridOfAgency.get(pid);
+            Set<String> grids = subGridOfAgency.get(agencyId);
             if(null == grids){
                 grids = new HashSet<>();
-                subGridOfAgency.put(pid,grids);
+                subGridOfAgency.put(agencyId,grids);
             }
             if(null != agency.getGridIds() && agency.getGridIds().size() > NumConstant.ZERO){
                 grids.addAll(agency.getGridIds());
@@ -204,18 +205,20 @@ public class UserServiceImpl implements UserService {
         }else{
             //不包含key
             Set<String> grids = new HashSet<>(agency.getGridIds());
-            subGridOfAgency.put(pid,grids);
+            subGridOfAgency.put(agencyId,grids);
         }
-
+        //外层是从顶级组织向下循环，所以循环到社区时跳出
         //定义递归出口
-        if(StringUtils.equals(ModuleConstant.AGENCY_LEVEL_COMMUNITY,agency.getLevel()) || null == agency.getSubAgencies() || agency.getSubAgencies().size() == NumConstant.ZERO){
-            return ;
+        if (StringUtils.equals(ModuleConstant.AGENCY_LEVEL_COMMUNITY, agency.getLevel())
+                || null == agency.getSubAgencies()
+                || agency.getSubAgencies().size() == NumConstant.ZERO) {
+            return;
         }
 
 
         //定义递归入口
         agency.getSubAgencies().forEach(obj -> {
-            initAgencyGridMap(pid,obj,subGridOfAgency);
+            initAgencyGridMap(agencyId,obj,subGridOfAgency);
         });
     }
 
@@ -223,7 +226,7 @@ public class UserServiceImpl implements UserService {
      * @Description 执行查询用户数据统计的逻辑
      * @param relation - agency(下级机关加网格) | grid(直属网格)
      * @param agencyId
-     * @param pid
+     * @param //pid 当前agency的父级组织id
      * @param customerId
      * @param gridIds  -  机关下所有网格集合/机关下直属网格集合
      * @param targetDate
@@ -233,10 +236,12 @@ public class UserServiceImpl implements UserService {
      * @author wangc
      * @date 2020.06.19 10:01
     **/
-    void queryUserData(String relation, String agencyId, String pid, String customerId, Set<String> gridIds, Date targetDate, UserStatisticalData dataPacket, DimIdGenerator.DimIdBean timeDimension){
-
-        if(StringUtils.isBlank(pid))
-            pid = NumConstant.ZERO_STR;
+    void queryUserData(String relation, String agencyId,
+                       // String pid,
+                       String customerId, Set<String> gridIds, Date targetDate, UserStatisticalData dataPacket, DimIdGenerator.DimIdBean timeDimension){
+        //pid没用到还传过来干啥-- 注释掉
+        // if(StringUtils.isBlank(pid))
+        //     pid = NumConstant.ZERO_STR;
 
         dataPacket.setCustomerId(customerId);
 
@@ -253,6 +258,9 @@ public class UserServiceImpl implements UserService {
 
         //指定日期 OR T-1
         calendar.setTime(targetDateCheck);
+        //calendar.get(Calendar.DATE) 当前日期数eg:2021-03-29  ：29
+        //calendar.getActualMinimum(Calendar.DAY_OF_MONTH)  :1
+
         //如果目标日期不是是当月的第一天
         if(calendar.get(Calendar.DATE) != calendar.getActualMinimum(Calendar.DAY_OF_MONTH)){
             //求出这个月的第一天
@@ -264,7 +272,6 @@ public class UserServiceImpl implements UserService {
 
             isMonthBeginning = false;
         }
-
         if(StringUtils.equals(ModuleConstant.DIM_SUB_AGENCY,relation)){
 
 
@@ -396,8 +403,6 @@ public class UserServiceImpl implements UserService {
             partiAgencyM.setWarmHeartedProportion(partiAgencyD.getWarmHeartedProportion());
             partiAgencyM.setCreatedBy(ModuleConstant.CREATED_BY_STATISTICAL_ROBOT);
 
-
-
             //如果是月初第一天，不再做日期区间查询
             if(isMonthBeginning) {
                 if (null == dataPacket.getRegAgencyMonthlyList()) {
@@ -416,6 +421,7 @@ public class UserServiceImpl implements UserService {
                 }
             }else{
                 //如果不是月初第一天
+                //targetDateCheck：定时任务参数中的date;calendar.getTime():date所属月第一天；
 
                 //本月注册用户增长数
                 Integer regIncrMonthly =  userDao.selectResiIncrWithinTimeRange(ModuleConstant.REG_OR_PARTI_FLAG_REG,gridIds,calendar.getTime(),targetDateCheck);

epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/user/UserDao.xml

@@ -40,11 +40,13 @@
     </choose>
 
     <choose>
+        <!--  不为空，默认查询创建时间在 昨天的增量 -->
         <when test='null != targetDate'>
-            AND CREATED_TIME <![CDATA[>=]]> #{targetDate} AND CREATED_TIME <![CDATA[<]]> DATE_SUB( #{targetDate}, INTERVAL - 1 DAY)
+            and DATE_FORMAT(CREATED_TIME,"%Y-%m-%d") = DATE_FORMAT(#{targetDate},"%Y-%m-%d")
         </when>
+        <!--  为空，默认查询昨天的增量 -->
         <otherwise>
-            AND CREATED_TIME <![CDATA[<]]>  CURDATE( ) AND CREATED_TIME <![CDATA[>=]]> DATE_SUB( CURDATE( ), INTERVAL 1 DAY )
+            and DATE_FORMAT(CREATED_TIME,"%Y-%m-%d") = DATE_FORMAT( DATE_SUB( CURDATE( ), INTERVAL 1 DAY ),"%Y-%m-%d")
         </otherwise>
     </choose>
     ) AS incr
@@ -76,11 +78,13 @@
         </otherwise>
     </choose>
     <choose>
+        <!--  不为空，包含传进来的这一天 -->
         <when test="null != targetDate">
-            AND CREATED_TIME <![CDATA[<]]> DATE_SUB( #{targetDate}, INTERVAL - 1 DAY )
+            and DATE_FORMAT(CREATED_TIME,"%Y-%m-%d")  <![CDATA[<=]]> DATE_FORMAT(#{targetDate},"%Y-%m-%d")
         </when>
+        <!--  为空，默认截止到昨天 -->
         <otherwise>
-            AND CREATED_TIME <![CDATA[<]]> CURDATE( )
+            and DATE_FORMAT(CREATED_TIME,"%Y-%m-%d")  <![CDATA[<=]]> DATE_FORMAT(DATE_SUB( CURDATE( ), INTERVAL 1 DAY ),"%Y-%m-%d")
         </otherwise>
     </choose>
 
@@ -121,10 +125,10 @@
 
     <choose>
         <when test='null != targetDate'>
-            AND CREATED_TIME <![CDATA[>=]]> #{targetDate} AND CREATED_TIME <![CDATA[<]]> DATE_SUB( #{targetDate}, INTERVAL - 1 DAY)
+            and DATE_FORMAT(CREATED_TIME,"%Y-%m-%d") = DATE_FORMAT(#{targetDate},"%Y-%m-%d")
         </when>
         <otherwise>
-            AND CREATED_TIME <![CDATA[<]]>  CURDATE( ) AND CREATED_TIME <![CDATA[>=]]> DATE_SUB( CURDATE( ), INTERVAL  1 DAY )
+            and DATE_FORMAT(CREATED_TIME,"%Y-%m-%d") = DATE_FORMAT(DATE_SUB( CURDATE( ), INTERVAL 1 DAY ),"%Y-%m-%d")
         </otherwise>
     </choose>
 </select>
@@ -164,10 +168,10 @@
 
     <choose>
         <when test='targetDate != null'>
-            AND CREATED_TIME <![CDATA[<]]> DATE_SUB( #{targetDate}, INTERVAL - 1 DAY )
+            and DATE_FORMAT(CREATED_TIME,"%Y-%m-%d")  <![CDATA[<=]]> DATE_FORMAT(#{targetDate},"%Y-%m-%d")
         </when>
         <otherwise>
-            AND CREATED_TIME <![CDATA[<]]> CURDATE( )
+            and DATE_FORMAT(CREATED_TIME,"%Y-%m-%d")  <![CDATA[<=]]> DATE_FORMAT(DATE_SUB( CURDATE( ), INTERVAL 1 DAY ),"%Y-%m-%d")
         </otherwise>
     </choose>
 </select>
@@ -190,10 +194,10 @@
     </foreach>
     <choose>
         <when test='null != targetDate'>
-            AND urole.CREATED_TIME <![CDATA[>=]]> #{targetDate} AND urole.CREATED_TIME <![CDATA[<]]> DATE_SUB( #{targetDate}, INTERVAL - 1 DAY)
+            and DATE_FORMAT(urole.CREATED_TIME,"%Y-%m-%d") = DATE_FORMAT(#{targetDate},"%Y-%m-%d")
         </when>
         <otherwise>
-            AND urole.CREATED_TIME <![CDATA[<]]>  CURDATE( ) AND urole.CREATED_TIME <![CDATA[>=]]> DATE_SUB( CURDATE( ), INTERVAL 1 DAY )
+            and DATE_FORMAT(urole.CREATED_TIME,"%Y-%m-%d") = DATE_FORMAT(DATE_SUB( CURDATE( ), INTERVAL 1 DAY ),"%Y-%m-%d")
         </otherwise>
     </choose>
 
@@ -221,10 +225,10 @@
 
     <choose>
         <when test='null != targetDate'>
-            AND urole.CREATED_TIME <![CDATA[<]]> DATE_SUB( #{targetDate}, INTERVAL - 1 DAY )
+            and DATE_FORMAT( urole.CREATED_TIME,"%Y-%m-%d")  <![CDATA[<=]]> DATE_FORMAT(#{targetDate},"%Y-%m-%d")
         </when>
         <otherwise>
-            AND urole.CREATED_TIME <![CDATA[<]]> CURDATE( )
+            and DATE_FORMAT(urole.CREATED_TIME,"%Y-%m-%d")  <![CDATA[<=]]> DATE_FORMAT(DATE_SUB( CURDATE( ), INTERVAL 1 DAY ),"%Y-%m-%d")
         </otherwise>
     </choose>
     AND erole.ROLE_NAME = '党员'
@@ -261,10 +265,10 @@
 
     <choose>
         <when test='null != targetDate'>
-            AND urole.CREATED_TIME <![CDATA[>=]]> #{targetDate} AND urole.CREATED_TIME <![CDATA[<]]> DATE_SUB( #{targetDate}, INTERVAL - 1 DAY)
+            and DATE_FORMAT(urole.CREATED_TIME,"%Y-%m-%d")=DATE_FORMAT(#{targetDate},"%Y-%m-%d")
         </when>
         <otherwise>
-            AND urole.CREATED_TIME <![CDATA[<]]>  CURDATE( ) AND urole.CREATED_TIME <![CDATA[>=]]> DATE_SUB( CURDATE( ), INTERVAL 1 DAY )
+            and DATE_FORMAT(urole.CREATED_TIME,"%Y-%m-%d")=DATE_FORMAT(DATE_SUB( CURDATE( ), INTERVAL 1 DAY ),"%Y-%m-%d")
         </otherwise>
     </choose>
     AND erole.ROLE_NAME = '热心居民'
@@ -290,10 +294,10 @@
     </foreach>
     <choose>
         <when test='null != targetDate'>
-            AND urole.CREATED_TIME <![CDATA[<]]> DATE_SUB( #{targetDate}, INTERVAL - 1 DAY )
+            and DATE_FORMAT(urole.CREATED_TIME,"%Y-%m-%d")  <![CDATA[<=]]> DATE_FORMAT(#{targetDate},"%Y-%m-%d")
         </when>
         <otherwise>
-            AND urole.CREATED_TIME <![CDATA[<]]> CURDATE( )
+            and DATE_FORMAT(urole.CREATED_TIME,"%Y-%m-%d")  <![CDATA[<=]]> DATE_FORMAT(DATE_SUB( CURDATE( ), INTERVAL 1 DAY ),"%Y-%m-%d")
         </otherwise>
     </choose>
     AND erole.ROLE_NAME = '热心居民';
@@ -332,8 +336,8 @@
             </foreach>
         </otherwise>
     </choose>
-
-    AND CREATED_TIME <![CDATA[>=]]> #{startDate} AND CREATED_TIME <![CDATA[<]]> DATE_SUB( #{endDate}, INTERVAL - 1 DAY)
+    AND CREATED_TIME <![CDATA[>=]]>  DATE_FORMAT(#{startDate},"%Y-%m-%d")
+    AND CREATED_TIME <![CDATA[<=]]>  DATE_FORMAT(#{endDate},"%Y-%m-%d")
 </select>
 
     <!-- 查询指定时间范围内居民增量 传参:注册用户or参与用户   |  网格Id集合  |  时间区间-->
@@ -368,10 +372,8 @@
             </foreach>
         </otherwise>
     </choose>
-
-
-    AND CREATED_TIME <![CDATA[>=]]> #{startDate} AND CREATED_TIME <![CDATA[<]]> DATE_SUB( #{endDate}, INTERVAL - 1 DAY)
-
+    AND CREATED_TIME <![CDATA[>=]]> DATE_FORMAT(#{startDate},"%Y-%m-%d")
+    AND CREATED_TIME <![CDATA[<=]]> DATE_FORMAT(#{endDate},"%Y-%m-%d")
 </select>
 
     <!-- 查询指定时间范围内党员的增量 传参:用户Id集合 | 时间区间 -->
@@ -394,9 +396,8 @@
             </foreach>
         </otherwise>
     </choose>
-
-
-    AND urole.CREATED_TIME <![CDATA[>=]]> #{startDate} AND urole.CREATED_TIME <![CDATA[<]]> DATE_SUB( #{endDate}, INTERVAL - 1 DAY)
+    AND urole.CREATED_TIME <![CDATA[>=]]> DATE_FORMAT(#{startDate},"%Y-%m-%d")
+    AND urole.CREATED_TIME <![CDATA[<=]]> DATE_FORMAT(#{endDate},"%Y-%m-%d")
     AND erole.ROLE_NAME = '党员'
 </select>
 
@@ -423,7 +424,8 @@
             </foreach>
         </otherwise>
     </choose>
-    AND urole.CREATED_TIME <![CDATA[>=]]> #{startDate} AND urole.CREATED_TIME <![CDATA[<]]> DATE_SUB( #{endDate}, INTERVAL - 1 DAY)
+    AND urole.CREATED_TIME <![CDATA[>=]]> DATE_FORMAT(#{startDate},"%Y-%m-%d")
+    AND urole.CREATED_TIME <![CDATA[<=]]> DATE_FORMAT(#{endDate},"%Y-%m-%d")
     AND erole.ROLE_NAME = '热心居民'
 </select>