网格人员类别分析

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/user/UserDao.java

@@ -278,5 +278,10 @@ public interface UserDao {
                                  @Param("agencyId") String agencyId);
 
     List<OrgIcUserTotalDTO> selectGridIcUserCategoryTotal(@Param("customerId")String customerId, @Param("dateId")String dateId);
+
+    List<OrgIcUserTotalDTO> selectGridIcUserCategoryIncr(@Param("customerId")String customerId,
+                                                         @Param("startDate")String startDate,
+                                                         @Param("endDate")String endDate,
+                                                         @Param("type")String type);
 }
 

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/entity/stats/FactIcuserCategoryAnalysisDailyEntity.java

@@ -27,7 +27,10 @@ public class FactIcuserCategoryAnalysisDailyEntity extends BaseEpmetEntity {
      * 预警配置ID
      */
 	private String configId;
-
+	/**
+	 * 类别列名，对个数据太麻烦了，在这个表存一下
+	 */
+	private String columnName;
     /**
      * 当前数据是截止到XXX的：yyyyMMdd
      */

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/screen/ScreenCustomerGridService.java

@@ -19,7 +19,6 @@ package com.epmet.service.evaluationindex.screen;
 
 
 import com.epmet.commons.mybatis.service.BaseService;
-import com.epmet.dto.CustomerGridDTO;
 import com.epmet.dto.extract.form.*;
 import com.epmet.dto.extract.result.GridInfoResultDTO;
 import com.epmet.dto.indexcollect.form.CustomerBizOrgFormDTO;
@@ -138,5 +137,5 @@ public interface ScreenCustomerGridService extends BaseService<ScreenCustomerGri
      */
     Date getLastSyncEndTime();
 
-    List<CustomerGridDTO> getByCustomerId(String customerId);
+    List<ScreenCustomerGridEntity> getByCustomerId(String customerId);
 }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/screen/impl/ScreenCustomerGridServiceImpl.java

@@ -29,7 +29,6 @@ import com.epmet.constant.DataSourceConstant;
 import com.epmet.constant.OrgSourceTypeConstant;
 import com.epmet.dao.evaluationindex.screen.DimLastSyncRecordDao;
 import com.epmet.dao.evaluationindex.screen.ScreenCustomerGridDao;
-import com.epmet.dto.CustomerGridDTO;
 import com.epmet.dto.extract.form.*;
 import com.epmet.dto.extract.result.GridInfoResultDTO;
 import com.epmet.dto.indexcollect.form.CustomerBizOrgFormDTO;
@@ -293,10 +292,9 @@ public class ScreenCustomerGridServiceImpl extends BaseServiceImpl<ScreenCustome
     }
 
     @Override
-    public List<CustomerGridDTO> getByCustomerId(String customerId) {
+    public List<ScreenCustomerGridEntity> getByCustomerId(String customerId) {
         LambdaQueryWrapper<ScreenCustomerGridEntity> wrapper = new LambdaQueryWrapper<>();
         wrapper.eq(ScreenCustomerGridEntity::getCustomerId, customerId);
-        List<ScreenCustomerGridEntity> list = baseDao.selectList(wrapper);
+        return baseDao.selectList(wrapper);
-        return ConvertUtils.sourceToTarget(list,CustomerGridDTO.class);
     }
 }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/stats/impl/FactIcuserCategoryAnalysisDailyServiceImpl.java

@@ -6,15 +6,16 @@ import com.epmet.commons.tools.constant.NumConstant;
 import com.epmet.commons.tools.enums.OrgTypeEnum;
 import com.epmet.constant.DataSourceConstant;
 import com.epmet.dao.stats.FactIcuserCategoryAnalysisDailyDao;
-import com.epmet.dto.CustomerGridDTO;
 import com.epmet.dto.customize.IcResiCategoryConfigDTO;
 import com.epmet.dto.user.result.OrgIcUserTotalDTO;
+import com.epmet.entity.evaluationindex.screen.ScreenCustomerGridEntity;
 import com.epmet.entity.stats.FactIcuserCategoryAnalysisDailyEntity;
 import com.epmet.service.customize.IcResiCategoryStatsConfigService;
 import com.epmet.service.evaluationindex.screen.ScreenCustomerAgencyService;
 import com.epmet.service.evaluationindex.screen.ScreenCustomerGridService;
 import com.epmet.service.stats.FactIcuserCategoryAnalysisDailyService;
 import com.epmet.service.user.UserService;
+import com.google.common.collect.Lists;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.MapUtils;
@@ -45,7 +46,8 @@ public class FactIcuserCategoryAnalysisDailyServiceImpl extends BaseServiceImpl<
     @Autowired
     private ScreenCustomerGridService gridService;
     @Autowired
-    private  UserService userService;
+    private UserService userService;
+
     /**
      * 数字社区-基础信息-人员类别分析-每个类别的人员总数、较上月迁入迁出人数
      *
@@ -55,99 +57,142 @@ public class FactIcuserCategoryAnalysisDailyServiceImpl extends BaseServiceImpl<
     @Override
     public void statUserCategoryGridDaily(String customerId, String dateId) {
         //查询出所有用语数据分析的类别column_name,没有则不计算
-        List<IcResiCategoryConfigDTO> columnList=icResiCategoryStatsConfigService.queryDataColumn(customerId);
+        List<IcResiCategoryConfigDTO> columnList = icResiCategoryStatsConfigService.queryDataColumn(customerId);
-        if(CollectionUtils.isEmpty(columnList)){
+        if (CollectionUtils.isEmpty(columnList)) {
+            return;
+        }
+        //所有网格查询出来
+        List<ScreenCustomerGridEntity> gridDTOList = gridService.getByCustomerId(customerId);
+        if (CollectionUtils.isEmpty(gridDTOList)) {
             return;
         }
-        // //所有网格和组织查询出来
-        // List<CustomerAgencyDTO> agencyDTOList=agencyService.getByCustomerId(customerId);
-        List<CustomerGridDTO> gridDTOList=gridService.getByCustomerId(customerId);
         //网格维度、组织维度初始好数据
-        List<FactIcuserCategoryAnalysisDailyEntity> initList=constructInitValue(customerId,dateId,gridDTOList,columnList);
+        List<FactIcuserCategoryAnalysisDailyEntity> initList = constructInitValue(customerId, dateId, gridDTOList, columnList);
-        if(CollectionUtils.isEmpty(initList)){
+        if (CollectionUtils.isEmpty(initList)) {
             return;
         }
-        // 批量插入数据
+        //当这3个数都为0的时候不插入数据库了
-
+        List<FactIcuserCategoryAnalysisDailyEntity> insertList = initList.stream().filter(entity -> (null != entity.getTotal() && entity.getTotal() > 0)
-
+                || (null != entity.getQcIncr() && entity.getQcIncr() > 0)
+                || (null != entity.getQrIncr() && entity.getQrIncr() > 0)).collect(Collectors.toList());
         //删除旧数据
-        clearOldDatas(customerId,dateId,OrgTypeEnum.GRID.getCode());
+        clearOldDatas(customerId, dateId, OrgTypeEnum.GRID.getCode());
+        // 批量插入数据
+        Lists.partition(insertList, NumConstant.ONE_HUNDRED).forEach(list -> {
+            insertBatch(list);
+        });
     }
 
-    private void clearOldDatas(String customerId, String dateId,String orgType) {
+    /**
+     * 删除旧数据
+     *
+     * @param customerId
+     * @param dateId
+     * @param orgType    grid、agency
+     */
+    private void clearOldDatas(String customerId, String dateId, String orgType) {
         int deleteNum;
         do {
             //每次删除1000条
-            deleteNum = baseDao.limitDelete(customerId,dateId,orgType);
+            deleteNum = baseDao.limitDelete(customerId, dateId, orgType);
         } while (deleteNum != NumConstant.ZERO);
     }
 
     /**
-     *
      * @param customerId
-     * @param dateId yyyyMMdd
+     * @param dateId      yyyyMMdd
      * @param gridDTOList 当前客户下所有的网格
-     * @param columnList 所有需要分析的类别对应的ic_resi_user的列名，目前只有18类
+     * @param columnList  所有需要分析的类别对应的ic_resi_user的列名，目前只有18类
      * @return
      */
-    private List<FactIcuserCategoryAnalysisDailyEntity> constructInitValue(String customerId,String dateId, List<CustomerGridDTO> gridDTOList,List<IcResiCategoryConfigDTO> columnList) {
+    private List<FactIcuserCategoryAnalysisDailyEntity> constructInitValue(String customerId, String dateId, List<ScreenCustomerGridEntity> gridDTOList, List<IcResiCategoryConfigDTO> columnList) {
-        List<FactIcuserCategoryAnalysisDailyEntity> list=new ArrayList<>();
+        List<FactIcuserCategoryAnalysisDailyEntity> list = new ArrayList<>();
-        if(CollectionUtils.isEmpty(gridDTOList)){
-            //没有配置的类别，退出吧
-            return list;
-        }
 
         //1、total
         // 查询明细ic_user_change_detailed变更明细表，各个网格的各个分类的总人数
-        Map<String, List<OrgIcUserTotalDTO>> totalMapList=userService.calGridIcUserTotal(customerId,dateId);
+        Map<String, List<OrgIcUserTotalDTO>> totalMapList = userService.calGridIcUserTotal(customerId, dateId);
 
-        String monthId=dateId.substring(NumConstant.ZERO,NumConstant.SIX);
+        String monthId = dateId.substring(NumConstant.ZERO, NumConstant.SIX);
-        String startDate=monthId.concat("01");
+        String startDate = monthId.concat("01");
-        String endDate=dateId;
+        String endDate = dateId;
 
         //2、迁入+30：eg:dateId:20211226,  [20211201,20211226]，类别sum值=1的
-        Map<String, List<OrgIcUserTotalDTO>> qrList=userService.calGridIcUserQr(customerId,dateId);
+        Map<String, List<OrgIcUserTotalDTO>> qrList = userService.calGridIcUserIncr(customerId, startDate, endDate, "qr");
 
         //3、迁出 -10 :eg:dateId:20211226,  [20211201,20211226]，类别sum值 !=1的
-        Map<String, List<OrgIcUserTotalDTO>> qrList=userService.calGridIcUserQr(customerId,dateId);
+        Map<String, List<OrgIcUserTotalDTO>> qcList = userService.calGridIcUserIncr(customerId, startDate, endDate, "qc");
 
-        for(IcResiCategoryConfigDTO config:columnList){
+        for (IcResiCategoryConfigDTO config : columnList) {
-            gridDTOList.forEach(grid->{
+            gridDTOList.forEach(grid -> {
-                FactIcuserCategoryAnalysisDailyEntity gridData=new FactIcuserCategoryAnalysisDailyEntity();
+                FactIcuserCategoryAnalysisDailyEntity gridData = new FactIcuserCategoryAnalysisDailyEntity();
                 gridData.setCustomerId(customerId);
                 gridData.setConfigId(config.getConfigId());
+                gridData.setColumnName(config.getColumnName());
                 gridData.setDateId(dateId);
-                gridData.setOrgId(grid.getId());
+                gridData.setOrgId(grid.getGridId());
                 gridData.setOrgType(OrgTypeEnum.GRID.getCode());
-                gridData.setPid(StringUtils.isNotBlank(grid.getPid())?grid.getPid(): NumConstant.ZERO_STR);
+                gridData.setPid(StringUtils.isNotBlank(grid.getParentAgencyId()) ? grid.getParentAgencyId() : NumConstant.ZERO_STR);
-                gridData.setPids(grid.getPids());
+                gridData.setPids(grid.getAllParentIds());
 
-                boolean zeroFlag=true;
+                //截止到目前累计值
-                if(MapUtils.isNotEmpty(totalMapList)&&totalMapList.containsKey(grid.getId())){
+                boolean totalZeroFlag = true;
+                if (MapUtils.isNotEmpty(totalMapList) && totalMapList.containsKey(grid.getGridId())) {
                     //当前网格下，有的类别
-                    List<OrgIcUserTotalDTO> crrentGrid=totalMapList.get(grid.getId());
+                    List<OrgIcUserTotalDTO> crrentGrid = totalMapList.get(grid.getGridId());
-                    if(CollectionUtils.isNotEmpty(crrentGrid)){
+                    if (CollectionUtils.isNotEmpty(crrentGrid)) {
                         //各个类别对应的数
-                        Map<String,Integer> categoryTotalMap=crrentGrid.stream().collect(Collectors.toMap(OrgIcUserTotalDTO::getColumnName, OrgIcUserTotalDTO::getTotal, (key1, key2) -> key2));
+                        Map<String, Integer> categoryTotalMap = crrentGrid.stream().collect(Collectors.toMap(OrgIcUserTotalDTO::getColumnName, OrgIcUserTotalDTO::getTotal, (key1, key2) -> key2));
-                        if(MapUtils.isNotEmpty(categoryTotalMap)&&categoryTotalMap.containsKey(config.getColumnName())){
+                        if (MapUtils.isNotEmpty(categoryTotalMap) && categoryTotalMap.containsKey(config.getColumnName())) {
-                            zeroFlag=false;
+                            totalZeroFlag = false;
                             gridData.setTotal(categoryTotalMap.get(config.getColumnName()));
                         }
                     }
                 }
-                if(zeroFlag){
+                if (totalZeroFlag) {
                     gridData.setTotal(NumConstant.ZERO);
                 }
-                gridData.setQcIncr(NumConstant.ZERO);
+
-                gridData.setQrIncr(NumConstant.ZERO);
+                //较上月迁入
+                boolean incrZeroFlag = true;
+                if (MapUtils.isNotEmpty(qrList) && qrList.containsKey(grid.getGridId())) {
+                    //当前网格下，有的类别
+                    List<OrgIcUserTotalDTO> crrentGrid = qrList.get(grid.getGridId());
+                    if (CollectionUtils.isNotEmpty(crrentGrid)) {
+                        //各个类别对应的数
+                        Map<String, Integer> categoryTotalMap = crrentGrid.stream().collect(Collectors.toMap(OrgIcUserTotalDTO::getColumnName, OrgIcUserTotalDTO::getTotal, (key1, key2) -> key2));
+                        if (MapUtils.isNotEmpty(categoryTotalMap) && categoryTotalMap.containsKey(config.getColumnName())) {
+                            incrZeroFlag = false;
+                            gridData.setQrIncr(categoryTotalMap.get(config.getColumnName()));
+                        }
+                    }
+                }
+                if (incrZeroFlag) {
+                    gridData.setQrIncr(NumConstant.ZERO);
+                }
+
+                // 较上月迁出：
+                boolean qcZeroFlag = true;
+                if (MapUtils.isNotEmpty(qcList) && qcList.containsKey(grid.getGridId())) {
+                    //当前网格下，有的类别
+                    List<OrgIcUserTotalDTO> crrentGridQc = qcList.get(grid.getGridId());
+                    if (CollectionUtils.isNotEmpty(crrentGridQc)) {
+                        //各个类别对应的数
+                        Map<String, Integer> categoryTotalMap = crrentGridQc.stream().collect(Collectors.toMap(OrgIcUserTotalDTO::getColumnName, OrgIcUserTotalDTO::getTotal, (key1, key2) -> key2));
+                        if (MapUtils.isNotEmpty(categoryTotalMap) && categoryTotalMap.containsKey(config.getColumnName())) {
+                            qcZeroFlag = false;
+                            gridData.setQcIncr(categoryTotalMap.get(config.getColumnName()));
+                        }
+                    }
+                }
+                if (qcZeroFlag) {
+                    gridData.setQcIncr(NumConstant.ZERO);
+                }
+
                 list.add(gridData);
             });
         }
         return list;
     }
 
-    public static void main(String[] args) {
+
-        String dateId="20211225";
-        System.out.println(dateId.substring(0,6));
-    }
 }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/user/UserService.java

@@ -176,4 +176,14 @@ public interface UserService {
      * @return
      */
     Map<String, List<OrgIcUserTotalDTO>> calGridIcUserTotal(String customerId, String dateId);
+
+    /**
+     *
+     * @param customerId
+     * @param startDate ：变动时间在此范围内的 yyyyMMdd
+     * @param endDate  :yyyyMMdd
+     * @param type:qr:迁入；qc:迁出
+     * @return
+     */
+    Map<String, List<OrgIcUserTotalDTO>> calGridIcUserIncr(String customerId, String startDate, String endDate, String type);
 }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/user/impl/UserServiceImpl.java

@@ -1201,4 +1201,30 @@ public class UserServiceImpl implements UserService {
         return resultMap;
     }
 
+    /**
+     * @param customerId
+     * @param startDate  ：变动时间在此范围内的 yyyyMMdd
+     * @param endDate    :yyyyMMdd
+     * @param type       :qr:迁入；qc:迁出
+     * @return
+     */
+    @Override
+    public Map<String, List<OrgIcUserTotalDTO>> calGridIcUserIncr(String customerId, String startDate, String endDate, String type) {
+        Map<String, List<OrgIcUserTotalDTO>> resultMap=new HashMap<>();
+        List<OrgIcUserTotalDTO> list=userDao.selectGridIcUserCategoryIncr(customerId,startDate,endDate,type);
+        if(CollectionUtils.isEmpty(list)){
+            return resultMap;
+        }
+        for(OrgIcUserTotalDTO dto:list){
+            if(resultMap.containsKey(dto.getOrgId())){
+                resultMap.get(dto.getOrgId()).add(dto);
+                continue;
+            }
+            List<OrgIcUserTotalDTO> newList=new ArrayList<>();
+            newList.add(dto);
+            resultMap.put(dto.getOrgId(),newList);
+        }
+        return resultMap;
+    }
+
 }

epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/user/UserDao.xml

@@ -1107,7 +1107,7 @@
         select
             temp.orgId,
             temp.FIELD_NAME as columnName,
-            count(distinct temp.IC_USER_ID) as total
+            count(temp.IC_USER_ID) as total
         from (
             SELECT
                 d.GRID_ID as orgId,
@@ -1120,9 +1120,40 @@
             d.DEL_FLAG = '0'
             AND d.CUSTOMER_ID = #{customerId}
             AND DATE_FORMAT( d.CREATED_TIME, '%Y%m%d' ) <= #{dateId}
-            group by d.orgId,d.IC_USER_ID,d.FIELD_NAME
+            group by d.GRID_ID,d.IC_USER_ID,d.FIELD_NAME
         )temp
         where temp.result=1
         group by temp.orgId,temp.FIELD_NAME
     </select>
+
+
+    <select id="selectGridIcUserCategoryIncr" parameterType="map" resultType="com.epmet.dto.user.result.OrgIcUserTotalDTO">
+        select
+            temp.orgId,
+            temp.FIELD_NAME as columnName,
+            count(temp.IC_USER_ID) as total
+        from (
+            SELECT
+                d.GRID_ID as orgId,
+                d.IC_USER_ID,
+                d.FIELD_NAME,
+                sum(d.`VALUE`)as result
+            FROM
+            ic_user_change_detailed d
+            WHERE
+            d.DEL_FLAG = '0'
+            AND d.CUSTOMER_ID = #{customerId}
+            AND DATE_FORMAT( d.CREATED_TIME, '%Y%m%d' ) &gt;= #{startDate}
+            AND DATE_FORMAT( d.CREATED_TIME, '%Y%m%d' ) &lt;= #{endDate}
+            group by d.GRID_ID,d.IC_USER_ID,d.FIELD_NAME
+        )temp
+        where 1=1
+        <if test="type != null and type == 'qr'">
+            and  temp.result=1
+        </if>
+        <if test="type != null and type == 'qc'">
+            and  temp.result !=1
+        </if>
+        group by temp.orgId,temp.FIELD_NAME
+    </select>
 </mapper>