党员数据抽取

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/controller/DemoController.java

@@ -637,8 +637,8 @@ public class DemoController {
     private PartyBaseInfoService partyBaseInfoService;
 
     @PostMapping("zxczxczxc")
-    public Result getZxcZxcZxc(){
+    public Result getZxcZxcZxc(@RequestBody CustomerIdAndDateIdFormDTO formDTO){
-        partyBaseInfoService.statsPartyMemberBaseInfoToScreen("45687aa479955f9d06204d415238f7cc","20200922");
+        partyBaseInfoService.statsPartyMemberBaseInfoToScreen(formDTO.getCustomerId(),formDTO.getDateId());
         return new Result();
     }
 

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/evaluationindex/screen/ScreenCustomerAgencyDao.java

@@ -18,6 +18,7 @@
 package com.epmet.dao.evaluationindex.screen;
 
 import com.epmet.commons.mybatis.dao.BaseDao;
+import com.epmet.dto.extract.form.PartyBaseInfoFormDTO;
 import com.epmet.dto.extract.result.CustomerAgencyInfoResultDTO;
 import com.epmet.dto.screen.result.TreeResultDTO;
 import com.epmet.dto.screencoll.form.CustomerAgencyFormDTO;
@@ -143,4 +144,12 @@ public interface ScreenCustomerAgencyDao extends BaseDao<ScreenCustomerAgencyEnt
      * @date 2020/9/23 11:11 上午
      */
     List<String> selectSubAgencyId(@Param("agencyId") String agencyId,@Param("customerId") String customerId);
+
+    /**
+     * @Description  查询客户下所有机关ID
+     * @param customerId
+     * @author zxc
+     * @date 2020/9/23 3:10 下午
+     */
+    List<PartyBaseInfoFormDTO> selectAllAgencyIdToParty(@Param("customerId")String customerId,@Param("dateId")String dateId);
 }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/evaluationindex/screen/ScreenCustomerGridDao.java

@@ -19,6 +19,7 @@ package com.epmet.dao.evaluationindex.screen;
 
 import com.epmet.commons.mybatis.dao.BaseDao;
 import com.epmet.dto.ScreenCustomerGridDTO;
+import com.epmet.dto.extract.form.PartyBaseInfoFormDTO;
 import com.epmet.dto.indexcal.PageQueryGridFormDTO;
 import com.epmet.dto.screencoll.form.CustomerGridFormDTO;
 import com.epmet.entity.evaluationindex.screen.ScreenCustomerGridEntity;
@@ -128,4 +129,13 @@ public interface ScreenCustomerGridDao extends BaseDao<ScreenCustomerGridEntity>
     ScreenCustomerGridEntity getLastUpdateGrid();
 
     ScreenCustomerGridEntity getByGridId(String gridId);
+
+    /**
+     * @Description  查询客户下所有网格ID
+     * @param customerId
+     * @param dateId
+     * @author zxc
+     * @date 2020/9/23 3:10 下午
+     */
+    List<PartyBaseInfoFormDTO> selectAllGridIdToParty(String customerId, String dateId);
 }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/extract/toscreen/impl/PartyBaseInfoServiceImpl.java

@@ -9,10 +9,13 @@ import com.epmet.dto.extract.result.UserCountResultDTO;
 import com.epmet.service.evaluationindex.extract.toscreen.PartyBaseInfoService;
 import com.epmet.service.evaluationindex.screen.ScreenCpcBaseDataService;
 import com.epmet.service.evaluationindex.screen.ScreenCustomerAgencyService;
+import com.epmet.service.evaluationindex.screen.ScreenCustomerGridService;
 import com.epmet.service.stats.DimCustomerPartymemberService;
 import com.epmet.service.stats.user.FactParticipationUserAgencyDailyService;
 import com.epmet.service.stats.user.FactParticipationUserGridDailyService;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.ListUtils;
+import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
@@ -39,6 +42,8 @@ public class PartyBaseInfoServiceImpl implements PartyBaseInfoService {
     @Autowired
     private ScreenCustomerAgencyService agencyService;
     @Autowired
+    private ScreenCustomerGridService gridService;
+    @Autowired
     private DimCustomerPartymemberService partyMemberService;
     @Autowired
     private FactParticipationUserGridDailyService userGridDailyService;
@@ -62,20 +67,22 @@ public class PartyBaseInfoServiceImpl implements PartyBaseInfoService {
             if (groupByLevel.containsKey(ScreenConstant.COMMUNITY)){
                 List<CustomerAgencyInfoResultDTO> customerGridInfoList = groupByLevel.get(ScreenConstant.COMMUNITY);
                 disPose(customerGridInfoList,true,customerId,dateId);
-            }else if (groupByLevel.containsKey(ScreenConstant.STREET)){
+            }
+            if (groupByLevel.containsKey(ScreenConstant.STREET)){
                 List<CustomerAgencyInfoResultDTO> customerGridInfoList = groupByLevel.get(ScreenConstant.STREET);
                 disPose(customerGridInfoList,false,customerId,dateId);
-            }else if (groupByLevel.containsKey(ScreenConstant.DISTRICT)){
+            }
+            if (groupByLevel.containsKey(ScreenConstant.DISTRICT)){
                 List<CustomerAgencyInfoResultDTO> customerGridInfoList = groupByLevel.get(ScreenConstant.DISTRICT);
                 disPose(customerGridInfoList,false,customerId,dateId);
-            }else if (groupByLevel.containsKey(ScreenConstant.CITY)){
+            }
+            if (groupByLevel.containsKey(ScreenConstant.CITY)){
                 List<CustomerAgencyInfoResultDTO> customerGridInfoList = groupByLevel.get(ScreenConstant.CITY);
                 disPose(customerGridInfoList,false,customerId,dateId);
-            }else if (groupByLevel.containsKey(ScreenConstant.PROVINCE)){
+            }
+            if (groupByLevel.containsKey(ScreenConstant.PROVINCE)){
                 List<CustomerAgencyInfoResultDTO> customerGridInfoList = groupByLevel.get(ScreenConstant.PROVINCE);
                 disPose(customerGridInfoList,false,customerId,dateId);
-            }else {
-                // TODO 干啥
             }
         }
         return true;
@@ -89,6 +96,7 @@ public class PartyBaseInfoServiceImpl implements PartyBaseInfoService {
      */
     public void disPose(List<CustomerAgencyInfoResultDTO> agencyIdList, Boolean isGrid, String customerId, String dateId){
         if (!CollectionUtils.isEmpty(agencyIdList)){
+            List<String> orgIds = agencyIdList.stream().map(m -> m.getAgencyId()).collect(Collectors.toList());
             List<PartyBaseInfoFormDTO> result = new ArrayList<>();
             if (isGrid == true){
                 List<UserCountResultDTO> userCountList = userGridDailyService.selectUserCount(customerId, dateId);
@@ -97,6 +105,7 @@ public class PartyBaseInfoServiceImpl implements PartyBaseInfoService {
                     String agencyId = agency.getAgencyId();
                     Map<String, Object> agencyMap = agencyService.selectAllSubAgencyId(agencyId, customerId);
                     List<String> gridIds = (List<String>) agencyMap.get(agencyId);
+                    orgIds.addAll(gridIds);
                     List<PartyMemberInfoResultDTO> partyMemberInfoList = partyMemberService.selectPartyMemberInfo(customerId, gridIds);
                     if (!CollectionUtils.isEmpty(partyMemberInfoList)){
                         // 为每个人赋值年龄
@@ -137,6 +146,8 @@ public class PartyBaseInfoServiceImpl implements PartyBaseInfoService {
                     PartyBaseInfoFormDTO form = new PartyBaseInfoFormDTO();
                     form.setOrgName(party.get(NumConstant.ZERO).getOrgName());
                     form.setOrgId(commAgencyId);
+                    form.setCustomerId(customerId);
+                    form.setDataEndTime(dateId);
                     form.setOrgType(ScreenConstant.AGENCY);
                     form.setParentId(party.get(NumConstant.ZERO).getParentId());
                     form.setAgeLevel1(party.stream().collect(Collectors.summingInt(PartyBaseInfoFormDTO::getAgeLevel1)));
@@ -150,30 +161,32 @@ public class PartyBaseInfoServiceImpl implements PartyBaseInfoService {
                     form.setRegisterUserCount(party.stream().collect(Collectors.summingInt(PartyBaseInfoFormDTO::getRegisterUserCount)));
                     result.add(form);
                 });
-                delAndInsert(result,customerId,dateId);
+                delAndInsert(result,customerId,dateId,orgIds);
             }else {
-                List<UserCountResultDTO> userCountRList = userAgencyDailyService.selectUserCount(customerId, dateId);
                 agencyIdList.forEach(agency -> {
                     String agencyId = agency.getAgencyId();
-                    // todo
                     List<PartyInfoResultDTO> partyInfoList = cpcBaseDataService.selectPartyInfo(customerId, dateId, agencyId);
-                    PartyBaseInfoFormDTO form = new PartyBaseInfoFormDTO();
+                    if (!CollectionUtils.isEmpty(partyInfoList)){
-                    form.setOrgName(partyInfoList.get(NumConstant.ZERO).getOrgName());
+                        PartyBaseInfoFormDTO form = new PartyBaseInfoFormDTO();
-                    form.setOrgId(agencyId);
+                        form.setOrgName(partyInfoList.get(NumConstant.ZERO).getOrgName());
-                    form.setOrgType(ScreenConstant.AGENCY);
+                        form.setOrgId(agencyId);
-                    form.setParentId(partyInfoList.get(NumConstant.ZERO).getParentId());
+                        form.setCustomerId(customerId);
-                    form.setAgeLevel1(partyInfoList.stream().collect(Collectors.summingInt(PartyInfoResultDTO::getAgeLevel1)));
+                        form.setDataEndTime(dateId);
-                    form.setAgeLevel2(partyInfoList.stream().collect(Collectors.summingInt(PartyInfoResultDTO::getAgeLevel2)));
+                        form.setOrgType(ScreenConstant.AGENCY);
-                    form.setAgeLevel3(partyInfoList.stream().collect(Collectors.summingInt(PartyInfoResultDTO::getAgeLevel3)));
+                        form.setParentId(partyInfoList.get(NumConstant.ZERO).getParentId());
-                    form.setAgeLevel4(partyInfoList.stream().collect(Collectors.summingInt(PartyInfoResultDTO::getAgeLevel4)));
+                        form.setAgeLevel1(partyInfoList.stream().collect(Collectors.summingInt(PartyInfoResultDTO::getAgeLevel1)));
-                    form.setAgeLevel5(partyInfoList.stream().collect(Collectors.summingInt(PartyInfoResultDTO::getAgeLevel5)));
+                        form.setAgeLevel2(partyInfoList.stream().collect(Collectors.summingInt(PartyInfoResultDTO::getAgeLevel2)));
-                    form.setAgeLevel6(partyInfoList.stream().collect(Collectors.summingInt(PartyInfoResultDTO::getAgeLevel6)));
+                        form.setAgeLevel3(partyInfoList.stream().collect(Collectors.summingInt(PartyInfoResultDTO::getAgeLevel3)));
-                    form.setPartyMemberCount(partyInfoList.stream().collect(Collectors.summingInt(PartyInfoResultDTO::getPartyMemberCount)));
+                        form.setAgeLevel4(partyInfoList.stream().collect(Collectors.summingInt(PartyInfoResultDTO::getAgeLevel4)));
-                    form.setResiTotal(partyInfoList.stream().collect(Collectors.summingInt(PartyInfoResultDTO::getResiTotal)));
+                        form.setAgeLevel5(partyInfoList.stream().collect(Collectors.summingInt(PartyInfoResultDTO::getAgeLevel5)));
-                    form.setRegisterUserCount(partyInfoList.stream().collect(Collectors.summingInt(PartyInfoResultDTO::getRegisterUserCount)));
+                        form.setAgeLevel6(partyInfoList.stream().collect(Collectors.summingInt(PartyInfoResultDTO::getAgeLevel6)));
-                    result.add(form);
+                        form.setPartyMemberCount(partyInfoList.stream().collect(Collectors.summingInt(PartyInfoResultDTO::getPartyMemberCount)));
+                        form.setResiTotal(partyInfoList.stream().collect(Collectors.summingInt(PartyInfoResultDTO::getResiTotal)));
+                        form.setRegisterUserCount(partyInfoList.stream().collect(Collectors.summingInt(PartyInfoResultDTO::getRegisterUserCount)));
+                        result.add(form);
+                    }
                 });
-                delAndInsert(result,customerId,dateId);
+                delAndInsert(result,customerId,dateId,orgIds);
             }
         }
     }
@@ -246,14 +259,34 @@ public class PartyBaseInfoServiceImpl implements PartyBaseInfoService {
      * @date 2020/9/23 10:13 上午
      */
     @Transactional(rollbackFor = Exception.class)
-    public void delAndInsert(List<PartyBaseInfoFormDTO> result, String customerId, String dateId){
+    public void delAndInsert(List<PartyBaseInfoFormDTO> result, String customerId, String dateId, List<String> orgIds){
+        List<PartyBaseInfoFormDTO> partyBaseInfoList = agencyService.selectAllAgencyIdToParty(customerId,dateId);
+        List<PartyBaseInfoFormDTO> resultList = gridService.selectAllGridIdToParty(customerId, dateId);
+        resultList.addAll(partyBaseInfoList);
+        List<PartyBaseInfoFormDTO> finalResult = new ArrayList<>();
+        resultList.forEach(rl -> {
+            orgIds.forEach(orgId -> {
+                if (rl.getOrgId().equals(orgId)){
+                    finalResult.add(rl);
+                }
+            });
+        });
         if (!CollectionUtils.isEmpty(result)){
-            List<String> orgIds = result.stream().map(m -> m.getOrgId()).collect(Collectors.toList());
+            finalResult.forEach(fr -> {
-            Integer delNum;
+                result.forEach(r -> {
-            do {
+                    if (fr.getOrgId().equals(r.getOrgId())){
-                delNum = cpcBaseDataService.deleteOldPartyBaseInfo(customerId, dateId, orgIds);
+                        BeanUtils.copyProperties(r,fr);
-            }while (delNum > NumConstant.ZERO);
+                    }
-            cpcBaseDataService.insertPartyBaseInfo(result);
+                });
+            });
         }
+        Integer delNum;
+        do {
+            delNum = cpcBaseDataService.deleteOldPartyBaseInfo(customerId, dateId, orgIds);
+        }while (delNum > NumConstant.ZERO);
+        List<List<PartyBaseInfoFormDTO>> partition = ListUtils.partition(finalResult, NumConstant.ONE_HUNDRED);
+        partition.forEach(p -> {
+            cpcBaseDataService.insertPartyBaseInfo(p);
+        });
     }
 }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/screen/ScreenCustomerAgencyService.java

@@ -17,6 +17,7 @@
 
 package com.epmet.service.evaluationindex.screen;
 
+import com.epmet.dto.extract.form.PartyBaseInfoFormDTO;
 import com.epmet.dto.extract.result.CustomerAgencyInfoResultDTO;
 
 import java.util.List;
@@ -66,4 +67,12 @@ public interface ScreenCustomerAgencyService{
      * @date 2020/9/23 11:11 上午
      */
     List<String> selectSubAgencyId(String agencyId, String customerId);
+
+    /**
+     * @Description  查询客户下所有机关ID
+     * @param customerId
+     * @author zxc
+     * @date 2020/9/23 3:10 下午
+     */
+    List<PartyBaseInfoFormDTO> selectAllAgencyIdToParty(String customerId,String dateId);
 }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/screen/ScreenCustomerGridService.java

@@ -19,6 +19,7 @@ package com.epmet.service.evaluationindex.screen;
 
 
 import com.epmet.commons.mybatis.service.BaseService;
+import com.epmet.dto.extract.form.PartyBaseInfoFormDTO;
 import com.epmet.entity.evaluationindex.screen.ScreenCustomerGridEntity;
 import com.epmet.entity.org.CustomerGridEntity;
 
@@ -36,4 +37,13 @@ public interface ScreenCustomerGridService extends BaseService<ScreenCustomerGri
     ScreenCustomerGridEntity getLastUpdateGrid();
 
     void addAndUpdateGrids(List<CustomerGridEntity> grids2Add, List<CustomerGridEntity> grids2Update);
+
+    /**
+     * @Description  查询客户下所有网格ID
+     * @param customerId
+     * @param dateId
+     * @author zxc
+     * @date 2020/9/23 3:10 下午
+     */
+    List<PartyBaseInfoFormDTO> selectAllGridIdToParty(String customerId, String dateId);
 }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/screen/impl/ScreenCpcBaseDataServiceImpl.java

@@ -17,7 +17,9 @@
 
 package com.epmet.service.evaluationindex.screen.impl;
 
+import com.epmet.commons.dynamic.datasource.annotation.DataSource;
 import com.epmet.commons.mybatis.service.impl.BaseServiceImpl;
+import com.epmet.constant.DataSourceConstant;
 import com.epmet.dao.evaluationindex.screen.ScreenCpcBaseDataDao;
 import com.epmet.dto.extract.form.PartyBaseInfoFormDTO;
 import com.epmet.dto.extract.result.PartyInfoResultDTO;
@@ -35,6 +37,7 @@ import java.util.List;
  * @since v1.0.0 2020-09-22
  */
 @Service
+@DataSource(DataSourceConstant.EVALUATION_INDEX)
 public class ScreenCpcBaseDataServiceImpl extends BaseServiceImpl<ScreenCpcBaseDataDao, ScreenCpcBaseDataEntity> implements ScreenCpcBaseDataService {
 
 

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/screen/impl/ScreenCustomerAgencyServiceImpl.java

@@ -22,6 +22,7 @@ import com.epmet.commons.tools.utils.DateUtils;
 import com.epmet.constant.OrgSourceTypeConstant;
 import com.epmet.dao.evaluationindex.screen.ScreenCustomerAgencyDao;
 import com.epmet.dao.evaluationindex.screen.ScreenCustomerGridDao;
+import com.epmet.dto.extract.form.PartyBaseInfoFormDTO;
 import com.epmet.dto.extract.result.CustomerAgencyInfoResultDTO;
 import com.epmet.dto.screen.result.TreeResultDTO;
 import com.epmet.constant.ScreenConstant;
@@ -147,6 +148,17 @@ public class ScreenCustomerAgencyServiceImpl implements ScreenCustomerAgencyServ
         return screenCustomerAgencyDao.selectSubAgencyId(agencyId, customerId);
     }
 
+    /**
+     * @Description  查询客户下所有机关ID
+     * @param customerId
+     * @author zxc
+     * @date 2020/9/23 3:10 下午
+     */
+    @Override
+    public List<PartyBaseInfoFormDTO> selectAllAgencyIdToParty(String customerId,String dateId) {
+        return screenCustomerAgencyDao.selectAllAgencyIdToParty(customerId,dateId);
+    }
+
     private void updateAgency(ScreenCustomerAgencyEntity exists) {
         screenCustomerAgencyDao.updateById(exists);
     }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/screen/impl/ScreenCustomerGridServiceImpl.java

@@ -24,6 +24,7 @@ import com.epmet.commons.tools.utils.DateUtils;
 import com.epmet.constant.DataSourceConstant;
 import com.epmet.constant.OrgSourceTypeConstant;
 import com.epmet.dao.evaluationindex.screen.ScreenCustomerGridDao;
+import com.epmet.dto.extract.form.PartyBaseInfoFormDTO;
 import com.epmet.entity.evaluationindex.screen.ScreenCustomerGridEntity;
 import com.epmet.entity.org.CustomerGridEntity;
 import com.epmet.service.evaluationindex.screen.ScreenCustomerGridService;
@@ -89,4 +90,16 @@ public class ScreenCustomerGridServiceImpl extends BaseServiceImpl<ScreenCustome
             }
         }
     }
+
+    /**
+     * @Description  查询客户下所有网格ID
+     * @param customerId
+     * @param dateId
+     * @author zxc
+     * @date 2020/9/23 3:10 下午
+     */
+    @Override
+    public List<PartyBaseInfoFormDTO> selectAllGridIdToParty(String customerId, String dateId) {
+        return screenCustomerGridDao.selectAllGridIdToParty(customerId, dateId);
+    }
 }

epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/evaluationindex/screen/ScreenCpcBaseDataDao.xml

@@ -120,10 +120,10 @@
         UPDATED_BY,
         UPDATED_TIME
         ) values
-        <foreach collection="list" item="item" separator=",">
+        <foreach collection="baseInfos" item="item" separator=",">
             (
             REPLACE(UUID(), '-', ''),
-            #{customerId},
+            #{item.customerId},
             #{item.orgType},
             #{item.orgId},
             #{item.parentId},
@@ -156,7 +156,7 @@
         and DATA_END_TIME = #{dateId}
         and
             (
-                <foreach collection="ordIds" item="orgId" separator=" OR ">
+                <foreach collection="orgIds" item="orgId" separator=" OR ">
                     org_id = #{orgId}
                 </foreach>
             )

epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/evaluationindex/screen/ScreenCustomerAgencyDao.xml

@@ -270,4 +270,20 @@
             AND CUSTOMER_ID = #{customerId}
             AND PID = #{agencyId}
     </select>
+
+    <!-- 查询客户下所有机关ID -->
+    <select id="selectAllAgencyIdToParty" resultType="com.epmet.dto.extract.form.PartyBaseInfoFormDTO">
+        SELECT
+            agency_id as orgId,
+            'agency' AS orgType,
+            AGENCY_NAME AS orgName,
+            CUSTOMER_ID AS customerId,
+            PID AS parentId,
+            #{dateId} AS dataEndTime
+        FROM
+            screen_customer_agency
+        WHERE
+            del_flag = '0'
+            AND CUSTOMER_ID = #{customerId}
+    </select>
 </mapper>

epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/evaluationindex/screen/ScreenCustomerGridDao.xml

@@ -217,4 +217,20 @@
         where DEL_FLAG = 0
           and GRID_ID = #{gridId}
     </select>
+
+    <!-- 查询客户下所有网格ID -->
+    <select id="selectAllGridIdToParty" resultType="com.epmet.dto.extract.form.PartyBaseInfoFormDTO">
+        SELECT
+            grid_id as orgId,
+            'grid' AS orgType,
+            grid_name AS orgName,
+            CUSTOMER_ID AS customerId,
+            PARENT_AGENCY_ID AS parentId,
+            #{dateId} AS dataEndTime
+        FROM
+            screen_customer_grid
+        WHERE
+            del_flag = '0'
+          AND CUSTOMER_ID = #{customerId}
+    </select>
 </mapper>