Merge remote-tracking branch 'origin/dev_import' into develop

epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/constant/SpecialCustomerOrgConstant.java

@@ -18,14 +18,19 @@ public interface SpecialCustomerOrgConstant {
      */
     String test="6e511da6816e53af4cda952365a26eb9";
 
-    // /**
-    //  * 榆山生产客户id
-    //  */
-    // String YUSHAN_PROD_CUSTOMER_ID = "46c55cb862d6d5e6d05d2ab61a1cc07e";
-    //
-    // /**
-    //  * 榆山测试客户id
-    //  */
-    // String YUSHAN_TEST_CUSTOMER_ID = "a4bbf298d8e427844038cee466f022ef";
+    /**
+     * 榆山生产客户id
+     */
+    String YUSHAN_PROD_CUSTOMER_ID = "46c55cb862d6d5e6d05d2ab61a1cc07e";
+
+    /**
+     * 榆山测试客户id
+     */
+    String YUSHAN_TEST_CUSTOMER_ID = "a4bbf298d8e427844038cee466f022ef";
+
+    /**
+     * 生产平阴客户id
+     */
+    String PING_YIN_CUSTOMER_ID="6f203e30de1a65aab7e69c058826cd80";
 
 }

epmet-commons/epmet-commons-tools/src/main/java/com/epmet/commons/tools/utils/YuShanSysApiService.java

@@ -5,6 +5,7 @@ import com.alibaba.fastjson.JSON;
 import com.epmet.commons.tools.constant.SpecialCustomerOrgConstant;
 import com.epmet.commons.tools.dto.form.UpdateUserPointsFormDTO;
 import com.epmet.commons.tools.dto.result.CustomerStaffInfoCacheResult;
+import com.epmet.commons.tools.enums.EnvEnum;
 import com.epmet.commons.tools.enums.ExternalApiEnum;
 import com.epmet.commons.tools.enums.ExternalServerEnum;
 import com.epmet.commons.tools.exception.EpmetErrorCode;
@@ -39,7 +40,16 @@ public class YuShanSysApiService {
         if (StringUtils.isBlank(customerId) || CollectionUtils.isEmpty(paramList)) {
             throw new RenException("参数错误");
         }
-        String serverUrl = ExternalServerEnum.getUrl(customerId);
+        if (!SpecialCustomerOrgConstant.PING_YIN_CUSTOMER_ID.equals(customerId)) {
+            return;
+        }
+        String serverUrl = null;
+        EnvEnum currentEnv = EnvEnum.getCurrentEnv();
+        if (EnvEnum.PROD.getCode().equals(currentEnv.getCode())) {
+            serverUrl = ExternalServerEnum.getUrl(SpecialCustomerOrgConstant.YUSHAN_PROD_CUSTOMER_ID);
+        } else if (EnvEnum.TEST.getCode().equals(currentEnv.getCode()) || EnvEnum.LOCAL.getCode().equals(currentEnv.getCode())) {
+            serverUrl = ExternalServerEnum.getUrl(SpecialCustomerOrgConstant.YUSHAN_TEST_CUSTOMER_ID);
+        }
         if (StringUtils.isBlank(serverUrl)) {
             log.warn(String.format("当前客户：%s,未配置服务器地址", customerId));
             return;

epmet-module/data-aggregator/data-aggregator-server/src/main/resources/mapper/datastats/DatsStatsDao.xml

@@ -331,17 +331,19 @@
     <select id="getSubAgencyProject" resultType="com.epmet.dataaggre.dto.datastats.result.SubAgencyProjectResultDTO">
         SELECT
             agency_id AS agencyId,
-            IFNULL(project_total, 0) AS "projectTotal",
-            IFNULL(pending_total, 0) AS "pendingTotal",
-            IFNULL(closed_total, 0) AS "closedProjectTotal"
+            <!-- 平阴三个街道的统计数据同一组织同一天会存在两条数据，一条是上报的一条是小程序统计的，所以查询的时候需要按组织分组sum -->
+            SUM(IFNULL(project_total, 0)) AS "projectTotal",
+            SUM(IFNULL(pending_total, 0)) AS "pendingTotal",
+            SUM(IFNULL(closed_total, 0)) AS "closedProjectTotal"
         FROM
             fact_agency_project_daily
         WHERE
             del_flag = '0'
         AND date_id = #{dateId}
-        <foreach item="agencyId" collection="agencyIds" open="AND (" separator="or" close=")" index="">
-            agency_id = #{agencyId}
+        <foreach collection="agencyIds" item="agencyId" open="AND agency_id IN (" separator="," close=")">
+            #{agencyId}
         </foreach>
+        GROUP BY agency_id
         ORDER BY
             project_total DESC
     </select>
@@ -349,17 +351,19 @@
     <select id="getSubGridProject" resultType="com.epmet.dataaggre.dto.datastats.result.SubGridProjectResultDTO">
         SELECT
             grid_id AS gridId,
-            IFNULL(project_total, 0) AS "projectTotal",
-            IFNULL(pending_total, 0) AS "pendingTotal",
-            IFNULL(closed_total, 0) AS "closedProjectTotal"
+        <!-- 平阴三个街道的统计数据同一网格同一天会存在两条数据，一条是上报的一条是小程序统计的，所以查询的时候需要按网格分组sum -->
+            SUM(IFNULL(project_total, 0)) AS "projectTotal",
+            SUM(IFNULL(pending_total, 0)) AS "pendingTotal",
+            SUM(IFNULL(closed_total, 0)) AS "closedProjectTotal"
         FROM
             fact_grid_project_daily
         WHERE
             del_flag = '0'
         AND date_id = #{dateId}
-        <foreach item="gridId" collection="gridIds" open="AND (" separator="or" close=")" index="">
-            grid_id = #{gridId}
+        <foreach collection="gridIds" item="gridId" open="AND grid_id IN (" separator="," close=")">
+            #{gridId}
         </foreach>
+        GROUP BY grid_id
         ORDER BY
             project_total DESC
     </select>
@@ -1004,7 +1008,8 @@
         FROM
         (
             SELECT DISTINCT
-                date_id dateId, project_total projectCount, closed_total closedProjectCount
+                <!-- 平阴的三个街道的数据在表里同一天会存在两条，一条是上报的一条是小程序统计的，所以查询时需要sum -->
+                date_id dateId, SUM(project_total) projectCount, SUM(closed_total) closedProjectCount
             <choose>
                 <when test='dataType == "agency"'>
                     , agency_id orgId
@@ -1028,6 +1033,7 @@
                 <if test='endTime != "" and endTime != null '>
                     AND date_id <![CDATA[<=]]> #{endTime}
                 </if>
+                GROUP BY date_id, agency_id
                 ORDER BY date_id ASC
             </if>
             <if test='sourceType != "" and sourceType == "end" and endTime != "" and endTime != null '>
@@ -1035,6 +1041,7 @@
                     AND date_id <![CDATA[>=]]> #{startTime}
                 </if>
                 AND date_id <![CDATA[<=]]> #{endTime}
+                GROUP BY date_id, agency_id
                 ORDER BY date_id DESC
             </if>
             <choose>

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/org/CustomerGridDao.java

@@ -25,6 +25,7 @@ import com.epmet.dto.org.CustomerStaffGridDTO;
 import com.epmet.dto.org.GridInfoDTO;
 import com.epmet.dto.org.form.GridBaseInfoFormDTO;
 import com.epmet.dto.org.result.CustomerGridDTO;
+import com.epmet.dto.stats.DimGridDTO;
 import com.epmet.dto.user.form.StaffBaseInfoFormDTO;
 import com.epmet.dto.user.result.GridUserInfoDTO;
 import com.epmet.entity.org.CustomerGridEntity;
@@ -118,4 +119,10 @@ public interface CustomerGridDao extends BaseDao<CustomerGridEntity> {
      * @Description 查询工作人员所属网格信息
      **/
     List<GridUserInfoDTO> getStaffGrid(StaffBaseInfoFormDTO formDTO);
+
+    /**
+     * @Author sun
+     * @Description 查询客户下有效网格列表
+     **/
+    List<DimGridDTO> gridListByCustomerId(@Param("customerId") String customerId);
 }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsProjectServiceImpl.java

@@ -1,17 +1,23 @@
 package com.epmet.service.impl;
 
 import com.epmet.commons.tools.constant.NumConstant;
+import com.epmet.commons.tools.utils.ConvertUtils;
 import com.epmet.commons.tools.utils.DateUtils;
+import com.epmet.constant.PingYinConstant;
 import com.epmet.constant.ProjectConstant;
+import com.epmet.dto.ProjectDTO;
 import com.epmet.dto.StatsFormDTO;
 import com.epmet.dto.project.form.MonthProjectListFormDTO;
 import com.epmet.dto.stats.DimAgencyDTO;
+import com.epmet.dto.stats.DimGridDTO;
 import com.epmet.entity.issue.IssueEntity;
+import com.epmet.entity.org.CustomerAgencyEntity;
 import com.epmet.entity.project.ProjectEntity;
 import com.epmet.entity.project.ProjectProcessEntity;
 import com.epmet.entity.stats.*;
 import com.epmet.service.Issue.IssueService;
 import com.epmet.service.StatsProjectService;
+import com.epmet.service.org.CustomerAgencyService;
 import com.epmet.service.org.CustomerGridService;
 import com.epmet.service.project.ProjectProcessService;
 import com.epmet.service.project.ProjectService;
@@ -63,6 +69,8 @@ public class StatsProjectServiceImpl implements StatsProjectService {
     private FactGridProjectMonthlyService factGridProjectMonthlyService;
     @Autowired
     private CustomerGridService customerGridService;
+    @Autowired
+    private CustomerAgencyService customerAgencyService;
 
     /**
      * @Author sun
@@ -108,10 +116,18 @@ public class StatsProjectServiceImpl implements StatsProjectService {
         DimIdGenerator.DimIdBean dimId = DimIdGenerator.getDimIdBean(date);
 
         //2:根据客户Id查询机关维度表数据
+        //2022.3.3  平阴数据统计时只统计了五个街道的 其他三个街道是上报的，现在这三个街道在小程序也产生了项目(例行工作)数据，
+        //但是dim维度表不能有这三个街道的组织，因此改查业务库组织列表数据   start  sun
+        List<DimAgencyDTO> dimAgencyList = new ArrayList<>();
+        if (customerId.equals(PingYinConstant.PROD_PING_YIN_CUSTOMER_ID)) {
+            List<CustomerAgencyEntity> agencyList = customerAgencyService.queryAgencyListByCustomerId(customerId);
+            dimAgencyList = ConvertUtils.sourceToTarget(agencyList, DimAgencyDTO.class);
+        } else {
             DimAgencyDTO dimAgencyDTO = new DimAgencyDTO();
             dimAgencyDTO.setCustomerId(customerId);
-        List<DimAgencyDTO> dimAgencyList = dimAgencyService.getDimAgencyList(dimAgencyDTO);
-
+            dimAgencyList = dimAgencyService.getDimAgencyList(dimAgencyDTO);
+        }
+        //2022.3.3  end  sun
        /*
        //2022.1.12 客户数据量大 调整计算逻辑，一千条已查询，封装数据,将这部分业务数据查询拿到子方法分页查询处理  sun
         //3:根据客户Id查询项目业务表已结案数据(查询传入日期及之前的数据)
@@ -650,7 +666,16 @@ public class StatsProjectServiceImpl implements StatsProjectService {
         DimIdGenerator.DimIdBean dimId = DimIdGenerator.getDimIdBean(date);
 
         //2:根据客户Id查询网格维度表数据
-        List<DimGridEntity> dimGridList = dimGridService.getGridListByCustomerId(customerId);
+        //2022.3.3  平阴数据统计时只统计了五个街道的 其他三个街道是上报的，现在这三个街道在小程序也产生了项目(例行工作)数据，
+        //但是dim维度表不能有这三个街道的组织，因此改查业务库网格列表数据   start  sun
+        List<DimGridEntity> dimGridList = new ArrayList<>();
+        if (customerId.equals(PingYinConstant.PROD_PING_YIN_CUSTOMER_ID)) {
+            List<DimGridDTO> gridList = customerGridService.gridListByCustomerId(customerId);
+            dimGridList = ConvertUtils.sourceToTarget(gridList, DimGridEntity.class);
+        } else {
+            dimGridList = dimGridService.getGridListByCustomerId(customerId);
+        }
+        //2022.3.3  end  sun
 
         //3:根据客户Id查询项目业务表已结案数据(查询传入日期及之前的数据)
         ProjectEntity projectEntity = new ProjectEntity();

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/org/CustomerGridService.java

@@ -8,9 +8,11 @@ import com.epmet.dto.org.CustomerStaffGridDTO;
 import com.epmet.dto.org.GridInfoDTO;
 import com.epmet.dto.org.form.GridBaseInfoFormDTO;
 import com.epmet.dto.org.result.CustomerGridDTO;
+import com.epmet.dto.stats.DimGridDTO;
 import com.epmet.dto.user.form.StaffBaseInfoFormDTO;
 import com.epmet.dto.user.result.GridUserInfoDTO;
 import com.epmet.entity.org.CustomerGridEntity;
+import com.epmet.entity.stats.DimGridEntity;
 
 import java.util.Date;
 import java.util.List;
@@ -99,4 +101,10 @@ public interface CustomerGridService extends BaseService<CustomerGridEntity> {
      * @Description 查询工作人员所属网格信息
      **/
     List<GridUserInfoDTO> getStaffGrid(StaffBaseInfoFormDTO formDTO);
+
+    /**
+     * @Author sun
+     * @Description 查询客户下有效网格列表
+     **/
+    List<DimGridDTO> gridListByCustomerId(String customerId);
 }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/org/impl/CustomerGridServiceImpl.java

@@ -12,6 +12,7 @@ import com.epmet.dto.org.CustomerStaffGridDTO;
 import com.epmet.dto.org.GridInfoDTO;
 import com.epmet.dto.org.form.GridBaseInfoFormDTO;
 import com.epmet.dto.org.result.CustomerGridDTO;
+import com.epmet.dto.stats.DimGridDTO;
 import com.epmet.dto.user.form.StaffBaseInfoFormDTO;
 import com.epmet.dto.user.result.GridUserInfoDTO;
 import com.epmet.entity.org.CustomerGridEntity;
@@ -147,4 +148,13 @@ public class CustomerGridServiceImpl extends BaseServiceImpl<CustomerGridDao, Cu
         return customerGridDao.getStaffGrid(formDTO);
     }
 
+    /**
+     * @Author sun
+     * @Description 查询客户下有效网格列表
+     **/
+    @Override
+    public List<DimGridDTO> gridListByCustomerId(String customerId) {
+        return customerGridDao.gridListByCustomerId(customerId);
+    }
+
 }

epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/evaluationindex/extract/FactOriginProjectOrgPeriodDailyDao.xml

@@ -40,12 +40,12 @@
         WHERE
         DEL_FLAG = '0'
         AND CUSTOMER_ID = #{customerId}
-        AND (DATE_FORMAT(INFORMED_DATE,"%Y%m%d") >= #{dateId} OR DATE_FORMAT(HANDLED_DATE,"%Y%m%d") >= #{dateId} OR DATE_FORMAT(PERIOD_TILL_REPLY_FIRSTLY,"%Y%m%d") >= #{dateId})
         <if test="null != list and list.size() > 0">
             <foreach collection="list" open="AND ( " close=" )" separator=" OR " item="item">
                 PROJECT_ID = #{item}
             </foreach>
         </if>
+        AND (DATE_FORMAT(INFORMED_DATE,"%Y%m%d") &gt;= #{dateId} OR DATE_FORMAT(HANDLED_DATE,"%Y%m%d") &gt;= #{dateId} OR DATE_FORMAT(PERIOD_TILL_REPLY_FIRSTLY,"%Y%m%d") &gt;= #{dateId})
     </delete>
 
     <select id="selectResponsiveness" resultType="com.epmet.dto.extract.result.OrgStatisticsResultDTO">

epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/org/CustomerGridDao.xml

@@ -185,4 +185,17 @@
         </if>
     </select>
 
+    <select id="gridListByCustomerId" resultType="com.epmet.dto.stats.DimGridDTO">
+         SELECT
+            id id,
+            customer_id customerId,
+            pid agencyId,
+            grid_name gridName
+        FROM
+            customer_grid
+        WHERE
+            del_flag = 0
+        AND customer_id = #{customerId}
+    </select>
+
 </mapper>

epmet-module/gov-org/gov-org-server/src/main/java/com/epmet/GovOrgApplication.java

@@ -13,6 +13,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.web.servlet.ServletComponentScan;
 import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
 import org.springframework.cloud.openfeign.EnableFeignClients;
+import org.springframework.scheduling.annotation.EnableAsync;
 
 /**
  *
@@ -24,6 +25,7 @@ import org.springframework.cloud.openfeign.EnableFeignClients;
 @EnableDiscoveryClient
 @EnableFeignClients
 @ServletComponentScan
+@EnableAsync
 public class GovOrgApplication {
 
     public static void main(String[] args) {

epmet-module/gov-org/gov-org-server/src/main/java/com/epmet/service/impl/BuildingServiceImpl.java

@@ -63,7 +63,6 @@ import java.util.stream.Collectors;
 
 @Slf4j
 @Service
-@EnableAsync
 public class BuildingServiceImpl implements BuildingService {
 
 

epmet-module/gov-org/gov-org-server/src/main/java/com/epmet/service/impl/HouseServiceImpl.java

@@ -74,7 +74,6 @@ import java.util.stream.Collectors;
 
 @Slf4j
 @Service
-@EnableAsync
 public class HouseServiceImpl implements HouseService, ResultDataResolver {
 
 

epmet-module/gov-org/gov-org-server/src/main/java/com/epmet/service/impl/IcNeighborHoodServiceImpl.java

@@ -22,7 +22,6 @@ import cn.afterturn.easypoi.excel.entity.ExportParams;
 import com.alibaba.excel.EasyExcel;
 import com.alibaba.excel.ExcelReader;
 import com.alibaba.excel.read.metadata.ReadSheet;
-import com.alibaba.fastjson.JSON;
 import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
 import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.baomidou.mybatisplus.core.metadata.IPage;
@@ -42,6 +41,7 @@ import com.epmet.commons.tools.redis.common.bean.GridInfoCache;
 import com.epmet.commons.tools.utils.ConvertUtils;
 import com.epmet.commons.tools.utils.Result;
 import com.epmet.constant.CustomerGridConstant;
+import com.epmet.constant.ImportErrorMsgConstants;
 import com.epmet.constants.ImportTaskConstants;
 import com.epmet.dao.CustomerGridDao;
 import com.epmet.dao.IcBuildingDao;
@@ -50,6 +50,7 @@ import com.epmet.dao.IcPropertyManagementDao;
 import com.epmet.dto.*;
 import com.epmet.dto.form.ImportInfoFormDTO;
 import com.epmet.dto.form.ImportTaskCommonFormDTO;
+import com.epmet.dto.result.ImportResultDTO;
 import com.epmet.dto.result.ImportTaskCommonResultDTO;
 import com.epmet.dto.result.InfoByNamesResultDTO;
 import com.epmet.dto.result.UploadImgResultDTO;
@@ -61,6 +62,7 @@ import com.epmet.model.ImportNeighborHoodInfoListener;
 import com.epmet.model.NeighborHoodInfoModel;
 import com.epmet.redis.IcHouseRedis;
 import com.epmet.service.*;
+import com.epmet.util.ExcelPoiUtils;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.ListUtils;
@@ -93,7 +95,6 @@ import java.util.stream.Collectors;
  */
 @Slf4j
 @Service
-@EnableAsync
 public class IcNeighborHoodServiceImpl extends BaseServiceImpl<IcNeighborHoodDao, IcNeighborHoodEntity> implements IcNeighborHoodService {
 
     @Autowired
@@ -248,16 +249,13 @@ public class IcNeighborHoodServiceImpl extends BaseServiceImpl<IcNeighborHoodDao
             ReadSheet readSheet = EasyExcel.readSheet(0).head(NeighborHoodInfoModel.class)
                     .registerReadListener(new ImportNeighborHoodInfoListener(formDTO,icHouseRedis,icBuildingDao,this,propertyManagementDao,epmetCommonServiceOpenFeignClient,importTask.getData().getTaskId()))
                     .build();
-            log.info("neighborhoodImport build readSheet num:{}",readSheet.getSheetNo());
             excelReader.read(readSheet);
         } catch (Exception e){
-            log.error("neighborhoodImport import exception", e);
             ImportTaskCommonFormDTO input = new ImportTaskCommonFormDTO();
             input.setOperatorId(formDTO.getUserId());
             input.setTaskId(importTask.getData().getTaskId());
             input.setProcessStatus(ImportTaskConstants.PROCESS_STATUS_FINISHED_FAIL);
-            Result result = epmetCommonServiceOpenFeignClient.finishImportTask(input);
-            log.info("neighborhoodImport finishImportTask result:{]", JSON.toJSONString(result));
+            epmetCommonServiceOpenFeignClient.finishImportTask(input);
         } finally {
             if (excelReader != null) {
                 excelReader.finish();