修改：

1.指标库维度同步，优化，新增的时候判断是否存在，存在的话则执行修改

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/evaluationindex/screen/DimLastSyncRecordDao.java

@@ -0,0 +1,48 @@
+/**
+ * Copyright 2018 人人开源 https://www.renren.io
+ * <p>
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ * <p>
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * <p>
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package com.epmet.dao.evaluationindex.screen;
+
+import com.epmet.commons.mybatis.dao.BaseDao;
+import com.epmet.entity.evaluationindex.screen.DimLastSyncRecordEntity;
+import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
+
+import java.util.Date;
+
+/**
+ * 
+ *
+ * @author generator generator@elink-cn.com
+ * @since v1.0.0 2021-12-28
+ */
+@Mapper
+public interface DimLastSyncRecordDao extends BaseDao<DimLastSyncRecordEntity> {
+    /**
+     * 查询最后一次同步时间
+     * @return
+     */
+    Date getDimLastSyncEndTime(@Param("subject") String subject);
+
+    /**
+     * 更新最后一次同步时间
+     * @param subject
+     * @param statsEndTime
+     * @return
+     */
+    int updateLastSyncEndTime(@Param("subject") String subject, @Param("statsEndTime") Date statsEndTime);
+}

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/entity/evaluationindex/screen/DimLastSyncRecordEntity.java

@@ -0,0 +1,51 @@
+/**
+ * Copyright 2018 人人开源 https://www.renren.io
+ * <p>
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ * <p>
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * <p>
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package com.epmet.entity.evaluationindex.screen;
+
+import com.baomidou.mybatisplus.annotation.TableName;
+
+import com.epmet.commons.mybatis.entity.BaseEpmetEntity;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+
+import java.util.Date;
+
+/**
+ * 
+ *
+ * @author generator generator@elink-cn.com
+ * @since v1.0.0 2021-12-28
+ */
+@Data
+@EqualsAndHashCode(callSuper=false)
+@TableName("dim_last_sync_record")
+public class DimLastSyncRecordEntity extends BaseEpmetEntity {
+
+	private static final long serialVersionUID = 1L;
+
+    /**
+     * 主体。agency:机关，department:部门，grid:网格
+     */
+	private String subject;
+
+    /**
+     * 最后一次同步时间
+     */
+	private Date lastSyncTime;
+
+}

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/entity/evaluationindex/screen/ScreenCustomerDeptEntity.java

@@ -87,4 +87,9 @@ public class ScreenCustomerDeptEntity extends BaseEpmetEntity {
 	 * desc: 是否参与上级计算yes:参与;no:不参与 add 01.14
 	 */
 	private String upToCal;
+
+	/**
+	 * desc: 是否显示 1显示 0不显示
+	 */
+	private String isDisplay;
 }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/entity/evaluationindex/screen/ScreenCustomerGridEntity.java

@@ -97,4 +97,9 @@ public class ScreenCustomerGridEntity extends BaseEpmetEntity {
 	 * 地区码
 	 */
 	private String code;
+
+	/**
+	 * desc: 是否显示 1显示 0不显示
+	 */
+	private String isDisplay;
 }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/screen/ScreenCustomerAgencyService.java

@@ -17,6 +17,7 @@
 
 package com.epmet.service.evaluationindex.screen;
 
+import cn.hutool.db.DaoTemplate;
 import com.epmet.commons.mybatis.service.BaseService;
 import com.epmet.dto.CustomerAgencyDTO;
 import com.epmet.dto.extract.form.PartyBaseInfoFormDTO;
@@ -27,6 +28,7 @@ import com.epmet.dto.extract.result.OrgNameResultDTO;
 import com.epmet.entity.evaluationindex.screen.ScreenCustomerAgencyEntity;
 import com.epmet.entity.org.CustomerAgencyEntity;
 
+import java.util.Date;
 import java.util.List;
 import java.util.Map;
 
@@ -61,7 +63,7 @@ public interface ScreenCustomerAgencyService extends BaseService<ScreenCustomerA
 
     ScreenCustomerAgencyEntity getLastUpdatedAgency();
 
-    void initAgencies(List<CustomerAgencyEntity> agencies2Add, List<CustomerAgencyEntity> agencies2Update);
+    void initAgencies(List<CustomerAgencyEntity> agencies2Add, List<CustomerAgencyEntity> agencies2Update, Date statsEndTime);
 
     /**
      * @Description  查询下一级机关的ID【即以agencyId为pid的agency】
@@ -164,4 +166,10 @@ public interface ScreenCustomerAgencyService extends BaseService<ScreenCustomerA
      */
     String getParentAgencyId(String agencyId);
 
+    /**
+     * 获取上次维度同步时间
+     * @return
+     */
+    Date getDimLastSyncTime();
+
 }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/screen/ScreenCustomerDeptService.java

@@ -21,6 +21,7 @@ import com.epmet.commons.mybatis.service.BaseService;
 import com.epmet.entity.evaluationindex.screen.ScreenCustomerDeptEntity;
 import com.epmet.entity.org.CustomerDepartmentEntity;
 
+import java.util.Date;
 import java.util.List;
 
 /**
@@ -39,6 +40,11 @@ public interface ScreenCustomerDeptService extends BaseService<ScreenCustomerDep
 
     ScreenCustomerDeptEntity getLastUpdateDept();
 
-    void addAndUpdateDepartments(List<CustomerDepartmentEntity> depts2Add, List<CustomerDepartmentEntity> depts2Update);
+    void addAndUpdateDepartments(List<CustomerDepartmentEntity> depts2Add, List<CustomerDepartmentEntity> depts2Update, Date syncEndTime);
 
+    /**
+     * 查询上次同步的截止时间
+     * @return
+     */
+    Date getLastSyncEndTime();
 }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/screen/ScreenCustomerGridService.java

@@ -27,6 +27,7 @@ import com.epmet.dto.screen.ScreenProjectGridDailyDTO;
 import com.epmet.entity.evaluationindex.screen.ScreenCustomerGridEntity;
 import com.epmet.entity.org.CustomerGridEntity;
 
+import java.util.Date;
 import java.util.List;
 import java.util.Map;
 
@@ -41,7 +42,7 @@ public interface ScreenCustomerGridService extends BaseService<ScreenCustomerGri
     ScreenCustomerGridEntity getLastAddGrid();
     ScreenCustomerGridEntity getLastUpdateGrid();
 
-    void addAndUpdateGrids(List<CustomerGridEntity> grids2Add, List<CustomerGridEntity> grids2Update);
+    void addAndUpdateGrids(List<CustomerGridEntity> grids2Add, List<CustomerGridEntity> grids2Update, Date syncEndTime);
 
     /**
      * @Description  查询客户下所有网格ID
@@ -129,4 +130,10 @@ public interface ScreenCustomerGridService extends BaseService<ScreenCustomerGri
      * @Date 2021/10/15 15:50
      */
     Map<String, ScreenCustomerGridDTO> getGridList(String customerId);
+
+    /**
+     * 查询上次同步截止时间
+     * @return
+     */
+    Date getLastSyncEndTime();
 }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/screen/impl/ScreenCustomerAgencyServiceImpl.java

@@ -25,6 +25,7 @@ import com.epmet.commons.tools.utils.DateUtils;
 import com.epmet.constant.DataSourceConstant;
 import com.epmet.constant.OrgSourceTypeConstant;
 import com.epmet.constant.ScreenConstant;
+import com.epmet.dao.evaluationindex.screen.DimLastSyncRecordDao;
 import com.epmet.dao.evaluationindex.screen.ScreenCustomerAgencyDao;
 import com.epmet.dao.evaluationindex.screen.ScreenCustomerGridDao;
 import com.epmet.dto.CustomerAgencyDTO;
@@ -34,6 +35,7 @@ import com.epmet.dto.extract.form.ScreenPartyLinkMassesDataFormDTO;
 import com.epmet.dto.extract.result.CustomerAgencyInfoResultDTO;
 import com.epmet.dto.extract.result.OrgNameResultDTO;
 import com.epmet.dto.screen.result.TreeResultDTO;
+import com.epmet.entity.evaluationindex.screen.DimLastSyncRecordEntity;
 import com.epmet.entity.evaluationindex.screen.ScreenCustomerAgencyEntity;
 import com.epmet.entity.org.CustomerAgencyEntity;
 import com.epmet.service.evaluationindex.screen.ScreenCustomerAgencyService;
@@ -63,6 +65,8 @@ public class ScreenCustomerAgencyServiceImpl extends BaseServiceImpl<ScreenCusto
     private ScreenCustomerAgencyDao screenCustomerAgencyDao;
     @Autowired
     private ScreenCustomerGridDao screenCustomerGridDao;
+    @Autowired
+    private DimLastSyncRecordDao dimLastSyncRecordDao;
 
     /**
      * @Description  根据agencyId，查询所有子级agencyId，【当机关的级别为 community时，所有子级为gridId】
@@ -116,12 +120,12 @@ public class ScreenCustomerAgencyServiceImpl extends BaseServiceImpl<ScreenCusto
 
     @Transactional(rollbackFor = Exception.class)
     @Override
-    public void initAgencies(List<CustomerAgencyEntity> agencies2Add, List<CustomerAgencyEntity> agencies2Update) {
+    public void initAgencies(List<CustomerAgencyEntity> agencies2Add, List<CustomerAgencyEntity> agencies2Update, Date statsEndTime) {
         String dateEndTime = DateUtils.format(new Date(), DateUtils.DATE_PATTERN_YYYYMMDD);
         if (!CollectionUtils.isEmpty(agencies2Add)) {
             // 添加
             for (CustomerAgencyEntity e : agencies2Add) {
-                addAgency(e, dateEndTime);
+                addOrUpdateAgency(e, dateEndTime);
             }
         }
 
@@ -137,11 +141,11 @@ public class ScreenCustomerAgencyServiceImpl extends BaseServiceImpl<ScreenCusto
                     exists.setDataEndTime(dateEndTime);
                     exists.setAreaCode(e.getAreaCode());
                     //如果没有区划 则默认暂时不显示
-                    if (StringUtils.isEmpty(e.getAreaCode())) {
+                    //if (StringUtils.isEmpty(e.getAreaCode())) {
-                        exists.setIsDisplay(NumConstant.ZERO_STR);
+                    //    exists.setIsDisplay(NumConstant.ZERO_STR);
-                    } else {
+                    //} else {
-                        exists.setIsDisplay(NumConstant.ONE_STR);
+                    //    exists.setIsDisplay(NumConstant.ONE_STR);
-                    }
+                    //}
                     exists.setAllParentNames(e.getAllParentName());
                     exists.setParentAreaCode(e.getParentAreaCode());
                     exists.setCode(e.getCode());
@@ -156,6 +160,19 @@ public class ScreenCustomerAgencyServiceImpl extends BaseServiceImpl<ScreenCusto
                 }
             }
         }
+        
+        // 保存同步记录
+        Date lastSyncEndTime = dimLastSyncRecordDao.getDimLastSyncEndTime("agency");
+        if (lastSyncEndTime == null) {
+            // 首次初始化，没有上次执行记录,插入一条
+            DimLastSyncRecordEntity insertRecord = new DimLastSyncRecordEntity();
+            insertRecord.setSubject("agency");
+            insertRecord.setLastSyncTime(statsEndTime);
+            dimLastSyncRecordDao.insert(insertRecord);
+        } else {
+            // 更新
+            dimLastSyncRecordDao.updateLastSyncEndTime("agency", statsEndTime);
+        }
     }
 
     /**
@@ -231,31 +248,62 @@ public class ScreenCustomerAgencyServiceImpl extends BaseServiceImpl<ScreenCusto
     }
 
     /**
-     * 添加agency
+     * 添加或者更新agency，防止情况：业务表中有数据被抽取过来，且这条数据已经由其他操作推到维度库了
      * @param e
      */
-    private void addAgency(CustomerAgencyEntity e, String dateEndTime) {
+    private void addOrUpdateAgency(CustomerAgencyEntity e, String dateEndTime) {
-        ScreenCustomerAgencyEntity cae = new ScreenCustomerAgencyEntity();
+
-        cae.setAgencyId(e.getId());
+        LambdaQueryWrapper<ScreenCustomerAgencyEntity> query = new LambdaQueryWrapper<>();
-        cae.setAgencyName(e.getOrganizationName());
+        query.eq(ScreenCustomerAgencyEntity::getCustomerId, e.getCustomerId());
-        cae.setAllParentNames(e.getAllParentName());
+        query.eq(ScreenCustomerAgencyEntity::getAgencyId, e.getId());
-        cae.setAreaCode(e.getAreaCode());
+        ScreenCustomerAgencyEntity existsOne = screenCustomerAgencyDao.selectOne(query);
-        //cae.setAreaMarks(e);
+        if (existsOne != null) {
-        //如果没有区划 则默认暂时不显示
+            // 已经存在
-        if (StringUtils.isEmpty(e.getAreaCode())) {
+            existsOne.setAgencyId(e.getId());
-            cae.setIsDisplay(NumConstant.ZERO_STR);
+            existsOne.setAgencyName(e.getOrganizationName());
+            existsOne.setAllParentNames(e.getAllParentName());
+            existsOne.setAreaCode(e.getAreaCode());
+            //cae.setAreaMarks(e);
+            //如果没有区划 则默认暂时不显示
+            //if (StringUtils.isEmpty(e.getAreaCode())) {
+            //    existsOne.setIsDisplay(NumConstant.ZERO_STR);
+            //} else {
+            //    existsOne.setIsDisplay(NumConstant.ONE_STR);
+            //}
+            existsOne.setCustomerId(e.getCustomerId());
+            existsOne.setDataEndTime(dateEndTime);
+            existsOne.setLevel(e.getLevel());
+            existsOne.setPid(e.getPid());
+            existsOne.setPids(e.getPids());
+            existsOne.setSourceType(OrgSourceTypeConstant.INTERNAL);
+            existsOne.setParentAreaCode(e.getParentAreaCode());
+            existsOne.setCode(e.getCode());
+            screenCustomerAgencyDao.updateById(existsOne);
+            return;
         } else {
-            cae.setIsDisplay(NumConstant.ONE_STR);
+            // 不存在，需要添加
+            ScreenCustomerAgencyEntity cae = new ScreenCustomerAgencyEntity();
+            cae.setAgencyId(e.getId());
+            cae.setAgencyName(e.getOrganizationName());
+            cae.setAllParentNames(e.getAllParentName());
+            cae.setAreaCode(e.getAreaCode());
+            //cae.setAreaMarks(e);
+            //如果没有区划 则默认暂时不显示
+            if (StringUtils.isEmpty(e.getAreaCode())) {
+                cae.setIsDisplay(NumConstant.ZERO_STR);
+            } else {
+                cae.setIsDisplay(NumConstant.ONE_STR);
+            }
+            cae.setCustomerId(e.getCustomerId());
+            cae.setDataEndTime(dateEndTime);
+            cae.setLevel(e.getLevel());
+            cae.setPid(e.getPid());
+            cae.setPids(e.getPids());
+            cae.setSourceType(OrgSourceTypeConstant.INTERNAL);
+            cae.setParentAreaCode(e.getParentAreaCode());
+            cae.setCode(e.getCode());
+            screenCustomerAgencyDao.insert(cae);
         }
-        cae.setCustomerId(e.getCustomerId());
-        cae.setDataEndTime(dateEndTime);
-        cae.setLevel(e.getLevel());
-        cae.setPid(e.getPid());
-        cae.setPids(e.getPids());
-        cae.setSourceType(OrgSourceTypeConstant.INTERNAL);
-        cae.setParentAreaCode(e.getParentAreaCode());
-        cae.setCode(e.getCode());
-        screenCustomerAgencyDao.insert(cae);
     }
 
     /**
@@ -355,4 +403,8 @@ public class ScreenCustomerAgencyServiceImpl extends BaseServiceImpl<ScreenCusto
         return baseDao.getParentAgencyId(agencyId);
     }
 
+    @Override
+    public Date getDimLastSyncTime() {
+        return dimLastSyncRecordDao.getDimLastSyncEndTime("agency");
+    }
 }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/screen/impl/ScreenCustomerDeptServiceImpl.java

@@ -20,13 +20,17 @@ package com.epmet.service.evaluationindex.screen.impl;
 import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
 import com.epmet.commons.dynamic.datasource.annotation.DataSource;
 import com.epmet.commons.mybatis.service.impl.BaseServiceImpl;
+import com.epmet.commons.tools.constant.NumConstant;
 import com.epmet.commons.tools.utils.DateUtils;
 import com.epmet.constant.DataSourceConstant;
 import com.epmet.constant.OrgSourceTypeConstant;
+import com.epmet.dao.evaluationindex.screen.DimLastSyncRecordDao;
 import com.epmet.dao.evaluationindex.screen.ScreenCustomerDeptDao;
+import com.epmet.entity.evaluationindex.screen.DimLastSyncRecordEntity;
 import com.epmet.entity.evaluationindex.screen.ScreenCustomerDeptEntity;
 import com.epmet.entity.org.CustomerDepartmentEntity;
 import com.epmet.service.evaluationindex.screen.ScreenCustomerDeptService;
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
@@ -46,6 +50,9 @@ public class ScreenCustomerDeptServiceImpl extends BaseServiceImpl<ScreenCustome
 
     @Autowired
     private ScreenCustomerDeptDao screenCustomerDeptDao;
+    
+    @Autowired
+    private DimLastSyncRecordDao dimLastSyncRecordDao;
 
     @Override
     public ScreenCustomerDeptEntity getLastAddDept() {
@@ -64,7 +71,7 @@ public class ScreenCustomerDeptServiceImpl extends BaseServiceImpl<ScreenCustome
      */
     @Override
     @Transactional(rollbackFor = Exception.class)
-    public void addAndUpdateDepartments(List<CustomerDepartmentEntity> depts2Add, List<CustomerDepartmentEntity> depts2Update) {
+    public void addAndUpdateDepartments(List<CustomerDepartmentEntity> depts2Add, List<CustomerDepartmentEntity> depts2Update, Date syncEndTime) {
         String dateStr = DateUtils.format(new Date(), "YYYYmmdd");
         for (CustomerDepartmentEntity dept : depts2Add) {
             ScreenCustomerDeptEntity e = screenCustomerDeptDao.selectByDeptId(dept.getId());
@@ -77,7 +84,22 @@ public class ScreenCustomerDeptServiceImpl extends BaseServiceImpl<ScreenCustome
                 e.setParentAgencyId(dept.getAgencyId());
                 e.setSourceType(OrgSourceTypeConstant.INTERNAL);
                 e.setAreaCode(dept.getAreaCode());
+                //如果没有区划 则默认暂时不显示
+                if (StringUtils.isEmpty(dept.getAreaCode())) {
+                    e.setIsDisplay(NumConstant.ZERO_STR);
+                } else {
+                    e.setIsDisplay(NumConstant.ONE_STR);
+                }
                 screenCustomerDeptDao.insert(e);
+            } else {
+                e.setCustomerId(dept.getCustomerId());
+                e.setDataEndTime(dateStr);
+                e.setDeptId(dept.getId());
+                e.setDeptName(dept.getDepartmentName());
+                e.setParentAgencyId(dept.getAgencyId());
+                e.setSourceType(OrgSourceTypeConstant.INTERNAL);
+                e.setAreaCode(dept.getAreaCode());
+                screenCustomerDeptDao.updateById(e);
             }
         }
 
@@ -100,6 +122,25 @@ public class ScreenCustomerDeptServiceImpl extends BaseServiceImpl<ScreenCustome
                 }
             }
         }
+        
+        // 存储同步记录
+        LambdaQueryWrapper<DimLastSyncRecordEntity> lastSyncRecordQuery = new LambdaQueryWrapper<>();
+        lastSyncRecordQuery.eq(DimLastSyncRecordEntity::getSubject, "department");
+
+        DimLastSyncRecordEntity syncRecord = dimLastSyncRecordDao.selectOne(lastSyncRecordQuery);
+        if (syncRecord != null) {
+            dimLastSyncRecordDao.updateLastSyncEndTime("department", syncEndTime);
+        } else {
+            syncRecord = new DimLastSyncRecordEntity();
+            syncRecord.setSubject("department");
+            syncRecord.setLastSyncTime(syncEndTime);
+            dimLastSyncRecordDao.insert(syncRecord);
+        }
+
     }
 
+    @Override
+    public Date getLastSyncEndTime() {
+        return dimLastSyncRecordDao.getDimLastSyncEndTime("department");
+    }
 }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/screen/impl/ScreenCustomerGridServiceImpl.java

@@ -22,16 +22,19 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
 import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
 import com.epmet.commons.dynamic.datasource.annotation.DataSource;
 import com.epmet.commons.mybatis.service.impl.BaseServiceImpl;
+import com.epmet.commons.tools.constant.NumConstant;
 import com.epmet.commons.tools.utils.ConvertUtils;
 import com.epmet.commons.tools.utils.DateUtils;
 import com.epmet.constant.DataSourceConstant;
 import com.epmet.constant.OrgSourceTypeConstant;
+import com.epmet.dao.evaluationindex.screen.DimLastSyncRecordDao;
 import com.epmet.dao.evaluationindex.screen.ScreenCustomerGridDao;
 import com.epmet.dto.extract.form.*;
 import com.epmet.dto.extract.result.GridInfoResultDTO;
 import com.epmet.dto.indexcollect.form.CustomerBizOrgFormDTO;
 import com.epmet.dto.screen.ScreenCustomerGridDTO;
 import com.epmet.dto.screen.ScreenProjectGridDailyDTO;
+import com.epmet.entity.evaluationindex.screen.DimLastSyncRecordEntity;
 import com.epmet.entity.evaluationindex.screen.ScreenCustomerGridEntity;
 import com.epmet.entity.org.CustomerGridEntity;
 import com.epmet.service.evaluationindex.screen.ScreenCustomerGridService;
@@ -57,6 +60,8 @@ public class ScreenCustomerGridServiceImpl extends BaseServiceImpl<ScreenCustome
 
     @Autowired
     private ScreenCustomerGridDao screenCustomerGridDao;
+    @Autowired
+    private DimLastSyncRecordDao dimLastSyncRecordDao;
 
     @Override
     public ScreenCustomerGridEntity getLastAddGrid() {
@@ -70,22 +75,39 @@ public class ScreenCustomerGridServiceImpl extends BaseServiceImpl<ScreenCustome
 
     @Transactional(rollbackFor = Exception.class)
     @Override
-    public void addAndUpdateGrids(List<CustomerGridEntity> grids2Add, List<CustomerGridEntity> grids2Update) {
+    public void addAndUpdateGrids(List<CustomerGridEntity> grids2Add, List<CustomerGridEntity> grids2Update, Date syncEndTime) {
         String dateStr = DateUtils.format(new Date(), "YYYYmmdd");
         for (CustomerGridEntity grid : grids2Add) {
             ScreenCustomerGridEntity screenGrid = screenCustomerGridDao.getByGridId(grid.getId());
             if (screenGrid == null) {
-                ScreenCustomerGridEntity insertOne = new ScreenCustomerGridEntity();
+                screenGrid = new ScreenCustomerGridEntity();
-                insertOne.setAllParentIds(grid.getPids());
+                screenGrid.setAllParentIds(grid.getPids());
-                insertOne.setCustomerId(grid.getCustomerId());
+                screenGrid.setCustomerId(grid.getCustomerId());
-                insertOne.setDataEndTime(dateStr);
+                screenGrid.setDataEndTime(dateStr);
-                insertOne.setGridId(grid.getId());
+                screenGrid.setGridId(grid.getId());
-                insertOne.setGridName(grid.getGridName());
+                screenGrid.setGridName(grid.getGridName());
-                insertOne.setParentAgencyId(grid.getPid());
+                screenGrid.setParentAgencyId(grid.getPid());
-                insertOne.setSourceType(OrgSourceTypeConstant.INTERNAL);
+                screenGrid.setSourceType(OrgSourceTypeConstant.INTERNAL);
-                insertOne.setAreaCode(grid.getAreaCode());
+                screenGrid.setAreaCode(grid.getAreaCode());
-                insertOne.setCode(grid.getCode());
+                screenGrid.setCode(grid.getCode());
-                screenCustomerGridDao.insert(insertOne);
+                //如果没有区划 则默认暂时不显示
+                if (StringUtils.isEmpty(grid.getAreaCode())) {
+                    screenGrid.setIsDisplay(NumConstant.ZERO_STR);
+                } else {
+                    screenGrid.setIsDisplay(NumConstant.ONE_STR);
+                }
+                screenCustomerGridDao.insert(screenGrid);
+            } else {
+                screenGrid.setAllParentIds(grid.getPids());
+                screenGrid.setCustomerId(grid.getCustomerId());
+                screenGrid.setDataEndTime(dateStr);
+                screenGrid.setGridId(grid.getId());
+                screenGrid.setGridName(grid.getGridName());
+                screenGrid.setParentAgencyId(grid.getPid());
+                screenGrid.setSourceType(OrgSourceTypeConstant.INTERNAL);
+                screenGrid.setAreaCode(grid.getAreaCode());
+                screenGrid.setCode(grid.getCode());
+                screenCustomerGridDao.updateById(screenGrid);
             }
         }
 
@@ -109,6 +131,19 @@ public class ScreenCustomerGridServiceImpl extends BaseServiceImpl<ScreenCustome
                 }
             }
         }
+
+        // 保存同步记录
+        Date lastSyncEndTime = dimLastSyncRecordDao.getDimLastSyncEndTime("grid");
+        if (lastSyncEndTime == null) {
+            // 首次初始化，没有上次执行记录,插入一条
+            DimLastSyncRecordEntity insertRecord = new DimLastSyncRecordEntity();
+            insertRecord.setSubject("grid");
+            insertRecord.setLastSyncTime(syncEndTime);
+            dimLastSyncRecordDao.insert(insertRecord);
+        } else {
+            // 更新
+            dimLastSyncRecordDao.updateLastSyncEndTime("grid", syncEndTime);
+        }
     }
 
     /**
@@ -250,4 +285,9 @@ public class ScreenCustomerGridServiceImpl extends BaseServiceImpl<ScreenCustome
         List<ScreenCustomerGridEntity> list = baseDao.selectList(wrapper);
         return ConvertUtils.sourceToTarget(list, ScreenCustomerGridDTO.class).stream().collect(Collectors.toMap(ScreenCustomerGridDTO::getGridId, Function.identity()));
     }
+
+    @Override
+    public Date getLastSyncEndTime() {
+        return dimLastSyncRecordDao.getDimLastSyncEndTime("grid");
+    }
 }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/EIDimServiceImpl.java

@@ -2,9 +2,6 @@ package com.epmet.service.impl;
 
 import com.epmet.dto.screen.ScreenProjectGridDailyDTO;
 import com.epmet.dto.screen.form.GridCenterPointForm;
-import com.epmet.entity.evaluationindex.screen.ScreenCustomerAgencyEntity;
-import com.epmet.entity.evaluationindex.screen.ScreenCustomerDeptEntity;
-import com.epmet.entity.evaluationindex.screen.ScreenCustomerGridEntity;
 import com.epmet.entity.org.CustomerAgencyEntity;
 import com.epmet.entity.org.CustomerDepartmentEntity;
 import com.epmet.entity.org.CustomerGridEntity;
@@ -45,10 +42,11 @@ public class EIDimServiceImpl implements EIDimService {
 
     @Override
     public void initAgencies() {
-        List<CustomerAgencyEntity> agencies2Add = listAgencies2Add();
+        Date statsEndTime = new Date();//DateUtils.integrate(new Date(), DateUtils.DATE_PATTERN);
-        List<CustomerAgencyEntity> agencies2Update = listAgencies2Update();
+        List<CustomerAgencyEntity> agencies2Add = listAgencies2Add(statsEndTime);
+        List<CustomerAgencyEntity> agencies2Update = listAgencies2Update(statsEndTime);
 
-        screenCustomerAgencyService.initAgencies(agencies2Add, agencies2Update);
+        screenCustomerAgencyService.initAgencies(agencies2Add, agencies2Update, statsEndTime);
     }
 
     /**
@@ -56,17 +54,9 @@ public class EIDimServiceImpl implements EIDimService {
      *
      * @return
      */
-    private List<CustomerAgencyEntity> listAgencies2Update() {
+    private List<CustomerAgencyEntity> listAgencies2Update(Date endTime) {
-        ScreenCustomerAgencyEntity e = screenCustomerAgencyService.getLastUpdatedAgency();
+        Date startTime = screenCustomerAgencyService.getDimLastSyncTime();
-
+        return originCustomerAgencyService.listAgenciesByUpdatedTime(startTime, endTime);
-        if (e != null) {
-            // 说明不是第一次初始化
-            Date startTime = e.getUpdatedTime();
-            Date endTime = new Date();
-            return originCustomerAgencyService.listAgenciesByUpdatedTime(startTime, endTime);
-        }
-
-        return new ArrayList<>();
     }
 
     /**
@@ -74,23 +64,17 @@ public class EIDimServiceImpl implements EIDimService {
      *
      * @return
      */
-    private List<CustomerAgencyEntity> listAgencies2Add() {
+    private List<CustomerAgencyEntity> listAgencies2Add(Date endTime) {
-        ScreenCustomerAgencyEntity e = screenCustomerAgencyService.getLastAddedAgency();
+        Date startTime = screenCustomerAgencyService.getDimLastSyncTime();
-
-        Date endTime = new Date();
-        Date startTime = null;
-        if (e != null) {
-            startTime = e.getCreatedTime();
-        }
-
         return originCustomerAgencyService.listAgenciesByCreateTime(null, startTime, endTime);
     }
 
     @Override
     public void initDepartments() {
-        List<CustomerDepartmentEntity> depts2Add = listDepts2Add();
+        Date syncEndTime = new Date();//DateUtils.integrate(new Date(), DateUtils.DATE_PATTERN);
-        List<CustomerDepartmentEntity> depts2Update = listDepts2Update();
+        List<CustomerDepartmentEntity> depts2Add = listDepts2Add(syncEndTime);
-        screenCustomerDeptService.addAndUpdateDepartments(depts2Add, depts2Update);
+        List<CustomerDepartmentEntity> depts2Update = listDepts2Update(syncEndTime);
+        screenCustomerDeptService.addAndUpdateDepartments(depts2Add, depts2Update, syncEndTime);
     }
 
     /**
@@ -98,17 +82,19 @@ public class EIDimServiceImpl implements EIDimService {
      *
      * @return
      */
-    private List<CustomerDepartmentEntity> listDepts2Update() {
+    private List<CustomerDepartmentEntity> listDepts2Update(Date syncEndTime) {
-        ScreenCustomerDeptEntity lastUpdateDept = screenCustomerDeptService.getLastUpdateDept();
+        //ScreenCustomerDeptEntity lastUpdateDept = screenCustomerDeptService.getLastUpdateDept();
-
+        //
-        if (lastUpdateDept != null) {
+        //if (lastUpdateDept != null) {
-            // 不是第一次初始化
+        //    // 不是第一次初始化
-            Date endTime = new Date();
+        //    Date endTime = new Date();
-            Date startTime = lastUpdateDept.getUpdatedTime();
+        //    Date startTime = lastUpdateDept.getUpdatedTime();
-            return originCustomerDepartmentService.listDepartmentsByUpdatedTime(startTime, endTime);
+        //    return originCustomerDepartmentService.listDepartmentsByUpdatedTime(startTime, endTime);
-        }
+        //}
-
+        //return new ArrayList<>();
-        return new ArrayList<>();
+
+        Date lastSyncEndTime = screenCustomerDeptService.getLastSyncEndTime();
+        return originCustomerDepartmentService.listDepartmentsByUpdatedTime(lastSyncEndTime, syncEndTime);
     }
 
     /**
@@ -116,23 +102,25 @@ public class EIDimServiceImpl implements EIDimService {
      *
      * @return
      */
-    private List<CustomerDepartmentEntity> listDepts2Add() {
+    private List<CustomerDepartmentEntity> listDepts2Add(Date syncEndTime) {
-        ScreenCustomerDeptEntity lastAddDept = screenCustomerDeptService.getLastAddDept();
+        //ScreenCustomerDeptEntity lastAddDept = screenCustomerDeptService.getLastAddDept();
-
+        //
-        Date startTime = null;
+        //Date startTime = null;
-        Date endTime = new Date();
+        //Date endTime = new Date();
-        if (lastAddDept != null) {
+        //if (lastAddDept != null) {
-            startTime = lastAddDept.getCreatedTime();
+        //    startTime = lastAddDept.getCreatedTime();
-        }
+        //}
-
+        
-        return originCustomerDepartmentService.listDepartmentsByCreatedTime(null, startTime, endTime);
+        Date startTime = screenCustomerDeptService.getLastSyncEndTime();
+        return originCustomerDepartmentService.listDepartmentsByCreatedTime(null, startTime, syncEndTime);
     }
 
     @Override
     public void initGrids() {
-        List<CustomerGridEntity> grids2Add = listGrids2Add();
+        Date syncEndTime = new Date();//DateUtils.integrate(new Date(), DateUtils.DATE_PATTERN);
-        List<CustomerGridEntity> grids2Update = listGrids2Update();
+        List<CustomerGridEntity> grids2Add = listGrids2Add(syncEndTime);
-        screenCustomerGridService.addAndUpdateGrids(grids2Add, grids2Update);
+        List<CustomerGridEntity> grids2Update = listGrids2Update(syncEndTime);
+        screenCustomerGridService.addAndUpdateGrids(grids2Add, grids2Update, syncEndTime);
     }
 
     @Override
@@ -159,26 +147,32 @@ public class EIDimServiceImpl implements EIDimService {
         return result;
     }
 
-    private List<CustomerGridEntity> listGrids2Update() {
+    private List<CustomerGridEntity> listGrids2Update(Date syncEndTime) {
-        ScreenCustomerGridEntity lastUpdateGrid = screenCustomerGridService.getLastUpdateGrid();
+        //ScreenCustomerGridEntity lastUpdateGrid = screenCustomerGridService.getLastUpdateGrid();
-        if (lastUpdateGrid != null) {
+        //if (lastUpdateGrid != null) {
-            Date endTime = new Date();
+        //    Date endTime = new Date();
-            Date startTime = lastUpdateGrid.getUpdatedTime();
+        //    Date startTime = lastUpdateGrid.getUpdatedTime();
-            return customerGridService.listUpdatedGridsByUpdateTime(startTime, endTime);
+        //    return customerGridService.listUpdatedGridsByUpdateTime(startTime, endTime);
-        }
+        //}
-
+        //
-        return new ArrayList();
+        //return new ArrayList();
+
+        Date lastSyncEndTime = screenCustomerGridService.getLastSyncEndTime();
+        return customerGridService.listUpdatedGridsByUpdateTime(lastSyncEndTime, syncEndTime);
     }
 
-    private List<CustomerGridEntity> listGrids2Add() {
+    private List<CustomerGridEntity> listGrids2Add(Date syncEndTime) {
-        ScreenCustomerGridEntity lastAddGrid = screenCustomerGridService.getLastAddGrid();
+        //ScreenCustomerGridEntity lastAddGrid = screenCustomerGridService.getLastAddGrid();
-
+        //
-        Date endTime = new Date();
+        //Date endTime = new Date();
-        Date startTime = null;
+        //Date startTime = null;
-        if (lastAddGrid != null) {
+        //if (lastAddGrid != null) {
-            startTime = lastAddGrid.getCreatedTime();
+        //    startTime = lastAddGrid.getCreatedTime();
-        }
+        //}
-
+        //
-        return customerGridService.listGridsByCreateTime(null, startTime, endTime);
+        //return customerGridService.listGridsByCreateTime(null, startTime, endTime);
+
+        Date lastSyncEndTime = screenCustomerGridService.getLastSyncEndTime();
+        return customerGridService.listGridsByCreateTime(null, lastSyncEndTime, syncEndTime);
     }
 }

epmet-module/data-statistical/data-statistical-server/src/main/resources/db/migration/V0.0.27__create_indexdimsyncrecord.sql

@@ -0,0 +1,15 @@
+create table dim_last_sync_record
+(
+    `ID`             varchar(64) primary key comment '主键',
+    `SUBJECT`        varchar(32) not null comment '主体。agency:机关，department:部门，grid:网格',
+    `LAST_SYNC_TIME` datetime    not null comment '最后一次同步时间',
+    `DEL_FLAG`       char(1)     NOT NULL DEFAULT '0' COMMENT '删除标识 0未删除；1已删除',
+    `REVISION`       int(11)     NOT NULL COMMENT '乐观锁',
+    `CREATED_BY`     varchar(64) NOT NULL COMMENT '创建人',
+    `CREATED_TIME`   datetime    NOT NULL COMMENT '创建时间',
+    `UPDATED_BY`     varchar(64) NOT NULL COMMENT '更新人',
+    `UPDATED_TIME`   datetime    NOT NULL COMMENT '更新时间'
+) default character set utf8mb4 engine InnoDB;
+
+create unique index dim_last_sync_record_SUBJECT_uindex
+	on dim_last_sync_record (SUBJECT);

epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/evaluationindex/screen/DimLastSyncRecordDao.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+
+<mapper namespace="com.epmet.dao.evaluationindex.screen.DimLastSyncRecordDao">
+
+    <resultMap type="com.epmet.entity.evaluationindex.screen.DimLastSyncRecordEntity" id="dimLastSyncRecordMap">
+        <result property="id" column="ID"/>
+        <result property="subject" column="SUBJECT"/>
+        <result property="lastSyncTime" column="LAST_SYNC_TIME"/>
+        <result property="delFlag" column="DEL_FLAG"/>
+        <result property="revision" column="REVISION"/>
+        <result property="createdBy" column="CREATED_BY"/>
+        <result property="createdTime" column="CREATED_TIME"/>
+        <result property="updatedBy" column="UPDATED_BY"/>
+        <result property="updatedTime" column="UPDATED_TIME"/>
+    </resultMap>
+    <update id="updateLastSyncEndTime">
+        update dim_last_sync_record
+        set LAST_SYNC_TIME = #{statsEndTime}
+        where SUBJECT = #{subject}
+          and DEL_FLAG = '0'
+    </update>
+
+    <!--查询最后一次维度同步时间-->
+    <select id="getDimLastSyncEndTime" resultType="java.util.Date">
+        select LAST_SYNC_TIME
+        from dim_last_sync_record
+        where SUBJECT = #{subject}
+          and DEL_FLAG = '0'
+    </select>
+</mapper>

epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/org/CustomerDepartmentDao.xml

@@ -44,7 +44,9 @@
         from customer_department cd
         LEFT JOIN customer_agency ca ON (
             cd.AGENCY_ID = ca.id and ca.DEL_FLAG='0')
-        where cd.UPDATED_TIME > #{startTime}
+        where
-            and cd.UPDATED_TIME <![CDATA[<=]]> #{endTime}
+            cd.CREATED_TIME <![CDATA[<]]> #{startTime}
+            and cd.UPDATED_TIME >= #{startTime}
+            and cd.UPDATED_TIME <![CDATA[<]]> #{endTime}
     </select>
 </mapper>

epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/org/CustomerGridDao.xml

@@ -71,6 +71,7 @@
         pids
         FROM customer_grid
         WHERE  SYNC_FLAG='1'
+        AND CREATED_TIME <![CDATA[<]]> #{startTime}
         AND UPDATED_TIME >= #{startTime}
         AND UPDATED_TIME <![CDATA[<]]> #{endTime}
     </select>

epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/org/StatsCustomerAgencyDao.xml

@@ -102,9 +102,10 @@
             district,
             PARENT_AREA_CODE as parentAreaCode
             from customer_agency
-            where  SYNC_FLAG='1'
+        where  SYNC_FLAG='1'
+            AND CREATED_TIME <![CDATA[<]]> #{startTime}
             AND UPDATED_TIME >= #{startTime}
-            and UPDATED_TIME <![CDATA[<=]]> #{endTime}
+            AND UPDATED_TIME <![CDATA[<]]> #{endTime}
     </select>
 
     <!-- 查询客户所属区域编码 -->