Merge remote-tracking branch 'remotes/origin/dev_bugfix_ljj' into dev

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/stats/FactPartyMemberVanguardAgencyDailyDao.java

@@ -37,8 +37,9 @@ public interface FactPartyMemberVanguardAgencyDailyDao extends BaseDao<FactParty
 	 * @date 2021/5/17 17:19
 	 * @param customerId
 	 * @param dateId
+	 * @param limit
 	 * @return int
 	 */
-	int deleteDailyData(@Param("customerId") String customerId, @Param("dateId")  String dateId);
-	
-}
+	int deleteDailyData(@Param("customerId") String customerId, @Param("dateId") String dateId, Integer limit);
+
+}

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/stats/FactPartyMemberVanguardGridDailyDao.java

@@ -36,7 +36,8 @@ public interface FactPartyMemberVanguardGridDailyDao extends BaseDao<FactPartyMe
 	 * @date 2021/5/17 17:19
 	 * @param customerId
 	 * @param dateId
+	 * @param limit
 	 * @return int
 	 */
-	int deleteDailyData(@Param("customerId") String customerId, @Param("dateId")  String dateId);
-}
+	int deleteDailyData(@Param("customerId") String customerId, @Param("dateId") String dateId, Integer limit);
+}

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsPartyMemberVanguardServiceImpl.java

@@ -6,7 +6,6 @@ import com.epmet.constant.OrgTypeConstant;
 import com.epmet.dto.StatsFormDTO;
 import com.epmet.dto.extract.result.OrgStatisticsResultDTO;
 import com.epmet.dto.stats.DimAgencyDTO;
-import com.epmet.dto.stats.DimGridDTO;
 import com.epmet.entity.stats.DimGridEntity;
 import com.epmet.entity.stats.FactPartyMemberVanguardAgencyDailyEntity;
 import com.epmet.entity.stats.FactPartyMemberVanguardGridDailyEntity;
@@ -16,6 +15,7 @@ import com.epmet.service.stats.*;
 import com.epmet.util.DimIdGenerator;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.collections4.ListUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Service;
 
@@ -191,10 +191,20 @@ public class StatsPartyMemberVanguardServiceImpl implements StatsPartyMemberVang
 					}
 				});
 			}
-			//删除旧数据
-			factPartyMemberVanguardAgencyDailyService.deleteDailyData(customerId, dimId.getDateId());
-			//批量插入
-			factPartyMemberVanguardAgencyDailyService.insertBatch(vanguardList);
+			//删除旧数据 每次删除1000
+			Integer effectRow;
+			do {
+				effectRow = factPartyMemberVanguardAgencyDailyService.deleteDailyData(customerId, dimId.getDateId(), NumConstant.ONE_THOUSAND);
+			} while (effectRow != null && effectRow > NumConstant.ZERO);
+
+			//批量插入 每次100
+			if (CollectionUtils.isEmpty(vanguardList)) {
+				log.warn("customerAgencyStats insert list is empty,customerId:{},dateId:{}", customerId, dimId.getDateId());
+			}
+
+			ListUtils.partition(vanguardList, NumConstant.ONE_HUNDRED)
+					.forEach(part -> factPartyMemberVanguardAgencyDailyService.insertBatch(part));
+
 		}
 	}
 	/**
@@ -327,10 +337,19 @@ public class StatsPartyMemberVanguardServiceImpl implements StatsPartyMemberVang
 					}
 				});
 			}
-			//删除旧数据
-			factPartyMemberVanguardGridDailyService.deleteDailyData(customerId, dimId.getDateId());
-			//批量插入
-			factPartyMemberVanguardGridDailyService.insertBatch(vanguardList);
+
+			//删除旧数据 每次删除1000
+			Integer effectRow;
+			do {
+				effectRow = factPartyMemberVanguardGridDailyService.deleteDailyData(customerId, dimId.getDateId(), NumConstant.ONE_THOUSAND);
+			} while (effectRow != null && effectRow > NumConstant.ZERO);
+
+			//批量插入 每次100
+			if (CollectionUtils.isEmpty(vanguardList)){
+				log.warn("customerGridStats insert list is empty,customerId:{},dateId:{}", customerId, dimId.getDateId());
+			}
+			ListUtils.partition(vanguardList, NumConstant.ONE_HUNDRED)
+					.forEach(part -> factPartyMemberVanguardGridDailyService.insertBatch(part));
 		}
 	}
 

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/stats/FactPartyMemberVanguardAgencyDailyService.java

@@ -99,7 +99,8 @@ public interface FactPartyMemberVanguardAgencyDailyService extends BaseService<F
      * @date 2021/5/17 15:55
      * @param customerId
      * @param dateId
+     * @param limit
      * @return void
      */
-    void deleteDailyData(String customerId, String dateId);
-}
+    Integer deleteDailyData(String customerId, String dateId, Integer limit);
+}

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/stats/FactPartyMemberVanguardGridDailyService.java

@@ -99,7 +99,8 @@ public interface FactPartyMemberVanguardGridDailyService extends BaseService<Fac
      * @date 2021/5/17 15:55
      * @param customerId
      * @param dateId
+     * @param limit
      * @return void
      */
-    void deleteDailyData(String customerId, String dateId);
-}
+    Integer deleteDailyData(String customerId, String dateId, Integer limit);
+}

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/stats/impl/FactPartyMemberVanguardAgencyDailyServiceImpl.java

@@ -105,14 +105,15 @@ public class FactPartyMemberVanguardAgencyDailyServiceImpl extends BaseServiceIm
      *
      * @param customerId
      * @param dateId
+     * @param limit
      * @return void
      * @author zhaoqifeng
      * @date 2021/5/17 15:55
      */
     @Override
     @Transactional(rollbackFor = Exception.class)
-    public void deleteDailyData(String customerId, String dateId) {
-        baseDao.deleteDailyData(customerId, dateId);
+    public Integer deleteDailyData(String customerId, String dateId, Integer limit) {
+        return baseDao.deleteDailyData(customerId, dateId, limit);
     }
 
-}
+}

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/stats/impl/FactPartyMemberVanguardGridDailyServiceImpl.java

@@ -105,14 +105,15 @@ public class FactPartyMemberVanguardGridDailyServiceImpl extends BaseServiceImpl
      *
      * @param customerId
      * @param dateId
+     * @param limit
      * @return void
      * @author zhaoqifeng
      * @date 2021/5/17 15:55
      */
     @Override
     @Transactional(rollbackFor = Exception.class)
-    public void deleteDailyData(String customerId, String dateId) {
-        baseDao.deleteDailyData(customerId, dateId);
+    public Integer deleteDailyData(String customerId, String dateId, Integer limit) {
+        return baseDao.deleteDailyData(customerId, dateId, limit);
     }
 
-}
+}

epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/stats/FactPartyMemberVanguardAgencyDailyDao.xml

@@ -34,7 +34,8 @@
     </resultMap>
     <delete id="deleteDailyData">
         delete from fact_party_member_vanguard_agency_daily where customer_id = #{customerId} and DATE_ID = #{dateId}
+        limit #{limit}
     </delete>
 
 
-</mapper>
+</mapper>

epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/stats/FactPartyMemberVanguardGridDailyDao.xml

@@ -34,7 +34,8 @@
     </resultMap>
     <delete id="deleteDailyData">
         delete from fact_party_member_vanguard_grid_daily where customer_id = #{customerId} and DATE_ID = #{dateId}
+        limit #{limit}
     </delete>
 
 
-</mapper>
+</mapper>