screen_project_data表数据抽取添加逻辑修改

eventreport_pingyin表批量插入改成单条插入

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/dao/evaluationindex/screen/ScreenProjectDataDao.java

@@ -133,4 +133,6 @@ public interface ScreenProjectDataDao extends BaseDao<ScreenProjectDataEntity> {
      * @Date 2022/3/16 9:48
      */
     void updateHistoryData(CustomerStaffGridDTO dto);
+
+    int getProjectCount(@Param("customerId") String customerId,@Param("dateId") String dateId);
 }

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/evaluationindex/screen/impl/ScreenProjectDataServiceImpl.java

@@ -260,10 +260,21 @@ public class ScreenProjectDataServiceImpl extends BaseServiceImpl<ScreenProjectD
         //库中无数据，范围性删除无意义，只是针对在数据库有历史数据的情况下，保证对同一天进行反
         //复采集时数不会重复
         if(!CollectionUtils.isEmpty(meta)){
+            int count = baseDao.getProjectCount(customerId, dateId);
+            if (meta.size() < count) {
+                //meta的数据量小于表里的数据量，意味着指定部分数据进行重新抽取，只需要删除指定的项目数据即可
+                List<String> projectIds = meta.stream().map(ScreenProjectDataDTO::getProjectId).collect(Collectors.toList());
+                List<List<String>> projectPartition = ListUtils.partition(projectIds, NumConstant.FIFTY);
+                projectPartition.forEach(part -> {
+                    baseDao.deleteByProjectIds(customerId, part);
+                });
+            } else {
                 int affectedRows = baseDao.deleteByProjectTimeAndCustomerId(customerId,dateId);
                 while (affectedRows > NumConstant.ZERO) {
                     affectedRows = baseDao.deleteByProjectTimeAndCustomerId(customerId, dateId);
                 }
+            }
+
             List<List<ScreenProjectDataDTO>> partition = ListUtils.partition(meta, NumConstant.ONE_HUNDRED);
             partition.forEach(part -> {
                 List<ScreenProjectDataEntity> list = ConvertUtils.sourceToTarget(part, ScreenProjectDataEntity.class);

epmet-module/data-statistical/data-statistical-server/src/main/resources/mapper/evaluationindex/screen/ScreenProjectDataDao.xml

@@ -539,4 +539,10 @@
             AND ORG_ID = #{agencyId}
             AND ORIGIN = 'agency'
     </update>
+    <select id="getProjectCount" resultType="int">
+        select count(ID)
+        from screen_project_data
+        where customer_id = #{customerId}
+        and DATA_END_TIME = #{dateId}
+    </select>
 </mapper>

epmet-module/open-data-worker/open-data-worker-server/src/main/java/com/epmet/opendata/service/impl/EventreportPingyinServiceImpl.java

@@ -162,7 +162,13 @@ public class EventreportPingyinServiceImpl extends BaseServiceImpl<EventreportPi
                 }
             });
         } else {
-            insertBatch(entityList);
+            for (EventreportPingyinEntity entity : entityList) {
+                try {
+                    baseDao.insert(entity);
+                } catch (Exception e) {
+                    log.error(e.toString());
+                }
+            }
         }