Browse Source

爬取新增范围

master
lichao 2 years ago
parent
commit
f9b699e450
  1. 2
      epmet-user/epmet-user-server/src/main/java/com/epmet/entity/IcResiUserTaskLogEntity.java
  2. 123
      epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/DataSyncConfigServiceImpl.java
  3. 1
      epmet-user/epmet-user-server/src/main/resources/mapper/IcResiUserTaskLogDao.xml

2
epmet-user/epmet-user-server/src/main/java/com/epmet/entity/IcResiUserTaskLogEntity.java

@ -41,4 +41,6 @@ public class IcResiUserTaskLogEntity extends BaseEpmetEntity {
*/ */
private String type; private String type;
private String dataScope;
} }

123
epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/DataSyncConfigServiceImpl.java

@ -278,32 +278,39 @@ public class DataSyncConfigServiceImpl extends BaseServiceImpl<DataSyncConfigDao
return; return;
} }
LambdaQueryWrapper<IcResiUserDeadTaskLogEntity> logEntityLambdaQueryWrapper = new LambdaQueryWrapper<IcResiUserDeadTaskLogEntity>().orderByDesc(IcResiUserDeadTaskLogEntity::getCreatedTime).last("limit 1");
IcResiUserDeadTaskLogEntity logEntity = icResiUserDeadTaskLogDao.selectOne(logEntityLambdaQueryWrapper);
List<NatUserInfoResultDTO> userInfoResultDTOS = new ArrayList<>();
boolean isNum = false;
int start = 0;
if (logEntity == null){
userInfoResultDTOS = getNatUserInfoFromDbForSiWang(0,50000);
if (userInfoResultDTOS.size() < 50000){
isNum = true;
}
}else{
if (logEntity.getDataType() == 1){
userInfoResultDTOS = getNatUserInfoFromDbForSiWang(logEntity.getDataCount(),50000);
isNum = true;
}else if (logEntity.getDataType() == 0){
start = logEntity.getDataCount();
}
}
IcResiUserDeadTaskLogEntity newLog = new IcResiUserDeadTaskLogEntity();
newLog.setCreatedTime(new Date());
List<NatUserInfoResultDTO> dbResiList = null; List<NatUserInfoResultDTO> dbResiList = null;
for (DataSyncConfigDTO config : configData) { for (DataSyncConfigDTO config : configData) {
String scope = config.getScopeList().stream().map(DataSyncScopeDTO::getId).collect(Collectors.joining(","));
// 查询上一次定时任务 居民位置记录
LambdaQueryWrapper<IcResiUserTaskLogEntity> logEntityLambdaQueryWrapper = new LambdaQueryWrapper<IcResiUserTaskLogEntity>().eq(IcResiUserTaskLogEntity::getDataScope,scope).eq(IcResiUserTaskLogEntity::getType,"siwang").orderByDesc(IcResiUserTaskLogEntity::getCreatedTime).last("limit 1");
IcResiUserTaskLogEntity logEntity = resiUserTaskLogDao.selectOne(logEntityLambdaQueryWrapper);
List<NatUserInfoResultDTO> userInfoResultDTOS = new ArrayList<>();
boolean isNum = false;
int start = 0;
if (logEntity == null){
userInfoResultDTOS = getNatUserInfoFromDbForSiWang(0,50000);
if (userInfoResultDTOS.size() < 50000){
isNum = true;
}
}else{
if (logEntity.getDataType() == 1){
userInfoResultDTOS = getNatUserInfoFromDbForSiWang(logEntity.getDataCount(),50000);
isNum = true;
}else if (logEntity.getDataType() == 0){
start = logEntity.getDataCount();
}
}
IcResiUserTaskLogEntity newLog = new IcResiUserTaskLogEntity();
newLog.setCreatedTime(new Date());
newLog.setDataScope(scope);
newLog.setType("siwang");
// 设置查询数据范围 // 设置查询数据范围
formDTO.setOrgList(config.getScopeList()); formDTO.setOrgList(config.getScopeList());
DataSyncEnum anEnum = DataSyncEnum.getEnum(config.getDataCode()); DataSyncEnum anEnum = DataSyncEnum.getEnum(config.getDataCode());
@ -347,7 +354,7 @@ public class DataSyncConfigServiceImpl extends BaseServiceImpl<DataSyncConfigDao
try { try {
//查询正常状态的居民 //查询正常状态的居民
siWang(dbResiList); siWang(dbResiList);
icResiUserDeadTaskLogDao.insert(newLog); resiUserTaskLogDao.insert(newLog);
log.info("======siWang信息拉取结束======"); log.info("======siWang信息拉取结束======");
} catch (Exception e) { } catch (Exception e) {
log.error("death thread execute exception", e); log.error("death thread execute exception", e);
@ -1662,42 +1669,48 @@ public class DataSyncConfigServiceImpl extends BaseServiceImpl<DataSyncConfigDao
return; return;
} }
// 查询上一次定时任务 居民位置记录
LambdaQueryWrapper<IcResiUserTaskLogEntity> logEntityLambdaQueryWrapper = new LambdaQueryWrapper<IcResiUserTaskLogEntity>().eq(IcResiUserTaskLogEntity::getType,"hunyin").orderByDesc(IcResiUserTaskLogEntity::getCreatedTime).last("limit 1");
IcResiUserTaskLogEntity logEntity = resiUserTaskLogDao.selectOne(logEntityLambdaQueryWrapper);
// userInfoResultDTOS 待处理的数据存放
List<NatUserInfoResultDTO> userInfoResultDTOS = new ArrayList<>();
boolean isNum = false;
int start = 0;
// 如果没有log数据 说明第一次进入循环
if (logEntity == null){
// 从0开始 优先获取规定的几个社区
userInfoResultDTOS = getNatUserInfoFromDbForSiWang(0,50000);
// 如果不满5000条数据 下面再从别的社区取 isNum = true
if (userInfoResultDTOS.size() < 50000){
isNum = true;
}
}else{
// 如果有log数据 判断一下上次停留在特殊数据还是普通数据 1是特殊数据(优先的几个社区)
if (logEntity.getDataType() == 1){
// 从上次停止的数据开始获取
userInfoResultDTOS = getNatUserInfoFromDbForSiWang(logEntity.getDataCount(),50000);
// 下面再从别的社区取 isNum = true 凑够50000
isNum = true;
}else if (logEntity.getDataType() == 0){
// 如果是普通数据 直接从次停留在特殊数据获取
start = logEntity.getDataCount();
}
}
// 建立一个最后插入log的标志
IcResiUserTaskLogEntity newLog = new IcResiUserTaskLogEntity();
newLog.setCreatedTime(new Date());
newLog.setType("hunyin");
List<NatUserInfoResultDTO> dbResiList = null; List<NatUserInfoResultDTO> dbResiList = null;
for (DataSyncConfigDTO config : configData) { for (DataSyncConfigDTO config : configData) {
String scope = config.getScopeList().stream().map(DataSyncScopeDTO::getId).collect(Collectors.joining(","));
// 查询上一次定时任务 居民位置记录
LambdaQueryWrapper<IcResiUserTaskLogEntity> logEntityLambdaQueryWrapper = new LambdaQueryWrapper<IcResiUserTaskLogEntity>().eq(IcResiUserTaskLogEntity::getDataScope,scope).eq(IcResiUserTaskLogEntity::getType,"hunyin").orderByDesc(IcResiUserTaskLogEntity::getCreatedTime).last("limit 1");
IcResiUserTaskLogEntity logEntity = resiUserTaskLogDao.selectOne(logEntityLambdaQueryWrapper);
// userInfoResultDTOS 待处理的数据存放
List<NatUserInfoResultDTO> userInfoResultDTOS = new ArrayList<>();
boolean isNum = false;
int start = 0;
// 如果没有log数据 说明第一次进入循环
if (logEntity == null){
// 从0开始 优先获取规定的几个社区
userInfoResultDTOS = getNatUserInfoFromDbForSiWang(0,50000);
// 如果不满5000条数据 下面再从别的社区取 isNum = true
if (userInfoResultDTOS.size() < 50000){
isNum = true;
}
}else{
// 如果有log数据 判断一下上次停留在特殊数据还是普通数据 1是特殊数据(优先的几个社区)
if (logEntity.getDataType() == 1){
// 从上次停止的数据开始获取
userInfoResultDTOS = getNatUserInfoFromDbForSiWang(logEntity.getDataCount(),50000);
// 下面再从别的社区取 isNum = true 凑够50000
isNum = true;
}else if (logEntity.getDataType() == 0){
// 如果是普通数据 直接从次停留在特殊数据获取
start = logEntity.getDataCount();
}
}
// 建立一个最后插入log的标志
IcResiUserTaskLogEntity newLog = new IcResiUserTaskLogEntity();
newLog.setCreatedTime(new Date());
newLog.setType("hunyin");
newLog.setDataScope(scope);
// 设置查询数据范围 // 设置查询数据范围
formDTO.setOrgList(config.getScopeList()); formDTO.setOrgList(config.getScopeList());
DataSyncEnum anEnum = DataSyncEnum.getEnum(config.getDataCode()); DataSyncEnum anEnum = DataSyncEnum.getEnum(config.getDataCode());

1
epmet-user/epmet-user-server/src/main/resources/mapper/IcResiUserTaskLogDao.xml

@ -8,6 +8,7 @@
<result property="customerId" column="CUSTOMER_ID"/> <result property="customerId" column="CUSTOMER_ID"/>
<result property="dataType" column="DATA_TYPE"/> <result property="dataType" column="DATA_TYPE"/>
<result property="dataCount" column="DATA_COUNT"/> <result property="dataCount" column="DATA_COUNT"/>
<result property="dataScope" column="DATA_SCOPE"/>
<result property="type" column="TYPE"/> <result property="type" column="TYPE"/>
<result property="delFlag" column="DEL_FLAG"/> <result property="delFlag" column="DEL_FLAG"/>
<result property="revision" column="REVISION"/> <result property="revision" column="REVISION"/>

Loading…
Cancel
Save