elink-star
/
epmet-cloud-lingshan
forked from luyan/epmet-cloud-lingshan
12
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

爬取新增范围

master
lichao 2 years ago
parent
a7eaf9a027
commit
f9b699e450
3 changed files with 71 additions and 55 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      epmet-user/epmet-user-server/src/main/java/com/epmet/entity/IcResiUserTaskLogEntity.java
  2. 123
      epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/DataSyncConfigServiceImpl.java
  3. 1
      epmet-user/epmet-user-server/src/main/resources/mapper/IcResiUserTaskLogDao.xml

2
epmet-user/epmet-user-server/src/main/java/com/epmet/entity/IcResiUserTaskLogEntity.java
View File

@ -41,4 +41,6 @@ public class IcResiUserTaskLogEntity extends BaseEpmetEntity {
*/
private String type;
private String dataScope;
}

123
epmet-user/epmet-user-server/src/main/java/com/epmet/service/impl/DataSyncConfigServiceImpl.java
View File

@ -278,32 +278,39 @@ public class DataSyncConfigServiceImpl extends BaseServiceImpl<DataSyncConfigDao
return;
}
LambdaQueryWrapper<IcResiUserDeadTaskLogEntity> logEntityLambdaQueryWrapper = new LambdaQueryWrapper<IcResiUserDeadTaskLogEntity>().orderByDesc(IcResiUserDeadTaskLogEntity::getCreatedTime).last("limit 1");
IcResiUserDeadTaskLogEntity logEntity = icResiUserDeadTaskLogDao.selectOne(logEntityLambdaQueryWrapper);
List<NatUserInfoResultDTO> userInfoResultDTOS = new ArrayList<>();
boolean isNum = false;
int start = 0;
if (logEntity == null){
userInfoResultDTOS = getNatUserInfoFromDbForSiWang(0,50000);
if (userInfoResultDTOS.size() < 50000){
isNum = true;
}
}else{
if (logEntity.getDataType() == 1){
userInfoResultDTOS = getNatUserInfoFromDbForSiWang(logEntity.getDataCount(),50000);
isNum = true;
}else if (logEntity.getDataType() == 0){
start = logEntity.getDataCount();
}
}
IcResiUserDeadTaskLogEntity newLog = new IcResiUserDeadTaskLogEntity();
newLog.setCreatedTime(new Date());
List<NatUserInfoResultDTO> dbResiList = null;
for (DataSyncConfigDTO config : configData) {
String scope = config.getScopeList().stream().map(DataSyncScopeDTO::getId).collect(Collectors.joining(","));
// 查询上一次定时任务 居民位置记录
LambdaQueryWrapper<IcResiUserTaskLogEntity> logEntityLambdaQueryWrapper = new LambdaQueryWrapper<IcResiUserTaskLogEntity>().eq(IcResiUserTaskLogEntity::getDataScope,scope).eq(IcResiUserTaskLogEntity::getType,"siwang").orderByDesc(IcResiUserTaskLogEntity::getCreatedTime).last("limit 1");
IcResiUserTaskLogEntity logEntity = resiUserTaskLogDao.selectOne(logEntityLambdaQueryWrapper);
List<NatUserInfoResultDTO> userInfoResultDTOS = new ArrayList<>();
boolean isNum = false;
int start = 0;
if (logEntity == null){
userInfoResultDTOS = getNatUserInfoFromDbForSiWang(0,50000);
if (userInfoResultDTOS.size() < 50000){
isNum = true;
}
}else{
if (logEntity.getDataType() == 1){
userInfoResultDTOS = getNatUserInfoFromDbForSiWang(logEntity.getDataCount(),50000);
isNum = true;
}else if (logEntity.getDataType() == 0){
start = logEntity.getDataCount();
}
}
IcResiUserTaskLogEntity newLog = new IcResiUserTaskLogEntity();
newLog.setCreatedTime(new Date());
newLog.setDataScope(scope);
newLog.setType("siwang");
// 设置查询数据范围
formDTO.setOrgList(config.getScopeList());
DataSyncEnum anEnum = DataSyncEnum.getEnum(config.getDataCode());
@ -347,7 +354,7 @@ public class DataSyncConfigServiceImpl extends BaseServiceImpl<DataSyncConfigDao
try {
//查询正常状态的居民
siWang(dbResiList);
icResiUserDeadTaskLogDao.insert(newLog);
resiUserTaskLogDao.insert(newLog);
log.info("======siWang信息拉取结束======");
} catch (Exception e) {
log.error("death thread execute exception", e);
@ -1662,42 +1669,48 @@ public class DataSyncConfigServiceImpl extends BaseServiceImpl<DataSyncConfigDao
return;
}
// 查询上一次定时任务 居民位置记录
LambdaQueryWrapper<IcResiUserTaskLogEntity> logEntityLambdaQueryWrapper = new LambdaQueryWrapper<IcResiUserTaskLogEntity>().eq(IcResiUserTaskLogEntity::getType,"hunyin").orderByDesc(IcResiUserTaskLogEntity::getCreatedTime).last("limit 1");
IcResiUserTaskLogEntity logEntity = resiUserTaskLogDao.selectOne(logEntityLambdaQueryWrapper);
// userInfoResultDTOS 待处理的数据存放
List<NatUserInfoResultDTO> userInfoResultDTOS = new ArrayList<>();
boolean isNum = false;
int start = 0;
// 如果没有log数据 说明第一次进入循环
if (logEntity == null){
// 从0开始 优先获取规定的几个社区
userInfoResultDTOS = getNatUserInfoFromDbForSiWang(0,50000);
// 如果不满5000条数据 下面再从别的社区取 isNum = true
if (userInfoResultDTOS.size() < 50000){
isNum = true;
}
}else{
// 如果有log数据 判断一下上次停留在特殊数据还是普通数据 1是特殊数据（优先的几个社区）
if (logEntity.getDataType() == 1){
// 从上次停止的数据开始获取
userInfoResultDTOS = getNatUserInfoFromDbForSiWang(logEntity.getDataCount(),50000);
// 下面再从别的社区取 isNum = true 凑够50000
isNum = true;
}else if (logEntity.getDataType() == 0){
// 如果是普通数据 直接从次停留在特殊数据获取
start = logEntity.getDataCount();
}
}
// 建立一个最后插入log的标志
IcResiUserTaskLogEntity newLog = new IcResiUserTaskLogEntity();
newLog.setCreatedTime(new Date());
newLog.setType("hunyin");
List<NatUserInfoResultDTO> dbResiList = null;
for (DataSyncConfigDTO config : configData) {
String scope = config.getScopeList().stream().map(DataSyncScopeDTO::getId).collect(Collectors.joining(","));
// 查询上一次定时任务 居民位置记录
LambdaQueryWrapper<IcResiUserTaskLogEntity> logEntityLambdaQueryWrapper = new LambdaQueryWrapper<IcResiUserTaskLogEntity>().eq(IcResiUserTaskLogEntity::getDataScope,scope).eq(IcResiUserTaskLogEntity::getType,"hunyin").orderByDesc(IcResiUserTaskLogEntity::getCreatedTime).last("limit 1");
IcResiUserTaskLogEntity logEntity = resiUserTaskLogDao.selectOne(logEntityLambdaQueryWrapper);
// userInfoResultDTOS 待处理的数据存放
List<NatUserInfoResultDTO> userInfoResultDTOS = new ArrayList<>();
boolean isNum = false;
int start = 0;
// 如果没有log数据 说明第一次进入循环
if (logEntity == null){
// 从0开始 优先获取规定的几个社区
userInfoResultDTOS = getNatUserInfoFromDbForSiWang(0,50000);
// 如果不满5000条数据 下面再从别的社区取 isNum = true
if (userInfoResultDTOS.size() < 50000){
isNum = true;
}
}else{
// 如果有log数据 判断一下上次停留在特殊数据还是普通数据 1是特殊数据（优先的几个社区）
if (logEntity.getDataType() == 1){
// 从上次停止的数据开始获取
userInfoResultDTOS = getNatUserInfoFromDbForSiWang(logEntity.getDataCount(),50000);
// 下面再从别的社区取 isNum = true 凑够50000
isNum = true;
}else if (logEntity.getDataType() == 0){
// 如果是普通数据 直接从次停留在特殊数据获取
start = logEntity.getDataCount();
}
}
// 建立一个最后插入log的标志
IcResiUserTaskLogEntity newLog = new IcResiUserTaskLogEntity();
newLog.setCreatedTime(new Date());
newLog.setType("hunyin");
newLog.setDataScope(scope);
// 设置查询数据范围
formDTO.setOrgList(config.getScopeList());
DataSyncEnum anEnum = DataSyncEnum.getEnum(config.getDataCode());

1
epmet-user/epmet-user-server/src/main/resources/mapper/IcResiUserTaskLogDao.xml
View File

@ -8,6 +8,7 @@
<result property="customerId" column="CUSTOMER_ID"/>
<result property="dataType" column="DATA_TYPE"/>
<result property="dataCount" column="DATA_COUNT"/>
<result property="dataScope" column="DATA_SCOPE"/>
<result property="type" column="TYPE"/>
<result property="delFlag" column="DEL_FLAG"/>
<result property="revision" column="REVISION"/>

Write Preview
Loading…
Cancel
Save