jianjun
5 years ago
14 changed files with 332 additions and 171 deletions
@ -1,30 +0,0 @@ |
|||||
package com.epmet.feign; |
|
||||
|
|
||||
import com.epmet.commons.tools.constant.ServiceConstant; |
|
||||
import com.epmet.commons.tools.utils.Result; |
|
||||
import com.epmet.dto.extract.form.ExtractFormDTO; |
|
||||
import com.epmet.feign.impl.FactExtractOpenFeignClientFallBack; |
|
||||
import org.springframework.cloud.openfeign.FeignClient; |
|
||||
import org.springframework.web.bind.annotation.PostMapping; |
|
||||
import org.springframework.web.bind.annotation.RequestBody; |
|
||||
|
|
||||
/** |
|
||||
* desc: 数据抽取 对外feign client |
|
||||
* |
|
||||
* @return: |
|
||||
* @date: 2020/6/22 17:39 |
|
||||
* @author: jianjun liu |
|
||||
*/ |
|
||||
@FeignClient(name = ServiceConstant.DATA_STATISTICAL_SERVER, fallback = FactExtractOpenFeignClientFallBack.class) |
|
||||
public interface FactExtractOpenFeignClient { |
|
||||
|
|
||||
/** |
|
||||
* desc: 【日】抽取业务数据到统计库 小组 话题 议题 项目 |
|
||||
* |
|
||||
* @date: 2020/6/22 9:09 |
|
||||
* @author: jianjun liu |
|
||||
*/ |
|
||||
@PostMapping(value = "data/stats/factorigin/extractall") |
|
||||
Result extractall(@RequestBody(required = false) ExtractFormDTO formDTO); |
|
||||
|
|
||||
} |
|
||||
@ -1,135 +0,0 @@ |
|||||
package com.epmet.controller; |
|
||||
|
|
||||
import com.epmet.commons.tools.utils.DateUtils; |
|
||||
import com.epmet.commons.tools.utils.Result; |
|
||||
import com.epmet.commons.tools.validator.ValidatorUtils; |
|
||||
import com.epmet.dto.extract.form.ExtractFormDTO; |
|
||||
import com.epmet.service.evaluationindex.extract.*; |
|
||||
import com.epmet.util.DimIdGenerator; |
|
||||
import org.apache.commons.lang3.StringUtils; |
|
||||
import org.springframework.beans.factory.annotation.Autowired; |
|
||||
import org.springframework.web.bind.annotation.PostMapping; |
|
||||
import org.springframework.web.bind.annotation.RequestBody; |
|
||||
import org.springframework.web.bind.annotation.RequestMapping; |
|
||||
import org.springframework.web.bind.annotation.RestController; |
|
||||
|
|
||||
import java.util.Date; |
|
||||
|
|
||||
|
|
||||
/** |
|
||||
* 原始数据清洗 |
|
||||
* |
|
||||
* @author yinzuomei@elink-cn.com |
|
||||
* @date 2020/9/15 11:06 |
|
||||
*/ |
|
||||
@RestController |
|
||||
@RequestMapping("factorigin") |
|
||||
public class FactOriginController { |
|
||||
@Autowired |
|
||||
private FactOriginExtractService factOriginExtractService; |
|
||||
@Autowired |
|
||||
private IssueExtractService issueExtractService; |
|
||||
@Autowired |
|
||||
private FactOriginTopicMainDailyService factOriginTopicMainDailyService; |
|
||||
@Autowired |
|
||||
private ProjectExtractService projectExtractService; |
|
||||
@Autowired |
|
||||
private GroupExtractService groupExtractService; |
|
||||
|
|
||||
/** |
|
||||
* desc:抽取业务数据到统计库 |
|
||||
* |
|
||||
* @param extractFormDTO 默认统计前一天 |
|
||||
* @return |
|
||||
*/ |
|
||||
@PostMapping("extractall") |
|
||||
public Result extractAll(@RequestBody ExtractFormDTO extractFormDTO) { |
|
||||
if (StringUtils.isNotBlank(extractFormDTO.getStartDate()) && StringUtils.isNotBlank(extractFormDTO.getEndDate())) { |
|
||||
Date startDate = DateUtils.stringToDate(extractFormDTO.getStartDate(), "yyyyMMdd"); |
|
||||
Date endDate = DateUtils.stringToDate(extractFormDTO.getEndDate(), "yyyyMMdd"); |
|
||||
do { |
|
||||
String dateDimId = DimIdGenerator.getDateDimId(startDate); |
|
||||
extractFormDTO.setDateId(dateDimId); |
|
||||
factOriginExtractService.extractAll(extractFormDTO); |
|
||||
startDate = DateUtils.addDateDays(DateUtils.stringToDate(extractFormDTO.getStartDate(), "yyyyMMdd"), 1); |
|
||||
} while (endDate.compareTo(startDate) >= 0); |
|
||||
} else { |
|
||||
factOriginExtractService.extractAll(extractFormDTO); |
|
||||
} |
|
||||
return new Result(); |
|
||||
} |
|
||||
|
|
||||
|
|
||||
/** |
|
||||
* @param extractFormDTO |
|
||||
* @Description 议题抽取(main) |
|
||||
* @author zxc |
|
||||
* @date 2020/9/15 2:02 下午 |
|
||||
*/ |
|
||||
@PostMapping("issueextractmain") |
|
||||
public Result issueExtractMain(@RequestBody ExtractFormDTO extractFormDTO) { |
|
||||
ValidatorUtils.validateEntity(extractFormDTO, ExtractFormDTO.ExtractForm.class); |
|
||||
issueExtractService.issueExtractMain(extractFormDTO); |
|
||||
return new Result(); |
|
||||
} |
|
||||
|
|
||||
/** |
|
||||
* @Description 议题抽取(log) |
|
||||
* @param extractFormDTO |
|
||||
* @author zxc |
|
||||
* @date 2020/9/16 9:41 上午 |
|
||||
*/ |
|
||||
@PostMapping("issueextractlog") |
|
||||
public Result issueExtractLog(@RequestBody ExtractFormDTO extractFormDTO){ |
|
||||
ValidatorUtils.validateEntity(extractFormDTO, ExtractFormDTO.ExtractForm.class); |
|
||||
issueExtractService.issueExtractLog(extractFormDTO); |
|
||||
return new Result(); |
|
||||
} |
|
||||
|
|
||||
/** |
|
||||
* @param extractFormDTO |
|
||||
* @return com.epmet.commons.tools.utils.Result |
|
||||
* @author yinzuomei |
|
||||
* @description 话题 (fact_origin_topic_main_daily 话题主表_日统计 fact_origin_topic_log_daily 话题明细_日统计) |
|
||||
* @Date 2020/9/15 13:39 |
|
||||
**/ |
|
||||
@PostMapping("topic") |
|
||||
public Result topicDataCleaning(@RequestBody ExtractFormDTO extractFormDTO) { |
|
||||
if (StringUtils.isNotBlank(extractFormDTO.getCustomerId()) && StringUtils.isNotBlank(extractFormDTO.getDateId())) { |
|
||||
factOriginTopicMainDailyService.topicCleaning(extractFormDTO); |
|
||||
} |
|
||||
return new Result(); |
|
||||
} |
|
||||
|
|
||||
@PostMapping("project") |
|
||||
public Result projectData(@RequestBody ExtractFormDTO extractFormDTO) { |
|
||||
projectExtractService.saveOriginProjectDaily(extractFormDTO); |
|
||||
return new Result(); |
|
||||
} |
|
||||
|
|
||||
/** |
|
||||
* @Description 项目节点历时数据清洗 |
|
||||
* @param param |
|
||||
* @return |
|
||||
* @author wangc |
|
||||
* @date 2020.09.20 16:11 |
|
||||
**/ |
|
||||
@PostMapping("projectorgperiodcleanning") |
|
||||
public Result projectOrgPeriodCleaning(@RequestBody ExtractFormDTO param){ |
|
||||
projectExtractService.extractProjectPeriodData(param); |
|
||||
return new Result(); |
|
||||
} |
|
||||
|
|
||||
/** |
|
||||
* @Description 小组数据清洗 |
|
||||
* @param |
|
||||
* @return |
|
||||
* @author wangc |
|
||||
* @date 2020.09.20 16:11 |
|
||||
**/ |
|
||||
@PostMapping("groupdatacleaning") |
|
||||
public Result groupDataCleaning(@RequestBody ExtractFormDTO param){ |
|
||||
groupExtractService.extractGroupData(param); |
|
||||
return new Result(); |
|
||||
} |
|
||||
} |
|
||||
@ -0,0 +1,37 @@ |
|||||
|
package com.epmet.controller; |
||||
|
|
||||
|
import com.epmet.commons.tools.utils.Result; |
||||
|
import com.epmet.dto.extract.form.ExtractIndexFormDTO; |
||||
|
import com.epmet.service.evaluationindex.extract.dataToIndex.IndexOriginExtractService; |
||||
|
import org.springframework.beans.factory.annotation.Autowired; |
||||
|
import org.springframework.web.bind.annotation.PostMapping; |
||||
|
import org.springframework.web.bind.annotation.RequestBody; |
||||
|
import org.springframework.web.bind.annotation.RequestMapping; |
||||
|
import org.springframework.web.bind.annotation.RestController; |
||||
|
|
||||
|
|
||||
|
/** |
||||
|
* 原始数据清洗 |
||||
|
* |
||||
|
* @author yinzuomei@elink-cn.com |
||||
|
* @date 2020/9/15 11:06 |
||||
|
*/ |
||||
|
@RestController |
||||
|
@RequestMapping("indexorigin") |
||||
|
public class IndexOriginExtractController { |
||||
|
@Autowired |
||||
|
private IndexOriginExtractService indexOriginExtractService; |
||||
|
|
||||
|
|
||||
|
/** |
||||
|
* desc:抽取统计数据到指标库 |
||||
|
* |
||||
|
* @param extractFormDTO 默认统计上一个月 |
||||
|
* @return |
||||
|
*/ |
||||
|
@PostMapping("extractall") |
||||
|
public Result extractAll(@RequestBody ExtractIndexFormDTO extractFormDTO) { |
||||
|
indexOriginExtractService.indexOriginExtractAll(extractFormDTO); |
||||
|
return new Result(); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,18 @@ |
|||||
|
package com.epmet.service.evaluationindex.extract.dataToIndex; |
||||
|
|
||||
|
import com.epmet.dto.extract.form.ExtractIndexFormDTO; |
||||
|
|
||||
|
/** |
||||
|
* desc：将统计库对象数据抽取到指标库 服务接口 |
||||
|
* |
||||
|
* @author yinzuomei@elink-cn.com |
||||
|
* @date 2020/9/17 14:05 |
||||
|
*/ |
||||
|
public interface IndexOriginExtractService { |
||||
|
/** |
||||
|
* desc：从统计库对象抽取指标数据 |
||||
|
* |
||||
|
* @param formDTO |
||||
|
*/ |
||||
|
void indexOriginExtractAll(ExtractIndexFormDTO formDTO); |
||||
|
} |
||||
@ -0,0 +1,105 @@ |
|||||
|
package com.epmet.service.evaluationindex.extract.dataToIndex.impl; |
||||
|
|
||||
|
import com.alibaba.fastjson.JSON; |
||||
|
import com.epmet.commons.tools.constant.NumConstant; |
||||
|
import com.epmet.commons.tools.utils.DateUtils; |
||||
|
import com.epmet.dto.extract.form.ExtractIndexFormDTO; |
||||
|
import com.epmet.service.evaluationindex.extract.CalCpcIndexService; |
||||
|
import com.epmet.service.evaluationindex.extract.CalGridIndexService; |
||||
|
import com.epmet.service.evaluationindex.extract.dataToIndex.IndexOriginExtractService; |
||||
|
import com.epmet.service.stats.DimCustomerService; |
||||
|
import com.epmet.util.DimIdGenerator; |
||||
|
import com.google.common.util.concurrent.ThreadFactoryBuilder; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.apache.commons.lang3.StringUtils; |
||||
|
import org.springframework.beans.factory.annotation.Autowired; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
import org.springframework.util.CollectionUtils; |
||||
|
|
||||
|
import java.util.ArrayList; |
||||
|
import java.util.Date; |
||||
|
import java.util.List; |
||||
|
import java.util.concurrent.*; |
||||
|
|
||||
|
/** |
||||
|
* desc：指标原始数据抽取服务实现类 |
||||
|
*/ |
||||
|
@Slf4j |
||||
|
@Service |
||||
|
public class IndexOriginExtractServiceImpl implements IndexOriginExtractService { |
||||
|
ThreadFactory namedThreadFactory = new ThreadFactoryBuilder() |
||||
|
.setNameFormat("indexOriginExtract-pool-%d").build(); |
||||
|
ExecutorService threadPool = new ThreadPoolExecutor(2, 8, |
||||
|
10L, TimeUnit.MINUTES, |
||||
|
new LinkedBlockingQueue<>(500), namedThreadFactory, new ThreadPoolExecutor.CallerRunsPolicy()); |
||||
|
|
||||
|
@Autowired |
||||
|
private CalCpcIndexService calCpcIndexService; |
||||
|
@Autowired |
||||
|
private CalGridIndexService calGridIndexService; |
||||
|
@Autowired |
||||
|
private DimCustomerService dimCustomerService; |
||||
|
|
||||
|
/** |
||||
|
* desc：从统计库对象抽取指标数据 |
||||
|
* |
||||
|
* @param formDTO |
||||
|
*/ |
||||
|
@Override |
||||
|
public void indexOriginExtractAll(ExtractIndexFormDTO formDTO) { |
||||
|
String monthId = formDTO.getMonthId(); |
||||
|
String customerId = formDTO.getCustomerId(); |
||||
|
if (StringUtils.isBlank(monthId)) { |
||||
|
monthId = DimIdGenerator.getDateDimId(DateUtils.addDateMonths(new Date(), -1)); |
||||
|
} |
||||
|
List<String> customerIds = new ArrayList<>(); |
||||
|
if (StringUtils.isNotBlank(customerId)) { |
||||
|
customerIds.add(customerId); |
||||
|
} |
||||
|
int pageNo = NumConstant.ONE; |
||||
|
int pageSize = NumConstant.ONE_HUNDRED; |
||||
|
customerIds = dimCustomerService.selectCustomerIdPage(pageNo, pageSize); |
||||
|
if (CollectionUtils.isEmpty(customerIds)) { |
||||
|
log.error("indexOriginExtractAll 获取客户Id为空"); |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
String finalMonthId = monthId; |
||||
|
customerIds.forEach(cId -> { |
||||
|
ExtractIndexFormDTO param = new ExtractIndexFormDTO(); |
||||
|
param.setCustomerId(cId); |
||||
|
param.setMonthId(finalMonthId); |
||||
|
submitJob(param); |
||||
|
}); |
||||
|
} |
||||
|
|
||||
|
|
||||
|
private void submitJob(ExtractIndexFormDTO param) { |
||||
|
final String customerId = param.getCustomerId(); |
||||
|
final String monthId = param.getMonthId(); |
||||
|
threadPool.submit(() -> { |
||||
|
try { |
||||
|
calCpcIndexService.calCpcPartyAbility(customerId, monthId); |
||||
|
} catch (Exception e) { |
||||
|
log.error("抽取【党员相关数据】发生异常,参数:" + JSON.toJSONString(param), e); |
||||
|
} |
||||
|
}); |
||||
|
threadPool.submit(() -> { |
||||
|
try { |
||||
|
calGridIndexService.calGridIndexGovernAbility(customerId, monthId); |
||||
|
} catch (Exception e) { |
||||
|
log.error("抽取【网格治理能力数据】发生异常,参数:" + JSON.toJSONString(param), e); |
||||
|
} |
||||
|
try { |
||||
|
calGridIndexService.calGridIndexPartyAbility(customerId, monthId); |
||||
|
} catch (Exception e) { |
||||
|
log.error("抽取【网格党建能力数据】发生异常,参数:" + JSON.toJSONString(param), e); |
||||
|
} |
||||
|
try { |
||||
|
calGridIndexService.calGridIndexServiceAbility(customerId, monthId); |
||||
|
} catch (Exception e) { |
||||
|
log.error("抽取【网格服务能力数据】发生异常,参数:" + JSON.toJSONString(param), e); |
||||
|
} |
||||
|
}); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,24 @@ |
|||||
|
package com.epmet.service; |
||||
|
|
||||
|
|
||||
|
import com.epmet.commons.tools.utils.Result; |
||||
|
import com.epmet.dto.extract.form.ExtractIndexFormDTO; |
||||
|
|
||||
|
/** |
||||
|
* desc: 统计库数据抽取到指标库服务类 |
||||
|
* |
||||
|
* @return: |
||||
|
* @date: 2020/6/22 10:24 |
||||
|
* @author: jianjun liu |
||||
|
*/ |
||||
|
public interface IndexOriginExtractTaskService { |
||||
|
|
||||
|
/** |
||||
|
* desc: 【月】抽取 数据到指标库 |
||||
|
* |
||||
|
* @param formDTO |
||||
|
* @date: 2020/6/22 9:09 |
||||
|
* @author: jianjun liu |
||||
|
*/ |
||||
|
Result indexOriginExtractJob(ExtractIndexFormDTO formDTO); |
||||
|
} |
||||
@ -0,0 +1,30 @@ |
|||||
|
package com.epmet.service.impl; |
||||
|
|
||||
|
import com.epmet.commons.tools.utils.Result; |
||||
|
import com.epmet.dto.extract.form.ExtractIndexFormDTO; |
||||
|
import com.epmet.feign.DataStatisticalOpenFeignClient; |
||||
|
import com.epmet.service.IndexOriginExtractTaskService; |
||||
|
import org.springframework.beans.factory.annotation.Autowired; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
|
||||
|
/** |
||||
|
* desc：指标原始数据统计 任务服务类 |
||||
|
*/ |
||||
|
@Service |
||||
|
public class IndexOriginExtractTaskServiceImpl implements IndexOriginExtractTaskService { |
||||
|
@Autowired |
||||
|
private DataStatisticalOpenFeignClient dataStatisticalOpenFeignClient; |
||||
|
|
||||
|
|
||||
|
/** |
||||
|
* desc: 【月】抽取 统计库对象数据到统计库 |
||||
|
* |
||||
|
* @param formDTO |
||||
|
* @date: 2020/6/22 9:09 |
||||
|
* @author: jianjun liu |
||||
|
*/ |
||||
|
@Override |
||||
|
public Result indexOriginExtractJob(ExtractIndexFormDTO formDTO) { |
||||
|
return dataStatisticalOpenFeignClient.indexOriginExtractAll(formDTO); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,38 @@ |
|||||
|
package com.epmet.task; |
||||
|
|
||||
|
import com.alibaba.fastjson.JSON; |
||||
|
import com.epmet.commons.tools.utils.Result; |
||||
|
import com.epmet.dto.extract.form.ExtractFormDTO; |
||||
|
import com.epmet.service.FactOriginExtractTaskService; |
||||
|
import org.apache.commons.lang3.StringUtils; |
||||
|
import org.slf4j.Logger; |
||||
|
import org.slf4j.LoggerFactory; |
||||
|
import org.springframework.beans.factory.annotation.Autowired; |
||||
|
import org.springframework.stereotype.Component; |
||||
|
|
||||
|
/** |
||||
|
* desc:数据抽取定时任务 定时任务 |
||||
|
*/ |
||||
|
@Component("indexOriginExtractTask") |
||||
|
public class IndexOriginExtractTask implements ITask { |
||||
|
private Logger logger = LoggerFactory.getLogger(getClass()); |
||||
|
|
||||
|
@Autowired |
||||
|
private FactOriginExtractTaskService factOriginExtractTaskService; |
||||
|
|
||||
|
|
||||
|
@Override |
||||
|
public void run(String params) { |
||||
|
logger.info("indexOriginExtractTask定时任务正在执行，参数为：{}", params); |
||||
|
ExtractFormDTO formDTO = new ExtractFormDTO(); |
||||
|
if (StringUtils.isNotBlank(params)) { |
||||
|
formDTO = JSON.parseObject(params, ExtractFormDTO.class); |
||||
|
} |
||||
|
Result result = factOriginExtractTaskService.factOriginExtractJob(formDTO); |
||||
|
if (result.success()) { |
||||
|
logger.info("indexOriginExtractTask定时任务执行成功"); |
||||
|
} else { |
||||
|
logger.error("indexOriginExtractTask定时任务执行失败:" + result.getMsg()); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
Loading…
Reference in new issue