jianjun
5 years ago
14 changed files with 332 additions and 171 deletions
@ -1,30 +0,0 @@ |
|||
package com.epmet.feign; |
|||
|
|||
import com.epmet.commons.tools.constant.ServiceConstant; |
|||
import com.epmet.commons.tools.utils.Result; |
|||
import com.epmet.dto.extract.form.ExtractFormDTO; |
|||
import com.epmet.feign.impl.FactExtractOpenFeignClientFallBack; |
|||
import org.springframework.cloud.openfeign.FeignClient; |
|||
import org.springframework.web.bind.annotation.PostMapping; |
|||
import org.springframework.web.bind.annotation.RequestBody; |
|||
|
|||
/** |
|||
* desc: 数据抽取 对外feign client |
|||
* |
|||
* @return: |
|||
* @date: 2020/6/22 17:39 |
|||
* @author: jianjun liu |
|||
*/ |
|||
@FeignClient(name = ServiceConstant.DATA_STATISTICAL_SERVER, fallback = FactExtractOpenFeignClientFallBack.class) |
|||
public interface FactExtractOpenFeignClient { |
|||
|
|||
/** |
|||
* desc: 【日】抽取业务数据到统计库 小组 话题 议题 项目 |
|||
* |
|||
* @date: 2020/6/22 9:09 |
|||
* @author: jianjun liu |
|||
*/ |
|||
@PostMapping(value = "data/stats/factorigin/extractall") |
|||
Result extractall(@RequestBody(required = false) ExtractFormDTO formDTO); |
|||
|
|||
} |
|||
@ -1,135 +0,0 @@ |
|||
package com.epmet.controller; |
|||
|
|||
import com.epmet.commons.tools.utils.DateUtils; |
|||
import com.epmet.commons.tools.utils.Result; |
|||
import com.epmet.commons.tools.validator.ValidatorUtils; |
|||
import com.epmet.dto.extract.form.ExtractFormDTO; |
|||
import com.epmet.service.evaluationindex.extract.*; |
|||
import com.epmet.util.DimIdGenerator; |
|||
import org.apache.commons.lang3.StringUtils; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.web.bind.annotation.PostMapping; |
|||
import org.springframework.web.bind.annotation.RequestBody; |
|||
import org.springframework.web.bind.annotation.RequestMapping; |
|||
import org.springframework.web.bind.annotation.RestController; |
|||
|
|||
import java.util.Date; |
|||
|
|||
|
|||
/** |
|||
* 原始数据清洗 |
|||
* |
|||
* @author yinzuomei@elink-cn.com |
|||
* @date 2020/9/15 11:06 |
|||
*/ |
|||
@RestController |
|||
@RequestMapping("factorigin") |
|||
public class FactOriginController { |
|||
@Autowired |
|||
private FactOriginExtractService factOriginExtractService; |
|||
@Autowired |
|||
private IssueExtractService issueExtractService; |
|||
@Autowired |
|||
private FactOriginTopicMainDailyService factOriginTopicMainDailyService; |
|||
@Autowired |
|||
private ProjectExtractService projectExtractService; |
|||
@Autowired |
|||
private GroupExtractService groupExtractService; |
|||
|
|||
/** |
|||
* desc:抽取业务数据到统计库 |
|||
* |
|||
* @param extractFormDTO 默认统计前一天 |
|||
* @return |
|||
*/ |
|||
@PostMapping("extractall") |
|||
public Result extractAll(@RequestBody ExtractFormDTO extractFormDTO) { |
|||
if (StringUtils.isNotBlank(extractFormDTO.getStartDate()) && StringUtils.isNotBlank(extractFormDTO.getEndDate())) { |
|||
Date startDate = DateUtils.stringToDate(extractFormDTO.getStartDate(), "yyyyMMdd"); |
|||
Date endDate = DateUtils.stringToDate(extractFormDTO.getEndDate(), "yyyyMMdd"); |
|||
do { |
|||
String dateDimId = DimIdGenerator.getDateDimId(startDate); |
|||
extractFormDTO.setDateId(dateDimId); |
|||
factOriginExtractService.extractAll(extractFormDTO); |
|||
startDate = DateUtils.addDateDays(DateUtils.stringToDate(extractFormDTO.getStartDate(), "yyyyMMdd"), 1); |
|||
} while (endDate.compareTo(startDate) >= 0); |
|||
} else { |
|||
factOriginExtractService.extractAll(extractFormDTO); |
|||
} |
|||
return new Result(); |
|||
} |
|||
|
|||
|
|||
/** |
|||
* @param extractFormDTO |
|||
* @Description 议题抽取(main) |
|||
* @author zxc |
|||
* @date 2020/9/15 2:02 下午 |
|||
*/ |
|||
@PostMapping("issueextractmain") |
|||
public Result issueExtractMain(@RequestBody ExtractFormDTO extractFormDTO) { |
|||
ValidatorUtils.validateEntity(extractFormDTO, ExtractFormDTO.ExtractForm.class); |
|||
issueExtractService.issueExtractMain(extractFormDTO); |
|||
return new Result(); |
|||
} |
|||
|
|||
/** |
|||
* @Description 议题抽取(log) |
|||
* @param extractFormDTO |
|||
* @author zxc |
|||
* @date 2020/9/16 9:41 上午 |
|||
*/ |
|||
@PostMapping("issueextractlog") |
|||
public Result issueExtractLog(@RequestBody ExtractFormDTO extractFormDTO){ |
|||
ValidatorUtils.validateEntity(extractFormDTO, ExtractFormDTO.ExtractForm.class); |
|||
issueExtractService.issueExtractLog(extractFormDTO); |
|||
return new Result(); |
|||
} |
|||
|
|||
/** |
|||
* @param extractFormDTO |
|||
* @return com.epmet.commons.tools.utils.Result |
|||
* @author yinzuomei |
|||
* @description 话题 (fact_origin_topic_main_daily 话题主表_日统计 fact_origin_topic_log_daily 话题明细_日统计) |
|||
* @Date 2020/9/15 13:39 |
|||
**/ |
|||
@PostMapping("topic") |
|||
public Result topicDataCleaning(@RequestBody ExtractFormDTO extractFormDTO) { |
|||
if (StringUtils.isNotBlank(extractFormDTO.getCustomerId()) && StringUtils.isNotBlank(extractFormDTO.getDateId())) { |
|||
factOriginTopicMainDailyService.topicCleaning(extractFormDTO); |
|||
} |
|||
return new Result(); |
|||
} |
|||
|
|||
@PostMapping("project") |
|||
public Result projectData(@RequestBody ExtractFormDTO extractFormDTO) { |
|||
projectExtractService.saveOriginProjectDaily(extractFormDTO); |
|||
return new Result(); |
|||
} |
|||
|
|||
/** |
|||
* @Description 项目节点历时数据清洗 |
|||
* @param param |
|||
* @return |
|||
* @author wangc |
|||
* @date 2020.09.20 16:11 |
|||
**/ |
|||
@PostMapping("projectorgperiodcleanning") |
|||
public Result projectOrgPeriodCleaning(@RequestBody ExtractFormDTO param){ |
|||
projectExtractService.extractProjectPeriodData(param); |
|||
return new Result(); |
|||
} |
|||
|
|||
/** |
|||
* @Description 小组数据清洗 |
|||
* @param |
|||
* @return |
|||
* @author wangc |
|||
* @date 2020.09.20 16:11 |
|||
**/ |
|||
@PostMapping("groupdatacleaning") |
|||
public Result groupDataCleaning(@RequestBody ExtractFormDTO param){ |
|||
groupExtractService.extractGroupData(param); |
|||
return new Result(); |
|||
} |
|||
} |
|||
@ -0,0 +1,37 @@ |
|||
package com.epmet.controller; |
|||
|
|||
import com.epmet.commons.tools.utils.Result; |
|||
import com.epmet.dto.extract.form.ExtractIndexFormDTO; |
|||
import com.epmet.service.evaluationindex.extract.dataToIndex.IndexOriginExtractService; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.web.bind.annotation.PostMapping; |
|||
import org.springframework.web.bind.annotation.RequestBody; |
|||
import org.springframework.web.bind.annotation.RequestMapping; |
|||
import org.springframework.web.bind.annotation.RestController; |
|||
|
|||
|
|||
/** |
|||
* 原始数据清洗 |
|||
* |
|||
* @author yinzuomei@elink-cn.com |
|||
* @date 2020/9/15 11:06 |
|||
*/ |
|||
@RestController |
|||
@RequestMapping("indexorigin") |
|||
public class IndexOriginExtractController { |
|||
@Autowired |
|||
private IndexOriginExtractService indexOriginExtractService; |
|||
|
|||
|
|||
/** |
|||
* desc:抽取统计数据到指标库 |
|||
* |
|||
* @param extractFormDTO 默认统计上一个月 |
|||
* @return |
|||
*/ |
|||
@PostMapping("extractall") |
|||
public Result extractAll(@RequestBody ExtractIndexFormDTO extractFormDTO) { |
|||
indexOriginExtractService.indexOriginExtractAll(extractFormDTO); |
|||
return new Result(); |
|||
} |
|||
} |
|||
@ -0,0 +1,18 @@ |
|||
package com.epmet.service.evaluationindex.extract.dataToIndex; |
|||
|
|||
import com.epmet.dto.extract.form.ExtractIndexFormDTO; |
|||
|
|||
/** |
|||
* desc：将统计库对象数据抽取到指标库 服务接口 |
|||
* |
|||
* @author yinzuomei@elink-cn.com |
|||
* @date 2020/9/17 14:05 |
|||
*/ |
|||
public interface IndexOriginExtractService { |
|||
/** |
|||
* desc：从统计库对象抽取指标数据 |
|||
* |
|||
* @param formDTO |
|||
*/ |
|||
void indexOriginExtractAll(ExtractIndexFormDTO formDTO); |
|||
} |
|||
@ -0,0 +1,105 @@ |
|||
package com.epmet.service.evaluationindex.extract.dataToIndex.impl; |
|||
|
|||
import com.alibaba.fastjson.JSON; |
|||
import com.epmet.commons.tools.constant.NumConstant; |
|||
import com.epmet.commons.tools.utils.DateUtils; |
|||
import com.epmet.dto.extract.form.ExtractIndexFormDTO; |
|||
import com.epmet.service.evaluationindex.extract.CalCpcIndexService; |
|||
import com.epmet.service.evaluationindex.extract.CalGridIndexService; |
|||
import com.epmet.service.evaluationindex.extract.dataToIndex.IndexOriginExtractService; |
|||
import com.epmet.service.stats.DimCustomerService; |
|||
import com.epmet.util.DimIdGenerator; |
|||
import com.google.common.util.concurrent.ThreadFactoryBuilder; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.apache.commons.lang3.StringUtils; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.stereotype.Service; |
|||
import org.springframework.util.CollectionUtils; |
|||
|
|||
import java.util.ArrayList; |
|||
import java.util.Date; |
|||
import java.util.List; |
|||
import java.util.concurrent.*; |
|||
|
|||
/** |
|||
* desc：指标原始数据抽取服务实现类 |
|||
*/ |
|||
@Slf4j |
|||
@Service |
|||
public class IndexOriginExtractServiceImpl implements IndexOriginExtractService { |
|||
ThreadFactory namedThreadFactory = new ThreadFactoryBuilder() |
|||
.setNameFormat("indexOriginExtract-pool-%d").build(); |
|||
ExecutorService threadPool = new ThreadPoolExecutor(2, 8, |
|||
10L, TimeUnit.MINUTES, |
|||
new LinkedBlockingQueue<>(500), namedThreadFactory, new ThreadPoolExecutor.CallerRunsPolicy()); |
|||
|
|||
@Autowired |
|||
private CalCpcIndexService calCpcIndexService; |
|||
@Autowired |
|||
private CalGridIndexService calGridIndexService; |
|||
@Autowired |
|||
private DimCustomerService dimCustomerService; |
|||
|
|||
/** |
|||
* desc：从统计库对象抽取指标数据 |
|||
* |
|||
* @param formDTO |
|||
*/ |
|||
@Override |
|||
public void indexOriginExtractAll(ExtractIndexFormDTO formDTO) { |
|||
String monthId = formDTO.getMonthId(); |
|||
String customerId = formDTO.getCustomerId(); |
|||
if (StringUtils.isBlank(monthId)) { |
|||
monthId = DimIdGenerator.getDateDimId(DateUtils.addDateMonths(new Date(), -1)); |
|||
} |
|||
List<String> customerIds = new ArrayList<>(); |
|||
if (StringUtils.isNotBlank(customerId)) { |
|||
customerIds.add(customerId); |
|||
} |
|||
int pageNo = NumConstant.ONE; |
|||
int pageSize = NumConstant.ONE_HUNDRED; |
|||
customerIds = dimCustomerService.selectCustomerIdPage(pageNo, pageSize); |
|||
if (CollectionUtils.isEmpty(customerIds)) { |
|||
log.error("indexOriginExtractAll 获取客户Id为空"); |
|||
return; |
|||
} |
|||
|
|||
String finalMonthId = monthId; |
|||
customerIds.forEach(cId -> { |
|||
ExtractIndexFormDTO param = new ExtractIndexFormDTO(); |
|||
param.setCustomerId(cId); |
|||
param.setMonthId(finalMonthId); |
|||
submitJob(param); |
|||
}); |
|||
} |
|||
|
|||
|
|||
private void submitJob(ExtractIndexFormDTO param) { |
|||
final String customerId = param.getCustomerId(); |
|||
final String monthId = param.getMonthId(); |
|||
threadPool.submit(() -> { |
|||
try { |
|||
calCpcIndexService.calCpcPartyAbility(customerId, monthId); |
|||
} catch (Exception e) { |
|||
log.error("抽取【党员相关数据】发生异常,参数:" + JSON.toJSONString(param), e); |
|||
} |
|||
}); |
|||
threadPool.submit(() -> { |
|||
try { |
|||
calGridIndexService.calGridIndexGovernAbility(customerId, monthId); |
|||
} catch (Exception e) { |
|||
log.error("抽取【网格治理能力数据】发生异常,参数:" + JSON.toJSONString(param), e); |
|||
} |
|||
try { |
|||
calGridIndexService.calGridIndexPartyAbility(customerId, monthId); |
|||
} catch (Exception e) { |
|||
log.error("抽取【网格党建能力数据】发生异常,参数:" + JSON.toJSONString(param), e); |
|||
} |
|||
try { |
|||
calGridIndexService.calGridIndexServiceAbility(customerId, monthId); |
|||
} catch (Exception e) { |
|||
log.error("抽取【网格服务能力数据】发生异常,参数:" + JSON.toJSONString(param), e); |
|||
} |
|||
}); |
|||
} |
|||
} |
|||
@ -0,0 +1,24 @@ |
|||
package com.epmet.service; |
|||
|
|||
|
|||
import com.epmet.commons.tools.utils.Result; |
|||
import com.epmet.dto.extract.form.ExtractIndexFormDTO; |
|||
|
|||
/** |
|||
* desc: 统计库数据抽取到指标库服务类 |
|||
* |
|||
* @return: |
|||
* @date: 2020/6/22 10:24 |
|||
* @author: jianjun liu |
|||
*/ |
|||
public interface IndexOriginExtractTaskService { |
|||
|
|||
/** |
|||
* desc: 【月】抽取 数据到指标库 |
|||
* |
|||
* @param formDTO |
|||
* @date: 2020/6/22 9:09 |
|||
* @author: jianjun liu |
|||
*/ |
|||
Result indexOriginExtractJob(ExtractIndexFormDTO formDTO); |
|||
} |
|||
@ -0,0 +1,30 @@ |
|||
package com.epmet.service.impl; |
|||
|
|||
import com.epmet.commons.tools.utils.Result; |
|||
import com.epmet.dto.extract.form.ExtractIndexFormDTO; |
|||
import com.epmet.feign.DataStatisticalOpenFeignClient; |
|||
import com.epmet.service.IndexOriginExtractTaskService; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.stereotype.Service; |
|||
|
|||
/** |
|||
* desc：指标原始数据统计 任务服务类 |
|||
*/ |
|||
@Service |
|||
public class IndexOriginExtractTaskServiceImpl implements IndexOriginExtractTaskService { |
|||
@Autowired |
|||
private DataStatisticalOpenFeignClient dataStatisticalOpenFeignClient; |
|||
|
|||
|
|||
/** |
|||
* desc: 【月】抽取 统计库对象数据到统计库 |
|||
* |
|||
* @param formDTO |
|||
* @date: 2020/6/22 9:09 |
|||
* @author: jianjun liu |
|||
*/ |
|||
@Override |
|||
public Result indexOriginExtractJob(ExtractIndexFormDTO formDTO) { |
|||
return dataStatisticalOpenFeignClient.indexOriginExtractAll(formDTO); |
|||
} |
|||
} |
|||
@ -0,0 +1,38 @@ |
|||
package com.epmet.task; |
|||
|
|||
import com.alibaba.fastjson.JSON; |
|||
import com.epmet.commons.tools.utils.Result; |
|||
import com.epmet.dto.extract.form.ExtractFormDTO; |
|||
import com.epmet.service.FactOriginExtractTaskService; |
|||
import org.apache.commons.lang3.StringUtils; |
|||
import org.slf4j.Logger; |
|||
import org.slf4j.LoggerFactory; |
|||
import org.springframework.beans.factory.annotation.Autowired; |
|||
import org.springframework.stereotype.Component; |
|||
|
|||
/** |
|||
* desc:数据抽取定时任务 定时任务 |
|||
*/ |
|||
@Component("indexOriginExtractTask") |
|||
public class IndexOriginExtractTask implements ITask { |
|||
private Logger logger = LoggerFactory.getLogger(getClass()); |
|||
|
|||
@Autowired |
|||
private FactOriginExtractTaskService factOriginExtractTaskService; |
|||
|
|||
|
|||
@Override |
|||
public void run(String params) { |
|||
logger.info("indexOriginExtractTask定时任务正在执行，参数为：{}", params); |
|||
ExtractFormDTO formDTO = new ExtractFormDTO(); |
|||
if (StringUtils.isNotBlank(params)) { |
|||
formDTO = JSON.parseObject(params, ExtractFormDTO.class); |
|||
} |
|||
Result result = factOriginExtractTaskService.factOriginExtractJob(formDTO); |
|||
if (result.success()) { |
|||
logger.info("indexOriginExtractTask定时任务执行成功"); |
|||
} else { |
|||
logger.error("indexOriginExtractTask定时任务执行失败:" + result.getMsg()); |
|||
} |
|||
} |
|||
} |
|||
Loading…
Reference in new issue