diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/entity/project/ProjectEntity.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/entity/project/ProjectEntity.java index 20a32c24a6..8244100898 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/entity/project/ProjectEntity.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/entity/project/ProjectEntity.java @@ -95,4 +95,7 @@ public class ProjectEntity extends BaseEpmetEntity { * */ private String locateDimension; + private Integer pageNo = 1; + private Integer pageSize = 20; + } diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsProjectServiceImpl.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsProjectServiceImpl.java index 3b1d1c45c8..4cbe3fd140 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsProjectServiceImpl.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsProjectServiceImpl.java @@ -1,5 +1,6 @@ package com.epmet.service.impl; +import cn.hutool.core.collection.CollectionUtil; import com.epmet.commons.tools.constant.NumConstant; import com.epmet.commons.tools.utils.DateUtils; import com.epmet.constant.ProjectConstant; @@ -112,6 +113,8 @@ public class StatsProjectServiceImpl implements StatsProjectService { dimAgencyDTO.setCustomerId(customerId); List dimAgencyList = dimAgencyService.getDimAgencyList(dimAgencyDTO); + /* + //2022.1.12 客户数据量大 调整计算逻辑，一千条已查询，封装数据,将这部分业务数据查询拿到子方法分页查询处理 sun //3:根据客户Id查询项目业务表已结案数据(查询传入日期及之前的数据) ProjectEntity projectEntity = new ProjectEntity(); projectEntity.setCustomerId(customerId); @@ -133,13 +136,14 @@ public class StatsProjectServiceImpl implements StatsProjectService { //4-3.遍历删除项目节点表查询的无效数据 processList.removeIf(next -> map.containsKey(next.getProjectId())); } - //20210721 sun end + //20210721 sun end */ //5:机关层级日月统计 if (null != dimAgencyList && dimAgencyList.size() > NumConstant.ZERO) { //5.1:执行机关日数据统计 try { - agencyDateProjectStats(customerId, dimId, date, dimAgencyList, projectList, processList); + //agencyDateProjectStats(customerId, dimId, date, dimAgencyList, projectList, processList); + agencyDateProjectStats(customerId, dimId, date, dimAgencyList); } catch (Exception e) { log.error(String.format(ProjectConstant.STATS_FAILED_PREFIX, "agencyDateProjectStats", customerId, dimId.getDateId()), e); } @@ -159,7 +163,238 @@ public class StatsProjectServiceImpl implements StatsProjectService { * @Author sun * @Description 数据-项目-机关日统计 **/ - private String agencyDateProjectStats(String customerId, DimIdGenerator.DimIdBean dimId, Date date, List dimAgencyList, List projectList, List processList) { + private String agencyDateProjectStats(String customerId, DimIdGenerator.DimIdBean dimId, Date date, List dimAgencyList) { + //机关日统计数据【agencyId->FactAgencyProjectDailyEntity】 + Map mapList = new HashMap<>(); + //计算百分比使用，保留小数点后两位 + NumberFormat numberFormat = NumberFormat.getInstance(); + numberFormat.setMaximumFractionDigits(6); + //存放已结案项目Id，用于统计已结案中已解决、未解决数 + Map closeMap = new HashMap<>(); + //存放前一日已结案项目Id，用于统计日增量中已结案项目的已解决、未解决数量 + Map closeDateMap = new HashMap<>(); + + //20210721 sun 业务逻辑调整，网格删除，组织没有删除情况，相应的组织层级数据统计应舍弃以删除网格数据 start + //处理逻辑：查询已删除网格下项目Id，将已查询的项目、节点数据中的脏数据剔除【项目库有张表有网格Id，可以查出每个项目所属网格，但是关联该表后sql查询效率极低固舍弃此方案】 + //4-1.查询客户下已删除网格列表下存在的项目Id集合 + List list = customerGridService.getdelGridProjectIdList(customerId); + //20210721 sun end + + //一、先分页查询节点数据，封装每个组织的项目总数、已结案总数;日增量中项目总数、已结案总数 + ProjectEntity projectEntity = new ProjectEntity(); + projectEntity.setCustomerId(customerId); + projectEntity.setCreatedTime(date); + projectEntity.setStatus(ProjectConstant.CLOSED); + projectEntity.setPageSize(NumConstant.ONE_THOUSAND); + int pageNo = NumConstant.ONE; + List processList = new ArrayList<>(); + do { + //1.一千条一循环查询节点数据，封装每个组织对应数据 + projectEntity.setPageNo(pageNo); + processList = projectProcessService.getProcessList(projectEntity); + pageNo++; + //遍历删除项目节点表查询的无效数据 + if (list.size() > NumConstant.ZERO) { + Map map = list.stream().collect(Collectors.toMap(String::toString, v -> v, (c1, c2) -> c1)); + processList.removeIf(next -> map.containsKey(next.getProjectId())); + } + //2:遍历机关维度数据,统计每个机关各项指标数 + for (DimAgencyDTO agency : dimAgencyList) { + //2-1:存放当前机关及所有下级机关Id + Map map = new HashMap<>(); + map.put(agency.getId(), agency.getId()); + String subPids = ((null == agency.getPids() || "".equals(agency.getPids())) ? agency.getId() : agency.getPids() + ":" + agency.getId()); + dimAgencyList.forEach(sub -> { + if (sub.getPids().contains(subPids)) { + map.put(sub.getId(), sub.getId()); + } + }); + //机关下截止当前日期的项目总数、已结案总数、已结案已解决总数、已结案未解决总数 + AtomicInteger projectTotal = new AtomicInteger(0); + AtomicInteger closedTotal = new AtomicInteger(0); + //日增量中项目总数、已结案总数、已结案已解决总数、已结案未解决总数 + AtomicInteger projectIncr = new AtomicInteger(0); + AtomicInteger closedIncr = new AtomicInteger(0); + //2-2:遍历进展数据，统计截止当日的项目总量、处理中总量、已结案总量以及日增量中的项目总量、处理中总量、已结案总量 + processList.forEach(process -> { + //当前机关及下级 + if (map.containsKey(process.getAgencyId())) { + //进展表中是创建项目状态的数据总数即为客户该机关下项目总数 + if (ProjectConstant.CREATED.equals(process.getOperation())) { + projectTotal.addAndGet(1); + if (date.equals(process.getCreatedTime())) { + //日增量总数 + projectIncr.addAndGet(1); + } + } + if (ProjectConstant.CLOSE.equals(process.getOperation())) { + //截止当前日期的结案总数 + closedTotal.addAndGet(1); + closeMap.put(process.getProjectId(), process.getProjectId()); + if (date.equals(process.getCreatedTime())) { + //日增量已结案总数 + closedIncr.addAndGet(1); + closeDateMap.put(process.getProjectId(), process.getProjectId()); + } + } + } + }); + + //2-3:封装机关日数据对象 + FactAgencyProjectDailyEntity entity = new FactAgencyProjectDailyEntity(); + if (mapList.containsKey(agency.getId())) { + entity = mapList.get(agency.getId()); + entity.setProjectTotal(entity.getProjectTotal() + projectTotal.intValue()); + entity.setPendingTotal(entity.getPendingTotal() + projectTotal.intValue() - closedTotal.intValue()); + entity.setClosedTotal(entity.getClosedTotal() + closedTotal.intValue()); + if (projectTotal.intValue() > NumConstant.ZERO) { + entity.setPendingRatio(new BigDecimal(numberFormat.format((float) entity.getPendingTotal() / (float) projectTotal.intValue()))); + entity.setClosedRatio(new BigDecimal(numberFormat.format((float) closedTotal.intValue() / (float) projectTotal.intValue()))); + } + entity.setProjectIncr(entity.getProjectIncr() + projectIncr.intValue()); + entity.setPendingIncr(entity.getPendingIncr() + projectIncr.intValue()); + entity.setClosedIncr(entity.getClosedIncr() + closedIncr.intValue()); + } else { + entity.setCustomerId(customerId); + entity.setAgencyId(agency.getId()); + entity.setParentId(agency.getPid()); + entity.setDateId(dimId.getDateId()); + entity.setWeekId(dimId.getWeekId()); + entity.setMonthId(dimId.getMonthId()); + entity.setQuarterId(dimId.getQuarterId()); + entity.setYearId(dimId.getYearId()); + entity.setProjectTotal(projectTotal.intValue()); + entity.setPendingTotal(projectTotal.intValue() - closedTotal.intValue()); + entity.setClosedTotal(closedTotal.intValue()); + if (projectTotal.intValue() > NumConstant.ZERO) { + entity.setPendingRatio(new BigDecimal(numberFormat.format((float) entity.getPendingTotal() / (float) projectTotal.intValue()))); + entity.setClosedRatio(new BigDecimal(numberFormat.format((float) closedTotal.intValue() / (float) projectTotal.intValue()))); + } + entity.setProjectIncr(projectIncr.intValue()); + entity.setPendingIncr(projectIncr.intValue()); + entity.setClosedIncr(closedIncr.intValue()); + } + mapList.put(agency.getId(), entity); + } + } while (!CollectionUtil.isEmpty(processList) && processList.size() == NumConstant.ONE_THOUSAND); + + //二、再分页查询项目表数据，封装每个组织的已结案已解决总数、已结案未解决总数;日增量中已结案已解决总数、已结案未解决总数 + int num = NumConstant.ONE; + List projectList = new ArrayList<>(); + do { + //3.一千条一循环查询项目数据，封装每个组织对应数据 + projectEntity.setPageNo(num); + projectList = projectService.getProjectList(projectEntity); + num++; + //遍历删除项目主表查询的无效数据 + if (list.size() > NumConstant.ZERO) { + Map map = list.stream().collect(Collectors.toMap(String::toString, v -> v, (c1, c2) -> c1)); + projectList.removeIf(next -> map.containsKey(next.getId())); + } + + //4.遍历机关维度数据,统计每个机关各项指标数 + for (DimAgencyDTO agency : dimAgencyList) { + //4-1:存放当前机关及所有下级机关Id + Map map = new HashMap<>(); + map.put(agency.getId(), agency.getId()); + String subPids = ((null == agency.getPids() || "".equals(agency.getPids())) ? agency.getId() : agency.getPids() + ":" + agency.getId()); + dimAgencyList.forEach(sub -> { + if (sub.getPids().contains(subPids)) { + map.put(sub.getId(), sub.getId()); + } + }); + //机关下截止当前日期的已结案已解决总数、已结案未解决总数 + AtomicInteger resolvedTotal = new AtomicInteger(0); + AtomicInteger unResolvedTotal = new AtomicInteger(0); + //日增量中项目已结案已解决总数、已结案未解决总数 + AtomicInteger resolvedIncr = new AtomicInteger(0); + AtomicInteger unResolvedIncr = new AtomicInteger(0); + //4-2:遍历项目业务数据，统计截止当日的已结案已解决总量、已结案未解决总量以及日增量中的已结案已解决总量、已结案未解决总量 + projectList.forEach(project -> { + //当前机关及下级 + if (map.containsKey(project.getAgencyId())) { + if (closeMap.containsKey(project.getId())) { + //已结案已解决 + if (ProjectConstant.RESOLVED.equals(project.getClosedStatus())) { + //截止当日的已结案已解决总量 + resolvedTotal.addAndGet(1); + if (closeDateMap.containsKey(project.getId())) { + //日增量中的已结案已解决总量 + resolvedIncr.addAndGet(1); + } + } + //已结案未解决 + if (ProjectConstant.UNRESOLVED.equals(project.getClosedStatus())) { + //截止当日的已结案未解决总量 + unResolvedTotal.addAndGet(1); + if (closeDateMap.containsKey(project.getId())) { + //日增量中的已结案未解决总量 + unResolvedIncr.addAndGet(1); + } + } + } + } + }); + + //1.4:封装机关日数据对象 + FactAgencyProjectDailyEntity entity = new FactAgencyProjectDailyEntity(); + if (mapList.containsKey(agency.getId())) { + entity = mapList.get(agency.getId()); + entity.setResolvedTotal(entity.getResolvedTotal() + resolvedTotal.intValue()); + entity.setUnresolvedTotal(entity.getUnresolvedTotal() + unResolvedTotal.intValue()); + if (entity.getClosedTotal() > NumConstant.ZERO) { + entity.setResolvedRatio(new BigDecimal(numberFormat.format((float) resolvedTotal.intValue() / (float) entity.getClosedTotal()))); + entity.setUnresolvedRatio(new BigDecimal(numberFormat.format((float) unResolvedTotal.intValue() / (float) entity.getClosedTotal()))); + } + entity.setResolvedIncr(entity.getResolvedIncr() + resolvedIncr.intValue()); + entity.setUnresolvedIncr(entity.getUnresolvedIncr() + unResolvedIncr.intValue()); + + + } else { + entity.setCustomerId(customerId); + entity.setAgencyId(agency.getId()); + entity.setParentId(agency.getPid()); + entity.setDateId(dimId.getDateId()); + entity.setWeekId(dimId.getWeekId()); + entity.setMonthId(dimId.getMonthId()); + entity.setQuarterId(dimId.getQuarterId()); + entity.setYearId(dimId.getYearId()); + entity.setResolvedTotal(resolvedTotal.intValue()); + entity.setUnresolvedTotal(unResolvedTotal.intValue()); + if (entity.getClosedTotal() > NumConstant.ZERO) { + entity.setResolvedRatio(new BigDecimal(numberFormat.format((float) resolvedTotal.intValue() / (float) entity.getClosedTotal()))); + entity.setUnresolvedRatio(new BigDecimal(numberFormat.format((float) unResolvedTotal.intValue() / (float) entity.getClosedTotal()))); + } + entity.setResolvedIncr(resolvedIncr.intValue()); + entity.setUnresolvedIncr(unResolvedIncr.intValue()); + + } + mapList.put(agency.getId(), entity); + } + } while (!CollectionUtil.isEmpty(projectList) && projectList.size() == NumConstant.ONE_THOUSAND); + + List projectDateEntityList = new ArrayList<>(mapList.values()); + //2:批量保存数据，先删后增 + if (null != projectDateEntityList && projectDateEntityList.size() > NumConstant.ZERO) { + //2.1:根据客户Id、日维度Id批量物理删除一下可能存在的历史数据 + FactAgencyProjectDailyEntity delEntity = new FactAgencyProjectDailyEntity(); + delEntity.setCustomerId(customerId); + delEntity.setDateId(dimId.getDateId()); + log.info("StatsProjectServiceImpl.agencyDateProjectStats-根据客户Id、日维度Id批量删除机关项目日统计表数据,对应客户Id:" + customerId + ",日维度Id:" + dimId.getDateId()); + factAgencyProjectDailyService.delDateProject(delEntity); + + //2.2:批量保存机关日统计数据 + log.info("StatsProjectServiceImpl.agencyDateProjectStats-批量新增机关项目日统计表数据,对应客户Id:" + customerId + ",日维度Id:" + dimId.getDateId()); + factAgencyProjectDailyService.insertBatch(projectDateEntityList); + } + + return null; + } + + /** + * 不考虑大数据量问题的方法【暂留】sun + */ + private String agencyDateProjectStats_old(String customerId, DimIdGenerator.DimIdBean dimId, Date date, List dimAgencyList, List projectList, List processList) { //批量机关日统计新增对象 List projectDateEntityList = new ArrayList<>(); //计算百分比使用，保留小数点后两位