elink-star
/
epmet-cloud-lingshan
forked from luyan/epmet-cloud-lingshan
12
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

【报表】批量导出-多线程及系列优化

master
wangxianzhang 3 years ago
parent
729bf41e1b
commit
fbb52ecf7c
2 changed files with 61 additions and 24 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 1
      epmet-commons/epmet-commons-feignclient/src/main/java/com/epmet/commons/feignclient/dtos/result/JimuReportDbDataResultDTO.java
  2. 82
      epmet-module/oper-customize/oper-customize-server/src/main/java/com/epmet/service/impl/IcCustomerReportServiceImpl.java

1
epmet-commons/epmet-commons-feignclient/src/main/java/com/epmet/commons/feignclient/dtos/result/JimuReportDbDataResultDTO.java
View File

@ -12,6 +12,7 @@ public class JimuReportDbDataResultDTO {
private String apiUrl;
private String apiMethod;
private String isList;
private String dbChName;
}
}

82
epmet-module/oper-customize/oper-customize-server/src/main/java/com/epmet/service/impl/IcCustomerReportServiceImpl.java
View File

@ -40,6 +40,7 @@ import com.epmet.service.IcReportFunService;
import feign.Response;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.ListUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
@ -57,6 +58,9 @@ import java.nio.file.Path;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@ -83,6 +87,8 @@ public class IcCustomerReportServiceImpl extends BaseServiceImpl<IcCustomerRepor
private IcReportFunService reportFunService;
@Autowired
private JiMuReportOpenFeignClient jiMuReportOpenFeignClient;
@Autowired
private ExecutorService executorService;
private QueryWrapper<IcCustomerReportEntity> getWrapper(Map<String, Object> params) {
@ -299,7 +305,7 @@ public class IcCustomerReportServiceImpl extends BaseServiceImpl<IcCustomerRepor
Path storePath = makeTemporaryDownloadDir(reportId);
// 4.生成压缩文件
Path zipFile = downloadAndComppress(storePath, reportId, paramKey, ids);
Path zipFile = downloadAndComppress(storePath, reportId, reportDb.getDbChName(), paramKey, ids);
// 5.下载
try (FileInputStream fis = new FileInputStream(zipFile.toFile())) {
@ -326,7 +332,7 @@ public class IcCustomerReportServiceImpl extends BaseServiceImpl<IcCustomerRepor
* @param ids
* @return 压缩文件路径
*/
private Path downloadAndComppress(Path storePath, String reportId, String paramKey, List<String> ids) {
private Path downloadAndComppress(Path storePath, String reportId, String reportName, String paramKey, List<String> ids) {
// 请求头
Map<String, Object> headers = new HashMap<>();
headers.put(Constant.AUTHORIZATION_HEADER, EpmetRequestHolder.getHeader(Constant.AUTHORIZATION_HEADER));
@ -341,9 +347,9 @@ public class IcCustomerReportServiceImpl extends BaseServiceImpl<IcCustomerRepor
param.getQueryParam().setParamKey(paramKey);
param.getQueryParam().setToken(EpmetRequestHolder.getLoginUserAuthorizationToken());
final ArrayList<File> files = new ArrayList<>();
ArrayList<File> files = new ArrayList<>();
final String currentTimeStr = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss"));
String currentTimeStr = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss"));
// 创建存放xlsx的临时目录
Path xlsxStorePath = storePath.resolve(currentTimeStr);
@ -353,38 +359,34 @@ public class IcCustomerReportServiceImpl extends BaseServiceImpl<IcCustomerRepor
throw new EpmetException("【报表批量导出】生成临时目录失败:" + xlsxStorePath.toString());
}
// 将所有id分片，交给多个线程并行处理
List<List<String>> idParts = ListUtils.partition(ids, 100);
// 1. 循环下载所有id对应的excel
for (String id : ids) {
param.getQueryParam().setId(id);
final Response response = jiMuReportOpenFeignClient.exportAllExcelStream(param);
// 取出文件后缀
final LinkedList<String> header = (LinkedList) response.headers().get("content-disposition");
final Matcher matcher = Pattern.compile("attachment;filename=(.+)(.xls|.xlsx)").matcher(header.get(0));
if (!matcher.matches()) {
log.error("【报表批量导出】批量导出失败，后缀不匹配。文件名：{}, 报表ID:{}, 业务数据ID：{}", header.get(0), reportId, id);
continue;
CountDownLatch cdl = new CountDownLatch(idParts.size());
for (List<String> idPart : idParts) {
CompletableFuture.runAsync(() -> {
downloadXlsByBatchByBizId(reportId, idPart, param, xlsxStorePath, files, cdl);
}, executorService);
}
final File excelFile = xlsxStorePath.resolve("file_" + id + matcher.group(2)).toFile();
files.add(excelFile);
try (FileOutputStream fos = new FileOutputStream(excelFile)) {
IOUtils.copy(response.body().asInputStream(), fos);
} catch (Exception e) {
log.error("【报表批量导出】生成临时表格文件失败,文件名：{}，报表ID:{}, 业务数据ID：{},错误信息:{}", header.get(0), reportId, id, ExceptionUtils.getErrorStackTrace(e));
}
// 等待多线程执行完成，再执行打包
try {
cdl.await();
} catch (InterruptedException e) {
log.error(ExceptionUtils.getErrorStackTrace(e));
throw new EpmetException("【报表批量导出】CountDownLatch等待被中断");
}
// 2,打包
Path zipFile = storePath.resolve("report_export_" + currentTimeStr + ".zip");
Path zipFile = storePath.resolve(reportName + "_" + currentTimeStr + ".zip");
try (final FileOutputStream fos = new FileOutputStream(zipFile.toFile());
final ZipOutputStream zos = new ZipOutputStream(fos)) {
// 循环每一个文件
for (File file : files) {
try (final FileInputStream fis = new FileInputStream(file.getAbsolutePath())) {
zos.putNextEntry(new ZipEntry("report_export_" + currentTimeStr + "/" + file.getName()));
zos.putNextEntry(new ZipEntry(reportName + "_" + currentTimeStr + "/" + file.getName()));
final byte[] buffer = new byte[10240];
for (int len; (len = fis.read(buffer)) > 0; ) {
zos.write(buffer, 0, len);
@ -406,6 +408,40 @@ public class IcCustomerReportServiceImpl extends BaseServiceImpl<IcCustomerRepor
return zipFile;
}
/**
* 批量下载xlsx
* @param reportId
* @param bizIds
* @param param
* @param xlsxStorePath
* @param files
*/
public void downloadXlsByBatchByBizId(String reportId, List<String> bizIds, JimuReportExportRequestDTO param, Path xlsxStorePath, ArrayList<File> files,
CountDownLatch cdl) {
for (String id : bizIds) {
param.getQueryParam().setId(id);
final Response response = jiMuReportOpenFeignClient.exportAllExcelStream(param);
// 取出文件后缀
final LinkedList<String> header = (LinkedList) response.headers().get("content-disposition");
final Matcher matcher = Pattern.compile("attachment;filename=(.+)(.xls|.xlsx)").matcher(header.get(0));
if (!matcher.matches()) {
log.error("【报表批量导出】批量导出失败，后缀不匹配。文件名：{}, 报表ID:{}, 业务数据ID：{}", header.get(0), reportId, id);
continue;
}
final File excelFile = xlsxStorePath.resolve("file_" + id + matcher.group(2)).toFile();
files.add(excelFile);
try (FileOutputStream fos = new FileOutputStream(excelFile)) {
IOUtils.copy(response.body().asInputStream(), fos);
} catch (Exception e) {
log.error("【报表批量导出】生成临时表格文件失败,文件名：{}，报表ID:{}, 业务数据ID：{},错误信息:{}", header.get(0), reportId, id, ExceptionUtils.getErrorStackTrace(e));
}
}
cdl.countDown();
}
/**
* 清理临时保存文件
*/

Write Preview
Loading…
Cancel
Save