From 0ec5501bc09bef604e62b8097032bcedbc75e095 Mon Sep 17 00:00:00 2001 From: jianjun Date: Mon, 27 Jul 2020 09:23:35 +0800 Subject: [PATCH] =?UTF-8?q?=E5=9B=BE=E7=89=87=E5=AE=A1=E6=A0=B8=EF=BC=8C?= =?UTF-8?q?=E5=88=86=E6=89=B9=E5=8F=91=E9=80=81=20=E6=9C=80=E5=A4=A7?= =?UTF-8?q?=E6=94=AF=E6=8C=81100=E4=B8=AA=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../impl/StatsPublicityServiceImpl.java | 2 +- .../scan/common/constant/SysConstant.java | 10 +- .../scan/service/impl/ScanServiceImpl.java | 91 +++++++++++++------ .../scan/support/result/SyncScanResult.java | 4 - 4 files changed, 72 insertions(+), 35 deletions(-) diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsPublicityServiceImpl.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsPublicityServiceImpl.java index c0f4b194a9..bcc55ede28 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsPublicityServiceImpl.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsPublicityServiceImpl.java @@ -613,7 +613,7 @@ public class StatsPublicityServiceImpl implements StatsPublicityService { for (ArticleTagsEntity tagEntity : articleTagsList) { ArticleEntity articleEntity = articleMap.get(tagEntity.getArticleId()); if (articleEntity == null) { - log.error("publicitySummary articleMap articleId:{} ", tagEntity.getArticleId()); + log.error("publicitySummary articleMap not contain articleId:{} ", tagEntity.getArticleId()); continue; } DimAgencyEntity dimAgencyEntity = dimAgencyEntityMap.get(articleEntity.getOrgId()); diff --git a/epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/common/constant/SysConstant.java b/epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/common/constant/SysConstant.java index a23795e538..8d7aa5d4d0 100644 --- a/epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/common/constant/SysConstant.java +++ b/epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/common/constant/SysConstant.java @@ -9,9 +9,15 @@ package com.epmet.openapi.scan.common.constant; public class SysConstant { public static final String UTF8 = "utf-8"; + /** + * 文本审核最大任务数 + */ + public static final Integer MAX_TASK_SIZE = 100; - public static final Integer MAX_TEXT_TASKS = 100; - public static final Integer MAX_IMG_TASKS = 10; + /** + * 图片审核最大任务数 + */ + public static final Integer MAX_SCAN_IMG_TASK_SIZE = 10; public static final String CODE = "code"; public static final String DATA = "data"; diff --git a/epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/service/impl/ScanServiceImpl.java b/epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/service/impl/ScanServiceImpl.java index 3756eb1764..2a5db725e2 100644 --- a/epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/service/impl/ScanServiceImpl.java +++ b/epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/service/impl/ScanServiceImpl.java @@ -24,6 +24,7 @@ import com.epmet.openapi.scan.support.result.ScanTaskResult; import com.epmet.openapi.scan.support.result.SceneDetailResult; import com.epmet.openapi.scan.support.result.SyncScanResult; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.ListUtils; import org.apache.http.HttpStatus; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -53,29 +54,19 @@ public class ScanServiceImpl implements ScanService { textScanParam.setBizType(bizType); List textTasks = textScanParam.getTasks(); - if (CollectionUtils.isEmpty(textTasks) || textTasks.size() > SysConstant.MAX_TEXT_TASKS) { - return new Result().error(SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getCode(), SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getMsg()); + if (CollectionUtils.isEmpty(textTasks) || textTasks.size() > SysConstant.MAX_TASK_SIZE) { + return new Result().error(SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getCode(), SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getMsg().concat(SysConstant.MAX_TASK_SIZE.toString())); } - TextScanRequest textScanRequest = new TextScanRequest(); - textScanRequest.setAcceptFormat(FormatType.JSON); // 指定api返回格式 - textScanRequest.setMethod(com.aliyuncs.http.MethodType.POST); // 指定请求方法 - textScanRequest.setEncoding(SysConstant.UTF8); - textScanRequest.setRegionId(regionId); + TextScanRequest textScanRequest = getTextScanRequest(); try { textScanRequest.setHttpContent(JSON.toJSONString(textScanParam).getBytes(SysConstant.UTF8), SysConstant.UTF8, FormatType.JSON); } catch (UnsupportedEncodingException e) { log.error("sendTextScan parse param exception", e); + return new Result().error(SysResponseEnum.SCAN_PARAM_ERROR.getCode(), SysResponseEnum.SCAN_PARAM_ERROR.getMsg()); } - - /** - * 请务必设置超时时间 - */ - textScanRequest.setConnectTimeout(3000); - textScanRequest.setReadTimeout(6000); - try { SyncScanResult textScanResult = executeSyncText(textScanRequest); return new Result().ok(textScanResult); @@ -85,24 +76,63 @@ public class ScanServiceImpl implements ScanService { } } + private TextScanRequest getTextScanRequest() { + TextScanRequest textScanRequest = new TextScanRequest(); + textScanRequest.setAcceptFormat(FormatType.JSON); // 指定api返回格式 + textScanRequest.setMethod(com.aliyuncs.http.MethodType.POST); // 指定请求方法 + textScanRequest.setEncoding(SysConstant.UTF8); + textScanRequest.setRegionId(regionId); + textScanRequest.setConnectTimeout(3000); + textScanRequest.setReadTimeout(6000); + return textScanRequest; + } + @Override public Result sendSyncImgScan(ImgScanParam imgScanParam) { - ImageSyncScanRequest imageSyncScanRequest = new ImageSyncScanRequest(); - imageSyncScanRequest.setAcceptFormat(FormatType.JSON); // 指定api返回格式 - imageSyncScanRequest.setMethod(com.aliyuncs.http.MethodType.POST); // 指定请求方法 - imageSyncScanRequest.setEncoding(SysConstant.UTF8); - imageSyncScanRequest.setRegionId(regionId); - //默认参数 // 鉴黄 暴恐涉政 imgScanParam.setScenes(ImgSceneEnum.getImgSceneList()); imgScanParam.setBizType(bizType); List imgTasks = imgScanParam.getTasks(); - if (CollectionUtils.isEmpty(imgTasks) || imgTasks.size() > SysConstant.MAX_IMG_TASKS) { - return new Result().error(SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getCode(), SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getMsg()); + if (CollectionUtils.isEmpty(imgTasks) || imgTasks.size() > SysConstant.MAX_TASK_SIZE) { + return new Result().error(SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getCode(), SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getMsg().concat(SysConstant.MAX_TASK_SIZE.toString())); + } + if (imgTasks.size() <= SysConstant.MAX_SCAN_IMG_TASK_SIZE) { + return doScanImg(imgScanParam); } + log.info("sendSyncImgScan tasks size:{} over 10", imgTasks.size()); + List> partition = ListUtils.partition(imgTasks, SysConstant.MAX_SCAN_IMG_TASK_SIZE); + SyncScanResult finalResult = new SyncScanResult(); + for (List tasks : partition) { + ImgScanParam scanParam = new ImgScanParam(); + scanParam.setBizType(imgScanParam.getBizType()); + scanParam.setScenes(imgScanParam.getScenes()); + scanParam.setTasks(tasks); + scanParam.setCallback(imgScanParam.getCallback()); + scanParam.setSeed(imgScanParam.getSeed()); + Result partResult = doScanImg(scanParam); + try { + Thread.sleep(5L); + } catch (InterruptedException e) { + log.error("sendSyncImgScan InterruptedException"); + } + if (partResult.success()) { + SyncScanResult data = partResult.getData(); + finalResult.getSuccessDataIds().addAll(data.getSuccessDataIds()); + finalResult.getFailDataIds().addAll(data.getFailDataIds()); + finalResult.getDetails().addAll(data.getDetails()); + } else { + return partResult; + } + } + return new Result().ok(finalResult); + + } + + private Result doScanImg(ImgScanParam imgScanParam) { + ImageSyncScanRequest imageSyncScanRequest = getImgScanRequest(); try { imageSyncScanRequest.setHttpContent(JSON.toJSONString(imgScanParam).getBytes(SysConstant.UTF8), SysConstant.UTF8, FormatType.JSON); } catch (UnsupportedEncodingException e) { @@ -110,12 +140,6 @@ public class ScanServiceImpl implements ScanService { return new Result().error(SysResponseEnum.SCAN_PARAM_ERROR.getCode(), SysResponseEnum.SCAN_PARAM_ERROR.getMsg()); } - /** - * 请务必设置超时时间 - */ - imageSyncScanRequest.setConnectTimeout(3000); - imageSyncScanRequest.setReadTimeout(6000); - try { SyncScanResult scanResult = executeSyncImg(imageSyncScanRequest); return new Result().ok(scanResult); @@ -125,6 +149,17 @@ public class ScanServiceImpl implements ScanService { } } + private ImageSyncScanRequest getImgScanRequest() { + ImageSyncScanRequest imageSyncScanRequest = new ImageSyncScanRequest(); + imageSyncScanRequest.setAcceptFormat(FormatType.JSON); // 指定api返回格式 + imageSyncScanRequest.setMethod(com.aliyuncs.http.MethodType.POST); // 指定请求方法 + imageSyncScanRequest.setEncoding(SysConstant.UTF8); + imageSyncScanRequest.setRegionId(regionId); + imageSyncScanRequest.setConnectTimeout(3000); + imageSyncScanRequest.setReadTimeout(6000); + return imageSyncScanRequest; + } + public SyncScanResult executeSyncText(AcsRequest textScanRequest) { SyncScanResult result = new SyncScanResult(); try { diff --git a/epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/support/result/SyncScanResult.java b/epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/support/result/SyncScanResult.java index c50589d7ce..8dcb757f64 100644 --- a/epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/support/result/SyncScanResult.java +++ b/epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/support/result/SyncScanResult.java @@ -39,8 +39,4 @@ public class SyncScanResult implements Serializable { } return isAllPass; } - - public void setAllPass(boolean allPass) { - isAllPass = allPass; - } }