diff --git a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsPublicityServiceImpl.java b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsPublicityServiceImpl.java index c0f4b194a9..bcc55ede28 100644 --- a/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsPublicityServiceImpl.java +++ b/epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsPublicityServiceImpl.java @@ -613,7 +613,7 @@ public class StatsPublicityServiceImpl implements StatsPublicityService { for (ArticleTagsEntity tagEntity : articleTagsList) { ArticleEntity articleEntity = articleMap.get(tagEntity.getArticleId()); if (articleEntity == null) { - log.error("publicitySummary articleMap articleId:{} ", tagEntity.getArticleId()); + log.error("publicitySummary articleMap not contain articleId:{} ", tagEntity.getArticleId()); continue; } DimAgencyEntity dimAgencyEntity = dimAgencyEntityMap.get(articleEntity.getOrgId()); diff --git a/epmet-openapi/epmet-openapi-scan/deploy/docker-compose-dev.yml b/epmet-openapi/epmet-openapi-scan/deploy/docker-compose-dev.yml index 5137c630ed..a9f630c5e0 100644 --- a/epmet-openapi/epmet-openapi-scan/deploy/docker-compose-dev.yml +++ b/epmet-openapi/epmet-openapi-scan/deploy/docker-compose-dev.yml @@ -2,7 +2,7 @@ version: "3.7" services: epmet-openapi-scan: container_name: epmet-openapi-scan-dev - image: 192.168.1.130:10080/epmet-cloud-dev/epmet-openapi-scan:0.3.23 + image: 192.168.1.130:10080/epmet-cloud-dev/epmet-openapi-scan:0.3.24 ports: - "8107:8107" network_mode: host # 不会创建新的网络 diff --git a/epmet-openapi/epmet-openapi-scan/deploy/docker-compose-prod.yml b/epmet-openapi/epmet-openapi-scan/deploy/docker-compose-prod.yml index 15c4000109..41652ae4d5 100644 --- a/epmet-openapi/epmet-openapi-scan/deploy/docker-compose-prod.yml +++ b/epmet-openapi/epmet-openapi-scan/deploy/docker-compose-prod.yml @@ -2,7 +2,7 @@ version: "3.7" services: epmet-openapi-scan: container_name: epmet-openapi-scan-prod - image: registry-vpc.cn-qingdao.aliyuncs.com/epmet-cloud-master/epmet-openapi-scan:0.3.23 + image: registry-vpc.cn-qingdao.aliyuncs.com/epmet-cloud-master/epmet-openapi-scan:0.3.24 ports: - "8107:8107" network_mode: host # 不会创建新的网络 diff --git a/epmet-openapi/epmet-openapi-scan/deploy/docker-compose-test.yml b/epmet-openapi/epmet-openapi-scan/deploy/docker-compose-test.yml index be5e1ee040..20bfb7607a 100644 --- a/epmet-openapi/epmet-openapi-scan/deploy/docker-compose-test.yml +++ b/epmet-openapi/epmet-openapi-scan/deploy/docker-compose-test.yml @@ -2,7 +2,7 @@ version: "3.7" services: epmet-openapi-scan: container_name: epmet-openapi-scan-test - image: registry-vpc.cn-qingdao.aliyuncs.com/epmet-cloud-release/epmet-openapi-scan:0.3.23 + image: registry-vpc.cn-qingdao.aliyuncs.com/epmet-cloud-release/epmet-openapi-scan:0.3.24 ports: - "8107:8107" network_mode: host # 不会创建新的网络 diff --git a/epmet-openapi/epmet-openapi-scan/pom.xml b/epmet-openapi/epmet-openapi-scan/pom.xml index 747107e901..97029c52f2 100644 --- a/epmet-openapi/epmet-openapi-scan/pom.xml +++ b/epmet-openapi/epmet-openapi-scan/pom.xml @@ -4,7 +4,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - 0.3.23 + 0.3.24 epmet-openapi-scan jar diff --git a/epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/common/constant/SysConstant.java b/epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/common/constant/SysConstant.java index a23795e538..8d7aa5d4d0 100644 --- a/epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/common/constant/SysConstant.java +++ b/epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/common/constant/SysConstant.java @@ -9,9 +9,15 @@ package com.epmet.openapi.scan.common.constant; public class SysConstant { public static final String UTF8 = "utf-8"; + /** + * 文本审核最大任务数 + */ + public static final Integer MAX_TASK_SIZE = 100; - public static final Integer MAX_TEXT_TASKS = 100; - public static final Integer MAX_IMG_TASKS = 10; + /** + * 图片审核最大任务数 + */ + public static final Integer MAX_SCAN_IMG_TASK_SIZE = 10; public static final String CODE = "code"; public static final String DATA = "data"; diff --git a/epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/service/impl/ScanServiceImpl.java b/epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/service/impl/ScanServiceImpl.java index 3756eb1764..2a5db725e2 100644 --- a/epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/service/impl/ScanServiceImpl.java +++ b/epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/service/impl/ScanServiceImpl.java @@ -24,6 +24,7 @@ import com.epmet.openapi.scan.support.result.ScanTaskResult; import com.epmet.openapi.scan.support.result.SceneDetailResult; import com.epmet.openapi.scan.support.result.SyncScanResult; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.ListUtils; import org.apache.http.HttpStatus; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -53,29 +54,19 @@ public class ScanServiceImpl implements ScanService { textScanParam.setBizType(bizType); List textTasks = textScanParam.getTasks(); - if (CollectionUtils.isEmpty(textTasks) || textTasks.size() > SysConstant.MAX_TEXT_TASKS) { - return new Result().error(SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getCode(), SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getMsg()); + if (CollectionUtils.isEmpty(textTasks) || textTasks.size() > SysConstant.MAX_TASK_SIZE) { + return new Result().error(SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getCode(), SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getMsg().concat(SysConstant.MAX_TASK_SIZE.toString())); } - TextScanRequest textScanRequest = new TextScanRequest(); - textScanRequest.setAcceptFormat(FormatType.JSON); // 指定api返回格式 - textScanRequest.setMethod(com.aliyuncs.http.MethodType.POST); // 指定请求方法 - textScanRequest.setEncoding(SysConstant.UTF8); - textScanRequest.setRegionId(regionId); + TextScanRequest textScanRequest = getTextScanRequest(); try { textScanRequest.setHttpContent(JSON.toJSONString(textScanParam).getBytes(SysConstant.UTF8), SysConstant.UTF8, FormatType.JSON); } catch (UnsupportedEncodingException e) { log.error("sendTextScan parse param exception", e); + return new Result().error(SysResponseEnum.SCAN_PARAM_ERROR.getCode(), SysResponseEnum.SCAN_PARAM_ERROR.getMsg()); } - - /** - * 请务必设置超时时间 - */ - textScanRequest.setConnectTimeout(3000); - textScanRequest.setReadTimeout(6000); - try { SyncScanResult textScanResult = executeSyncText(textScanRequest); return new Result().ok(textScanResult); @@ -85,24 +76,63 @@ public class ScanServiceImpl implements ScanService { } } + private TextScanRequest getTextScanRequest() { + TextScanRequest textScanRequest = new TextScanRequest(); + textScanRequest.setAcceptFormat(FormatType.JSON); // 指定api返回格式 + textScanRequest.setMethod(com.aliyuncs.http.MethodType.POST); // 指定请求方法 + textScanRequest.setEncoding(SysConstant.UTF8); + textScanRequest.setRegionId(regionId); + textScanRequest.setConnectTimeout(3000); + textScanRequest.setReadTimeout(6000); + return textScanRequest; + } + @Override public Result sendSyncImgScan(ImgScanParam imgScanParam) { - ImageSyncScanRequest imageSyncScanRequest = new ImageSyncScanRequest(); - imageSyncScanRequest.setAcceptFormat(FormatType.JSON); // 指定api返回格式 - imageSyncScanRequest.setMethod(com.aliyuncs.http.MethodType.POST); // 指定请求方法 - imageSyncScanRequest.setEncoding(SysConstant.UTF8); - imageSyncScanRequest.setRegionId(regionId); - //默认参数 // 鉴黄 暴恐涉政 imgScanParam.setScenes(ImgSceneEnum.getImgSceneList()); imgScanParam.setBizType(bizType); List imgTasks = imgScanParam.getTasks(); - if (CollectionUtils.isEmpty(imgTasks) || imgTasks.size() > SysConstant.MAX_IMG_TASKS) { - return new Result().error(SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getCode(), SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getMsg()); + if (CollectionUtils.isEmpty(imgTasks) || imgTasks.size() > SysConstant.MAX_TASK_SIZE) { + return new Result().error(SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getCode(), SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getMsg().concat(SysConstant.MAX_TASK_SIZE.toString())); + } + if (imgTasks.size() <= SysConstant.MAX_SCAN_IMG_TASK_SIZE) { + return doScanImg(imgScanParam); } + log.info("sendSyncImgScan tasks size:{} over 10", imgTasks.size()); + List> partition = ListUtils.partition(imgTasks, SysConstant.MAX_SCAN_IMG_TASK_SIZE); + SyncScanResult finalResult = new SyncScanResult(); + for (List tasks : partition) { + ImgScanParam scanParam = new ImgScanParam(); + scanParam.setBizType(imgScanParam.getBizType()); + scanParam.setScenes(imgScanParam.getScenes()); + scanParam.setTasks(tasks); + scanParam.setCallback(imgScanParam.getCallback()); + scanParam.setSeed(imgScanParam.getSeed()); + Result partResult = doScanImg(scanParam); + try { + Thread.sleep(5L); + } catch (InterruptedException e) { + log.error("sendSyncImgScan InterruptedException"); + } + if (partResult.success()) { + SyncScanResult data = partResult.getData(); + finalResult.getSuccessDataIds().addAll(data.getSuccessDataIds()); + finalResult.getFailDataIds().addAll(data.getFailDataIds()); + finalResult.getDetails().addAll(data.getDetails()); + } else { + return partResult; + } + } + return new Result().ok(finalResult); + + } + + private Result doScanImg(ImgScanParam imgScanParam) { + ImageSyncScanRequest imageSyncScanRequest = getImgScanRequest(); try { imageSyncScanRequest.setHttpContent(JSON.toJSONString(imgScanParam).getBytes(SysConstant.UTF8), SysConstant.UTF8, FormatType.JSON); } catch (UnsupportedEncodingException e) { @@ -110,12 +140,6 @@ public class ScanServiceImpl implements ScanService { return new Result().error(SysResponseEnum.SCAN_PARAM_ERROR.getCode(), SysResponseEnum.SCAN_PARAM_ERROR.getMsg()); } - /** - * 请务必设置超时时间 - */ - imageSyncScanRequest.setConnectTimeout(3000); - imageSyncScanRequest.setReadTimeout(6000); - try { SyncScanResult scanResult = executeSyncImg(imageSyncScanRequest); return new Result().ok(scanResult); @@ -125,6 +149,17 @@ public class ScanServiceImpl implements ScanService { } } + private ImageSyncScanRequest getImgScanRequest() { + ImageSyncScanRequest imageSyncScanRequest = new ImageSyncScanRequest(); + imageSyncScanRequest.setAcceptFormat(FormatType.JSON); // 指定api返回格式 + imageSyncScanRequest.setMethod(com.aliyuncs.http.MethodType.POST); // 指定请求方法 + imageSyncScanRequest.setEncoding(SysConstant.UTF8); + imageSyncScanRequest.setRegionId(regionId); + imageSyncScanRequest.setConnectTimeout(3000); + imageSyncScanRequest.setReadTimeout(6000); + return imageSyncScanRequest; + } + public SyncScanResult executeSyncText(AcsRequest textScanRequest) { SyncScanResult result = new SyncScanResult(); try { diff --git a/epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/support/result/SyncScanResult.java b/epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/support/result/SyncScanResult.java index c50589d7ce..8dcb757f64 100644 --- a/epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/support/result/SyncScanResult.java +++ b/epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/support/result/SyncScanResult.java @@ -39,8 +39,4 @@ public class SyncScanResult implements Serializable { } return isAllPass; } - - public void setAllPass(boolean allPass) { - isAllPass = allPass; - } }