Merge remote-tracking branch 'origin/dev' into dev

epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsPublicityServiceImpl.java

@@ -613,7 +613,7 @@ public class StatsPublicityServiceImpl implements StatsPublicityService {
 				for (ArticleTagsEntity tagEntity : articleTagsList) {
 					ArticleEntity articleEntity = articleMap.get(tagEntity.getArticleId());
 					if (articleEntity == null) {
-						log.error("publicitySummary articleMap articleId:{} ", tagEntity.getArticleId());
+						log.error("publicitySummary articleMap not contain  articleId:{} ", tagEntity.getArticleId());
 						continue;
 					}
 					DimAgencyEntity dimAgencyEntity = dimAgencyEntityMap.get(articleEntity.getOrgId());

epmet-openapi/epmet-openapi-scan/deploy/docker-compose-dev.yml

@@ -2,7 +2,7 @@ version: "3.7"
 services:
   epmet-openapi-scan:
     container_name: epmet-openapi-scan-dev
-    image: 192.168.1.130:10080/epmet-cloud-dev/epmet-openapi-scan:0.3.23
+    image: 192.168.1.130:10080/epmet-cloud-dev/epmet-openapi-scan:0.3.24
     ports:
       - "8107:8107"
     network_mode: host # 不会创建新的网络

epmet-openapi/epmet-openapi-scan/deploy/docker-compose-prod.yml

@@ -2,7 +2,7 @@ version: "3.7"
 services:
   epmet-openapi-scan:
     container_name: epmet-openapi-scan-prod
-    image: registry-vpc.cn-qingdao.aliyuncs.com/epmet-cloud-master/epmet-openapi-scan:0.3.23
+    image: registry-vpc.cn-qingdao.aliyuncs.com/epmet-cloud-master/epmet-openapi-scan:0.3.24
     ports:
       - "8107:8107"
     network_mode: host # 不会创建新的网络

epmet-openapi/epmet-openapi-scan/deploy/docker-compose-test.yml

@@ -2,7 +2,7 @@ version: "3.7"
 services:
   epmet-openapi-scan:
     container_name: epmet-openapi-scan-test
-    image: registry-vpc.cn-qingdao.aliyuncs.com/epmet-cloud-release/epmet-openapi-scan:0.3.23
+    image: registry-vpc.cn-qingdao.aliyuncs.com/epmet-cloud-release/epmet-openapi-scan:0.3.24
     ports:
       - "8107:8107"
     network_mode: host # 不会创建新的网络

epmet-openapi/epmet-openapi-scan/pom.xml

@@ -4,7 +4,7 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 
     <modelVersion>4.0.0</modelVersion>
-    <version>0.3.23</version>
+    <version>0.3.24</version>
     <artifactId>epmet-openapi-scan</artifactId>
     <packaging>jar</packaging>
 

epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/common/constant/SysConstant.java

@@ -9,9 +9,15 @@ package com.epmet.openapi.scan.common.constant;
 public class SysConstant {
 
 	public static final String UTF8 = "utf-8";
+	/**
+	 * 文本审核最大任务数
+	 */
+	public static final Integer MAX_TASK_SIZE = 100;
 
-	public static final Integer MAX_TEXT_TASKS = 100;
+	/**
-	public static final Integer MAX_IMG_TASKS = 10;
+	 * 图片审核最大任务数
+	 */
+	public static final Integer MAX_SCAN_IMG_TASK_SIZE = 10;
 
 	public static final String CODE = "code";
 	public static final String DATA = "data";

epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/service/impl/ScanServiceImpl.java

@@ -24,6 +24,7 @@ import com.epmet.openapi.scan.support.result.ScanTaskResult;
 import com.epmet.openapi.scan.support.result.SceneDetailResult;
 import com.epmet.openapi.scan.support.result.SyncScanResult;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.collections4.ListUtils;
 import org.apache.http.HttpStatus;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
@@ -53,29 +54,19 @@ public class ScanServiceImpl implements ScanService {
 		textScanParam.setBizType(bizType);
 
 		List<TextTask> textTasks = textScanParam.getTasks();
-		if (CollectionUtils.isEmpty(textTasks) || textTasks.size() > SysConstant.MAX_TEXT_TASKS) {
+		if (CollectionUtils.isEmpty(textTasks) || textTasks.size() > SysConstant.MAX_TASK_SIZE) {
-			return new Result<SyncScanResult>().error(SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getCode(), SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getMsg());
+			return new Result<SyncScanResult>().error(SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getCode(), SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getMsg().concat(SysConstant.MAX_TASK_SIZE.toString()));
 		}
 
-		TextScanRequest textScanRequest = new TextScanRequest();
+		TextScanRequest textScanRequest = getTextScanRequest();
-		textScanRequest.setAcceptFormat(FormatType.JSON); // 指定api返回格式
-		textScanRequest.setMethod(com.aliyuncs.http.MethodType.POST); // 指定请求方法
-		textScanRequest.setEncoding(SysConstant.UTF8);
-		textScanRequest.setRegionId(regionId);
 
 		try {
 			textScanRequest.setHttpContent(JSON.toJSONString(textScanParam).getBytes(SysConstant.UTF8), SysConstant.UTF8, FormatType.JSON);
 		} catch (UnsupportedEncodingException e) {
 			log.error("sendTextScan parse param exception", e);
+			return new Result<SyncScanResult>().error(SysResponseEnum.SCAN_PARAM_ERROR.getCode(), SysResponseEnum.SCAN_PARAM_ERROR.getMsg());
 		}
 
-
-		/**
-		 * 请务必设置超时时间
-		 */
-		textScanRequest.setConnectTimeout(3000);
-		textScanRequest.setReadTimeout(6000);
-
 		try {
 			SyncScanResult textScanResult = executeSyncText(textScanRequest);
 			return new Result<SyncScanResult>().ok(textScanResult);
@@ -85,24 +76,63 @@ public class ScanServiceImpl implements ScanService {
 		}
 	}
 
+	private TextScanRequest getTextScanRequest() {
+		TextScanRequest textScanRequest = new TextScanRequest();
+		textScanRequest.setAcceptFormat(FormatType.JSON); // 指定api返回格式
+		textScanRequest.setMethod(com.aliyuncs.http.MethodType.POST); // 指定请求方法
+		textScanRequest.setEncoding(SysConstant.UTF8);
+		textScanRequest.setRegionId(regionId);
+		textScanRequest.setConnectTimeout(3000);
+		textScanRequest.setReadTimeout(6000);
+		return textScanRequest;
+	}
+
 	@Override
 	public Result<SyncScanResult> sendSyncImgScan(ImgScanParam imgScanParam) {
-		ImageSyncScanRequest imageSyncScanRequest = new ImageSyncScanRequest();
-		imageSyncScanRequest.setAcceptFormat(FormatType.JSON); // 指定api返回格式
-		imageSyncScanRequest.setMethod(com.aliyuncs.http.MethodType.POST); // 指定请求方法
-		imageSyncScanRequest.setEncoding(SysConstant.UTF8);
-		imageSyncScanRequest.setRegionId(regionId);
-
 		//默认参数
 		// 鉴黄 暴恐涉政
 		imgScanParam.setScenes(ImgSceneEnum.getImgSceneList());
 		imgScanParam.setBizType(bizType);
 
 		List<ImgTask> imgTasks = imgScanParam.getTasks();
-		if (CollectionUtils.isEmpty(imgTasks) || imgTasks.size() > SysConstant.MAX_IMG_TASKS) {
+		if (CollectionUtils.isEmpty(imgTasks) || imgTasks.size() > SysConstant.MAX_TASK_SIZE) {
-			return new Result<SyncScanResult>().error(SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getCode(), SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getMsg());
+			return new Result<SyncScanResult>().error(SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getCode(), SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getMsg().concat(SysConstant.MAX_TASK_SIZE.toString()));
+		}
+		if (imgTasks.size() <= SysConstant.MAX_SCAN_IMG_TASK_SIZE) {
+			return doScanImg(imgScanParam);
 		}
+		log.info("sendSyncImgScan tasks size:{} over 10", imgTasks.size());
+		List<List<ImgTask>> partition = ListUtils.partition(imgTasks, SysConstant.MAX_SCAN_IMG_TASK_SIZE);
+		SyncScanResult finalResult = new SyncScanResult();
+		for (List<ImgTask> tasks : partition) {
+			ImgScanParam scanParam = new ImgScanParam();
+			scanParam.setBizType(imgScanParam.getBizType());
+			scanParam.setScenes(imgScanParam.getScenes());
+			scanParam.setTasks(tasks);
+			scanParam.setCallback(imgScanParam.getCallback());
+			scanParam.setSeed(imgScanParam.getSeed());
+			Result<SyncScanResult> partResult = doScanImg(scanParam);
+			try {
+				Thread.sleep(5L);
+			} catch (InterruptedException e) {
+				log.error("sendSyncImgScan InterruptedException");
+			}
+			if (partResult.success()) {
+				SyncScanResult data = partResult.getData();
+				finalResult.getSuccessDataIds().addAll(data.getSuccessDataIds());
+				finalResult.getFailDataIds().addAll(data.getFailDataIds());
+				finalResult.getDetails().addAll(data.getDetails());
+			} else {
+				return partResult;
+			}
+		}
+		return new Result<SyncScanResult>().ok(finalResult);
+
 
+	}
+
+	private Result<SyncScanResult> doScanImg(ImgScanParam imgScanParam) {
+		ImageSyncScanRequest imageSyncScanRequest = getImgScanRequest();
 		try {
 			imageSyncScanRequest.setHttpContent(JSON.toJSONString(imgScanParam).getBytes(SysConstant.UTF8), SysConstant.UTF8, FormatType.JSON);
 		} catch (UnsupportedEncodingException e) {
@@ -110,12 +140,6 @@ public class ScanServiceImpl implements ScanService {
 			return new Result<SyncScanResult>().error(SysResponseEnum.SCAN_PARAM_ERROR.getCode(), SysResponseEnum.SCAN_PARAM_ERROR.getMsg());
 		}
 
-		/**
-		 * 请务必设置超时时间
-		 */
-		imageSyncScanRequest.setConnectTimeout(3000);
-		imageSyncScanRequest.setReadTimeout(6000);
-
 		try {
 			SyncScanResult scanResult = executeSyncImg(imageSyncScanRequest);
 			return new Result<SyncScanResult>().ok(scanResult);
@@ -125,6 +149,17 @@ public class ScanServiceImpl implements ScanService {
 		}
 	}
 
+	private ImageSyncScanRequest getImgScanRequest() {
+		ImageSyncScanRequest imageSyncScanRequest = new ImageSyncScanRequest();
+		imageSyncScanRequest.setAcceptFormat(FormatType.JSON); // 指定api返回格式
+		imageSyncScanRequest.setMethod(com.aliyuncs.http.MethodType.POST); // 指定请求方法
+		imageSyncScanRequest.setEncoding(SysConstant.UTF8);
+		imageSyncScanRequest.setRegionId(regionId);
+		imageSyncScanRequest.setConnectTimeout(3000);
+		imageSyncScanRequest.setReadTimeout(6000);
+		return imageSyncScanRequest;
+	}
+
 	public SyncScanResult executeSyncText(AcsRequest<?> textScanRequest) {
 		SyncScanResult result = new SyncScanResult();
 		try {

epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/support/result/SyncScanResult.java

@@ -39,8 +39,4 @@ public class SyncScanResult implements Serializable {
 		}
 		return isAllPass;
 	}
-
-	public void setAllPass(boolean allPass) {
-		isAllPass = allPass;
-	}
 }