rongchao
/
epmet-cloud-rizhao
1
0
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

图片审核，分批发送 最大支持100个任务

dev
jianjun 5 years ago
parent
669306f6db
commit
0ec5501bc0
4 changed files with 72 additions and 35 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsPublicityServiceImpl.java
  2. 10
      epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/common/constant/SysConstant.java
  3. 91
      epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/service/impl/ScanServiceImpl.java
  4. 4
      epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/support/result/SyncScanResult.java

2
epmet-module/data-statistical/data-statistical-server/src/main/java/com/epmet/service/impl/StatsPublicityServiceImpl.java
View File

@ -613,7 +613,7 @@ public class StatsPublicityServiceImpl implements StatsPublicityService {
for (ArticleTagsEntity tagEntity : articleTagsList) {
ArticleEntity articleEntity = articleMap.get(tagEntity.getArticleId());
if (articleEntity == null) {
log.error("publicitySummary articleMap articleId:{} ", tagEntity.getArticleId());
log.error("publicitySummary articleMap not contain articleId:{} ", tagEntity.getArticleId());
continue;
}
DimAgencyEntity dimAgencyEntity = dimAgencyEntityMap.get(articleEntity.getOrgId());

10
epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/common/constant/SysConstant.java
View File

@ -9,9 +9,15 @@ package com.epmet.openapi.scan.common.constant;
public class SysConstant {
public static final String UTF8 = "utf-8";
/**
* 文本审核最大任务数
*/
public static final Integer MAX_TASK_SIZE = 100;
public static final Integer MAX_TEXT_TASKS = 100;
public static final Integer MAX_IMG_TASKS = 10;
/**
* 图片审核最大任务数
*/
public static final Integer MAX_SCAN_IMG_TASK_SIZE = 10;
public static final String CODE = "code";
public static final String DATA = "data";

91
epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/service/impl/ScanServiceImpl.java
View File

@ -24,6 +24,7 @@ import com.epmet.openapi.scan.support.result.ScanTaskResult;
import com.epmet.openapi.scan.support.result.SceneDetailResult;
import com.epmet.openapi.scan.support.result.SyncScanResult;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.ListUtils;
import org.apache.http.HttpStatus;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
@ -53,29 +54,19 @@ public class ScanServiceImpl implements ScanService {
textScanParam.setBizType(bizType);
List<TextTask> textTasks = textScanParam.getTasks();
if (CollectionUtils.isEmpty(textTasks) || textTasks.size() > SysConstant.MAX_TEXT_TASKS) {
return new Result<SyncScanResult>().error(SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getCode(), SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getMsg());
if (CollectionUtils.isEmpty(textTasks) || textTasks.size() > SysConstant.MAX_TASK_SIZE) {
return new Result<SyncScanResult>().error(SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getCode(), SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getMsg().concat(SysConstant.MAX_TASK_SIZE.toString()));
}
TextScanRequest textScanRequest = new TextScanRequest();
textScanRequest.setAcceptFormat(FormatType.JSON); // 指定api返回格式
textScanRequest.setMethod(com.aliyuncs.http.MethodType.POST); // 指定请求方法
textScanRequest.setEncoding(SysConstant.UTF8);
textScanRequest.setRegionId(regionId);
TextScanRequest textScanRequest = getTextScanRequest();
try {
textScanRequest.setHttpContent(JSON.toJSONString(textScanParam).getBytes(SysConstant.UTF8), SysConstant.UTF8, FormatType.JSON);
} catch (UnsupportedEncodingException e) {
log.error("sendTextScan parse param exception", e);
return new Result<SyncScanResult>().error(SysResponseEnum.SCAN_PARAM_ERROR.getCode(), SysResponseEnum.SCAN_PARAM_ERROR.getMsg());
}
/**
* 请务必设置超时时间
*/
textScanRequest.setConnectTimeout(3000);
textScanRequest.setReadTimeout(6000);
try {
SyncScanResult textScanResult = executeSyncText(textScanRequest);
return new Result<SyncScanResult>().ok(textScanResult);
@ -85,24 +76,63 @@ public class ScanServiceImpl implements ScanService {
}
}
private TextScanRequest getTextScanRequest() {
TextScanRequest textScanRequest = new TextScanRequest();
textScanRequest.setAcceptFormat(FormatType.JSON); // 指定api返回格式
textScanRequest.setMethod(com.aliyuncs.http.MethodType.POST); // 指定请求方法
textScanRequest.setEncoding(SysConstant.UTF8);
textScanRequest.setRegionId(regionId);
textScanRequest.setConnectTimeout(3000);
textScanRequest.setReadTimeout(6000);
return textScanRequest;
}
@Override
public Result<SyncScanResult> sendSyncImgScan(ImgScanParam imgScanParam) {
ImageSyncScanRequest imageSyncScanRequest = new ImageSyncScanRequest();
imageSyncScanRequest.setAcceptFormat(FormatType.JSON); // 指定api返回格式
imageSyncScanRequest.setMethod(com.aliyuncs.http.MethodType.POST); // 指定请求方法
imageSyncScanRequest.setEncoding(SysConstant.UTF8);
imageSyncScanRequest.setRegionId(regionId);
//默认参数
// 鉴黄 暴恐涉政
imgScanParam.setScenes(ImgSceneEnum.getImgSceneList());
imgScanParam.setBizType(bizType);
List<ImgTask> imgTasks = imgScanParam.getTasks();
if (CollectionUtils.isEmpty(imgTasks) || imgTasks.size() > SysConstant.MAX_IMG_TASKS) {
return new Result<SyncScanResult>().error(SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getCode(), SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getMsg());
if (CollectionUtils.isEmpty(imgTasks) || imgTasks.size() > SysConstant.MAX_TASK_SIZE) {
return new Result<SyncScanResult>().error(SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getCode(), SysResponseEnum.SCAN_TASK_LIST_PARAM_ERROR.getMsg().concat(SysConstant.MAX_TASK_SIZE.toString()));
}
if (imgTasks.size() <= SysConstant.MAX_SCAN_IMG_TASK_SIZE) {
return doScanImg(imgScanParam);
}
log.info("sendSyncImgScan tasks size:{} over 10", imgTasks.size());
List<List<ImgTask>> partition = ListUtils.partition(imgTasks, SysConstant.MAX_SCAN_IMG_TASK_SIZE);
SyncScanResult finalResult = new SyncScanResult();
for (List<ImgTask> tasks : partition) {
ImgScanParam scanParam = new ImgScanParam();
scanParam.setBizType(imgScanParam.getBizType());
scanParam.setScenes(imgScanParam.getScenes());
scanParam.setTasks(tasks);
scanParam.setCallback(imgScanParam.getCallback());
scanParam.setSeed(imgScanParam.getSeed());
Result<SyncScanResult> partResult = doScanImg(scanParam);
try {
Thread.sleep(5L);
} catch (InterruptedException e) {
log.error("sendSyncImgScan InterruptedException");
}
if (partResult.success()) {
SyncScanResult data = partResult.getData();
finalResult.getSuccessDataIds().addAll(data.getSuccessDataIds());
finalResult.getFailDataIds().addAll(data.getFailDataIds());
finalResult.getDetails().addAll(data.getDetails());
} else {
return partResult;
}
}
return new Result<SyncScanResult>().ok(finalResult);
}
private Result<SyncScanResult> doScanImg(ImgScanParam imgScanParam) {
ImageSyncScanRequest imageSyncScanRequest = getImgScanRequest();
try {
imageSyncScanRequest.setHttpContent(JSON.toJSONString(imgScanParam).getBytes(SysConstant.UTF8), SysConstant.UTF8, FormatType.JSON);
} catch (UnsupportedEncodingException e) {
@ -110,12 +140,6 @@ public class ScanServiceImpl implements ScanService {
return new Result<SyncScanResult>().error(SysResponseEnum.SCAN_PARAM_ERROR.getCode(), SysResponseEnum.SCAN_PARAM_ERROR.getMsg());
}
/**
* 请务必设置超时时间
*/
imageSyncScanRequest.setConnectTimeout(3000);
imageSyncScanRequest.setReadTimeout(6000);
try {
SyncScanResult scanResult = executeSyncImg(imageSyncScanRequest);
return new Result<SyncScanResult>().ok(scanResult);
@ -125,6 +149,17 @@ public class ScanServiceImpl implements ScanService {
}
}
private ImageSyncScanRequest getImgScanRequest() {
ImageSyncScanRequest imageSyncScanRequest = new ImageSyncScanRequest();
imageSyncScanRequest.setAcceptFormat(FormatType.JSON); // 指定api返回格式
imageSyncScanRequest.setMethod(com.aliyuncs.http.MethodType.POST); // 指定请求方法
imageSyncScanRequest.setEncoding(SysConstant.UTF8);
imageSyncScanRequest.setRegionId(regionId);
imageSyncScanRequest.setConnectTimeout(3000);
imageSyncScanRequest.setReadTimeout(6000);
return imageSyncScanRequest;
}
public SyncScanResult executeSyncText(AcsRequest<?> textScanRequest) {
SyncScanResult result = new SyncScanResult();
try {

4
epmet-openapi/epmet-openapi-scan/src/main/java/com/epmet/openapi/scan/support/result/SyncScanResult.java
View File

@ -39,8 +39,4 @@ public class SyncScanResult implements Serializable {
}
return isAllPass;
}
public void setAllPass(boolean allPass) {
isAllPass = allPass;
}
}

Write Preview
Loading…
Cancel
Save