Skip to content

M
middleware-automatic
Commit 7475fd26 by shentao
Options

Merge branch 'release' into 'master' 

Release

See merge request !113
parents ab20f51a 6b7d21cf
Showing with 532 additions and 143 deletions
middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/pojo/AIMark.java
......@@ -18,6 +18,11 @@ public class AIMark implements Serializable {
* 分层id
*/
private Integer monitorLevelId;
private List<Integer> monitorLevelIds;
private String type;
/**
* 接口信息
*/
......@@ -83,6 +88,22 @@ public class AIMark implements Serializable {
this.projectId = projectId;
}
public List<Integer> getMonitorLevelIds() {
return monitorLevelIds;
}
public void setMonitorLevelIds(List<Integer> monitorLevelIds) {
this.monitorLevelIds = monitorLevelIds;
}
public String getType() {
return type;
}
public void setType(String type) {
this.type = type;
}
public static class TagRelation implements Serializable {
/**标签id*/
private Long id;
......@@ -99,6 +120,27 @@ public class AIMark implements Serializable {
/**标签关联value*/
private String relateValue;
/***是否包含前提条件*/
private Boolean hasRequirements;
/**前提条件*/
private List<Requirement> requirements;
public Boolean getHasRequirements() {
return hasRequirements;
}
public void setHasRequirements(Boolean hasRequirements) {
this.hasRequirements = hasRequirements;
}
public List<Requirement> getRequirements() {
return requirements;
}
public void setRequirements(List<Requirement> requirements) {
this.requirements = requirements;
}
public Long getId() {
return id;
}
......@@ -156,6 +198,28 @@ public class AIMark implements Serializable {
}
}
public static class Requirement implements Serializable {
private String requirementKey;
private String requirementValue;
public String getRequirementKey() {
return requirementKey;
}
public void setRequirementKey(String requirementKey) {
this.requirementKey = requirementKey;
}
public String getRequirementValue() {
return requirementValue;
}
public void setRequirementValue(String requirementValue) {
this.requirementValue = requirementValue;
}
}
public static class InterfaceRelation implements Serializable {
/** 接口地址*/
private String url;
......
middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/pojo/AiApiResult.java
......@@ -16,10 +16,30 @@ public class AiApiResult {
private String noise;
private String negLabel;
private String project;
private String result;
private Long createAt;
public String getNegLabel() {
return negLabel;
}
public void setNegLabel(String negLabel) {
this.negLabel = negLabel;
}
public String getProject() {
return project;
}
public void setProject(String project) {
this.project = project;
}
public String getId() {
return id;
}
......
middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/pojo/enums/AiInterfaceEnums.java 0 → 100644
package com.zhiwei.middleware.automatic.server.pojo.enums;
/**
* author：liu-yu
* date: 2024/7/9 10:03
**/
public enum AiInterfaceEnums {
FEI_HE("飞鹤", "62baa19f4d9a553ac3e7cd9b", "feihe-compet-tagger"),
LAO_XIANG_JI("老乡鸡", "665d8791b36dd11c5e453955", "laoxiangji-tagger");
final String project;
final String projectId;
final String waitFor;
AiInterfaceEnums(String project, String projectId, String waitFor) {
this.project = project;
this.projectId = projectId;
this.waitFor = waitFor;
}
public String getProject() {
return project;
}
public String getProjectId() {
return projectId;
}
public String getWaitFor() {
return waitFor;
}
public static AiInterfaceEnums create(String projectId) {
for (AiInterfaceEnums enums : AiInterfaceEnums.values()) {
if (enums.getProjectId().equals(projectId)) {
return enums;
}
}
throw new IllegalStateException("该项目不支持ai标注：" + projectId);
}
}
middleware-automatic-center-server/src/main/java/com/zhiwei/middleware/automatic/server/mission/AsyncTask.java
......@@ -3,6 +3,7 @@ package com.zhiwei.middleware.automatic.server.mission;
import com.zhiwei.middleware.automatic.server.common.GenericAttribute;
import com.zhiwei.middleware.automatic.server.dao.AiMarkCountDao;
import com.zhiwei.middleware.automatic.server.pojo.AiMarkCount;
import com.zhiwei.middleware.automatic.server.pojo.enums.AiInterfaceEnums;
import com.zhiwei.middleware.automatic.server.util.RedissonUtil;
import com.zhiwei.middleware.automatic.server.util.TimeUtil;
import com.zhiwei.middleware.automatic.server.util.Tools;
......@@ -59,13 +60,16 @@ public class AsyncTask {
public void syncDayCount() {
String timeKey = TimeUtil.COMMON_TIME_FORMAT.format(System.currentTimeMillis() - GenericAttribute.DAY_TIME);
AiMarkCount aiMarkCount = new AiMarkCount();
aiMarkCount.setTime(timeKey);
aiMarkCount.setTotal(redissonUtil.getCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, GenericAttribute.KEY_TOTAL)));
aiMarkCount.setResError(redissonUtil.getCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, GenericAttribute.KEY_RES_ERROR)));
aiMarkCount.setResIncomplete(redissonUtil.getCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, GenericAttribute.KEY_RES_INCOMPLETE)));
aiMarkCount.setNotMarkRelation(redissonUtil.getCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, GenericAttribute.KEY_NOT_MARK_RELATION)));
aiMarkCount.setSuccess(redissonUtil.getCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, GenericAttribute.KEY_SUCCESS)));
aiMarkCountDao.insert(aiMarkCount);
for (AiInterfaceEnums enums : AiInterfaceEnums.values()) {
AiMarkCount aiMarkCount = new AiMarkCount();
aiMarkCount.setTime(timeKey);
aiMarkCount.setTotal(redissonUtil.getCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, enums.getProject(), GenericAttribute.KEY_TOTAL)));
aiMarkCount.setResError(redissonUtil.getCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, enums.getProject(), GenericAttribute.KEY_RES_ERROR)));
aiMarkCount.setResIncomplete(redissonUtil.getCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, enums.getProject(), GenericAttribute.KEY_RES_INCOMPLETE)));
aiMarkCount.setNotMarkRelation(redissonUtil.getCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, enums.getProject(), GenericAttribute.KEY_NOT_MARK_RELATION)));
aiMarkCount.setSuccess(redissonUtil.getCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, enums.getProject(), GenericAttribute.KEY_SUCCESS)));
aiMarkCountDao.insert(aiMarkCount);
}
}
}
middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/task/aiTask/AbstractAiTaskService.java 0 → 100644
package com.zhiwei.middleware.automatic.son.task.aiTask;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.base.entity.subclass.mark.MarkInfo;
import com.zhiwei.middleware.automatic.server.common.GenericAttribute;
import com.zhiwei.middleware.automatic.server.pojo.AIMark;
import com.zhiwei.middleware.automatic.server.pojo.AiApiResult;
import com.zhiwei.middleware.automatic.server.pojo.AiInterfaceParam;
import com.zhiwei.middleware.automatic.server.pojo.enums.AiInterfaceEnums;
import com.zhiwei.middleware.automatic.server.util.RedissonUtil;
import com.zhiwei.middleware.automatic.son.dao.AiApiResultDao;
import com.zhiwei.middleware.automatic.son.util.MarkInfoUtil;
import com.zhiwei.middleware.automatic.son.util.OkHttpUtil;
import com.zhiwei.middleware.automatic.son.util.Tools;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
/**
* author：liu-yu
* date: 2024/7/9 10:57
**/
public abstract class AbstractAiTaskService implements AiTaskService {
private static final Logger log = LogManager.getLogger(AbstractAiTaskService.class);
private final AiInterfaceEnums aiInterfaceEnums;
private final RedissonUtil redissonUtil;
private final AiApiResultDao aiApiResultDao;
public AbstractAiTaskService(AiInterfaceEnums aiInterfaceEnums, RedissonUtil redissonUtil,
AiApiResultDao aiApiResultDao) {
this.aiInterfaceEnums = aiInterfaceEnums;
this.redissonUtil = redissonUtil;
this.aiApiResultDao = aiApiResultDao;
}
protected JSONObject aiInterface(AiInterfaceParam aiParam, String timeKey) {
JSONObject param = aiInterfaceParamBuild(aiParam);
long now = System.currentTimeMillis();
AIMark.InterfaceRelation interfaceRelation = aiParam.getAiMark().getInterfaceRelation();
String s = httpResult(interfaceRelation.getMethod(), interfaceRelation.getUrl(), param.toString(), interfaceRelation.getHeader().getInnerMap());
requestMarkTotalCount(timeKey);
if (StringUtils.isEmpty(s)) {
requestErrorCount(timeKey);
log.error("ai标注失败，项目：{}，数据id：{}，耗时：{}，错误信息：ai接口返回为null" , aiInterfaceEnums.getProject(),
aiParam.getJson().getString("id"), System.currentTimeMillis() - now);
return null;
}
long endTime = System.currentTimeMillis() - now;
AiApiResult aiRes = aiApiResultConversion(s);
if (Objects.isNull(aiRes)) {
requestResIncomplete(timeKey);
log.info("项目：{}，数据url:{}，耗时：{}，ai接口返回信息不完整：{}", aiInterfaceEnums.getProject(), param.getJSONObject("data").getString(GenericAttribute.URL), endTime, s);
return null;
}
log.info("项目：{}，数据url：{}，ai标签：{}，耗时：{}，具体数据:{}", aiInterfaceEnums.getProject(), param.getJSONObject("data").getString(GenericAttribute.URL), aiRes.getSent(), endTime, aiRes.getResult());
List<AIMark.TagRelation> tagRelations = aiParam.getAiMark().getTagRelations();
Map<String, List<AIMark.TagRelation>> relateGroup = tagRelations.stream().collect(Collectors.groupingBy(AIMark.TagRelation::getRelateKey, Collectors.toList()));
List<AIMark.TagRelation> tagRelationList = relateGroup.values().stream().map(relations -> relations.stream().filter(x -> relateValueVerify(x, aiRes)).findFirst().orElse(null)).filter(Objects::nonNull).collect(Collectors.toList());
if (tagRelationList.isEmpty()) {
markNotRelation(timeKey);
log.info("数据url：{}，ai标签：{}，没有绑定关系，以过滤", param.getJSONObject("data").getString(GenericAttribute.URL), aiRes.getSent());
return null;
}
JSONObject json = aiParam.getJson();
String mtag = tagRelationList.stream().map(AIMark.TagRelation::getUniqueId).collect(Collectors.joining());
json.put(GenericAttribute.ES_M_TAG, mtag);
json.put(GenericAttribute.ES_M_PERSON, GenericAttribute.AI_PERSON);
json.put(GenericAttribute.ES_M_TIME, System.currentTimeMillis());
json.put(GenericAttribute.ES_M_GROUP, aiParam.getAiMark().getProject());
json.put(GenericAttribute.ES_M_GROUP_ID, aiParam.getAiMark().getProjectId());
log.info("数据url：{}，ai标签：{}，标注标签：{}，成功标注", param.getJSONObject("data").getString(GenericAttribute.URL), aiRes.getSent(), mtag);
return json;
}
protected JSONObject aiInterfaceParamBuild(AiInterfaceParam aiParam) {
JSONObject param = new JSONObject();
param.put("wait_for", new String[]{aiInterfaceEnums.getWaitFor()});
JSONObject data = new JSONObject();
JSONObject json = aiParam.getJson();
data.put("author", json.getString(GenericAttribute.ES_SOURCE));
data.put("pub_date", Instant.ofEpochMilli(json.getLong(GenericAttribute.ES_TIME)).toString());
List<String> indFullText = json.getJSONArray(GenericAttribute.ES_IND_FULL_TEXT).toJavaList(String.class);
data.put(GenericAttribute.ES_TITLE, indFullText.get(0));
data.put("snippet", indFullText.get(1));
String answerUrl = json.getString(GenericAttribute.ES_URL_ANSWER);
String questionUrl = json.getString(GenericAttribute.ES_URL_QUESTION);
String url = Objects.nonNull(answerUrl) ? answerUrl : Objects.nonNull(questionUrl) ? questionUrl : json.getString(GenericAttribute.URL);
data.put(GenericAttribute.URL, url);
JSONObject metadata = httpParamByMetadata(aiParam);
if (Objects.nonNull(metadata)) {
data.put("metadata", metadata);
}
param.put("data", data);
return param;
}
@Override
public MarkInfo executor(AiInterfaceParam aiParam, String timeKey) {
JSONObject jsonObject = aiInterface(aiParam, timeKey);
if (Objects.isNull(jsonObject)) {
return null;
}
MarkInfo markInfo = MarkInfoUtil.transformToMarkInfo(jsonObject);
markSuccess(timeKey);
return markInfo;
}
@Override
public AiInterfaceEnums getType() {
return aiInterfaceEnums;
}
@Override
public String httpResult(String httpType, String url, String paramInfo, Map<String, Object> header) {
switch (httpType) {
case "POST" :
return OkHttpUtil.httpPost(url, paramInfo, header);
case "PUT" :
return OkHttpUtil.httpPut(url, paramInfo, header);
}
return null;
}
@Override
public AiApiResult aiApiResultConversion(String interfaceRes) {
JSONObject res = JSONObject.parseObject(interfaceRes);
if (Objects.nonNull(res)) {
JSONObject waitRes = res.getJSONObject(aiInterfaceEnums.getWaitFor());
if (Objects.nonNull(waitRes)) {
JSONObject result = waitRes.getJSONObject("result");
if (Objects.nonNull(result)) {
aiApiResultTitleSubString(result);
if (waitRes.getString("status").equals("DONE") || (!StringUtils.isEmpty(result.getString("sent")) && StringUtils.isEmpty("dedup"))) {
AiApiResult aiRes = new AiApiResult();
aiRes.setId(waitRes.getString("result_id"));
aiRes.setCreateAt(System.currentTimeMillis());
aiRes.setInfo_class(result.getString("info_class"));
aiRes.setNoise(result.getString("denoise"));
aiRes.setSent(result.getString("sent"));
aiRes.setStatus(waitRes.getString("status"));
aiRes.setNegLabel(waitRes.getString("neg_label"));
aiRes.setResult(res.toJSONString());
aiRes.setProject(aiInterfaceEnums.getProject());
aiApiResultDao.insert(aiRes);
return aiRes;
} else {
return aiApiResultDao.findOne(new Query(Criteria.where("_id").is(result.getString("dedup"))));
}
}
}
}
return null;
}
@Override
public void requestMarkTotalCount(String timeKey) {
redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, aiInterfaceEnums.getProject(), GenericAttribute.KEY_TOTAL), 1);
}
@Override
public void requestErrorCount(String timeKey) {
redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, aiInterfaceEnums.getProject(), GenericAttribute.KEY_RES_ERROR), 1);
}
@Override
public void requestResIncomplete(String timeKey) {
redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, aiInterfaceEnums.getProject(), GenericAttribute.KEY_RES_INCOMPLETE), 1);
}
@Override
public void markNotRelation(String timeKey) {
redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, aiInterfaceEnums.getProject(), GenericAttribute.KEY_NOT_MARK_RELATION), 1);
}
@Override
public void markSuccess(String timeKey) {
redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, aiInterfaceEnums.getProject(), GenericAttribute.KEY_SUCCESS), 1);
}
private boolean relateValueVerify(AIMark.TagRelation tagRelation, AiApiResult result) {
String relateValue = getResultValue(tagRelation.getRelateKey(), result);
if (StringUtils.isEmpty(relateValue)) {
return false;
}
if (!relateValue.equals(tagRelation.getRelateValue())) {
return false;
}
if (Boolean.TRUE.equals(tagRelation.getHasRequirements())) {
return tagRelation.getRequirements().stream().allMatch(e -> e.getRequirementValue().equals(getResultValue(e.getRequirementKey(), result)));
}
return true;
}
private String getResultValue(String relateKey, AiApiResult result) {
String relateValue = null;
switch (relateKey) {
case "sent" :
relateValue = result.getSent();
break;
case "info_class":
relateValue = result.getInfo_class();
break;
case "" :
relateValue = result.getNegLabel();
break;
}
return relateValue;
}
private void aiApiResultTitleSubString(JSONObject json) {
String title = json.getString(GenericAttribute.ES_TITLE);
if (Objects.nonNull(title) && title.length() > 100) {
json.put(GenericAttribute.ES_TITLE, title.substring(0, 100));
}
String snippet = json.getString("snippet");
if (Objects.nonNull(snippet) && snippet.length() > 100) {
json.put("snippet", snippet.substring(0, 100));
}
}
}
middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/task/aiTask/AiTaskService.java 0 → 100644
package com.zhiwei.middleware.automatic.son.task.aiTask;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.base.entity.subclass.mark.MarkInfo;
import com.zhiwei.middleware.automatic.server.pojo.AiApiResult;
import com.zhiwei.middleware.automatic.server.pojo.AiInterfaceParam;
import com.zhiwei.middleware.automatic.server.pojo.enums.AiInterfaceEnums;
import java.util.Map;
/**
* author：liu-yu
* date: 2024/7/9 10:46
**/
public interface AiTaskService {
AiInterfaceEnums getType();
MarkInfo executor(AiInterfaceParam aiParam, String timeKey);
/**
* 构建请求参数 (metadata)
* @param aiParam aiParam
* @return 请求参数
*/
JSONObject httpParamByMetadata(AiInterfaceParam aiParam);
/**
* 接口请求
* @param httpType 请求类型
* @param url url
* @param header header
* @return 接口返回值
*/
String httpResult(String httpType, String url, String paramInfo, Map<String, Object> header);
/**
* 接口返回值转换
* @param interfaceRes 接口返回值
* @return AiApiResult
*/
AiApiResult aiApiResultConversion(String interfaceRes);
void requestMarkTotalCount(String timeKey);
void requestErrorCount(String timeKey);
void requestResIncomplete(String timeKey);
void markNotRelation(String timeKey);
void markSuccess(String timeKey);
}
middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/task/aiTask/AiTaskServiceFeiHe.java 0 → 100644
package com.zhiwei.middleware.automatic.son.task.aiTask;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.middleware.automatic.server.pojo.AiInterfaceParam;
import com.zhiwei.middleware.automatic.server.pojo.enums.AiInterfaceEnums;
import com.zhiwei.middleware.automatic.server.util.RedissonUtil;
import com.zhiwei.middleware.automatic.son.dao.AiApiResultDao;
import org.springframework.stereotype.Service;
/**
* author：liu-yu
* date: 2024/7/9 14:16
**/
@Service
public class AiTaskServiceFeiHe extends AbstractAiTaskService {
public AiTaskServiceFeiHe(RedissonUtil redissonUtil, AiApiResultDao aiApiResultDao) {
super(AiInterfaceEnums.FEI_HE, redissonUtil, aiApiResultDao);
}
@Override
public JSONObject httpParamByMetadata(AiInterfaceParam aiParam) {
JSONObject metadata = new JSONObject();
metadata.put("product", aiParam.getAiMark().getInterfaceRelation().getProduct());
metadata.put("platform", aiParam.getPt());
metadata.put("platform_category", aiParam.getPt());
return metadata;
}
}
middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/task/aiTask/AiTaskServiceHandler.java 0 → 100644
package com.zhiwei.middleware.automatic.son.task.aiTask;
import com.zhiwei.base.entity.subclass.mark.MarkInfo;
import com.zhiwei.middleware.automatic.server.pojo.AiInterfaceParam;
import com.zhiwei.middleware.automatic.son.task.holder.ApplicationContextHolder;
import java.util.HashMap;
import java.util.Map;
/**
* author：liu-yu
* date: 2024/7/9 14:20
**/
public class AiTaskServiceHandler {
private static final Map<String, AiTaskService> SERVICE_MAP = new HashMap<>();
private AiTaskServiceHandler() {
Map<String, AiTaskService> beansOfType = ApplicationContextHolder.getInstance().getBeansOfType(AiTaskService.class);
beansOfType.forEach((k, v) -> SERVICE_MAP.put(v.getType().getProjectId(), v));
}
public static AiTaskServiceHandler getInstance() {
return AiTaskServiceHandlerHolder.HANDLER;
}
public MarkInfo sendAIRequest(AiInterfaceParam aiParam, String timeKey) {
return SERVICE_MAP.get(aiParam.getAiMark().getProjectId()).executor(aiParam, timeKey);
}
private static class AiTaskServiceHandlerHolder {
private static final AiTaskServiceHandler HANDLER = new AiTaskServiceHandler();
}
}
middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/task/aiTask/AiTaskServiceLXJ.java 0 → 100644
package com.zhiwei.middleware.automatic.son.task.aiTask;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.middleware.automatic.server.pojo.AiInterfaceParam;
import com.zhiwei.middleware.automatic.server.pojo.enums.AiInterfaceEnums;
import com.zhiwei.middleware.automatic.server.util.RedissonUtil;
import com.zhiwei.middleware.automatic.son.dao.AiApiResultDao;
import org.springframework.stereotype.Service;
/**
* author：liu-yu
* date: 2024/7/9 14:19
**/
@Service
public class AiTaskServiceLXJ extends AbstractAiTaskService {
public AiTaskServiceLXJ(RedissonUtil redissonUtil, AiApiResultDao aiApiResultDao) {
super(AiInterfaceEnums.LAO_XIANG_JI, redissonUtil, aiApiResultDao);
}
@Override
public JSONObject httpParamByMetadata(AiInterfaceParam aiParam) {
return null;
}
}
middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/task/service/TaskServiceAiMark.java
......@@ -3,27 +3,18 @@ package com.zhiwei.middleware.automatic.son.task.service;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.base.entity.subclass.mark.MarkInfo;
import com.zhiwei.middleware.automatic.server.common.GenericAttribute;
import com.zhiwei.middleware.automatic.server.pojo.AIMark;
import com.zhiwei.middleware.automatic.server.pojo.AiApiResult;
import com.zhiwei.middleware.automatic.server.pojo.AiInterfaceParam;
import com.zhiwei.middleware.automatic.server.pojo.AutoTask;
import com.zhiwei.middleware.automatic.server.pojo.enums.TaskType;
import com.zhiwei.middleware.automatic.server.util.RedissonUtil;
import com.zhiwei.middleware.automatic.son.dao.AiApiResultDao;
import com.zhiwei.middleware.automatic.son.dubbo.DubboHandler;
import com.zhiwei.middleware.automatic.son.util.MarkInfoUtil;
import com.zhiwei.middleware.automatic.son.util.OkHttpUtil;
import com.zhiwei.middleware.automatic.son.util.Tools;
import org.apache.commons.lang3.StringUtils;
import com.zhiwei.middleware.automatic.son.task.aiTask.AiTaskServiceHandler;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.stereotype.Service;
import java.time.Instant;
import java.util.*;
import java.util.stream.Collectors;
......@@ -40,18 +31,16 @@ public class TaskServiceAiMark implements TaskService {
private final RedissonUtil redissonUtil;
private final AiApiResultDao aiApiResultDao;
private final DubboHandler dubboHandler;
private final TaskType AI_MARK = TaskType.AI_MARK;
public TaskServiceAiMark(@Qualifier("aiMarkExecutor") ThreadPoolTaskExecutor markAiExecutor,
RedissonUtil redissonUtil, AiApiResultDao aiApiResultDao,
RedissonUtil redissonUtil,
DubboHandler dubboHandler) {
this.markAiExecutor = markAiExecutor;
this.redissonUtil = redissonUtil;
this.aiApiResultDao = aiApiResultDao;
this.dubboHandler = dubboHandler;
}
......@@ -79,7 +68,7 @@ public class TaskServiceAiMark implements TaskService {
List<MarkInfo> list = new ArrayList<>();
for (AiInterfaceParam param : data) {
try {
MarkInfo markInfo = aiInterface(param, timeKey);
MarkInfo markInfo = AiTaskServiceHandler.getInstance().sendAIRequest(param, timeKey);
if (Objects.nonNull(markInfo)) {
list.add(markInfo);
}
......@@ -89,125 +78,6 @@ public class TaskServiceAiMark implements TaskService {
}
dubboHandler.markUpsert(list);
}
private MarkInfo aiInterface(AiInterfaceParam aiParam, String timeKey) {
JSONObject param = aiInterfaceParamBuild(aiParam);
long now = System.currentTimeMillis();
String s = OkHttpUtil.httpPut(aiParam.getAiMark().getInterfaceRelation().getUrl(), param.toJSONString(), aiParam.getAiMark().getInterfaceRelation().getHeader().getInnerMap());
redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, GenericAttribute.KEY_TOTAL), 1);
if (Objects.isNull(s)) {
redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, GenericAttribute.KEY_RES_ERROR), 1);
log.error("ai标注失败，数据id：{}，耗时：{}，错误信息：ai接口返回为null" , aiParam.getJson().getString("id"), System.currentTimeMillis() - now);
return null;
}
long endTime = System.currentTimeMillis() - now;
AiApiResult aiRes = aiApiResultConversion(s);
if (Objects.isNull(aiRes)) {
redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, GenericAttribute.KEY_RES_INCOMPLETE), 1);
log.info("数据url:{}，耗时：{}，ai接口返回信息不完整：{}", param.getJSONObject("data").getString(GenericAttribute.URL), endTime, s);
return null;
}
log.info("数据url：{}，ai标签：{}，耗时：{}，具体数据:{}", param.getJSONObject("data").getString(GenericAttribute.URL), aiRes.getSent(), endTime, aiRes.getResult());
List<AIMark.TagRelation> tagRelations = aiParam.getAiMark().getTagRelations();
Map<String, List<AIMark.TagRelation>> relateGroup = tagRelations.stream().collect(Collectors.groupingBy(AIMark.TagRelation::getRelateKey, Collectors.toList()));
List<AIMark.TagRelation> tagRelationList = relateGroup.values().stream().map(relations -> relations.stream().filter(x -> relateValueVerify(x, aiRes)).findFirst().orElse(null)).filter(Objects::nonNull).collect(Collectors.toList());
if (tagRelationList.isEmpty()) {
redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, GenericAttribute.KEY_NOT_MARK_RELATION), 1);
log.info("数据url：{}，ai标签：{}，没有绑定关系，以过滤", param.getJSONObject("data").getString(GenericAttribute.URL), aiRes.getSent());
return null;
}
JSONObject json = aiParam.getJson();
String mtag = tagRelationList.stream().map(AIMark.TagRelation::getUniqueId).collect(Collectors.joining());
json.put(GenericAttribute.ES_M_TAG, mtag);
json.put(GenericAttribute.ES_M_PERSON, GenericAttribute.AI_PERSON);
json.put(GenericAttribute.ES_M_TIME, System.currentTimeMillis());
json.put(GenericAttribute.ES_M_GROUP, aiParam.getAiMark().getProject());
json.put(GenericAttribute.ES_M_GROUP_ID, aiParam.getAiMark().getProjectId());
MarkInfo markInfo = MarkInfoUtil.transformToMarkInfo(json);
redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, GenericAttribute.KEY_SUCCESS), 1);
log.info("数据url：{}，ai标签：{}，标注标签：{}，成功标注", param.getJSONObject("data").getString(GenericAttribute.URL), aiRes.getSent(), mtag);
return markInfo;
}
private boolean relateValueVerify(AIMark.TagRelation tagRelation, AiApiResult result) {
String relateValue;
switch (tagRelation.getRelateKey()) {
case "sent" :
relateValue = result.getSent();
break;
case "info_class":
relateValue = result.getInfo_class();
break;
default:relateValue = "";
break;
}
if (StringUtils.isEmpty(relateValue)) {
return false;
}
return relateValue.equals(tagRelation.getRelateValue());
}
private AiApiResult aiApiResultConversion(String s) {
JSONObject res = JSONObject.parseObject(s);
if (Objects.nonNull(res)) {
JSONObject feiHeiRes = res.getJSONObject("feihe-compet-tagger");
if (Objects.nonNull(feiHeiRes)) {
JSONObject result = feiHeiRes.getJSONObject("result");
if (Objects.nonNull(result)) {
aiApiResultTitleSubString(result);
if (feiHeiRes.getString("status").equals("DONE")) {
AiApiResult aiRes = new AiApiResult();
aiRes.setId(feiHeiRes.getString("result_id"));
aiRes.setCreateAt(System.currentTimeMillis());
aiRes.setInfo_class(result.getString("info_class"));
aiRes.setNoise(result.getString("denoise"));
aiRes.setSent(result.getString("sent"));
aiRes.setStatus(feiHeiRes.getString("status"));
aiRes.setResult(res.toJSONString());
aiApiResultDao.insert(aiRes);
return aiRes;
} else {
return aiApiResultDao.findOne(new Query(Criteria.where("_id").is(result.getString("dedup"))));
}
}
}
}
return null;
}
private void aiApiResultTitleSubString(JSONObject json) {
String title = json.getString(GenericAttribute.ES_TITLE);
if (Objects.nonNull(title) && title.length() > 100) {
json.put(GenericAttribute.ES_TITLE, title.substring(0, 100));
}
String snippet = json.getString("snippet");
if (Objects.nonNull(snippet) && snippet.length() > 100) {
json.put("snippet", snippet.substring(0, 100));
}
}
private JSONObject aiInterfaceParamBuild(AiInterfaceParam aiParam) {
JSONObject param = new JSONObject();
param.put("wait_for", new String[]{"feihe-compet-tagger"});
JSONObject data = new JSONObject();
JSONObject json = aiParam.getJson();
JSONObject metadata = new JSONObject();
metadata.put("product", aiParam.getAiMark().getInterfaceRelation().getProduct());
metadata.put("platform", aiParam.getPt());
metadata.put("platform_category", aiParam.getPt());
data.put("metadata", metadata);
data.put("author", json.getString(GenericAttribute.ES_SOURCE));
data.put("pub_date", Instant.ofEpochMilli(json.getLong(GenericAttribute.ES_TIME)).toString());
List<String> indFullText = json.getJSONArray(GenericAttribute.ES_IND_FULL_TEXT).toJavaList(String.class);
data.put(GenericAttribute.ES_TITLE, indFullText.get(0));
data.put("snippet", indFullText.get(1));
String answerUrl = json.getString(GenericAttribute.ES_URL_ANSWER);
String questionUrl = json.getString(GenericAttribute.ES_URL_QUESTION);
String url = Objects.nonNull(answerUrl) ? answerUrl : Objects.nonNull(questionUrl) ? questionUrl : json.getString(GenericAttribute.URL);
data.put(GenericAttribute.URL, url);
param.put("data", data);
return param;
}
}
middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/util/OkHttpUtil.java
......@@ -18,9 +18,17 @@ public class OkHttpUtil {
public static String httpPut(String url, String info, Map<String, Object> header) {
Request.Builder builder = new Request.Builder().url(url).put(RequestBody.create(MediaType.parse("application/json; charset=utf-8"), info));
for (Map.Entry<String, Object> entry : header.entrySet()) {
builder.addHeader(entry.getKey(), String.valueOf(entry.getValue()));
builderHeader(builder, header);
try (Response response = okHttpClient.newCall(builder.build()).execute()) {
return response.body().string();
} catch (Exception e) {
return null;
}
}
public static String httpPost(String url, String info, Map<String, Object> header) {
Request.Builder builder = new Request.Builder().url(url).post(RequestBody.create(MediaType.parse("application/json; charset=utf-8"), info));
builderHeader(builder, header);
try (Response response = okHttpClient.newCall(builder.build()).execute()) {
return response.body().string();
} catch (Exception e) {
......@@ -28,4 +36,10 @@ public class OkHttpUtil {
}
}
private static void builderHeader(Request.Builder builder, Map<String, Object> header) {
for (Map.Entry<String, Object> entry : header.entrySet()) {
builder.addHeader(entry.getKey(), String.valueOf(entry.getValue()));
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment