Commit c9ca5670 by liuyu

2024/07/19 多项目ai标注优化

parent 6956c938
......@@ -31,6 +31,9 @@ public class AIMark implements Serializable {
* 标签对应关系
*/
private List<TagRelation> tagRelations;
private List<String> fields;
/**
* 项目名
*/
......@@ -64,6 +67,14 @@ public class AIMark implements Serializable {
this.interfaceRelation = interfaceRelation;
}
public List<String> getFields() {
return fields;
}
public void setFields(List<String> fields) {
this.fields = fields;
}
public List<TagRelation> getTagRelations() {
return tagRelations;
}
......
package com.zhiwei.middleware.automatic.server.pojo;
import java.util.List;
import java.util.Objects;
/**
......@@ -24,6 +25,8 @@ public class AiApiResult {
private String sensitive;
private List<AiTagRelation> tagList;
private String project;
private String result;
......@@ -56,6 +59,14 @@ public class AiApiResult {
return builder.toString();
}
public List<AiTagRelation> getTagList() {
return tagList;
}
public void setTagList(List<AiTagRelation> tagList) {
this.tagList = tagList;
}
public String getSensitive() {
return sensitive;
}
......
package com.zhiwei.middleware.automatic.server.pojo;
/**
* author:liu-yu
* date: 2024/7/18 14:02
**/
public class AiMarkProjectCache {
private String id;
private String project;
private String projectId;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getProject() {
return project;
}
public void setProject(String project) {
this.project = project;
}
public String getProjectId() {
return projectId;
}
public void setProjectId(String projectId) {
this.projectId = projectId;
}
}
package com.zhiwei.middleware.automatic.server.pojo;
/**
* author:liu-yu
* date: 2024/7/15 14:56
**/
public class AiTagRelation {
private String name;
private String value;
public AiTagRelation() {}
public AiTagRelation(String name, String value) {
this.name = name;
this.value = value;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getValue() {
return value;
}
public void setValue(String value) {
this.value = value;
}
}
......@@ -8,11 +8,7 @@ public enum AiInterfaceEnums {
FEI_HE("飞鹤", "62baa19f4d9a553ac3e7cd9b"),
LAO_XIANG_JI("老乡鸡", "665d8791b36dd11c5e453955"),
JIA_NAN_KE_JI("嘉楠科技", "65efecab7d8b3a478260e7b6"),
CHANG_XIN_CUN_CHU("长鑫存储", "65b89d4515e6c42baff23e74");
COMMON("常用", "常用");
final String project;
......@@ -30,13 +26,4 @@ public enum AiInterfaceEnums {
public String getProjectId() {
return projectId;
}
public static AiInterfaceEnums create(String projectId) {
for (AiInterfaceEnums enums : AiInterfaceEnums.values()) {
if (enums.getProjectId().equals(projectId)) {
return enums;
}
}
throw new IllegalStateException("该项目不支持ai标注:" + projectId);
}
}
package com.zhiwei.middleware.automatic.server.dao;
import com.zhiwei.middleware.automatic.server.pojo.AiMarkProjectCache;
import java.util.List;
/**
* author:liu-yu
* date: 2024/7/18 14:04
**/
public interface AiMarkProjectCacheDao {
List<AiMarkProjectCache> findAll();
}
package com.zhiwei.middleware.automatic.server.dao.impl;
import com.zhiwei.middleware.automatic.server.dao.AiMarkProjectCacheDao;
import com.zhiwei.middleware.automatic.server.pojo.AiMarkProjectCache;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.stereotype.Component;
import java.util.List;
/**
* author:liu-yu
* date: 2024/7/18 14:04
**/
@Component
public class AiMarkProjectCacheDaoImpl implements AiMarkProjectCacheDao {
private final MongoTemplate mongoTemplate;
private static final String COLLECTION_NAME = "marker_ai_mark_project";
public AiMarkProjectCacheDaoImpl(@Qualifier("markerMongoTemplate") MongoTemplate mongoTemplate) {
this.mongoTemplate = mongoTemplate;
}
@Override
public List<AiMarkProjectCache> findAll() {
return mongoTemplate.findAll(AiMarkProjectCache.class, COLLECTION_NAME);
}
}
......@@ -2,8 +2,9 @@ package com.zhiwei.middleware.automatic.server.mission;
import com.zhiwei.middleware.automatic.server.common.GenericAttribute;
import com.zhiwei.middleware.automatic.server.dao.AiMarkCountDao;
import com.zhiwei.middleware.automatic.server.dao.AiMarkProjectCacheDao;
import com.zhiwei.middleware.automatic.server.pojo.AiMarkCount;
import com.zhiwei.middleware.automatic.server.pojo.enums.AiInterfaceEnums;
import com.zhiwei.middleware.automatic.server.pojo.AiMarkProjectCache;
import com.zhiwei.middleware.automatic.server.util.RedissonUtil;
import com.zhiwei.middleware.automatic.server.util.TimeUtil;
import com.zhiwei.middleware.automatic.server.util.Tools;
......@@ -25,6 +26,8 @@ public class AsyncTask {
private final RedissonUtil redissonUtil;
private final AiMarkProjectCacheDao aiMarkProjectCacheDao;
public static Long DAY_START_TIME;
public static Long DAY_END_TIME;
......@@ -32,10 +35,12 @@ public class AsyncTask {
public static String CURRENT_TIME;
public AsyncTask(@Qualifier("hangzhouMongoTemplate") MongoTemplate hangZhouMongo,
AiMarkCountDao aiMarkCountDao, RedissonUtil redissonUtil) {
AiMarkCountDao aiMarkCountDao, RedissonUtil redissonUtil,
AiMarkProjectCacheDao aiMarkProjectCacheDao) {
this.hangZhouMongo = hangZhouMongo;
this.aiMarkCountDao = aiMarkCountDao;
this.redissonUtil = redissonUtil;
this.aiMarkProjectCacheDao = aiMarkProjectCacheDao;
}
public List<String> findAllGroup() {
......@@ -60,7 +65,7 @@ public class AsyncTask {
public void syncDayCount() {
String timeKey = TimeUtil.COMMON_TIME_FORMAT.format(System.currentTimeMillis() - GenericAttribute.DAY_TIME);
for (AiInterfaceEnums enums : AiInterfaceEnums.values()) {
for (AiMarkProjectCache enums : aiMarkProjectCacheDao.findAll()) {
AiMarkCount aiMarkCount = new AiMarkCount();
aiMarkCount.setTime(timeKey);
aiMarkCount.setTotal(redissonUtil.getCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, enums.getProject(), GenericAttribute.KEY_TOTAL)));
......
......@@ -6,6 +6,7 @@ import com.zhiwei.middleware.automatic.server.common.GenericAttribute;
import com.zhiwei.middleware.automatic.server.pojo.AIMark;
import com.zhiwei.middleware.automatic.server.pojo.AiApiResult;
import com.zhiwei.middleware.automatic.server.pojo.AiInterfaceParam;
import com.zhiwei.middleware.automatic.server.pojo.AiTagRelation;
import com.zhiwei.middleware.automatic.server.pojo.enums.AiInterfaceEnums;
import com.zhiwei.middleware.automatic.server.util.RedissonUtil;
import com.zhiwei.middleware.automatic.son.dao.AiApiResultDao;
......@@ -19,9 +20,7 @@ import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.*;
import java.util.stream.Collectors;
/**
......@@ -50,27 +49,29 @@ public abstract class AbstractAiTaskService implements AiTaskService {
long now = System.currentTimeMillis();
AIMark.InterfaceRelation interfaceRelation = aiParam.getAiMark().getInterfaceRelation();
String s = httpResult(interfaceRelation.getMethod(), interfaceRelation.getUrl(), param.toString(), interfaceRelation.getHeader().getInnerMap());
requestMarkTotalCount(timeKey);
String project = aiParam.getAiMark().getProject();
requestMarkTotalCount(timeKey, project);
if (StringUtils.isEmpty(s)) {
requestErrorCount(timeKey);
log.error("ai标注失败,项目:{},数据id:{},耗时:{},错误信息:ai接口返回为null" , aiInterfaceEnums.getProject(),
requestErrorCount(timeKey, project);
log.error("ai标注失败,项目:{},数据id:{},耗时:{},错误信息:ai接口返回为null" , project,
aiParam.getJson().getString("id"), System.currentTimeMillis() - now);
return null;
}
long endTime = System.currentTimeMillis() - now;
AiApiResult aiRes = aiApiResultConversion(s, aiParam);
if (Objects.isNull(aiRes)) {
requestResIncomplete(timeKey);
log.info("项目:{},数据url:{},耗时:{},ai接口返回信息不完整:{}", aiInterfaceEnums.getProject(), param.getJSONObject("data").getString(GenericAttribute.URL), endTime, s);
requestResIncomplete(timeKey, project);
log.info("项目:{},数据url:{},耗时:{},ai接口返回信息不完整:{}", project, param.getJSONObject("data").getString(GenericAttribute.URL), endTime, s);
return null;
}
log.info("项目:{},数据url:{},ai标签:{},耗时:{},具体数据:{}", aiInterfaceEnums.getProject(), param.getJSONObject("data").getString(GenericAttribute.URL), aiRes.toString(), endTime, aiRes.getResult());
log.info("项目:{},数据url:{},ai标签:{},耗时:{},具体数据:{}", project, param.getJSONObject("data").getString(GenericAttribute.URL), aiRes.toString(), endTime, aiRes.getResult());
List<AIMark.TagRelation> tagRelations = aiParam.getAiMark().getTagRelations();
Map<String, List<AIMark.TagRelation>> relateGroup = tagRelations.stream().collect(Collectors.groupingBy(AIMark.TagRelation::getRelateKey, Collectors.toList()));
List<AIMark.TagRelation> tagRelationList = relateGroup.values().stream().map(relations -> relations.stream().filter(x -> relateValueVerify(x, aiRes)).findFirst().orElse(null)).filter(Objects::nonNull).collect(Collectors.toList());
Map<String, AiTagRelation> aiTagGroup = aiRes.getTagList().stream().collect(Collectors.toMap(AiTagRelation::getName, e -> e));
List<AIMark.TagRelation> tagRelationList = relateGroup.values().stream().map(relations -> relations.stream().filter(x -> relateValueVerify(x, aiTagGroup)).findFirst().orElse(null)).filter(Objects::nonNull).collect(Collectors.toList());
if (tagRelationList.isEmpty()) {
markNotRelation(timeKey);
log.info("数据url:{},ai标签:{},没有绑定关系,以过滤", param.getJSONObject("data").getString(GenericAttribute.URL), aiRes.toString());
markNotRelation(timeKey, project);
log.info("项目:{},数据url:{},ai标签:{},没有绑定关系,以过滤", project, param.getJSONObject("data").getString(GenericAttribute.URL), aiRes.toString());
return null;
}
JSONObject json = aiParam.getJson();
......@@ -81,7 +82,7 @@ public abstract class AbstractAiTaskService implements AiTaskService {
json.put(GenericAttribute.ES_M_TIME, System.currentTimeMillis());
json.put(GenericAttribute.ES_M_GROUP, aiParam.getAiMark().getProject());
json.put(GenericAttribute.ES_M_GROUP_ID, aiParam.getAiMark().getProjectId());
log.info("数据url:{},ai标签:{},标注标签:{},成功标注", param.getJSONObject("data").getString(GenericAttribute.URL), aiRes.toString(), logTag);
log.info("项目:{},数据url:{},ai标签:{},标注标签:{},成功标注", project, param.getJSONObject("data").getString(GenericAttribute.URL), aiRes.toString(), logTag);
return json;
}
......@@ -114,7 +115,7 @@ public abstract class AbstractAiTaskService implements AiTaskService {
return null;
}
MarkInfo markInfo = MarkInfoUtil.transformToMarkInfo(jsonObject);
markSuccess(timeKey);
markSuccess(timeKey, aiParam.getAiMark().getProject());
return markInfo;
}
......@@ -160,8 +161,17 @@ public abstract class AbstractAiTaskService implements AiTaskService {
aiRes.setNegLabel(result.getString("neg_label"));
aiRes.setRelated(result.getString("related"));
aiRes.setSensitive(result.getString("sensitive"));
List<AiTagRelation> tagList = new ArrayList<>();
for (String field : aiParam.getAiMark().getFields()) {
String value = result.getString(field);
if (StringUtils.isEmpty(value)) {
continue;
}
tagList.add(new AiTagRelation(field, value));
}
aiRes.setTagList(tagList);
aiRes.setResult(res.toJSONString());
aiRes.setProject(aiInterfaceEnums.getProject());
aiRes.setProject(aiParam.getAiMark().getProject());
aiApiResultDao.insert(aiRes);
return aiRes;
} else {
......@@ -170,32 +180,33 @@ public abstract class AbstractAiTaskService implements AiTaskService {
}
@Override
public void requestMarkTotalCount(String timeKey) {
redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, aiInterfaceEnums.getProject(), GenericAttribute.KEY_TOTAL), 1);
public void requestMarkTotalCount(String timeKey, String project) {
redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, project, GenericAttribute.KEY_TOTAL), 1);
}
@Override
public void requestErrorCount(String timeKey) {
redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, aiInterfaceEnums.getProject(), GenericAttribute.KEY_RES_ERROR), 1);
public void requestErrorCount(String timeKey, String project) {
redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, project, GenericAttribute.KEY_RES_ERROR), 1);
}
@Override
public void requestResIncomplete(String timeKey) {
redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, aiInterfaceEnums.getProject(), GenericAttribute.KEY_RES_INCOMPLETE), 1);
public void requestResIncomplete(String timeKey, String project) {
redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, project, GenericAttribute.KEY_RES_INCOMPLETE), 1);
}
@Override
public void markNotRelation(String timeKey) {
redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, aiInterfaceEnums.getProject(), GenericAttribute.KEY_NOT_MARK_RELATION), 1);
public void markNotRelation(String timeKey, String project) {
redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, project, GenericAttribute.KEY_NOT_MARK_RELATION), 1);
}
@Override
public void markSuccess(String timeKey) {
redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, aiInterfaceEnums.getProject(), GenericAttribute.KEY_SUCCESS), 1);
public void markSuccess(String timeKey, String project) {
redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, project, GenericAttribute.KEY_SUCCESS), 1);
}
private boolean relateValueVerify(AIMark.TagRelation tagRelation, AiApiResult result) {
String relateValue = getResultValue(tagRelation.getRelateKey(), result);
private boolean relateValueVerify(AIMark.TagRelation tagRelation, Map<String, AiTagRelation> aiTagGroup) {
AiTagRelation aiTagRelationGroup = aiTagGroup.get(tagRelation.getRelateKey());
String relateValue = Objects.nonNull(aiTagRelationGroup) ? aiTagRelationGroup.getValue() : "";
if (StringUtils.isEmpty(relateValue)) {
return false;
}
......@@ -203,7 +214,10 @@ public abstract class AbstractAiTaskService implements AiTaskService {
return false;
}
if (Boolean.TRUE.equals(tagRelation.getHasRequirements())) {
return tagRelation.getRequirements().stream().allMatch(e -> e.getRequirementValue().equals(getResultValue(e.getRequirementKey(), result)));
return tagRelation.getRequirements().stream().allMatch(e -> {
AiTagRelation aiTagRelation = aiTagGroup.get(e.getRequirementKey());
return Objects.nonNull(aiTagRelation) && aiTagRelation.getValue().equals(e.getRequirementValue());
});
}
return true;
}
......
......@@ -41,13 +41,13 @@ public interface AiTaskService {
*/
AiApiResult aiApiResultConversion(String interfaceRes, AiInterfaceParam aiParam);
void requestMarkTotalCount(String timeKey);
void requestMarkTotalCount(String timeKey, String project);
void requestErrorCount(String timeKey);
void requestErrorCount(String timeKey, String project);
void requestResIncomplete(String timeKey);
void requestResIncomplete(String timeKey, String project);
void markNotRelation(String timeKey);
void markNotRelation(String timeKey, String project);
void markSuccess(String timeKey);
void markSuccess(String timeKey, String project);
}
package com.zhiwei.middleware.automatic.son.task.aiTask;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.middleware.automatic.server.pojo.AiInterfaceParam;
import com.zhiwei.middleware.automatic.server.pojo.enums.AiInterfaceEnums;
import com.zhiwei.middleware.automatic.server.util.RedissonUtil;
import com.zhiwei.middleware.automatic.son.dao.AiApiResultDao;
import org.springframework.stereotype.Service;
/**
* author:liu-yu
* date: 2024/7/15 11:04
**/
@Service
public class AiTaskServiceCJCC extends AbstractAiTaskService {
public AiTaskServiceCJCC(RedissonUtil redissonUtil, AiApiResultDao aiApiResultDao) {
super(AiInterfaceEnums.CHANG_XIN_CUN_CHU, redissonUtil, aiApiResultDao);
}
@Override
public JSONObject httpParamByMetadata(AiInterfaceParam aiParam) {
return null;
}
}
......@@ -12,9 +12,9 @@ import org.springframework.stereotype.Service;
* date: 2024/7/9 14:19
**/
@Service
public class AiTaskServiceLXJ extends AbstractAiTaskService {
public AiTaskServiceLXJ(RedissonUtil redissonUtil, AiApiResultDao aiApiResultDao) {
super(AiInterfaceEnums.LAO_XIANG_JI, redissonUtil, aiApiResultDao);
public class AiTaskServiceCommon extends AbstractAiTaskService {
public AiTaskServiceCommon(RedissonUtil redissonUtil, AiApiResultDao aiApiResultDao) {
super(AiInterfaceEnums.COMMON, redissonUtil, aiApiResultDao);
}
@Override
......
......@@ -2,10 +2,12 @@ package com.zhiwei.middleware.automatic.son.task.aiTask;
import com.zhiwei.base.entity.subclass.mark.MarkInfo;
import com.zhiwei.middleware.automatic.server.pojo.AiInterfaceParam;
import com.zhiwei.middleware.automatic.server.pojo.enums.AiInterfaceEnums;
import com.zhiwei.middleware.automatic.son.task.holder.ApplicationContextHolder;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
/**
* author:liu-yu
......@@ -24,7 +26,11 @@ public class AiTaskServiceHandler {
}
public MarkInfo sendAIRequest(AiInterfaceParam aiParam, String timeKey) {
return SERVICE_MAP.get(aiParam.getAiMark().getProjectId()).executor(aiParam, timeKey);
AiTaskService aiTaskService = SERVICE_MAP.get(aiParam.getAiMark().getProjectId());
if (Objects.isNull(aiTaskService)) {
aiTaskService = SERVICE_MAP.get(AiInterfaceEnums.COMMON.getProjectId());
}
return aiTaskService.executor(aiParam, timeKey);
}
private static class AiTaskServiceHandlerHolder {
......
package com.zhiwei.middleware.automatic.son.task.aiTask;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.middleware.automatic.server.pojo.AiInterfaceParam;
import com.zhiwei.middleware.automatic.server.pojo.enums.AiInterfaceEnums;
import com.zhiwei.middleware.automatic.server.util.RedissonUtil;
import com.zhiwei.middleware.automatic.son.dao.AiApiResultDao;
import org.springframework.stereotype.Service;
/**
* author:liu-yu
* date: 2024/7/10 16:43
**/
@Service
public class AiTaskServiceJNKJ extends AbstractAiTaskService {
public AiTaskServiceJNKJ(RedissonUtil redissonUtil, AiApiResultDao aiApiResultDao) {
super(AiInterfaceEnums.JIA_NAN_KE_JI, redissonUtil, aiApiResultDao);
}
@Override
public JSONObject httpParamByMetadata(AiInterfaceParam aiParam) {
return null;
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment