Merge branch 'feature' into 'release'

2024/07/10 老乡鸡ai标注接入、ai统计添加项目信息、标签命中添加前提条件。

See merge request !112

middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/pojo/AIMark.java

@@ -18,6 +18,11 @@ public class AIMark implements Serializable {
      * 分层id
      */
     private Integer monitorLevelId;
+
+    private List<Integer> monitorLevelIds;
+
+    private String type;
+
     /**
      * 接口信息
      */
@@ -83,6 +88,22 @@ public class AIMark implements Serializable {
         this.projectId = projectId;
     }
 
+    public List<Integer> getMonitorLevelIds() {
+        return monitorLevelIds;
+    }
+
+    public void setMonitorLevelIds(List<Integer> monitorLevelIds) {
+        this.monitorLevelIds = monitorLevelIds;
+    }
+
+    public String getType() {
+        return type;
+    }
+
+    public void setType(String type) {
+        this.type = type;
+    }
+
     public static class TagRelation implements Serializable {
         /**标签id*/
         private Long id;
@@ -99,6 +120,27 @@ public class AIMark implements Serializable {
         /**标签关联value*/
         private String relateValue;
 
+        /***是否包含前提条件*/
+        private Boolean hasRequirements;
+        /**前提条件*/
+        private List<Requirement> requirements;
+
+        public Boolean getHasRequirements() {
+            return hasRequirements;
+        }
+
+        public void setHasRequirements(Boolean hasRequirements) {
+            this.hasRequirements = hasRequirements;
+        }
+
+        public List<Requirement> getRequirements() {
+            return requirements;
+        }
+
+        public void setRequirements(List<Requirement> requirements) {
+            this.requirements = requirements;
+        }
+
         public Long getId() {
             return id;
         }
@@ -156,6 +198,28 @@ public class AIMark implements Serializable {
         }
     }
 
+    public static class Requirement implements Serializable {
+
+        private String requirementKey;
+        private String requirementValue;
+
+        public String getRequirementKey() {
+            return requirementKey;
+        }
+
+        public void setRequirementKey(String requirementKey) {
+            this.requirementKey = requirementKey;
+        }
+
+        public String getRequirementValue() {
+            return requirementValue;
+        }
+
+        public void setRequirementValue(String requirementValue) {
+            this.requirementValue = requirementValue;
+        }
+    }
+
     public static class InterfaceRelation implements Serializable {
         /** 接口地址*/
         private String url;

middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/pojo/AiApiResult.java

@@ -16,10 +16,30 @@ public class AiApiResult {
 
     private String noise;
 
+    private String negLabel;
+
+    private String project;
+
     private String result;
 
     private Long createAt;
 
+    public String getNegLabel() {
+        return negLabel;
+    }
+
+    public void setNegLabel(String negLabel) {
+        this.negLabel = negLabel;
+    }
+
+    public String getProject() {
+        return project;
+    }
+
+    public void setProject(String project) {
+        this.project = project;
+    }
+
     public String getId() {
         return id;
     }

middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/pojo/enums/AiInterfaceEnums.java

+package com.zhiwei.middleware.automatic.server.pojo.enums;
+
+/**
+ * author：liu-yu
+ * date: 2024/7/9 10:03
+ **/
+public enum AiInterfaceEnums {
+
+    FEI_HE("飞鹤", "62baa19f4d9a553ac3e7cd9b", "feihe-compet-tagger"),
+
+    LAO_XIANG_JI("老乡鸡", "665d8791b36dd11c5e453955", "laoxiangji-tagger");
+
+    final String project;
+
+    final String projectId;
+
+    final String waitFor;
+
+    AiInterfaceEnums(String project, String projectId, String waitFor) {
+        this.project = project;
+        this.projectId = projectId;
+        this.waitFor = waitFor;
+    }
+
+    public String getProject() {
+        return project;
+    }
+
+    public String getProjectId() {
+        return projectId;
+    }
+
+    public String getWaitFor() {
+        return waitFor;
+    }
+
+    public static AiInterfaceEnums create(String projectId) {
+        for (AiInterfaceEnums enums : AiInterfaceEnums.values()) {
+            if (enums.getProjectId().equals(projectId)) {
+                return enums;
+            }
+        }
+        throw new IllegalStateException("该项目不支持ai标注：" + projectId);
+    }
+}

middleware-automatic-center-server/src/main/java/com/zhiwei/middleware/automatic/server/mission/AsyncTask.java

@@ -3,6 +3,7 @@ package com.zhiwei.middleware.automatic.server.mission;
 import com.zhiwei.middleware.automatic.server.common.GenericAttribute;
 import com.zhiwei.middleware.automatic.server.dao.AiMarkCountDao;
 import com.zhiwei.middleware.automatic.server.pojo.AiMarkCount;
+import com.zhiwei.middleware.automatic.server.pojo.enums.AiInterfaceEnums;
 import com.zhiwei.middleware.automatic.server.util.RedissonUtil;
 import com.zhiwei.middleware.automatic.server.util.TimeUtil;
 import com.zhiwei.middleware.automatic.server.util.Tools;
@@ -59,13 +60,16 @@ public class AsyncTask {
 
     public void syncDayCount() {
         String timeKey = TimeUtil.COMMON_TIME_FORMAT.format(System.currentTimeMillis() - GenericAttribute.DAY_TIME);
-        AiMarkCount aiMarkCount = new AiMarkCount();
-        aiMarkCount.setTime(timeKey);
-        aiMarkCount.setTotal(redissonUtil.getCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, GenericAttribute.KEY_TOTAL)));
-        aiMarkCount.setResError(redissonUtil.getCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, GenericAttribute.KEY_RES_ERROR)));
-        aiMarkCount.setResIncomplete(redissonUtil.getCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, GenericAttribute.KEY_RES_INCOMPLETE)));
-        aiMarkCount.setNotMarkRelation(redissonUtil.getCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, GenericAttribute.KEY_NOT_MARK_RELATION)));
-        aiMarkCount.setSuccess(redissonUtil.getCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, GenericAttribute.KEY_SUCCESS)));
-        aiMarkCountDao.insert(aiMarkCount);
+        for (AiInterfaceEnums enums : AiInterfaceEnums.values()) {
+            AiMarkCount aiMarkCount = new AiMarkCount();
+            aiMarkCount.setTime(timeKey);
+            aiMarkCount.setTotal(redissonUtil.getCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, enums.getProject(), GenericAttribute.KEY_TOTAL)));
+            aiMarkCount.setResError(redissonUtil.getCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, enums.getProject(), GenericAttribute.KEY_RES_ERROR)));
+            aiMarkCount.setResIncomplete(redissonUtil.getCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, enums.getProject(), GenericAttribute.KEY_RES_INCOMPLETE)));
+            aiMarkCount.setNotMarkRelation(redissonUtil.getCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, enums.getProject(), GenericAttribute.KEY_NOT_MARK_RELATION)));
+            aiMarkCount.setSuccess(redissonUtil.getCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, enums.getProject(), GenericAttribute.KEY_SUCCESS)));
+            aiMarkCountDao.insert(aiMarkCount);
+        }
+
     }
 }

middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/task/aiTask/AbstractAiTaskService.java

+package com.zhiwei.middleware.automatic.son.task.aiTask;
+
+import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.base.entity.subclass.mark.MarkInfo;
+import com.zhiwei.middleware.automatic.server.common.GenericAttribute;
+import com.zhiwei.middleware.automatic.server.pojo.AIMark;
+import com.zhiwei.middleware.automatic.server.pojo.AiApiResult;
+import com.zhiwei.middleware.automatic.server.pojo.AiInterfaceParam;
+import com.zhiwei.middleware.automatic.server.pojo.enums.AiInterfaceEnums;
+import com.zhiwei.middleware.automatic.server.util.RedissonUtil;
+import com.zhiwei.middleware.automatic.son.dao.AiApiResultDao;
+import com.zhiwei.middleware.automatic.son.util.MarkInfoUtil;
+import com.zhiwei.middleware.automatic.son.util.OkHttpUtil;
+import com.zhiwei.middleware.automatic.son.util.Tools;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.data.mongodb.core.query.Query;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+/**
+ * author：liu-yu
+ * date: 2024/7/9 10:57
+ **/
+public abstract class AbstractAiTaskService implements AiTaskService {
+
+    private static final Logger log = LogManager.getLogger(AbstractAiTaskService.class);
+
+    private final AiInterfaceEnums aiInterfaceEnums;
+
+    private final RedissonUtil redissonUtil;
+
+    private final AiApiResultDao aiApiResultDao;
+
+    public AbstractAiTaskService(AiInterfaceEnums aiInterfaceEnums, RedissonUtil redissonUtil,
+                                 AiApiResultDao aiApiResultDao) {
+        this.aiInterfaceEnums = aiInterfaceEnums;
+        this.redissonUtil = redissonUtil;
+        this.aiApiResultDao = aiApiResultDao;
+    }
+
+    protected JSONObject aiInterface(AiInterfaceParam aiParam, String timeKey) {
+        JSONObject param = aiInterfaceParamBuild(aiParam);
+        long now = System.currentTimeMillis();
+        AIMark.InterfaceRelation interfaceRelation = aiParam.getAiMark().getInterfaceRelation();
+        String s = httpResult(interfaceRelation.getMethod(), interfaceRelation.getUrl(), param.toString(), interfaceRelation.getHeader().getInnerMap());
+        requestMarkTotalCount(timeKey);
+        if (StringUtils.isEmpty(s)) {
+            requestErrorCount(timeKey);
+            log.error("ai标注失败，项目：{}，数据id：{}，耗时：{}，错误信息：ai接口返回为null" , aiInterfaceEnums.getProject(),
+                    aiParam.getJson().getString("id"), System.currentTimeMillis() - now);
+            return null;
+        }
+        long endTime = System.currentTimeMillis() - now;
+        AiApiResult aiRes = aiApiResultConversion(s);
+        if (Objects.isNull(aiRes)) {
+            requestResIncomplete(timeKey);
+            log.info("项目：{}，数据url:{}，耗时：{}，ai接口返回信息不完整：{}", aiInterfaceEnums.getProject(), param.getJSONObject("data").getString(GenericAttribute.URL), endTime, s);
+            return null;
+        }
+        log.info("项目：{}，数据url：{}，ai标签：{}，耗时：{}，具体数据:{}", aiInterfaceEnums.getProject(), param.getJSONObject("data").getString(GenericAttribute.URL), aiRes.getSent(), endTime, aiRes.getResult());
+        List<AIMark.TagRelation> tagRelations = aiParam.getAiMark().getTagRelations();
+        Map<String, List<AIMark.TagRelation>> relateGroup = tagRelations.stream().collect(Collectors.groupingBy(AIMark.TagRelation::getRelateKey, Collectors.toList()));
+        List<AIMark.TagRelation> tagRelationList = relateGroup.values().stream().map(relations -> relations.stream().filter(x -> relateValueVerify(x, aiRes)).findFirst().orElse(null)).filter(Objects::nonNull).collect(Collectors.toList());
+        if (tagRelationList.isEmpty()) {
+            markNotRelation(timeKey);
+            log.info("数据url：{}，ai标签：{}，没有绑定关系，以过滤", param.getJSONObject("data").getString(GenericAttribute.URL), aiRes.getSent());
+            return null;
+        }
+        JSONObject json = aiParam.getJson();
+        String mtag = tagRelationList.stream().map(AIMark.TagRelation::getUniqueId).collect(Collectors.joining());
+        json.put(GenericAttribute.ES_M_TAG, mtag);
+        json.put(GenericAttribute.ES_M_PERSON, GenericAttribute.AI_PERSON);
+        json.put(GenericAttribute.ES_M_TIME, System.currentTimeMillis());
+        json.put(GenericAttribute.ES_M_GROUP, aiParam.getAiMark().getProject());
+        json.put(GenericAttribute.ES_M_GROUP_ID, aiParam.getAiMark().getProjectId());
+        log.info("数据url：{}，ai标签：{}，标注标签：{}，成功标注", param.getJSONObject("data").getString(GenericAttribute.URL), aiRes.getSent(), mtag);
+        return json;
+    }
+
+    protected JSONObject aiInterfaceParamBuild(AiInterfaceParam aiParam) {
+        JSONObject param = new JSONObject();
+        param.put("wait_for", new String[]{aiInterfaceEnums.getWaitFor()});
+        JSONObject data = new JSONObject();
+        JSONObject json = aiParam.getJson();
+        data.put("author", json.getString(GenericAttribute.ES_SOURCE));
+        data.put("pub_date", Instant.ofEpochMilli(json.getLong(GenericAttribute.ES_TIME)).toString());
+        List<String> indFullText = json.getJSONArray(GenericAttribute.ES_IND_FULL_TEXT).toJavaList(String.class);
+        data.put(GenericAttribute.ES_TITLE, indFullText.get(0));
+        data.put("snippet", indFullText.get(1));
+        String answerUrl = json.getString(GenericAttribute.ES_URL_ANSWER);
+        String questionUrl = json.getString(GenericAttribute.ES_URL_QUESTION);
+        String url = Objects.nonNull(answerUrl) ? answerUrl : Objects.nonNull(questionUrl) ? questionUrl : json.getString(GenericAttribute.URL);
+        data.put(GenericAttribute.URL, url);
+        JSONObject metadata = httpParamByMetadata(aiParam);
+        if (Objects.nonNull(metadata)) {
+            data.put("metadata", metadata);
+        }
+        param.put("data", data);
+        return param;
+    }
+
+    @Override
+    public MarkInfo executor(AiInterfaceParam aiParam, String timeKey) {
+        JSONObject jsonObject = aiInterface(aiParam, timeKey);
+        if (Objects.isNull(jsonObject)) {
+            return null;
+        }
+        MarkInfo markInfo = MarkInfoUtil.transformToMarkInfo(jsonObject);
+        markSuccess(timeKey);
+        return markInfo;
+    }
+
+    @Override
+    public AiInterfaceEnums getType() {
+        return aiInterfaceEnums;
+    }
+
+    @Override
+    public String httpResult(String httpType, String url, String paramInfo, Map<String, Object> header) {
+        switch (httpType) {
+            case "POST" :
+                return OkHttpUtil.httpPost(url, paramInfo, header);
+            case "PUT" :
+                return OkHttpUtil.httpPut(url, paramInfo, header);
+        }
+        return null;
+    }
+
+    @Override
+    public AiApiResult aiApiResultConversion(String interfaceRes) {
+        JSONObject res = JSONObject.parseObject(interfaceRes);
+        if (Objects.nonNull(res)) {
+            JSONObject waitRes = res.getJSONObject(aiInterfaceEnums.getWaitFor());
+            if (Objects.nonNull(waitRes)) {
+                JSONObject result = waitRes.getJSONObject("result");
+                if (Objects.nonNull(result)) {
+                    aiApiResultTitleSubString(result);
+                    if (waitRes.getString("status").equals("DONE") || (!StringUtils.isEmpty(result.getString("sent")) && StringUtils.isEmpty("dedup"))) {
+                        AiApiResult aiRes = new AiApiResult();
+                        aiRes.setId(waitRes.getString("result_id"));
+                        aiRes.setCreateAt(System.currentTimeMillis());
+                        aiRes.setInfo_class(result.getString("info_class"));
+                        aiRes.setNoise(result.getString("denoise"));
+                        aiRes.setSent(result.getString("sent"));
+                        aiRes.setStatus(waitRes.getString("status"));
+                        aiRes.setNegLabel(waitRes.getString("neg_label"));
+                        aiRes.setResult(res.toJSONString());
+                        aiRes.setProject(aiInterfaceEnums.getProject());
+                        aiApiResultDao.insert(aiRes);
+                        return aiRes;
+                    } else {
+                        return aiApiResultDao.findOne(new Query(Criteria.where("_id").is(result.getString("dedup"))));
+                    }
+                }
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public void requestMarkTotalCount(String timeKey) {
+        redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, aiInterfaceEnums.getProject(), GenericAttribute.KEY_TOTAL), 1);
+    }
+
+    @Override
+    public void requestErrorCount(String timeKey) {
+        redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, aiInterfaceEnums.getProject(), GenericAttribute.KEY_RES_ERROR), 1);
+    }
+
+    @Override
+    public void requestResIncomplete(String timeKey) {
+        redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, aiInterfaceEnums.getProject(), GenericAttribute.KEY_RES_INCOMPLETE), 1);
+    }
+
+    @Override
+    public void markNotRelation(String timeKey) {
+        redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, aiInterfaceEnums.getProject(), GenericAttribute.KEY_NOT_MARK_RELATION), 1);
+    }
+
+    @Override
+    public void markSuccess(String timeKey) {
+        redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, aiInterfaceEnums.getProject(), GenericAttribute.KEY_SUCCESS), 1);
+    }
+
+    private boolean relateValueVerify(AIMark.TagRelation tagRelation, AiApiResult result) {
+        String relateValue = getResultValue(tagRelation.getRelateKey(), result);
+        if (StringUtils.isEmpty(relateValue)) {
+            return false;
+        }
+        if (!relateValue.equals(tagRelation.getRelateValue())) {
+            return false;
+        }
+        if (Boolean.TRUE.equals(tagRelation.getHasRequirements())) {
+            return tagRelation.getRequirements().stream().allMatch(e -> e.getRequirementValue().equals(getResultValue(e.getRequirementKey(), result)));
+        }
+        return true;
+    }
+
+    private String getResultValue(String relateKey, AiApiResult result) {
+        String relateValue = null;
+        switch (relateKey) {
+            case "sent" :
+                relateValue = result.getSent();
+                break;
+            case "info_class":
+                relateValue = result.getInfo_class();
+                break;
+            case "" :
+                relateValue = result.getNegLabel();
+                break;
+        }
+        return relateValue;
+    }
+
+    private void aiApiResultTitleSubString(JSONObject json) {
+        String title = json.getString(GenericAttribute.ES_TITLE);
+        if (Objects.nonNull(title) && title.length() > 100) {
+            json.put(GenericAttribute.ES_TITLE, title.substring(0, 100));
+        }
+        String snippet = json.getString("snippet");
+        if (Objects.nonNull(snippet) && snippet.length() > 100) {
+            json.put("snippet", snippet.substring(0, 100));
+        }
+    }
+}

middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/task/aiTask/AiTaskService.java

+package com.zhiwei.middleware.automatic.son.task.aiTask;
+
+import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.base.entity.subclass.mark.MarkInfo;
+import com.zhiwei.middleware.automatic.server.pojo.AiApiResult;
+import com.zhiwei.middleware.automatic.server.pojo.AiInterfaceParam;
+import com.zhiwei.middleware.automatic.server.pojo.enums.AiInterfaceEnums;
+
+import java.util.Map;
+
+/**
+ * author：liu-yu
+ * date: 2024/7/9 10:46
+ **/
+public interface AiTaskService {
+
+    AiInterfaceEnums getType();
+
+    MarkInfo executor(AiInterfaceParam aiParam, String timeKey);
+
+    /**
+     * 构建请求参数 (metadata)
+     * @param aiParam aiParam
+     * @return 请求参数
+     */
+    JSONObject httpParamByMetadata(AiInterfaceParam aiParam);
+
+    /**
+     * 接口请求
+     * @param httpType 请求类型
+     * @param url url
+     * @param header header
+     * @return 接口返回值
+     */
+    String httpResult(String httpType, String url, String paramInfo, Map<String, Object> header);
+
+    /**
+     * 接口返回值转换
+     * @param interfaceRes 接口返回值
+     * @return AiApiResult
+     */
+    AiApiResult aiApiResultConversion(String interfaceRes);
+
+    void requestMarkTotalCount(String timeKey);
+
+    void requestErrorCount(String timeKey);
+
+    void requestResIncomplete(String timeKey);
+
+    void markNotRelation(String timeKey);
+
+    void markSuccess(String timeKey);
+}

middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/task/aiTask/AiTaskServiceFeiHe.java

+package com.zhiwei.middleware.automatic.son.task.aiTask;
+
+import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.middleware.automatic.server.pojo.AiInterfaceParam;
+import com.zhiwei.middleware.automatic.server.pojo.enums.AiInterfaceEnums;
+import com.zhiwei.middleware.automatic.server.util.RedissonUtil;
+import com.zhiwei.middleware.automatic.son.dao.AiApiResultDao;
+import org.springframework.stereotype.Service;
+
+/**
+ * author：liu-yu
+ * date: 2024/7/9 14:16
+ **/
+@Service
+public class AiTaskServiceFeiHe extends AbstractAiTaskService {
+    public AiTaskServiceFeiHe(RedissonUtil redissonUtil, AiApiResultDao aiApiResultDao) {
+        super(AiInterfaceEnums.FEI_HE, redissonUtil, aiApiResultDao);
+    }
+
+    @Override
+    public JSONObject httpParamByMetadata(AiInterfaceParam aiParam) {
+        JSONObject metadata = new JSONObject();
+        metadata.put("product", aiParam.getAiMark().getInterfaceRelation().getProduct());
+        metadata.put("platform", aiParam.getPt());
+        metadata.put("platform_category", aiParam.getPt());
+        return metadata;
+    }
+
+
+}

middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/task/aiTask/AiTaskServiceHandler.java

+package com.zhiwei.middleware.automatic.son.task.aiTask;
+
+import com.zhiwei.base.entity.subclass.mark.MarkInfo;
+import com.zhiwei.middleware.automatic.server.pojo.AiInterfaceParam;
+import com.zhiwei.middleware.automatic.son.task.holder.ApplicationContextHolder;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * author：liu-yu
+ * date: 2024/7/9 14:20
+ **/
+public class AiTaskServiceHandler {
+
+    private static final Map<String, AiTaskService> SERVICE_MAP = new HashMap<>();
+    private AiTaskServiceHandler() {
+        Map<String, AiTaskService> beansOfType = ApplicationContextHolder.getInstance().getBeansOfType(AiTaskService.class);
+        beansOfType.forEach((k, v) -> SERVICE_MAP.put(v.getType().getProjectId(), v));
+    }
+
+    public static AiTaskServiceHandler getInstance() {
+        return AiTaskServiceHandlerHolder.HANDLER;
+    }
+
+    public MarkInfo sendAIRequest(AiInterfaceParam aiParam, String timeKey) {
+        return SERVICE_MAP.get(aiParam.getAiMark().getProjectId()).executor(aiParam, timeKey);
+    }
+
+    private static class AiTaskServiceHandlerHolder {
+        private static final AiTaskServiceHandler HANDLER = new AiTaskServiceHandler();
+    }
+}

middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/task/aiTask/AiTaskServiceLXJ.java

+package com.zhiwei.middleware.automatic.son.task.aiTask;
+
+import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.middleware.automatic.server.pojo.AiInterfaceParam;
+import com.zhiwei.middleware.automatic.server.pojo.enums.AiInterfaceEnums;
+import com.zhiwei.middleware.automatic.server.util.RedissonUtil;
+import com.zhiwei.middleware.automatic.son.dao.AiApiResultDao;
+import org.springframework.stereotype.Service;
+
+/**
+ * author：liu-yu
+ * date: 2024/7/9 14:19
+ **/
+@Service
+public class AiTaskServiceLXJ extends AbstractAiTaskService {
+    public AiTaskServiceLXJ(RedissonUtil redissonUtil, AiApiResultDao aiApiResultDao) {
+        super(AiInterfaceEnums.LAO_XIANG_JI, redissonUtil, aiApiResultDao);
+    }
+
+    @Override
+    public JSONObject httpParamByMetadata(AiInterfaceParam aiParam) {
+        return null;
+    }
+}

middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/task/service/TaskServiceAiMark.java

@@ -3,27 +3,18 @@ package com.zhiwei.middleware.automatic.son.task.service;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.base.entity.subclass.mark.MarkInfo;
 import com.zhiwei.middleware.automatic.server.common.GenericAttribute;
-import com.zhiwei.middleware.automatic.server.pojo.AIMark;
-import com.zhiwei.middleware.automatic.server.pojo.AiApiResult;
 import com.zhiwei.middleware.automatic.server.pojo.AiInterfaceParam;
 import com.zhiwei.middleware.automatic.server.pojo.AutoTask;
 import com.zhiwei.middleware.automatic.server.pojo.enums.TaskType;
 import com.zhiwei.middleware.automatic.server.util.RedissonUtil;
-import com.zhiwei.middleware.automatic.son.dao.AiApiResultDao;
 import com.zhiwei.middleware.automatic.son.dubbo.DubboHandler;
-import com.zhiwei.middleware.automatic.son.util.MarkInfoUtil;
-import com.zhiwei.middleware.automatic.son.util.OkHttpUtil;
-import com.zhiwei.middleware.automatic.son.util.Tools;
-import org.apache.commons.lang3.StringUtils;
+import com.zhiwei.middleware.automatic.son.task.aiTask.AiTaskServiceHandler;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.data.mongodb.core.query.Criteria;
-import org.springframework.data.mongodb.core.query.Query;
 import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
 import org.springframework.stereotype.Service;
 
-import java.time.Instant;
 import java.util.*;
 import java.util.stream.Collectors;
 
@@ -40,18 +31,16 @@ public class TaskServiceAiMark implements TaskService {
     
     private final RedissonUtil redissonUtil;
 
-    private final AiApiResultDao aiApiResultDao;
 
     private final DubboHandler dubboHandler;
 
     private final TaskType AI_MARK = TaskType.AI_MARK;
 
     public TaskServiceAiMark(@Qualifier("aiMarkExecutor") ThreadPoolTaskExecutor markAiExecutor,
-                             RedissonUtil redissonUtil, AiApiResultDao aiApiResultDao,
+                             RedissonUtil redissonUtil,
                              DubboHandler dubboHandler) {
         this.markAiExecutor = markAiExecutor;
         this.redissonUtil = redissonUtil;
-        this.aiApiResultDao = aiApiResultDao;
         this.dubboHandler = dubboHandler;
     }
 
@@ -79,7 +68,7 @@ public class TaskServiceAiMark implements TaskService {
         List<MarkInfo> list = new ArrayList<>();
         for (AiInterfaceParam param : data) {
             try {
-                MarkInfo markInfo = aiInterface(param, timeKey);
+                MarkInfo markInfo = AiTaskServiceHandler.getInstance().sendAIRequest(param, timeKey);
                 if (Objects.nonNull(markInfo)) {
                     list.add(markInfo);
                 }
@@ -89,125 +78,6 @@ public class TaskServiceAiMark implements TaskService {
         }
         dubboHandler.markUpsert(list);
     }
-
-    private MarkInfo aiInterface(AiInterfaceParam aiParam, String timeKey) {
-        JSONObject param = aiInterfaceParamBuild(aiParam);
-        long now = System.currentTimeMillis();
-        String s = OkHttpUtil.httpPut(aiParam.getAiMark().getInterfaceRelation().getUrl(), param.toJSONString(), aiParam.getAiMark().getInterfaceRelation().getHeader().getInnerMap());
-        redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, GenericAttribute.KEY_TOTAL), 1);
-        if (Objects.isNull(s)) {
-            redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, GenericAttribute.KEY_RES_ERROR), 1);
-            log.error("ai标注失败，数据id：{}，耗时：{}，错误信息：ai接口返回为null" , aiParam.getJson().getString("id"), System.currentTimeMillis() - now);
-            return null;
-        }
-        long endTime = System.currentTimeMillis() - now;
-        AiApiResult aiRes = aiApiResultConversion(s);
-        if (Objects.isNull(aiRes)) {
-            redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, GenericAttribute.KEY_RES_INCOMPLETE), 1);
-            log.info("数据url:{}，耗时：{}，ai接口返回信息不完整：{}", param.getJSONObject("data").getString(GenericAttribute.URL), endTime, s);
-            return null;
-        }
-        log.info("数据url：{}，ai标签：{}，耗时：{}，具体数据:{}", param.getJSONObject("data").getString(GenericAttribute.URL), aiRes.getSent(), endTime, aiRes.getResult());
-        List<AIMark.TagRelation> tagRelations = aiParam.getAiMark().getTagRelations();
-        Map<String, List<AIMark.TagRelation>> relateGroup = tagRelations.stream().collect(Collectors.groupingBy(AIMark.TagRelation::getRelateKey, Collectors.toList()));
-        List<AIMark.TagRelation> tagRelationList = relateGroup.values().stream().map(relations -> relations.stream().filter(x -> relateValueVerify(x, aiRes)).findFirst().orElse(null)).filter(Objects::nonNull).collect(Collectors.toList());
-        if (tagRelationList.isEmpty()) {
-            redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, GenericAttribute.KEY_NOT_MARK_RELATION), 1);
-            log.info("数据url：{}，ai标签：{}，没有绑定关系，以过滤", param.getJSONObject("data").getString(GenericAttribute.URL), aiRes.getSent());
-            return null;
-        }
-        JSONObject json = aiParam.getJson();
-        String mtag = tagRelationList.stream().map(AIMark.TagRelation::getUniqueId).collect(Collectors.joining());
-        json.put(GenericAttribute.ES_M_TAG, mtag);
-        json.put(GenericAttribute.ES_M_PERSON, GenericAttribute.AI_PERSON);
-        json.put(GenericAttribute.ES_M_TIME, System.currentTimeMillis());
-        json.put(GenericAttribute.ES_M_GROUP, aiParam.getAiMark().getProject());
-        json.put(GenericAttribute.ES_M_GROUP_ID, aiParam.getAiMark().getProjectId());
-        MarkInfo markInfo = MarkInfoUtil.transformToMarkInfo(json);
-        redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, GenericAttribute.KEY_SUCCESS), 1);
-        log.info("数据url：{}，ai标签：{}，标注标签：{}，成功标注", param.getJSONObject("data").getString(GenericAttribute.URL), aiRes.getSent(), mtag);
-        return markInfo;
-    }
-
-    private boolean relateValueVerify(AIMark.TagRelation tagRelation, AiApiResult result) {
-        String relateValue;
-        switch (tagRelation.getRelateKey()) {
-            case "sent" :
-                relateValue = result.getSent();
-                break;
-            case "info_class":
-                relateValue = result.getInfo_class();
-                break;
-            default:relateValue = "";
-                break;
-        }
-        if (StringUtils.isEmpty(relateValue)) {
-            return false;
-        }
-        return relateValue.equals(tagRelation.getRelateValue());
-    }
-
-    private AiApiResult aiApiResultConversion(String s) {
-        JSONObject res = JSONObject.parseObject(s);
-        if (Objects.nonNull(res)) {
-            JSONObject feiHeiRes = res.getJSONObject("feihe-compet-tagger");
-            if (Objects.nonNull(feiHeiRes)) {
-                JSONObject result = feiHeiRes.getJSONObject("result");
-                if (Objects.nonNull(result)) {
-                    aiApiResultTitleSubString(result);
-                    if (feiHeiRes.getString("status").equals("DONE")) {
-                        AiApiResult aiRes = new AiApiResult();
-                        aiRes.setId(feiHeiRes.getString("result_id"));
-                        aiRes.setCreateAt(System.currentTimeMillis());
-                        aiRes.setInfo_class(result.getString("info_class"));
-                        aiRes.setNoise(result.getString("denoise"));
-                        aiRes.setSent(result.getString("sent"));
-                        aiRes.setStatus(feiHeiRes.getString("status"));
-                        aiRes.setResult(res.toJSONString());
-                        aiApiResultDao.insert(aiRes);
-                        return aiRes;
-                    } else {
-                        return aiApiResultDao.findOne(new Query(Criteria.where("_id").is(result.getString("dedup"))));
-                    }
-                }
-            }
-        }
-        return null;
-    }
-
-    private void aiApiResultTitleSubString(JSONObject json) {
-        String title = json.getString(GenericAttribute.ES_TITLE);
-        if (Objects.nonNull(title) && title.length() > 100) {
-            json.put(GenericAttribute.ES_TITLE, title.substring(0, 100));
-        }
-        String snippet = json.getString("snippet");
-        if (Objects.nonNull(snippet) && snippet.length() > 100) {
-            json.put("snippet", snippet.substring(0, 100));
-        }
-    }
-
-    private JSONObject aiInterfaceParamBuild(AiInterfaceParam aiParam) {
-        JSONObject param = new JSONObject();
-        param.put("wait_for", new String[]{"feihe-compet-tagger"});
-        JSONObject data = new JSONObject();
-        JSONObject json = aiParam.getJson();
-        JSONObject metadata = new JSONObject();
-        metadata.put("product", aiParam.getAiMark().getInterfaceRelation().getProduct());
-        metadata.put("platform", aiParam.getPt());
-        metadata.put("platform_category", aiParam.getPt());
-        data.put("metadata", metadata);
-        data.put("author", json.getString(GenericAttribute.ES_SOURCE));
-        data.put("pub_date", Instant.ofEpochMilli(json.getLong(GenericAttribute.ES_TIME)).toString());
-        List<String> indFullText = json.getJSONArray(GenericAttribute.ES_IND_FULL_TEXT).toJavaList(String.class);
-        data.put(GenericAttribute.ES_TITLE, indFullText.get(0));
-        data.put("snippet", indFullText.get(1));
-        String answerUrl = json.getString(GenericAttribute.ES_URL_ANSWER);
-        String questionUrl = json.getString(GenericAttribute.ES_URL_QUESTION);
-        String url = Objects.nonNull(answerUrl) ? answerUrl : Objects.nonNull(questionUrl) ? questionUrl : json.getString(GenericAttribute.URL);
-        data.put(GenericAttribute.URL, url);
-        param.put("data", data);
-        return param;
-    }
 }
 
 

middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/util/OkHttpUtil.java

@@ -18,9 +18,17 @@ public class OkHttpUtil {
 
     public static String httpPut(String url, String info, Map<String, Object> header) {
         Request.Builder builder = new Request.Builder().url(url).put(RequestBody.create(MediaType.parse("application/json; charset=utf-8"), info));
-        for (Map.Entry<String, Object> entry : header.entrySet()) {
-            builder.addHeader(entry.getKey(), String.valueOf(entry.getValue()));
+        builderHeader(builder, header);
+        try (Response response = okHttpClient.newCall(builder.build()).execute()) {
+            return response.body().string();
+        } catch (Exception e) {
+            return null;
         }
+    }
+
+    public static String httpPost(String url, String info, Map<String, Object> header) {
+        Request.Builder builder = new Request.Builder().url(url).post(RequestBody.create(MediaType.parse("application/json; charset=utf-8"), info));
+        builderHeader(builder, header);
         try (Response response = okHttpClient.newCall(builder.build()).execute()) {
             return response.body().string();
         } catch (Exception e) {
@@ -28,4 +36,10 @@ public class OkHttpUtil {
         }
     }
 
+    private static void builderHeader(Request.Builder builder, Map<String, Object> header) {
+        for (Map.Entry<String, Object> entry : header.entrySet()) {
+            builder.addHeader(entry.getKey(), String.valueOf(entry.getValue()));
+        }
+    }
+
 }