2024/06/13 ai标注接口服务

middleware-automatic-center-client-autoconfigure/pom.xml

@@ -11,13 +11,13 @@
     </parent>
 
     <artifactId>middleware-automatic-center-autoconfigure</artifactId>
-    <version>1.0.0.3-SNAPSHOT</version>
+    <version>1.0.0.4-SNAPSHOT</version>
     <packaging>jar</packaging>
 
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <java.version>1.8</java.version>
-        <automatic.version>1.0.0.3-SNAPSHOT</automatic.version>
+        <automatic.version>1.0.0.4-SNAPSHOT</automatic.version>
     </properties>
 
     <dependencies>

middleware-automatic-center-client-autoconfigure/src/main/java/com/zhiwei/middleware/automatic/configuration/AutoMaticClientConfiguration.java

@@ -2,7 +2,6 @@ package com.zhiwei.middleware.automatic.configuration;
 
 
 import com.zhiwei.middleware.automatic.server.core.*;
-import com.zhiwei.middleware.automatic.server.dubbo.service.AutoMaticService;
 import org.springframework.boot.autoconfigure.AutoConfigureAfter;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
@@ -23,4 +22,10 @@ public class AutoMaticClientConfiguration {
                 properties.getConsumer(), properties.getConfigCenterConfig());
     }
 
+    @Bean
+    @ConditionalOnMissingBean(AiMarkMaticClient.class)
+    public AiMarkMaticClient aiMarkClient(AutoMaticClientConfigurationProperties properties) {
+        return AutoMaticClientFactory.getAiMarkMaticClient(properties.getApplication(), properties.getRegistry(), properties.getConsumer(), properties.getConfigCenterConfig());
+    }
+
 }

middleware-automatic-center-client/pom.xml

@@ -11,7 +11,7 @@
     </parent>
 
     <artifactId>middleware-automatic-center-client</artifactId>
-    <version>1.0.0.3-SNAPSHOT</version>
+    <version>1.0.0.4-SNAPSHOT</version>
 
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/common/GenericAttribute.java

@@ -4,6 +4,8 @@ public class GenericAttribute {
 
     public static final boolean IS_TEST = false;
 
+    public static final String REDIS_MARK_AI_KEY = "aiMarkKey";
+
     public static final String REDIS_QUEUE_ONE_KEY = "autoDataOneQueue";
 
     public static final String REDIS_QUEUE_MULTI_KEY = "autoDataMultiQueue";
@@ -23,6 +25,8 @@ public class GenericAttribute {
 
     public static final String AUTO_PERSON = "自动化机器人";
 
+    public static final String AI_PERSON = "ai机器人";
+
     public static final long AUTO_CID = 100040002;
 
     public static final String ES_M_TIME = "mtime";
@@ -56,4 +60,17 @@ public class GenericAttribute {
     public static final Long DAY_TIME = 1000L * 60 * 60 * 24;
 
     public static final Long WEEK_TIME = DAY_TIME * 7;
+
+    public static final String ES_SOURCE = "source";
+
+    public static final String ES_TIME = "time";
+    
+    public static final String ES_IND_FULL_TEXT = "ind_full_text";
+
+    public static final String ES_URL_QUESTION = "question_url";
+    public static final String ES_URL_ANSWER = "answer_url";
+
+    public static final String ES_M_GROUP = "mgroup";
+
+    public static final String ES_M_GROUP_ID = "mgroup_id";
 }

middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/core/AiMarkMaticClient.java

+package com.zhiwei.middleware.automatic.server.core;
+
+import com.zhiwei.middleware.automatic.server.dubbo.service.AiMarkMaticService;
+import com.zhiwei.middleware.automatic.server.pojo.AiInterfaceParam;
+
+import java.util.List;
+
+/**
+ * author：liu-yu
+ * date: 2024/6/12 14:10
+ **/
+public class AiMarkMaticClient {
+
+    private final AiMarkMaticService service;
+
+    public AiMarkMaticClient(AiMarkMaticService service) {
+        this.service = service;
+    }
+
+    public void autoAiMark(AiInterfaceParam param) {
+        service.autoAiMark(param);
+    }
+
+    public void autoAiMark(List<AiInterfaceParam> paramList) {
+        service.autoAiMark(paramList);
+    }
+}

middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/core/AutoMaticClientFactory.java

 package com.zhiwei.middleware.automatic.server.core;
 
+import com.zhiwei.middleware.automatic.server.dubbo.service.AiMarkMaticService;
 import com.zhiwei.middleware.automatic.server.dubbo.service.AutoMaticService;
 import org.apache.dubbo.common.utils.StringUtils;
 import org.apache.dubbo.config.*;
@@ -90,6 +91,16 @@ public class AutoMaticClientFactory {
         return new AutoMaticClient(createInstance(AutoMaticService.class, registry, group, appName, null));
     }
 
+    public static AiMarkMaticClient getAiMarkMaticClient(String registry, String group, String appName) {
+        return new AiMarkMaticClient(createInstance(AiMarkMaticService.class, registry, group, appName, null));
+    }
+
+    public static AiMarkMaticClient getAiMarkMaticClient(String registry, String group, String appName, long timeout) {
+        ConfigCenterConfig configCenterConfig = new ConfigCenterConfig();
+        configCenterConfig.setTimeout(timeout);
+        return new AiMarkMaticClient(createInstance(AiMarkMaticService.class, registry, group, appName, configCenterConfig));
+    }
+
     /**
      * 获取自动标注client
      * @return 自动标注client
@@ -98,4 +109,9 @@ public class AutoMaticClientFactory {
                                                      ConsumerConfig consumer, ConfigCenterConfig configCenterConfig) {
         return new AutoMaticClient(createInstance(AutoMaticService.class, application, registry, consumer, configCenterConfig));
     }
+
+    public static AiMarkMaticClient getAiMarkMaticClient(ApplicationConfig application, RegistryConfig registry,
+                                                    ConsumerConfig consumer, ConfigCenterConfig configCenterConfig)  {
+        return new AiMarkMaticClient(createInstance(AiMarkMaticService.class, application, registry, consumer, configCenterConfig));
+    }
 }

middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/dubbo/service/AiMarkMaticService.java

+package com.zhiwei.middleware.automatic.server.dubbo.service;
+
+import com.zhiwei.middleware.automatic.server.pojo.AiInterfaceParam;
+
+import java.util.List;
+
+/**
+ * author：liu-yu
+ * date: 2024/6/12 13:53
+ **/
+public interface AiMarkMaticService {
+
+    void autoAiMark(AiInterfaceParam param);
+
+    void autoAiMark(List<AiInterfaceParam> paramList);
+}

middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/pojo/AIMark.java

+package com.zhiwei.middleware.automatic.server.pojo;
+
+import com.alibaba.fastjson.JSONObject;
+
+import java.io.Serializable;
+import java.util.List;
+
+/**
+ * @ClassName: AIMark
+ * @Description: ai标注
+ * @author: YaoGuoDong
+ * @create: 2024-06-11 15:08
+ **/
+public class AIMark implements Serializable {
+
+    private String id;
+    /**
+     * 分层id
+     */
+    private Integer monitorLevelId;
+    /**
+     * 接口信息
+     */
+    private InterfaceRelation interfaceRelation;
+    /**
+     * 标签对应关系
+     */
+    private List<TagRelation> tagRelations;
+    /**
+     * 项目名
+     */
+    private String project;
+    /**
+     * 项目id
+     */
+    private String projectId;
+
+    public String getId() {
+        return id;
+    }
+
+    public void setId(String id) {
+        this.id = id;
+    }
+
+    public Integer getMonitorLevelId() {
+        return monitorLevelId;
+    }
+
+    public void setMonitorLevelId(Integer monitorLevelId) {
+        this.monitorLevelId = monitorLevelId;
+    }
+
+    public InterfaceRelation getInterfaceRelation() {
+        return interfaceRelation;
+    }
+
+    public void setInterfaceRelation(InterfaceRelation interfaceRelation) {
+        this.interfaceRelation = interfaceRelation;
+    }
+
+    public List<TagRelation> getTagRelations() {
+        return tagRelations;
+    }
+
+    public void setTagRelations(List<TagRelation> tagRelations) {
+        this.tagRelations = tagRelations;
+    }
+
+    public String getProject() {
+        return project;
+    }
+
+    public void setProject(String project) {
+        this.project = project;
+    }
+
+    public String getProjectId() {
+        return projectId;
+    }
+
+    public void setProjectId(String projectId) {
+        this.projectId = projectId;
+    }
+
+    public static class TagRelation implements Serializable {
+        /**标签id*/
+        private Long id;
+        /**标签组id*/
+        private Long groupId;
+        /**标签组合id mtag*/
+        private String uniqueId;
+        /**标签名*/
+        private String name;
+        /**标签组名*/
+        private String groupName;
+        /**标签关联key*/
+        private String relateKey;
+        /**标签关联value*/
+        private String relateValue;
+
+        public Long getId() {
+            return id;
+        }
+
+        public void setId(Long id) {
+            this.id = id;
+        }
+
+        public Long getGroupId() {
+            return groupId;
+        }
+
+        public void setGroupId(Long groupId) {
+            this.groupId = groupId;
+        }
+
+        public String getUniqueId() {
+            return uniqueId;
+        }
+
+        public void setUniqueId(String uniqueId) {
+            this.uniqueId = uniqueId;
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public void setName(String name) {
+            this.name = name;
+        }
+
+        public String getGroupName() {
+            return groupName;
+        }
+
+        public void setGroupName(String groupName) {
+            this.groupName = groupName;
+        }
+
+        public String getRelateKey() {
+            return relateKey;
+        }
+
+        public void setRelateKey(String relateKey) {
+            this.relateKey = relateKey;
+        }
+
+        public String getRelateValue() {
+            return relateValue;
+        }
+
+        public void setRelateValue(String relateValue) {
+            this.relateValue = relateValue;
+        }
+    }
+
+    public static class InterfaceRelation implements Serializable {
+        /** 接口地址*/
+        private String url;
+        /** 接口对应信息*/
+        private String product;
+        /**接口方法 PUT*/
+        private String method;
+        /** 接口头信息*/
+        private JSONObject header;
+
+        public String getUrl() {
+            return url;
+        }
+
+        public void setUrl(String url) {
+            this.url = url;
+        }
+
+        public String getProduct() {
+            return product;
+        }
+
+        public void setProduct(String product) {
+            this.product = product;
+        }
+
+        public String getMethod() {
+            return method;
+        }
+
+        public void setMethod(String method) {
+            this.method = method;
+        }
+
+        public JSONObject getHeader() {
+            return header;
+        }
+
+        public void setHeader(JSONObject header) {
+            this.header = header;
+        }
+    }
+}

middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/pojo/AiApiResult.java

+package com.zhiwei.middleware.automatic.server.pojo;
+
+/**
+ * author：liu-yu
+ * date: 2024/6/13 9:46
+ **/
+public class AiApiResult {
+
+    private String id;
+
+    private String status;
+
+    private String info_class;
+
+    private String sent;
+
+    private String noise;
+
+    private String result;
+
+    private Long createAt;
+
+    public String getId() {
+        return id;
+    }
+
+    public void setId(String id) {
+        this.id = id;
+    }
+
+    public String getStatus() {
+        return status;
+    }
+
+    public void setStatus(String status) {
+        this.status = status;
+    }
+
+    public String getInfo_class() {
+        return info_class;
+    }
+
+    public void setInfo_class(String info_class) {
+        this.info_class = info_class;
+    }
+
+    public String getSent() {
+        return sent;
+    }
+
+    public void setSent(String sent) {
+        this.sent = sent;
+    }
+
+    public String getNoise() {
+        return noise;
+    }
+
+    public void setNoise(String noise) {
+        this.noise = noise;
+    }
+
+    public String getResult() {
+        return result;
+    }
+
+    public void setResult(String result) {
+        this.result = result;
+    }
+
+    public Long getCreateAt() {
+        return createAt;
+    }
+
+    public void setCreateAt(Long createAt) {
+        this.createAt = createAt;
+    }
+}

middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/pojo/AiInterfaceParam.java

+package com.zhiwei.middleware.automatic.server.pojo;
+
+import com.alibaba.fastjson.JSONObject;
+
+import java.io.Serializable;
+
+/**
+ * author：liu-yu
+ * date: 2024/6/11 17:40
+ **/
+public class AiInterfaceParam implements Serializable {
+
+    /**
+     * 源数据
+     */
+    private JSONObject json;
+
+    /**
+     *
+     * 接口参数
+     */
+    private AIMark aiMark;
+    
+    private String pt;
+
+    public JSONObject getJson() {
+        return json;
+    }
+
+    public void setJson(JSONObject json) {
+        this.json = json;
+    }
+
+    public AIMark getAiMark() {
+        return aiMark;
+    }
+
+    public void setAiMark(AIMark aiMark) {
+        this.aiMark = aiMark;
+    }
+
+    public String getPt() {
+        return pt;
+    }
+
+    public void setPt(String pt) {
+        this.pt = pt;
+    }
+}

middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/pojo/enums/TaskType.java

@@ -10,8 +10,9 @@ public enum TaskType {
     TEMPLATE_CLEAR_RETRY("template_clear_retry", null, "模板重置模板重试任务"),
     TEMPLATE_LOST_RESET("template_lost_reset", null, "过期重置模板清除任务"),
     TEMPLATE_DAY_CLEAR("template_day_clear", null, "模板每日清除任务"),
-    TEMPLATE_LOST_REMOVE("template_lost_remove", null, "过期模板清除任务");
+    TEMPLATE_LOST_REMOVE("template_lost_remove", null, "过期模板清除任务"),
 
+    AI_MARK("aiMark", "aiMark", "ai标注接口");
     final String type;
 
     final String cacheId;

middleware-automatic-center-server/pom.xml

@@ -21,7 +21,7 @@
         <qbjc-bean.version>1.1.4.1-SNAPSHOT</qbjc-bean.version>
         <nlp-aggree.version>0.0.5-SNAPSHOT</nlp-aggree.version>
         <dubbo-server.version>2.7.4.1</dubbo-server.version>
-        <automatic.version>1.0.0.3-SNAPSHOT</automatic.version>
+        <automatic.version>1.0.0.4-SNAPSHOT</automatic.version>
         <base.version>2.0.0-SNAPSHOT</base.version>
     </properties>
 

middleware-automatic-center-server/src/main/java/com/zhiwei/middleware/automatic/server/dubbo/service/impl/AiMarkMaticServiceImpl.java

+package com.zhiwei.middleware.automatic.server.dubbo.service.impl;
+
+import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.middleware.automatic.server.common.GenericAttribute;
+import com.zhiwei.middleware.automatic.server.core.TaskManager;
+import com.zhiwei.middleware.automatic.server.dubbo.service.AiMarkMaticService;
+import com.zhiwei.middleware.automatic.server.pojo.AiInterfaceParam;
+import com.zhiwei.middleware.automatic.server.pojo.AutoTask;
+import com.zhiwei.middleware.automatic.server.pojo.enums.TaskType;
+import com.zhiwei.middleware.automatic.server.util.RedissonUtil;
+import com.zhiwei.middleware.automatic.server.util.Tools;
+import org.apache.dubbo.config.annotation.Service;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * author：liu-yu
+ * date: 2024/6/12 14:46
+ **/
+@Service
+public class AiMarkMaticServiceImpl implements AiMarkMaticService {
+
+    private final RedissonUtil redissonUtil;
+
+    public AiMarkMaticServiceImpl(RedissonUtil redissonUtil) {
+        this.redissonUtil = redissonUtil;
+    }
+
+    @Override
+    public void autoAiMark(AiInterfaceParam param) {
+        autoAiMarkCommon(Collections.singletonList(param));
+    }
+
+    @Override
+    public void autoAiMark(List<AiInterfaceParam> paramList) {
+        autoAiMarkCommon(paramList);
+    }
+
+    private void autoAiMarkCommon(List<AiInterfaceParam> paramList) {
+        TaskType aiMark = TaskType.AI_MARK;
+        AutoTask autoTask = new AutoTask(aiMark.getType(), null);
+        String sourceKey = Tools.assembleKey(GenericAttribute.REDIS_MARK_AI_KEY, Tools.randomUUID());
+        redissonUtil.setList(sourceKey, paramList.stream().map(JSONObject::toJSONString).collect(Collectors.toList()));
+        autoTask.getParamSource().put(aiMark.getCacheId(), sourceKey);
+        TaskManager.getInstance().putTask(autoTask);
+    }
+}

middleware-automatic-center-server/src/main/resources/redisson-local.yaml

@@ -15,5 +15,5 @@ singleServerConfig:
   dnsMonitoringInterval: 5000
 threads: 0
 nettyThreads: 0
-codec: !<org.redisson.codec.JsonJacksonCodec> {}
+codec: !<org.redisson.client.codec.StringCodec> {}
 transportMode: "NIO"
\ No newline at end of file

middleware-automatic-center-son/pom.xml

@@ -24,7 +24,7 @@
         <qbjc-bean.version>1.1.4.1-SNAPSHOT</qbjc-bean.version>
         <kafka.version>2.4.1.RELEASE</kafka.version>
         <base.version>2.0.0-SNAPSHOT</base.version>
-        <automatic.version>1.0.0.3-SNAPSHOT</automatic.version>
+        <automatic.version>1.0.0.4-SNAPSHOT</automatic.version>
         <marker.version>1.2.3-SNAPSHOT</marker.version>
         <filter.version>1.1.6-SNAPSHOT</filter.version>
         <nlp-aggree.version>0.0.5-SNAPSHOT</nlp-aggree.version>

middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/config/TaskPoolConfig.java

@@ -34,4 +34,22 @@ public class TaskPoolConfig {
         return new ThreadPoolExecutorTimeout(1000L * 60 * 30, 32, 32,10000L,
                 TimeUnit.MILLISECONDS, 1000, "模板任务线程池");
     }
+
+    @Bean("aiMarkExecutor")
+    public ThreadPoolTaskExecutor aiMarkExecutor() {
+        ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
+        // 配置核心线程数
+        executor.setCorePoolSize(2);
+        // 配置最大线程数
+        executor.setMaxPoolSize(2);
+        // 配置线程池中的线程的名称前缀
+        executor.setThreadNamePrefix("aiMark-executor-");
+        executor.setQueueCapacity(100);
+        // rejection-policy：当pool已经达到max size的时候，如何处理新任务
+        // CALLER_RUNS：不在新线程中执行任务，而是有调用者所在的线程来执行
+        executor.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy());
+        // 执行初始化
+        executor.initialize();
+        return executor;
+    }
 }

middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/dao/AiApiResultDao.java

+package com.zhiwei.middleware.automatic.son.dao;
+
+import com.zhiwei.middleware.automatic.server.pojo.AiApiResult;
+import org.springframework.data.mongodb.core.query.Query;
+
+/**
+ * author：liu-yu
+ * date: 2024/6/13 10:02
+ **/
+public interface AiApiResultDao {
+
+    void insert(AiApiResult aiApiResult);
+
+    AiApiResult findOne(Query query);
+}

middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/dao/impl/AiApiResultDaoImpl.java

+package com.zhiwei.middleware.automatic.son.dao.impl;
+
+import com.zhiwei.middleware.automatic.server.pojo.AiApiResult;
+import com.zhiwei.middleware.automatic.son.dao.AiApiResultDao;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.data.mongodb.core.query.Query;
+import org.springframework.stereotype.Component;
+
+/**
+ * author：liu-yu
+ * date: 2024/6/13 10:03
+ **/
+@Component
+public class AiApiResultDaoImpl implements AiApiResultDao {
+
+    private final MongoTemplate mongoTemplate;
+
+    private static final String COLLECTION_NAME = "marker_ai_api_result";
+
+    public AiApiResultDaoImpl(@Qualifier("markerMongoTemplate") MongoTemplate mongoTemplate) {
+        this.mongoTemplate = mongoTemplate;
+    }
+    @Override
+    public void insert(AiApiResult aiApiResult) {
+        mongoTemplate.insert(aiApiResult, COLLECTION_NAME);
+    }
+
+    @Override
+    public AiApiResult findOne(Query query) {
+        return mongoTemplate.findOne(query, AiApiResult.class, COLLECTION_NAME);
+    }
+}

middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/dubbo/DubboHandler.java

@@ -16,6 +16,7 @@ import org.springframework.stereotype.Component;
 
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 @Component
 public class DubboHandler {
@@ -41,8 +42,10 @@ public class DubboHandler {
     }
 
     public void markUpsert(List<MarkInfo> collect) {
+        if (Objects.nonNull(collect) && !collect.isEmpty()) {
             markerClient.upsert(collect);
         }
+    }
 
     /**
      * 批量查询标签结果

middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/task/service/TaskServiceAiMark.java

+package com.zhiwei.middleware.automatic.son.task.service;
+
+import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.base.entity.subclass.mark.MarkInfo;
+import com.zhiwei.middleware.automatic.server.common.GenericAttribute;
+import com.zhiwei.middleware.automatic.server.pojo.AIMark;
+import com.zhiwei.middleware.automatic.server.pojo.AiApiResult;
+import com.zhiwei.middleware.automatic.server.pojo.AiInterfaceParam;
+import com.zhiwei.middleware.automatic.server.pojo.AutoTask;
+import com.zhiwei.middleware.automatic.server.pojo.enums.TaskType;
+import com.zhiwei.middleware.automatic.server.util.RedissonUtil;
+import com.zhiwei.middleware.automatic.son.dao.AiApiResultDao;
+import com.zhiwei.middleware.automatic.son.dubbo.DubboHandler;
+import com.zhiwei.middleware.automatic.son.util.MarkInfoUtil;
+import com.zhiwei.middleware.automatic.son.util.OkHttpUtil;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.data.mongodb.core.query.Query;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
+import org.springframework.stereotype.Service;
+
+import java.time.Instant;
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * author：liu-yu
+ * date: 2024/6/12 15:20
+ **/
+@Service("TaskServiceAiMark")
+public class TaskServiceAiMark implements TaskService {
+
+    private static final Logger log = LogManager.getLogger(TaskServiceAiMark.class);
+
+    private final ThreadPoolTaskExecutor markAiExecutor;
+    
+    private final RedissonUtil redissonUtil;
+
+    private final AiApiResultDao aiApiResultDao;
+
+    private final DubboHandler dubboHandler;
+
+    private final TaskType AI_MARK = TaskType.AI_MARK;
+
+    public TaskServiceAiMark(@Qualifier("aiMarkExecutor") ThreadPoolTaskExecutor markAiExecutor,
+                             RedissonUtil redissonUtil, AiApiResultDao aiApiResultDao,
+                             DubboHandler dubboHandler) {
+        this.markAiExecutor = markAiExecutor;
+        this.redissonUtil = redissonUtil;
+        this.aiApiResultDao = aiApiResultDao;
+        this.dubboHandler = dubboHandler;
+    }
+
+    @Override
+    public TaskType supports(String type) {
+        return AI_MARK.getType().equals(type) ? AI_MARK : null;
+    }
+
+    @Override
+    public void runTask(AutoTask autoTask) {
+        List<String> list = redissonUtil.getList(autoTask.getParamSource().getString(AI_MARK.getCacheId()));
+        if (Objects.isNull(list) || list.isEmpty()) {
+            return;
+        }
+        List<AiInterfaceParam> data = list.stream().map(e -> JSONObject.parseObject(e).toJavaObject(AiInterfaceParam.class)).collect(Collectors.toList());
+        markAiExecutor.execute(() -> aiMark(data));
+    }
+
+    @Override
+    public void executorStatus() {
+
+    }
+
+    private void aiMark(List<AiInterfaceParam> data) {
+        List<MarkInfo> list = new ArrayList<>();
+        for (AiInterfaceParam param : data) {
+            try {
+                MarkInfo markInfo = aiInterface(param);
+                if (Objects.nonNull(markInfo)) {
+                    list.add(markInfo);
+                }
+            } catch (Exception e) {
+                log.error("ai标注失败，数据id：{}，错误信息：" , param.getJson().getString("_id"), e);
+            }
+        }
+        dubboHandler.markUpsert(list);
+    }
+
+    private MarkInfo aiInterface(AiInterfaceParam aiParam) {
+        JSONObject param = aiInterfaceParamBuild(aiParam);
+        long now = System.currentTimeMillis();
+        String s = OkHttpUtil.httpPut(aiParam.getAiMark().getInterfaceRelation().getUrl(), param.toJSONString(), aiParam.getAiMark().getInterfaceRelation().getHeader().getInnerMap());
+        if (Objects.isNull(s)) {
+            log.error("ai标注失败，数据id：{}，错误信息：ai接口返回为null" , aiParam.getJson().getString("_id"));
+            return null;
+        }
+        long endTime = System.currentTimeMillis() - now;
+        AiApiResult aiRes = aiApiResultConversion(s);
+        if (Objects.isNull(aiRes)) {
+            log.info("数据url:{}，耗时：{}，ai接口返回信息不完整：{}", param.getString(GenericAttribute.URL), endTime, s);
+            return null;
+        }
+        log.info("数据url：{}，ai标签：{}，耗时：{}，具体数据:{}", param.getString(GenericAttribute.URL), aiRes.getSent(), endTime, aiRes.getResult());
+        List<AIMark.TagRelation> tagRelations = aiParam.getAiMark().getTagRelations();
+        AIMark.TagRelation tagRelation = tagRelations.stream().filter(e -> relateValueVerify(e, aiRes)).findFirst().orElse(null);
+        if (Objects.isNull(tagRelation)) {
+            log.info("数据url：{}，ai标签：{}，没有绑定关系，以过滤", param.getString(GenericAttribute.URL), aiRes.getSent());
+            return null;
+        }
+        JSONObject json = aiParam.getJson();
+        json.put(GenericAttribute.ES_M_TAG, tagRelation.getUniqueId());
+        json.put(GenericAttribute.ES_M_PERSON, GenericAttribute.AI_PERSON);
+        json.put(GenericAttribute.ES_M_TIME, System.currentTimeMillis());
+        json.put(GenericAttribute.ES_M_GROUP, aiParam.getAiMark().getProject());
+        json.put(GenericAttribute.ES_M_GROUP_ID, aiParam.getAiMark().getProjectId());
+        MarkInfo markInfo = MarkInfoUtil.transformToMarkInfo(json);
+        log.info("数据url：{}，ai标签：{}，标注标签：{}，成功标注", param.getString(GenericAttribute.URL), aiRes.getSent(), tagRelation.getName());
+        return markInfo;
+    }
+
+    private boolean relateValueVerify(AIMark.TagRelation tagRelation, AiApiResult result) {
+        String relateValue;
+        switch (tagRelation.getRelateKey()) {
+            case "sent" :
+                relateValue = result.getSent();
+                break;
+            case "info_class":
+                relateValue = result.getInfo_class();
+                break;
+            default:relateValue = "";
+                break;
+        }
+        return relateValue.equals(tagRelation.getRelateValue());
+    }
+
+    private AiApiResult aiApiResultConversion(String s) {
+        JSONObject res = JSONObject.parseObject(s);
+        JSONObject feiHeiRes = res.getJSONObject("feihe-compet-tagger");
+        if (Objects.nonNull(feiHeiRes)) {
+            JSONObject result = feiHeiRes.getJSONObject("result");
+            if (Objects.nonNull(result)) {
+                aiApiResultTitleSubString(result);
+                if (feiHeiRes.getString("status").equals("DONE")) {
+                    AiApiResult aiRes = new AiApiResult();
+                    aiRes.setId(feiHeiRes.getString("result_id"));
+                    aiRes.setCreateAt(System.currentTimeMillis());
+                    aiRes.setInfo_class(result.getString("info_class"));
+                    aiRes.setNoise(result.getString("denoise"));
+                    aiRes.setSent(result.getString("sent"));
+                    aiRes.setStatus(feiHeiRes.getString("status"));
+                    aiRes.setResult(res.toJSONString());
+                    aiApiResultDao.insert(aiRes);
+                    return aiRes;
+                } else {
+                    return aiApiResultDao.findOne(new Query(Criteria.where("_id").is(result.getString("dedup"))));
+                }
+            }
+        }
+        return null;
+    }
+
+    private void aiApiResultTitleSubString(JSONObject json) {
+        String title = json.getString(GenericAttribute.ES_TITLE);
+        if (Objects.nonNull(title) && title.length() > 100) {
+            json.put(GenericAttribute.ES_TITLE, title.substring(0, 100));
+        }
+        String snippet = json.getString("snippet");
+        if (Objects.nonNull(snippet) && snippet.length() > 100) {
+            json.put("snippet", snippet.substring(0, 100));
+        }
+    }
+
+    private JSONObject aiInterfaceParamBuild(AiInterfaceParam aiParam) {
+        JSONObject param = new JSONObject();
+        param.put("wait_for", new String[]{"feihe-compet-tagger"});
+        JSONObject data = new JSONObject();
+        JSONObject json = aiParam.getJson();
+        JSONObject metadata = new JSONObject();
+        metadata.put("product", aiParam.getAiMark().getInterfaceRelation().getProduct());
+        metadata.put("platform", aiParam.getPt());
+        metadata.put("platform_category", aiParam.getPt());
+        data.put("metadata", metadata);
+        data.put("author", json.getString(GenericAttribute.ES_SOURCE));
+        data.put("pub_date", Instant.ofEpochMilli(json.getLong(GenericAttribute.ES_TIME)).toString());
+        List<String> indFullText = json.getJSONArray(GenericAttribute.ES_IND_FULL_TEXT).toJavaList(String.class);
+        data.put(GenericAttribute.ES_TITLE, indFullText.get(0));
+        data.put("snippet", indFullText.get(1));
+        String answerUrl = json.getString(GenericAttribute.ES_URL_ANSWER);
+        String questionUrl = json.getString(GenericAttribute.ES_URL_QUESTION);
+        String url = Objects.nonNull(answerUrl) ? answerUrl : Objects.nonNull(questionUrl) ? questionUrl : json.getString(GenericAttribute.URL);
+        data.put(GenericAttribute.URL, url);
+        param.put("data", data);
+        return param;
+    }
+}
+
+

middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/task/service/TaskServiceCommon.java

@@ -248,7 +248,7 @@ public class TaskServiceCommon extends BaseTaskTypePair<TaskServiceCommon.TaskCo
             TemplateTitleVo templateTitleVo = titleVoMap.get(aggreTitle);
             String aggreeTag = templateTitleVo.getMtag();
             sourceObj.put(GenericAttribute.ES_M_TAG, aggreeTag);
-            sourceObj.put(GenericAttribute.ES_M_PERSON, "自动化机器人");
+            sourceObj.put(GenericAttribute.ES_M_PERSON, GenericAttribute.AUTO_PERSON);
             sourceObj.put(GenericAttribute.ES_M_TIME, new Date().getTime());
             log.info("项目:{} 模板标题:{} MarkSum:{} Tag:{}被标注标题：{}相似度：{}", group, aggreTitle, templateTitleVo.getMarkSum(), aggreeTag,
                     title, similarMap.get("similar"));

middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/util/OkHttpUtil.java

+package com.zhiwei.middleware.automatic.son.util;
+
+import okhttp3.*;
+
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * author：liu-yu
+ * date: 2024/6/12 17:59
+ **/
+public class OkHttpUtil {
+    public static final OkHttpClient okHttpClient = new OkHttpClient.Builder()
+            .connectTimeout(60, TimeUnit.SECONDS)
+            .readTimeout(60, TimeUnit.SECONDS)
+            .writeTimeout(60, TimeUnit.SECONDS)
+            .retryOnConnectionFailure(true).build();
+
+    public static String httpPut(String url, String info, Map<String, Object> header) {
+        Request.Builder builder = new Request.Builder().url(url).put(RequestBody.create(MediaType.parse("application/json; charset=utf-8"), info));
+        for (Map.Entry<String, Object> entry : header.entrySet()) {
+            builder.addHeader(entry.getKey(), String.valueOf(entry.getValue()));
+        }
+        try (Response response = okHttpClient.newCall(builder.build()).execute()) {
+            return response.body().string();
+        } catch (Exception e) {
+            return null;
+        }
+    }
+
+}