Merge branch 'release' into 'master'

Release

See merge request !107

middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/common/GenericAttribute.java

@@ -73,4 +73,16 @@ public class GenericAttribute {
     public static final String ES_M_GROUP = "mgroup";
 
     public static final String ES_M_GROUP_ID = "mgroup_id";
+
+    public static final String KEY_TOTAL = "total";
+
+    public static final String KEY_RES_ERROR = "resError";
+
+    public static final String KEY_RES_INCOMPLETE = "resIncomplete";
+
+    public static final String KEY_NOT_MARK_RELATION = "notMarkRelation";
+
+    public static final String KEY_SUCCESS = "success";
+
+    public static final String KEY_AI = "ai:count:";
 }

middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/pojo/AiMarkCount.java

+package com.zhiwei.middleware.automatic.server.pojo;
+
+/**
+ * author：liu-yu
+ * date: 2024/6/27 14:15
+ **/
+public class AiMarkCount {
+    private String id;
+
+    private String time;
+
+    private Long total;
+
+    private Long resError;
+
+    private Long resIncomplete;
+
+    private Long notMarkRelation;
+
+    private Long success;
+
+    public String getId() {
+        return id;
+    }
+
+    public void setId(String id) {
+        this.id = id;
+    }
+
+    public String getTime() {
+        return time;
+    }
+
+    public void setTime(String time) {
+        this.time = time;
+    }
+
+    public Long getTotal() {
+        return total;
+    }
+
+    public void setTotal(Long total) {
+        this.total = total;
+    }
+
+    public Long getResError() {
+        return resError;
+    }
+
+    public void setResError(Long resError) {
+        this.resError = resError;
+    }
+
+    public Long getResIncomplete() {
+        return resIncomplete;
+    }
+
+    public void setResIncomplete(Long resIncomplete) {
+        this.resIncomplete = resIncomplete;
+    }
+
+    public Long getNotMarkRelation() {
+        return notMarkRelation;
+    }
+
+    public void setNotMarkRelation(Long notMarkRelation) {
+        this.notMarkRelation = notMarkRelation;
+    }
+
+    public Long getSuccess() {
+        return success;
+    }
+
+    public void setSuccess(Long success) {
+        this.success = success;
+    }
+}

middleware-automatic-center-server/src/main/java/com/zhiwei/middleware/automatic/server/dao/AiMarkCountDao.java

+package com.zhiwei.middleware.automatic.server.dao;
+
+import com.zhiwei.middleware.automatic.server.pojo.AiMarkCount;
+
+/**
+ * author：liu-yu
+ * date: 2024/6/27 17:33
+ **/
+public interface AiMarkCountDao {
+
+    void insert(AiMarkCount aiMarkCount);
+}

middleware-automatic-center-server/src/main/java/com/zhiwei/middleware/automatic/server/dao/impl/AiMarkCountDaoImpl.java

+package com.zhiwei.middleware.automatic.server.dao.impl;
+
+import com.zhiwei.middleware.automatic.server.dao.AiMarkCountDao;
+import com.zhiwei.middleware.automatic.server.pojo.AiMarkCount;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.stereotype.Component;
+
+/**
+ * author：liu-yu
+ * date: 2024/6/27 17:34
+ **/
+@Component
+public class AiMarkCountDaoImpl implements AiMarkCountDao {
+
+    private final MongoTemplate mongoTemplate;
+
+    private static final String COLLECTION_NAME = "marker_ai_mark_count";
+
+    public AiMarkCountDaoImpl(@Qualifier("markerMongoTemplate") MongoTemplate mongoTemplate) {
+        this.mongoTemplate = mongoTemplate;
+    }
+    @Override
+    public void insert(AiMarkCount aiMarkCount) {
+        mongoTemplate.insert(aiMarkCount, COLLECTION_NAME);
+    }
+}

middleware-automatic-center-server/src/main/java/com/zhiwei/middleware/automatic/server/dubbo/service/impl/AiMarkMaticServiceImpl.java

@@ -4,6 +4,7 @@ import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.middleware.automatic.server.common.GenericAttribute;
 import com.zhiwei.middleware.automatic.server.core.TaskManager;
 import com.zhiwei.middleware.automatic.server.dubbo.service.AiMarkMaticService;
+import com.zhiwei.middleware.automatic.server.mission.AsyncTask;
 import com.zhiwei.middleware.automatic.server.pojo.AiInterfaceParam;
 import com.zhiwei.middleware.automatic.server.pojo.AutoTask;
 import com.zhiwei.middleware.automatic.server.pojo.enums.TaskType;
@@ -44,6 +45,7 @@ public class AiMarkMaticServiceImpl implements AiMarkMaticService {
         String sourceKey = Tools.assembleKey(GenericAttribute.REDIS_MARK_AI_KEY, Tools.randomUUID());
         redissonUtil.setList(sourceKey, paramList.stream().map(JSONObject::toJSONString).collect(Collectors.toList()));
         autoTask.getParamSource().put(aiMark.getCacheId(), sourceKey);
+        autoTask.getParamSource().put(GenericAttribute.ES_TIME, AsyncTask.CURRENT_TIME);
         TaskManager.getInstance().putTask(autoTask);
     }
 }

middleware-automatic-center-server/src/main/java/com/zhiwei/middleware/automatic/server/mission/AsyncTask.java

 package com.zhiwei.middleware.automatic.server.mission;
 
 import com.zhiwei.middleware.automatic.server.common.GenericAttribute;
+import com.zhiwei.middleware.automatic.server.dao.AiMarkCountDao;
+import com.zhiwei.middleware.automatic.server.pojo.AiMarkCount;
+import com.zhiwei.middleware.automatic.server.util.RedissonUtil;
+import com.zhiwei.middleware.automatic.server.util.TimeUtil;
+import com.zhiwei.middleware.automatic.server.util.Tools;
 import com.zhiwei.qbjc.bean.pojo.common.Project;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.data.mongodb.core.MongoTemplate;
@@ -15,12 +20,21 @@ public class AsyncTask {
 
     private final MongoTemplate hangZhouMongo;
 
+    private final AiMarkCountDao aiMarkCountDao;
+
+    private final RedissonUtil redissonUtil;
+
     public static Long DAY_START_TIME;
 
     public static Long DAY_END_TIME;
 
-    public AsyncTask(@Qualifier("hangzhouMongoTemplate") MongoTemplate hangZhouMongo) {
+    public static String CURRENT_TIME;
+
+    public AsyncTask(@Qualifier("hangzhouMongoTemplate") MongoTemplate hangZhouMongo,
+                     AiMarkCountDao aiMarkCountDao, RedissonUtil redissonUtil) {
         this.hangZhouMongo = hangZhouMongo;
+        this.aiMarkCountDao = aiMarkCountDao;
+        this.redissonUtil = redissonUtil;
     }
 
     public List<String> findAllGroup() {
@@ -35,5 +49,23 @@ public class AsyncTask {
         calendar.set(Calendar.HOUR_OF_DAY, 0);
         DAY_START_TIME = calendar.getTimeInMillis();
         DAY_END_TIME = DAY_START_TIME + GenericAttribute.DAY_TIME;
+        initTimeKey();
+    }
+
+    private static void initTimeKey() {
+        Long now = System.currentTimeMillis();
+        CURRENT_TIME = TimeUtil.COMMON_TIME_FORMAT.format(now);
+    }
+
+    public void syncDayCount() {
+        String timeKey = TimeUtil.COMMON_TIME_FORMAT.format(System.currentTimeMillis() - GenericAttribute.DAY_TIME);
+        AiMarkCount aiMarkCount = new AiMarkCount();
+        aiMarkCount.setTime(timeKey);
+        aiMarkCount.setTotal(redissonUtil.getCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, GenericAttribute.KEY_TOTAL)));
+        aiMarkCount.setResError(redissonUtil.getCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, GenericAttribute.KEY_RES_ERROR)));
+        aiMarkCount.setResIncomplete(redissonUtil.getCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, GenericAttribute.KEY_RES_INCOMPLETE)));
+        aiMarkCount.setNotMarkRelation(redissonUtil.getCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, GenericAttribute.KEY_NOT_MARK_RELATION)));
+        aiMarkCount.setSuccess(redissonUtil.getCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, GenericAttribute.KEY_SUCCESS)));
+        aiMarkCountDao.insert(aiMarkCount);
     }
 }

middleware-automatic-center-server/src/main/java/com/zhiwei/middleware/automatic/server/mission/ScheduledMission.java

@@ -96,6 +96,16 @@ public class ScheduledMission {
         }
     }
 
+    @Scheduled(cron = "0 0 9 * * ? ")
+    public void syncDayCount() {
+        try {
+            asyncTask.syncDayCount();
+        } catch (Exception e) {
+            log.error("ai标注每日统计失败：", e);
+        }
+    }
+
+
     private void putTask(String group, long startTime, long endTime, boolean splitFilter) {
         AutoTask autoTask = new AutoTask(TaskType.TEMPLATE.getType(), group, splitFilter);
         autoTask.getParamSource().put(GenericAttribute.START_PARAM, startTime);

middleware-automatic-center-server/src/main/java/com/zhiwei/middleware/automatic/server/util/TimeUtil.java

@@ -11,6 +11,8 @@ public class TimeUtil {
 
     public static final FastDateFormat CONTENT_DF = FastDateFormat.getInstance("yyyy-MM-dd HH:mm");
 
+    public static final FastDateFormat COMMON_TIME_FORMAT = FastDateFormat.getInstance("yyyy-MM-dd");
+
     public static final FastDateFormat TIME_FORMAT = FastDateFormat.getInstance("yyyy-MM-dd HH:mm:ss");
 
     public static final FastDateFormat DW_INDEX = FastDateFormat.getInstance("yyyyMM");

middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/dao/impl/EsDaoImpl.java

@@ -64,7 +64,7 @@ public class EsDaoImpl implements EsDao {
         if (size >= 0) {
             builder.size(size);
         }
-        builder.sort("_id", SortOrder.ASC);
+        builder.sort("id", SortOrder.ASC);
         String scrollId = "";
         while (null != scrollId) {
             if (objects.length > 0) {

middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/task/service/TaskServiceAiMark.java

@@ -13,6 +13,7 @@ import com.zhiwei.middleware.automatic.son.dao.AiApiResultDao;
 import com.zhiwei.middleware.automatic.son.dubbo.DubboHandler;
 import com.zhiwei.middleware.automatic.son.util.MarkInfoUtil;
 import com.zhiwei.middleware.automatic.son.util.OkHttpUtil;
+import com.zhiwei.middleware.automatic.son.util.Tools;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -66,7 +67,7 @@ public class TaskServiceAiMark implements TaskService {
             return;
         }
         List<AiInterfaceParam> data = list.stream().map(e -> JSONObject.parseObject(e).toJavaObject(AiInterfaceParam.class)).collect(Collectors.toList());
-        markAiExecutor.execute(() -> aiMark(data));
+        markAiExecutor.execute(() -> aiMark(data, autoTask.getParamSource().getString(GenericAttribute.ES_TIME)));
     }
 
     @Override
@@ -74,11 +75,11 @@ public class TaskServiceAiMark implements TaskService {
 
     }
 
-    private void aiMark(List<AiInterfaceParam> data) {
+    private void aiMark(List<AiInterfaceParam> data, String timeKey) {
         List<MarkInfo> list = new ArrayList<>();
         for (AiInterfaceParam param : data) {
             try {
-                MarkInfo markInfo = aiInterface(param);
+                MarkInfo markInfo = aiInterface(param, timeKey);
                 if (Objects.nonNull(markInfo)) {
                     list.add(markInfo);
                 }
@@ -89,35 +90,42 @@ public class TaskServiceAiMark implements TaskService {
         dubboHandler.markUpsert(list);
     }
 
-    private MarkInfo aiInterface(AiInterfaceParam aiParam) {
+    private MarkInfo aiInterface(AiInterfaceParam aiParam, String timeKey) {
         JSONObject param = aiInterfaceParamBuild(aiParam);
         long now = System.currentTimeMillis();
         String s = OkHttpUtil.httpPut(aiParam.getAiMark().getInterfaceRelation().getUrl(), param.toJSONString(), aiParam.getAiMark().getInterfaceRelation().getHeader().getInnerMap());
+        redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, GenericAttribute.KEY_TOTAL), 1);
         if (Objects.isNull(s)) {
-            log.error("ai标注失败，数据id：{}，错误信息：ai接口返回为null" , aiParam.getJson().getString("_id"));
+            redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, GenericAttribute.KEY_RES_ERROR), 1);
+            log.error("ai标注失败，数据id：{}，错误信息：ai接口返回为null" , aiParam.getJson().getString("id"));
             return null;
         }
         long endTime = System.currentTimeMillis() - now;
         AiApiResult aiRes = aiApiResultConversion(s);
         if (Objects.isNull(aiRes)) {
+            redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, GenericAttribute.KEY_RES_INCOMPLETE), 1);
             log.info("数据url:{}，耗时：{}，ai接口返回信息不完整：{}", param.getJSONObject("data").getString(GenericAttribute.URL), endTime, s);
             return null;
         }
         log.info("数据url：{}，ai标签：{}，耗时：{}，具体数据:{}", param.getJSONObject("data").getString(GenericAttribute.URL), aiRes.getSent(), endTime, aiRes.getResult());
         List<AIMark.TagRelation> tagRelations = aiParam.getAiMark().getTagRelations();
-        AIMark.TagRelation tagRelation = tagRelations.stream().filter(e -> relateValueVerify(e, aiRes)).findFirst().orElse(null);
-        if (Objects.isNull(tagRelation)) {
+        Map<String, List<AIMark.TagRelation>> relateGroup = tagRelations.stream().collect(Collectors.groupingBy(AIMark.TagRelation::getRelateKey, Collectors.toList()));
+        List<AIMark.TagRelation> tagRelationList = relateGroup.values().stream().map(relations -> relations.stream().filter(x -> relateValueVerify(x, aiRes)).findFirst().orElse(null)).filter(Objects::nonNull).collect(Collectors.toList());
+        if (tagRelationList.isEmpty()) {
+            redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, GenericAttribute.KEY_NOT_MARK_RELATION), 1);
             log.info("数据url：{}，ai标签：{}，没有绑定关系，以过滤", param.getJSONObject("data").getString(GenericAttribute.URL), aiRes.getSent());
             return null;
         }
         JSONObject json = aiParam.getJson();
-        json.put(GenericAttribute.ES_M_TAG, tagRelation.getUniqueId());
+        String mtag = tagRelationList.stream().map(AIMark.TagRelation::getUniqueId).collect(Collectors.joining());
+        json.put(GenericAttribute.ES_M_TAG, mtag);
         json.put(GenericAttribute.ES_M_PERSON, GenericAttribute.AI_PERSON);
         json.put(GenericAttribute.ES_M_TIME, System.currentTimeMillis());
         json.put(GenericAttribute.ES_M_GROUP, aiParam.getAiMark().getProject());
         json.put(GenericAttribute.ES_M_GROUP_ID, aiParam.getAiMark().getProjectId());
         MarkInfo markInfo = MarkInfoUtil.transformToMarkInfo(json);
-        log.info("数据url：{}，ai标签：{}，标注标签：{}，成功标注", param.getJSONObject("data").getString(GenericAttribute.URL), aiRes.getSent(), tagRelation.getName());
+        redissonUtil.putCount(Tools.assembleKey(GenericAttribute.KEY_AI, timeKey, GenericAttribute.KEY_SUCCESS), 1);
+        log.info("数据url：{}，ai标签：{}，标注标签：{}，成功标注", param.getJSONObject("data").getString(GenericAttribute.URL), aiRes.getSent(), mtag);
         return markInfo;
     }