Merge branch 'feature' into 'release'

2023年05/22 模板查询dubbo接口。当日模板记录清空。

See merge request !44

middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/core/AutoMaticClient.java

@@ -32,12 +32,14 @@ public class AutoMaticClient {
          autoMaticService.resetTemplate(group, templateTitle);
     }
 
-    public Map<String, TemplateTitleVo> getTemplateTitleByProjectByLive(String project) {
-        return autoMaticService.getTemplateTitleByProjectByLive(project);
+    public Map<String, Object> getTemplateTitleByProjectByLive(String project, String searchInfo, List<String> tags, long startTime,
+                                                                        long endTime, int page, int size, String orderField, boolean isAsc) {
+        return autoMaticService.getTemplateTitleByProjectByLive(project, searchInfo, tags, startTime, endTime, page, size, orderField, isAsc);
     }
 
-    public Map<String, TemplateTitleVo> getTemplateTitleByProjectByLost(String project) {
-        return autoMaticService.getTemplateTitleByProjectByLost(project);
+    public Map<String, Object> getTemplateTitleByProjectByLost(String project, String searchInfo, List<String> tags, long startTime,
+                                                                        long endTime, int page, int size, String orderField, boolean isAsc) {
+        return autoMaticService.getTemplateTitleByProjectByLost(project, searchInfo, tags, startTime, endTime, page, size, orderField, isAsc);
     }
 
     public List<String> getMupdateByTemplateTitle(String group, String templateTitle) {

middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/core/AutoMaticClientFactory.java

@@ -80,9 +80,9 @@ public class AutoMaticClientFactory {
     }
 
 
-    public static AutoMaticClient getAutoMaticClient(String registry, String group, String appName, long timeOut) {
+    public static AutoMaticClient getAutoMaticClient(String registry, String group, String appName, long timeout) {
         ConfigCenterConfig configCenterConfig = new ConfigCenterConfig();
-        configCenterConfig.setTimeout(timeOut);
+        configCenterConfig.setTimeout(timeout);
         return new AutoMaticClient(createInstance(AutoMaticService.class, registry, group, appName, configCenterConfig));
     }
 

middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/dubbo/service/AutoMaticService.java

@@ -33,14 +33,16 @@ public interface AutoMaticService {
      * @param project 项目
      * @return 模板集
      */
-    Map<String, TemplateTitleVo> getTemplateTitleByProjectByLive(String project);
+    Map<String, Object> getTemplateTitleByProjectByLive(String project, String searchInfo, List<String> tags, long startTime,
+                                                              long endTime, int page, int size, String orderField, boolean isAsc);
 
     /**
      * 获取项目文本模板
      * @param project 项目
      * @return 模板集
      */
-    Map<String, TemplateTitleVo> getTemplateTitleByProjectByLost(String project);
+    Map<String, Object> getTemplateTitleByProjectByLost(String project, String searchInfo, List<String> tags, long startTime,
+                                                              long endTime, int page, int size, String orderField, boolean isAsc);
 
     /**
      * 根据模板标题获取数据(仅最新100条)

middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/pojo/TemplateRecord.java

@@ -12,6 +12,8 @@ public class TemplateRecord {
      */
     private String templateId;
 
+    private String esId;
+
     /**
      * 特征值
      */
@@ -24,9 +26,10 @@ public class TemplateRecord {
 
     public TemplateRecord() {}
 
-    public TemplateRecord(String templateId, String mupdate) {
+    public TemplateRecord(String templateId, String esId, String mupdate) {
         this.templateId = templateId;
         this.mupdate = mupdate;
+        this.esId = esId;
         this.createAt = System.currentTimeMillis();
     }
 
@@ -61,4 +64,12 @@ public class TemplateRecord {
     public void setCreateAt(Long createAt) {
         this.createAt = createAt;
     }
+
+    public String getEsId() {
+        return esId;
+    }
+
+    public void setEsId(String esId) {
+        this.esId = esId;
+    }
 }

middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/pojo/TemplateTitleVo.java

@@ -128,6 +128,8 @@ public class TemplateTitleVo implements Serializable {
 
     public void accSum(long markSum, long daySum) {
         this.markSum = markSum + this.daySum;
+        if (this.daySum != 0) {
             this.daySum += daySum;
         }
+    }
 }

middleware-automatic-center-server/src/main/java/com/zhiwei/middleware/automatic/server/dubbo/service/AutoMaticService.java

@@ -37,14 +37,16 @@ public interface AutoMaticService {
      * @param project 项目
      * @return 模板集
      */
-    Map<String, TemplateTitleVo> getTemplateTitleByProjectByLive(String project);
+    Map<String, Object> getTemplateTitleByProjectByLive(String project, String searchInfo, List<String> tags, long startTime,
+                                                               long endTime, int page, int size, String orderField, boolean isAsc);
 
     /**
      * 获取项目文本模板
      * @param project 项目
      * @return 模板集
      */
-    Map<String, TemplateTitleVo> getTemplateTitleByProjectByLost(String project);
+    Map<String, Object> getTemplateTitleByProjectByLost(String project, String searchInfo, List<String> tags, long startTime,
+                                                               long endTime, int page, int size, String orderField, boolean isAsc);
 
     /**
      * 根据模板标题获取数据(仅最新100条)

middleware-automatic-center-server/src/main/java/com/zhiwei/middleware/automatic/server/dubbo/service/impl/AutoMaticServiceImpl.java

@@ -67,13 +67,15 @@ public class AutoMaticServiceImpl implements AutoMaticService {
     }
 
     @Override
-    public Map<String, TemplateTitleVo> getTemplateTitleByProjectByLive(String project) {
-        return templateTitleService.getTemplateTitleByProjectLive(project);
+    public Map<String, Object> getTemplateTitleByProjectByLive( String project, String searchInfo, List<String> tags, long startTime,
+                                                                      long endTime, int page, int size, String orderField, boolean isAsc) {
+        return templateTitleService.searchTemplateTitle(GenericAttribute.REDIS_MAP_KEY, project, searchInfo, tags, startTime, endTime, page, size, orderField, isAsc);
     }
 
     @Override
-    public Map<String, TemplateTitleVo> getTemplateTitleByProjectByLost(String project) {
-        return templateTitleService.getTemplateTitleByProjectLost(project);
+    public Map<String, Object> getTemplateTitleByProjectByLost( String project, String searchInfo, List<String> tags, long startTime,
+                                                                      long endTime, int page, int size, String orderField, boolean isAsc) {
+        return templateTitleService.searchTemplateTitle(GenericAttribute.REDIS_MAP_LOSE_KEY, project, searchInfo, tags, startTime, endTime, page, size, orderField, isAsc);
     }
 
     @Override

middleware-automatic-center-server/src/main/java/com/zhiwei/middleware/automatic/server/service/TemplateTitleService.java

@@ -21,6 +21,22 @@ public interface TemplateTitleService {
     Map<String, TemplateTitleVo> getTemplateTitleByProjectLost(String project);
 
     /**
+     * 模板查询
+     * @param project 项目
+     * @param searchInfo 查询信息
+     * @param tags 标签信息
+     * @param startTime 开始时间
+     * @param endTime 结束时间
+     * @param page 分页
+     * @param size size
+     * @param orderField 排序字段
+     * @param isAsc 排序
+     * @return 模板
+     */
+    Map<String, Object> searchTemplateTitle(String key, String project, String searchInfo, List<String> tags, long startTime,
+                                           long endTime, int page, int size, String orderField, boolean isAsc);
+
+    /**
      * 根据模板标题获取数据(仅最新100条)
      *
      * @param group 项目

middleware-automatic-center-server/src/main/java/com/zhiwei/middleware/automatic/server/service/impl/TemplateTitleServiceImpl.java

@@ -46,6 +46,62 @@ public class TemplateTitleServiceImpl implements TemplateTitleService {
     }
 
     @Override
+    public Map<String, Object> searchTemplateTitle(String redisKey, String project, String searchInfo, List<String> tags,
+                                                   long startTime, long endTime, int page, int size, String orderField, boolean isAsc) {
+        Map<String, TemplateTitleVo> templateTitleVoMap = getTemplateTitleByProject(redisKey, project);
+        int limit = (page - 1) * size;
+        List<Map<String, Object>> res = new ArrayList<>();
+        Map<String, Object> resultMap = new HashMap<>();
+        // 分页起始节点大于总模板数
+        if (limit > templateTitleVoMap.size()) {
+            return resultMap;
+        }
+        int i = 0;
+        for (String key : templateTitleVoMap.keySet()) {
+            // 返回数量达到指定分页数
+            if (res.size() == size) {
+                break;
+            }
+            TemplateTitleVo templateTitleVo = templateTitleVoMap.get(key);
+            long createTime = templateTitleVo.getCreateTime().getTime();
+            long updateTime = templateTitleVo.getUpdateTime().getTime();
+            // 时间过滤
+            if (0L != startTime && 0L != endTime && !(updateTime >= startTime && updateTime <= endTime)) {
+                continue;
+            }
+            String mtag = Tools.sortTag(templateTitleVo.getMtag());
+            // 标签过滤
+            if (!Tools.filterTag(tags, mtag)) {
+                continue;
+            }
+            // 文本过滤
+            if (!StringUtils.isEmpty(searchInfo) && !key.contains(searchInfo)) {
+                continue;
+            }
+            i++;
+            // 分页过滤
+            if (limit >= i) {
+                continue;
+            }
+            Map<String, Object> map = new HashMap<>();
+            map.put("templateTitle", key);
+            map.put("project", project);
+            map.put("searchInfo", searchInfo);
+            map.put("createTime", createTime);
+            map.put("updateTime", updateTime);
+            map.put("markSum", templateTitleVo.getMarkSum());
+            map.put("daySum", templateTitleVo.getDaySum());
+            map.put("mtag", templateTitleVo.getMtag());
+            map.put("url", templateTitleVo.getUrl());
+            res.add(map);
+        }
+        Tools.sortByField(res, orderField, isAsc);
+        resultMap.put("count", templateTitleVoMap.size());
+        resultMap.put("list", res);
+        return resultMap;
+    }
+
+    @Override
     public List<String> getMupdateByTemplateTitle(String group, String templateTitle) {
         try {
             TemplateTitleVo templateTitleVo = getTemplateTitleByProjectLive(group).get(templateTitle);
@@ -119,11 +175,11 @@ public class TemplateTitleServiceImpl implements TemplateTitleService {
             }
         }
         if (Objects.nonNull(titleVo)) {
-            setMarkCount(group, titleVo);
             res.put("isMatched", true);
             Map<String, Object> dataMap = new HashMap<>();
             dataMap.put("title", titleVo.getTemplateTitle());
             dataMap.put("markSum", titleVo.getMarkSum());
+            dataMap.put("daySum", titleVo.getDaySum());
             dataMap.put("updateTime", titleVo.getUpdateTime().getTime());
             dataMap.put("createTime", titleVo.getCreateTime().getTime());
             dataMap.put("mtag", titleVo.getMtag());
@@ -141,8 +197,4 @@ public class TemplateTitleServiceImpl implements TemplateTitleService {
         }
         return Tools.restoreTMap(mapValue, TemplateTitleVo.class);
     }
-
-    private void setMarkCount(String group, TemplateTitleVo vo) {
-        vo.setMarkSum(redissonUtil.getCount(Tools.assembleKey(COUNT_KEY, group, vo.getId())));
-    }
 }

middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/task/service/TaskServiceCommon.java

@@ -235,11 +235,12 @@ public class TaskServiceCommon extends BaseTaskTypePair<TaskServiceCommon.TaskCo
             try {
                 String[] updates = dubboHandler.getMupdates(markInfo.filterInfo());
                 templateTitleVo.refreshMark();
+                String id = sourceObj.getString("_id");
 //                // 特征值记录
-                templateTitleService.insertTemplateRecord(new TemplateRecord(templateTitleVo.getId(), updates[0]));
+                templateTitleService.insertTemplateRecord(new TemplateRecord(templateTitleVo.getId(), id, updates[0]));
                 // 测试环境临时添加，用于对比
                 templateRecordDao.tempRecord(new TemplateTempRecord(templateTitleVo.getId(), templateTitleVo.getTemplateTitle(), group, getUrl(markInfo),
-                        sourceObj.getString("_id"), templateTitleVo.getMtag()));
+                        id, templateTitleVo.getMtag()));
                 return true;
             } catch (Exception e) {
                 log.error("记录事件采集-标注数据特征值失败", e);

middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/task/service/TaskServiceTemplate.java

@@ -84,9 +84,9 @@ public class TaskServiceTemplate extends BaseTaskTypePair<TaskServiceTemplate.Ta
             long now = System.currentTimeMillis();
             TaskTemplateFunctional pair = getPairValue(autoTask.getType());
             pair.template(autoTask);
-            log.info("{}结束, 项目:{}，开始时间：{}，结束时间：{}，耗时：{}", getPairKey(autoTask.getType()).getMessage(), autoTask.getGroup(),
-                    Tools.TIME_FORMAT.format(autoTask.getParamSource().getLong(GenericAttribute.START_PARAM)),
-                    Tools.TIME_FORMAT.format(autoTask.getParamSource().getLong(GenericAttribute.START_PARAM)), System.currentTimeMillis() - now);
+//            log.info("{}结束, 项目:{}，开始时间：{}，结束时间：{}，耗时：{}", getPairKey(autoTask.getType()).getMessage(), autoTask.getGroup(),
+//                    Tools.TIME_FORMAT.format(autoTask.getParamSource().getLong(GenericAttribute.START_PARAM)),
+//                    Tools.TIME_FORMAT.format(autoTask.getParamSource().getLong(GenericAttribute.START_PARAM)), System.currentTimeMillis() - now);
             }, autoTask));
     }
 
@@ -139,7 +139,7 @@ public class TaskServiceTemplate extends BaseTaskTypePair<TaskServiceTemplate.Ta
                 return;
             }
             log.info("发现{}组数据{}条,聚合中...", group, sourceList.size());
-            projectDataTemplate(group, sourceList);
+            projectDataTemplate(group, sourceList, Objects.nonNull(autoTask.isSplitFilter()) ? autoTask.isSplitFilter() : false);
         } catch (Exception e) {
             log.error("自动聚合模板更新失败，项目：{}", group, e);
         }
@@ -184,7 +184,7 @@ public class TaskServiceTemplate extends BaseTaskTypePair<TaskServiceTemplate.Ta
      * @param group 项目
      * @param sourceList 数据集
      */
-    private void projectDataTemplate(String group, List<Map<String, Object>> sourceList) {
+    private void projectDataTemplate(String group, List<Map<String, Object>> sourceList, boolean isBig) {
         //聚合模板
         Map<String, TemplateTitleVo> aggregation = aggregation(transferMark(sourceList));
         //旧的聚合模板
@@ -209,7 +209,7 @@ public class TaskServiceTemplate extends BaseTaskTypePair<TaskServiceTemplate.Ta
                     return true;
                 }).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
         // 新旧模板合并 且更新模板
-        mergeTemplate(group, aggregation, templateTitleByProject);
+        mergeTemplate(group, aggregation, templateTitleByProject, isBig);
     }
 
     /**
@@ -246,7 +246,7 @@ public class TaskServiceTemplate extends BaseTaskTypePair<TaskServiceTemplate.Ta
      * @param oldTemplate 旧模板
      * @param newTemplate 新模板
      */
-    private void mergeTemplate(String group, Map<String, TemplateTitleVo> oldTemplate, Map<String, TemplateTitleVo> newTemplate) {
+    private void mergeTemplate(String group, Map<String, TemplateTitleVo> oldTemplate, Map<String, TemplateTitleVo> newTemplate, boolean isBig) {
         for (Map.Entry<String, TemplateTitleVo> newEntry : newTemplate.entrySet()) {
             List<String> templateKeys = oldTemplate.keySet().stream()
                     .filter(e -> CosineSimilarity.calculateTextSimWithBrand(newEntry.getKey(), e) >= 0.96)
@@ -259,6 +259,9 @@ public class TaskServiceTemplate extends BaseTaskTypePair<TaskServiceTemplate.Ta
                 for (String oldKey : templateKeys) {
                     TemplateTitleVo templateTitleVo = oldTemplate.get(oldKey);
                     templateTitleVo.setMtag(newEntry.getValue().getMtag());
+                    if (isBig) {
+                        templateTitleVo.setDaySum(0);
+                    }
                     templateTitleService.setTemplateValue(group, oldKey, templateTitleVo);
                 }
             }