Merge branch 'release' into 'master'

Release

See merge request !35

middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/common/GenericAttribute.java

@@ -45,6 +45,8 @@ public class GenericAttribute {
 
     public static final String KEY = "task";
 
+    public static final String LOCK = "lock";
+
     public static final String RUNNING = "running";
 
     public static final String URL = "url";

middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/pojo/TemplateTitleVo.java

@@ -15,7 +15,8 @@ public class TemplateTitleVo implements Serializable {
     private String templateTitle;
     private Date updateTime;
     private Date createTime;
-    private Long markSum;
+    private long markSum;
+    private long daySum;
     private String mtag;
     private String url;
     private TemplateStatus status;
@@ -65,14 +66,22 @@ public class TemplateTitleVo implements Serializable {
         this.createTime = createTime;
     }
 
-    public Long getMarkSum() {
+    public long getMarkSum() {
         return markSum;
     }
 
-    public void setMarkSum(Long markSum) {
+    public void setMarkSum(long markSum) {
         this.markSum = markSum;
     }
 
+    public long getDaySum() {
+        return daySum;
+    }
+
+    public void setDaySum(long daySum) {
+        this.daySum = daySum;
+    }
+
     public String getMtag() {
         return mtag;
     }
@@ -103,14 +112,22 @@ public class TemplateTitleVo implements Serializable {
         this.createTime = new Date();
         this.templateTitle = templateTitle;
         this.markSum = 0L;
+        this.daySum = 0L;
         this.mtag = mtag;
         this.url = url;
         this.status = TemplateStatus.运行中;
     }
 
-    public TemplateTitleVo() {}
+    public TemplateTitleVo() {
+    }
 
     public void refreshMark() {
         this.updateTime = new Date();
+        this.daySum++;
+    }
+
+    public void accSum(long markSum, long daySum) {
+        this.markSum = markSum + this.daySum;
+        this.daySum += daySum;
     }
 }

middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/util/RedissonUtil.java

@@ -53,6 +53,16 @@ public class RedissonUtil {
         bucket.delete();
     }
 
+    public void unlock(String lockKey) {
+        RLock lock = redissonClient.getLock(lockKey);
+        lock.forceUnlock();
+    }
+
+    public boolean tryLock(String lockKey, long waitTime, int leaseTime) throws InterruptedException {
+        RLock lock = redissonClient.getLock(lockKey);
+        return lock.tryLock(waitTime, leaseTime, TimeUnit.SECONDS);
+    }
+
     public void putCount(String key, long count) {
         RAtomicLong atomicLong = redissonClient.getAtomicLong(redisKey(key));
         atomicLong.addAndGet(count);

middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/service/TemplateTitleService.java

@@ -19,10 +19,9 @@ public interface TemplateTitleService {
     /**
      * 添加项目文本模板
      * @param project 项目
-     * @param vos 模板集
-     * @return 模板集数量
+     * @param vo 模板
      */
-    void setTemplateTitleByProject(String project, Map<String, TemplateTitleVo> vos);
+    void setTemplateTitleByProject(String project, TemplateTitleVo vo);
 
     /**
      * 修正模板标题的markTag 如果不存在就会增加
@@ -46,4 +45,20 @@ public interface TemplateTitleService {
      * @return 是否成功
      */
     boolean resetTemplate (String group, String templateTitle);
+
+    /**
+     * 替换模板
+     * @param group 项目
+     * @param title title
+     * @param templateTitleVo 模板
+     */
+    void setTemplateValue(String group, String title, TemplateTitleVo templateTitleVo);
+
+    /**
+     * 替换模板标签
+     * @param group 项目
+     * @param title title
+     * @param fixTag 标签
+     */
+    void setTemplateValue(String group, String title, String fixTag);
 }

middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/service/impl/TemplateTitleServiceImpl.java

@@ -30,6 +30,7 @@ import org.springframework.stereotype.Service;
 
 import java.io.IOException;
 import java.util.*;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 @Service
@@ -75,11 +76,9 @@ public class TemplateTitleServiceImpl implements TemplateTitleService {
     }
 
     @Override
-    public synchronized void setTemplateTitleByProject(String project, Map<String, TemplateTitleVo> vos) {
+    public void setTemplateTitleByProject(String project, TemplateTitleVo vo) {
         String key = Tools.assembleKey(GenericAttribute.REDIS_MAP_KEY, project);
-        for (Map.Entry<String, TemplateTitleVo> entry : vos.entrySet()) {
-            redissonUtil.setMapValue(key, entry.getKey(), JSONObject.toJSONString(entry.getValue()));
-        }
+        redissonUtil.setMapValue(key, vo.getTemplateTitle(), JSONObject.toJSONString(vo));
     }
 
     @Override
@@ -87,17 +86,7 @@ public class TemplateTitleServiceImpl implements TemplateTitleService {
         try {
             // 防止增加模板标题有带符号的问题
             templateTitle = Tools.filterSymbol(templateTitle);
-            String key = Tools.assembleKey(GenericAttribute.REDIS_MAP_KEY, group);
-            String mapKeyValue = redissonUtil.getMapKeyValue(key, templateTitle);
-            if (Tools.isEmpty(mapKeyValue)) {
-                return false;
-            }
-            TemplateTitleVo titleVo = JSONObject.parseObject(mapKeyValue, TemplateTitleVo.class);
-            String oldTag = titleVo.getMtag();
-            titleVo.setMtag(fixTag);
-            redissonUtil.setMapValue(key, templateTitle, JSONObject.toJSONString(titleVo));
-            log.info("修改模板标签成功: group:{} templateTitle:{} oldTag:{} fixTag:{}", group, templateTitle, oldTag,
-                    fixTag);
+            setTemplateValue(group, templateTitle, fixTag);
             return true;
         } catch (Exception e) {
             e.printStackTrace();
@@ -110,9 +99,9 @@ public class TemplateTitleServiceImpl implements TemplateTitleService {
     }
 
     @Override
-    public boolean resetTemplate(String group, String templateTitle) {
+    public boolean resetTemplate(String group, String title) {
         // 防止增加模板标题有带符号的问题
-        templateTitle = Tools.filterSymbol(templateTitle);
+        String templateTitle = Tools.filterSymbol(title);
         Map<String, TemplateTitleVo> map = getTemplateTitleByProject(group);
         if (Objects.isNull(map.get(templateTitle))) {
             return false;
@@ -121,7 +110,7 @@ public class TemplateTitleServiceImpl implements TemplateTitleService {
         // 只要在运行中的模板 才能重置
         if (Objects.isNull(templateTitleVo.getStatus()) || templateTitleVo.getStatus().name().equals(TemplateStatus.运行中.name())) {
             templateTitleVo.setStatus(TemplateStatus.重置中);
-            setTemplateTitleByProject(group, map);
+            setTemplateValue(group, templateTitle, templateTitleVo);
             // 修改模板的标注信息
             executor.execute(() -> {
                 try {
@@ -132,7 +121,7 @@ public class TemplateTitleServiceImpl implements TemplateTitleService {
                     templateTitleVo.setStatus(TemplateStatus.重置失败);
                     log.error("重置模板：修改聚和集错误，title:{}，以加入重试队列", templateTitleVo.getTemplateTitle());
                 } finally {
-                    setTemplateTitleByProject(group, map);
+                    setTemplateValue(group, templateTitle, templateTitleVo);
                 }
             });
             return true;
@@ -140,6 +129,46 @@ public class TemplateTitleServiceImpl implements TemplateTitleService {
         return false;
     }
 
+    @Override
+    public void setTemplateValue(String group, String title, TemplateTitleVo templateTitleVo) {
+        setTemplateValue(group, title, templateTitleVo, null);
+    }
+
+    @Override
+    public void setTemplateValue(String group, String title, String fixTag) {
+        setTemplateValue(group, title, null, fixTag);
+    }
+
+    private void setTemplateValue(String group, String title, TemplateTitleVo templateTitleVo, String fixTag) {
+        String lockKey = Tools.assembleKey(GenericAttribute.LOCK, title);
+        try {
+            if (redissonUtil.tryLock(lockKey, 5, 5)) {
+                String mapKey = Tools.assembleKey(GenericAttribute.REDIS_MAP_KEY, group);
+                String mapKeyValue = redissonUtil.getMapKeyValue(mapKey, title);
+                TemplateTitleVo titleVo = JSONObject.parseObject(mapKeyValue, TemplateTitleVo.class);
+                if (Objects.isNull(titleVo)) {
+                    return;
+                }
+                if (Objects.isNull(templateTitleVo)) {
+                    templateTitleVo = titleVo;
+                }
+                if (Objects.isNull(fixTag)) {
+                    templateTitleVo.accSum(titleVo.getMarkSum(), titleVo.getDaySum());
+                } else {
+                    String oldTag = templateTitleVo.getMtag();
+                    templateTitleVo.setMtag(fixTag);
+                    log.info("修改模板标签成功: group:{} templateTitle:{} oldTag:{} fixTag:{}", group, title, oldTag,
+                            fixTag);
+                }
+                redissonUtil.setMapValue(mapKey, title, JSONObject.toJSONString(templateTitleVo));
+            }
+        } catch (Exception e) {
+            log.error("修改模板失败，项目：{}，title:{}", group, title, e);
+        } finally {
+            redissonUtil.unlock(lockKey);
+        }
+    }
+
     /**
      * 模板自动标注的历史数据修改为新标签
      * @param templateTitleVo 模板

middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/task/service/TaskServiceCommon.java

@@ -159,6 +159,7 @@ public class TaskServiceCommon extends BaseTaskTypePair<TaskServiceCommon.TaskCo
         for (Map.Entry<String, List<MarkInfo>> entry : groupMap.entrySet()) {
             try {
                 Map<String, TemplateTitleVo> template = templateTitleService.getTemplateTitleByProject(entry.getKey());
+                clearTemplateDaySum(template);
                 asyncTitleMark(entry.getKey(), entry.getValue(), template);
             } catch (Exception e) {
                 log.error("自动标注处理失败，项目：{}", entry.getKey(), e);
@@ -166,6 +167,12 @@ public class TaskServiceCommon extends BaseTaskTypePair<TaskServiceCommon.TaskCo
         }
     }
 
+    private void clearTemplateDaySum(Map<String, TemplateTitleVo> template) {
+        for (Map.Entry<String, TemplateTitleVo> entry : template.entrySet()) {
+            entry.getValue().setDaySum(0L);
+        }
+    }
+
     /**
      * 异步等待
      * @param group 项目
@@ -195,6 +202,14 @@ public class TaskServiceCommon extends BaseTaskTypePair<TaskServiceCommon.TaskCo
             String title = Tools.filterSymbol(markInfo.getSourceObj().getString(GenericAttribute.ES_TITLE));
             return null != title && title.length() > 6;
         }).filter(e -> autoMark(group, e, titleVoMap)).collect(Collectors.toList());
+        // 模板更新
+        for (Map.Entry<String, TemplateTitleVo> entry : titleVoMap.entrySet()) {
+            if (entry.getValue().getDaySum() == 0) {
+                continue;
+            }
+            templateTitleService.setTemplateValue(group, entry.getKey(), entry.getValue());
+        }
+        // 自动标注数据发送到标注中间件
 //        dubboHandler.markUpsert(newList);
     }
 
@@ -220,7 +235,7 @@ public class TaskServiceCommon extends BaseTaskTypePair<TaskServiceCommon.TaskCo
                 // 特征值记录
                 templateTitleService.insertTemplateRecord(new TemplateRecord(templateTitleVo.getId(), updates[0]));
                 // 模板标注count累加
-                redissonUtil.putCount(Tools.assembleKey(COUNT_KEY, group, templateTitleVo.getId()), 1);
+//                redissonUtil.putCount(Tools.assembleKey(COUNT_KEY, group, templateTitleVo.getId()), 1);
                 // 测试环境临时添加，用于对比
                 templateRecordDao.tempRecord(new TemplateTempRecord(templateTitleVo.getId(), templateTitleVo.getTemplateTitle(), group, getUrl(markInfo),
                         sourceObj.getString("_id"), templateTitleVo.getMtag()));

middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/task/service/TaskServiceTemplate.java

@@ -195,7 +195,7 @@ public class TaskServiceTemplate extends BaseTaskTypePair<TaskServiceTemplate.Ta
                     return true;
                 }).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
         // 新旧模板合并 且更新模板
-        templateTitleService.setTemplateTitleByProject(group,  mergeTemplate(aggregation, templateTitleByProject));
+        mergeTemplate(group, aggregation, templateTitleByProject);
     }
 
     /**
@@ -229,22 +229,23 @@ public class TaskServiceTemplate extends BaseTaskTypePair<TaskServiceTemplate.Ta
      * @param oldTemplate 旧模板
      * @param newTemplate 新模板
      */
-    private Map<String, TemplateTitleVo> mergeTemplate(Map<String, TemplateTitleVo> oldTemplate, Map<String, TemplateTitleVo> newTemplate) {
+    private void mergeTemplate(String group, Map<String, TemplateTitleVo> oldTemplate, Map<String, TemplateTitleVo> newTemplate) {
         for (Map.Entry<String, TemplateTitleVo> newEntry : newTemplate.entrySet()) {
             List<String> templateKeys = oldTemplate.keySet().stream()
                     .filter(e -> CosineSimilarity.calculateTextSimWithBrand(newEntry.getKey(), e) >= 0.96)
                     .collect(Collectors.toList());
             // 添加模板
             if (templateKeys.isEmpty()) {
-                oldTemplate.put(newEntry.getKey(), newEntry.getValue());
+                templateTitleService.setTemplateTitleByProject(group, newEntry.getValue());
             } else {
                 // 更新标签
                 for (String oldKey : templateKeys) {
-                    oldTemplate.get(oldKey).setMtag(newEntry.getValue().getMtag());
+                    TemplateTitleVo templateTitleVo = oldTemplate.get(oldKey);
+                    templateTitleVo.setMtag(newEntry.getValue().getMtag());
+                    templateTitleService.setTemplateValue(group, oldKey, templateTitleVo);
                 }
             }
         }
-        return oldTemplate;
     }
 
     /**