Merge branch 'release' into 'master'

Release

See merge request !55

middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/common/GenericAttribute.java

@@ -53,5 +53,7 @@ public class GenericAttribute {
 
     public static final String URL = "url";
 
-    public static final Long WEEK_TIME = 1000L * 60 * 60 * 24 * 7;
+    public static final Long DAY_TIME = 1000L * 60 * 60 * 24;
+
+    public static final Long WEEK_TIME = DAY_TIME * 7;
 }

middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/pojo/AutoTask.java

 package com.zhiwei.middleware.automatic.server.pojo;
 
 import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.middleware.automatic.server.common.GenericAttribute;
 
 public class AutoTask {
 
@@ -31,6 +32,15 @@ public class AutoTask {
         this.splitFilter = splitFilter;
     }
 
+    public AutoTask(String type, String group, Long startTime, Long endTime) {
+        this.type = type;
+        this.group = group;
+        this.splitFilter = null;
+        this.paramSource = new JSONObject();
+        this.paramSource.put(GenericAttribute.START_PARAM, startTime);
+        this.paramSource.put(GenericAttribute.END_PARAM, endTime);
+    }
+
 
     public JSONObject getParamSource() {
         return paramSource;

middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/pojo/TemplateTitleVo.java

@@ -126,11 +126,8 @@ public class TemplateTitleVo implements Serializable {
         this.daySum++;
     }
 
-    public void accSum(long markSum, long daySum) {
-        // daySum就不需要更新
-        if (this.daySum != 0) {
-            this.daySum += daySum;
-            this.markSum = markSum + this.daySum;
-        }
+    public void accSum(long daySum) {
+        this.daySum += daySum;
+        this.markSum += daySum;
     }
 }

middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/pojo/enums/TaskType.java

@@ -8,7 +8,8 @@ public enum TaskType {
     TEMPLATE_RESET("template_reset", null, "模板重置任务"),
     TEMPLATE_RECORD("template_record", null, "模板特征值清除记录任务"),
     TEMPLATE_CLEAR_RETRY("template_clear_retry", null, "模板重置模板重试任务"),
-    TEMPLATE_LOST_RESET("template_lost_reset", null, "过期重置模板清除任务");
+    TEMPLATE_LOST_RESET("template_lost_reset", null, "过期重置模板清除任务"),
+    TEMPLATE_DAY_CLEAR("template_day_clear", null, "模板每日清除任务");
 
     final String type;
 

middleware-automatic-center-server/src/main/java/com/zhiwei/middleware/automatic/server/mission/ScheduledMission.java

@@ -74,16 +74,38 @@ public class ScheduledMission {
         }
     }
 
+    @Scheduled(cron = "0 0 0 * * ? ")
+    @Async("autMarkExecutor")
+    public void templateDayClear() {
+        try {
+            Calendar calendar = Calendar.getInstance();
+            // 聚合1天,文章时间和标注时间都在1天内
+            calendar.add(Calendar.DAY_OF_MONTH, -1);
+            Long startTime = calendar.getTimeInMillis();
+            Long endTime = startTime + GenericAttribute.DAY_TIME;
+            for (String project : asyncTask.findAllGroup()) {
+                AutoTask autoTask = new AutoTask(TaskType.TEMPLATE_DAY_CLEAR.getType(), project, startTime, endTime);
+                log.info("任务类型：{}，项目：{}，开始时间：{}，结束时间：{}已创建", autoTask.getType(), autoTask.getGroup(),startTime, endTime);
+                TaskManager.getInstance().putTask(autoTask);
+            }
+        } catch (Exception e) {
+            log.error("模板每日清除任务构建失败", e);
+        }
+    }
+
     private void putTask(String group, long startTime, long endTime, boolean splitFilter) {
         AutoTask autoTask = new AutoTask(TaskType.TEMPLATE.getType(), group, splitFilter);
         autoTask.getParamSource().put(GenericAttribute.START_PARAM, startTime);
         autoTask.getParamSource().put(GenericAttribute.END_PARAM, endTime);
+        log.info("任务类型：{}，项目：{}，开始时间：{}，结束时间：{}已创建", autoTask.getType(), autoTask.getGroup(), startTime, endTime);
         TaskManager.getInstance().putTask(autoTask);
     }
 
     private void putTaskByRecord() {
         AutoTask autoTask = new AutoTask(TaskType.TEMPLATE_RECORD.getType(), null);
-        autoTask.getParamSource().put(GenericAttribute.END_PARAM, System.currentTimeMillis() - MONTH);
+        Long endTime = System.currentTimeMillis() - MONTH;
+        autoTask.getParamSource().put(GenericAttribute.END_PARAM, endTime);
+        log.info("任务类型：{}，项目：{}，结束时间：{}已创建", autoTask.getType(), autoTask.getGroup(), endTime);
         TaskManager.getInstance().putTask(autoTask);
     }
 
@@ -94,12 +116,14 @@ public class ScheduledMission {
             if (value.getStatus() == TemplateStatus.重置失败) {
                 AutoTask autoTask = new AutoTask(TaskType.TEMPLATE_CLEAR_RETRY.getType(), group);
                 autoTask.getParamSource().put(GenericAttribute.TEMPLATE_TITLE, entry.getKey());
+                log.info("任务类型：{}，项目：{}，标题：{}已创建", autoTask.getType(), autoTask.getGroup(), entry.getKey());
                 TaskManager.getInstance().putTask(autoTask);
                 continue;
             }
             if (value.getUpdateTime().getTime() > (System.currentTimeMillis() - GenericAttribute.WEEK_TIME)) {
                 AutoTask autoTask = new AutoTask(TaskType.TEMPLATE_LOST_RESET.getType(), group);
                 autoTask.getParamSource().put(GenericAttribute.TEMPLATE_TITLE, entry.getKey());
+                log.info("任务类型：{}，项目：{}，标题：{}已创建", autoTask.getType(), autoTask.getGroup(), entry.getKey());
                 TaskManager.getInstance().putTask(autoTask);
             }
         }

middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/service/TemplateTitleService.java

@@ -51,7 +51,7 @@ public interface TemplateTitleService {
     boolean resetTemplate (String group, String templateTitle);
 
     /**
-     * 替换模板
+     * 替换模板 全部
      * @param group 项目
      * @param title title
      * @param templateTitleVo 模板
@@ -59,7 +59,15 @@ public interface TemplateTitleService {
     void setTemplateValue(String group, String title, TemplateTitleVo templateTitleVo);
 
     /**
-     * 替换模板标签
+     * 替换模板 数值累加
+     * @param group 项目
+     * @param title title
+     * @param daySum 数值
+     */
+    void setTemplateValue(String group, String title, Long daySum);
+
+    /**
+     * 替换模板 标签
      * @param group 项目
      * @param title title
      * @param fixTag 标签
@@ -80,4 +88,10 @@ public interface TemplateTitleService {
      * @return 是否成功
      */
     boolean modifyTemplateMarkerInfo (String title, String project);
+
+    /**
+     * 清除模板每日统计
+     * @param group 项目
+     */
+    void clearDaySum(String group, Long startTime, Long endTime);
 }

middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/service/impl/TemplateTitleServiceImpl.java

@@ -118,12 +118,17 @@ public class TemplateTitleServiceImpl implements TemplateTitleService {
 
     @Override
     public void setTemplateValue(String group, String title, TemplateTitleVo templateTitleVo) {
-        setTemplateValue(group, title, templateTitleVo, null);
+        setTemplateValue(group, title, templateTitleVo, null, null);
     }
 
     @Override
     public void setTemplateValue(String group, String title, String fixTag) {
-        setTemplateValue(group, title, null, fixTag);
+        setTemplateValue(group, title, null, fixTag, null);
+    }
+
+    @Override
+    public void setTemplateValue(String group, String title, Long daySum) {
+        setTemplateValue(group, title, null, null, daySum);
     }
 
     @Override
@@ -179,7 +184,19 @@ public class TemplateTitleServiceImpl implements TemplateTitleService {
         return true;
     }
 
-    private void setTemplateValue(String group, String title, TemplateTitleVo templateTitleVo, String fixTag) {
+    @Override
+    public void clearDaySum(String group, Long startTime, Long endTime) {
+        Map<String, TemplateTitleVo> templateTitleVoMap = getTemplateTitleByProjectLive(group);
+        for (Map.Entry<String, TemplateTitleVo> vo : templateTitleVoMap.entrySet()) {
+            TemplateTitleVo value = vo.getValue();
+            long count = templateRecordDao.count(new Query(Criteria.where("templateId").is(value.getId()).and("createAt").gte(startTime).lt(endTime)));
+            value.setDaySum(0L);
+            value.setMarkSum(count);
+            setTemplateValue(group, vo.getKey(), value);
+        }
+    }
+
+    private void setTemplateValue(String group, String title, TemplateTitleVo templateTitleVo, String fixTag, Long daySum) {
         String lockKey = Tools.assembleKey(GenericAttribute.LOCK, title);
         try {
             if (redissonUtil.tryLock(lockKey, 5, 5)) {
@@ -189,23 +206,22 @@ public class TemplateTitleServiceImpl implements TemplateTitleService {
                 if (Objects.isNull(titleVo)) {
                     return;
                 }
-                // templateTitleVo，fixTag 必定只有一个有值
-                // 当传入的模板为空，表示不需要修改模板
                 if (Objects.isNull(templateTitleVo)) {
                     templateTitleVo = titleVo;
                 }
-                // 标签为null表示需要更新模板
-                if (Objects.isNull(fixTag)) {
-                    // templateTitleVo.daySum = 0 代表清空day模板，场景：大聚合时清空daySum。daySum != 0代表更新数值，场景：自动标注数值统计
-                    templateTitleVo.accSum(titleVo.getMarkSum(), titleVo.getDaySum());
-                } else {
+                if (Objects.nonNull(fixTag)) {
                     // 修改标签
                     String oldTag = templateTitleVo.getMtag();
                     templateTitleVo.setMtag(fixTag);
                     log.info("修改模板标签成功: group:{} templateTitle:{} oldTag:{} fixTag:{}", group, title, oldTag,
                             fixTag);
                 }
+                // 表示累计模板
+                if (Objects.nonNull(daySum)) {
+                    templateTitleVo.accSum(daySum);
+                }
                 redissonUtil.setMapValue(mapKey, title, JSONObject.toJSONString(templateTitleVo));
+                log.info("模板更新，id：{}，标注总量：{}，标注天级：{}，标签：{}", templateTitleVo.getId(), templateTitleVo.getMarkSum(), templateTitleVo.getDaySum(), templateTitleVo.getMtag());
             }
         } catch (Exception e) {
             log.error("修改模板失败，项目：{}，title:{}", group, title, e);

middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/task/service/TaskServiceCommon.java

@@ -210,7 +210,7 @@ public class TaskServiceCommon extends BaseTaskTypePair<TaskServiceCommon.TaskCo
             if (entry.getValue().getDaySum() == 0) {
                 continue;
             }
-            templateTitleService.setTemplateValue(group, entry.getKey(), entry.getValue());
+            templateTitleService.setTemplateValue(group, entry.getKey(),  entry.getValue().getDaySum());
         }
         // 自动标注数据发送到标注中间件
         dubboHandler.markUpsert(newList);

middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/task/service/TaskServiceTemplate.java

@@ -71,6 +71,7 @@ public class TaskServiceTemplate extends BaseTaskTypePair<TaskServiceTemplate.Ta
         taskCache(TaskType.TEMPLATE_CLEAR_RETRY, this::templateLost);
         taskCache(TaskType.TEMPLATE_RECORD, this::templateRecordClean);
         taskCache(TaskType.TEMPLATE_LOST_RESET, this::templateLost);
+        taskCache(TaskType.TEMPLATE_DAY_CLEAR, this::templateDayClear);
     }
 
     @Override
@@ -142,7 +143,7 @@ public class TaskServiceTemplate extends BaseTaskTypePair<TaskServiceTemplate.Ta
                 return;
             }
             log.info("发现{}组，开始：{}，结束：{}，数据{}条,聚合中...", group, Tools.TIME_FORMAT.format(startTime), Tools.TIME_FORMAT.format(endTime), sourceList.size());
-            projectDataTemplate(group, sourceList, Objects.nonNull(autoTask.isSplitFilter()) ? autoTask.isSplitFilter() : false);
+            projectDataTemplate(group, sourceList);
             log.info("发现{}组，开始：{}，结束：{}，数据{}条,聚合完成", group, Tools.TIME_FORMAT.format(startTime), Tools.TIME_FORMAT.format(endTime), sourceList.size());
             TaskManager.getInstance().removeBucketCache(autoTask);
         } catch (Exception e) {
@@ -158,6 +159,10 @@ public class TaskServiceTemplate extends BaseTaskTypePair<TaskServiceTemplate.Ta
                 autoTask.getGroup());
     }
 
+    private void templateDayClear(AutoTask autoTask) {
+
+    }
+
     /**
      * 查询该项目 指定时间范围的数据
      * @param mgroup 项目
@@ -189,7 +194,7 @@ public class TaskServiceTemplate extends BaseTaskTypePair<TaskServiceTemplate.Ta
      * @param group 项目
      * @param sourceList 数据集
      */
-    private void projectDataTemplate(String group, List<Map<String, Object>> sourceList, boolean isBig) {
+    private void projectDataTemplate(String group, List<Map<String, Object>> sourceList) {
         //聚合模板
         Map<String, TemplateTitleVo> newTemplate = aggregation(transferMark(sourceList));
         //旧的聚合模板
@@ -214,7 +219,7 @@ public class TaskServiceTemplate extends BaseTaskTypePair<TaskServiceTemplate.Ta
                     return true;
                 }).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
         // 新旧模板合并 且更新模板
-        mergeTemplate(group, oldTemplate, newTemplate, isBig);
+        mergeTemplate(group, oldTemplate, newTemplate);
     }
 
     /**
@@ -251,7 +256,7 @@ public class TaskServiceTemplate extends BaseTaskTypePair<TaskServiceTemplate.Ta
      * @param oldTemplate 旧模板
      * @param newTemplate 新模板
      */
-    private void mergeTemplate(String group, Map<String, TemplateTitleVo> oldTemplate, Map<String, TemplateTitleVo> newTemplate, boolean isBig) {
+    private void mergeTemplate(String group, Map<String, TemplateTitleVo> oldTemplate, Map<String, TemplateTitleVo> newTemplate) {
         for (Map.Entry<String, TemplateTitleVo> newEntry : newTemplate.entrySet()) {
             List<String> templateKeys = oldTemplate.keySet().stream()
                     .filter(e -> CosineSimilarity.calculateTextSimWithBrand(newEntry.getKey(), e) >= 0.96)
@@ -263,11 +268,9 @@ public class TaskServiceTemplate extends BaseTaskTypePair<TaskServiceTemplate.Ta
                 // 更新标签
                 for (String oldKey : templateKeys) {
                     TemplateTitleVo templateTitleVo = oldTemplate.get(oldKey);
-                    templateTitleVo.setMtag(newEntry.getValue().getMtag());
-                    if (isBig) {
-                        templateTitleVo.setDaySum(0);
+                    if (!templateTitleVo.getMtag().equals(newEntry.getValue().getMtag())) {
+                        templateTitleService.setTemplateValue(group, oldKey, newEntry.getValue().getMtag());
                     }
-                    templateTitleService.setTemplateValue(group, oldKey, templateTitleVo);
                 }
             }
         }