Skip to content

M
middleware-automatic
Commit 2706f518 by liuyu
Options

2023年05/24 聚合模板数值累加bug修复。每日数值清空任务添加。serve端日志添加。

parent b9251121
Showing with 96 additions and 29 deletions
middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/common/GenericAttribute.java
......@@ -53,5 +53,7 @@ public class GenericAttribute {
public static final String URL = "url";
public static final Long WEEK_TIME = 1000L * 60 * 60 * 24 * 7;
public static final Long DAY_TIME = 1000L * 60 * 60 * 24;
public static final Long WEEK_TIME = DAY_TIME * 7;
}
middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/pojo/AutoTask.java
package com.zhiwei.middleware.automatic.server.pojo;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.middleware.automatic.server.common.GenericAttribute;
public class AutoTask {
......@@ -31,6 +32,15 @@ public class AutoTask {
this.splitFilter = splitFilter;
}
public AutoTask(String type, String group, Long startTime, Long endTime) {
this.type = type;
this.group = group;
this.splitFilter = null;
this.paramSource = new JSONObject();
this.paramSource.put(GenericAttribute.START_PARAM, startTime);
this.paramSource.put(GenericAttribute.END_PARAM, endTime);
}
public JSONObject getParamSource() {
return paramSource;
......
middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/pojo/TemplateTitleVo.java
......@@ -126,11 +126,8 @@ public class TemplateTitleVo implements Serializable {
this.daySum++;
}
public void accSum(long markSum, long daySum) {
// daySum就不需要更新
if (this.daySum != 0) {
public void accSum(long daySum) {
this.daySum += daySum;
this.markSum = markSum + this.daySum;
}
this.markSum += daySum;
}
}
middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/pojo/enums/TaskType.java
......@@ -8,7 +8,8 @@ public enum TaskType {
TEMPLATE_RESET("template_reset", null, "模板重置任务"),
TEMPLATE_RECORD("template_record", null, "模板特征值清除记录任务"),
TEMPLATE_CLEAR_RETRY("template_clear_retry", null, "模板重置模板重试任务"),
TEMPLATE_LOST_RESET("template_lost_reset", null, "过期重置模板清除任务");
TEMPLATE_LOST_RESET("template_lost_reset", null, "过期重置模板清除任务"),
TEMPLATE_DAY_CLEAR("template_day_clear", null, "模板每日清除任务");
final String type;
......
middleware-automatic-center-server/src/main/java/com/zhiwei/middleware/automatic/server/mission/ScheduledMission.java
......@@ -74,16 +74,38 @@ public class ScheduledMission {
}
}
@Scheduled(cron = "0 0 0 * * ? ")
@Async("autMarkExecutor")
public void templateDayClear() {
try {
Calendar calendar = Calendar.getInstance();
// 聚合1天,文章时间和标注时间都在1天内
calendar.add(Calendar.DAY_OF_MONTH, -1);
Long startTime = calendar.getTimeInMillis();
Long endTime = startTime + GenericAttribute.DAY_TIME;
for (String project : asyncTask.findAllGroup()) {
AutoTask autoTask = new AutoTask(TaskType.TEMPLATE_DAY_CLEAR.getType(), project, startTime, endTime);
log.info("任务类型：{}，项目：{}，开始时间：{}，结束时间：{}已创建", autoTask.getType(), autoTask.getGroup(),startTime, endTime);
TaskManager.getInstance().putTask(autoTask);
}
} catch (Exception e) {
log.error("模板每日清除任务构建失败", e);
}
}
private void putTask(String group, long startTime, long endTime, boolean splitFilter) {
AutoTask autoTask = new AutoTask(TaskType.TEMPLATE.getType(), group, splitFilter);
autoTask.getParamSource().put(GenericAttribute.START_PARAM, startTime);
autoTask.getParamSource().put(GenericAttribute.END_PARAM, endTime);
log.info("任务类型：{}，项目：{}，开始时间：{}，结束时间：{}已创建", autoTask.getType(), autoTask.getGroup(), startTime, endTime);
TaskManager.getInstance().putTask(autoTask);
}
private void putTaskByRecord() {
AutoTask autoTask = new AutoTask(TaskType.TEMPLATE_RECORD.getType(), null);
autoTask.getParamSource().put(GenericAttribute.END_PARAM, System.currentTimeMillis() - MONTH);
Long endTime = System.currentTimeMillis() - MONTH;
autoTask.getParamSource().put(GenericAttribute.END_PARAM, endTime);
log.info("任务类型：{}，项目：{}，结束时间：{}已创建", autoTask.getType(), autoTask.getGroup(), endTime);
TaskManager.getInstance().putTask(autoTask);
}
......@@ -94,12 +116,14 @@ public class ScheduledMission {
if (value.getStatus() == TemplateStatus.重置失败) {
AutoTask autoTask = new AutoTask(TaskType.TEMPLATE_CLEAR_RETRY.getType(), group);
autoTask.getParamSource().put(GenericAttribute.TEMPLATE_TITLE, entry.getKey());
log.info("任务类型：{}，项目：{}，标题：{}已创建", autoTask.getType(), autoTask.getGroup(), entry.getKey());
TaskManager.getInstance().putTask(autoTask);
continue;
}
if (value.getUpdateTime().getTime() > (System.currentTimeMillis() - GenericAttribute.WEEK_TIME)) {
AutoTask autoTask = new AutoTask(TaskType.TEMPLATE_LOST_RESET.getType(), group);
autoTask.getParamSource().put(GenericAttribute.TEMPLATE_TITLE, entry.getKey());
log.info("任务类型：{}，项目：{}，标题：{}已创建", autoTask.getType(), autoTask.getGroup(), entry.getKey());
TaskManager.getInstance().putTask(autoTask);
}
}
......
middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/service/TemplateTitleService.java
......@@ -51,7 +51,7 @@ public interface TemplateTitleService {
boolean resetTemplate (String group, String templateTitle);
/**
* 替换模板
* 替换模板 全部
* @param group 项目
* @param title title
* @param templateTitleVo 模板
......@@ -59,7 +59,15 @@ public interface TemplateTitleService {
void setTemplateValue(String group, String title, TemplateTitleVo templateTitleVo);
/**
* 替换模板标签
* 替换模板 数值累加
* @param group 项目
* @param title title
* @param daySum 数值
*/
void setTemplateValue(String group, String title, Long daySum);
/**
* 替换模板 标签
* @param group 项目
* @param title title
* @param fixTag 标签
......@@ -80,4 +88,10 @@ public interface TemplateTitleService {
* @return 是否成功
*/
boolean modifyTemplateMarkerInfo (String title, String project);
/**
* 清除模板每日统计
* @param group 项目
*/
void clearDaySum(String group, Long startTime, Long endTime);
}
middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/service/impl/TemplateTitleServiceImpl.java
......@@ -118,12 +118,17 @@ public class TemplateTitleServiceImpl implements TemplateTitleService {
@Override
public void setTemplateValue(String group, String title, TemplateTitleVo templateTitleVo) {
setTemplateValue(group, title, templateTitleVo, null);
setTemplateValue(group, title, templateTitleVo, null, null);
}
@Override
public void setTemplateValue(String group, String title, String fixTag) {
setTemplateValue(group, title, null, fixTag);
setTemplateValue(group, title, null, fixTag, null);
}
@Override
public void setTemplateValue(String group, String title, Long daySum) {
setTemplateValue(group, title, null, null, daySum);
}
@Override
......@@ -179,7 +184,19 @@ public class TemplateTitleServiceImpl implements TemplateTitleService {
return true;
}
private void setTemplateValue(String group, String title, TemplateTitleVo templateTitleVo, String fixTag) {
@Override
public void clearDaySum(String group, Long startTime, Long endTime) {
Map<String, TemplateTitleVo> templateTitleVoMap = getTemplateTitleByProjectLive(group);
for (Map.Entry<String, TemplateTitleVo> vo : templateTitleVoMap.entrySet()) {
TemplateTitleVo value = vo.getValue();
long count = templateRecordDao.count(new Query(Criteria.where("templateId").is(value.getId()).and("createAt").gte(startTime).lt(endTime)));
value.setDaySum(0L);
value.setMarkSum(count);
setTemplateValue(group, vo.getKey(), value);
}
}
private void setTemplateValue(String group, String title, TemplateTitleVo templateTitleVo, String fixTag, Long daySum) {
String lockKey = Tools.assembleKey(GenericAttribute.LOCK, title);
try {
if (redissonUtil.tryLock(lockKey, 5, 5)) {
......@@ -189,23 +206,22 @@ public class TemplateTitleServiceImpl implements TemplateTitleService {
if (Objects.isNull(titleVo)) {
return;
}
// templateTitleVo，fixTag 必定只有一个有值
// 当传入的模板为空，表示不需要修改模板
if (Objects.isNull(templateTitleVo)) {
templateTitleVo = titleVo;
}
// 标签为null表示需要更新模板
if (Objects.isNull(fixTag)) {
// templateTitleVo.daySum = 0 代表清空day模板，场景：大聚合时清空daySum。daySum != 0代表更新数值，场景：自动标注数值统计
templateTitleVo.accSum(titleVo.getMarkSum(), titleVo.getDaySum());
} else {
if (Objects.nonNull(fixTag)) {
// 修改标签
String oldTag = templateTitleVo.getMtag();
templateTitleVo.setMtag(fixTag);
log.info("修改模板标签成功: group:{} templateTitle:{} oldTag:{} fixTag:{}", group, title, oldTag,
fixTag);
}
// 表示累计模板
if (Objects.nonNull(daySum)) {
templateTitleVo.accSum(daySum);
}
redissonUtil.setMapValue(mapKey, title, JSONObject.toJSONString(templateTitleVo));
log.info("模板更新，id：{}，标注总量：{}，标注天级：{}，标签：{}", templateTitleVo.getId(), templateTitleVo.getMarkSum(), templateTitleVo.getDaySum(), templateTitleVo.getMtag());
}
} catch (Exception e) {
log.error("修改模板失败，项目：{}，title:{}", group, title, e);
......
middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/task/service/TaskServiceCommon.java
......@@ -210,7 +210,7 @@ public class TaskServiceCommon extends BaseTaskTypePair<TaskServiceCommon.TaskCo
if (entry.getValue().getDaySum() == 0) {
continue;
}
templateTitleService.setTemplateValue(group, entry.getKey(), entry.getValue());
templateTitleService.setTemplateValue(group, entry.getKey(), entry.getValue().getDaySum());
}
// 自动标注数据发送到标注中间件
dubboHandler.markUpsert(newList);
......
middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/task/service/TaskServiceTemplate.java
......@@ -71,6 +71,7 @@ public class TaskServiceTemplate extends BaseTaskTypePair<TaskServiceTemplate.Ta
taskCache(TaskType.TEMPLATE_CLEAR_RETRY, this::templateLost);
taskCache(TaskType.TEMPLATE_RECORD, this::templateRecordClean);
taskCache(TaskType.TEMPLATE_LOST_RESET, this::templateLost);
taskCache(TaskType.TEMPLATE_DAY_CLEAR, this::templateDayClear);
}
@Override
......@@ -142,7 +143,7 @@ public class TaskServiceTemplate extends BaseTaskTypePair<TaskServiceTemplate.Ta
return;
}
log.info("发现{}组，开始：{}，结束：{}，数据{}条,聚合中...", group, Tools.TIME_FORMAT.format(startTime), Tools.TIME_FORMAT.format(endTime), sourceList.size());
projectDataTemplate(group, sourceList, Objects.nonNull(autoTask.isSplitFilter()) ? autoTask.isSplitFilter() : false);
projectDataTemplate(group, sourceList);
log.info("发现{}组，开始：{}，结束：{}，数据{}条,聚合完成", group, Tools.TIME_FORMAT.format(startTime), Tools.TIME_FORMAT.format(endTime), sourceList.size());
TaskManager.getInstance().removeBucketCache(autoTask);
} catch (Exception e) {
......@@ -158,6 +159,10 @@ public class TaskServiceTemplate extends BaseTaskTypePair<TaskServiceTemplate.Ta
autoTask.getGroup());
}
private void templateDayClear(AutoTask autoTask) {
}
/**
* 查询该项目 指定时间范围的数据
* @param mgroup 项目
......@@ -189,7 +194,7 @@ public class TaskServiceTemplate extends BaseTaskTypePair<TaskServiceTemplate.Ta
* @param group 项目
* @param sourceList 数据集
*/
private void projectDataTemplate(String group, List<Map<String, Object>> sourceList, boolean isBig) {
private void projectDataTemplate(String group, List<Map<String, Object>> sourceList) {
//聚合模板
Map<String, TemplateTitleVo> newTemplate = aggregation(transferMark(sourceList));
//旧的聚合模板
......@@ -214,7 +219,7 @@ public class TaskServiceTemplate extends BaseTaskTypePair<TaskServiceTemplate.Ta
return true;
}).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
// 新旧模板合并 且更新模板
mergeTemplate(group, oldTemplate, newTemplate, isBig);
mergeTemplate(group, oldTemplate, newTemplate);
}
/**
......@@ -251,7 +256,7 @@ public class TaskServiceTemplate extends BaseTaskTypePair<TaskServiceTemplate.Ta
* @param oldTemplate 旧模板
* @param newTemplate 新模板
*/
private void mergeTemplate(String group, Map<String, TemplateTitleVo> oldTemplate, Map<String, TemplateTitleVo> newTemplate, boolean isBig) {
private void mergeTemplate(String group, Map<String, TemplateTitleVo> oldTemplate, Map<String, TemplateTitleVo> newTemplate) {
for (Map.Entry<String, TemplateTitleVo> newEntry : newTemplate.entrySet()) {
List<String> templateKeys = oldTemplate.keySet().stream()
.filter(e -> CosineSimilarity.calculateTextSimWithBrand(newEntry.getKey(), e) >= 0.96)
......@@ -263,11 +268,9 @@ public class TaskServiceTemplate extends BaseTaskTypePair<TaskServiceTemplate.Ta
// 更新标签
for (String oldKey : templateKeys) {
TemplateTitleVo templateTitleVo = oldTemplate.get(oldKey);
templateTitleVo.setMtag(newEntry.getValue().getMtag());
if (isBig) {
templateTitleVo.setDaySum(0);
if (!templateTitleVo.getMtag().equals(newEntry.getValue().getMtag())) {
templateTitleService.setTemplateValue(group, oldKey, newEntry.getValue().getMtag());
}
templateTitleService.setTemplateValue(group, oldKey, templateTitleVo);
}
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment