Skip to content

M
middleware-automatic
Commit 3b7f0d54 by liuyu
Options

2023年05/17 标注模板更新加锁

parent 5f8b3d5a
Showing with 121 additions and 32 deletions
middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/common/GenericAttribute.java
......@@ -45,6 +45,8 @@ public class GenericAttribute {
public static final String KEY = "task";
public static final String LOCK = "lock";
public static final String RUNNING = "running";
public static final String URL = "url";
......
middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/pojo/TemplateTitleVo.java
......@@ -15,7 +15,8 @@ public class TemplateTitleVo implements Serializable {
private String templateTitle;
private Date updateTime;
private Date createTime;
private Long markSum;
private long markSum;
private long daySum;
private String mtag;
private String url;
private TemplateStatus status;
......@@ -65,14 +66,22 @@ public class TemplateTitleVo implements Serializable {
this.createTime = createTime;
}
public Long getMarkSum() {
public long getMarkSum() {
return markSum;
}
public void setMarkSum(Long markSum) {
public void setMarkSum(long markSum) {
this.markSum = markSum;
}
public long getDaySum() {
return daySum;
}
public void setDaySum(long daySum) {
this.daySum = daySum;
}
public String getMtag() {
return mtag;
}
......@@ -103,14 +112,22 @@ public class TemplateTitleVo implements Serializable {
this.createTime = new Date();
this.templateTitle = templateTitle;
this.markSum = 0L;
this.daySum = 0L;
this.mtag = mtag;
this.url = url;
this.status = TemplateStatus.运行中;
}
public TemplateTitleVo() {}
public TemplateTitleVo() {
}
public void refreshMark() {
this.updateTime = new Date();
this.daySum++;
}
public void accSum(long markSum, long daySum) {
this.markSum = markSum + this.daySum;
this.daySum += daySum;
}
}
middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/util/RedissonUtil.java
......@@ -53,6 +53,16 @@ public class RedissonUtil {
bucket.delete();
}
public void unlock(String lockKey) {
RLock lock = redissonClient.getLock(lockKey);
lock.forceUnlock();
}
public boolean tryLock(String lockKey, long waitTime, int leaseTime) throws InterruptedException {
RLock lock = redissonClient.getLock(lockKey);
return lock.tryLock(waitTime, leaseTime, TimeUnit.SECONDS);
}
public void putCount(String key, long count) {
RAtomicLong atomicLong = redissonClient.getAtomicLong(redisKey(key));
atomicLong.addAndGet(count);
......
middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/service/TemplateTitleService.java
......@@ -19,10 +19,9 @@ public interface TemplateTitleService {
/**
* 添加项目文本模板
* @param project 项目
* @param vos 模板集
* @return 模板集数量
* @param vo 模板
*/
void setTemplateTitleByProject(String project, Map<String, TemplateTitleVo> vos);
void setTemplateTitleByProject(String project, TemplateTitleVo vo);
/**
* 修正模板标题的markTag 如果不存在就会增加
......@@ -46,4 +45,20 @@ public interface TemplateTitleService {
* @return 是否成功
*/
boolean resetTemplate (String group, String templateTitle);
/**
* 替换模板
* @param group 项目
* @param title title
* @param templateTitleVo 模板
*/
void setTemplateValue(String group, String title, TemplateTitleVo templateTitleVo);
/**
* 替换模板标签
* @param group 项目
* @param title title
* @param fixTag 标签
*/
void setTemplateValue(String group, String title, String fixTag);
}
middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/service/impl/TemplateTitleServiceImpl.java
......@@ -30,6 +30,7 @@ import org.springframework.stereotype.Service;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@Service
......@@ -75,11 +76,9 @@ public class TemplateTitleServiceImpl implements TemplateTitleService {
}
@Override
public synchronized void setTemplateTitleByProject(String project, Map<String, TemplateTitleVo> vos) {
public void setTemplateTitleByProject(String project, TemplateTitleVo vo) {
String key = Tools.assembleKey(GenericAttribute.REDIS_MAP_KEY, project);
for (Map.Entry<String, TemplateTitleVo> entry : vos.entrySet()) {
redissonUtil.setMapValue(key, entry.getKey(), JSONObject.toJSONString(entry.getValue()));
}
redissonUtil.setMapValue(key, vo.getTemplateTitle(), JSONObject.toJSONString(vo));
}
@Override
......@@ -87,17 +86,7 @@ public class TemplateTitleServiceImpl implements TemplateTitleService {
try {
// 防止增加模板标题有带符号的问题
templateTitle = Tools.filterSymbol(templateTitle);
String key = Tools.assembleKey(GenericAttribute.REDIS_MAP_KEY, group);
String mapKeyValue = redissonUtil.getMapKeyValue(key, templateTitle);
if (Tools.isEmpty(mapKeyValue)) {
return false;
}
TemplateTitleVo titleVo = JSONObject.parseObject(mapKeyValue, TemplateTitleVo.class);
String oldTag = titleVo.getMtag();
titleVo.setMtag(fixTag);
redissonUtil.setMapValue(key, templateTitle, JSONObject.toJSONString(titleVo));
log.info("修改模板标签成功: group:{} templateTitle:{} oldTag:{} fixTag:{}", group, templateTitle, oldTag,
fixTag);
setTemplateValue(group, templateTitle, fixTag);
return true;
} catch (Exception e) {
e.printStackTrace();
......@@ -110,9 +99,9 @@ public class TemplateTitleServiceImpl implements TemplateTitleService {
}
@Override
public boolean resetTemplate(String group, String templateTitle) {
public boolean resetTemplate(String group, String title) {
// 防止增加模板标题有带符号的问题
templateTitle = Tools.filterSymbol(templateTitle);
String templateTitle = Tools.filterSymbol(title);
Map<String, TemplateTitleVo> map = getTemplateTitleByProject(group);
if (Objects.isNull(map.get(templateTitle))) {
return false;
......@@ -121,7 +110,7 @@ public class TemplateTitleServiceImpl implements TemplateTitleService {
// 只要在运行中的模板 才能重置
if (Objects.isNull(templateTitleVo.getStatus()) || templateTitleVo.getStatus().name().equals(TemplateStatus.运行中.name())) {
templateTitleVo.setStatus(TemplateStatus.重置中);
setTemplateTitleByProject(group, map);
setTemplateValue(group, templateTitle, templateTitleVo);
// 修改模板的标注信息
executor.execute(() -> {
try {
......@@ -132,7 +121,7 @@ public class TemplateTitleServiceImpl implements TemplateTitleService {
templateTitleVo.setStatus(TemplateStatus.重置失败);
log.error("重置模板：修改聚和集错误，title:{}，以加入重试队列", templateTitleVo.getTemplateTitle());
} finally {
setTemplateTitleByProject(group, map);
setTemplateValue(group, templateTitle, templateTitleVo);
}
});
return true;
......@@ -140,6 +129,46 @@ public class TemplateTitleServiceImpl implements TemplateTitleService {
return false;
}
@Override
public void setTemplateValue(String group, String title, TemplateTitleVo templateTitleVo) {
setTemplateValue(group, title, templateTitleVo, null);
}
@Override
public void setTemplateValue(String group, String title, String fixTag) {
setTemplateValue(group, title, null, fixTag);
}
private void setTemplateValue(String group, String title, TemplateTitleVo templateTitleVo, String fixTag) {
String lockKey = Tools.assembleKey(GenericAttribute.LOCK, title);
try {
if (redissonUtil.tryLock(lockKey, 5, 5)) {
String mapKey = Tools.assembleKey(GenericAttribute.REDIS_MAP_KEY, group);
String mapKeyValue = redissonUtil.getMapKeyValue(mapKey, title);
TemplateTitleVo titleVo = JSONObject.parseObject(mapKeyValue, TemplateTitleVo.class);
if (Objects.isNull(titleVo)) {
return;
}
if (Objects.isNull(templateTitleVo)) {
templateTitleVo = titleVo;
}
if (Objects.isNull(fixTag)) {
templateTitleVo.accSum(titleVo.getMarkSum(), titleVo.getDaySum());
} else {
String oldTag = templateTitleVo.getMtag();
templateTitleVo.setMtag(fixTag);
log.info("修改模板标签成功: group:{} templateTitle:{} oldTag:{} fixTag:{}", group, title, oldTag,
fixTag);
}
redissonUtil.setMapValue(mapKey, title, JSONObject.toJSONString(templateTitleVo));
}
} catch (Exception e) {
log.error("修改模板失败，项目：{}，title:{}", group, title, e);
} finally {
redissonUtil.unlock(lockKey);
}
}
/**
* 模板自动标注的历史数据修改为新标签
* @param templateTitleVo 模板
......
middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/task/service/TaskServiceCommon.java
......@@ -159,6 +159,7 @@ public class TaskServiceCommon extends BaseTaskTypePair<TaskServiceCommon.TaskCo
for (Map.Entry<String, List<MarkInfo>> entry : groupMap.entrySet()) {
try {
Map<String, TemplateTitleVo> template = templateTitleService.getTemplateTitleByProject(entry.getKey());
clearTemplateDaySum(template);
asyncTitleMark(entry.getKey(), entry.getValue(), template);
} catch (Exception e) {
log.error("自动标注处理失败，项目：{}", entry.getKey(), e);
......@@ -166,6 +167,12 @@ public class TaskServiceCommon extends BaseTaskTypePair<TaskServiceCommon.TaskCo
}
}
private void clearTemplateDaySum(Map<String, TemplateTitleVo> template) {
for (Map.Entry<String, TemplateTitleVo> entry : template.entrySet()) {
entry.getValue().setDaySum(0L);
}
}
/**
* 异步等待
* @param group 项目
......@@ -195,6 +202,14 @@ public class TaskServiceCommon extends BaseTaskTypePair<TaskServiceCommon.TaskCo
String title = Tools.filterSymbol(markInfo.getSourceObj().getString(GenericAttribute.ES_TITLE));
return null != title && title.length() > 6;
}).filter(e -> autoMark(group, e, titleVoMap)).collect(Collectors.toList());
// 模板更新
for (Map.Entry<String, TemplateTitleVo> entry : titleVoMap.entrySet()) {
if (entry.getValue().getDaySum() == 0) {
continue;
}
templateTitleService.setTemplateValue(group, entry.getKey(), entry.getValue());
}
// 自动标注数据发送到标注中间件
// dubboHandler.markUpsert(newList);
}
......@@ -220,7 +235,7 @@ public class TaskServiceCommon extends BaseTaskTypePair<TaskServiceCommon.TaskCo
// 特征值记录
templateTitleService.insertTemplateRecord(new TemplateRecord(templateTitleVo.getId(), updates[0]));
// 模板标注count累加
redissonUtil.putCount(Tools.assembleKey(COUNT_KEY, group, templateTitleVo.getId()), 1);
// redissonUtil.putCount(Tools.assembleKey(COUNT_KEY, group, templateTitleVo.getId()), 1);
// 测试环境临时添加，用于对比
templateRecordDao.tempRecord(new TemplateTempRecord(templateTitleVo.getId(), templateTitleVo.getTemplateTitle(), group, getUrl(markInfo),
sourceObj.getString("_id"), templateTitleVo.getMtag()));
......
middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/task/service/TaskServiceTemplate.java
......@@ -195,7 +195,7 @@ public class TaskServiceTemplate extends BaseTaskTypePair<TaskServiceTemplate.Ta
return true;
}).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
// 新旧模板合并 且更新模板
templateTitleService.setTemplateTitleByProject(group, mergeTemplate(aggregation, templateTitleByProject));
mergeTemplate(group, aggregation, templateTitleByProject);
}
/**
......@@ -229,22 +229,23 @@ public class TaskServiceTemplate extends BaseTaskTypePair<TaskServiceTemplate.Ta
* @param oldTemplate 旧模板
* @param newTemplate 新模板
*/
private Map<String, TemplateTitleVo> mergeTemplate(Map<String, TemplateTitleVo> oldTemplate, Map<String, TemplateTitleVo> newTemplate) {
private void mergeTemplate(String group, Map<String, TemplateTitleVo> oldTemplate, Map<String, TemplateTitleVo> newTemplate) {
for (Map.Entry<String, TemplateTitleVo> newEntry : newTemplate.entrySet()) {
List<String> templateKeys = oldTemplate.keySet().stream()
.filter(e -> CosineSimilarity.calculateTextSimWithBrand(newEntry.getKey(), e) >= 0.96)
.collect(Collectors.toList());
// 添加模板
if (templateKeys.isEmpty()) {
oldTemplate.put(newEntry.getKey(), newEntry.getValue());
templateTitleService.setTemplateTitleByProject(group, newEntry.getValue());
} else {
// 更新标签
for (String oldKey : templateKeys) {
oldTemplate.get(oldKey).setMtag(newEntry.getValue().getMtag());
TemplateTitleVo templateTitleVo = oldTemplate.get(oldKey);
templateTitleVo.setMtag(newEntry.getValue().getMtag());
templateTitleService.setTemplateValue(group, oldKey, templateTitleVo);
}
}
}
return oldTemplate;
}
/**
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment