Commit 4e1bb0b5 by shentao

Merge branch 'release' into 'master'

Release

See merge request !8
parents 009e5701 f33fdd70
......@@ -36,15 +36,21 @@
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-1.2-api</artifactId>
<!-- <scope>provided</scope>-->
<scope>compile</scope>
<version>${log4j.version}</version>
<scope>provided</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-log4j12 -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<!-- <scope>provided</scope>-->
<scope>compile</scope>
<version>${slf4j.version}</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
......
......@@ -44,4 +44,6 @@ public class GenericAttribute {
public static final String FIX_TAG = "fixTag";
public static final String KEY = "task";
public static final String URL = "url";
}
......@@ -3,9 +3,9 @@ package com.zhiwei.middleware.automatic.server.pojo.enums;
public enum TaskType {
COMMON_ONE("common_one","common", "commonCache"),
COMMON_TWO("common_two","common", "commonCache"),
TEMPLATE("template", "template", ""),
TEMPLATE_MODIFY("template_modify","template", ""),
TEMPLATE_RESET("template_reset","template", "");
TEMPLATE("template", "template", null),
TEMPLATE_MODIFY("template_modify","template", null),
TEMPLATE_RESET("template_reset","template", null);
final String type;
final String name;
......
......@@ -101,6 +101,10 @@
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
<exclusion>
<artifactId>slf4j-api</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
</exclusions>
</dependency>
<!--日志整合-->
......@@ -108,16 +112,6 @@
<groupId>com.zhiwei</groupId>
<artifactId>push-log</artifactId>
<version>${push-log.version}</version>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-1.2-api</artifactId>
</exclusion>
</exclusions>
</dependency>
......@@ -161,26 +155,6 @@
<scope>provided</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.dubbo/dubbo-spring-boot-starter -->
<dependency>
<groupId>org.apache.dubbo</groupId>
<artifactId>dubbo-spring-boot-starter</artifactId>
<version>${dubbo-server.version}</version>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.dubbo</groupId>
<artifactId>dubbo</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.dubbo</groupId>
......
......@@ -13,7 +13,7 @@ public class Server {
try {
SpringApplication.run(Server.class, args);
} catch (Exception e) {
System.out.println(e.getMessage());
log.error("自动标注中间件server启动失败", e);
}
log.info("时间:,自动标注中间件启动成功");
}
......
......@@ -20,11 +20,11 @@ public class TemplateRecordDaoImpl implements TemplateRecordDao {
@Override
public List<TemplateRecord> findTemplateRecord(Query query) {
return mongoTemplate.find(query, TemplateRecord.class, "automaticmark_template_record");
return mongoTemplate.find(query, TemplateRecord.class, "automaticmark_template_record_new");
}
@Override
public long count(Query query) {
return mongoTemplate.count(query, TemplateRecord.class, "automaticmark_template_record");
return mongoTemplate.count(query, TemplateRecord.class, "automaticmark_template_record_new");
}
}
......@@ -4,7 +4,7 @@
<Configuration status="WARN">
<Properties> <!-- 配置日志文件输出目录 -->
<Property name="LOG_HOME">./log/</Property>
<property name="APP_NAME">automatic-center-server</property>
<property name="APP_NAME">middleware-automatic-center-server</property>
</Properties>
<Appenders>
<!-- 定义日志输出地 -->
......
......@@ -10,6 +10,7 @@
<modelVersion>4.0.0</modelVersion>
<artifactId>middleware-automatic-center-son</artifactId>
<version>1.0-SNAPSHOT</version>
<properties>
<maven.compiler.source>8</maven.compiler.source>
......@@ -33,27 +34,6 @@
<dependencies>
<!-- https://mvnrepository.com/artifact/org.apache.dubbo/dubbo-spring-boot-starter -->
<dependency>
<groupId>org.apache.dubbo</groupId>
<artifactId>dubbo-spring-boot-starter</artifactId>
<version>${dubbo-server.version}</version>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.dubbo</groupId>
<artifactId>dubbo</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.dubbo</groupId>
<artifactId>dubbo</artifactId>
......@@ -93,6 +73,26 @@
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-1.2-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.dubbo</groupId>
<artifactId>dubbo</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.dubbo</groupId>
<artifactId>dubbo-spring-boot-starter</artifactId>
</exclusion>
</exclusions>
</dependency>
......@@ -172,6 +172,12 @@
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>${es.version}</version>
<exclusions>
<exclusion>
<artifactId>log4j-api</artifactId>
<groupId>org.apache.logging.log4j</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
......@@ -210,6 +216,14 @@
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<!--日志整合-->
......@@ -217,16 +231,6 @@
<groupId>com.zhiwei</groupId>
<artifactId>push-log</artifactId>
<version>${push-log.version}</version>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-1.2-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
......
......@@ -20,21 +20,21 @@ public class TemplateRecordDaoImpl implements TemplateRecordDao {
@Override
public List<TemplateRecord> findTemplateRecord(Query query) {
return mongoTemplate.find(query, TemplateRecord.class, "automaticmark_template_record");
return mongoTemplate.find(query, TemplateRecord.class, "automaticmark_template_record_new");
}
@Override
public void insertTemplateRecord(TemplateRecord templateRecord) {
mongoTemplate.insert(templateRecord, "automaticmark_template_record");
mongoTemplate.insert(templateRecord, "automaticmark_template_record_new");
}
@Override
public long count(Query query) {
return mongoTemplate.count(query, TemplateRecord.class, "automaticmark_template_record");
return mongoTemplate.count(query, TemplateRecord.class, "automaticmark_template_record_new");
}
@Override
public void removeTemplateRecord(Query query) {
mongoTemplate.remove(query, TemplateRecord.class, "automaticmark_template_record");
mongoTemplate.remove(query, TemplateRecord.class, "automaticmark_template_record_new");
}
}
......@@ -49,23 +49,13 @@ public class TaskManager implements ApplicationRunner {
public void pullTask() {
try {
List<AutoTask> tasks = redissonUtil.pullQueue(GenericAttribute.KEY, LIMIT)
.stream()
.map(e -> JSONObject.parseObject(e).toJavaObject(AutoTask.class))
.collect(Collectors.toList());
tasks.forEach(e -> {
TaskType taskType = Objects.requireNonNull(TaskType.create(e.getType()));
TaskService taskService = TaskServiceHandler.getInstance()
.getTaskService(taskType.getName());
if (taskService.thresholdWarn()) {
log.error("任务类型:{},当前运行任务已到达最大核心数", taskService.getTaskType());
for (String s : redissonUtil.pullQueue(GenericAttribute.KEY, LIMIT)) {
AutoTask autoTask = JSONObject.parseObject(s).toJavaObject(AutoTask.class);
String cacheId = TaskServiceHandler.getInstance().taskExecute(autoTask);
if (!Strings.isEmpty(cacheId)) {
redissonUtil.deleteList(autoTask.getParamSource().getString(cacheId));
}
log.info("任务类型:{},开始执行,信息:{}", taskService.getTaskType(), JSONObject.toJSONString(e));
taskService.runTask(e);
if (!Strings.isEmpty(taskType.getCacheId())) {
redissonUtil.deleteList(e.getParamSource().getString(taskType.getCacheId()));
}
});
} catch (Exception e) {
log.error("任务管理器,任务执行失败:", e);
}
......
package com.zhiwei.middleware.automatic.son.task.holder;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.middleware.automatic.server.pojo.AutoTask;
import com.zhiwei.middleware.automatic.server.pojo.enums.TaskType;
import com.zhiwei.middleware.automatic.son.task.service.TaskService;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.context.ApplicationContext;
import java.util.HashMap;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
public class TaskServiceHandler {
private static final Map<String, TaskService> SERVICE_MAP = new HashMap<>();
private static final Logger log = LogManager.getLogger(TaskServiceHandler.class);
private static final List<TaskService> SERVICE_LIST = new ArrayList<>();
private TaskServiceHandler() {
ApplicationContext applicationContext = ApplicationContextHolder.getInstance();
Map<String, TaskService> beansOfType = applicationContext.getBeansOfType(TaskService.class);
beansOfType.forEach((k, v) -> SERVICE_MAP.put(v.getTaskType(), v));
beansOfType.forEach((k, v) -> SERVICE_LIST.add(v));
}
public static TaskServiceHandler getInstance() {
return TaskServiceHandlerHolder.TASK_SERVICE_HANDLER;
}
public TaskService getTaskService(String type) {
return SERVICE_MAP.get(type);
public String taskExecute(AutoTask autoTask) {
for (TaskService taskService : SERVICE_LIST) {
TaskType taskType = taskService.supports(autoTask.getType());
if (Objects.nonNull(taskType)) {
if (taskService.thresholdWarn()) {
log.error("任务类型:{},当前运行任务已到达最大核心数", autoTask.getType());
}
log.info("任务类型:{},开始执行,信息:{}", autoTask.getType(), JSONObject.toJSONString(autoTask));
taskService.runTask(autoTask);
return taskType.getCacheId();
}
}
return null;
}
private static class TaskServiceHandlerHolder {
......
package com.zhiwei.middleware.automatic.son.task.service;
import com.zhiwei.middleware.automatic.server.pojo.AutoTask;
import com.zhiwei.middleware.automatic.server.pojo.enums.TaskType;
public interface TaskService {
/**
* 回去任务名字
* 任务类型匹配
* @return 名字
*/
String getTaskType();
TaskType supports(String type);
/**
* 任务运行
......
......@@ -16,6 +16,7 @@ import com.zhiwei.middleware.automatic.son.util.CosineSimilarity;
import com.zhiwei.middleware.automatic.son.util.MarkInfoUtil;
import com.zhiwei.middleware.automatic.son.util.RedissonUtil;
import com.zhiwei.middleware.automatic.son.util.Tools;
import javafx.util.Pair;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Qualifier;
......@@ -31,6 +32,8 @@ public class TaskServiceCommon implements TaskService {
private static final Logger log = LogManager.getLogger(TaskServiceCommon.class);
private static final Map<String, Pair<TaskType, TaskCommonFunctional>> TYPE_MAP = new HashMap<>();
private final RedissonUtil redissonUtil;
private final TemplateTitleService templateTitleService;
......@@ -48,16 +51,20 @@ public class TaskServiceCommon implements TaskService {
this.templateTitleService = templateTitleService;
this.dubboHandler = dubboHandler;
this.autoMarkExecutor = autoMarkExecutor;
TYPE_MAP.put(TaskType.COMMON_TWO.getType(), new Pair<>(TaskType.COMMON_TWO, this::getMultiAutoInfo));
TYPE_MAP.put(TaskType.COMMON_ONE.getType(), new Pair<>(TaskType.COMMON_ONE, this::getOneAutoInfo));
}
@Override
public String getTaskType() {
return TaskType.COMMON_ONE.getName();
public TaskType supports(String type) {
Pair<TaskType, TaskCommonFunctional> pair = TYPE_MAP.get(type);
return Objects.isNull(pair) ? null : pair.getKey();
}
@Override
public void runTask(AutoTask autoTask) {
Map<String, List<MarkInfo>> dataSourceInfo = getDataSourceInfo(autoTask);
Pair<TaskType, TaskCommonFunctional> pair = Objects.requireNonNull(TYPE_MAP.get(autoTask.getType()));
Map<String, List<MarkInfo>> dataSourceInfo = pair.getValue().getDataSource(autoTask.getParamSource().getString(pair.getKey().getCacheId()));
if (dataSourceInfo.isEmpty()) {
log.error("本地任务可用数据为空,taskType:{}", autoTask.getType());
}
......@@ -70,23 +77,6 @@ public class TaskServiceCommon implements TaskService {
}
/**
* 获取源数据
* @param autoTask 任务
* @return 源数据按项目分组
*/
private Map<String, List<MarkInfo>> getDataSourceInfo(AutoTask autoTask) {
TaskType taskType = TaskType.create(autoTask.getType());
switch (Objects.requireNonNull(taskType)) {
case COMMON_ONE:
return getOneAutoInfo(autoTask.getParamSource().getString(taskType.getCacheId()));
case COMMON_TWO:
return getMultiAutoInfo(autoTask.getParamSource().getString(taskType.getCacheId()));
default:
return new HashMap<>();
}
}
/**
* 获取单个项目标注源数据
* @param key redis缓存key
* @return 源数据按项目分组
......@@ -206,7 +196,7 @@ public class TaskServiceCommon implements TaskService {
sourceObj.put(GenericAttribute.ES_M_TAG, aggreTag);
sourceObj.put(GenericAttribute.ES_M_PERSON, "自动化机器人");
sourceObj.put(GenericAttribute.ES_M_TIME, new Date().getTime());
log.info("模板标题:{} MarkSum:{} Tag:{}被标注标题:{}相似度:{}", aggreTitle, templateTitleVo.getMarkSum(), aggreTag,
log.info("项目:{} 模板标题:{} MarkSum:{} Tag:{}被标注标题:{}相似度:{}", group, aggreTitle, templateTitleVo.getMarkSum(), aggreTag,
title, similarMap.get("similar"));
// 刷新一下标注量和标注时间,
......@@ -265,4 +255,10 @@ public class TaskServiceCommon implements TaskService {
return null;
}
@FunctionalInterface
public interface TaskCommonFunctional {
Map<String, List<MarkInfo>> getDataSource(String group);
}
}
......@@ -15,6 +15,7 @@ import com.zhiwei.middleware.automatic.son.util.CosineSimilarity;
import com.zhiwei.middleware.automatic.son.util.Tools;
import com.zhiwei.nlp.AggreeBootStarter;
import com.zhiwei.nlp.vo.KResult;
import javafx.util.Pair;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.index.query.BoolQueryBuilder;
......@@ -27,10 +28,7 @@ import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.util.Calendar;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
......@@ -39,6 +37,8 @@ public class TaskServiceTemplate implements TaskService {
private final Logger log = LogManager.getLogger(TaskServiceTemplate.class);
private static final Map<String, Pair<TaskType, TaskTemplateFunctional>> TYPE_MAP = new HashMap<>();
private final TemplateTitleService templateTitleService;
private final EsDao esDao;
......@@ -57,16 +57,23 @@ public class TaskServiceTemplate implements TaskService {
this.esDao = esDao;
this.esIndexes = esIndexes;
this.executor = executor;
TYPE_MAP.put(TaskType.TEMPLATE.getType(), new Pair<>(TaskType.TEMPLATE, this::runTask));
TYPE_MAP.put(TaskType.TEMPLATE_MODIFY.getType(), new Pair<>(TaskType.TEMPLATE_MODIFY, this::templateModify));
TYPE_MAP.put(TaskType.TEMPLATE_RESET.getType(), new Pair<>(TaskType.TEMPLATE_RESET, this::templateReset));
}
@Override
public String getTaskType() {
return TaskType.TEMPLATE.getName();
public TaskType supports(String type) {
Pair<TaskType, TaskTemplateFunctional> par = TYPE_MAP.get(type);
return Objects.isNull(par) ? null : par.getKey();
}
@Override
public void runTask(AutoTask autoTask) {
executor.execute(() -> switchTask(autoTask));
executor.execute(() -> {
Pair<TaskType, TaskTemplateFunctional> pair = Objects.requireNonNull(TYPE_MAP.get(autoTask.getType()));
pair.getValue().template(autoTask.getParamSource());
});
}
@Override
......@@ -75,27 +82,35 @@ public class TaskServiceTemplate implements TaskService {
return false;
}
private void switchTask (AutoTask autoTask) {
JSONObject paramSource = autoTask.getParamSource();
String group = paramSource.getString(GenericAttribute.GROUP_PARAM);
switch (Objects.requireNonNull(TaskType.create(autoTask.getType()))) {
case TEMPLATE_RESET:
templateTitleService.resetTemplate(group, paramSource.getString(GenericAttribute.TEMPLATE_TITLE));
break;
case TEMPLATE_MODIFY:
templateTitleService.modifyTemplateTitle(group, paramSource.getString(GenericAttribute.TEMPLATE_TITLE),
paramSource.getString(GenericAttribute.FIX_TAG));
break;
case TEMPLATE:
runTask(group, paramSource.getLong(GenericAttribute.START_PARAM), paramSource.getLong(GenericAttribute.END_PARAM));
break;
/**
* 模板修改任务
* @param json 参数
*/
private void templateModify(JSONObject json) {
templateTitleService.modifyTemplateTitle(json.getString(GenericAttribute.GROUP_PARAM), json.getString(GenericAttribute.TEMPLATE_TITLE),
json.getString(GenericAttribute.FIX_TAG));
}
/**
* 模板重置任务
* @param json 参数
*/
private void templateReset(JSONObject json) {
templateTitleService.resetTemplate(json.getString(GenericAttribute.GROUP_PARAM), json.getString(GenericAttribute.TEMPLATE_TITLE));
}
private void runTask(String group, Long startTime, Long endTime) {
/**
* 模板构建任务
* @param json 参数
*/
private void runTask(JSONObject json) {
String group = json.getString(GenericAttribute.GROUP_PARAM);
try {
Long startTime = json.getLong(GenericAttribute.START_PARAM);
Long endTime = json.getLong(GenericAttribute.END_PARAM);
//源数据
List<Map<String, Object>> sourceList = findRecentTimeData(group,startTime,
List<Map<String, Object>> sourceList = findRecentTimeData(group, startTime,
endTime);
if (sourceList.isEmpty()) {
return;
......@@ -230,4 +245,9 @@ public class TaskServiceTemplate implements TaskService {
private QueryBuilder autoRobotQueryBuilder() {
return QueryBuilders.termQuery("mperson", GenericAttribute.AUTO_PERSON);
}
@FunctionalInterface
public interface TaskTemplateFunctional {
void template(JSONObject json);
}
}
......@@ -4,7 +4,7 @@
<Configuration status="WARN">
<Properties> <!-- 配置日志文件输出目录 -->
<Property name="LOG_HOME">./log/</Property>
<property name="APP_NAME">automatic-center-server</property>
<property name="APP_NAME">middleware-automatic-center-son</property>
</Properties>
<Appenders>
<!-- 定义日志输出地 -->
......
prod.robot.push.address=https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=e2218c6e-af6a-4296-9d75-7178b941a3b5
prod.robot.push.enable=false
prod.robot.push.enable=true
prod.robot.push.filterclass=org.apache.dubbo.common.Version,com.alibaba.dubbo.common.Version,org.apache.dubbo.monitor.dubbo.DubboMonitor,com.alibaba.dubbo.monitor.dubbo.DubboMonitor
prod.robot.push.level=error
prod.robot.push.app.name=automatic-server-prod
......
......@@ -52,27 +52,7 @@
</exclusions>
</dependency>
<!-- 日志依赖 -->
<!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-1.2-api -->
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-1.2-api</artifactId>
<version>${log4j.version}</version>
<scope>provided</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-log4j12 -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>${slf4j.version}</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</dependencyManagement>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment