Skip to content

B
brandkbs2
Commit c80d9698 by shenjunjie
Options

Merge branch 'feature' into 'dev' 

Feature

See merge request !461
parents b1bcbde8 41c52dad
Showing with 171 additions and 21 deletions
pom.xml
......@@ -35,7 +35,7 @@
<dubbo.version>2.7.4.1</dubbo.version>
<curator.version>2.12.0</curator.version>
<push.log.version>2.17.0-SNAPSHOT</push.log.version>
<event-client.version>1.0.9.6-SNAPSHOT</event-client.version>
<event-client.version>1.0.9.8-SNAPSHOT</event-client.version>
<cleaner-unified-filter.version>1.1.5-SNAPSHOT</cleaner-unified-filter.version>
</properties>
......
src/main/java/com/zhiwei/brandkbs2/common/RedisKeyPrefix.java
......@@ -56,6 +56,7 @@ public class RedisKeyPrefix {
public static final String CUSTOM_EVENT_LIST = "BRANDKBS:CUSTOM_EVENT:LIST:";
public static final String CUSTOM_EVENT_ANALYZE_SHARE = "BRANDKBS:CUSTOM_EVENT:SHARE:";
public static final String CUSTOM_EVENT_ANALYZE = "BRANDKBS:CUSTOM_EVENT:ANALYZE:";
public static final String EVENT_AGG_TITLE_LIST = "BRANDKBS:EVENT:AGG_TITLE:LIST:";
/**
* 热点相关缓存KEY
......
src/main/java/com/zhiwei/brandkbs2/controller/app/AppArticleController.java
......@@ -57,7 +57,7 @@ public class AppArticleController extends BaseController {
@ApiOperation("原始数据列表")
@PostMapping("/origin/list")
@LogRecord(description = "舆情库-原始数据列表")
@LogRecord(description = "舆情库-原始数据")
public ResponseResult getOriginList(@RequestBody MarkSearchDTO markSearchDTO) {
return ResponseResult.success(markDataService.getOriginList(markSearchDTO));
}
......@@ -117,14 +117,14 @@ public class AppArticleController extends BaseController {
@ApiOperation("舆情列表")
@PostMapping("/mark/list")
@LogRecord(description = "舆情库-舆情列表")
@LogRecord(description = "舆情库-舆情数据")
public ResponseResult getYuqingMarkList(@RequestBody MarkSearchDTO markSearchDTO) {
return ResponseResult.success(markDataService.getYuqingMarkList(markSearchDTO));
}
@ApiOperation("舆情列表-生成聚合")
@PostMapping("/mark/aggree")
@LogRecord(description = "舆情库-舆情列表-聚合")
@LogRecord(description = "舆情库-有效舆情-聚合")
public ResponseResult generateYuqingMarkAggreeList(@RequestBody JSONObject info) {
Long startTime = info.getLong("startTime");
Long endTime = info.getLong("endTime");
......@@ -188,7 +188,7 @@ public class AppArticleController extends BaseController {
@ApiImplicitParam(name = "startTime", value = "开始时间", paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long")
})
@LogRecord(description = "舆情库-舆情分析")
// @LogRecord(description = "舆情库-舆情分析")
@GetMapping("/emotion/proportion")
public ResponseResult getMarkEmotionProportion(@RequestParam(value = "startTime", required = false) Long startTime,
@RequestParam(value = "endTime", required = false) Long endTime) {
......@@ -448,6 +448,7 @@ public class AppArticleController extends BaseController {
@ApiOperation("新-舆情分析-舆情总量")
@GetMapping("/analyze/amount")
@LogRecord(description = "舆情库-舆情分析")
public ResponseResult getYuqingAmount(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "planId", required = false) String planId) {
......
src/main/java/com/zhiwei/brandkbs2/controller/app/AppEventController.java
......@@ -4,13 +4,11 @@ import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.aop.LogRecord;
import com.zhiwei.brandkbs2.auth.Auth;
import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.controller.BaseController;
import com.zhiwei.brandkbs2.enmus.RoleEnum;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.dto.EventSearchDTO;
import com.zhiwei.brandkbs2.service.EventService;
import com.zhiwei.middleware.event.core.EventMonitorClient;
import com.zhiwei.middleware.event.pojo.dto.BrandkbsEventSearchDTO;
import com.zhiwei.middleware.event.pojo.dto.EventDTO;
import io.swagger.annotations.Api;
......@@ -142,7 +140,7 @@ public class AppEventController extends BaseController {
}
@ApiOperation("前台事件库-全网事件库-行业热点-列表")
@LogRecord(description = "事件库-全网事件库-行业热点")
@LogRecord(description = "事件库-全网事件库")
@GetMapping("/getWholeNetworkEventsList")
public ResponseResult getWholeNetworkEventsList(@RequestParam(value = "firstType", required = false, defaultValue = "") String firstType,
@RequestParam(value = "start", required = false, defaultValue = "0") long start,
......@@ -187,7 +185,7 @@ public class AppEventController extends BaseController {
}
@ApiOperation("前台事件库-全网事件库-企业危机-危机库列表")
@LogRecord(description = "事件库-全网事件库-企业危机")
@LogRecord(description = "事件库-危机案例库")
@GetMapping("/crisisList")
public ResponseResult crisisList(@RequestParam(value = "page",defaultValue = "1") Integer page,
@RequestParam(value = "pageSize",defaultValue = "10") Integer pageSize,
......@@ -221,7 +219,7 @@ public class AppEventController extends BaseController {
}
@ApiOperation("前台事件库-品牌事件库-获取品牌事件库事件列表")
@LogRecord(description = "事件库-品牌事件库")
@LogRecord(description = "事件库-品牌事件库-事件中心")
@PostMapping("/eventList")
public ResponseResult getEventListMiddleware(@RequestBody BrandkbsEventSearchDTO dto){
return ResponseResult.success(eventService.getEventListMiddleware(dto));
......@@ -234,7 +232,7 @@ public class AppEventController extends BaseController {
}
@ApiOperation("前台事件库-事件监测-获取事件监测列表")
@LogRecord(description = "事件库-事件分析")
@LogRecord(description = "事件库-事件监测")
@GetMapping("/eventMonitor")
public ResponseResult getEventMonitor(@RequestParam(value = "page", defaultValue = "1") Integer page,
@RequestParam(value = "pageSize", defaultValue = "10") Integer pageSize){
......@@ -280,7 +278,7 @@ public class AppEventController extends BaseController {
@ApiOperation("前台事件库-事件详情-获取事件基础信息")
@GetMapping("/detail/baseInfo")
@LogRecord(values = "title", description = "事件库-品牌事件库", arguments = false, entity = true)
@LogRecord(values = "title", description = "事件库-品牌事件库-事件中心", arguments = false, entity = true)
@Auth(role = RoleEnum.NO_AUTHORISE)
public ResponseResult eventBaseInfo(@RequestParam(value = "eventId") String eventId){
return ResponseResult.success(eventService.eventBaseInfo(eventId));
......@@ -343,4 +341,18 @@ public class AppEventController extends BaseController {
List<String> eventIds = json.getJSONArray("eventIds").toJavaList(String.class);
return ResponseResult.success(eventService.scheduleEventStatus(eventIds));
}
@ApiOperation("前台事件库-品牌事件库标题聚合分析")
@GetMapping("/analyze/aggregate-titles")
@LogRecord(description = "事件库-品牌事件库-标题聚合分析")
public ResponseResult getLastNews(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime) {
return ResponseResult.success(eventService.getEventAggTitleList(startTime, endTime, true));
}
@ApiOperation("前台事件库-品牌事件库-分析")
@PostMapping("/analyze")
public ResponseResult aggTitleAnalyze(@RequestBody EventDTO dto){
return eventService.aggTitleAnalyze(dto);
}
}
src/main/java/com/zhiwei/brandkbs2/dao/EventMiddlewareDao.java
......@@ -338,4 +338,10 @@ public interface EventMiddlewareDao {
* @return
*/
ReturnData scheduleEventStatus(List<String> eventIds);
/**
* 添加新事件
* @param eventDTO 事件传输类
*/
ReturnData addEvent(EventDTO eventDTO);
}
src/main/java/com/zhiwei/brandkbs2/dao/impl/EventMiddlewareDaoImpl.java
......@@ -231,4 +231,9 @@ public class EventMiddlewareDaoImpl implements EventMiddlewareDao {
public ReturnData scheduleEventStatus(List<String> eventIds) {
return eventMonitorClient.scheduleEventStatus(eventIds);
}
@Override
public ReturnData addEvent(EventDTO eventDTO) {
return eventClient.addEvent(eventDTO);
}
}
src/main/java/com/zhiwei/brandkbs2/service/EventService.java
......@@ -4,6 +4,7 @@ import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportEventDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportEventDataDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.UploadEventDTO;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.Event;
import com.zhiwei.brandkbs2.pojo.EventDisseminationTrend;
import com.zhiwei.brandkbs2.pojo.dto.EventDataDTO;
......@@ -436,4 +437,17 @@ public interface EventService {
* @return
*/
JSONObject scheduleEventStatus(List<String> eventIds);
/**
* 品牌事件库-聚合标题列表
* @param startTime
* @param endTime
* @return
*/
List<JSONObject> getEventAggTitleList(Long startTime, Long endTime, boolean cache);
/**
* 品牌事件库-标题聚合分析-分析
*/
ResponseResult aggTitleAnalyze(EventDTO eventDTO);
}
src/main/java/com/zhiwei/brandkbs2/service/MarkFlowService.java
......@@ -3,6 +3,7 @@ package com.zhiwei.brandkbs2.service;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.base.entity.subclass.mark.MarkInfo;
import com.zhiwei.brandkbs2.pojo.MarkFlowEntity;
import com.zhiwei.brandkbs2.pojo.dto.TagFilterDTO;
import lombok.AllArgsConstructor;
import lombok.Data;
......@@ -83,10 +84,13 @@ public interface MarkFlowService {
private String planId;
private Long gid;
public MarkInfoSource(JSONObject json, String projectId, String contendId) {
private List<TagFilterDTO> tagFilters;
public MarkInfoSource(JSONObject json, String projectId, String contendId, List<TagFilterDTO> tagFilters) {
this.json = json;
this.projectId = projectId;
this.contendId = contendId;
this.tagFilters = tagFilters;
}
public MarkInfoSource(JSONObject json, String linkedGroupId) {
......
src/main/java/com/zhiwei/brandkbs2/service/TaskService.java
......@@ -29,6 +29,11 @@ public interface TaskService{
void customEventCache();
/**
* 事件标题聚合分析缓存
*/
void eventAggTitleCache();
/**
* 生成简报任务并推送
*/
void generateReportAndSend();
......
src/main/java/com/zhiwei/brandkbs2/service/impl/EventServiceImpl.java
package com.zhiwei.brandkbs2.service.impl;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
......@@ -23,15 +24,13 @@ import com.zhiwei.brandkbs2.enmus.response.EventCodeEnum;
import com.zhiwei.brandkbs2.exception.ExceptionCast;
import com.zhiwei.brandkbs2.listener.ApplicationProjectListener;
import com.zhiwei.brandkbs2.model.CommonCodeEnum;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.*;
import com.zhiwei.brandkbs2.pojo.dto.EventDataDTO;
import com.zhiwei.brandkbs2.pojo.dto.EventSearchDTO;
import com.zhiwei.brandkbs2.pojo.dto.YqEventDTO;
import com.zhiwei.brandkbs2.pojo.vo.*;
import com.zhiwei.brandkbs2.service.EventDataService;
import com.zhiwei.brandkbs2.service.EventService;
import com.zhiwei.brandkbs2.service.ProjectService;
import com.zhiwei.brandkbs2.service.UserService;
import com.zhiwei.brandkbs2.service.*;
import com.zhiwei.brandkbs2.util.MongoUtil;
import com.zhiwei.brandkbs2.util.RedisUtil;
import com.zhiwei.brandkbs2.util.Tools;
......@@ -115,6 +114,9 @@ public class EventServiceImpl implements EventService {
@Resource(name = "projectServiceImpl")
private ProjectService projectService;
@Resource(name = "markDataServiceImpl")
MarkDataService markDataService;
@Resource(name = "userServiceImpl")
private UserService userService;
......@@ -893,6 +895,57 @@ public class EventServiceImpl implements EventService {
return (JSONObject) eventMiddlewareDao.scheduleEventStatus(eventIds).getData();
}
@Override
public List<JSONObject> getEventAggTitleList(Long startTime, Long endTime, boolean cache) {
String projectId = UserThreadLocal.getProjectId();
String redisKey = RedisUtil.getEventAggTitleListKEY(projectId, startTime, endTime);
String resultStr = redisUtil.get(redisKey);
if (cache && StringUtils.isNotEmpty(resultStr)) {
return JSON.parseArray(resultStr, JSONObject.class);
}
List<JSONObject> resultList = markDataService.getLastNews(startTime, endTime, null, 50, true);
redisUtil.setExpire(redisKey, JSON.toJSONString(resultList));
return resultList;
}
@Override
public ResponseResult aggTitleAnalyze(EventDTO dto) {
long now = System.currentTimeMillis();
if (now < dto.getEndTime() && getMonitoringEventsCount() >= 5){
return ResponseResult.failure("结束时间为未来时间判断为监测事件，监测事件数量不得超过5");
}
Project project = projectService.getProjectById(UserThreadLocal.getProjectId());
// submitter submitterId
ServletRequestAttributes requestAttributes = (ServletRequestAttributes) RequestContextHolder.getRequestAttributes();
HttpServletRequest request = Objects.requireNonNull(requestAttributes).getRequest();
String submitterId = JwtUtil.unsign(request.getHeader(jwtKey), Map.class).get(GenericAttribute.USER_ID).toString();
String submitter = userService.queryUserInfo(submitterId, project.getId()).getNickname();
// 事件基础信息
dto.setProject(project.getBrandLinkedGroup());
dto.setProjectId(project.getBrandLinkedGroupId());
dto.setSubmitter(submitter);
dto.setSubmitterId(submitterId);
// 数据源
dto.setDataSource(project.getBrandLinkedGroup());
// 品见关联信息
List<BrandkbsBasicInfo> brandkbsBasicInfos = new ArrayList<>(1);
BrandkbsBasicInfo brandkbsBasicInfo = new BrandkbsBasicInfo();
brandkbsBasicInfo.setBrandkbsProject(project.getProjectName());
brandkbsBasicInfo.setBrandkbsProjectId(project.getId());
brandkbsBasicInfo.setBrandkbsBrand(project.getBrandName());
// 主品牌品牌id使用项目id
brandkbsBasicInfo.setBrandkbsBrandId(project.getId());
brandkbsBasicInfos.add(brandkbsBasicInfo);
dto.setBrandkbsInfos(brandkbsBasicInfos);
// 结束时间大于当前时间为监测事件
if (now < dto.getEndTime()) {
eventMiddlewareDao.addEventMonitor(dto);
return ResponseResult.success();
}
eventMiddlewareDao.addEvent(dto);
return ResponseResult.success();
}
@Data
public static class MongoQueryUtil {
......
src/main/java/com/zhiwei/brandkbs2/service/impl/MarkFlowServiceImpl.java
......@@ -15,7 +15,9 @@ import com.zhiwei.brandkbs2.enmus.ChannelEmotion;
import com.zhiwei.brandkbs2.exception.ExceptionCast;
import com.zhiwei.brandkbs2.model.CommonCodeEnum;
import com.zhiwei.brandkbs2.pojo.MarkFlowEntity;
import com.zhiwei.brandkbs2.pojo.dto.TagFilterDTO;
import com.zhiwei.brandkbs2.service.MarkFlowService;
import com.zhiwei.brandkbs2.service.TagFilterService;
import com.zhiwei.brandkbs2.util.RedisUtil;
import com.zhiwei.brandkbs2.util.Tools;
import com.zhiwei.middleware.automaticmark.vo.Keyword;
......@@ -24,6 +26,7 @@ import com.zhiwei.middleware.mark.service.MarkerClient;
import com.zhiwei.middleware.mark.vo.QueryResult;
import com.zhiwei.qbjc.bean.pojo.common.Tag;
import com.zhiwei.qbjc.bean.tools.BeanTools;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.stereotype.Service;
......@@ -59,13 +62,11 @@ public class MarkFlowServiceImpl implements MarkFlowService {
@Resource(name = "filterClient")
UnifiedFilterClient filterClient;
@Resource(name = "highlightWordDao")
HighlightWordDao highlightWordDao;
@Override
public JSONObject createMarkFlowInfo(MarkInfoSource markInfoSource) {
JSONObject resultInfo = createInfoWithEmotion(markInfoSource.getJson());
resultInfo.put("sourceDetails", getSourceDetails(markInfoSource));
resultInfo.put("customTags", getCustomTagsInfo(markInfoSource));
return resultInfo;
}
......@@ -196,6 +197,34 @@ public class MarkFlowServiceImpl implements MarkFlowService {
return JSON.parseObject(data, MarkFlowEntity.class);
}
/**
* 获取自定义标签
* @param markInfoSource
* @return
*/
private List<JSONObject> getCustomTagsInfo(MarkInfoSource markInfoSource){
List<JSONObject> res = new ArrayList<>();
JSONObject json = markInfoSource.getJson();
List<JSONObject> markCacheMaps = json.getJSONArray(GenericAttribute.ES_MARK_CACHE_MAPS).toJavaList(JSONObject.class);
if (CollectionUtils.isEmpty(markCacheMaps)){
return null;
}
// 已配置的标签筛选
List<TagFilterDTO> tagFilters = markInfoSource.getTagFilters();
List<String> groupNames = tagFilters.stream().map(TagFilterDTO::getGroupName).collect(Collectors.toList());
for (JSONObject markCacheMap : markCacheMaps) {
String groupName = markCacheMap.getString("group_name");
if (!groupNames.contains(groupName) || Objects.equals(Constant.EMOTION_LABEL_KEY, groupName)){
continue;
}
JSONObject jsonObject = new JSONObject();
jsonObject.put("uniqueId", markCacheMap.getString("unique_id"));
jsonObject.put("name", markCacheMap.getString("name"));
res.add(jsonObject);
}
return res;
}
private JSONObject createInfoWithEmotion(JSONObject json) {
JSONObject info = new JSONObject();
String emotion = Tools.getEmotion(json);
......
src/main/java/com/zhiwei/brandkbs2/service/impl/TaskServiceImpl.java
......@@ -210,6 +210,21 @@ public class TaskServiceImpl implements TaskService {
}, cacheServiceExecutor)).toArray(CompletableFuture[]::new)).join();
}
@Override
public void eventAggTitleCache() {
AtomicInteger total = new AtomicInteger();
List<Long[]> timeList = Arrays.asList(commonService.getTimeRangeDay(), commonService.getTimeRangeFormatWeek(), commonService.getTimeRangeFormatMonth());
CompletableFuture.allOf(GlobalPojo.PROJECT_MAP.values().stream().map(project -> CompletableFuture.supplyAsync(() -> {
UserThreadLocal.set(new UserInfo().setProjectId(project.getId()));
timeList.forEach(times -> {
// 事件标题聚合缓存
eventService.getEventAggTitleList(times[0], times[1], false);
});
log.info("项目:{}-事件标题聚合缓存已完成:{}个", project.getProjectName(), total.incrementAndGet());
return null;
}, cacheServiceExecutor)).toArray(CompletableFuture[]::new)).join();
}
private Pair<List<Channel>, List<Channel>> batchHandle(List<Map.Entry<ChannelIndex, ChannelIndex.Record>> batchList, Map<ChannelIndex,
ChannelIndex.Record> newRecordMap) {
List<Channel> insertList = Collections.synchronizedList(new ArrayList<>());
......
src/main/java/com/zhiwei/brandkbs2/task/ControlCenter.java
......@@ -44,7 +44,8 @@ public class ControlCenter {
log.info("定时按天缓存数据-启动");
try {
taskService.messageFlowCache();
taskService.customEventCache();
// taskService.customEventCache();
taskService.eventAggTitleCache();
taskService.yuqingAnalyzeHighWordCache();
} catch (Exception e) {
log.error("定时按天缓存数据-出错", e);
......
src/main/java/com/zhiwei/brandkbs2/util/RedisUtil.java
......@@ -27,6 +27,10 @@ public class RedisUtil {
return RedisKeyPrefix.CUSTOM_EVENT_LIST + Tools.concat(projectId, startTimeStr, endTimeStr);
}
public static String getEventAggTitleListKEY(String projectId, Long startTimeStr, Long endTimeStr) {
return RedisKeyPrefix.EVENT_AGG_TITLE_LIST + Tools.concat(projectId, startTimeStr, endTimeStr);
}
public static String getShotPageKey(String id, String projectId) {
return RedisKeyPrefix.REDIS_SYSTEM_COPY + "SHOT_PAGE:" + projectId + "-" + id;
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment