Commit 5fce5dd7 by shenjunjie

Merge branch 'release' into 'master'

Release

See merge request !458
parents 9baefda2 d5130f37
......@@ -35,7 +35,7 @@
<dubbo.version>2.7.4.1</dubbo.version>
<curator.version>2.12.0</curator.version>
<push.log.version>2.17.0-SNAPSHOT</push.log.version>
<event-client.version>1.0.9.6-SNAPSHOT</event-client.version>
<event-client.version>1.0.9.8-SNAPSHOT</event-client.version>
<cleaner-unified-filter.version>1.1.5-SNAPSHOT</cleaner-unified-filter.version>
</properties>
......
......@@ -56,6 +56,7 @@ public class RedisKeyPrefix {
public static final String CUSTOM_EVENT_LIST = "BRANDKBS:CUSTOM_EVENT:LIST:";
public static final String CUSTOM_EVENT_ANALYZE_SHARE = "BRANDKBS:CUSTOM_EVENT:SHARE:";
public static final String CUSTOM_EVENT_ANALYZE = "BRANDKBS:CUSTOM_EVENT:ANALYZE:";
public static final String EVENT_AGG_TITLE_LIST = "BRANDKBS:EVENT:AGG_TITLE:LIST:";
/**
* 热点相关缓存KEY
......
......@@ -57,7 +57,7 @@ public class AppArticleController extends BaseController {
@ApiOperation("原始数据列表")
@PostMapping("/origin/list")
@LogRecord(description = "舆情库-原始数据列表")
@LogRecord(description = "舆情库-原始数据")
public ResponseResult getOriginList(@RequestBody MarkSearchDTO markSearchDTO) {
return ResponseResult.success(markDataService.getOriginList(markSearchDTO));
}
......@@ -117,14 +117,14 @@ public class AppArticleController extends BaseController {
@ApiOperation("舆情列表")
@PostMapping("/mark/list")
@LogRecord(description = "舆情库-舆情列表")
@LogRecord(description = "舆情库-舆情数据")
public ResponseResult getYuqingMarkList(@RequestBody MarkSearchDTO markSearchDTO) {
return ResponseResult.success(markDataService.getYuqingMarkList(markSearchDTO));
}
@ApiOperation("舆情列表-生成聚合")
@PostMapping("/mark/aggree")
@LogRecord(description = "舆情库-舆情列表-聚合")
@LogRecord(description = "舆情库-有效舆情-聚合")
public ResponseResult generateYuqingMarkAggreeList(@RequestBody JSONObject info) {
Long startTime = info.getLong("startTime");
Long endTime = info.getLong("endTime");
......@@ -188,7 +188,7 @@ public class AppArticleController extends BaseController {
@ApiImplicitParam(name = "startTime", value = "开始时间", paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long")
})
@LogRecord(description = "舆情库-舆情分析")
// @LogRecord(description = "舆情库-舆情分析")
@GetMapping("/emotion/proportion")
public ResponseResult getMarkEmotionProportion(@RequestParam(value = "startTime", required = false) Long startTime,
@RequestParam(value = "endTime", required = false) Long endTime) {
......@@ -448,6 +448,7 @@ public class AppArticleController extends BaseController {
@ApiOperation("新-舆情分析-舆情总量")
@GetMapping("/analyze/amount")
@LogRecord(description = "舆情库-舆情分析")
public ResponseResult getYuqingAmount(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "planId", required = false) String planId) {
......
......@@ -4,13 +4,11 @@ import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.aop.LogRecord;
import com.zhiwei.brandkbs2.auth.Auth;
import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.controller.BaseController;
import com.zhiwei.brandkbs2.enmus.RoleEnum;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.dto.EventSearchDTO;
import com.zhiwei.brandkbs2.service.EventService;
import com.zhiwei.middleware.event.core.EventMonitorClient;
import com.zhiwei.middleware.event.pojo.dto.BrandkbsEventSearchDTO;
import com.zhiwei.middleware.event.pojo.dto.EventDTO;
import io.swagger.annotations.Api;
......@@ -142,7 +140,7 @@ public class AppEventController extends BaseController {
}
@ApiOperation("前台事件库-全网事件库-行业热点-列表")
@LogRecord(description = "事件库-全网事件库-行业热点")
@LogRecord(description = "事件库-全网事件库")
@GetMapping("/getWholeNetworkEventsList")
public ResponseResult getWholeNetworkEventsList(@RequestParam(value = "firstType", required = false, defaultValue = "") String firstType,
@RequestParam(value = "start", required = false, defaultValue = "0") long start,
......@@ -187,7 +185,7 @@ public class AppEventController extends BaseController {
}
@ApiOperation("前台事件库-全网事件库-企业危机-危机库列表")
@LogRecord(description = "事件库-全网事件库-企业危机")
@LogRecord(description = "事件库-危机案例库")
@GetMapping("/crisisList")
public ResponseResult crisisList(@RequestParam(value = "page",defaultValue = "1") Integer page,
@RequestParam(value = "pageSize",defaultValue = "10") Integer pageSize,
......@@ -221,7 +219,7 @@ public class AppEventController extends BaseController {
}
@ApiOperation("前台事件库-品牌事件库-获取品牌事件库事件列表")
@LogRecord(description = "事件库-品牌事件库")
@LogRecord(description = "事件库-品牌事件库-事件中心")
@PostMapping("/eventList")
public ResponseResult getEventListMiddleware(@RequestBody BrandkbsEventSearchDTO dto){
return ResponseResult.success(eventService.getEventListMiddleware(dto));
......@@ -234,7 +232,7 @@ public class AppEventController extends BaseController {
}
@ApiOperation("前台事件库-事件监测-获取事件监测列表")
@LogRecord(description = "事件库-事件分析")
@LogRecord(description = "事件库-事件监测")
@GetMapping("/eventMonitor")
public ResponseResult getEventMonitor(@RequestParam(value = "page", defaultValue = "1") Integer page,
@RequestParam(value = "pageSize", defaultValue = "10") Integer pageSize){
......@@ -280,7 +278,7 @@ public class AppEventController extends BaseController {
@ApiOperation("前台事件库-事件详情-获取事件基础信息")
@GetMapping("/detail/baseInfo")
@LogRecord(values = "title", description = "事件库-品牌事件库", arguments = false, entity = true)
@LogRecord(values = "title", description = "事件库-品牌事件库-事件中心", arguments = false, entity = true)
@Auth(role = RoleEnum.NO_AUTHORISE)
public ResponseResult eventBaseInfo(@RequestParam(value = "eventId") String eventId){
return ResponseResult.success(eventService.eventBaseInfo(eventId));
......@@ -343,4 +341,18 @@ public class AppEventController extends BaseController {
List<String> eventIds = json.getJSONArray("eventIds").toJavaList(String.class);
return ResponseResult.success(eventService.scheduleEventStatus(eventIds));
}
@ApiOperation("前台事件库-品牌事件库标题聚合分析")
@GetMapping("/analyze/aggregate-titles")
@LogRecord(description = "事件库-品牌事件库-标题聚合分析")
public ResponseResult getLastNews(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime) {
return ResponseResult.success(eventService.getEventAggTitleList(startTime, endTime, false));
}
@ApiOperation("前台事件库-品牌事件库-分析")
@PostMapping("/analyze")
public ResponseResult aggTitleAnalyze(@RequestBody EventDTO dto){
return eventService.aggTitleAnalyze(dto);
}
}
......@@ -338,4 +338,10 @@ public interface EventMiddlewareDao {
* @return
*/
ReturnData scheduleEventStatus(List<String> eventIds);
/**
* 添加新事件
* @param eventDTO 事件传输类
*/
ReturnData addEvent(EventDTO eventDTO);
}
......@@ -231,4 +231,9 @@ public class EventMiddlewareDaoImpl implements EventMiddlewareDao {
public ReturnData scheduleEventStatus(List<String> eventIds) {
return eventMonitorClient.scheduleEventStatus(eventIds);
}
@Override
public ReturnData addEvent(EventDTO eventDTO) {
return eventClient.addEvent(eventDTO);
}
}
......@@ -4,6 +4,7 @@ import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportEventDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportEventDataDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.UploadEventDTO;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.Event;
import com.zhiwei.brandkbs2.pojo.EventDisseminationTrend;
import com.zhiwei.brandkbs2.pojo.dto.EventDataDTO;
......@@ -436,4 +437,17 @@ public interface EventService {
* @return
*/
JSONObject scheduleEventStatus(List<String> eventIds);
/**
* 品牌事件库-聚合标题列表
* @param startTime
* @param endTime
* @return
*/
List<JSONObject> getEventAggTitleList(Long startTime, Long endTime, boolean cache);
/**
* 品牌事件库-标题聚合分析-分析
*/
ResponseResult aggTitleAnalyze(EventDTO eventDTO);
}
......@@ -29,6 +29,11 @@ public interface TaskService{
void customEventCache();
/**
* 事件标题聚合分析缓存
*/
void eventAggTitleCache();
/**
* 生成简报任务并推送
*/
void generateReportAndSend();
......
package com.zhiwei.brandkbs2.service.impl;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
......@@ -23,15 +24,13 @@ import com.zhiwei.brandkbs2.enmus.response.EventCodeEnum;
import com.zhiwei.brandkbs2.exception.ExceptionCast;
import com.zhiwei.brandkbs2.listener.ApplicationProjectListener;
import com.zhiwei.brandkbs2.model.CommonCodeEnum;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.*;
import com.zhiwei.brandkbs2.pojo.dto.EventDataDTO;
import com.zhiwei.brandkbs2.pojo.dto.EventSearchDTO;
import com.zhiwei.brandkbs2.pojo.dto.YqEventDTO;
import com.zhiwei.brandkbs2.pojo.vo.*;
import com.zhiwei.brandkbs2.service.EventDataService;
import com.zhiwei.brandkbs2.service.EventService;
import com.zhiwei.brandkbs2.service.ProjectService;
import com.zhiwei.brandkbs2.service.UserService;
import com.zhiwei.brandkbs2.service.*;
import com.zhiwei.brandkbs2.util.MongoUtil;
import com.zhiwei.brandkbs2.util.RedisUtil;
import com.zhiwei.brandkbs2.util.Tools;
......@@ -115,6 +114,9 @@ public class EventServiceImpl implements EventService {
@Resource(name = "projectServiceImpl")
private ProjectService projectService;
@Resource(name = "markDataServiceImpl")
MarkDataService markDataService;
@Resource(name = "userServiceImpl")
private UserService userService;
......@@ -893,6 +895,57 @@ public class EventServiceImpl implements EventService {
return (JSONObject) eventMiddlewareDao.scheduleEventStatus(eventIds).getData();
}
@Override
public List<JSONObject> getEventAggTitleList(Long startTime, Long endTime, boolean cache) {
String projectId = UserThreadLocal.getProjectId();
String redisKey = RedisUtil.getEventAggTitleListKEY(projectId, startTime, endTime);
String resultStr = redisUtil.get(redisKey);
if (cache && StringUtils.isNotEmpty(resultStr)) {
return JSON.parseArray(resultStr, JSONObject.class);
}
List<JSONObject> resultList = markDataService.getLastNews(startTime, endTime, null, 50, true);
redisUtil.setExpire(redisKey, JSON.toJSONString(resultList));
return resultList;
}
@Override
public ResponseResult aggTitleAnalyze(EventDTO dto) {
long now = System.currentTimeMillis();
if (now < dto.getEndTime() && getMonitoringEventsCount() >= 5){
return ResponseResult.failure("结束时间为未来时间判断为监测事件,监测事件数量不得超过5");
}
Project project = projectService.getProjectById(UserThreadLocal.getProjectId());
// submitter submitterId
ServletRequestAttributes requestAttributes = (ServletRequestAttributes) RequestContextHolder.getRequestAttributes();
HttpServletRequest request = Objects.requireNonNull(requestAttributes).getRequest();
String submitterId = JwtUtil.unsign(request.getHeader(jwtKey), Map.class).get(GenericAttribute.USER_ID).toString();
String submitter = userService.queryUserInfo(submitterId, project.getId()).getNickname();
// 事件基础信息
dto.setProject(project.getBrandLinkedGroup());
dto.setProjectId(project.getBrandLinkedGroupId());
dto.setSubmitter(submitter);
dto.setSubmitterId(submitterId);
// 数据源
dto.setDataSource(project.getBrandLinkedGroup());
// 品见关联信息
List<BrandkbsBasicInfo> brandkbsBasicInfos = new ArrayList<>(1);
BrandkbsBasicInfo brandkbsBasicInfo = new BrandkbsBasicInfo();
brandkbsBasicInfo.setBrandkbsProject(project.getProjectName());
brandkbsBasicInfo.setBrandkbsProjectId(project.getId());
brandkbsBasicInfo.setBrandkbsBrand(project.getBrandName());
// 主品牌品牌id使用项目id
brandkbsBasicInfo.setBrandkbsBrandId(project.getId());
brandkbsBasicInfos.add(brandkbsBasicInfo);
dto.setBrandkbsInfos(brandkbsBasicInfos);
// 结束时间大于当前时间为监测事件
if (now < dto.getEndTime()) {
eventMiddlewareDao.addEventMonitor(dto);
return ResponseResult.success();
}
eventMiddlewareDao.addEvent(dto);
return ResponseResult.success();
}
@Data
public static class MongoQueryUtil {
......
......@@ -910,9 +910,6 @@ public class MarkDataServiceImpl implements MarkDataService {
}
private BaseMap getFirstArticle(Long startTime, Long endTime, String aggTitle, String projectId, String contendId, String planId, boolean include) throws IOException{
if (StringUtils.isBlank(aggTitle)){
return null;
}
// 索引
String[] indexes = esClientDao.getIndexes();
// postFilter
......@@ -2522,21 +2519,35 @@ public class MarkDataServiceImpl implements MarkDataService {
// 负面趋势图
Pair<String, List<LineVO>> negativeSpreadTendency = getSpreadTendency(startTime, endTime, planId, EmotionEnum.NEGATIVE.getName());
Map<String, BaseMap> baseMaps = new HashMap<>(2);
CompletableFuture.allOf(Stream.of(spreadTendency.getLeft(), negativeSpreadTendency.getLeft()).map(aggTitle -> CompletableFuture.runAsync(() -> {
try {
baseMaps.put(aggTitle, getFirstArticle(startTime, endTime, aggTitle, projectId, Constant.PRIMARY_CONTEND_ID, planId, true));
} catch (IOException ignored) {
}
}, executor)).toArray(CompletableFuture[]::new)).join();
BaseMap totalBaseMaps = null;
BaseMap negativeBaseMaps = null;
if (Objects.nonNull(negativeSpreadTendency.getLeft()) && Objects.nonNull(spreadTendency.getLeft())) {
CompletableFuture.allOf(Stream.of(spreadTendency.getLeft(), negativeSpreadTendency.getLeft()).map(aggTitle -> CompletableFuture.runAsync(() -> {
try {
baseMaps.put(aggTitle, getFirstArticle(startTime, endTime, aggTitle, projectId, Constant.PRIMARY_CONTEND_ID, planId, true));
} catch (IOException ignored) {
}
}, executor)).toArray(CompletableFuture[]::new)).join();
}
// 趋势图标题为空,此时说明最高点的时间段发文平台全为微博,此时取最高点时间段内时间最早的微博发文
if (Objects.isNull(spreadTendency.getLeft()) || Objects.isNull(negativeSpreadTendency.getLeft())){
totalBaseMaps = getWeiboFirstArticle(endTime, spreadTendency.getRight(), planId, projectId, null);
negativeBaseMaps = getWeiboFirstArticle(endTime, spreadTendency.getRight(), planId, projectId, EmotionEnum.NEGATIVE.getName());
}
// 最高点
BaseMap baseMap = baseMaps.get(spreadTendency.getLeft());
BaseMap baseMap = Objects.isNull(baseMaps.get(spreadTendency.getLeft())) ? totalBaseMaps : baseMaps.get(spreadTendency.getLeft());
JSONObject highestJson = new JSONObject();
highestJson.put("title", Objects.isNull(baseMap) ? null : baseMap.getTitle());
highestJson.put("url", Objects.isNull(baseMap) ? null : baseMap.getUrl());
BaseMap negativeBaseMap = baseMaps.get(negativeSpreadTendency.getLeft());
if (Objects.nonNull(baseMap)) {
highestJson.put("title", baseMap.getTitle());
highestJson.put("url", baseMap.getUrl());
}
// 负面最高点
BaseMap negativeBaseMap = Objects.isNull(baseMaps.get(negativeSpreadTendency.getLeft())) ? negativeBaseMaps : baseMaps.get(negativeSpreadTendency.getLeft());
JSONObject negativeHighestJson = new JSONObject();
negativeHighestJson.put("title", Objects.isNull(negativeBaseMap) ? null : negativeBaseMap.getTitle());
negativeHighestJson.put("url", Objects.isNull(negativeBaseMap) ? null : negativeBaseMap.getUrl());
if (Objects.nonNull(negativeBaseMap)) {
negativeHighestJson.put("title", negativeBaseMap.getTitle());
negativeHighestJson.put("url", negativeBaseMap.getUrl());
}
res.putAll(platformSpreadTendency);
res.put("总量", spreadTendency.getRight());
res.put("负面", negativeSpreadTendency.getRight());
......@@ -2548,6 +2559,54 @@ public class MarkDataServiceImpl implements MarkDataService {
return res;
}
/**
* 获取时间段内的微博最早文章
*
* @param lines
* @param planId
* @param projectId
* @param emotion
* @return
* @throws IOException
*/
private BaseMap getWeiboFirstArticle(Long endTime, List<LineVO> lines, String planId, String projectId, String emotion) throws IOException {
if (CollectionUtils.isEmpty(lines)){
return null;
}
// 最高点的时间为开始时间
LineVO startLine = lines.stream().max(Comparator.comparingLong(LineVO::getCount)).get();
Long startTime = startLine.getDate();
int startIndex = lines.indexOf(startLine);
// 最高点后一个点的时间为结束时间
endTime = startIndex == lines.size() - 1 ? endTime : lines.get(startIndex + 1).getDate();
// 索引
String[] indexes = esClientDao.getIndexes();
// postFilter
BoolQueryBuilder postFilter;
if (Objects.isNull(planId)) {
postFilter = projectContendIdQuery(projectId, Constant.PRIMARY_CONTEND_ID);
}else {
postFilter = EsQueryTools.assembleCacheMapsPlanQuery(projectId, planId);
}
// emotion
if (StringUtils.isNotEmpty(emotion) && !EmotionEnum.ALL.getName().equals(emotion)) {
postFilter.must(QueryBuilders.termQuery("brandkbs_mark_cache_maps.name.keyword", emotion));
}
// weibo平台
postFilter.must(QueryBuilders.termQuery("platform_id", "5d02236e6395002a7c380b79"));
postFilter.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
//sort
FieldSortBuilder sort = new FieldSortBuilder("time").order(SortOrder.ASC);
//hits
SearchHits hits = esClientDao.searchHits(indexes, postFilter, null, null, sort, 0, 1, null);
if (0 == hits.getTotalHits().value) {
return new BaseMap();
}
BaseMap baseMap = Tools.getBaseFromEsMap(hits.getAt(0).getSourceAsMap());
baseMap.setTitle(baseMap.getContent());
return baseMap;
}
@Override
public List<JSONObject> getActiveChannels(Long startTime, Long endTime, String planId) {
List<JSONObject> list = new ArrayList<>();
......@@ -2789,14 +2848,14 @@ public class MarkDataServiceImpl implements MarkDataService {
res.add(new LineVO(count, time));
});
// 走势图最高点的聚合标题
String maxAggTitle = "";
String maxAggTitle = null;
Optional<? extends Histogram.Bucket> maxBucket = buckets.stream().max(Comparator.comparingLong(MultiBucketsAggregation.Bucket::getDocCount));
if (maxBucket.isPresent()){
Map<String, Aggregation> map = maxBucket.get().getAggregations().asMap();
ParsedStringTerms countTeam = (ParsedStringTerms) map.get("titleAgg");
List<? extends Terms.Bucket> titleBuckets = countTeam.getBuckets();
// 没有拿到聚合标题,说明此时发文的平台为微博
maxAggTitle = CollectionUtils.isEmpty(titleBuckets) ? "" : titleBuckets.get(0).getKeyAsString();
// 没有拿到聚合标题,说明此时发文的平台为微博
maxAggTitle = CollectionUtils.isEmpty(titleBuckets) ? null : titleBuckets.get(0).getKeyAsString();
}
return Pair.of(maxAggTitle, res);
}
......
......@@ -210,6 +210,21 @@ public class TaskServiceImpl implements TaskService {
}, cacheServiceExecutor)).toArray(CompletableFuture[]::new)).join();
}
@Override
public void eventAggTitleCache() {
AtomicInteger total = new AtomicInteger();
List<Long[]> timeList = Arrays.asList(commonService.getTimeRangeDay(), commonService.getTimeRangeFormatWeek(), commonService.getTimeRangeFormatMonth());
CompletableFuture.allOf(GlobalPojo.PROJECT_MAP.values().stream().map(project -> CompletableFuture.supplyAsync(() -> {
UserThreadLocal.set(new UserInfo().setProjectId(project.getId()));
timeList.forEach(times -> {
// 事件标题聚合缓存
eventService.getEventAggTitleList(times[0], times[1], false);
});
log.info("项目:{}-事件标题聚合缓存已完成:{}个", project.getProjectName(), total.incrementAndGet());
return null;
}, cacheServiceExecutor)).toArray(CompletableFuture[]::new)).join();
}
private Pair<List<Channel>, List<Channel>> batchHandle(List<Map.Entry<ChannelIndex, ChannelIndex.Record>> batchList, Map<ChannelIndex,
ChannelIndex.Record> newRecordMap) {
List<Channel> insertList = Collections.synchronizedList(new ArrayList<>());
......
......@@ -44,7 +44,8 @@ public class ControlCenter {
log.info("定时按天缓存数据-启动");
try {
taskService.messageFlowCache();
taskService.customEventCache();
// taskService.customEventCache();
taskService.eventAggTitleCache();
taskService.yuqingAnalyzeHighWordCache();
} catch (Exception e) {
log.error("定时按天缓存数据-出错", e);
......
......@@ -27,6 +27,10 @@ public class RedisUtil {
return RedisKeyPrefix.CUSTOM_EVENT_LIST + Tools.concat(projectId, startTimeStr, endTimeStr);
}
public static String getEventAggTitleListKEY(String projectId, Long startTimeStr, Long endTimeStr) {
return RedisKeyPrefix.EVENT_AGG_TITLE_LIST + Tools.concat(projectId, startTimeStr, endTimeStr);
}
public static String getShotPageKey(String id, String projectId) {
return RedisKeyPrefix.REDIS_SYSTEM_COPY + "SHOT_PAGE:" + projectId + "-" + id;
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment