Commit 83707cc7 by shenjunjie

es-index nested结构调整

parent 4d0e1f69
......@@ -15,7 +15,8 @@ public class GenericAttribute {
* es index
*/
public static final String ES_INDEX_PRE = "brandkbs2";
public static final String ES_INDEX_TEST = "brandkbs2_test";
public static final String ES_INDEX_TEST = "brandkbs2_test";
// public static final String ES_INDEX_TEST = "brandkbs2_2022";
public static final String ES_CHANNEL_INDEX_TEST = "brandkbs2_channel_record_test";
/**
* es ind_title
......@@ -34,7 +35,7 @@ public class GenericAttribute {
*/
public static final String ES_MEDIA_TYPE = "media_type";
// es platform_id
public static final String ES_PLATFORM_ID= "platform_id";
public static final String ES_PLATFORM_ID = "platform_id";
// es channel_influence
public static final String ES_CHANNEL_INDEX = "channel_influence";
/**
......@@ -50,33 +51,61 @@ public class GenericAttribute {
public static final String ES_CHANNEL_FID = "fid";
public static final String ES_CNAME = "cname";
/** es c2 **/
/**
* es c2
**/
public static final String ES_C2 = "c2";
/** es c4 **/
/**
* es c4
**/
public static final String ES_C4 = "c4";
/** es c5 **/
/**
* es c5
**/
public static final String ES_C5 = "c5";
/** es foreign **/
/**
* es foreign
**/
public static final String ES_FOREIGN = "foreign";
/** es source **/
/**
* es source
**/
public static final String ES_SOURCE = "source";
/** es real_source **/
/**
* es real_source
**/
public static final String ES_REAL_SOURCE = "real_source";
/** es time **/
/**
* es time
**/
public static final String ES_TIME = "time";
/** es ctime **/
/**
* es ctime
**/
public static final String ES_CTIME = "ctime";
/** es mtime **/
/**
* es mtime
**/
public static final String ES_MTIME = "mtime";
/** es mtag **/
/**
* es mtag
**/
public static final String ES_MTAG = "mtag";
/** es mgroup **/
/**
* es mgroup
**/
public static final String ES_MGROUP = "mgroup";
/** es forward **/
/**
* es forward
**/
public static final String ES_FORWARD = "forward";
/** es brandkbs_cache_maps **/
/**
* es brandkbs_cache_maps
**/
public static final String ES_BRANDKBS_CACHE_MAPS = "brandkbs_cache_maps";
/** es mark_cache_maps **/
/**
* es mark_cache_maps
**/
public static final String ES_MARK_CACHE_MAPS = "brandkbs_mark_cache_maps";
public static final String ES_LINKED_GROUP_ID = "linked_group_id";
......@@ -92,7 +121,7 @@ public class GenericAttribute {
public static final String USER_ID = "userId";
public static final String ROLE_ID = "roleId";
public static final String PROJECT_ID = "projectId";
public static final String AVATAR_URL= "avatarUrl";
public static final String AVATAR_URL = "avatarUrl";
// public enum ChannelParam{
// 负面稿件数("negativeArticles"),
......
......@@ -55,8 +55,8 @@ public class RedisKeyPrefix {
/**
* 热点相关缓存KEY
*/
public static final String HOT_RANKLIST = "brandkbs:hot:rankList:";
public static final String HOT_LIST = "brandkbs:hot:list:";
public static final String HOT_RANK_LIST = "BRANDKBS:HOT:RANK_LIST:";
public static final String HOT_LIST = "BRANDKBS:HOT:LIST:";
/**
* 项目简报报相关缓存KEY
......
......@@ -175,4 +175,12 @@ public class AppArticleController extends BaseController {
return ResponseResult.success();
}
@ApiOperation("舆情简报-获取报告结果")
@ApiImplicitParam(name = "id", value = "报告ID", required = true, paramType = "path", dataType = "String")
@GetMapping("/analyze/{id}")
public ResponseResult getReportAnalyze(@PathVariable String id) {
return ResponseResult.success(reportService.getPcReportAnalyze(id, true));
}
}
......@@ -9,6 +9,7 @@ import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.dto.EventSearchDTO;
import com.zhiwei.brandkbs2.service.EventService;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiOperation;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
......@@ -138,4 +139,18 @@ public class AppEventController extends BaseController {
JSONObject jsonObject = JSON.parseObject(responseEntity.getBody());
return ResponseResult.success(jsonObject);
}
@ApiOperation("最新事件")
@GetMapping("/getLastEventTop")
public ResponseResult getLastEventTop() {
return ResponseResult.success(eventService.getLastEventTop(null, 5));
}
@ApiOperation("关联事件")
@ApiImplicitParam(name = "keyword", value = "关键词", paramType = "query", dataType = "String")
@GetMapping("/getRelevanceEvent")
public ResponseResult getRelevanceEvent(@RequestParam(value = "keyword", required = false) String keyword) {
return ResponseResult.success(eventService.getLastEventTop(keyword, 5));
}
}
......@@ -99,7 +99,7 @@ public class AppHotController extends BaseController {
public ResponseResult hot(){
ResponseEntity<String> responseEntity = restTemplate.getForEntity(longTimeInListSearchUrl, String.class,"weibo");
JSONObject jsonObject = JSON.parseObject(responseEntity.getBody());
String redisKey = RedisKeyPrefix.HOT_RANKLIST;
String redisKey = RedisKeyPrefix.HOT_RANK_LIST;
String result = redisUtil.get(redisKey);
//当舆论场崩溃时从缓存里获取
if(jsonObject.getBoolean("state")){
......
......@@ -156,9 +156,4 @@ public interface BaseMongoDao<T extends AbstractBaseMongo> {
return criteria;
}
@FunctionalInterface
interface VoidSetFunction {
<T> void set(T t);
}
}
......@@ -94,4 +94,7 @@ public interface EventDao extends BaseMongoDao<Event> {
*/
List<Event> getEventsByProjectIdAndContendId(Long startTime, Long endTime, String emotion, String projectId, String contendId, int limit);
List<Event> getEventsByTotalChannelVolumeTop(Long startTime, Long endTime, String emotion, String projectId, String contendId, int limit);
}
......@@ -105,6 +105,17 @@ public class EventDaoImpl extends BaseMongoDaoImpl<Event> implements EventDao {
return mongoTemplate.find(query, clazz, COLLECTION_NAME);
}
@Override
public List<Event> getEventsByTotalChannelVolumeTop(Long startTime, Long endTime, String emotion, String projectId, String contendId, int limit) {
Query query = new Query();
Criteria criteria = eventCountCriteria(startTime, endTime, emotion, projectId, contendId);
query.addCriteria(criteria);
query.limit(limit);
// 按影响力排序
addSort(query, "{\"negativeArticleVolume\":\"descend\"}");
return mongoTemplate.find(query, clazz, COLLECTION_NAME);
}
private Criteria eventCountCriteria(Long startTime, Long endTime, String emotion, String projectId, String contendId) {
Criteria criteria = Criteria.where("projectId").is(projectId);
criteria.and("contendId").is(contendId);
......
package com.zhiwei.brandkbs2.dao.impl;
import com.zhiwei.brandkbs2.dao.EventDisseminationTrendDao;
import com.zhiwei.brandkbs2.pojo.Event;
import com.zhiwei.brandkbs2.pojo.EventDisseminationTrend;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Component;
......@@ -21,6 +20,6 @@ public class EventDisseminationTrendDaoImpl extends BaseMongoDaoImpl<EventDissem
@Override
public EventDisseminationTrend findOne(Query query) {
return mongoTemplate.findOne(query, EventDisseminationTrend.class);
return mongoTemplate.findOne(query, EventDisseminationTrend.class, COLLECTION_NAME);
}
}
......@@ -4,12 +4,19 @@ import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.common.GlobalPojo;
import com.zhiwei.pushlog.tools.Tools;
import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.lucene.search.join.ScoreMode;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.NestedQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import java.util.*;
import static com.zhiwei.brandkbs2.common.GenericAttribute.ES_BRANDKBS_CACHE_MAPS;
import static com.zhiwei.brandkbs2.util.Tools.concat;
/**
* @ClassName: EsQueryTool
* @Description es语句辅助类
......@@ -83,27 +90,51 @@ public class EsQueryTools {
return tagQuery;
}
/**
* 媒体类型请求匹配
*
* @param mediaTypes
* @return
*/
public static BoolQueryBuilder assembleMediaTypeQuery(List<String> mediaTypes) {
BoolQueryBuilder tagQuery = QueryBuilders.boolQuery();
mediaTypes.forEach(e -> {
tagQuery.should(QueryBuilders.termQuery("brandkbs_cache_maps.channel_type.keyword", e));
});
return tagQuery;
public static BoolQueryBuilder assembleCacheMapsQuery(String projectId, String linkedGroupId, String contendId) {
return assembleCacheMapsQuery(projectId, linkedGroupId, contendId, null);
}
public static BoolQueryBuilder assembleCacheMapsQuery(String projectId, String linkedGroupId, String contendId, List<String> mediaTypes) {
BoolQueryBuilder nestedBoolQueryBuilder = QueryBuilders.boolQuery();
// must key
nestedBoolQueryBuilder.must(cacheMapsNestedQuery(QueryBuilders.termQuery("brandkbs_cache_maps.key.keyword", concat(projectId, linkedGroupId, contendId))));
// mediaType
if (CollectionUtils.isNotEmpty(mediaTypes)) {
BoolQueryBuilder mediaTypeQueryBuilder = QueryBuilders.boolQuery();
mediaTypes.forEach(e -> {
mediaTypeQueryBuilder.should(cacheMapsNestedQuery(QueryBuilders.termQuery("brandkbs_cache_maps.channel_type.keyword", e)));
});
nestedBoolQueryBuilder.must(mediaTypeQueryBuilder);
}
return nestedBoolQueryBuilder;
}
public static NestedQueryBuilder cacheMapsNestedQuery(QueryBuilder query) {
return new NestedQueryBuilder(ES_BRANDKBS_CACHE_MAPS, query, ScoreMode.None);
}
// /**
// * 媒体类型请求匹配
// *
// * @param mediaTypes
// * @return
// */
// public static BoolQueryBuilder assembleMediaTypeQuery(List<String> mediaTypes) {
// BoolQueryBuilder tagQuery = QueryBuilders.boolQuery();
// mediaTypes.forEach(e -> {
// // TODO 1
// tagQuery.should(QueryBuilders.termQuery("brandkbs_cache_maps.channel_type.keyword", e));
// });
// return tagQuery;
// }
public static BoolQueryBuilder assembleSourceQuery(String sourceKeyword) {
BoolQueryBuilder channelBoolQueryBuilder = QueryBuilders.boolQuery();
String[] keys = sourceKeyword.trim().split("\\|");
for (String key : keys) {
String channelRegex = getAllRegex(key);
BoolQueryBuilder keyQueryBuilder = QueryBuilders.boolQuery();
keyQueryBuilder.must(QueryBuilders.regexpQuery("source.keyword", ".*" + channelRegex + ".*"));
keyQueryBuilder.must(QueryBuilders.regexpQuery("source", ".*" + channelRegex + ".*"));
channelBoolQueryBuilder.should(keyQueryBuilder);
}
return channelBoolQueryBuilder;
......
package com.zhiwei.brandkbs2.pojo;
import com.zhiwei.brandkbs2.enmus.EmotionEnum;
import com.zhiwei.brandkbs2.util.Tools;
import lombok.Getter;
import lombok.Setter;
import org.springframework.data.mongodb.core.mapping.Document;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
......@@ -15,15 +19,15 @@ import java.util.Map;
@Getter
@Setter
@Document(collection = "brandkbs_event_dissemination_trend")
public class EventDisseminationTrend extends AbstractBaseMongo{
public class EventDisseminationTrend extends AbstractBaseMongo {
/**
* 负面稿件传播
*/
private List<Map<String,Object>> negativeSpread;
private List<Map<String, Object>> negativeSpread;
/**
* 总稿件传播
*/
private List<Map<String,Object>> totalSpread;
private List<Map<String, Object>> totalSpread;
/**
* 类型 按小时、按天
*/
......@@ -32,4 +36,61 @@ public class EventDisseminationTrend extends AbstractBaseMongo{
* 关联事件id
*/
private String eventId;
public static EventDisseminationTrend createFromType(List<EventData> list, Long startTime, Long endTime, String type, String eventId) {
EventDisseminationTrend eventDisseminationTrend = new EventDisseminationTrend();
eventDisseminationTrend.setEventId(eventId);
eventDisseminationTrend.setType(type);
switch (type) {
case "小时": {
setSpreadAll(list, Tools.parseToHours(startTime, endTime), eventDisseminationTrend);
break;
}
case "天": {
setSpreadAll(list, Tools.parseToDays(startTime, endTime), eventDisseminationTrend);
break;
}
}
return eventDisseminationTrend;
}
private static void setSpreadAll(List<EventData> list, List<Map<String, Long>> rangeTimeList, EventDisseminationTrend trend) {
long start;
long end;
long negCount = 0;
long count = 0;
int index = 0;
List<Map<String, Object>> negativeSpread = new ArrayList<>();
List<Map<String, Object>> totalSpread = new ArrayList<>();
for (EventData eventData : list) {
Map<String, Long> map = rangeTimeList.get(index);
start = map.get("startTime");
end = map.get("endTime");
if (eventData.getTime() >= start && eventData.getTime() < end) {
// 数据统计
count++;
if (EmotionEnum.NEGATIVE.getName().equals(eventData.getEmotion())) {
negCount++;
}
} else {
// 总统计
Map<String, Object> element = new HashMap<>();
element.put("time", start);
element.put("sum", count);
negativeSpread.add(element);
// 负面统计
Map<String, Object> negElement = new HashMap<>();
negElement.put("time", start);
negElement.put("sum", negCount);
totalSpread.add(negElement);
// 推进节点并重置计量值
count = 0;
negCount = 0;
index++;
}
}
trend.setTotalSpread(totalSpread);
trend.setNegativeSpread(negativeSpread);
}
}
......@@ -18,6 +18,11 @@ public class EventTopArticlesAnalysis extends AbstractBaseMongo {
* 聚合标题
*/
private String aggTitle;
/**
* 首条标题
*/
private String url;
/**
* 聚合情感
*/
......@@ -49,6 +54,7 @@ public class EventTopArticlesAnalysis extends AbstractBaseMongo {
eventTopArticlesAnalysis.setEmotion(templateData.getEmotion());
eventTopArticlesAnalysis.setCount(count);
eventTopArticlesAnalysis.setTime(templateData.getTime());
eventTopArticlesAnalysis.setUrl(templateData.getUrl());
eventTopArticlesAnalysis.setTimePoint(timePoint);
eventTopArticlesAnalysis.setType("按时间");
eventTopArticlesAnalysis.setEventId(eventId);
......@@ -61,6 +67,7 @@ public class EventTopArticlesAnalysis extends AbstractBaseMongo {
eventTopArticlesAnalysis.setEmotion(templateData.getEmotion());
eventTopArticlesAnalysis.setCount(count);
eventTopArticlesAnalysis.setTime(templateData.getTime());
eventTopArticlesAnalysis.setUrl(templateData.getUrl());
eventTopArticlesAnalysis.setType("按数量");
eventTopArticlesAnalysis.setEventId(eventId);
return eventTopArticlesAnalysis;
......
......@@ -5,6 +5,7 @@ import com.zhiwei.brandkbs2.easyexcel.dto.ExportEventDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportEventDataDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.UploadEventDTO;
import com.zhiwei.brandkbs2.pojo.Event;
import com.zhiwei.brandkbs2.pojo.EventData;
import com.zhiwei.brandkbs2.pojo.EventDisseminationTrend;
import com.zhiwei.brandkbs2.pojo.dto.EventDataDTO;
import com.zhiwei.brandkbs2.pojo.dto.EventSearchDTO;
......@@ -37,8 +38,8 @@ public interface EventService {
/**
* 批量添加舆情事件数据
*
* @param contendId 竞品id
* @param yqEventList 舆情事件列表
* @param contendId 竞品id
* @param yqEventList 舆情事件列表
*/
void addYqEvents(String contendId, List<YqEventDTO> yqEventList);
......@@ -73,7 +74,7 @@ public interface EventService {
* 稿件数据上传
*
* @param contendId 竞品id
* @param file 文件
* @param file 文件
* @return 文件信息
*/
JSONObject uploadEventDatas(String contendId, MultipartFile file);
......@@ -168,17 +169,17 @@ public interface EventService {
/**
* 添加文件上传事件信息
*
* @param contendId 竞品id
* @param fileUrl 文件路径
* @param contendId 竞品id
* @param fileUrl 文件路径
*/
void addFileEvents(String contendId, String fileUrl);
/**
* 添加文件上传事件信息
*
* @param projectId 项目id
* @param contendId 竞品id
* @param eventDTO 事件传输对象
* @param projectId 项目id
* @param contendId 竞品id
* @param eventDTO 事件传输对象
*/
void addFileEvent(String projectId, String contendId, UploadEventDTO eventDTO);
......@@ -218,12 +219,14 @@ public interface EventService {
/**
* 获取品牌事件搜索条件
*
* @return
*/
JSONObject getEventsSearchCriteria();
/**
* 获取品牌事件列表信息
*
* @param eventSearchDTO 事件搜索类
* @return
*/
......@@ -231,6 +234,7 @@ public interface EventService {
/**
* 事件详情-基础静态信息
*
* @param eventId
* @return
*/
......@@ -238,6 +242,7 @@ public interface EventService {
/**
* 事件详情-传播趋势图
*
* @param eventId
* @param type
* @return
......@@ -246,6 +251,7 @@ public interface EventService {
/**
* 事件详情-渠道发声
*
* @param eventId
* @param type
* @param page
......@@ -253,10 +259,11 @@ public interface EventService {
* @param sorter
* @return
*/
List<JSONObject> getEventChannelVoices(String eventId, String type, int page, int pageSize, String sorter);
PageVO<JSONObject> getEventChannelVoices(String eventId, String type, int page, int pageSize, String sorter);
/**
* 事件详情-热门文章分析
*
* @param id
* @param type
* @param emotion
......@@ -264,4 +271,12 @@ public interface EventService {
* @return
*/
JSONObject getEventTopArticlesAnalysis(String id, String type, String emotion, String aggTitle);
/**
* 获得最新的事件
*
* @return
*/
List<JSONObject> getLastEventTop(String keyword,int limit);
}
......@@ -62,4 +62,14 @@ public interface IndexService {
*/
JSONObject getSpreadingTend(Long startTime, Long endTime, boolean cache);
/**
* 获取主品牌传播趋势
*
* @param startTime 开始时间
* @param endTime 结束时间
* @param cache 是否优先读取缓存
* @return 主品牌传播趋势
*/
JSONObject getSpreadingTend(Long startTime, Long endTime, String projectId, String contendId, boolean cache);
}
......@@ -40,7 +40,6 @@ import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.io.IOException;
import java.text.ParseException;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
......@@ -134,7 +133,7 @@ public class CustomEventServiceImpl implements CustomEventService {
if (Objects.isNull(customEvent)) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("自定义事件数据异常"));
}
if (!customEvent.getStatus()) {
if (Boolean.FALSE.equals(customEvent.getStatus())) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("该事件数据更新中,无法修改该事件信息!"));
}
// 修改自定义事件及清空历史数据
......@@ -180,7 +179,7 @@ public class CustomEventServiceImpl implements CustomEventService {
if (Objects.isNull(customEvent)) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("自定义事件数据异常"));
}
if (!customEvent.getStatus()) {
if (Boolean.FALSE.equals(customEvent.getStatus())) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("该事件数据更新中!"));
}
customEvent.setStatus(false);
......@@ -210,7 +209,7 @@ public class CustomEventServiceImpl implements CustomEventService {
try {
return getCustomEventAnalyzeInner(id, cache);
} catch (Exception e) {
log.info("自定义事件:{} 数据更新异常", id, e);
log.info("自定义事件:{} 获取异常", id, e);
}
return new JSONObject();
}
......@@ -231,9 +230,6 @@ public class CustomEventServiceImpl implements CustomEventService {
}
String redisKey = RedisKeyPrefix.CUSTOM_EVENT_ANALYZE + id;
if (cache) {
if (Boolean.FALSE.equals(customEvent.getStatus())) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("该事件数据更新中,请勿重复更新!"));
}
String resultStr = redisUtil.get(redisKey);
if (StringUtils.isNotEmpty(resultStr)) {
return JSON.parseObject(resultStr);
......@@ -510,7 +506,7 @@ public class CustomEventServiceImpl implements CustomEventService {
// 获得首发标题
List<BaseMap> list = entry.getValue().stream().sorted(Comparator.comparingLong(BaseMap::getTime)).limit(1).collect(Collectors.toList());
JSONObject result = new JSONObject();
result.put("title", list.get(0).getTitle());
result.put("title", list.get(0).getTitleNullOptionalContent());
result.put("url", list.get(0).getUrl());
result.put("num", entry.getValue().size());
res.add(result);
......
......@@ -5,10 +5,12 @@ import com.google.common.collect.Lists;
import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.dao.EventDao;
import com.zhiwei.brandkbs2.dao.EventDataDao;
import com.zhiwei.brandkbs2.dao.EventDisseminationTrendDao;
import com.zhiwei.brandkbs2.dao.EventTopArticlesAnalysisDao;
import com.zhiwei.brandkbs2.enmus.EmotionEnum;
import com.zhiwei.brandkbs2.pojo.Event;
import com.zhiwei.brandkbs2.pojo.EventData;
import com.zhiwei.brandkbs2.pojo.EventDisseminationTrend;
import com.zhiwei.brandkbs2.pojo.EventTopArticlesAnalysis;
import com.zhiwei.brandkbs2.service.EventDataService;
import com.zhiwei.brandkbs2.service.MarkDataService;
......@@ -17,9 +19,13 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.joda.time.Period;
import org.joda.time.PeriodType;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.text.MessageFormat;
import java.util.*;
import java.util.concurrent.atomic.AtomicLong;
......@@ -50,6 +56,9 @@ public class EventDataServiceImpl implements EventDataService {
@Resource(name = "eventTopArticlesAnalysisDao")
private EventTopArticlesAnalysisDao eventTopArticlesAnalysisDao;
@Resource(name = "eventDisseminationTrendDao")
private EventDisseminationTrendDao eventDisseminationTrendDao;
private static final String TOP_DETAIL_TEMPLATE = "Top媒体传播方向:全局Top8的文章中,{0}报道方向占比最高,为{1}%,{2}报道方向占比{3}%。Top报道方向中,《{4}》相似文章数最多,总文章数为{5}篇。";
@Override
......@@ -104,6 +113,7 @@ public class EventDataServiceImpl implements EventDataService {
// 删除历史数据
eventDataDao.deleteByEventId(event.getId(), event.getCollectionName());
eventTopArticlesAnalysisDao.deleteByEventId(event.getId());
eventDisseminationTrendDao.deleteOneByQuery(Query.query(Criteria.where("eventId").is(event.getId())));
}
private void setEventStaticState(Event event, List<EventData> eventDataList) {
......@@ -142,12 +152,25 @@ public class EventDataServiceImpl implements EventDataService {
JSONObject articlePlatformProportions = new JSONObject();
platformIdCounts.forEach((platform, count) -> articlePlatformProportions.put(platform, getProportion(count.get(), totalDisseminationVolume)));
// 热门文章分析描述
String eventTopArticlesAnalysisDetail = MessageFormat.format(TOP_DETAIL_TEMPLATE, topArticlesAnalysis(eventDataList, event));
String eventTopArticlesAnalysisDetail = MessageFormat.format(TOP_DETAIL_TEMPLATE, topArticlesAnalysis(eventDataList, event).toArray());
event.setStaticState(totalDisseminationVolume, totalChannelVolumes.size(), negativeArticleVolume, articleEmotionProportions,
articlePlatformProportions, eventTopArticlesAnalysisDetail);
// 传播分析
eventDisseminationTrend(eventDataList, event);
eventDao.updateOne(event);
}
private void eventDisseminationTrend(List<EventData> eventDataList, Event event) {
// 按时间排序
eventDataList = eventDataList.stream().sorted(Comparator.comparingLong(EventData::getTime)).collect(Collectors.toList());
Long startTime = event.getStartTime();
Long endTime = null == event.getEndTime() ? System.currentTimeMillis() : event.getEndTime();
EventDisseminationTrend hourTrend = EventDisseminationTrend.createFromType(eventDataList, startTime, endTime, "小时", event.getId());
EventDisseminationTrend dayTrend = EventDisseminationTrend.createFromType(eventDataList, startTime, endTime, "天", event.getId());
eventDisseminationTrendDao.insertOne(hourTrend);
eventDisseminationTrendDao.insertOne(dayTrend);
}
private List<String> topArticlesAnalysis(List<EventData> eventDataList, Event event) {
// 标题前8数据
List<Map.Entry<String, List<EventData>>> top8Titles =
......@@ -199,7 +222,7 @@ public class EventDataServiceImpl implements EventDataService {
long sum = emotionMap.values().stream().mapToLong(AtomicLong::get).sum();
emotionMap.entrySet().stream().sorted((x, y) -> Long.compare(y.getValue().get(), x.getValue().get())).limit(2).forEach(entry -> {
details.add(entry.getKey());
details.add(String.valueOf((entry.getValue().get()) / sum));
details.add(new BigDecimal((double) (entry.getValue().get() * 100) / sum).setScale(2, RoundingMode.UP).toString());
});
details.add(top8Titles.get(0).getKey());
details.add(String.valueOf(top8Titles.get(0).getValue().size()));
......
......@@ -22,10 +22,7 @@ import com.zhiwei.brandkbs2.enmus.EventTagEnum;
import com.zhiwei.brandkbs2.exception.ExceptionCast;
import com.zhiwei.brandkbs2.listener.ApplicationProjectListener;
import com.zhiwei.brandkbs2.model.CommonCodeEnum;
import com.zhiwei.brandkbs2.pojo.Event;
import com.zhiwei.brandkbs2.pojo.EventData;
import com.zhiwei.brandkbs2.pojo.EventDisseminationTrend;
import com.zhiwei.brandkbs2.pojo.EventTopArticlesAnalysis;
import com.zhiwei.brandkbs2.pojo.*;
import com.zhiwei.brandkbs2.pojo.dto.EventDataDTO;
import com.zhiwei.brandkbs2.pojo.dto.EventSearchDTO;
import com.zhiwei.brandkbs2.pojo.dto.YqEventDTO;
......@@ -538,6 +535,8 @@ public class EventServiceImpl implements EventService {
if (Objects.nonNull(eventSearchDTO.getStartTime()) && Objects.nonNull(eventSearchDTO.getEndTime())) {
query.addCriteria(Criteria.where("startTime").gte(eventSearchDTO.getStartTime()).lt(eventSearchDTO.getEndTime()));
}
// 关键词
eventDao.addKeywordFuzz(query, eventSearchDTO.getKeyword(), "title");
// 传播量
if (Objects.nonNull(eventSearchDTO.getTotalDisseminationVolumes())) {
Long[] totalDisseminationVolumes = eventSearchDTO.getTotalDisseminationVolumes();
......@@ -568,8 +567,7 @@ public class EventServiceImpl implements EventService {
sortMap.get(event.getId()).setFirstEventData(r);
return null;
})).toArray(CompletableFuture[]::new)).join();
return PageVO.createPageVo(total, eventSearchDTO.getPage(), eventSearchDTO.getPageSize(),
eventList.stream().map(event -> sortMap.get(event.getId())).collect(Collectors.toList()));
return PageVO.createPageVo(total, eventSearchDTO.getPage(), eventSearchDTO.getPageSize(), eventList.stream().map(event -> sortMap.get(event.getId())).collect(Collectors.toList()));
}
@Override
......@@ -589,31 +587,30 @@ public class EventServiceImpl implements EventService {
}
@Override
public List<JSONObject> getEventChannelVoices(String eventId, String type, int page, int pageSize, String sorter) {
public PageVO<JSONObject> getEventChannelVoices(String eventId, String type, int page, int pageSize, String sorter) {
Event event = getEventById(eventId);
Query query = Query.query(Criteria.where("eventId").is(eventId));
if (Objects.equals("重要渠道", type)) {
query.addCriteria(Criteria.where("mediaType").in(IMPORTANT_CHANNEL_LIST));
}
mongoUtil.start(page, pageSize, query);
// 总数
// 总量
long total = eventDataDao.count(query, event.getCollectionName());
mongoUtil.start(page, pageSize, query);
// 排序
eventDataDao.addSort(query, sorter);
// 数据
List<EventData> eventDataList = eventDataDao.findList(query, event.getCollectionName());
// 分组排序
Map<String, List<EventData>> groupMap = new HashMap<>();
for (EventData eventData : eventDataList) {
groupMap.putIfAbsent(eventData.getMediaType(), new ArrayList<>());
groupMap.get(eventData.getMediaType()).add(eventData);
}
return groupMap.entrySet().stream().map(entry -> {
List<JSONObject> collect = eventDataList.stream().map(data -> {
JSONObject json = new JSONObject();
json.put("type", entry.getKey());
json.put("list", entry.getValue());
json.put("title", data.getTitle());
json.put("url", data.getUrl());
json.put("time", data.getTime());
json.put("source", data.getSource());
json.put("emotion", data.getEmotion());
json.put("mediaType", data.getMediaType());
return json;
}).collect(Collectors.toList());
return PageVO.createPageVo(total, page, pageSize, collect);
}
@Override
......@@ -632,7 +629,7 @@ public class EventServiceImpl implements EventService {
switch (type) {
case "按时间":
Map<String, List<EventTopArticlesAnalysisVO>> collect = eventTopArticlesAnalyses.stream().map(eventTopArticlesAnalysis -> {
boolean highLight = Objects.equals(aggTitle, eventTopArticlesAnalysis.getAggTitle());
boolean highLight = null == aggTitle || Objects.equals(aggTitle, eventTopArticlesAnalysis.getAggTitle());
return new EventTopArticlesAnalysisVO(eventTopArticlesAnalysis, highLight);
}).collect(Collectors.groupingBy(EventTopArticlesAnalysisVO::getTimePoint));
res.put("dataDay", collect);
......@@ -647,6 +644,21 @@ public class EventServiceImpl implements EventService {
}
}
@Override
public List<JSONObject> getLastEventTop(String keyword, int limit) {
Query query = Query.query(Criteria.where("projectId").is(UserThreadLocal.getProjectId()));
eventDao.addKeywordFuzz(query, keyword, "title");
eventDao.addSort(query, "{\"startTime\":\"descend\"}");
query.limit(limit);
return eventDao.findList(query).stream().map(event -> {
JSONObject result = new JSONObject();
result.put("id", event.getId());
result.put("title", event.getTitle());
result.put("total", event.getTotalDisseminationVolume());
return result;
}).sorted((o1, o2) -> Long.compare((long) o2.get("total"), (long) o1.get("total"))).collect(Collectors.toList());
}
/**
* 获取时间筛选条件
*
......
......@@ -99,7 +99,7 @@ public class IndexServiceImpl implements IndexService {
jsonObject.put("compare", lastRangeCount == 0 ? 0d : (rangeCount - lastRangeCount) / (double) lastRangeCount);
List<Map<String, Long>> dayList = Tools.parseToDaysWithBase(startTime, endTime);
jsonObject.put("spread", this.getArticleSpreadWithBrand(projectId, Constant.PRIMARY_CONTEND_ID, dayList));
redisUtil.set(redisKey, JSON.toJSONString(jsonObject));
redisUtil.setExpire(redisKey, JSON.toJSONString(jsonObject));
} catch (Exception e) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("getYuqingAmount异常"), e);
}
......@@ -156,7 +156,7 @@ public class IndexServiceImpl implements IndexService {
jsonObject.put("compare", oldPosPro == 0 ? 0d : (posPro - oldPosPro) / oldPosPro);
double avgPosPro = totalNormalCount == 0 ? 0d : totalPositiveCount / (double) totalNormalCount;
jsonObject.put("avgPosPro", avgPosPro);
redisUtil.set(redisKey, JSON.toJSONString(jsonObject));
redisUtil.setExpire(redisKey, JSON.toJSONString(jsonObject));
} catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("es查询异常"), e);
}
......@@ -212,7 +212,7 @@ public class IndexServiceImpl implements IndexService {
}
}
jsonObject.put("spread", lineList);
redisUtil.set(redisKey, JSONObject.toJSONString(jsonObject));
redisUtil.setExpire(redisKey, JSONObject.toJSONString(jsonObject));
} catch (Exception e) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("getEventAmount异常"), e);
}
......@@ -237,7 +237,7 @@ public class IndexServiceImpl implements IndexService {
try {
long normalCount = markDataService.getYuqingMarkCount(startTime, endTime, EmotionEnum.ALL.getName(), projectId, contendId);
List<JSONObject> platforms = getPlatformProportionWithPlatform(startTime, endTime, EmotionEnum.ALL.getName(), projectId, contendId, normalCount);
redisUtil.set(redisKey, JSON.toJSONString(platforms));
redisUtil.setExpire(redisKey, JSON.toJSONString(platforms));
return platforms;
} catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("es查询异常"), e);
......@@ -247,6 +247,11 @@ public class IndexServiceImpl implements IndexService {
@Override
public JSONObject getSpreadingTend(Long startTime, Long endTime, boolean cache) {
return getSpreadingTend(startTime, endTime, UserThreadLocal.getProjectId(), "0", cache);
}
@Override
public JSONObject getSpreadingTend(Long startTime, Long endTime, String projectId, String contendId, boolean cache) {
boolean flag = true;
if (null == startTime || null == endTime) {
Long[] timeRangeMonth = commonService.getTimeRangeMonth();
......@@ -254,9 +259,7 @@ public class IndexServiceImpl implements IndexService {
endTime = timeRangeMonth[1];
flag = false;
}
String projectId = UserThreadLocal.getProjectId();
String linkedGroupId = projectService.getProjectById(projectId).getBrandLinkedGroupId();
String contendId = "0";
String redisKey = RedisUtil.getIndexSpread(projectId, startTime, endTime);
String resultStr;
// 返回缓存
......@@ -287,7 +290,7 @@ public class IndexServiceImpl implements IndexService {
summary.put("topEvent", this.getTopEvent(startTime, endTime, projectId, contendId));
summary.put("topSource", this.getTopSource(startTime, endTime, projectId, linkedGroupId, contendId));
resJson.put("summary", summary);
redisUtil.set(redisKey, JSON.toJSONString(resJson));
redisUtil.setExpire(redisKey, JSON.toJSONString(resJson));
} catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("es查询异常"), e);
}
......@@ -305,7 +308,7 @@ public class IndexServiceImpl implements IndexService {
} catch (IOException ignored) {
count = -1;
}
result.put("platform", platformId);
result.put("platform", GlobalPojo.getPlatformNameById(platformId));
result.put("num", count);
result.put("proportion", normalCount == 0 ? 0d : count / (double) normalCount);
resultList.add(result);
......
......@@ -566,17 +566,13 @@ public class MarkDataServiceImpl implements MarkDataService {
String contendId = dto.getContendId();
// PostFilter 后置过滤器
// BoolQueryBuilder postFilter = projectLinkedGroupQuery(projectId, linkedGroupId);
BoolQueryBuilder postFilter = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, contendId);
BoolQueryBuilder postFilter = EsQueryTools.assembleCacheMapsQuery(projectId, linkedGroupId, contendId, dto.getMediaTypes());
// time
postFilter.must(QueryBuilders.rangeQuery(dto.getTimeType()).gte(dto.getStartTime()).lt(dto.getEndTime()));
// platform
if (CollectionUtils.isNotEmpty(dto.getPlatforms())) {
postFilter.must(EsQueryTools.assemblePlatformQuery(Tools.getPlatformByIds(dto.getPlatforms())));
}
// mediaTypes
if (CollectionUtils.isNotEmpty(dto.getMediaTypes())) {
postFilter.must(EsQueryTools.assembleMediaTypeQuery(dto.getMediaTypes()));
}
// tags
if (CollectionUtils.isNotEmpty(dto.getTags())) {
postFilter.must(EsQueryTools.assembleTagQuery(dto.getTags()));
......@@ -735,13 +731,12 @@ public class MarkDataServiceImpl implements MarkDataService {
// 聚合查询
TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("count").field("brandkbs_mark_cache_maps.name.keyword");
// query
BoolQueryBuilder query = QueryBuilders.boolQuery();
BoolQueryBuilder query = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, "0");
// keyword
if (StringUtils.isNotEmpty(keyword)) {
String[] fieldSearch = "标题".equals(searchField) ? new String[]{GenericAttribute.ES_IND_TITLE} : new String[]{GenericAttribute.ES_IND_FULL_TEXT};
query.must(EsQueryTools.assembleNormalKeywordQuery(keyword, fieldSearch));
}
query.must(QueryBuilders.termQuery("brandkbs_cache_maps.linked_group_id.keyword", linkedGroupId)).must(QueryBuilders.termQuery("brandkbs_cache_maps.project_id.keyword", projectId));
// time range
query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
......@@ -807,7 +802,7 @@ public class MarkDataServiceImpl implements MarkDataService {
BoolQueryBuilder query = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, contendId);
query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime))
// 过滤微博
.mustNot(QueryBuilders.termQuery("platform_id.keyword", "5d02236e6395002a7c380b79"));
.mustNot(QueryBuilders.termQuery("platform_id", "5d02236e6395002a7c380b79"));
if (null != emotion && !Objects.equals(emotion, EmotionEnum.ALL.getName())) {
query.must(QueryBuilders.termQuery("brandkbs_mark_cache_maps.name.keyword", emotion));
}
......@@ -865,7 +860,7 @@ public class MarkDataServiceImpl implements MarkDataService {
timeBuilder.lt(endTime);
}
query.must(timeBuilder);
return esClientDao.searchScroll(query, 10000, fetchSource);
return esClientDao.searchScroll(query, 2000, fetchSource);
} catch (Exception e) {
log.error("searchByEvent-", e);
}
......@@ -958,17 +953,13 @@ public class MarkDataServiceImpl implements MarkDataService {
String contendId = dto.getContendId();
String contendLinkedGroupId = projectService.getProjectByContendId(projectId, dto.getContendId()).getBrandLinkedGroupId();
// PostFilter 后置过滤器
BoolQueryBuilder postFilter = projectLinkedGroupContendIdQuery(projectId, contendLinkedGroupId, contendId);
BoolQueryBuilder postFilter = EsQueryTools.assembleCacheMapsQuery(projectId, contendLinkedGroupId, contendId, dto.getMediaTypes());
// time
postFilter.must(QueryBuilders.rangeQuery(dto.getTimeType()).gte(dto.getStartTime()).lt(dto.getEndTime()));
// platform
if (CollectionUtils.isNotEmpty(dto.getPlatforms())) {
postFilter.must(EsQueryTools.assemblePlatformQuery(Tools.getPlatformByIds(dto.getPlatforms())));
}
// mediaTypes
if (CollectionUtils.isNotEmpty(dto.getMediaTypes())) {
postFilter.must(EsQueryTools.assembleMediaTypeQuery(dto.getMediaTypes()));
}
// tags
if (CollectionUtils.isNotEmpty(dto.getTags())) {
postFilter.must(EsQueryTools.assembleTagQuery(dto.getTags()));
......@@ -1359,7 +1350,7 @@ public class MarkDataServiceImpl implements MarkDataService {
return url;
}
private void defaultSearchFilter(SearchFilterDTO dto){
private void defaultSearchFilter(SearchFilterDTO dto) {
String projectId = UserThreadLocal.getProjectId();
dto.setProjectId(projectId);
if (Objects.isNull(dto.getStartTime()) || Objects.isNull(dto.getEndTime())) {
......@@ -1368,15 +1359,15 @@ public class MarkDataServiceImpl implements MarkDataService {
dto.setStartTime(DateUtils.addDays(now, -1).getTime());
}
// 平台为空时,传全部
if (dto.getPlatforms().size() == 0 ){
dto.setPlatforms(Arrays.asList("微博","微信","新闻","平媒","自媒体","论坛","视频","短视频"));
if (dto.getPlatforms().size() == 0) {
dto.setPlatforms(Arrays.asList("微博", "微信", "新闻", "平媒", "自媒体", "论坛", "视频", "短视频"));
}
// 平台
dto.setPlatformsStr(StringUtils.join(dto.getPlatforms(), ","));
// sort
dto.setSortOf("ctime+" + dto.getSorter());
// matchFields
if(dto.getMatchType().equals("标题")){
if (dto.getMatchType().equals("标题")) {
dto.setMatchFields("title");
}
}
......@@ -1399,7 +1390,7 @@ public class MarkDataServiceImpl implements MarkDataService {
filter.must(QueryBuilders.termQuery("brandkbs_mark_cache_maps.name.keyword", emotion));
}
if (null != platformId) {
filter.must(QueryBuilders.termQuery("platform_id.keyword", platformId));
filter.must(QueryBuilders.termQuery("platform_id", platformId));
}
return esClientDao.count(filter);
}
......@@ -1526,8 +1517,9 @@ public class MarkDataServiceImpl implements MarkDataService {
DateHistogramAggregationBuilder daysAggregationBuilder = AggregationBuilders.dateHistogram("dayAgg").field("time").calendarInterval(DateHistogramInterval.DAY);
TermsAggregationBuilder emotionAggregationBuilder = AggregationBuilders.terms("count").field("brandkbs_mark_cache_maps.name.keyword");
// query
BoolQueryBuilder query = QueryBuilders.boolQuery();
query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime)).must(QueryBuilders.termQuery("brandkbs_cache_maps.project_id.keyword", projectId)).must(QueryBuilders.termQuery("brandkbs_cache_maps.linked_group_id.keyword", linkedGroupId));
BoolQueryBuilder query = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, "0");
// time
query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
// 情感倾向限制
query.must(QueryBuilders.termQuery("brandkbs_mark_cache_maps.group_name.keyword", "情感倾向"));
// response
......@@ -1561,10 +1553,10 @@ public class MarkDataServiceImpl implements MarkDataService {
// 索引
String[] indexes = esClientDao.getIndexes();
// 聚合请求
TermsAggregationBuilder platformAggregationBuilder = AggregationBuilders.terms("platform_count").field("platform_id.keyword").order(BucketOrder.count(false));
TermsAggregationBuilder platformAggregationBuilder = AggregationBuilders.terms("platform_count").field("platform_id").order(BucketOrder.count(false));
TermsAggregationBuilder emotionAggregationBuilder = AggregationBuilders.terms("emotion_count").field("brandkbs_mark_cache_maps.name.keyword");
// postFilter
BoolQueryBuilder query = projectLinkedGroupQuery(projectId, linkedGroupId);
BoolQueryBuilder query = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, "0");
// time
query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
// 情感倾向限制
......@@ -1631,7 +1623,7 @@ public class MarkDataServiceImpl implements MarkDataService {
// 获取索引
String[] indexes = esClientDao.getIndexes();
// 聚合请求
TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("count").field("platform_id.keyword").order(BucketOrder.count(false)).size(100);
TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("count").field("platform_id").order(BucketOrder.count(false)).size(100);
// query
BoolQueryBuilder query = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, contendId);
// keyword
......@@ -1669,9 +1661,9 @@ public class MarkDataServiceImpl implements MarkDataService {
EsClientDao.SearchHelper searchHelper = EsClientDao.createSearchHelper();
BoolQueryBuilder postFilter = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, "0");
postFilter.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
postFilter.must(QueryBuilders.existsQuery("brandkbs_cache_maps.channel_emotion"));
postFilter.must(EsQueryTools.cacheMapsNestedQuery(QueryBuilders.existsQuery("brandkbs_cache_maps.channel_emotion")));
searchHelper.setPostFilter(postFilter);
searchHelper.setFetchSource(new String[]{"brandkbs_cache_maps"});
searchHelper.setFetchSource(new String[]{GenericAttribute.ES_BRANDKBS_CACHE_MAPS});
List<SearchResponse> searchResponses = esClientDao.searchScrollResponse(searchHelper);
Map<Integer, AtomicInteger> emotionMap = new HashMap<>();
emotionMap.put(ChannelEmotion.POSITIVE.getState(), new AtomicInteger());
......@@ -1701,7 +1693,7 @@ public class MarkDataServiceImpl implements MarkDataService {
for (ImportantChannelEnum channelEnum : mediaTypeList) {
BoolQueryBuilder postFilter = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, "0");
postFilter.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
postFilter.must(QueryBuilders.matchQuery("brandkbs_cache_maps.channel_type.keyword", channelEnum.getState()));
postFilter.must(EsQueryTools.cacheMapsNestedQuery(QueryBuilders.matchQuery("brandkbs_cache_maps.channel_type.keyword", channelEnum.getState())));
Long count = esClientDao.count(postFilter);
mediaTypeMap.put(channelEnum.name().toLowerCase() + "Percent", count);
}
......@@ -1719,7 +1711,7 @@ public class MarkDataServiceImpl implements MarkDataService {
// fetchSource
searchHelper.setFetchSource(new String[]{"ind_title", "ind_full_text", "c5", "foreign", "brandkbs_mark_cache_maps"});
// postFilter
BoolQueryBuilder postFilter = projectLinkedGroupQuery(projectId, linkedGroupId);
BoolQueryBuilder postFilter = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, "0");
postFilter.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
searchHelper.setPostFilter(postFilter);
searchHelper.setSize(10000);
......@@ -1737,13 +1729,10 @@ public class MarkDataServiceImpl implements MarkDataService {
return textList;
}
@Deprecated
private static BoolQueryBuilder projectLinkedGroupQuery(String projectId, String linkedGroupId) {
return QueryBuilders.boolQuery().must(QueryBuilders.termQuery("brandkbs_cache_maps.project_id.keyword", projectId)).must(QueryBuilders.termQuery("brandkbs_cache_maps.linked_group_id.keyword", linkedGroupId));
}
protected static BoolQueryBuilder projectLinkedGroupContendIdQuery(String projectId, String linkedGroupId, String contendId) {
return QueryBuilders.boolQuery().must(QueryBuilders.termQuery("brandkbs_cache_maps.key.keyword", Tools.concat(projectId, linkedGroupId, contendId)));
return QueryBuilders.boolQuery().must(QueryBuilders.termQuery("brandkbs_cache_maps.key.keyword", Tools.concat(projectId, linkedGroupId, contendId))) ;
// TODO 上线后开启
// return EsQueryTools.assembleCacheMapsQuery(projectId, linkedGroupId, contendId);
}
private List<MarkFlowEntity> getMarkFlowEntity(MarkSearchDTO markSearchDTO, SearchHits searchHits) {
......
......@@ -5,19 +5,23 @@ import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Maps;
import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.common.RedisKeyPrefix;
import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.dao.EventDao;
import com.zhiwei.brandkbs2.dao.EventDataDao;
import com.zhiwei.brandkbs2.dao.ReportDao;
import com.zhiwei.brandkbs2.dao.ReportSettingsDao;
import com.zhiwei.brandkbs2.enmus.EmotionEnum;
import com.zhiwei.brandkbs2.enmus.ReportTypeEnum;
import com.zhiwei.brandkbs2.exception.ExceptionCast;
import com.zhiwei.brandkbs2.listener.ApplicationProjectListener;
import com.zhiwei.brandkbs2.model.CommonCodeEnum;
import com.zhiwei.brandkbs2.pojo.AbstractProject;
import com.zhiwei.brandkbs2.pojo.Report;
import com.zhiwei.brandkbs2.pojo.ReportSettings;
import com.zhiwei.brandkbs2.pojo.*;
import com.zhiwei.brandkbs2.pojo.dto.ReportDTO;
import com.zhiwei.brandkbs2.pojo.dto.ReportSearchDTO;
import com.zhiwei.brandkbs2.pojo.dto.ReportSettingsDTO;
import com.zhiwei.brandkbs2.pojo.vo.PageVO;
import com.zhiwei.brandkbs2.service.IndexService;
import com.zhiwei.brandkbs2.service.MarkDataService;
import com.zhiwei.brandkbs2.service.ProjectService;
import com.zhiwei.brandkbs2.service.ReportService;
import com.zhiwei.brandkbs2.util.MongoUtil;
......@@ -25,6 +29,7 @@ import com.zhiwei.brandkbs2.util.RedisUtil;
import com.zhiwei.brandkbs2.util.Tools;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
......@@ -36,11 +41,10 @@ import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;
import static com.zhiwei.brandkbs2.config.Constant.HOUR_FORMAT;
/**
* @ClassName: ReportServiceImpl
* @Description 报告业务接口实现类
......@@ -57,9 +61,21 @@ public class ReportServiceImpl implements ReportService {
@Resource(name = "reportSettingsDao")
ReportSettingsDao reportSettingsDao;
@Resource(name = "eventDao")
EventDao eventDao;
@Resource(name = "eventDataDao")
EventDataDao eventDataDao;
@Resource(name = "projectServiceImpl")
ProjectService projectService;
@Resource(name = "markDataServiceImpl")
MarkDataService markDataService;
@Resource(name = "indexServiceImpl")
IndexService indexService;
@Resource(name = "mongoUtil")
com.zhiwei.brandkbs2.util.MongoUtil mongoUtil;
......@@ -132,8 +148,7 @@ public class ReportServiceImpl implements ReportService {
List<JSONObject> result = new ArrayList<>();
Map<String, JSONObject> map = new HashMap<>();
// Map<String, List<JSONObject>> collect = list.stream().collect(Collectors.groupingBy(json -> json.getString(groupName)));
Map<String, Integer> collect = list.stream().collect(Collectors.toMap(json -> json.getString(groupName), json -> json.getInteger(countName),
Integer::sum));
Map<String, Integer> collect = list.stream().collect(Collectors.toMap(json -> json.getString(groupName), json -> json.getInteger(countName), Integer::sum));
// collect.put("不限", collect.values().stream().mapToInt(v -> v).sum());
collect.forEach((type, count) -> {
JSONObject json = new JSONObject();
......@@ -213,8 +228,13 @@ public class ReportServiceImpl implements ReportService {
result.put("status", false);
return result;
}
JSONObject result = getPcReportResult(report);
stringRedisTemplate.opsForValue().set(redisKey, JSON.toJSONString(result));
JSONObject result = null;
try {
result = getPcReportResult(report);
} catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("es查询异常"));
}
redisUtil.set(redisKey, JSON.toJSONString(result));
return result;
}
......@@ -273,41 +293,190 @@ public class ReportServiceImpl implements ReportService {
* @param report 报告对象
* @return 月报结果
*/
private JSONObject getPcReportResult(Report report) {
private JSONObject getPcReportResult(Report report) throws IOException {
Long startTime = report.getStartTime();
Long endTime = report.getEndTime();
String startTimeStr = HOUR_FORMAT.format(startTime);
String endTimeStr = HOUR_FORMAT.format(endTime);
String lastStartTimeStr = HOUR_FORMAT.format(startTime - (endTime - startTime));
Long lastStartTimeStr = startTime - (endTime - startTime);
String projectId = report.getProjectId();
String contendId = "0";
String linkedGroupId = projectService.getProjectByContendId(projectId, contendId).getBrandLinkedGroupId();
JSONObject result = new JSONObject();
result.put("title", report.getTitle());
result.put("startTime", startTime);
result.put("endTime", endTime);
result.put("status", true);
result.put("type", ReportTypeEnum.getInstanceByState(report.getType()).getDescribe());
// TODO
result.put("todo", "尚未完成");
result.put("brandSummary", this.getPcBrandSummary(startTime, endTime, lastStartTimeStr, projectId, linkedGroupId, contendId));
List<Map<String, Long>> dayList = Tools.parseToDays(startTime, DateUtils.addDays(new Date(endTime), -1).getTime());
result.put("brandSpread", this.getPcBrandSpread(dayList, projectId, linkedGroupId, contendId));
result.put("brandEvent", this.getPcBrandEvent(startTime, endTime, projectId, contendId));
List<JSONObject> contendCompare = report.getContends().stream().map(brandContendId -> {
JSONObject json = new JSONObject();
AbstractProject abstractProject = projectService.getProjectByContendId(projectId, brandContendId);
json.put("id", brandContendId);
json.put("brand", abstractProject.getBrandName());
json.put("brandCompare", indexService.getSpreadingTend(startTime, endTime, projectId, contendId, true));
return json;
}).collect(Collectors.toList());
result.put("contendCompare", contendCompare);
return result;
}
/**
* 获取品牌总览页面数据
*
* @param startTime 开始时间
* @param endTime 结束时间
* @param lastStartTime 上个月开始时间
* @param projectId 项目ID
* @return 总览页面数据
*/
private JSONObject getPcBrandSummary(Long startTime, Long endTime, Long lastStartTime, String projectId, String linkedGroupId, String contendId) throws IOException {
JSONObject result = new JSONObject();
//本月舆情总量数据
long curTotal = markDataService.getYuqingMarkCount(startTime, endTime, EmotionEnum.ALL.getName(), projectId, contendId);
result.put("curTotal", curTotal);
//上月舆情总量数据
long lastTotal = markDataService.getYuqingMarkCount(lastStartTime, startTime, EmotionEnum.ALL.getName(), projectId, contendId);
result.put("lastTotal", lastTotal);
result.put("compare", lastTotal == 0 ? 0 : (curTotal - lastTotal) * 1.0 / lastTotal);
//获取时间范围内总正面稿件数
long curPositiveTotal = markDataService.getYuqingMarkCount(startTime, endTime, EmotionEnum.POSITIVE.getName(), projectId, contendId);
result.put("curPosTotal", curPositiveTotal);
result.put("curPosPro", curTotal == 0 ? 0 : curPositiveTotal * 1.0 / curTotal);
//正面事件传播量top
List<Event> topPosEventList = eventDao.getEventsByTotalChannelVolumeTop(startTime, endTime, EmotionEnum.POSITIVE.getName(), projectId, contendId, 3);
if (CollectionUtils.isEmpty(topPosEventList)) {
List<Map.Entry<String, Integer>> topPosArticleList = markDataService.getMarkTopTitle(startTime, endTime, EmotionEnum.POSITIVE.getName(), projectId, linkedGroupId, contendId, 3);
result.put("topPosSummary", this.getTopArticlesMsg(startTime, endTime, projectId, linkedGroupId, contendId, topPosArticleList));
} else {
result.put("topPosSummary", this.getTopEventMsg(topPosEventList));
}
//获取上个周期时间范围内总正面稿件数
long lastPositiveTotal = markDataService.getYuqingMarkCount(lastStartTime, startTime,
EmotionEnum.POSITIVE.getName(), projectId, contendId);
result.put("lastPosTotal", lastPositiveTotal);
result.put("posCompare", lastPositiveTotal == 0 ? 0 : (curPositiveTotal - lastPositiveTotal) * 1.0 / lastPositiveTotal);
//获取时间范围内总中性稿件数
long curNeutralTotal = markDataService.getYuqingMarkCount(startTime, endTime, EmotionEnum.NEUTRAL.getName(), projectId, contendId);
result.put("curNeuTotal", curNeutralTotal);
result.put("curNeuPro", curTotal == 0 ? 0 : curNeutralTotal * 1.0 / curTotal);
//中性事件传播量top
List<Event> topNeuEventList = eventDao.getEventsByTotalChannelVolumeTop(startTime, endTime, EmotionEnum.NEUTRAL.getName(), projectId, contendId, 4);
if (CollectionUtils.isEmpty(topNeuEventList)) {
List<Map.Entry<String, Integer>> topNeuArticleList = markDataService.getMarkTopTitle(startTime, endTime, EmotionEnum.NEUTRAL.getName(), projectId,
linkedGroupId, contendId, 4);
result.put("topNeuSummary", this.getTopArticlesMsg(startTime, endTime, projectId, linkedGroupId, contendId, topNeuArticleList));
} else {
result.put("topNeuSummary", this.getTopEventMsg(topNeuEventList));
}
//获取上个周期时间范围内总中性稿件数
long lastNeutralTotal = markDataService.getYuqingMarkCount(lastStartTime, startTime, EmotionEnum.NEUTRAL.getName(), projectId, contendId);
result.put("lastNeuTotal", lastNeutralTotal);
result.put("neuCompare", lastNeutralTotal == 0 ? 0 : (curNeutralTotal - lastNeutralTotal) * 1.0 / lastNeutralTotal);
//获取时间范围内总负面稿件数
long curNegativeTotal = markDataService.getYuqingMarkCount(startTime, endTime, EmotionEnum.NEGATIVE.getName(), projectId, contendId);
result.put("curNegTotal", curNegativeTotal);
result.put("curNegPro", curTotal == 0 ? 0 : curNegativeTotal * 1.0 / curTotal);
//中性事件传播量top
List<Event> topNegEventList = eventDao.getEventsByTotalChannelVolumeTop(startTime, endTime, EmotionEnum.NEGATIVE.getName(), projectId, contendId, 4);
if (CollectionUtils.isEmpty(topNegEventList)) {
List<Map.Entry<String, Integer>> topNegArticleList = markDataService.getMarkTopTitle(startTime, endTime, EmotionEnum.NEGATIVE.getName(),
projectId, linkedGroupId, contendId, 4);
result.put("topNegSummary", this.getTopArticlesMsg(startTime, endTime, projectId, linkedGroupId, contendId, topNegArticleList));
} else {
result.put("topNegSummary", this.getTopEventMsg(topNegEventList));
}
//获取上个周期时间范围内总负面稿件数
long lastNegativeTotal = markDataService.getYuqingMarkCount(lastStartTime, startTime, EmotionEnum.NEGATIVE.getName(), projectId, contendId);
result.put("lastNegTotal", lastNegativeTotal);
result.put("negCompare", lastNegativeTotal == 0 ? 0 : (curNegativeTotal - lastNegativeTotal) * 1.0 / lastNegativeTotal);
return result;
}
/**
* 获取报告类型聚合结果
* 获取品牌传播分析页面数据
*
* @param pid 项目ID
* @param uid 用户ID
* @return 报告类型聚合结果
* @param dayList 分段时间集合
* @param projectId 项目ID
* @return 传播分析页面数据
*/
private List<JSONObject> getReportTypeAggCount(int pid, int uid) {
List<String> types = Arrays.stream(ReportTypeEnum.values()).map(ReportTypeEnum::getState).collect(Collectors.toList());
return types.stream()
.map(type -> {
JSONObject result = new JSONObject();
result.put("type", type);
// result.put("count", reportDao.selectReportCountByTimeAndTypeAndProjectIdAndUserId(null, null, type, pid, uid));
return result;
})
.filter(json -> json.getIntValue("count") > 0)
private JSONObject getPcBrandSpread(List<Map<String, Long>> dayList, String projectId, String linkedGroupId, String contendId) throws IOException {
JSONObject result = new JSONObject();
List<JSONObject> lineList = new ArrayList<>(dayList.size());
//获取传播趋势
for (Map<String, Long> map : dayList) {
Long startTime = map.get("startTime");
Long endTime = map.get("endTime");
//统计时间段总稿件个数
long normalCount = markDataService.getYuqingMarkCount(startTime, endTime, EmotionEnum.ALL.getName(), projectId, contendId);
//统计时间段正面稿件个数
long posNormalCount = markDataService.getYuqingMarkCount(startTime, endTime, EmotionEnum.POSITIVE.getName(), projectId, contendId);
//统计时间段中性稿件个数
long neuNormalCount = markDataService.getYuqingMarkCount(startTime, endTime, EmotionEnum.NEUTRAL.getName(), projectId, contendId);
//统计时间段负面稿件个数
long negNormalCount = markDataService.getYuqingMarkCount(startTime, endTime, EmotionEnum.NEGATIVE.getName(), projectId, contendId);
JSONObject line = new JSONObject();
line.put("normalCount", normalCount);
line.put("posNormalCount", posNormalCount);
line.put("neuNormalCount", neuNormalCount);
line.put("negNormalCount", negNormalCount);
line.put("time", startTime);
lineList.add(line);
}
result.put("spread", lineList);
//获取稿件的传播峰值
List<Long> dateList = lineList.stream()
.sorted((o1, o2) -> o2.getInteger("normalCount").compareTo(o1.getInteger("normalCount")))
.map(json -> json.getLong("time"))
.collect(Collectors.toList());
result.put("normalMaxTimes", dateList.size() > 2 ? dateList.subList(0, 3) : dateList);
if (CollectionUtils.isNotEmpty(dateList)) {
Long startTime = dateList.get(0);
Long endTime = DateUtils.addDays(new Date(startTime), 1).getTime();
//获取最热稿件相关信息
List<Map.Entry<String, Integer>> resList = markDataService.getMarkTopTitle(startTime, endTime, EmotionEnum.ALL.getName(), projectId, linkedGroupId, contendId, 1);
if (CollectionUtils.isNotEmpty(resList)) {
Map.Entry<String, Integer> entry = resList.get(0);
BaseMap baseMap = markDataService.getFirstArticle(startTime, endTime, entry.getKey(), projectId, linkedGroupId, contendId);
result.put("topArticle", baseMap.getTitle());
result.put("topArticleUrl", baseMap.getUrl());
result.put("topArticleEmotion", baseMap.getEmotion());
}
}
return result;
}
/**
* 获取品牌事件数据
*
* @param startTime 开始时间
* @param endTime 结束时间
* @param projectId 项目ID
* @return 品牌事件数据
*/
private List<JSONObject> getPcBrandEvent(Long startTime, Long endTime, String projectId, String contendId) {
List<String> emotions = Arrays.asList(EmotionEnum.POSITIVE.getName(), EmotionEnum.NEGATIVE.getName());
return emotions.stream().map(emotion -> {
JSONObject result = new JSONObject();
result.put("name", Objects.equals(emotion, EmotionEnum.POSITIVE.getName()) ? "正面事件" : Objects.equals(emotion, EmotionEnum.NEGATIVE.getName()) ? "负面事件" : "中性事件");
List<Event> topEventList = eventDao.getEventsByTotalChannelVolumeTop(startTime, endTime, emotion, projectId, contendId, 10);
result.put("events", topEventList.stream().map(event -> {
JSONObject topEvent = new JSONObject();
boolean hasAnalyze = Objects.equals(event.getContendId(), Constant.PRIMARY_CONTEND_ID);
topEvent.put("id", event.getId());
String title = event.getTitle();
topEvent.put("title", title.length() <= 30 ? title : title.substring(0, 30) + "...");
topEvent.put("influence", event.getInfluence());
topEvent.put("articleCount", event.getTotalDisseminationVolume());
//查询首发稿件
EventData firstData = eventDataDao.findFirstData(event.getId(), event.getCollectionName());
topEvent.put("firstSource", Objects.isNull(firstData) ? "" : firstData.getSource());
topEvent.put("hasAnalyze", hasAnalyze);
return topEvent;
}).collect(Collectors.toList()));
return result;
}).collect(Collectors.toList());
}
private List<ReportSettings> getReportSettingsList() {
......@@ -319,4 +488,49 @@ public class ReportServiceImpl implements ReportService {
return reports;
}
/**
* 获取top稿件信息
*
* @param startTime 开始时间
* @param endTime 结束时间
* @param projectId 项目ID
* @param topArticleList top稿件集合
* @return top稿件信息
*/
private List<JSONObject> getTopArticlesMsg(Long startTime, Long endTime, String projectId, String linkedGroupId, String contendId, List<Map.Entry<String, Integer>> topArticleList) {
return topArticleList.stream().map(entry -> {
JSONObject topPosArticle = null;
try {
String aggTitle = entry.getKey();
BaseMap firstArticle = markDataService.getFirstArticle(startTime, endTime, aggTitle, projectId, linkedGroupId, contendId);
topPosArticle = new JSONObject();
topPosArticle.put("title", firstArticle.getTitle());
topPosArticle.put("url", firstArticle.getUrl());
} catch (IOException e) {
log.error("getTopArticlesMsg-getFirstArticle-", e);
}
return topPosArticle;
}).filter(Objects::nonNull).collect(Collectors.toList());
}
/**
* 获取top事件信息
*
* @param topEventList top事件集合
* @return top事件信息
*/
private List<JSONObject> getTopEventMsg(List<Event> topEventList) {
return topEventList.stream()
.map(event -> {
boolean hasAnalyze = Objects.equals(event.getContendId(), Constant.PRIMARY_CONTEND_ID);
JSONObject jsonObject = new JSONObject();
jsonObject.put("id", event.getId());
String title = event.getTitle();
jsonObject.put("title", title.length() <= 30 ? title : title.substring(0, 30) + "...");
jsonObject.put("hasAnalyze", hasAnalyze);
return jsonObject;
}).collect(Collectors.toList());
}
}
......@@ -37,7 +37,6 @@ import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
......@@ -446,40 +445,13 @@ public class Tools {
/**
* 日期取整,去掉时分秒
*
* @param time
* @param pattern
* @return
*/
public static Long truncDate(Long time, String pattern){
Long result = null;
if (time != null) {
Date date = new Date(time);
Calendar c = Calendar.getInstance();
c.setTime(date);
if (!pattern.contains("yyyy")) {
c.set(Calendar.YEAR, 1970);
}
if (!pattern.contains("MM")) {
c.set(Calendar.MONTH, 0);
}
if (!pattern.contains("dd")) {
c.set(Calendar.DAY_OF_MONTH, 1);
}
if (!pattern.contains("HH")) {
c.set(Calendar.HOUR_OF_DAY, 0);
}
if (!pattern.contains("mm")) {
c.set(Calendar.MINUTE, 0);
}
if (!pattern.contains("ss")) {
c.set(Calendar.SECOND, 0);
}
if (!pattern.contains("SSS")) {
c.set(Calendar.MILLISECOND, 0);
}
result = c.getTime().getTime();
}
return result;
public static Long truncDate(Long time, String pattern) {
return truncDate(new Date(time), pattern).getTime();
}
/**
......@@ -821,7 +793,6 @@ public class Tools {
* @return 按小时分割的map集合
*/
public static List<Map<String, Long>> parseToHours(Long startTime, Long endTime) {
FastDateFormat df = HOUR_FORMAT;
Date start = Tools.truncDate(new Date(startTime), Constant.HOUR_PATTERN);
Date end = Tools.truncDate(new Date(endTime), Constant.HOUR_PATTERN);
Period periodHour = new Period(start.getTime(), end.getTime(), PeriodType.hours());
......@@ -881,13 +852,15 @@ public class Tools {
*/
public static boolean isContains(List<String> keywords, String content) {
boolean contains = true;
content = content.toLowerCase();
if (null != keywords) {
content = content.toLowerCase();
// 按空格分割必须全部命中
for (String keyword : keywords) {
for (String word : keyword.split(" ")) {
if (!content.contains(word.toLowerCase())) {
//一个不命中,直接返回false
return false;
for (String keyword : keywords) {
for (String word : keyword.split(" ")) {
if (!content.contains(word.toLowerCase())) {
//一个不命中,直接返回false
return false;
}
}
}
}
......@@ -895,7 +868,7 @@ public class Tools {
}
public static JSONObject getBrandkbsHitMap(Map<String, Object> esMap, String hitKey) {
List<Map<String, Object>> cacheMaps = (List<Map<String, Object>>) esMap.get("brandkbs_cache_maps");
List<Map<String, Object>> cacheMaps = (List<Map<String, Object>>) esMap.get(GenericAttribute.ES_BRANDKBS_CACHE_MAPS);
for (Map<String, Object> cacheMap : cacheMaps) {
if (hitKey.equals(cacheMap.get("key"))) {
return new JSONObject(cacheMap);
......
spring.profiles.active=pro
spring.profiles.active=local
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment