Skip to content

B
brandkbs2
Commit 08a6dad5 by shenjunjie
Options

Merge branch 'feature' into 'dev' 

Feature

See merge request !28
parents 2a9e8f84 590feb2f
Showing with 853 additions and 177 deletions
pom.xml
......@@ -232,6 +232,11 @@
<artifactId>nlp-aggree</artifactId>
<version>0.0.5-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.zhiwei.tools</groupId>
<artifactId>zhiwei-tools</artifactId>
<version>0.3.3-SNAPSHOT</version>
</dependency>
</dependencies>
<build>
......
src/main/java/com/zhiwei/brandkbs2/common/GenericAttribute.java
......@@ -15,7 +15,8 @@ public class GenericAttribute {
* es index
*/
public static final String ES_INDEX_PRE = "brandkbs2";
public static final String ES_INDEX_TEST = "brandkbs2_test";
// public static final String ES_INDEX_TEST = "brandkbs2_test";
public static final String ES_INDEX_TEST = "brandkbs2_2022";
public static final String ES_CHANNEL_INDEX_TEST = "brandkbs2_channel_record_test";
/**
* es ind_title
......@@ -34,7 +35,7 @@ public class GenericAttribute {
*/
public static final String ES_MEDIA_TYPE = "media_type";
// es platform_id
public static final String ES_PLATFORM_ID= "platform_id";
public static final String ES_PLATFORM_ID = "platform_id";
// es channel_influence
public static final String ES_CHANNEL_INDEX = "channel_influence";
/**
......@@ -50,33 +51,61 @@ public class GenericAttribute {
public static final String ES_CHANNEL_FID = "fid";
public static final String ES_CNAME = "cname";
/** es c2 **/
/**
* es c2
**/
public static final String ES_C2 = "c2";
/** es c4 **/
/**
* es c4
**/
public static final String ES_C4 = "c4";
/** es c5 **/
/**
* es c5
**/
public static final String ES_C5 = "c5";
/** es foreign **/
/**
* es foreign
**/
public static final String ES_FOREIGN = "foreign";
/** es source **/
/**
* es source
**/
public static final String ES_SOURCE = "source";
/** es real_source **/
/**
* es real_source
**/
public static final String ES_REAL_SOURCE = "real_source";
/** es time **/
/**
* es time
**/
public static final String ES_TIME = "time";
/** es ctime **/
/**
* es ctime
**/
public static final String ES_CTIME = "ctime";
/** es mtime **/
/**
* es mtime
**/
public static final String ES_MTIME = "mtime";
/** es mtag **/
/**
* es mtag
**/
public static final String ES_MTAG = "mtag";
/** es mgroup **/
/**
* es mgroup
**/
public static final String ES_MGROUP = "mgroup";
/** es forward **/
/**
* es forward
**/
public static final String ES_FORWARD = "forward";
/** es brandkbs_cache_maps **/
/**
* es brandkbs_cache_maps
**/
public static final String ES_BRANDKBS_CACHE_MAPS = "brandkbs_cache_maps";
/** es mark_cache_maps **/
/**
* es mark_cache_maps
**/
public static final String ES_MARK_CACHE_MAPS = "brandkbs_mark_cache_maps";
public static final String ES_LINKED_GROUP_ID = "linked_group_id";
......@@ -92,7 +121,7 @@ public class GenericAttribute {
public static final String USER_ID = "userId";
public static final String ROLE_ID = "roleId";
public static final String PROJECT_ID = "projectId";
public static final String AVATAR_URL= "avatarUrl";
public static final String AVATAR_URL = "avatarUrl";
// public enum ChannelParam{
// 负面稿件数("negativeArticles"),
......
src/main/java/com/zhiwei/brandkbs2/common/RedisKeyPrefix.java
......@@ -55,8 +55,8 @@ public class RedisKeyPrefix {
/**
* 热点相关缓存KEY
*/
public static final String HOT_RANKLIST = "brandkbs:hot:rankList:";
public static final String HOT_LIST = "brandkbs:hot:list:";
public static final String HOT_RANK_LIST = "BRANDKBS:HOT:RANK_LIST:";
public static final String HOT_LIST = "BRANDKBS:HOT:LIST:";
/**
* 项目简报报相关缓存KEY
......
src/main/java/com/zhiwei/brandkbs2/controller/app/AppArticleController.java
......@@ -175,4 +175,12 @@ public class AppArticleController extends BaseController {
return ResponseResult.success();
}
@ApiOperation("舆情简报-获取报告结果")
@ApiImplicitParam(name = "id", value = "报告ID", required = true, paramType = "path", dataType = "String")
@GetMapping("/analyze/{id}")
public ResponseResult getReportAnalyze(@PathVariable String id) {
return ResponseResult.success(reportService.getPcReportAnalyze(id, true));
}
}
src/main/java/com/zhiwei/brandkbs2/controller/app/AppEventController.java
......@@ -9,6 +9,7 @@ import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.dto.EventSearchDTO;
import com.zhiwei.brandkbs2.service.EventService;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiOperation;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
......@@ -138,4 +139,18 @@ public class AppEventController extends BaseController {
JSONObject jsonObject = JSON.parseObject(responseEntity.getBody());
return ResponseResult.success(jsonObject);
}
@ApiOperation("最新事件")
@GetMapping("/getLastEventTop")
public ResponseResult getLastEventTop() {
return ResponseResult.success(eventService.getLastEventTop(null, 5));
}
@ApiOperation("关联事件")
@ApiImplicitParam(name = "keyword", value = "关键词", paramType = "query", dataType = "String")
@GetMapping("/getRelevanceEvent")
public ResponseResult getRelevanceEvent(@RequestParam(value = "keyword", required = false) String keyword) {
return ResponseResult.success(eventService.getLastEventTop(keyword, 5));
}
}
src/main/java/com/zhiwei/brandkbs2/controller/app/AppHotController.java
......@@ -99,7 +99,7 @@ public class AppHotController extends BaseController {
public ResponseResult hot(){
ResponseEntity<String> responseEntity = restTemplate.getForEntity(longTimeInListSearchUrl, String.class,"weibo");
JSONObject jsonObject = JSON.parseObject(responseEntity.getBody());
String redisKey = RedisKeyPrefix.HOT_RANKLIST;
String redisKey = RedisKeyPrefix.HOT_RANK_LIST;
String result = redisUtil.get(redisKey);
//当舆论场崩溃时从缓存里获取
if(jsonObject.getBoolean("state")){
......
src/main/java/com/zhiwei/brandkbs2/controller/app/AppSearchController.java
......@@ -4,19 +4,25 @@ package com.zhiwei.brandkbs2.controller.app;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.auth.Auth;
import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.enmus.RoleEnum;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.dto.SearchFilterDTO;
import com.zhiwei.brandkbs2.service.MarkDataService;
import com.zhiwei.brandkbs2.util.Tools;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.apache.commons.lang3.time.DateUtils;
import org.joda.time.Period;
import org.joda.time.PeriodType;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.client.RestTemplate;
import javax.annotation.Resource;
import java.util.Date;
import java.util.Objects;
/**
......@@ -42,6 +48,9 @@ public class AppSearchController {
@Value("${ef.search.url}")
private String getEfSearchUrl;
@Resource(name = "markDataServiceImpl")
MarkDataService markDataService;
@ApiOperation("搜索-查热点")
@GetMapping("/hot/list")
public ResponseResult searchHotList(@RequestParam(value = "pageSize",defaultValue ="10") Integer limit,
......@@ -76,4 +85,18 @@ public class AppSearchController {
JSONObject result = JSON.parseObject(responseEntity.getBody());
return ResponseResult.success(result);
}
@ApiOperation("搜索-全网搜")
@GetMapping("/searchWhole")
public ResponseResult searchWholeNetwork(@RequestBody SearchFilterDTO dto){
long time = DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), -89).getTime();
if (time > dto.getStartTime()){
return ResponseResult.failure("仅能搜索近3个月内信息");
}
Period periodDay = new Period(dto.getStartTime(), dto.getEndTime(), PeriodType.days());
if (periodDay.getDays() > 30) {
return ResponseResult.failure("时间跨度不能超过30天");
}
return ResponseResult.success(markDataService.searchWholeNetwork(dto));
}
}
src/main/java/com/zhiwei/brandkbs2/dao/BaseMongoDao.java
......@@ -156,9 +156,4 @@ public interface BaseMongoDao<T extends AbstractBaseMongo> {
return criteria;
}
@FunctionalInterface
interface VoidSetFunction {
<T> void set(T t);
}
}
src/main/java/com/zhiwei/brandkbs2/dao/EventDao.java
......@@ -94,4 +94,7 @@ public interface EventDao extends BaseMongoDao<Event> {
*/
List<Event> getEventsByProjectIdAndContendId(Long startTime, Long endTime, String emotion, String projectId, String contendId, int limit);
List<Event> getEventsByTotalChannelVolumeTop(Long startTime, Long endTime, String emotion, String projectId, String contendId, int limit);
}
src/main/java/com/zhiwei/brandkbs2/dao/impl/EventDaoImpl.java
......@@ -105,6 +105,17 @@ public class EventDaoImpl extends BaseMongoDaoImpl<Event> implements EventDao {
return mongoTemplate.find(query, clazz, COLLECTION_NAME);
}
@Override
public List<Event> getEventsByTotalChannelVolumeTop(Long startTime, Long endTime, String emotion, String projectId, String contendId, int limit) {
Query query = new Query();
Criteria criteria = eventCountCriteria(startTime, endTime, emotion, projectId, contendId);
query.addCriteria(criteria);
query.limit(limit);
// 按影响力排序
addSort(query, "{\"negativeArticleVolume\":\"descend\"}");
return mongoTemplate.find(query, clazz, COLLECTION_NAME);
}
private Criteria eventCountCriteria(Long startTime, Long endTime, String emotion, String projectId, String contendId) {
Criteria criteria = Criteria.where("projectId").is(projectId);
criteria.and("contendId").is(contendId);
......
src/main/java/com/zhiwei/brandkbs2/dao/impl/EventDisseminationTrendDaoImpl.java
package com.zhiwei.brandkbs2.dao.impl;
import com.zhiwei.brandkbs2.dao.EventDisseminationTrendDao;
import com.zhiwei.brandkbs2.pojo.Event;
import com.zhiwei.brandkbs2.pojo.EventDisseminationTrend;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Component;
......@@ -21,6 +20,6 @@ public class EventDisseminationTrendDaoImpl extends BaseMongoDaoImpl<EventDissem
@Override
public EventDisseminationTrend findOne(Query query) {
return mongoTemplate.findOne(query, EventDisseminationTrend.class);
return mongoTemplate.findOne(query, EventDisseminationTrend.class, COLLECTION_NAME);
}
}
src/main/java/com/zhiwei/brandkbs2/es/EsQueryTools.java
......@@ -4,12 +4,19 @@ import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.common.GlobalPojo;
import com.zhiwei.pushlog.tools.Tools;
import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.lucene.search.join.ScoreMode;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.NestedQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import java.util.*;
import static com.zhiwei.brandkbs2.common.GenericAttribute.ES_BRANDKBS_CACHE_MAPS;
import static com.zhiwei.brandkbs2.util.Tools.concat;
/**
* @ClassName: EsQueryTool
* @Description es语句辅助类
......@@ -83,27 +90,51 @@ public class EsQueryTools {
return tagQuery;
}
/**
* 媒体类型请求匹配
*
* @param mediaTypes
* @return
*/
public static BoolQueryBuilder assembleMediaTypeQuery(List<String> mediaTypes) {
BoolQueryBuilder tagQuery = QueryBuilders.boolQuery();
public static BoolQueryBuilder assembleCacheMapsQuery(String projectId, String linkedGroupId, String contendId) {
return assembleCacheMapsQuery(projectId, linkedGroupId, contendId, null);
}
public static BoolQueryBuilder assembleCacheMapsQuery(String projectId, String linkedGroupId, String contendId, List<String> mediaTypes) {
BoolQueryBuilder nestedBoolQueryBuilder = QueryBuilders.boolQuery();
// must key
nestedBoolQueryBuilder.must(cacheMapsNestedQuery(QueryBuilders.termQuery("brandkbs_cache_maps.key.keyword", concat(projectId, linkedGroupId, contendId))));
// mediaType
if (CollectionUtils.isNotEmpty(mediaTypes)) {
BoolQueryBuilder mediaTypeQueryBuilder = QueryBuilders.boolQuery();
mediaTypes.forEach(e -> {
tagQuery.should(QueryBuilders.termQuery("brandkbs_cache_maps.channel_type.keyword", e));
mediaTypeQueryBuilder.should(cacheMapsNestedQuery(QueryBuilders.termQuery("brandkbs_cache_maps.channel_type.keyword", e)));
});
return tagQuery;
nestedBoolQueryBuilder.must(mediaTypeQueryBuilder);
}
return nestedBoolQueryBuilder;
}
public static NestedQueryBuilder cacheMapsNestedQuery(QueryBuilder query) {
return new NestedQueryBuilder(ES_BRANDKBS_CACHE_MAPS, query, ScoreMode.None);
}
// /**
// * 媒体类型请求匹配
// *
// * @param mediaTypes
// * @return
// */
// public static BoolQueryBuilder assembleMediaTypeQuery(List<String> mediaTypes) {
// BoolQueryBuilder tagQuery = QueryBuilders.boolQuery();
// mediaTypes.forEach(e -> {
// // TODO 1
// tagQuery.should(QueryBuilders.termQuery("brandkbs_cache_maps.channel_type.keyword", e));
// });
// return tagQuery;
// }
public static BoolQueryBuilder assembleSourceQuery(String sourceKeyword) {
BoolQueryBuilder channelBoolQueryBuilder = QueryBuilders.boolQuery();
String[] keys = sourceKeyword.trim().split("\\|");
for (String key : keys) {
String channelRegex = getAllRegex(key);
BoolQueryBuilder keyQueryBuilder = QueryBuilders.boolQuery();
keyQueryBuilder.must(QueryBuilders.regexpQuery("source.keyword", ".*" + channelRegex + ".*"));
keyQueryBuilder.must(QueryBuilders.regexpQuery("source", ".*" + channelRegex + ".*"));
channelBoolQueryBuilder.should(keyQueryBuilder);
}
return channelBoolQueryBuilder;
......
src/main/java/com/zhiwei/brandkbs2/pojo/EventDisseminationTrend.java
package com.zhiwei.brandkbs2.pojo;
import com.zhiwei.brandkbs2.enmus.EmotionEnum;
import com.zhiwei.brandkbs2.util.Tools;
import lombok.Getter;
import lombok.Setter;
import org.springframework.data.mongodb.core.mapping.Document;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
......@@ -15,15 +19,15 @@ import java.util.Map;
@Getter
@Setter
@Document(collection = "brandkbs_event_dissemination_trend")
public class EventDisseminationTrend extends AbstractBaseMongo{
public class EventDisseminationTrend extends AbstractBaseMongo {
/**
* 负面稿件传播
*/
private List<Map<String,Object>> negativeSpread;
private List<Map<String, Object>> negativeSpread;
/**
* 总稿件传播
*/
private List<Map<String,Object>> totalSpread;
private List<Map<String, Object>> totalSpread;
/**
* 类型 按小时、按天
*/
......@@ -32,4 +36,61 @@ public class EventDisseminationTrend extends AbstractBaseMongo{
* 关联事件id
*/
private String eventId;
public static EventDisseminationTrend createFromType(List<EventData> list, Long startTime, Long endTime, String type, String eventId) {
EventDisseminationTrend eventDisseminationTrend = new EventDisseminationTrend();
eventDisseminationTrend.setEventId(eventId);
eventDisseminationTrend.setType(type);
switch (type) {
case "小时": {
setSpreadAll(list, Tools.parseToHours(startTime, endTime), eventDisseminationTrend);
break;
}
case "天": {
setSpreadAll(list, Tools.parseToDays(startTime, endTime), eventDisseminationTrend);
break;
}
}
return eventDisseminationTrend;
}
private static void setSpreadAll(List<EventData> list, List<Map<String, Long>> rangeTimeList, EventDisseminationTrend trend) {
long start;
long end;
long negCount = 0;
long count = 0;
int index = 0;
List<Map<String, Object>> negativeSpread = new ArrayList<>();
List<Map<String, Object>> totalSpread = new ArrayList<>();
for (EventData eventData : list) {
Map<String, Long> map = rangeTimeList.get(index);
start = map.get("startTime");
end = map.get("endTime");
if (eventData.getTime() >= start && eventData.getTime() < end) {
// 数据统计
count++;
if (EmotionEnum.NEGATIVE.getName().equals(eventData.getEmotion())) {
negCount++;
}
} else {
// 总统计
Map<String, Object> element = new HashMap<>();
element.put("time", start);
element.put("sum", count);
negativeSpread.add(element);
// 负面统计
Map<String, Object> negElement = new HashMap<>();
negElement.put("time", start);
negElement.put("sum", negCount);
totalSpread.add(negElement);
// 推进节点并重置计量值
count = 0;
negCount = 0;
index++;
}
}
trend.setTotalSpread(totalSpread);
trend.setNegativeSpread(negativeSpread);
}
}
src/main/java/com/zhiwei/brandkbs2/pojo/EventTopArticlesAnalysis.java
......@@ -18,6 +18,11 @@ public class EventTopArticlesAnalysis extends AbstractBaseMongo {
* 聚合标题
*/
private String aggTitle;
/**
* 首条标题
*/
private String url;
/**
* 聚合情感
*/
......@@ -49,6 +54,7 @@ public class EventTopArticlesAnalysis extends AbstractBaseMongo {
eventTopArticlesAnalysis.setEmotion(templateData.getEmotion());
eventTopArticlesAnalysis.setCount(count);
eventTopArticlesAnalysis.setTime(templateData.getTime());
eventTopArticlesAnalysis.setUrl(templateData.getUrl());
eventTopArticlesAnalysis.setTimePoint(timePoint);
eventTopArticlesAnalysis.setType("按时间");
eventTopArticlesAnalysis.setEventId(eventId);
......@@ -61,6 +67,7 @@ public class EventTopArticlesAnalysis extends AbstractBaseMongo {
eventTopArticlesAnalysis.setEmotion(templateData.getEmotion());
eventTopArticlesAnalysis.setCount(count);
eventTopArticlesAnalysis.setTime(templateData.getTime());
eventTopArticlesAnalysis.setUrl(templateData.getUrl());
eventTopArticlesAnalysis.setType("按数量");
eventTopArticlesAnalysis.setEventId(eventId);
return eventTopArticlesAnalysis;
......
src/main/java/com/zhiwei/brandkbs2/pojo/dto/SearchFilterDTO.java
package com.zhiwei.brandkbs2.pojo.dto;
import com.alibaba.fastjson.JSONObject;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import java.util.List;
import java.util.*;
@Data
@ApiModel("全网搜搜索传输类")
......@@ -44,7 +45,7 @@ public class SearchFilterDTO {
* 关键字
*/
@ApiModelProperty(value = "关键字")
private String keyword;
private String search;
/**
* 平台
......@@ -53,13 +54,15 @@ public class SearchFilterDTO {
private List<String> platforms;
/**
* 平台(真正用于搜索的)
* 平台(查商业数据库用)
*/
@ApiModelProperty(value = "平台(查商业数据库用)")
private String platformsStr;
/**
* 数据类型 1:原创, 2:转发, 7:评论
* 数据类型 1:原创, 2:转发, 7:评论 (查商业数据库用)
*/
@ApiModelProperty(value = "数据类型(查商业数据库用)")
private String filterForward = "1,2";
/**
......@@ -69,13 +72,27 @@ public class SearchFilterDTO {
private String sorter;
/**
* 排序字段(真正用于搜索的)
* 排序字段(查商业数据库用)
*/
@ApiModelProperty(value = "排序字段(查商业数据库用)")
private String sortOf;
/**
* 匹配域名 默认标题+内容
* 来源
*/
@ApiModelProperty(value = "来源")
private String source;
/**
* 过滤关键字
*/
@ApiModelProperty(value = "过滤关键字")
private String filterWords;
/**
* 匹配域名 默认标题+内容(查商业数据库用)
*/
@ApiModelProperty(value = "匹配域名(查商业数据库用)")
private String matchFields = null;
/**
......@@ -84,4 +101,48 @@ public class SearchFilterDTO {
@ApiModelProperty(value = "查询类型")
private String matchType;
/**
* 重要渠道，{father:"区域",son:"北京"}，{father:"全部"}
*/
@ApiModelProperty(value = "重要渠道")
private Map<String, String> sensitiveChannels;
/**
* 粉丝数
*/
@ApiModelProperty(value = "粉丝数")
private String fans;
/**
* 过滤类型 标题/全文
*/
@ApiModelProperty(value = "过滤类型 标题/全文")
private String filterType;
public Map<String, Object> toMap() {
Map<String, Object> map = new HashMap<>();
map.put("timeType","time");
map.put("pt", getPt());
map.put("matchType", matchType);
map.put("search", search);
map.put("startTime", startTime);
map.put("endTime", endTime);
map.put("source", source);
map.put("sort", Objects.nonNull(sorter) ? sorter : "DESC");
map.put("page", page);
map.put("pageSize", pageSize);
map.put("notWords", filterWords);
map.put("sensitiveChannels", sensitiveChannels);
map.put("fans", fans);
map.put("filterType", filterType);
return map;
}
public List<String> getPt() {
return Objects.nonNull(platforms) ? this.platforms : Collections.emptyList();
}
public String getMatchType() {
return Objects.nonNull(this.matchType) ? this.matchType : "标题";
}
}
src/main/java/com/zhiwei/brandkbs2/service/EventService.java
......@@ -5,6 +5,7 @@ import com.zhiwei.brandkbs2.easyexcel.dto.ExportEventDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportEventDataDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.UploadEventDTO;
import com.zhiwei.brandkbs2.pojo.Event;
import com.zhiwei.brandkbs2.pojo.EventData;
import com.zhiwei.brandkbs2.pojo.EventDisseminationTrend;
import com.zhiwei.brandkbs2.pojo.dto.EventDataDTO;
import com.zhiwei.brandkbs2.pojo.dto.EventSearchDTO;
......@@ -218,12 +219,14 @@ public interface EventService {
/**
* 获取品牌事件搜索条件
*
* @return
*/
JSONObject getEventsSearchCriteria();
/**
* 获取品牌事件列表信息
*
* @param eventSearchDTO 事件搜索类
* @return
*/
......@@ -231,6 +234,7 @@ public interface EventService {
/**
* 事件详情-基础静态信息
*
* @param eventId
* @return
*/
......@@ -238,6 +242,7 @@ public interface EventService {
/**
* 事件详情-传播趋势图
*
* @param eventId
* @param type
* @return
......@@ -246,6 +251,7 @@ public interface EventService {
/**
* 事件详情-渠道发声
*
* @param eventId
* @param type
* @param page
......@@ -253,10 +259,11 @@ public interface EventService {
* @param sorter
* @return
*/
List<JSONObject> getEventChannelVoices(String eventId, String type, int page, int pageSize, String sorter);
PageVO<JSONObject> getEventChannelVoices(String eventId, String type, int page, int pageSize, String sorter);
/**
* 事件详情-热门文章分析
*
* @param id
* @param type
* @param emotion
......@@ -264,4 +271,12 @@ public interface EventService {
* @return
*/
JSONObject getEventTopArticlesAnalysis(String id, String type, String emotion, String aggTitle);
/**
* 获得最新的事件
*
* @return
*/
List<JSONObject> getLastEventTop(String keyword,int limit);
}
src/main/java/com/zhiwei/brandkbs2/service/IndexService.java
......@@ -62,4 +62,14 @@ public interface IndexService {
*/
JSONObject getSpreadingTend(Long startTime, Long endTime, boolean cache);
/**
* 获取主品牌传播趋势
*
* @param startTime 开始时间
* @param endTime 结束时间
* @param cache 是否优先读取缓存
* @return 主品牌传播趋势
*/
JSONObject getSpreadingTend(Long startTime, Long endTime, String projectId, String contendId, boolean cache);
}
src/main/java/com/zhiwei/brandkbs2/service/impl/CustomEventServiceImpl.java
......@@ -40,7 +40,6 @@ import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.io.IOException;
import java.text.ParseException;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
......@@ -134,7 +133,7 @@ public class CustomEventServiceImpl implements CustomEventService {
if (Objects.isNull(customEvent)) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("自定义事件数据异常"));
}
if (!customEvent.getStatus()) {
if (Boolean.FALSE.equals(customEvent.getStatus())) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("该事件数据更新中,无法修改该事件信息！"));
}
// 修改自定义事件及清空历史数据
......@@ -180,7 +179,7 @@ public class CustomEventServiceImpl implements CustomEventService {
if (Objects.isNull(customEvent)) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("自定义事件数据异常"));
}
if (!customEvent.getStatus()) {
if (Boolean.FALSE.equals(customEvent.getStatus())) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("该事件数据更新中!"));
}
customEvent.setStatus(false);
......@@ -210,7 +209,7 @@ public class CustomEventServiceImpl implements CustomEventService {
try {
return getCustomEventAnalyzeInner(id, cache);
} catch (Exception e) {
log.info("自定义事件:{} 数据更新异常", id, e);
log.info("自定义事件:{} 获取异常", id, e);
}
return new JSONObject();
}
......@@ -231,9 +230,6 @@ public class CustomEventServiceImpl implements CustomEventService {
}
String redisKey = RedisKeyPrefix.CUSTOM_EVENT_ANALYZE + id;
if (cache) {
if (Boolean.FALSE.equals(customEvent.getStatus())) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("该事件数据更新中，请勿重复更新！"));
}
String resultStr = redisUtil.get(redisKey);
if (StringUtils.isNotEmpty(resultStr)) {
return JSON.parseObject(resultStr);
......@@ -510,7 +506,7 @@ public class CustomEventServiceImpl implements CustomEventService {
// 获得首发标题
List<BaseMap> list = entry.getValue().stream().sorted(Comparator.comparingLong(BaseMap::getTime)).limit(1).collect(Collectors.toList());
JSONObject result = new JSONObject();
result.put("title", list.get(0).getTitle());
result.put("title", list.get(0).getTitleNullOptionalContent());
result.put("url", list.get(0).getUrl());
result.put("num", entry.getValue().size());
res.add(result);
......
src/main/java/com/zhiwei/brandkbs2/service/impl/EventDataServiceImpl.java
......@@ -5,10 +5,12 @@ import com.google.common.collect.Lists;
import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.dao.EventDao;
import com.zhiwei.brandkbs2.dao.EventDataDao;
import com.zhiwei.brandkbs2.dao.EventDisseminationTrendDao;
import com.zhiwei.brandkbs2.dao.EventTopArticlesAnalysisDao;
import com.zhiwei.brandkbs2.enmus.EmotionEnum;
import com.zhiwei.brandkbs2.pojo.Event;
import com.zhiwei.brandkbs2.pojo.EventData;
import com.zhiwei.brandkbs2.pojo.EventDisseminationTrend;
import com.zhiwei.brandkbs2.pojo.EventTopArticlesAnalysis;
import com.zhiwei.brandkbs2.service.EventDataService;
import com.zhiwei.brandkbs2.service.MarkDataService;
......@@ -17,9 +19,13 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.joda.time.Period;
import org.joda.time.PeriodType;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.text.MessageFormat;
import java.util.*;
import java.util.concurrent.atomic.AtomicLong;
......@@ -50,6 +56,9 @@ public class EventDataServiceImpl implements EventDataService {
@Resource(name = "eventTopArticlesAnalysisDao")
private EventTopArticlesAnalysisDao eventTopArticlesAnalysisDao;
@Resource(name = "eventDisseminationTrendDao")
private EventDisseminationTrendDao eventDisseminationTrendDao;
private static final String TOP_DETAIL_TEMPLATE = "Top媒体传播方向：全局Top8的文章中，{0}报道方向占比最高，为{1}%，{2}报道方向占比{3}%。Top报道方向中，《{4}》相似文章数最多，总文章数为{5}篇。";
@Override
......@@ -104,6 +113,7 @@ public class EventDataServiceImpl implements EventDataService {
// 删除历史数据
eventDataDao.deleteByEventId(event.getId(), event.getCollectionName());
eventTopArticlesAnalysisDao.deleteByEventId(event.getId());
eventDisseminationTrendDao.deleteOneByQuery(Query.query(Criteria.where("eventId").is(event.getId())));
}
private void setEventStaticState(Event event, List<EventData> eventDataList) {
......@@ -142,12 +152,25 @@ public class EventDataServiceImpl implements EventDataService {
JSONObject articlePlatformProportions = new JSONObject();
platformIdCounts.forEach((platform, count) -> articlePlatformProportions.put(platform, getProportion(count.get(), totalDisseminationVolume)));
// 热门文章分析描述
String eventTopArticlesAnalysisDetail = MessageFormat.format(TOP_DETAIL_TEMPLATE, topArticlesAnalysis(eventDataList, event));
String eventTopArticlesAnalysisDetail = MessageFormat.format(TOP_DETAIL_TEMPLATE, topArticlesAnalysis(eventDataList, event).toArray());
event.setStaticState(totalDisseminationVolume, totalChannelVolumes.size(), negativeArticleVolume, articleEmotionProportions,
articlePlatformProportions, eventTopArticlesAnalysisDetail);
// 传播分析
eventDisseminationTrend(eventDataList, event);
eventDao.updateOne(event);
}
private void eventDisseminationTrend(List<EventData> eventDataList, Event event) {
// 按时间排序
eventDataList = eventDataList.stream().sorted(Comparator.comparingLong(EventData::getTime)).collect(Collectors.toList());
Long startTime = event.getStartTime();
Long endTime = null == event.getEndTime() ? System.currentTimeMillis() : event.getEndTime();
EventDisseminationTrend hourTrend = EventDisseminationTrend.createFromType(eventDataList, startTime, endTime, "小时", event.getId());
EventDisseminationTrend dayTrend = EventDisseminationTrend.createFromType(eventDataList, startTime, endTime, "天", event.getId());
eventDisseminationTrendDao.insertOne(hourTrend);
eventDisseminationTrendDao.insertOne(dayTrend);
}
private List<String> topArticlesAnalysis(List<EventData> eventDataList, Event event) {
// 标题前8数据
List<Map.Entry<String, List<EventData>>> top8Titles =
......@@ -199,7 +222,7 @@ public class EventDataServiceImpl implements EventDataService {
long sum = emotionMap.values().stream().mapToLong(AtomicLong::get).sum();
emotionMap.entrySet().stream().sorted((x, y) -> Long.compare(y.getValue().get(), x.getValue().get())).limit(2).forEach(entry -> {
details.add(entry.getKey());
details.add(String.valueOf((entry.getValue().get()) / sum));
details.add(new BigDecimal((double) (entry.getValue().get() * 100) / sum).setScale(2, RoundingMode.UP).toString());
});
details.add(top8Titles.get(0).getKey());
details.add(String.valueOf(top8Titles.get(0).getValue().size()));
......
src/main/java/com/zhiwei/brandkbs2/service/impl/EventServiceImpl.java
......@@ -5,6 +5,7 @@ import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.common.RedisKeyPrefix;
import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.dao.EventDao;
import com.zhiwei.brandkbs2.dao.EventDataDao;
import com.zhiwei.brandkbs2.dao.EventDisseminationTrendDao;
......@@ -538,6 +539,8 @@ public class EventServiceImpl implements EventService {
if (Objects.nonNull(eventSearchDTO.getStartTime()) && Objects.nonNull(eventSearchDTO.getEndTime())) {
query.addCriteria(Criteria.where("startTime").gte(eventSearchDTO.getStartTime()).lt(eventSearchDTO.getEndTime()));
}
// 关键词
eventDao.addKeywordFuzz(query, eventSearchDTO.getKeyword(), "title");
// 传播量
if (Objects.nonNull(eventSearchDTO.getTotalDisseminationVolumes())) {
Long[] totalDisseminationVolumes = eventSearchDTO.getTotalDisseminationVolumes();
......@@ -568,8 +571,7 @@ public class EventServiceImpl implements EventService {
sortMap.get(event.getId()).setFirstEventData(r);
return null;
})).toArray(CompletableFuture[]::new)).join();
return PageVO.createPageVo(total, eventSearchDTO.getPage(), eventSearchDTO.getPageSize(),
eventList.stream().map(event -> sortMap.get(event.getId())).collect(Collectors.toList()));
return PageVO.createPageVo(total, eventSearchDTO.getPage(), eventSearchDTO.getPageSize(), eventList.stream().map(event -> sortMap.get(event.getId())).collect(Collectors.toList()));
}
@Override
......@@ -589,31 +591,30 @@ public class EventServiceImpl implements EventService {
}
@Override
public List<JSONObject> getEventChannelVoices(String eventId, String type, int page, int pageSize, String sorter) {
public PageVO<JSONObject> getEventChannelVoices(String eventId, String type, int page, int pageSize, String sorter) {
Event event = getEventById(eventId);
Query query = Query.query(Criteria.where("eventId").is(eventId));
if (Objects.equals("重要渠道", type)) {
query.addCriteria(Criteria.where("mediaType").in(IMPORTANT_CHANNEL_LIST));
}
mongoUtil.start(page, pageSize, query);
// 总数
// 总量
long total = eventDataDao.count(query, event.getCollectionName());
mongoUtil.start(page, pageSize, query);
// 排序
eventDataDao.addSort(query, sorter);
// 数据
List<EventData> eventDataList = eventDataDao.findList(query, event.getCollectionName());
// 分组排序
Map<String, List<EventData>> groupMap = new HashMap<>();
for (EventData eventData : eventDataList) {
groupMap.putIfAbsent(eventData.getMediaType(), new ArrayList<>());
groupMap.get(eventData.getMediaType()).add(eventData);
}
return groupMap.entrySet().stream().map(entry -> {
List<JSONObject> collect = eventDataList.stream().map(data -> {
JSONObject json = new JSONObject();
json.put("type", entry.getKey());
json.put("list", entry.getValue());
json.put("title", data.getTitle());
json.put("url", data.getUrl());
json.put("time", data.getTime());
json.put("source", data.getSource());
json.put("emotion", data.getEmotion());
json.put("mediaType", data.getMediaType());
return json;
}).collect(Collectors.toList());
return PageVO.createPageVo(total, page, pageSize, collect);
}
@Override
......@@ -632,10 +633,10 @@ public class EventServiceImpl implements EventService {
switch (type) {
case "按时间":
Map<String, List<EventTopArticlesAnalysisVO>> collect = eventTopArticlesAnalyses.stream().map(eventTopArticlesAnalysis -> {
boolean highLight = Objects.equals(aggTitle, eventTopArticlesAnalysis.getAggTitle());
boolean highLight = null == aggTitle || Objects.equals(aggTitle, eventTopArticlesAnalysis.getAggTitle());
return new EventTopArticlesAnalysisVO(eventTopArticlesAnalysis, highLight);
}).collect(Collectors.groupingBy(EventTopArticlesAnalysisVO::getTimePoint));
res.put("dataDay", collect);
res.put("dataDay", Tools.sortTimeKeyMap(collect, Constant.SPEC_DAY_FORMAT, false));
res.put("dataAmount", null);
return res;
case "按数量":
......@@ -647,6 +648,21 @@ public class EventServiceImpl implements EventService {
}
}
@Override
public List<JSONObject> getLastEventTop(String keyword, int limit) {
Query query = Query.query(Criteria.where("projectId").is(UserThreadLocal.getProjectId()));
eventDao.addKeywordFuzz(query, keyword, "title");
eventDao.addSort(query, "{\"startTime\":\"descend\"}");
query.limit(limit);
return eventDao.findList(query).stream().map(event -> {
JSONObject result = new JSONObject();
result.put("id", event.getId());
result.put("title", event.getTitle());
result.put("total", event.getTotalDisseminationVolume());
return result;
}).sorted((o1, o2) -> Long.compare((long) o2.get("total"), (long) o1.get("total"))).collect(Collectors.toList());
}
/**
* 获取时间筛选条件
*
......
src/main/java/com/zhiwei/brandkbs2/service/impl/IndexServiceImpl.java
......@@ -99,7 +99,7 @@ public class IndexServiceImpl implements IndexService {
jsonObject.put("compare", lastRangeCount == 0 ? 0d : (rangeCount - lastRangeCount) / (double) lastRangeCount);
List<Map<String, Long>> dayList = Tools.parseToDaysWithBase(startTime, endTime);
jsonObject.put("spread", this.getArticleSpreadWithBrand(projectId, Constant.PRIMARY_CONTEND_ID, dayList));
redisUtil.set(redisKey, JSON.toJSONString(jsonObject));
redisUtil.setExpire(redisKey, JSON.toJSONString(jsonObject));
} catch (Exception e) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("getYuqingAmount异常"), e);
}
......@@ -156,7 +156,7 @@ public class IndexServiceImpl implements IndexService {
jsonObject.put("compare", oldPosPro == 0 ? 0d : (posPro - oldPosPro) / oldPosPro);
double avgPosPro = totalNormalCount == 0 ? 0d : totalPositiveCount / (double) totalNormalCount;
jsonObject.put("avgPosPro", avgPosPro);
redisUtil.set(redisKey, JSON.toJSONString(jsonObject));
redisUtil.setExpire(redisKey, JSON.toJSONString(jsonObject));
} catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("es查询异常"), e);
}
......@@ -212,7 +212,7 @@ public class IndexServiceImpl implements IndexService {
}
}
jsonObject.put("spread", lineList);
redisUtil.set(redisKey, JSONObject.toJSONString(jsonObject));
redisUtil.setExpire(redisKey, JSONObject.toJSONString(jsonObject));
} catch (Exception e) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("getEventAmount异常"), e);
}
......@@ -237,7 +237,7 @@ public class IndexServiceImpl implements IndexService {
try {
long normalCount = markDataService.getYuqingMarkCount(startTime, endTime, EmotionEnum.ALL.getName(), projectId, contendId);
List<JSONObject> platforms = getPlatformProportionWithPlatform(startTime, endTime, EmotionEnum.ALL.getName(), projectId, contendId, normalCount);
redisUtil.set(redisKey, JSON.toJSONString(platforms));
redisUtil.setExpire(redisKey, JSON.toJSONString(platforms));
return platforms;
} catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("es查询异常"), e);
......@@ -247,6 +247,11 @@ public class IndexServiceImpl implements IndexService {
@Override
public JSONObject getSpreadingTend(Long startTime, Long endTime, boolean cache) {
return getSpreadingTend(startTime, endTime, UserThreadLocal.getProjectId(), "0", cache);
}
@Override
public JSONObject getSpreadingTend(Long startTime, Long endTime, String projectId, String contendId, boolean cache) {
boolean flag = true;
if (null == startTime || null == endTime) {
Long[] timeRangeMonth = commonService.getTimeRangeMonth();
......@@ -254,9 +259,7 @@ public class IndexServiceImpl implements IndexService {
endTime = timeRangeMonth[1];
flag = false;
}
String projectId = UserThreadLocal.getProjectId();
String linkedGroupId = projectService.getProjectById(projectId).getBrandLinkedGroupId();
String contendId = "0";
String redisKey = RedisUtil.getIndexSpread(projectId, startTime, endTime);
String resultStr;
// 返回缓存
......@@ -287,7 +290,7 @@ public class IndexServiceImpl implements IndexService {
summary.put("topEvent", this.getTopEvent(startTime, endTime, projectId, contendId));
summary.put("topSource", this.getTopSource(startTime, endTime, projectId, linkedGroupId, contendId));
resJson.put("summary", summary);
redisUtil.set(redisKey, JSON.toJSONString(resJson));
redisUtil.setExpire(redisKey, JSON.toJSONString(resJson));
} catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("es查询异常"), e);
}
......@@ -305,7 +308,7 @@ public class IndexServiceImpl implements IndexService {
} catch (IOException ignored) {
count = -1;
}
result.put("platform", platformId);
result.put("platform", GlobalPojo.getPlatformNameById(platformId));
result.put("num", count);
result.put("proportion", normalCount == 0 ? 0d : count / (double) normalCount);
resultList.add(result);
......
src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java
......@@ -2,6 +2,7 @@ package com.zhiwei.brandkbs2.service.impl;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.ImmutableMap;
import com.zhiwei.brandkbs2.auth.UserThreadLocal;
......@@ -32,6 +33,7 @@ import com.zhiwei.brandkbs2.util.RedisUtil;
import com.zhiwei.brandkbs2.util.TextUtil;
import com.zhiwei.brandkbs2.util.Tools;
import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform;
import com.zhiwei.tools.istarshine.IstarShineUtil;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
......@@ -60,9 +62,13 @@ import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.joda.time.Period;
import org.joda.time.PeriodType;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
import javax.annotation.Resource;
import java.io.IOException;
......@@ -84,6 +90,18 @@ public class MarkDataServiceImpl implements MarkDataService {
private static final String[] EVENT_FETCH_SOURCE = new String[]{"id", "ind_full_text", "c5", "real_source", "source", "mtime", "time", "url", "mtag", "foreign", "brandkbs_mark_cache_maps"};
@Value("${istarshine.addIStarShineKSData.url}")
private String addIStarShineKSDataUrl;
@Value("${istarshine.getIStarShineKSInfoData.url}")
private String getIStarShineKSInfoDataUrl;
@Value("${whole.search.url}")
private String wholeSearchUrl;
@Autowired
private RestTemplate restTemplate;
@Resource(name = "projectServiceImpl")
private ProjectService projectService;
......@@ -548,17 +566,13 @@ public class MarkDataServiceImpl implements MarkDataService {
String contendId = dto.getContendId();
// PostFilter 后置过滤器
// BoolQueryBuilder postFilter = projectLinkedGroupQuery(projectId, linkedGroupId);
BoolQueryBuilder postFilter = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, contendId);
BoolQueryBuilder postFilter = EsQueryTools.assembleCacheMapsQuery(projectId, linkedGroupId, contendId, dto.getMediaTypes());
// time
postFilter.must(QueryBuilders.rangeQuery(dto.getTimeType()).gte(dto.getStartTime()).lt(dto.getEndTime()));
// platform
if (CollectionUtils.isNotEmpty(dto.getPlatforms())) {
postFilter.must(EsQueryTools.assemblePlatformQuery(Tools.getPlatformByIds(dto.getPlatforms())));
}
// mediaTypes
if (CollectionUtils.isNotEmpty(dto.getMediaTypes())) {
postFilter.must(EsQueryTools.assembleMediaTypeQuery(dto.getMediaTypes()));
}
// tags
if (CollectionUtils.isNotEmpty(dto.getTags())) {
postFilter.must(EsQueryTools.assembleTagQuery(dto.getTags()));
......@@ -717,13 +731,12 @@ public class MarkDataServiceImpl implements MarkDataService {
// 聚合查询
TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("count").field("brandkbs_mark_cache_maps.name.keyword");
// query
BoolQueryBuilder query = QueryBuilders.boolQuery();
BoolQueryBuilder query = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, "0");
// keyword
if (StringUtils.isNotEmpty(keyword)) {
String[] fieldSearch = "标题".equals(searchField) ? new String[]{GenericAttribute.ES_IND_TITLE} : new String[]{GenericAttribute.ES_IND_FULL_TEXT};
query.must(EsQueryTools.assembleNormalKeywordQuery(keyword, fieldSearch));
}
query.must(QueryBuilders.termQuery("brandkbs_cache_maps.linked_group_id.keyword", linkedGroupId)).must(QueryBuilders.termQuery("brandkbs_cache_maps.project_id.keyword", projectId));
// time range
query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
......@@ -789,7 +802,7 @@ public class MarkDataServiceImpl implements MarkDataService {
BoolQueryBuilder query = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, contendId);
query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime))
// 过滤微博
.mustNot(QueryBuilders.termQuery("platform_id.keyword", "5d02236e6395002a7c380b79"));
.mustNot(QueryBuilders.termQuery("platform_id", "5d02236e6395002a7c380b79"));
if (null != emotion && !Objects.equals(emotion, EmotionEnum.ALL.getName())) {
query.must(QueryBuilders.termQuery("brandkbs_mark_cache_maps.name.keyword", emotion));
}
......@@ -847,7 +860,7 @@ public class MarkDataServiceImpl implements MarkDataService {
timeBuilder.lt(endTime);
}
query.must(timeBuilder);
return esClientDao.searchScroll(query, 10000, fetchSource);
return esClientDao.searchScroll(query, 2000, fetchSource);
} catch (Exception e) {
log.error("searchByEvent-", e);
}
......@@ -940,17 +953,13 @@ public class MarkDataServiceImpl implements MarkDataService {
String contendId = dto.getContendId();
String contendLinkedGroupId = projectService.getProjectByContendId(projectId, dto.getContendId()).getBrandLinkedGroupId();
// PostFilter 后置过滤器
BoolQueryBuilder postFilter = projectLinkedGroupContendIdQuery(projectId, contendLinkedGroupId, contendId);
BoolQueryBuilder postFilter = EsQueryTools.assembleCacheMapsQuery(projectId, contendLinkedGroupId, contendId, dto.getMediaTypes());
// time
postFilter.must(QueryBuilders.rangeQuery(dto.getTimeType()).gte(dto.getStartTime()).lt(dto.getEndTime()));
// platform
if (CollectionUtils.isNotEmpty(dto.getPlatforms())) {
postFilter.must(EsQueryTools.assemblePlatformQuery(Tools.getPlatformByIds(dto.getPlatforms())));
}
// mediaTypes
if (CollectionUtils.isNotEmpty(dto.getMediaTypes())) {
postFilter.must(EsQueryTools.assembleMediaTypeQuery(dto.getMediaTypes()));
}
// tags
if (CollectionUtils.isNotEmpty(dto.getTags())) {
postFilter.must(EsQueryTools.assembleTagQuery(dto.getTags()));
......@@ -1220,42 +1229,145 @@ public class MarkDataServiceImpl implements MarkDataService {
@Override
public JSONObject searchWholeNetwork(SearchFilterDTO dto) {
// Project project = projectService.getProjectById(dto.getProjectId());
// if (1 == project.getWholeSearchDataSource()){
// defaultSearchFilter(dto);
// try {
// JSONObject params = IstarShineUtil.getKSSearchParam(keyword, null, dto.getStartTime(), dto.getEndTime(), platforms,
// filterForward, sortOf, dto.getPage(), dto.getPageSize(), matchFields,null,null);
//
// JSONObject taskEntity = new JSONObject();
//
// taskEntity.put("appId", "6183571e0d710000f6003a12"); //应用id,由张志伟提供给使用者
// taskEntity.put("taskType", ""); //可为null, 任务类型
// taskEntity.put("taskDescription", ""); //可为null， 任务描述
// taskEntity.put("userName", "品见"); //提交任务的人
// taskEntity.put("searchParams", params); //任务详情参数
//
// ResponseEntity<JSONObject> entity = restTemplate.postForEntity(url, taskEntity, JSONObject.class);
// JSONObject result = entity.getBody().getJSONObject("data");
// return searchWholeBackUp(result.get("id").toString());
// }
// }
return null; //TODO
Project project = projectService.getProjectById(dto.getProjectId());
JSONObject jsonObject = new JSONObject();
if (2 == project.getWholeSearchDataSource()){ // 查商业数据库
defaultSearchFilter(dto);
try {
JSONObject params = IstarShineUtil.getKSSearchParam(dto.getSearch(), null, dto.getStartTime(), dto.getEndTime(), dto.getPlatformsStr(),
dto.getFilterForward(), dto.getSortOf(), dto.getPage(), dto.getPageSize(), dto.getMatchFields(), null, null);
JSONObject taskEntity = new JSONObject();
taskEntity.put("appId", "6183571e0d710000f6003a12"); // 应用id, 由张志伟提供给使用者
taskEntity.put("taskType", ""); // 可为null, 任务类型
taskEntity.put("taskDescription", ""); // 可为null, 任务描述
taskEntity.put("userName", "品见"); // 提交任务的人
taskEntity.put("searchParams", params); // 任务详情参数
ResponseEntity<JSONObject> responseEntity = restTemplate.postForEntity(addIStarShineKSDataUrl, taskEntity, JSONObject.class);
JSONObject result = Objects.requireNonNull(responseEntity.getBody()).getJSONObject("data");
return searchWholeBackUp(result.get("id").toString());
}catch (Exception e){
log.error("error searchWholeNetwork - ", e);
}
}else { // 查舆情库
ResponseEntity<JSONObject> responseEntity = restTemplate.postForEntity(wholeSearchUrl, dto.toMap(), JSONObject.class);
return searchWholeOpinion(responseEntity, jsonObject);
}
return jsonObject;
}
private JSONObject searchWholeOpinion(ResponseEntity<JSONObject> responseEntity, JSONObject jsonObject){
JSONObject bodyData = Objects.requireNonNull(responseEntity.getBody()).getJSONObject("data");
JSONObject data = bodyData.getJSONObject("data");
JSONArray listArray = data.getJSONArray("list");
List<JSONObject> resultList = new ArrayList<>();
private void defaultSearchFilter(SearchFilterDTO dto){
for (Object o : listArray) {
JSONObject list = (JSONObject) o;
JSONObject result = new JSONObject();
result.put("platform", list.get("real_source"));
JSONArray contentArray = list.getJSONArray("ind_full_text");
assert contentArray == null;
result.put("title", contentArray.get(0));
result.put("content", contentArray.get(1));
if (3 == contentArray.size()) {
result.put("content", result.getString("content") + contentArray.get(2));
}
result.put("channel", list.get("source"));
String url = list.getString("url");
// 如果url为空,则查询question_url字段
if (null == url) {
url = list.getString("question_url");
}
result.put("url", url);
result.put("time", list.get("time"));
resultList.add(result);
}
jsonObject.put("list", resultList);
jsonObject.put("page", data.get("page"));
jsonObject.put("pageLimit", data.get("pageLimit"));
jsonObject.put("total", data.get("total"));
jsonObject.put("totalPage", data.get("totalPage"));
jsonObject.put("isBackUp", false);
return jsonObject;
}
public JSONObject searchWholeBackUp(String taskId) {
JSONObject jsonObject = new JSONObject();
List<JSONObject> resultList = new ArrayList<>();
for (int i = 0; i < 30; i++) {
ResponseEntity<JSONObject> responseEntity = restTemplate.getForEntity(getIStarShineKSInfoDataUrl, JSONObject.class, taskId);
JSONObject data = Objects.requireNonNull(responseEntity.getBody()).getJSONObject("data");
JSONObject task = data.getJSONObject("task");
if(1 == task.getInteger("crawlerFinish")){
//pageSize
Integer receiveCount = task.getInteger("receiveCount");
Integer searchCount = task.getInteger("searchCount");
int totalPage = searchCount % receiveCount == 0 ? searchCount / receiveCount : (searchCount / receiveCount + 1);
JSONArray listArray = data.getJSONArray("data");
for (Object o : listArray) {
JSONObject list = (JSONObject) o;
JSONObject result = new JSONObject();
result.put("platform", list.getJSONObject("publisher").get("platform"));
result.put("title", list.get("title"));
result.put("content", list.get("content"));
result.put("channel", list.get("channel"));
result.put("url", getUrl(list));
result.put("time", list.get("ctime"));
resultList.add(result);
}
jsonObject.put("list",resultList);
//jsonObject.put("page",data1.get("page"));
jsonObject.put("pageLimit",receiveCount);
jsonObject.put("total",searchCount);
jsonObject.put("totalPage",totalPage);
jsonObject.put("isBackUp",true);
return jsonObject;
}
Tools.sleep(500L);
}
return jsonObject;
}
/**
* 获取全网搜接口信息url
*
* @param json
* @return
*/
private static String getUrl(JSONObject json) {
String url = json.getString("url");
// 如果url为空,则查询QA类型url字段
if (null == url) {
url = json.getString("answer_url");
}
if (null == url) {
url = json.getString("question_url");
}
return url;
}
private void defaultSearchFilter(SearchFilterDTO dto) {
String projectId = UserThreadLocal.getProjectId();
dto.setProjectId(projectId);
if (Objects.isNull(dto.getStartTime()) || Objects.isNull(dto.getEndTime())) {
Date now = new Date();
dto.setEndTime(now.getTime());
dto.setStartTime(DateUtils.addDays(now, -1).getTime());
}
// 平台为空时，传全部
if (dto.getPlatforms().size() == 0 ){
dto.setPlatforms(Arrays.asList("微博","微信","新闻","平媒","自媒体","论坛","视频","短视频"));
if (dto.getPlatforms().size() == 0) {
dto.setPlatforms(Arrays.asList("微博", "微信", "新闻", "平媒", "自媒体", "论坛", "视频", "短视频"));
}
// 平台
dto.setPlatformsStr(StringUtils.join(dto.getPlatforms(), ",")); //平台
dto.setPlatformsStr(StringUtils.join(dto.getPlatforms(), ","));
// sort
dto.setSortOf("ctime+" + dto.getSorter());
// matchFields
if(dto.getMatchType().equals("标题")){
if (dto.getMatchType().equals("标题")) {
dto.setMatchFields("title");
}
}
......@@ -1278,7 +1390,7 @@ public class MarkDataServiceImpl implements MarkDataService {
filter.must(QueryBuilders.termQuery("brandkbs_mark_cache_maps.name.keyword", emotion));
}
if (null != platformId) {
filter.must(QueryBuilders.termQuery("platform_id.keyword", platformId));
filter.must(QueryBuilders.termQuery("platform_id", platformId));
}
return esClientDao.count(filter);
}
......@@ -1372,9 +1484,12 @@ public class MarkDataServiceImpl implements MarkDataService {
*/
private void defaultContendDTO(MarkSearchDTO dto) {
if (Objects.isNull(dto.getStartTime()) || Objects.isNull(dto.getEndTime())) {
Date now = new Date();
dto.setEndTime(now.getTime());
dto.setStartTime(DateUtils.addDays(now, -29).getTime());
Long[] timeRangeMonth = commonService.getTimeRangeMonth();
dto.setEndTime(timeRangeMonth[1]);
dto.setStartTime(timeRangeMonth[0]);
}else{
dto.setStartTime(Tools.truncDate(dto.getStartTime(), Constant.DAY_PATTERN));
dto.setEndTime(Tools.truncDate(dto.getEndTime(), Constant.DAY_PATTERN));
}
String projectId = UserThreadLocal.getProjectId();
dto.setProjectId(projectId);
......@@ -1402,8 +1517,9 @@ public class MarkDataServiceImpl implements MarkDataService {
DateHistogramAggregationBuilder daysAggregationBuilder = AggregationBuilders.dateHistogram("dayAgg").field("time").calendarInterval(DateHistogramInterval.DAY);
TermsAggregationBuilder emotionAggregationBuilder = AggregationBuilders.terms("count").field("brandkbs_mark_cache_maps.name.keyword");
// query
BoolQueryBuilder query = QueryBuilders.boolQuery();
query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime)).must(QueryBuilders.termQuery("brandkbs_cache_maps.project_id.keyword", projectId)).must(QueryBuilders.termQuery("brandkbs_cache_maps.linked_group_id.keyword", linkedGroupId));
BoolQueryBuilder query = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, "0");
// time
query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
// 情感倾向限制
query.must(QueryBuilders.termQuery("brandkbs_mark_cache_maps.group_name.keyword", "情感倾向"));
// response
......@@ -1437,10 +1553,10 @@ public class MarkDataServiceImpl implements MarkDataService {
// 索引
String[] indexes = esClientDao.getIndexes();
// 聚合请求
TermsAggregationBuilder platformAggregationBuilder = AggregationBuilders.terms("platform_count").field("platform_id.keyword").order(BucketOrder.count(false));
TermsAggregationBuilder platformAggregationBuilder = AggregationBuilders.terms("platform_count").field("platform_id").order(BucketOrder.count(false));
TermsAggregationBuilder emotionAggregationBuilder = AggregationBuilders.terms("emotion_count").field("brandkbs_mark_cache_maps.name.keyword");
// postFilter
BoolQueryBuilder query = projectLinkedGroupQuery(projectId, linkedGroupId);
BoolQueryBuilder query = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, "0");
// time
query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
// 情感倾向限制
......@@ -1507,7 +1623,7 @@ public class MarkDataServiceImpl implements MarkDataService {
// 获取索引
String[] indexes = esClientDao.getIndexes();
// 聚合请求
TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("count").field("platform_id.keyword").order(BucketOrder.count(false)).size(100);
TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("count").field("platform_id").order(BucketOrder.count(false)).size(100);
// query
BoolQueryBuilder query = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, contendId);
// keyword
......@@ -1545,9 +1661,9 @@ public class MarkDataServiceImpl implements MarkDataService {
EsClientDao.SearchHelper searchHelper = EsClientDao.createSearchHelper();
BoolQueryBuilder postFilter = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, "0");
postFilter.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
postFilter.must(QueryBuilders.existsQuery("brandkbs_cache_maps.channel_emotion"));
postFilter.must(EsQueryTools.cacheMapsNestedQuery(QueryBuilders.existsQuery("brandkbs_cache_maps.channel_emotion")));
searchHelper.setPostFilter(postFilter);
searchHelper.setFetchSource(new String[]{"brandkbs_cache_maps"});
searchHelper.setFetchSource(new String[]{GenericAttribute.ES_BRANDKBS_CACHE_MAPS});
List<SearchResponse> searchResponses = esClientDao.searchScrollResponse(searchHelper);
Map<Integer, AtomicInteger> emotionMap = new HashMap<>();
emotionMap.put(ChannelEmotion.POSITIVE.getState(), new AtomicInteger());
......@@ -1577,7 +1693,7 @@ public class MarkDataServiceImpl implements MarkDataService {
for (ImportantChannelEnum channelEnum : mediaTypeList) {
BoolQueryBuilder postFilter = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, "0");
postFilter.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
postFilter.must(QueryBuilders.matchQuery("brandkbs_cache_maps.channel_type.keyword", channelEnum.getState()));
postFilter.must(EsQueryTools.cacheMapsNestedQuery(QueryBuilders.matchQuery("brandkbs_cache_maps.channel_type.keyword", channelEnum.getState())));
Long count = esClientDao.count(postFilter);
mediaTypeMap.put(channelEnum.name().toLowerCase() + "Percent", count);
}
......@@ -1595,7 +1711,7 @@ public class MarkDataServiceImpl implements MarkDataService {
// fetchSource
searchHelper.setFetchSource(new String[]{"ind_title", "ind_full_text", "c5", "foreign", "brandkbs_mark_cache_maps"});
// postFilter
BoolQueryBuilder postFilter = projectLinkedGroupQuery(projectId, linkedGroupId);
BoolQueryBuilder postFilter = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, "0");
postFilter.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
searchHelper.setPostFilter(postFilter);
searchHelper.setSize(10000);
......@@ -1613,13 +1729,10 @@ public class MarkDataServiceImpl implements MarkDataService {
return textList;
}
@Deprecated
private static BoolQueryBuilder projectLinkedGroupQuery(String projectId, String linkedGroupId) {
return QueryBuilders.boolQuery().must(QueryBuilders.termQuery("brandkbs_cache_maps.project_id.keyword", projectId)).must(QueryBuilders.termQuery("brandkbs_cache_maps.linked_group_id.keyword", linkedGroupId));
}
protected static BoolQueryBuilder projectLinkedGroupContendIdQuery(String projectId, String linkedGroupId, String contendId) {
return QueryBuilders.boolQuery().must(QueryBuilders.termQuery("brandkbs_cache_maps.key.keyword", Tools.concat(projectId, linkedGroupId, contendId)));
return QueryBuilders.boolQuery().must(QueryBuilders.termQuery("brandkbs_cache_maps.key.keyword", Tools.concat(projectId, linkedGroupId, contendId))) ;
// TODO 上线后开启
// return EsQueryTools.assembleCacheMapsQuery(projectId, linkedGroupId, contendId);
}
private List<MarkFlowEntity> getMarkFlowEntity(MarkSearchDTO markSearchDTO, SearchHits searchHits) {
......
src/main/java/com/zhiwei/brandkbs2/service/impl/ReportServiceImpl.java
......@@ -5,19 +5,23 @@ import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Maps;
import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.common.RedisKeyPrefix;
import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.dao.EventDao;
import com.zhiwei.brandkbs2.dao.EventDataDao;
import com.zhiwei.brandkbs2.dao.ReportDao;
import com.zhiwei.brandkbs2.dao.ReportSettingsDao;
import com.zhiwei.brandkbs2.enmus.EmotionEnum;
import com.zhiwei.brandkbs2.enmus.ReportTypeEnum;
import com.zhiwei.brandkbs2.exception.ExceptionCast;
import com.zhiwei.brandkbs2.listener.ApplicationProjectListener;
import com.zhiwei.brandkbs2.model.CommonCodeEnum;
import com.zhiwei.brandkbs2.pojo.AbstractProject;
import com.zhiwei.brandkbs2.pojo.Report;
import com.zhiwei.brandkbs2.pojo.ReportSettings;
import com.zhiwei.brandkbs2.pojo.*;
import com.zhiwei.brandkbs2.pojo.dto.ReportDTO;
import com.zhiwei.brandkbs2.pojo.dto.ReportSearchDTO;
import com.zhiwei.brandkbs2.pojo.dto.ReportSettingsDTO;
import com.zhiwei.brandkbs2.pojo.vo.PageVO;
import com.zhiwei.brandkbs2.service.IndexService;
import com.zhiwei.brandkbs2.service.MarkDataService;
import com.zhiwei.brandkbs2.service.ProjectService;
import com.zhiwei.brandkbs2.service.ReportService;
import com.zhiwei.brandkbs2.util.MongoUtil;
......@@ -25,6 +29,7 @@ import com.zhiwei.brandkbs2.util.RedisUtil;
import com.zhiwei.brandkbs2.util.Tools;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
......@@ -36,11 +41,10 @@ import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;
import static com.zhiwei.brandkbs2.config.Constant.HOUR_FORMAT;
/**
* @ClassName: ReportServiceImpl
* @Description 报告业务接口实现类
......@@ -57,9 +61,21 @@ public class ReportServiceImpl implements ReportService {
@Resource(name = "reportSettingsDao")
ReportSettingsDao reportSettingsDao;
@Resource(name = "eventDao")
EventDao eventDao;
@Resource(name = "eventDataDao")
EventDataDao eventDataDao;
@Resource(name = "projectServiceImpl")
ProjectService projectService;
@Resource(name = "markDataServiceImpl")
MarkDataService markDataService;
@Resource(name = "indexServiceImpl")
IndexService indexService;
@Resource(name = "mongoUtil")
com.zhiwei.brandkbs2.util.MongoUtil mongoUtil;
......@@ -132,8 +148,7 @@ public class ReportServiceImpl implements ReportService {
List<JSONObject> result = new ArrayList<>();
Map<String, JSONObject> map = new HashMap<>();
// Map<String, List<JSONObject>> collect = list.stream().collect(Collectors.groupingBy(json -> json.getString(groupName)));
Map<String, Integer> collect = list.stream().collect(Collectors.toMap(json -> json.getString(groupName), json -> json.getInteger(countName),
Integer::sum));
Map<String, Integer> collect = list.stream().collect(Collectors.toMap(json -> json.getString(groupName), json -> json.getInteger(countName), Integer::sum));
// collect.put("不限", collect.values().stream().mapToInt(v -> v).sum());
collect.forEach((type, count) -> {
JSONObject json = new JSONObject();
......@@ -213,8 +228,13 @@ public class ReportServiceImpl implements ReportService {
result.put("status", false);
return result;
}
JSONObject result = getPcReportResult(report);
stringRedisTemplate.opsForValue().set(redisKey, JSON.toJSONString(result));
JSONObject result = null;
try {
result = getPcReportResult(report);
} catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("es查询异常"));
}
redisUtil.set(redisKey, JSON.toJSONString(result));
return result;
}
......@@ -273,41 +293,190 @@ public class ReportServiceImpl implements ReportService {
* @param report 报告对象
* @return 月报结果
*/
private JSONObject getPcReportResult(Report report) {
private JSONObject getPcReportResult(Report report) throws IOException {
Long startTime = report.getStartTime();
Long endTime = report.getEndTime();
String startTimeStr = HOUR_FORMAT.format(startTime);
String endTimeStr = HOUR_FORMAT.format(endTime);
String lastStartTimeStr = HOUR_FORMAT.format(startTime - (endTime - startTime));
Long lastStartTimeStr = startTime - (endTime - startTime);
String projectId = report.getProjectId();
String contendId = "0";
String linkedGroupId = projectService.getProjectByContendId(projectId, contendId).getBrandLinkedGroupId();
JSONObject result = new JSONObject();
result.put("title", report.getTitle());
result.put("startTime", startTime);
result.put("endTime", endTime);
result.put("status", true);
result.put("type", ReportTypeEnum.getInstanceByState(report.getType()).getDescribe());
// TODO
result.put("todo", "尚未完成");
result.put("brandSummary", this.getPcBrandSummary(startTime, endTime, lastStartTimeStr, projectId, linkedGroupId, contendId));
List<Map<String, Long>> dayList = Tools.parseToDays(startTime, DateUtils.addDays(new Date(endTime), -1).getTime());
result.put("brandSpread", this.getPcBrandSpread(dayList, projectId, linkedGroupId, contendId));
result.put("brandEvent", this.getPcBrandEvent(startTime, endTime, projectId, contendId));
List<JSONObject> contendCompare = report.getContends().stream().map(brandContendId -> {
JSONObject json = new JSONObject();
AbstractProject abstractProject = projectService.getProjectByContendId(projectId, brandContendId);
json.put("id", brandContendId);
json.put("brand", abstractProject.getBrandName());
json.put("brandCompare", indexService.getSpreadingTend(startTime, endTime, projectId, contendId, true));
return json;
}).collect(Collectors.toList());
result.put("contendCompare", contendCompare);
return result;
}
/**
* 获取报告类型聚合结果
* 获取品牌总览页面数据
*
* @param pid 项目ID
* @param uid 用户ID
* @return 报告类型聚合结果
* @param startTime 开始时间
* @param endTime 结束时间
* @param lastStartTime 上个月开始时间
* @param projectId 项目ID
* @return 总览页面数据
*/
private List<JSONObject> getReportTypeAggCount(int pid, int uid) {
List<String> types = Arrays.stream(ReportTypeEnum.values()).map(ReportTypeEnum::getState).collect(Collectors.toList());
return types.stream()
.map(type -> {
private JSONObject getPcBrandSummary(Long startTime, Long endTime, Long lastStartTime, String projectId, String linkedGroupId, String contendId) throws IOException {
JSONObject result = new JSONObject();
result.put("type", type);
// result.put("count", reportDao.selectReportCountByTimeAndTypeAndProjectIdAndUserId(null, null, type, pid, uid));
//本月舆情总量数据
long curTotal = markDataService.getYuqingMarkCount(startTime, endTime, EmotionEnum.ALL.getName(), projectId, contendId);
result.put("curTotal", curTotal);
//上月舆情总量数据
long lastTotal = markDataService.getYuqingMarkCount(lastStartTime, startTime, EmotionEnum.ALL.getName(), projectId, contendId);
result.put("lastTotal", lastTotal);
result.put("compare", lastTotal == 0 ? 0 : (curTotal - lastTotal) * 1.0 / lastTotal);
//获取时间范围内总正面稿件数
long curPositiveTotal = markDataService.getYuqingMarkCount(startTime, endTime, EmotionEnum.POSITIVE.getName(), projectId, contendId);
result.put("curPosTotal", curPositiveTotal);
result.put("curPosPro", curTotal == 0 ? 0 : curPositiveTotal * 1.0 / curTotal);
//正面事件传播量top
List<Event> topPosEventList = eventDao.getEventsByTotalChannelVolumeTop(startTime, endTime, EmotionEnum.POSITIVE.getName(), projectId, contendId, 3);
if (CollectionUtils.isEmpty(topPosEventList)) {
List<Map.Entry<String, Integer>> topPosArticleList = markDataService.getMarkTopTitle(startTime, endTime, EmotionEnum.POSITIVE.getName(), projectId, linkedGroupId, contendId, 3);
result.put("topPosSummary", this.getTopArticlesMsg(startTime, endTime, projectId, linkedGroupId, contendId, topPosArticleList));
} else {
result.put("topPosSummary", this.getTopEventMsg(topPosEventList));
}
//获取上个周期时间范围内总正面稿件数
long lastPositiveTotal = markDataService.getYuqingMarkCount(lastStartTime, startTime,
EmotionEnum.POSITIVE.getName(), projectId, contendId);
result.put("lastPosTotal", lastPositiveTotal);
result.put("posCompare", lastPositiveTotal == 0 ? 0 : (curPositiveTotal - lastPositiveTotal) * 1.0 / lastPositiveTotal);
//获取时间范围内总中性稿件数
long curNeutralTotal = markDataService.getYuqingMarkCount(startTime, endTime, EmotionEnum.NEUTRAL.getName(), projectId, contendId);
result.put("curNeuTotal", curNeutralTotal);
result.put("curNeuPro", curTotal == 0 ? 0 : curNeutralTotal * 1.0 / curTotal);
//中性事件传播量top
List<Event> topNeuEventList = eventDao.getEventsByTotalChannelVolumeTop(startTime, endTime, EmotionEnum.NEUTRAL.getName(), projectId, contendId, 4);
if (CollectionUtils.isEmpty(topNeuEventList)) {
List<Map.Entry<String, Integer>> topNeuArticleList = markDataService.getMarkTopTitle(startTime, endTime, EmotionEnum.NEUTRAL.getName(), projectId,
linkedGroupId, contendId, 4);
result.put("topNeuSummary", this.getTopArticlesMsg(startTime, endTime, projectId, linkedGroupId, contendId, topNeuArticleList));
} else {
result.put("topNeuSummary", this.getTopEventMsg(topNeuEventList));
}
//获取上个周期时间范围内总中性稿件数
long lastNeutralTotal = markDataService.getYuqingMarkCount(lastStartTime, startTime, EmotionEnum.NEUTRAL.getName(), projectId, contendId);
result.put("lastNeuTotal", lastNeutralTotal);
result.put("neuCompare", lastNeutralTotal == 0 ? 0 : (curNeutralTotal - lastNeutralTotal) * 1.0 / lastNeutralTotal);
//获取时间范围内总负面稿件数
long curNegativeTotal = markDataService.getYuqingMarkCount(startTime, endTime, EmotionEnum.NEGATIVE.getName(), projectId, contendId);
result.put("curNegTotal", curNegativeTotal);
result.put("curNegPro", curTotal == 0 ? 0 : curNegativeTotal * 1.0 / curTotal);
//中性事件传播量top
List<Event> topNegEventList = eventDao.getEventsByTotalChannelVolumeTop(startTime, endTime, EmotionEnum.NEGATIVE.getName(), projectId, contendId, 4);
if (CollectionUtils.isEmpty(topNegEventList)) {
List<Map.Entry<String, Integer>> topNegArticleList = markDataService.getMarkTopTitle(startTime, endTime, EmotionEnum.NEGATIVE.getName(),
projectId, linkedGroupId, contendId, 4);
result.put("topNegSummary", this.getTopArticlesMsg(startTime, endTime, projectId, linkedGroupId, contendId, topNegArticleList));
} else {
result.put("topNegSummary", this.getTopEventMsg(topNegEventList));
}
//获取上个周期时间范围内总负面稿件数
long lastNegativeTotal = markDataService.getYuqingMarkCount(lastStartTime, startTime, EmotionEnum.NEGATIVE.getName(), projectId, contendId);
result.put("lastNegTotal", lastNegativeTotal);
result.put("negCompare", lastNegativeTotal == 0 ? 0 : (curNegativeTotal - lastNegativeTotal) * 1.0 / lastNegativeTotal);
return result;
})
.filter(json -> json.getIntValue("count") > 0)
}
/**
* 获取品牌传播分析页面数据
*
* @param dayList 分段时间集合
* @param projectId 项目ID
* @return 传播分析页面数据
*/
private JSONObject getPcBrandSpread(List<Map<String, Long>> dayList, String projectId, String linkedGroupId, String contendId) throws IOException {
JSONObject result = new JSONObject();
List<JSONObject> lineList = new ArrayList<>(dayList.size());
//获取传播趋势
for (Map<String, Long> map : dayList) {
Long startTime = map.get("startTime");
Long endTime = map.get("endTime");
//统计时间段总稿件个数
long normalCount = markDataService.getYuqingMarkCount(startTime, endTime, EmotionEnum.ALL.getName(), projectId, contendId);
//统计时间段正面稿件个数
long posNormalCount = markDataService.getYuqingMarkCount(startTime, endTime, EmotionEnum.POSITIVE.getName(), projectId, contendId);
//统计时间段中性稿件个数
long neuNormalCount = markDataService.getYuqingMarkCount(startTime, endTime, EmotionEnum.NEUTRAL.getName(), projectId, contendId);
//统计时间段负面稿件个数
long negNormalCount = markDataService.getYuqingMarkCount(startTime, endTime, EmotionEnum.NEGATIVE.getName(), projectId, contendId);
JSONObject line = new JSONObject();
line.put("normalCount", normalCount);
line.put("posNormalCount", posNormalCount);
line.put("neuNormalCount", neuNormalCount);
line.put("negNormalCount", negNormalCount);
line.put("time", startTime);
lineList.add(line);
}
result.put("spread", lineList);
//获取稿件的传播峰值
List<Long> dateList = lineList.stream()
.sorted((o1, o2) -> o2.getInteger("normalCount").compareTo(o1.getInteger("normalCount")))
.map(json -> json.getLong("time"))
.collect(Collectors.toList());
result.put("normalMaxTimes", dateList.size() > 2 ? dateList.subList(0, 3) : dateList);
if (CollectionUtils.isNotEmpty(dateList)) {
Long startTime = dateList.get(0);
Long endTime = DateUtils.addDays(new Date(startTime), 1).getTime();
//获取最热稿件相关信息
List<Map.Entry<String, Integer>> resList = markDataService.getMarkTopTitle(startTime, endTime, EmotionEnum.ALL.getName(), projectId, linkedGroupId, contendId, 1);
if (CollectionUtils.isNotEmpty(resList)) {
Map.Entry<String, Integer> entry = resList.get(0);
BaseMap baseMap = markDataService.getFirstArticle(startTime, endTime, entry.getKey(), projectId, linkedGroupId, contendId);
result.put("topArticle", baseMap.getTitle());
result.put("topArticleUrl", baseMap.getUrl());
result.put("topArticleEmotion", baseMap.getEmotion());
}
}
return result;
}
/**
* 获取品牌事件数据
*
* @param startTime 开始时间
* @param endTime 结束时间
* @param projectId 项目ID
* @return 品牌事件数据
*/
private List<JSONObject> getPcBrandEvent(Long startTime, Long endTime, String projectId, String contendId) {
List<String> emotions = Arrays.asList(EmotionEnum.POSITIVE.getName(), EmotionEnum.NEGATIVE.getName());
return emotions.stream().map(emotion -> {
JSONObject result = new JSONObject();
result.put("name", Objects.equals(emotion, EmotionEnum.POSITIVE.getName()) ? "正面事件" : Objects.equals(emotion, EmotionEnum.NEGATIVE.getName()) ? "负面事件" : "中性事件");
List<Event> topEventList = eventDao.getEventsByTotalChannelVolumeTop(startTime, endTime, emotion, projectId, contendId, 10);
result.put("events", topEventList.stream().map(event -> {
JSONObject topEvent = new JSONObject();
boolean hasAnalyze = Objects.equals(event.getContendId(), Constant.PRIMARY_CONTEND_ID);
topEvent.put("id", event.getId());
String title = event.getTitle();
topEvent.put("title", title.length() <= 30 ? title : title.substring(0, 30) + "...");
topEvent.put("influence", event.getInfluence());
topEvent.put("articleCount", event.getTotalDisseminationVolume());
//查询首发稿件
EventData firstData = eventDataDao.findFirstData(event.getId(), event.getCollectionName());
topEvent.put("firstSource", Objects.isNull(firstData) ? "" : firstData.getSource());
topEvent.put("hasAnalyze", hasAnalyze);
return topEvent;
}).collect(Collectors.toList()));
return result;
}).collect(Collectors.toList());
}
private List<ReportSettings> getReportSettingsList() {
......@@ -319,4 +488,49 @@ public class ReportServiceImpl implements ReportService {
return reports;
}
/**
* 获取top稿件信息
*
* @param startTime 开始时间
* @param endTime 结束时间
* @param projectId 项目ID
* @param topArticleList top稿件集合
* @return top稿件信息
*/
private List<JSONObject> getTopArticlesMsg(Long startTime, Long endTime, String projectId, String linkedGroupId, String contendId, List<Map.Entry<String, Integer>> topArticleList) {
return topArticleList.stream().map(entry -> {
JSONObject topPosArticle = null;
try {
String aggTitle = entry.getKey();
BaseMap firstArticle = markDataService.getFirstArticle(startTime, endTime, aggTitle, projectId, linkedGroupId, contendId);
topPosArticle = new JSONObject();
topPosArticle.put("title", firstArticle.getTitle());
topPosArticle.put("url", firstArticle.getUrl());
} catch (IOException e) {
log.error("getTopArticlesMsg-getFirstArticle-", e);
}
return topPosArticle;
}).filter(Objects::nonNull).collect(Collectors.toList());
}
/**
* 获取top事件信息
*
* @param topEventList top事件集合
* @return top事件信息
*/
private List<JSONObject> getTopEventMsg(List<Event> topEventList) {
return topEventList.stream()
.map(event -> {
boolean hasAnalyze = Objects.equals(event.getContendId(), Constant.PRIMARY_CONTEND_ID);
JSONObject jsonObject = new JSONObject();
jsonObject.put("id", event.getId());
String title = event.getTitle();
jsonObject.put("title", title.length() <= 30 ? title : title.substring(0, 30) + "...");
jsonObject.put("hasAnalyze", hasAnalyze);
return jsonObject;
}).collect(Collectors.toList());
}
}
src/main/java/com/zhiwei/brandkbs2/util/Tools.java
......@@ -37,6 +37,7 @@ import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel;
import java.text.ParseException;
import java.util.*;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
......@@ -446,6 +447,17 @@ public class Tools {
/**
* 日期取整，去掉时分秒
*
* @param time
* @param pattern
* @return
*/
public static Long truncDate(Long time, String pattern) {
return truncDate(new Date(time), pattern).getTime();
}
/**
* 日期取整，去掉时分秒
*
* @param date
* @return Date
* @Title: truncDate
......@@ -782,7 +794,6 @@ public class Tools {
* @return 按小时分割的map集合
*/
public static List<Map<String, Long>> parseToHours(Long startTime, Long endTime) {
FastDateFormat df = HOUR_FORMAT;
Date start = Tools.truncDate(new Date(startTime), Constant.HOUR_PATTERN);
Date end = Tools.truncDate(new Date(endTime), Constant.HOUR_PATTERN);
Period periodHour = new Period(start.getTime(), end.getTime(), PeriodType.hours());
......@@ -805,8 +816,8 @@ public class Tools {
/**
* 解析时间转换成按月基数时间的集合
*
* @param startTime 开始时间
* @param endTime 结束时间
* @param startLong 开始时间
* @param endLong 结束时间
* @return 按月基数分割时间的集合
*/
public static List<Map<String, Long>> parseToDaysWithBase(Long startLong, Long endLong) {
......@@ -842,6 +853,7 @@ public class Tools {
*/
public static boolean isContains(List<String> keywords, String content) {
boolean contains = true;
if (null != keywords) {
content = content.toLowerCase();
// 按空格分割必须全部命中
for (String keyword : keywords) {
......@@ -852,11 +864,12 @@ public class Tools {
}
}
}
}
return contains;
}
public static JSONObject getBrandkbsHitMap(Map<String, Object> esMap, String hitKey) {
List<Map<String, Object>> cacheMaps = (List<Map<String, Object>>) esMap.get("brandkbs_cache_maps");
List<Map<String, Object>> cacheMaps = (List<Map<String, Object>>) esMap.get(GenericAttribute.ES_BRANDKBS_CACHE_MAPS);
for (Map<String, Object> cacheMap : cacheMaps) {
if (hitKey.equals(cacheMap.get("key"))) {
return new JSONObject(cacheMap);
......@@ -865,6 +878,26 @@ public class Tools {
return new JSONObject();
}
public static <T> Map<String, T> sortTimeKeyMap(Map<String, T> map, FastDateFormat timePattern, boolean isDesc) {
Map<String, T> res = new LinkedHashMap<>();
List<Map.Entry<String, T>> sortKey = map.entrySet().stream().sorted((x, y) -> {
try {
long xTime = timePattern.parse(x.getKey()).getTime();
long yTime = timePattern.parse(y.getKey()).getTime();
if (isDesc) {
return Long.compare(yTime, xTime);
}
return Long.compare(xTime, yTime);
} catch (ParseException e) {
return 0;
}
}).collect(Collectors.toList());
for (Map.Entry<String, T> longTEntry : sortKey) {
res.put(longTEntry.getKey(), longTEntry.getValue());
}
return res;
}
public static <T> Map<Long, T> sortTimeKeyMap(Map<Long, T> map, boolean isDesc) {
Map<Long, T> res = new LinkedHashMap<>();
List<Map.Entry<Long, T>> sortKey = map.entrySet().stream().sorted((x, y) -> {
......
src/main/resources/application-local.properties
......@@ -76,3 +76,7 @@ ef.searchCriteria.url=https://ef.zhiweidata.com/index/getSearchKey.do
ef.filterNew.url=https://ef.zhiweidata.com/filterNew.do?firstType={1}&start={2}&end={3}&page={4}
ef.captcha.url=https://ef.zhiweidata.com/captchaApi/getNewCaptcha.do.do?type=test
ef.checkCaptcha.url=https://ef.zhiweidata.com/captchaApi/checkeCaptcha.do.do?id={1}&captcha={2}&type=test
#\u5168\u7F51\u641C\u5916\u90E8\u63A5\u53E3
istarshine.addIStarShineKSData.url=https://istarshine-service.zhiweidata.com/istarshine/ks/addIStarShineKSData
istarshine.getIStarShineKSInfoData.url=https://istarshine-service.zhiweidata.com/istarshine/ks/getIStarShineKSInfoData?taskId={1}&appId=6183571e0d710000f6003a12
whole.search.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/search/yuqing
\ No newline at end of file
src/main/resources/application.properties
spring.profiles.active=pro
spring.profiles.active=local
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment