Skip to content

B
brandkbs2
Commit 053f2b38 by shenjunjie
Options

Merge branch 'feature' into 'dev' 

舆情库-舆情分析

See merge request !7
parents 54695e3d a5619849
Showing with 563 additions and 140 deletions
src/main/java/com/zhiwei/brandkbs2/common/RedisKeyPrefix.java
......@@ -15,8 +15,12 @@ public class RedisKeyPrefix {
private RedisKeyPrefix() {
}
public static final String MARK_ANALYZE_SUMMARY = "BRANDKBS:MARK:ANALYZE:SUMMARY:";
public static final String ARTICLE_SPREAD = "BRANDKBS:MARK:ANALYZE:SPREAD:";
public static final String MARK_ANALYZE_SUMMARY = "BRANDKBS:MARK:ANALYZE_SUMMARY:";
public static final String MARK_PLATFORM_PROPORTION = "BRANDKBS:MARK:PLATFORM_PROPORTION:";
public static final String MARK_SPREAD = "BRANDKBS:MARK:SPREAD:";
public static final String MARK_CHANNEL_PROPORTION = "BRANDKBS:MARK:CHANNEL_PROPORTION:";
public static final String MARK_EMOTION_PROPORTION = "BRANDKBS:MARK:EMOTION_PROPORTION:";
public static final String MARK_HIGH_WORD = "BRANDKBS:MARK:HIGH_WORD:";
private static final String SEPARATOR = ":";
......
src/main/java/com/zhiwei/brandkbs2/controller/AppArticleController.java
......@@ -19,38 +19,88 @@ import javax.annotation.Resource;
/**
* @author sjj
* @ClassName AppArticleController
* @Description 提供前台稿件相关信息展示
* @Description 提供前台舆情库相关信息展示
* @date 2022-06-20 09:27
*/
@RestController
@RequestMapping("/app/yuqing")
@Api(tags = "前台稿件展示接口", description = "提供前台稿件相关信息展示")
@Api(tags = "前台舆情库展示接口", description = "提供前台舆情库相关信息展示")
@Auth(role = RoleEnum.CUSTOMER)
public class AppArticleController extends BaseController {
@Resource(name = "markDataServiceImpl")
IMarkDataService markDataService;
@ApiOperation("舆情标注数据列表")
@GetMapping("/list/mark")
@ApiOperation("舆情列表")
@GetMapping("/mark/list")
public ResponseResult getYuqingMarkList(MarkSearchDTO markSearchDTO) {
return ResponseResult.success(markDataService.getYuqingMarkList(markSearchDTO));
}
@ApiOperation("舆情标注数据搜索条件")
@GetMapping("/list/mark/criteria")
@ApiOperation("舆情列表-搜索条件")
@GetMapping("/mark/list/criteria")
public ResponseResult getYuqingMark(@RequestParam(required = false) String linkedGroupId) {
return ResponseResult.success(markDataService.getYuqingMarkCriteria(linkedGroupId));
}
@ApiOperation("舆情标注数据提要")
@ApiOperation("舆情分析-提要")
@ApiImplicitParams({@ApiImplicitParam(name = "startTime", value = "开始时间", paramType = "query", dataType = "long"), @ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long")})
@GetMapping("/analyze/summary")
public ResponseResult getAnalyzeSummary(@RequestParam(value = "startTime", required = false) Long startTime, @RequestParam(value = "endTime", required = false) Long endTime) {
return ResponseResult.success(markDataService.getAnalyzeSummary(startTime, endTime, true));
}
@ApiOperation("舆情分析-传播趋势")
@ApiImplicitParams({
@ApiImplicitParam(name = "startTime", value = "开始时间", paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long")
})
@GetMapping("/analyze/summary")
public ResponseResult getArticleSummary(@RequestParam(value = "startTime", required = false) Long startTime,
@GetMapping("/spread")
public ResponseResult getMarkSpread(@RequestParam(value = "startTime", required = false) Long startTime,
@RequestParam(value = "endTime", required = false) Long endTime) {
return ResponseResult.success(markDataService.getAnalyzeSummary(startTime, endTime, true));
return ResponseResult.success(markDataService.getMarkSpread(startTime, endTime, true));
}
@ApiOperation("舆情分析-情感分布")
@ApiImplicitParams({
@ApiImplicitParam(name = "startTime", value = "开始时间", paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long")
})
@GetMapping("/emotion/proportion")
public ResponseResult getMarkEmotionProportion(@RequestParam(value = "startTime", required = false) Long startTime,
@RequestParam(value = "endTime", required = false) Long endTime) {
return ResponseResult.success(markDataService.getMarkEmotionProportion(startTime, endTime, true));
}
@ApiOperation("舆情分析-平台分布")
@ApiImplicitParams({
@ApiImplicitParam(name = "startTime", value = "开始时间", paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long")
})
@GetMapping("/platform/proportion")
public ResponseResult getMarkPlatformProportion(@RequestParam(value = "startTime", required = false) Long startTime,
@RequestParam(value = "endTime", required = false) Long endTime) {
return ResponseResult.success(markDataService.getMarkPlatformProportion(startTime, endTime, true));
}
@ApiOperation("舆情分析-渠道分布")
@ApiImplicitParams({@ApiImplicitParam(name = "startTime", value = "开始时间", paramType = "query", dataType = "long"), @ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long")})
@GetMapping("/channel/proportion")
public ResponseResult getMarkChannelProportion(@RequestParam(value = "startTime", required = false) Long startTime, @RequestParam(value = "endTime", required = false) Long endTime) {
return ResponseResult.success(markDataService.getMarkChannelProportion(startTime, endTime, true));
}
@ApiOperation("舆情分析-高频词分布")
@ApiImplicitParams({
@ApiImplicitParam(name = "startTime", value = "开始时间", paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long")
})
@GetMapping("/highWord")
public ResponseResult getMarkHighWord(@RequestParam(value = "startTime", required = false) Long startTime,
@RequestParam(value = "endTime", required = false) Long endTime) {
return ResponseResult.success(markDataService.getMarkHighWord(startTime, endTime));
}
}
src/main/java/com/zhiwei/brandkbs2/controller/CommonController.java
......@@ -4,6 +4,7 @@ import com.zhiwei.brandkbs2.auth.Auth;
import com.zhiwei.brandkbs2.enmus.RoleEnum;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.service.ICommonService;
import com.zhiwei.brandkbs2.service.IProjectService;
import com.zhiwei.middleware.mark.pojo.enums.TagField;
import com.zhiwei.middleware.mark.vo.MarkerTag;
import io.swagger.annotations.Api;
......@@ -34,6 +35,9 @@ public class CommonController extends BaseController {
@Resource(name = "commonServiceImpl")
ICommonService commonService;
@Resource(name = "projectServiceImpl")
IProjectService projectService;
@ApiOperation("获取情感倾向标签信息")
@ApiImplicitParam(name = "linkedGroupId", value = "关联项目ID", required = true, paramType = "path", dataType = "string")
@GetMapping("/get/tag/emotion/{linkedGroupId}")
......@@ -56,6 +60,13 @@ public class CommonController extends BaseController {
}
}
@ApiOperation("获取当前用户拥有的所有项目及品牌列表")
@GetMapping("/user/getUserAllProjects")
@Auth(role = RoleEnum.CUSTOMER)
public ResponseResult getUserAllProjects() {
return ResponseResult.success(projectService.getUserAllProjects());
}
@ApiOperation("测试接口")
@GetMapping("/test")
public ResponseResult test() {
......
src/main/java/com/zhiwei/brandkbs2/es/EsClientDao.java
......@@ -5,6 +5,8 @@ import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.pojo.ChannelIndex;
import com.zhiwei.brandkbs2.pojo.Event;
import com.zhiwei.brandkbs2.util.Tools;
import lombok.Getter;
import lombok.Setter;
import org.apache.commons.lang3.time.FastDateFormat;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
......@@ -226,12 +228,62 @@ public class EsClientDao {
// return builder;
// }
public SearchHits searchHits(String[] indexes, QueryBuilder postFilter, QueryBuilder query, FieldSortBuilder sort, int from, int size, HighlightBuilder highlighter) throws IOException {
return searchResponse(indexes, postFilter, query, null, sort, from, size, highlighter).getHits();
public SearchHits searchHits(String[] indexes, QueryBuilder postFilter, QueryBuilder query, String[] fetchSource, FieldSortBuilder sort, int from, int size, HighlightBuilder highlighter) throws IOException {
return searchResponse(indexes, postFilter, query, null, fetchSource, sort, from, size, highlighter).getHits();
}
public SearchHits searchHits(SearchHelper searchHelper) throws IOException {
return searchHits(searchRequest(searchHelper));
}
public SearchResponse searchResponse(String[] indexes, QueryBuilder postFilter, QueryBuilder query, AggregationBuilder aggregationBuilder,
FieldSortBuilder sort, int from, int size, HighlightBuilder highlighter) throws IOException {
String[] fetchSource, FieldSortBuilder sort, int from, int size, HighlightBuilder highlighter) throws IOException {
return searchResponse(searchRequest(indexes, postFilter, query, aggregationBuilder, fetchSource, sort, from, size, highlighter));
}
public SearchResponse searchResponse(SearchHelper searchHelper) throws IOException {
return searchResponse(searchRequest(searchHelper));
}
public List<SearchResponse> searchScrollResponse(SearchHelper searchHelper) throws IOException {
return retryTemplate.execute(context -> scrollSearch(searchRequest(searchHelper)));
}
public Long count(String[] indexes, QueryBuilder postFilter, QueryBuilder query) throws IOException {
CountRequest countRequest = new CountRequest();
BoolQueryBuilder countQuery = QueryBuilders.boolQuery();
if (!Objects.isNull(query)) {
countQuery.must(query);
}
if (!Objects.isNull(postFilter)) {
countQuery.must(postFilter);
}
countRequest.indices(indexes);
countRequest.query(countQuery);
CountResponse response = retryTemplate.execute(context -> esClient.count(countRequest, RequestOptions.DEFAULT));
return response.getCount();
}
public static SearchHelper createSearchHelper() {
return new SearchHelper();
}
private SearchHits searchHits(SearchRequest searchRequest) throws IOException {
return searchResponse(searchRequest).getHits();
}
private SearchResponse searchResponse(SearchRequest searchRequest) throws IOException {
return retryTemplate.execute(context -> esClient.search(searchRequest, RequestOptions.DEFAULT));
}
private SearchRequest searchRequest(SearchHelper searchHelper) {
defaultInit(searchHelper);
return searchRequest(searchHelper.getIndexes(), searchHelper.getPostFilter(), searchHelper.getQuery(), searchHelper.getAggregationBuilder(), searchHelper.getFetchSource(), searchHelper.getSort(), searchHelper.getFrom(), searchHelper.getSize(), searchHelper.getHighlighter());
}
private SearchRequest searchRequest(String[] indexes, QueryBuilder postFilter, QueryBuilder query, AggregationBuilder aggregationBuilder,
String[] fetchSource,
FieldSortBuilder sort, int from, int size, HighlightBuilder highlighter) {
SearchRequest searchRequest = new SearchRequest();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchRequest.indices(indexes);
......@@ -244,35 +296,69 @@ public class EsClientDao {
if (!Objects.isNull(aggregationBuilder)) {
searchSourceBuilder.aggregation(aggregationBuilder);
}
if (!Objects.isNull(fetchSource)) {
searchSourceBuilder.fetchSource(fetchSource, null);
}
if (!Objects.isNull(sort)) {
searchSourceBuilder.sort(sort);
}
if (from >= 0) {
searchSourceBuilder.from(from);
}
if (size >= 0) {
if (size == 0) {
searchSourceBuilder.size(10000);
} else if (size > 0) {
searchSourceBuilder.size(size);
}
if (!Objects.isNull(highlighter)) {
searchSourceBuilder.highlighter(highlighter);
}
searchRequest.source(searchSourceBuilder);
return retryTemplate.execute(context -> esClient.search(searchRequest, RequestOptions.DEFAULT));
return searchRequest;
}
public Long count(String[] indexes, QueryBuilder postFilter, QueryBuilder query) throws IOException {
CountRequest countRequest = new CountRequest();
BoolQueryBuilder countQuery = QueryBuilders.boolQuery();
if (!Objects.isNull(query)) {
countQuery.must(query);
private List<SearchResponse> scrollSearch(SearchRequest searchRequest) throws IOException {
List<SearchResponse> res = new ArrayList<>();
searchRequest.scroll(TIME_VALUE);
SearchResponse searchResponse = esClient.search(searchRequest, RequestOptions.DEFAULT);
String scrollId = searchResponse.getScrollId();
res.add(searchResponse);
while (Objects.nonNull(scrollId)) {
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(TIME_VALUE);
searchResponse = esClient.scroll(scrollRequest, RequestOptions.DEFAULT);
scrollId = searchResponse.getScrollId();
if (1 > searchResponse.getHits().getHits().length) {
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId);
esClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
scrollId = null;
} else {
res.add(searchResponse);
}
if (!Objects.isNull(postFilter)) {
countQuery.must(postFilter);
}
countRequest.indices(indexes);
countRequest.query(countQuery);
CountResponse response = retryTemplate.execute(context -> esClient.count(countRequest, RequestOptions.DEFAULT));
return response.getCount();
return res;
}
private void defaultInit(SearchHelper searchHelper) {
if (null == searchHelper.getIndexes()) {
searchHelper.setIndexes(EsClientDao.this.getIndexes());
}
}
@Setter
@Getter
public static class SearchHelper {
String[] indexes;
QueryBuilder postFilter;
QueryBuilder query;
AggregationBuilder aggregationBuilder;
String[] fetchSource;
FieldSortBuilder sort;
int from;
int size;
HighlightBuilder highlighter;
}
}
src/main/java/com/zhiwei/brandkbs2/pojo/BaseMap.java
......@@ -23,6 +23,8 @@ public class BaseMap {
*/
private String title;
private String incompleteContent;
/**
* content
*/
......@@ -58,4 +60,12 @@ public class BaseMap {
*/
private String emotion;
public String getTitleNullOptionalContent() {
if (null != title) {
return title;
} else {
return incompleteContent;
}
}
}
src/main/java/com/zhiwei/brandkbs2/pojo/Channel.java
......@@ -55,7 +55,7 @@ public class Channel extends ChannelIndex {
/**
* 渠道指数
*/
private Double EmotionIndex = 0d;
private Double emotionIndex = 0d;
/**
* 经验评级
......
src/main/java/com/zhiwei/brandkbs2/pojo/EventData.java
......@@ -91,7 +91,7 @@ public class EventData extends AbstractBaseMongo {
data.setRealSource(baseMap.getRealSource());
data.setSource(baseMap.getSource());
data.setUrl(baseMap.getUrl());
data.setTitle(baseMap.getTitle());
data.setTitle(baseMap.getTitleNullOptionalContent());
data.setAggTitle(Tools.filterSpecialCharacter(baseMap.getTitle()));
data.setContent(baseMap.getContent());
data.setEmotion(baseMap.getEmotion());
......
src/main/java/com/zhiwei/brandkbs2/pojo/dto/MarkSearchDTO.java
......@@ -95,7 +95,7 @@ public class MarkSearchDTO {
* 排序字段
*/
@ApiModelProperty(value = "排序字段")
private String sorter;
private String sorter = "{\"mtime\":\"descend\"}";
/**
* 是否聚合
......
src/main/java/com/zhiwei/brandkbs2/service/IMarkDataService.java
......@@ -5,6 +5,8 @@ import com.zhiwei.brandkbs2.pojo.MarkFlowEntity;
import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
import com.zhiwei.brandkbs2.pojo.vo.PageVO;
import java.util.List;
/**
* @ClassName: IMarkDataService
* @Description: 标注数据服务抽象类
......@@ -21,7 +23,6 @@ public interface IMarkDataService {
/**
* 舆情标注数据搜索条件
*
*/
JSONObject getYuqingMarkCriteria(String linkedGroup);
......@@ -31,7 +32,54 @@ public interface IMarkDataService {
* @param startTime 开始时间
* @param endTime 结束时间
* @param cache 是否启用缓存
* @return 舆情消息流提要信息
* @return 舆情标注数据提要信息
*/
JSONObject getAnalyzeSummary(Long startTime, Long endTime, boolean cache);
/**
* 获取舆情标注数据传播趋势
*
* @param startTime 开始时间
* @param endTime 结束时间
* @return 获取舆情标注数据传播趋势
*/
List<JSONObject> getMarkSpread(Long startTime, Long endTime, boolean cache);
/**
* 获取舆情标注数据舆情情感分布
*
* @param startTime 开始时间
* @param endTime 结束时间
* @return 获取舆情标注数据舆情情感分布
*/
JSONObject getMarkEmotionProportion(Long startTime, Long endTime, boolean cache);
/**
* 获取舆情标注数据舆情渠道分布
*
* @param startTime 开始时间
* @param endTime 结束时间
* @param cache 是否启用缓存
* @return 获取舆情标注数据舆情渠道分布
*/
List<JSONObject> getMarkChannelProportion(Long startTime, Long endTime, boolean cache);
/**
* 获取舆情标注数据舆情平台分布
*
* @param startTime 开始时间
* @param endTime 结束时间
* @return 获取舆情标注数据舆情平台分布
*/
List<JSONObject> getMarkPlatformProportion(Long startTime, Long endTime, boolean cache);
/**
* 获取舆情标注数据舆情高频词分布
*
* @param startTime 开始时间
* @param endTime 结束时间
* @return 获取舆情标注数据舆情高频词分布
*/
List<JSONObject> getMarkHighWord(Long startTime, Long endTime);
}
src/main/java/com/zhiwei/brandkbs2/service/impl/ChannelServiceImpl.java
......@@ -100,7 +100,7 @@ public class ChannelServiceImpl implements IChannelService {
JSONObject json = new JSONObject();
BaseMap baseMap = Tools.getBaseFromEsMap(sourceJson);
json.put("id", sourceJson.get("id"));
json.put("title", baseMap.getTitle());
json.put("title",baseMap.getTitleNullOptionalContent());
json.put("url", baseMap.getUrl());
json.put("time", new Date(baseMap.getTime()));
json.put("emotion", baseMap.getEmotion());
......
src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java
......@@ -4,7 +4,6 @@ package com.zhiwei.brandkbs2.service.impl;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.ImmutableMap;
import com.sun.org.apache.bcel.internal.generic.RETURN;
import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.common.GlobalPojo;
......@@ -16,15 +15,13 @@ import com.zhiwei.brandkbs2.es.EsClientDao;
import com.zhiwei.brandkbs2.es.EsQueryTools;
import com.zhiwei.brandkbs2.exception.ExceptionCast;
import com.zhiwei.brandkbs2.model.CommonCodeEnum;
import com.zhiwei.brandkbs2.pojo.BaseMap;
import com.zhiwei.brandkbs2.pojo.MarkFlowEntity;
import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
import com.zhiwei.brandkbs2.pojo.vo.CustomTagVo;
import com.zhiwei.brandkbs2.pojo.vo.PageVO;
import com.zhiwei.brandkbs2.pojo.vo.ProjectVO;
import com.zhiwei.brandkbs2.service.ICommonService;
import com.zhiwei.brandkbs2.service.ICustomTagService;
import com.zhiwei.brandkbs2.service.IMarkDataService;
import com.zhiwei.brandkbs2.service.IProjectService;
import com.zhiwei.brandkbs2.service.*;
import com.zhiwei.brandkbs2.util.AggreeUtil;
import com.zhiwei.brandkbs2.util.Tools;
import com.zhiwei.middleware.mark.pojo.enums.TagField;
......@@ -34,11 +31,10 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.script.Script;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilders;
......@@ -47,11 +43,9 @@ import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggre
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.bucket.histogram.ParsedDateHistogram;
import org.elasticsearch.search.aggregations.bucket.terms.ParsedLongTerms;
import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.FieldSortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
......@@ -89,6 +83,9 @@ public class MarkDataServiceImpl implements IMarkDataService {
@Resource(name = "customTagServiceImpl")
private ICustomTagService customTagService;
@Resource(name = "tagFilterServiceImpl")
ITagFilterService tagFilterService;
@Autowired
private StringRedisTemplate stringRedisTemplate;
......@@ -138,8 +135,10 @@ public class MarkDataServiceImpl implements IMarkDataService {
json.put("name", markerTag.getName());
return json;
}).collect(Collectors.toList()));
//重要发声方
// 重要发声方
result.put("mediaTypeList", ImportantChannelEnum.getAllTagExceptSpec());
// 其他标签组(筛选器)
result.put("tagFilterList", tagFilterService.getTagFilter());
// 自定义标签列表
List<CustomTagVo> customTagList = customTagService.findCustomTagList(1, 1).getList();
if (CollectionUtils.isNotEmpty(customTagList)) {
......@@ -151,12 +150,12 @@ public class MarkDataServiceImpl implements IMarkDataService {
@Override
public JSONObject getAnalyzeSummary(Long startTime, Long endTime, boolean cache) {
try {
String projectId = UserThreadLocal.getProjectId();
String linkedGroupId = projectService.getProjectVOById(projectId).getBrandLinkedGroupId();
if (Objects.isNull(startTime) || Objects.isNull(endTime)) {
endTime = DateUtils.addDays(Tools.truncDate(new Date(), DAY_PATTERN), 1).getTime();
startTime = DateUtils.addMonths(new Date(endTime), -1).getTime();
}
String projectId = UserThreadLocal.getProjectId();
String linkedGroupId = projectService.getProjectVOById(projectId).getBrandLinkedGroupId();
String redisKey = RedisKeyPrefix.MARK_ANALYZE_SUMMARY + Tools.concat(projectId, startTime, endTime);
String resultStr;
// 返回缓存
......@@ -169,35 +168,165 @@ public class MarkDataServiceImpl implements IMarkDataService {
emotionAnalyzeAdd(startTime, endTime, projectId, linkedGroupId, result);
// 获取标注数据传播趋势
List<JSONObject> markSpread = getMarkSpread(startTime, endTime, projectId, true);
markSpread.stream().max(Comparator.comparing(o -> o.getInteger("posNormalCount"))).ifPresent(e -> result.put("posNormalMaxTime", e.getDate("time")));
markSpread.stream().max(Comparator.comparing(o -> o.getInteger("posNormalCount"))).ifPresent(e -> result.put("posMaxTime", e.getDate("time")));
// 获取所有平台稿件倾向稿件数量信息
List<JSONObject> platformsCount = getPlatformMarkEmotionCount(startTime, endTime, projectId, linkedGroupId);
platformsCount.stream().max(Comparator.comparing(o -> o.getInteger(EmotionEnum.POSITIVE.toString()))).ifPresent(e -> result.put("posTopPlatform",
e.getString("platform")));
// String posTopArticleTitle = this.getArticleTopTitle(startTime, endTime, EmotionEnum.POSITIVE.getState(), pid, Constant.PRIMARY_CONTENDID, 1).get(0).getKey();
// if (StringUtils.isNotEmpty(posTopArticleTitle)) {
// String posTopArtTitle = this.selectFirstNormalByAggTitleAndTimeAndProjectIdAndContendId
// (posTopArticleTitle, startTimeStr, endTimeStr, pid, Constant.PRIMARY_CONTENDID).getTitle();
// result.put("posTopArtTitle", posTopArtTitle.length() <= 30 ? posTopArtTitle : posTopArtTitle.substring(0, 30));
// }
// result.put("negNormalMaxTime", articleSpread.stream().max(Comparator.comparing(o -> o.getInteger("negNormalCount"))).get().getDate("time"));
// result.put("negTopPlatform", platformsCount.stream().max(Comparator.comparing(o -> o.getInteger(EmotionEnum.NEGATIVE.toString()))).get().getString("platform"));
// String negTopArticleTitle = this.getArticleTopTitle(startTime, endTime, EmotionEnum.NEGATIVE.getState(), pid, Constant.PRIMARY_CONTENDID, 1).get(0).getKey();
// if (StringUtils.isNotEmpty(negTopArticleTitle)) {
// String negTopArtTitle = this.selectFirstNormalByAggTitleAndTimeAndProjectIdAndContendId
// (negTopArticleTitle, startTimeStr, endTimeStr, pid, Constant.PRIMARY_CONTENDID).getTitle();
// result.put("negTopArtTitle", negTopArtTitle.length() <= 30 ? negTopArtTitle : negTopArtTitle.substring(0, 30));
// }
// List<JSONObject> articlePlatformProportion = this.getArticlePlatformProportion(startTime, endTime, pid, true);
// result.put("platformRank", articlePlatformProportion.stream()
// .filter(articlePlatform -> articlePlatform.getLongValue("count") > 0)
// .sorted((o1, o2) -> o2.getLong("count").compareTo(o1.getLong("count")))
// .collect(Collectors.toList()));
// stringRedisTemplate.opsForValue().set(redisKey, JSON.toJSONString(result), 1, TimeUnit.HOURS);
// return result;
// 正面舆情峰值平台
platformsCount.stream().max(Comparator.comparing(o -> o.getInteger(EmotionEnum.POSITIVE.getName()))).ifPresent(e -> result.put("posTopPlatform",
e.getString("platform_id")));
// 负面舆情峰值平台
platformsCount.stream().max(Comparator.comparing(o -> o.getInteger(EmotionEnum.NEGATIVE.getName()))).ifPresent(e -> result.put("negTopPlatform", e.getString("platform_id")));
// 获取时间段正面情感数据最多的标题
String posTopTitle = getMarkTopTitle(startTime, endTime, EmotionEnum.POSITIVE.getName(), projectId, linkedGroupId, 1).get(0).getKey();
result.put("posTopTitle", posTopTitle);
// 正面舆情峰值时间
markSpread.stream().max(Comparator.comparing(o -> o.getInteger("negNormalCount"))).ifPresent(e -> result.put("negMaxTime", e.getLong("time")));
// 获取时间段负面情感数据最多的标题
String negTopTitle = getMarkTopTitle(startTime, endTime, EmotionEnum.NEGATIVE.getName(), projectId, linkedGroupId, 1).get(0).getKey();
result.put("negTopTitle", negTopTitle);
List<JSONObject> articlePlatformProportion = getMarkPlatformProportion(startTime, endTime, projectId, linkedGroupId, true);
result.put("platformRank", articlePlatformProportion.stream().filter(articlePlatform -> articlePlatform.getLongValue("count") > 0).sorted((o1, o2) -> o2.getLong("count").compareTo(o1.getLong("count"))).collect(Collectors.toList()));
stringRedisTemplate.opsForValue().set(redisKey, JSON.toJSONString(result), 1, TimeUnit.HOURS);
return result;
} catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("es查询异常"), e);
}
return null;
}
@Override
public List<JSONObject> getMarkSpread(Long startTime, Long endTime, boolean cache) {
try {
String projectId = UserThreadLocal.getProjectId();
return getMarkSpread(startTime, endTime, projectId, cache);
} catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("es查询异常"), e);
}
return null;
}
@Override
public JSONObject getMarkEmotionProportion(Long startTime, Long endTime, boolean cache) {
try {
if (Objects.isNull(startTime) || Objects.isNull(endTime)) {
endTime = DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), 1).getTime();
startTime = DateUtils.addMonths(new Date(endTime), -1).getTime();
}
String projectId = UserThreadLocal.getProjectId();
String linkedGroupId = projectService.getProjectVOById(projectId).getBrandLinkedGroupId();
String redisKey = RedisKeyPrefix.MARK_EMOTION_PROPORTION + Tools.concat(projectId, startTime, endTime);
// 返回缓存
String resultStr;
if (cache && (StringUtils.isNotEmpty(resultStr = stringRedisTemplate.opsForValue().get(redisKey)))) {
return JSON.parseObject(resultStr);
}
long lastStartTime = startTime - (endTime - startTime);
List<String> emotionList = Arrays.asList(EmotionEnum.POSITIVE.getName(), EmotionEnum.NEUTRAL.getName(), EmotionEnum.NEGATIVE.getName());
List<JSONObject> curEmotions = getMarkEmotionsCount(startTime, endTime, null, null, projectId, linkedGroupId, emotionList);
List<JSONObject> oldEmotions = getMarkEmotionsCount(lastStartTime, startTime, null, null, projectId, linkedGroupId, emotionList);
JSONObject result = new JSONObject();
result.put("curCount", curEmotions.stream().mapToLong(emotion -> emotion.getLongValue("count")).sum());
result.put("curPosCount", curEmotions.get(0).getLongValue("count"));
result.put("curNeuCount", curEmotions.get(1).getLongValue("count"));
result.put("curNegCount", curEmotions.get(2).getLongValue("count"));
result.put("lastCount", oldEmotions.stream().mapToLong(emotion -> emotion.getLongValue("count")).sum());
result.put("lastPosCount", oldEmotions.get(0).getLongValue("count"));
result.put("lastNeuCount", oldEmotions.get(1).getLongValue("count"));
result.put("lastNegCount", oldEmotions.get(2).getLongValue("count"));
stringRedisTemplate.opsForValue().set(redisKey, JSON.toJSONString(result), 1, TimeUnit.HOURS);
return result;
} catch (IOException e) {
e.printStackTrace();
ExceptionCast.cast(CommonCodeEnum.FAIL.message("es查询异常"), e);
}
return null;
}
@Override
public List<JSONObject> getMarkChannelProportion(Long startTime, Long endTime, boolean cache) {
if (Objects.isNull(startTime) || Objects.isNull(endTime)) {
endTime = DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), 1).getTime();
startTime = DateUtils.addMonths(new Date(endTime), -1).getTime();
}
String projectId = UserThreadLocal.getProjectId();
String linkedGroupId = projectService.getProjectVOById(projectId).getBrandLinkedGroupId();
String redisKey = RedisKeyPrefix.MARK_CHANNEL_PROPORTION + Tools.concat(projectId, startTime, endTime);
String resultStr;
if (cache && StringUtils.isNotEmpty(resultStr = stringRedisTemplate.opsForValue().get(redisKey))) {
return JSON.parseArray(resultStr, JSONObject.class);
}
// emotion范围
List<String> emotionList = Arrays.asList(EmotionEnum.POSITIVE.getName(), EmotionEnum.NEUTRAL.getName(), EmotionEnum.NEGATIVE.getName());
List<JSONObject> channelEmotions = getChannelEmotionsCount(startTime, endTime, null, null, projectId, linkedGroupId, emotionList);
long articlesCount = channelEmotions.stream().mapToLong(channelEmotion -> channelEmotion.getLongValue("count")).sum();
for (JSONObject result : channelEmotions) {
int channelEmotion = result.getIntValue("channelEmotion");
long channelEmotionCount = result.getLongValue("count");
List<String> mediaTypeList = Arrays.asList(
ImportantChannelEnum.YANGJI.getState(),
ImportantChannelEnum.KEJI.getState(),
ImportantChannelEnum.CAIJING.getState(),
ImportantChannelEnum.QITA.getState());
List<JSONObject> mediaTypeResultList = new ArrayList<>(mediaTypeList.size());
for (String mediaType : mediaTypeList) {
BoolQueryBuilder boolQueryBuilder = projectLinkedGroupQuery(projectId, linkedGroupId);
boolQueryBuilder.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime))
// TODO channel_emotion
.must(QueryBuilders.termQuery("channel_emotion", channelEmotion));
// TODO 其他类别标签
if (ImportantChannelEnum.QITA.getState().equals(mediaType)) {
boolQueryBuilder.mustNot(QueryBuilders.matchQuery("tag", ImportantChannelEnum.YANGJI.getState()))
.mustNot(QueryBuilders.matchQuery("tag", ImportantChannelEnum.KEJI.getState()))
.mustNot(QueryBuilders.matchQuery("tag", ImportantChannelEnum.CAIJING.getState()));
} else {
boolQueryBuilder.must(QueryBuilders.matchQuery("tag", mediaType));
}
result.put("mediaTypePro", mediaTypeResultList);
result.put("proportion", articlesCount == 0 ? 0d : channelEmotionCount * 1.0 / articlesCount);
}
stringRedisTemplate.opsForValue().set(redisKey, JSON.toJSONString(channelEmotions), 1, TimeUnit.HOURS);
}
return channelEmotions;
}
@Override
public List<JSONObject> getMarkPlatformProportion(Long startTime, Long endTime, boolean cache) {
List<JSONObject> res = null;
try {
if (Objects.isNull(startTime) || Objects.isNull(endTime)) {
endTime = DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), 1).getTime();
startTime = DateUtils.addMonths(new Date(endTime), -1).getTime();
}
String projectId = UserThreadLocal.getProjectId();
String linkedGroupId = projectService.getProjectVOById(projectId).getBrandLinkedGroupId();
res = getMarkPlatformProportion(startTime, endTime, projectId, linkedGroupId, cache);
} catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("es查询异常"), e);
}
return res;
}
@Override
public List<JSONObject> getMarkHighWord(Long startTime, Long endTime) {
try {
if (Objects.isNull(startTime) || Objects.isNull(endTime)) {
endTime = DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), 1).getTime();
startTime = DateUtils.addMonths(new Date(endTime), -1).getTime();
}
String projectId = UserThreadLocal.getProjectId();
String linkedGroupId = projectService.getProjectVOById(projectId).getBrandLinkedGroupId();
String redisKey = RedisKeyPrefix.MARK_HIGH_WORD + Tools.concat(projectId, startTime, endTime);
String resultStr = stringRedisTemplate.opsForValue().get(redisKey);
if (StringUtils.isNotEmpty(resultStr)) {
return JSON.parseArray(resultStr, JSONObject.class);
}
List<String> text = getMarkTitleAndContent(startTime, endTime, projectId,linkedGroupId);
// 分析热评词
// AnsjSeg ansjSeg = new AnsjSeg();
// List<Map.Entry<String, Integer>> words = (List<Map.Entry<String, Integer>>) ansjSeg.getFenCi(text, 30).get("word");
// List<JSONObject> hotwords = Tools.getHotwords(words);
// stringRedisTemplate.opsForValue().set(redisKey, JSON.toJSONString(hotwords), 1, TimeUnit.DAYS);
} catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("es查询异常"), e);
}
return null;
}
......@@ -224,9 +353,7 @@ public class MarkDataServiceImpl implements IMarkDataService {
// PostFilter 后置过滤器
BoolQueryBuilder postFilter = QueryBuilders.boolQuery();
// linkedGroupId projectId
postFilter.must(QueryBuilders.termQuery("brandkbs_cache_maps.linkedGroupId.keyword", dto.getLinkedGroupId())).must(
QueryBuilders.termQuery("brandkbs_cache_maps.projectId.keyword", UserThreadLocal.getProjectId())
);
postFilter.must(QueryBuilders.termQuery("brandkbs_cache_maps.linkedGroupId.keyword", dto.getLinkedGroupId())).must(QueryBuilders.termQuery("brandkbs_cache_maps.projectId.keyword", UserThreadLocal.getProjectId()));
// time
postFilter.must(QueryBuilders.rangeQuery(dto.getTimeType()).lte(dto.getStartTime()).lt(dto.getEndTime()));
// platform
......@@ -247,8 +374,7 @@ public class MarkDataServiceImpl implements IMarkDataService {
// keyword
if (StringUtils.isNotEmpty(dto.getKeyword())) {
// Query
String[] fieldSearch = "标题".equals(dto.getSearchField()) ? new String[]{GenericAttribute.ES_IND_TITLE}
: new String[]{GenericAttribute.ES_IND_FULL_TEXT};
String[] fieldSearch = "标题".equals(dto.getSearchField()) ? new String[]{GenericAttribute.ES_IND_TITLE} : new String[]{GenericAttribute.ES_IND_FULL_TEXT};
query.must(EsQueryTools.assembleNormalKeywordQuery(dto.getKeyword(), fieldSearch));
}
// sourceKeyword
......@@ -256,18 +382,23 @@ public class MarkDataServiceImpl implements IMarkDataService {
query.must(QueryBuilders.matchQuery(GenericAttribute.ES_SOURCE, dto.getSourceKeyword()));
}
// sort
FieldSortBuilder sort = SortBuilders.fieldSort(GenericAttribute.ES_GID).order(SortOrder.DESC);
FieldSortBuilder sort = null;
for (Map.Entry<String, Object> entry : JSONObject.parseObject(dto.getSorter()).entrySet()) {
if (entry.getValue().toString().contains("desc")) {
sort = SortBuilders.fieldSort(entry.getKey()).order(SortOrder.DESC);
} else {
sort = SortBuilders.fieldSort(entry.getKey()).order(SortOrder.ASC);
}
}
// from size
int from = (dto.getPage() - 1) * dto.getPageSize();
// HighlightBuilder ???
SearchHits searchHits = esClientDao.searchHits(indexes, postFilter, query, sort, from, dto.getPageSize(), null);
SearchHits searchHits = esClientDao.searchHits(indexes, postFilter, query, null, sort, from, dto.getPageSize(), null);
//平台计量统计 仅第一页且无平台限制
Map<String, Long> counts = new HashMap<>();
if (1 == dto.getPage() && CollectionUtils.isEmpty(dto.getPlatforms())) {
for (MessagePlatform platform : GlobalPojo.PLATFORMS) {
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery()
.must(query)
.must(EsQueryTools.assemblePlatformQuery(Collections.singletonList(platform)));
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery().must(query).must(EsQueryTools.assemblePlatformQuery(Collections.singletonList(platform)));
Long count = esClientDao.count(indexes, postFilter, queryBuilder);
counts.put(platform.getName(), count);
}
......@@ -276,10 +407,9 @@ public class MarkDataServiceImpl implements IMarkDataService {
}
private List<List<JSONObject>> aggregation(SearchHits searchHits, String uniqueField) {
List<JSONObject> jsonList =
Arrays.stream(searchHits.getHits()).map(hit -> new JSONObject(hit.getSourceAsMap())).collect(Collectors.toList());
List<JSONObject> jsonList = Arrays.stream(searchHits.getHits()).map(hit -> new JSONObject(hit.getSourceAsMap())).collect(Collectors.toList());
Map<String, String> idTitle = jsonList.stream().collect(Collectors.toMap(json -> json.getString(uniqueField),
json -> Tools.getBaseFromEsMap(json).getTitle()));
json -> Tools.getBaseFromEsMap(json).getTitleNullOptionalContent()));
List<List<String>> kResult = AggreeUtil.getKResult(idTitle);
return AggreeUtil.sortByTimeAsc(kResult, jsonList, uniqueField);
}
......@@ -368,20 +498,19 @@ public class MarkDataServiceImpl implements IMarkDataService {
BoolQueryBuilder query = QueryBuilders.boolQuery();
// keyword
if (StringUtils.isNotEmpty(keyword)) {
String[] fieldSearch = "标题".equals(searchField) ? new String[]{GenericAttribute.ES_IND_TITLE}
: new String[]{GenericAttribute.ES_IND_FULL_TEXT};
String[] fieldSearch = "标题".equals(searchField) ? new String[]{GenericAttribute.ES_IND_TITLE} : new String[]{GenericAttribute.ES_IND_FULL_TEXT};
query.must(EsQueryTools.assembleNormalKeywordQuery(keyword, fieldSearch));
}
// postFilter
BoolQueryBuilder postFilter = QueryBuilders.boolQuery();
postFilter.must(QueryBuilders.termQuery("brandkbs_cache_maps.linkedGroupId.keyword", linkedGroupId))
.must(QueryBuilders.termQuery("brandkbs_cache_maps.projectId.keyword", projectId));
postFilter.must(QueryBuilders.termQuery("brandkbs_cache_maps.linkedGroupId.keyword", linkedGroupId)).must(QueryBuilders.termQuery("brandkbs_cache_maps.projectId.keyword", projectId));
// time range
postFilter.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
List<JSONObject> resultList = new ArrayList<>(emotionList.size());
Map<String, Aggregation> aggMap = esClientDao.searchResponse(indexes, postFilter, query, aggregationBuilder, null, 0, 0, null).getAggregations().asMap();
ParsedLongTerms teamAgg = (ParsedLongTerms) aggMap.get("count");
Map<String, Aggregation> aggMap =
esClientDao.searchResponse(indexes, postFilter, query, aggregationBuilder, null, null, 0, 0, null).getAggregations().asMap();
ParsedStringTerms teamAgg = (ParsedStringTerms) aggMap.get("count");
List<? extends Terms.Bucket> buckets = teamAgg.getBuckets();
for (String emotion : emotionList) {
List<? extends Terms.Bucket> res = buckets.stream().filter(bucket -> bucket.getKeyAsString().equals(emotion)).collect(Collectors.toList());
......@@ -400,7 +529,7 @@ public class MarkDataServiceImpl implements IMarkDataService {
endTime = DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), 1).getTime();
startTime = DateUtils.addMonths(new Date(endTime), -1).getTime();
}
String redisKey = RedisKeyPrefix.ARTICLE_SPREAD + Tools.concat(projectId, startTime, endTime);
String redisKey = RedisKeyPrefix.MARK_SPREAD + Tools.concat(projectId, startTime, endTime);
String resultStr;
if (cache && StringUtils.isNotEmpty(resultStr = stringRedisTemplate.opsForValue().get(redisKey))) {
return JSON.parseArray(resultStr, JSONObject.class);
......@@ -411,9 +540,9 @@ public class MarkDataServiceImpl implements IMarkDataService {
result.forEach(day -> {
JSONObject line = new JSONObject();
line.put("normalCount", day.getIntValue("total"));
line.put("posNormalCount", day.getIntValue(EmotionEnum.POSITIVE.toString()));
line.put("neuNormalCount", day.getIntValue(EmotionEnum.NEUTRAL.toString()));
line.put("negNormalCount", day.getIntValue(EmotionEnum.NEGATIVE.toString()));
line.put("posNormalCount", day.getIntValue(EmotionEnum.POSITIVE.getName()));
line.put("neuNormalCount", day.getIntValue(EmotionEnum.NEUTRAL.getName()));
line.put("negNormalCount", day.getIntValue(EmotionEnum.NEGATIVE.getName()));
line.put("time", day.getLongValue("date"));
lineList.add(line);
});
......@@ -435,22 +564,20 @@ public class MarkDataServiceImpl implements IMarkDataService {
public List<Map.Entry<String, Integer>> getMarkTopTitle(Long startTime, Long endTime, String emotion, String projectId, String linkedGroupId, int size) throws IOException {
// 索引
String[] indexes = esClientDao.getIndexes();
//搜索源构建对象
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
// TODO script使用
Script script = new Script("params._source.ind_full_title");
Script script = new Script("params._source.ind_title");
TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("titles").script(script).order(BucketOrder.count(false)).size(60000);
// postFilter
BoolQueryBuilder postFilter = projectLinkedGroupQuery(projectId, linkedGroupId);
postFilter.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime))
// 过滤微博
.mustNot(QueryBuilders.termQuery("platform_id", "5d02236e6395002a7c380b79"));
if (emotion != null) {
.mustNot(QueryBuilders.termQuery("platform_id.keyword", "5d02236e6395002a7c380b79"));
if (null != emotion) {
postFilter.must(QueryBuilders.termQuery("mark_cache_maps.name.keyword", emotion));
}
Map<String, Integer> result = new HashMap<>();
SearchResponse searchResponse = esClientDao.searchResponse(indexes, postFilter, null, aggregationBuilder, null, 0, 0, null);
SearchResponse searchResponse = esClientDao.searchResponse(indexes, postFilter, null, aggregationBuilder, null, null, 0, 0, null);
Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
ParsedStringTerms teamAgg = (ParsedStringTerms) aggMap.get("titles");
List<? extends Terms.Bucket> buckets = teamAgg.getBuckets();
......@@ -461,8 +588,7 @@ public class MarkDataServiceImpl implements IMarkDataService {
result.merge(aggTitle, num, Integer::sum);
});
// 去掉 “分享一篇文章” 的标题
return result.entrySet().stream().filter(data -> !"分享一篇文章".equals(data.getKey())).sorted(Map.Entry.comparingByValue(Comparator.reverseOrder()))
.limit(size).collect(Collectors.toList());
return result.entrySet().stream().filter(data -> !"分享一篇文章".equals(data.getKey())).sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())).limit(size).collect(Collectors.toList());
}
/**
......@@ -479,30 +605,26 @@ public class MarkDataServiceImpl implements IMarkDataService {
// 索引
String[] indexes = esClientDao.getIndexes();
// ??? TODO 为什么是16h offset
DateHistogramAggregationBuilder daysAggregationBuilder = AggregationBuilders.dateHistogram("dayAgg").field("time")
.calendarInterval(DateHistogramInterval.DAY);
TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("count").field("mark_cache_maps.name.keyword");
DateHistogramAggregationBuilder daysAggregationBuilder = AggregationBuilders.dateHistogram("dayAgg").field("time").calendarInterval(DateHistogramInterval.DAY);
TermsAggregationBuilder emotionAggregationBuilder = AggregationBuilders.terms("count").field("mark_cache_maps.name.keyword");
// postFilter
BoolQueryBuilder postFilter = QueryBuilders.boolQuery();
postFilter.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime))
.must(QueryBuilders.termQuery("brandkbs_cache_maps.projectId.keyword", projectId))
.must(QueryBuilders.termQuery("brandkbs_cache_maps.linkedGroupId.keyword", linkedGroupId));
postFilter.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime)).must(QueryBuilders.termQuery("brandkbs_cache_maps.projectId.keyword", projectId)).must(QueryBuilders.termQuery("brandkbs_cache_maps.linkedGroupId.keyword", linkedGroupId));
// 情感倾向限制
postFilter.must(QueryBuilders.termQuery("mark_cache_maps.group_name.keyword", "情感倾向"));
// response
SearchResponse searchResponse = esClientDao.searchResponse(indexes, postFilter, null, aggregationBuilder, null, 0, 0, null);
SearchResponse searchResponse = esClientDao.searchResponse(indexes, postFilter, null,
daysAggregationBuilder.subAggregation(emotionAggregationBuilder), null, null, 0, 0, null);
Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
ParsedDateHistogram dayAggTeam = (ParsedDateHistogram) aggMap.get("dayAgg");
List<? extends Histogram.Bucket> buckets = dayAggTeam.getBuckets();
buckets.forEach(bucket -> {
JSONObject result = new JSONObject();
Map<String, Aggregation> map = bucket.getAggregations().asMap();
ParsedLongTerms countTeam = (ParsedLongTerms) map.get("count");
ParsedStringTerms countTeam = (ParsedStringTerms) map.get("count");
List<? extends Terms.Bucket> list = countTeam.getBuckets();
result.put("date", bucket.getKeyAsString());
result.put("total", bucket.getDocCount());
result.put(EmotionEnum.POSITIVE.getName(), 0);
result.put(EmotionEnum.NEUTRAL.getName(), 0);
result.put(EmotionEnum.NEGATIVE.getName(), 0);
emotionCount(result, list);
emotionCount(result, bucket.getDocCount(), list);
ResultList.add(result);
});
return ResultList;
......@@ -522,37 +644,35 @@ public class MarkDataServiceImpl implements IMarkDataService {
// 索引
String[] indexes = esClientDao.getIndexes();
// 聚合请求
TermsAggregationBuilder platformAggregationBuilder = AggregationBuilders.terms("platform_count").field("platform_id").order(BucketOrder.count(false));
TermsAggregationBuilder platformAggregationBuilder = AggregationBuilders.terms("platform_count").field("platform_id.keyword").order(BucketOrder.count(false));
TermsAggregationBuilder emotionAggregationBuilder = AggregationBuilders.terms("emotion_count").field("mark_cache_maps.name.keyword");
// postFilter
BoolQueryBuilder postFilter = QueryBuilders.boolQuery();
postFilter.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime))
.must(QueryBuilders.termQuery("brandkbs_cache_maps.projectId.keyword", projectId))
.must(QueryBuilders.termQuery("brandkbs_cache_maps.linkedGroupId.keyword", linkedGroupId));
SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, postFilter,
platformAggregationBuilder.subAggregation(emotionAggregationBuilder), null, 0, 0, null);
BoolQueryBuilder postFilter = projectLinkedGroupQuery(projectId, linkedGroupId);
// time
postFilter.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
// 情感倾向限制
postFilter.must(QueryBuilders.termQuery("mark_cache_maps.group_name.keyword", "情感倾向"));
SearchResponse searchResponse = esClientDao.searchResponse(indexes, postFilter, null,
platformAggregationBuilder.subAggregation(emotionAggregationBuilder), null, null, 0, 0, null);
Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
ParsedStringTerms teamAgg = (ParsedStringTerms) aggMap.get("platform_count");
List<? extends Terms.Bucket> buckets = teamAgg.getBuckets();
buckets.forEach(bucket -> {
JSONObject result = new JSONObject();
Map<String, Aggregation> map = bucket.getAggregations().asMap();
ParsedLongTerms countTeam = (ParsedLongTerms) map.get("emotion_count");
ParsedStringTerms countTeam = (ParsedStringTerms) map.get("emotion_count");
List<? extends Terms.Bucket> list = countTeam.getBuckets();
result.put("platform", bucket.getKeyAsString());
result.put("total", bucket.getDocCount());
result.put(EmotionEnum.POSITIVE.toString(), 0);
result.put(EmotionEnum.NEUTRAL.toString(), 0);
result.put(EmotionEnum.NEGATIVE.toString(), 0);
emotionCount(result, list);
result.put("platform_id", bucket.getKeyAsString());
emotionCount(result, bucket.getDocCount(), list);
ResultList.add(result);
});
return ResultList;
}
private void emotionCount(JSONObject result, List<? extends Terms.Bucket> list) {
private void emotionCount(JSONObject result, long total, List<? extends Terms.Bucket> list) {
result.put(EmotionEnum.POSITIVE.getName(), 0);
result.put(EmotionEnum.NEUTRAL.getName(), 0);
result.put(EmotionEnum.NEGATIVE.getName(), 0);
list.forEach(data -> {
String emotion = data.getKeyAsString();
if (EmotionEnum.POSITIVE.getName().equals(emotion)) {
......@@ -565,9 +685,102 @@ public class MarkDataServiceImpl implements IMarkDataService {
});
}
private List<JSONObject> getMarkPlatformProportion(Long startTime, Long endTime, String projectId, String linkedGroupId, boolean cache) throws IOException {
String redisKey = RedisKeyPrefix.MARK_PLATFORM_PROPORTION + Tools.concat(projectId, startTime, endTime);
String resultStr;
if (cache && StringUtils.isNotEmpty(resultStr = stringRedisTemplate.opsForValue().get(redisKey))) {
return JSON.parseArray(resultStr, JSONObject.class);
}
List<JSONObject> platformList = commonService.getQbjcPlatform("id", "name");
List<JSONObject> platformsCount = getPlatformsCount(startTime, endTime, null, null, projectId, linkedGroupId, platformList);
long articlesCount = platformsCount.stream().mapToLong(platform -> platform.getLongValue("count")).sum();
List<JSONObject> resultList = platformsCount.stream().peek(platform -> platform.put("proportion", platform.getLongValue("count") * 1.0 / articlesCount)).collect(Collectors.toList());
stringRedisTemplate.opsForValue().set(redisKey, JSON.toJSONString(resultList), 1, TimeUnit.HOURS);
return resultList;
}
/**
* 获取所有平台稿件数量信息
*
* @param startTime 开始时间
* @param endTime 结束时间
* @param keyword 关键词
* @param searchField 检索字段
* @param projectId 项目ID
* @param platformList 平台集合
* @return 所有平台稿件数量信息
*/
private List<JSONObject> getPlatformsCount(Long startTime, Long endTime, String keyword, String searchField, String projectId, String linedGroupId, List<JSONObject> platformList) throws IOException {
// 获取索引
String[] indexes = esClientDao.getIndexes();
// 聚合请求
TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("count").field("platform_id.keyword").order(BucketOrder.count(false)).size(100);
// query
BoolQueryBuilder query = QueryBuilders.boolQuery();
// keyword
if (StringUtils.isNotEmpty(keyword)) {
String[] fieldSearch = "标题".equals(searchField) ? new String[]{GenericAttribute.ES_IND_TITLE} : new String[]{GenericAttribute.ES_IND_FULL_TEXT};
query.must(EsQueryTools.assembleNormalKeywordQuery(keyword, fieldSearch));
}
// postFilter
BoolQueryBuilder postFilter = projectLinkedGroupQuery(projectId, linedGroupId);
// time
postFilter.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
List<JSONObject> platformResultList = new ArrayList<>(platformList.size());
SearchResponse searchResponse = esClientDao.searchResponse(indexes, postFilter, query, aggregationBuilder, null, null, 0, 0, null);
Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
ParsedStringTerms teamAgg = (ParsedStringTerms) aggMap.get("count");
List<? extends Terms.Bucket> buckets = teamAgg.getBuckets();
for (JSONObject platform : platformList) {
String platformId = platform.getString("id");
String platformName = platform.getString("name");
List<? extends Terms.Bucket> resultList = buckets.stream().filter(bucket -> platformId.equals(bucket.getKeyAsString())).collect(Collectors.toList());
JSONObject platformResult = new JSONObject();
platformResult.put("platform", platformName);
platformResult.put("count", CollectionUtils.isEmpty(resultList) ? 0 : resultList.get(0).getDocCount());
platformResultList.add(platformResult);
}
return platformResultList;
}
/**
* 获取所有渠道倾向稿件数量信息
* TODO 未完成
*
* @return 所有稿件倾向稿件数量信息
*/
private List<JSONObject> getChannelEmotionsCount(Long startTime, Long endTime, String keyword, String searchField, String projectId, String linkedGroupId,
List<String> emotionList) {
return Collections.emptyList();
}
private List<String> getMarkTitleAndContent(Long startTime, Long endTime, String projectId, String linkedGroupId) throws IOException {
EsClientDao.SearchHelper searchHelper = EsClientDao.createSearchHelper();
// sort
searchHelper.setSort(SortBuilders.fieldSort("id").order(SortOrder.DESC));
// fetchSource
searchHelper.setFetchSource(new String[]{"ind_title", "ind_full_text"});
// postFilter
BoolQueryBuilder postFilter = projectLinkedGroupQuery(projectId, linkedGroupId);
postFilter.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
searchHelper.setPostFilter(postFilter);
List<String> textList = new ArrayList<>();
List<SearchResponse> searchResponses = esClientDao.searchScrollResponse(searchHelper);
for (SearchResponse searchResponse : searchResponses) {
for (SearchHit searchHit : searchResponse.getHits().getHits()) {
Map<String, Object> source = searchHit.getSourceAsMap();
BaseMap baseMap = Tools.getBaseFromEsMap(source);
String title = baseMap.getTitle();
String content = baseMap.getContent();
textList.add(title + content);
}
}
return textList;
}
private BoolQueryBuilder projectLinkedGroupQuery(String projectId, String linkedGroupId) {
return QueryBuilders.boolQuery().must(QueryBuilders.termQuery("brandkbs_cache_maps.projectId.keyword", projectId))
.must(QueryBuilders.termQuery("brandkbs_cache_maps.linkedGroupId.keyword", linkedGroupId));
return QueryBuilders.boolQuery().must(QueryBuilders.termQuery("brandkbs_cache_maps.projectId.keyword", projectId)).must(QueryBuilders.termQuery("brandkbs_cache_maps.linkedGroupId.keyword", linkedGroupId));
}
}
\ No newline at end of file
src/main/java/com/zhiwei/brandkbs2/util/Tools.java
......@@ -323,7 +323,7 @@ public class Tools {
case INCOMPLETE:
IncompleteText incompleteText = IncompleteText.restoreFromEs(map);
// 短文本使用content做title展示
res.setTitle(incompleteText.getContent());
res.setIncompleteContent(incompleteText.getContent());
res.setContent(incompleteText.getContent());
break;
case QA:
......
src/main/resources/application-dev.properties
......@@ -6,11 +6,11 @@ jwt.hour=120
brandkbs.file.url=/usr/local/sources/brandkbs2/files/
brandkbs.img.url=/usr/local/sources/brandkbs2/images/
brandkbs.head.url=/usr/local/sources/brandkbs2/head/
brandkbs.image.url=https://brandkbs2.test.zhiweidata.com/brandkbs2/images/
brandkbs.image.url=https://brandkbs2.test.zhiweidata.com/brandkbs/images/
#redis
spring.redis.host=192.168.0.39
spring.redis.database=7
spring.redis.database=12
spring.redis.port=7373
#mongo\u914D\u7F6E
......@@ -22,8 +22,8 @@ mongo.autoConnectRetry=true
mongo.socketKeepAlive=true
mongo.socketTimeout=120000
mongo.slaveOk=true
primary.uri=mongodb://brandkbs2:3vh65l$i6qQA@192.168.0.150:27017,192.168.0.151:27017,192.168.0.152:27017/qbjc?authSource=admin
secondary.uri=mongodb://qbjcuser:qbjc1q2w3e4r@192.168.0.150:27017,192.168.0.151:27017,192.168.0.152:27017/qbjc?authSource=admin
primary.uri=mongodb://brandkbs2:3vh65l$i6qQA@192.168.0.150:27017,192.168.0.151:27017,192.168.0.152:27017/brandkbs2?authSource=admin
secondary.uri=mongodb://qbjcuser:asSADf5ffs@192.168.0.150:27017,192.168.0.151:27017,192.168.0.152:27017/qbjc?authSource=admin
#es
es.clusterNodes=192.168.0.130:9200
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment