Skip to content

B
brandkbs2
Commit d3b75160 by 陈健智
Options

2022/8/16 14:06 竞品库,查热点,查危机,全网事件库-查事件

parent 790d1431
Showing with 635 additions and 51 deletions
src/main/java/com/zhiwei/brandkbs2/common/RedisKeyPrefix.java
......@@ -60,10 +60,24 @@ public class RedisKeyPrefix {
*/
private static final String EVENT_ANALYZE_PROGRESS = "BRANDKBS:EVENT:ANALYZE:PROGRESS:";
/**
* 竞品库-获取竞品对比舆情解读数据（PC
*/
public static final String INDEX_COMPARE_SUMMARY_PC = "BRANDKBS:CONTEND:COMPARESUMMARY:PC:";
/**
* 竞品库-获取竞品对比传播分析数据
*/
public static final String INDEX_COMPARE_ANALYZE = "BRANDKBS:CONTEND:COMPAREANALYZE:";
public static String eventAnalysisProgress(String eventId, String projectId) {
return RedisKeyPrefix.generateRedisKey(RedisKeyPrefix.EVENT_ANALYZE_PROGRESS, projectId, eventId);
}
public static String eventAnalysisProgress(String eventId) {
return RedisKeyPrefix.generateRedisKey(RedisKeyPrefix.EVENT_ANALYZE_PROGRESS, UserThreadLocal.getProjectId(), eventId);
}
public static String yuqingProgressKey(String linkedGroupId) {
return yuqingProgressKey(UserThreadLocal.getProjectId(), linkedGroupId);
}
......
src/main/java/com/zhiwei/brandkbs2/config/Constant.java
......@@ -51,4 +51,9 @@ public class Constant {
return -1L;
}
/**
* 主品牌默认竞品ID
*/
public static final String PRIMARY_CONTENDID = "0";
}
src/main/java/com/zhiwei/brandkbs2/controller/app/AppContendController.java 0 → 100644
package com.zhiwei.brandkbs2.controller.app;
import com.zhiwei.brandkbs2.auth.Auth;
import com.zhiwei.brandkbs2.controller.BaseController;
import com.zhiwei.brandkbs2.enmus.RoleEnum;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
import com.zhiwei.brandkbs2.service.MarkDataService;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiImplicitParams;
import io.swagger.annotations.ApiOperation;
import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource;
import java.io.IOException;
import java.text.ParseException;
import java.util.List;
/**
* @ClassName AppContendController
* @Description 提供前台竞品库相关信息展示
* @author cjz
* @date 2022-08-09 10:22
*/
@RestController
@RequestMapping("/app/contend")
@Api(tags = "前台竞品库展示接口",description = "提供前台竞品库相关信息展示")
@Auth(role = RoleEnum.CUSTOMER)
public class AppContendController extends BaseController {
@Resource(name = "markDataServiceImpl")
MarkDataService markDataService;
@ApiOperation("竞品库-竞品舆情-搜索条件")
@GetMapping("/searchCriteria")
public ResponseResult getContendSearchCriteria(@RequestParam(required = false) String linkedGroupId) {
return ResponseResult.success(markDataService.getContendSearchCriteria(linkedGroupId));
}
@ApiOperation("竞品库-竞品舆情-搜索结果列表")
@GetMapping("/list")
public ResponseResult getContendSearchList(@RequestBody MarkSearchDTO markSearchDTO) throws IOException {
return ResponseResult.success(markDataService.getContendSearchList(markSearchDTO));
}
@ApiOperation("竞品库-竞品分析-舆情总结页面")
@GetMapping("/summary")
public ResponseResult getSummary(@RequestBody MarkSearchDTO markSearchDTO) throws IOException {
return ResponseResult.success(markDataService.getContendSummary(markSearchDTO, true));
}
@ApiOperation("竞品库-竞品对比-传播分析页面")
@GetMapping("/spread/analyze")
public ResponseResult getSpreadAnalyze(@RequestBody MarkSearchDTO markSearchDTO) throws IOException {
return ResponseResult.success(markDataService.getContendSpreadAnalyze(markSearchDTO, 5, true));
}
}
src/main/java/com/zhiwei/brandkbs2/controller/app/AppCrisisController.java
......@@ -26,8 +26,6 @@ import org.springframework.web.client.RestTemplate;
@Api(tags = "前台危机展示接口",description = "提供前台危机相关信息展示")
@Auth(role = RoleEnum.CUSTOMER)
public class AppCrisisController extends BaseController {
@Value("${crisis.search.url}")
private String crisisSearchUrl;
@Value("${crisis.searchTags.url}")
private String crisisTagsUrl;
......@@ -47,16 +45,6 @@ public class AppCrisisController extends BaseController {
@Autowired
private RestTemplate restTemplate;
@ApiOperation("危机库-查危机")
@GetMapping("/crisisSearch")
public ResponseResult crisisSearch(@RequestParam(value = "page",defaultValue = "1") Integer page,
@RequestParam(value = "pageSize",defaultValue = "3") Integer pageSize,
@RequestParam("keyword") String keyword){
ResponseEntity<String> responseEntity = restTemplate.getForEntity(crisisSearchUrl, String.class, page, pageSize, keyword);
Object data = JSON.parseObject(responseEntity.getBody()).get("data");
return ResponseResult.success(data);
}
@ApiOperation("危机库-通过标签查危机")
@GetMapping("/crisisTags")
public ResponseResult crisisTags(@RequestParam(value = "page",defaultValue = "1") Integer page,
......
src/main/java/com/zhiwei/brandkbs2/controller/app/AppHotController.java
......@@ -5,6 +5,7 @@ import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.auth.Auth;
import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.common.RedisKeyPrefix;
import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.controller.BaseController;
import com.zhiwei.brandkbs2.enmus.RoleEnum;
import com.zhiwei.brandkbs2.model.ResponseResult;
......@@ -48,9 +49,6 @@ public class AppHotController extends BaseController {
@Value("${trends.longTimeInListSearchByInner.url}")
private String trendsListUrl;
@Value("${trends.findHotSearchESDataInTimeByInner.url}")
private String trendsSearchUrl;
@Value("${trends.longTimeInListSearch.url}")
private String longTimeInListSearchUrl;
......@@ -74,7 +72,7 @@ public class AppHotController extends BaseController {
* douyin 抖音
* bilibili-ranking B站
*/
@ApiOperation("热点库")
@ApiOperation("热点库-列表")
@GetMapping("/getHotList")
public ResponseResult getHotList(@RequestParam(value = "sortType",defaultValue = "realTime") String sortType,
@RequestParam(value = "type",defaultValue = "weibo") String type){
......@@ -96,21 +94,6 @@ public class AppHotController extends BaseController {
}
}
@ApiOperation("热点库-查热点")
@GetMapping("/searchHotList")
public ResponseResult searchHotList(@RequestParam(value = "pageSize",defaultValue ="10") Integer limit,
@RequestParam(value = "page",defaultValue ="1") Integer page,
@RequestParam(value = "type",defaultValue = "weibo") String type,
@RequestParam(value = "word") String word){
ResponseEntity<JSONObject> jsonObjectResponseEntity = restTemplate.getForEntity(trendsSearchUrl, JSONObject.class, limit, page, type, word);
JSONObject body = jsonObjectResponseEntity.getBody();
if(Objects.nonNull(body)){
return ResponseResult.success(body);
}else{
return ResponseResult.failure("响应超时");
}
}
@ApiOperation("热点库-热点榜单")
@GetMapping("/hot")
public ResponseResult hot(){
......@@ -142,14 +125,14 @@ public class AppHotController extends BaseController {
Date startDate = DateUtils.addHours(endDate, -24);
List<Map.Entry<String, Integer>> markTopTitleList =
markDataService.getMarkTopTitle(startDate.getTime(), endDate.getTime(), null, projectId, linkedGroupId,"0", size);
markDataService.getMarkTopTitle(startDate.getTime(), endDate.getTime(), null, projectId, linkedGroupId, Constant.PRIMARY_CONTENDID, size);
List<JSONObject> resultList = markTopTitleList.stream().map(map -> {
JSONObject resultJsonObject = new JSONObject();
resultJsonObject.put("title", map.getKey());
resultJsonObject.put("num", map.getValue());
try {
BaseMap firstArticle = markDataService.getFirstArticle(startDate.getTime(), endDate.getTime(), map.getKey(), projectId, linkedGroupId);
BaseMap firstArticle = markDataService.getFirstArticle(startDate.getTime(), endDate.getTime(), map.getKey(), projectId, linkedGroupId, Constant.PRIMARY_CONTENDID);
resultJsonObject.put("content", firstArticle.getContent());
resultJsonObject.put("url", firstArticle.getUrl());
resultJsonObject.put("realSource", firstArticle.getRealSource());
......
src/main/java/com/zhiwei/brandkbs2/controller/app/AppSearchController.java 0 → 100644
package com.zhiwei.brandkbs2.controller.app;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.auth.Auth;
import com.zhiwei.brandkbs2.enmus.RoleEnum;
import com.zhiwei.brandkbs2.model.ResponseResult;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.client.RestTemplate;
import java.util.Objects;
/**
* @ClassName AppSearchController
* @Description 提供前台搜索相关功能
* @author cjz
* @date 2022-08-15 16:17
*/
@RestController
@RequestMapping("/app/search")
@Api(tags = "前台搜索相关接口",description = "提供搜索相关功能")
@Auth(role = RoleEnum.CUSTOMER)
public class AppSearchController {
@Autowired
private RestTemplate restTemplate;
@Value("${trends.findHotSearchESDataInTimeByInner.url}")
private String trendsSearchUrl;
@Value("${crisis.search.url}")
private String crisisSearchUrl;
@Value("${ef.search.url}")
private String getEfSearchUrl;
@ApiOperation("搜索-查热点")
@GetMapping("/hot/list")
public ResponseResult searchHotList(@RequestParam(value = "pageSize",defaultValue ="10") Integer limit,
@RequestParam(value = "page",defaultValue ="1") Integer page,
@RequestParam(value = "type",defaultValue = "weibo") String type,
@RequestParam(value = "word") String word){
ResponseEntity<JSONObject> jsonObjectResponseEntity = restTemplate.getForEntity(trendsSearchUrl, JSONObject.class, limit, page, type, word);
JSONObject body = jsonObjectResponseEntity.getBody();
if(Objects.nonNull(body)){
return ResponseResult.success(body);
}else{
return ResponseResult.failure("响应超时");
}
}
@ApiOperation("搜索-查危机")
@GetMapping("/crisisSearch")
public ResponseResult crisisSearch(@RequestParam(value = "page",defaultValue = "1") Integer page,
@RequestParam(value = "pageSize",defaultValue = "3") Integer pageSize,
@RequestParam("keyword") String keyword){
ResponseEntity<String> responseEntity = restTemplate.getForEntity(crisisSearchUrl, String.class, page, pageSize, keyword);
Object data = JSON.parseObject(responseEntity.getBody()).get("data");
return ResponseResult.success(data);
}
@ApiOperation("搜索-全网事件库-查事件")
@GetMapping("/getWholeNetworkEvents")
public ResponseResult getWholeNetworkEvents(@RequestParam("keyword") String keyword,
@RequestParam(value = "page",defaultValue = "1") Integer page){
String name = keyword.trim();
ResponseEntity<String> responseEntity = restTemplate.getForEntity(getEfSearchUrl, String.class, name, page);
JSONObject jsonObject = JSON.parseObject(responseEntity.getBody());
return ResponseResult.success(jsonObject);
}
}
src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java
......@@ -8,8 +8,10 @@ import com.zhiwei.brandkbs2.pojo.dto.ExportAppYuqingDTO;
import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
import com.zhiwei.brandkbs2.pojo.vo.PageVO;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.web.bind.annotation.RequestParam;
import java.io.IOException;
import java.text.ParseException;
import java.util.List;
import java.util.Map;
......@@ -141,7 +143,7 @@ public interface MarkDataService {
* @param linkedGroupId 关联组id
* @return 首发稿件
*/
BaseMap getFirstArticle(Long startTime, Long endTime, String aggTitle, String projectId, String linkedGroupId) throws IOException;
BaseMap getFirstArticle(Long startTime, Long endTime, String aggTitle, String projectId, String linkedGroupId, String contendId) throws IOException;
/**
* 搜索标注数据通过事件
......@@ -163,4 +165,36 @@ public interface MarkDataService {
*/
List<JSONObject> searchMarkDataByTime(String projectId, String linkedGroupId, String contendId, Long startTime, Long endTime, String... fetchSource);
/**
* 竞品库-获取搜索条件
* @param linkedGroupId 关联组Id
* @return 搜索条件
*/
JSONObject getContendSearchCriteria(String linkedGroupId);
/**
* 竞品库-获取搜索结果列表
* @param markSearchDTO 标注数据搜索传输类
* @return 搜索结果列表
*/
PageVO<MarkFlowEntity> getContendSearchList(MarkSearchDTO markSearchDTO) throws IOException;
/**
* 竞品库-获取竞品对比舆情解读数据
*
* @param markSearchDTO 标注数据搜索传输类
* @param cache 是否启用缓存
* @return 竞品对比舆情解读数据
*/
JSONObject getContendSummary(MarkSearchDTO markSearchDTO, boolean cache) throws IOException;
/**
* 竞品库-获取竞品对比传播分析数据
* @param markSearchDTO 标注数据搜索传输类
* @param hotArticleSize 热门稿件数量
* @param cache 是否启用缓存
* @return
*/
JSONObject getContendSpreadAnalyze(MarkSearchDTO markSearchDTO, int hotArticleSize, boolean cache) throws IOException;
}
src/main/java/com/zhiwei/brandkbs2/service/impl/CustomEventServiceImpl.java
......@@ -99,7 +99,7 @@ public class CustomEventServiceImpl implements CustomEventService {
JSONObject result = new JSONObject();
String aggTitle = map.getKey();
try {
BaseMap firstArticle = markDataService.getFirstArticle(startTime, endTime, aggTitle, projectId, linkedGroupId);
BaseMap firstArticle = markDataService.getFirstArticle(startTime, endTime, aggTitle, projectId, linkedGroupId, Constant.PRIMARY_CONTENDID);
result.put("title", firstArticle.getTitle());
result.put("time", firstArticle.getTime());
result.put("url", firstArticle.getUrl());
......
src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java
......@@ -8,6 +8,7 @@ import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.common.GlobalPojo;
import com.zhiwei.brandkbs2.common.RedisKeyPrefix;
import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.dao.AggreeResultDao;
import com.zhiwei.brandkbs2.enmus.ChannelEmotion;
import com.zhiwei.brandkbs2.enmus.EmotionEnum;
......@@ -17,10 +18,7 @@ import com.zhiwei.brandkbs2.es.EsQueryTools;
import com.zhiwei.brandkbs2.exception.ExceptionCast;
import com.zhiwei.brandkbs2.listener.ApplicationProjectListener;
import com.zhiwei.brandkbs2.model.CommonCodeEnum;
import com.zhiwei.brandkbs2.pojo.AggreeResult;
import com.zhiwei.brandkbs2.pojo.BaseMap;
import com.zhiwei.brandkbs2.pojo.Event;
import com.zhiwei.brandkbs2.pojo.MarkFlowEntity;
import com.zhiwei.brandkbs2.pojo.*;
import com.zhiwei.brandkbs2.pojo.dto.ExportAppYuqingDTO;
import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
import com.zhiwei.brandkbs2.pojo.vo.CustomTagVo;
......@@ -57,13 +55,18 @@ import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilde
import org.elasticsearch.search.sort.FieldSortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.joda.time.Period;
import org.joda.time.PeriodType;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
......@@ -355,7 +358,8 @@ public class MarkDataServiceImpl implements MarkDataService {
topTitle = negTopTitle.get(0).getKey();
}
result.put("negTopTitle", topTitle);
List<JSONObject> articlePlatformProportion = getMarkPlatformProportion(startTime, endTime, projectId, linkedGroupId, true);
// 舆情库默认contendId为0
List<JSONObject> articlePlatformProportion = getMarkPlatformProportion(startTime, endTime, projectId, linkedGroupId, Constant.PRIMARY_CONTENDID, true);
result.put("platformRank", articlePlatformProportion.stream().filter(articlePlatform -> articlePlatform.getLongValue("count") > 0).sorted((o1, o2) -> o2.getLong("count").compareTo(o1.getLong("count"))).collect(Collectors.toList()));
redisUtil.setExpire(redisKey, JSON.toJSONString(result));
return result;
......@@ -445,7 +449,8 @@ public class MarkDataServiceImpl implements MarkDataService {
endTime = timeRange[1];
String projectId = UserThreadLocal.getProjectId();
String linkedGroupId = projectService.getProjectVOById(projectId).getBrandLinkedGroupId();
res = getMarkPlatformProportion(startTime, endTime, projectId, linkedGroupId, cache);
// 舆情库默认contendId为0
res = getMarkPlatformProportion(startTime, endTime, projectId, linkedGroupId, Constant.PRIMARY_CONTENDID, cache);
} catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("es查询异常"), e);
}
......@@ -531,11 +536,13 @@ public class MarkDataServiceImpl implements MarkDataService {
private EsClientDao.SearchHelper createSearchHelperByMarkSearchDTO(MarkSearchDTO dto) {
EsClientDao.SearchHelper helper = EsClientDao.createSearchHelper();
// linkedGroupId projectId
// linkedGroupId、projectId、contendId
String projectId = dto.getProjectId();
String linkedGroupId = dto.getLinkedGroupId();
String contendId = dto.getContendId();
// PostFilter 后置过滤器
BoolQueryBuilder postFilter = projectLinkedGroupQuery(projectId, linkedGroupId);
// BoolQueryBuilder postFilter = projectLinkedGroupQuery(projectId, linkedGroupId);
BoolQueryBuilder postFilter = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, contendId);
// time
postFilter.must(QueryBuilders.rangeQuery(dto.getTimeType()).gte(dto.getStartTime()).lt(dto.getEndTime()));
// platform
......@@ -599,7 +606,6 @@ public class MarkDataServiceImpl implements MarkDataService {
helper.setSize(0);
// TODO 本地调试不滚动查询
// List<SearchResponse> searchResponses = esClientDao.searchScrollResponse(helper);
//singletonList：只能存放一个元素，多一个或者少一个都会导致异常
List<SearchResponse> searchResponses = Collections.singletonList(esClientDao.searchResponse(helper));
return Pair.of(searchResponses.stream().map(SearchResponse::getHits).toArray(SearchHits[]::new), null);
}
......@@ -608,7 +614,8 @@ public class MarkDataServiceImpl implements MarkDataService {
Map<String, Long> counts = new HashMap<>();
if (1 == dto.getPage() && CollectionUtils.isEmpty(dto.getPlatforms())) {
for (MessagePlatform platform : GlobalPojo.PLATFORMS) {
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery().must(helper.getPostFilter()).must(EsQueryTools.assemblePlatformQuery(Collections.singletonList(platform)));
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery().must(helper.getPostFilter())
.must(EsQueryTools.assemblePlatformQuery(Collections.singletonList(platform)));
Long count = esClientDao.count(esClientDao.getIndexes(), queryBuilder, null);
counts.put(platform.getName(), count);
}
......@@ -800,11 +807,11 @@ public class MarkDataServiceImpl implements MarkDataService {
}
@Override
public BaseMap getFirstArticle(Long startTime, Long endTime, String aggTitle, String projectId, String linkedGroupId) throws IOException {
public BaseMap getFirstArticle(Long startTime, Long endTime, String aggTitle, String projectId, String linkedGroupId, String contendId) throws IOException {
// 索引
String[] indexes = esClientDao.getIndexes();
// postFilter
BoolQueryBuilder postFilter = projectLinkedGroupQuery(projectId, linkedGroupId);
BoolQueryBuilder postFilter = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, contendId);
postFilter.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime)).must(QueryBuilders.termQuery("agg_title.keyword", aggTitle));
//sort
FieldSortBuilder sort = new FieldSortBuilder("time").order(SortOrder.ASC);
......@@ -843,6 +850,421 @@ public class MarkDataServiceImpl implements MarkDataService {
return Collections.emptyList();
}
@Override
public JSONObject getContendSearchCriteria(String linkedGroupId) {
String projectId = UserThreadLocal.getProjectId();
if (null == linkedGroupId) {
linkedGroupId = projectService.getProjectVOById(projectId).getBrandLinkedGroupId();
}
JSONObject result = new JSONObject();
// 搜索时间
result.put("times", Arrays.asList("今天", "24小时", "三天", "七天", "近30天"));
// 平台
result.put("platformList", commonService.getQbjcPlatform("id", "name"));
// 情感标签
result.put("emotionList", commonService.getEmotionTagsWithSort(projectId, linkedGroupId).stream().map(markerTag -> {
JSONObject json = new JSONObject();
json.put("uniqueId", markerTag.getUniqueId());
json.put("name", markerTag.getName());
return json;
}).collect(Collectors.toList()));
// 重要发声方
result.put("mediaTypeList", ImportantChannelEnum.getAllTagExceptSpec());
// 竞品品牌
Project project = projectService.getProjectById(projectId);
List<JSONObject> contendNameList = project.getContendList().stream().map(contend -> {
JSONObject jsonObject = new JSONObject();
jsonObject.put("contendName", contend.getBrandName());
jsonObject.put("contendId", contend.getId());
return jsonObject;
}).collect(Collectors.toList());
JSONObject allContend = new JSONObject();
allContend.put("contendName", "全部");
allContend.put("contendId", "-1");
contendNameList.add(0, allContend);
result.put("contendNameList", contendNameList);
// 其他标签组(筛选器)
result.put("tagFilterList", tagFilterService.getTagFilter());
// 自定义标签列表
List<CustomTagVo> customTagList = customTagService.findCustomTagList(1, 1).getList();
if (CollectionUtils.isNotEmpty(customTagList)) {
result.put("customTag", customTagList.get(0).generateSearchSonTagMaps());
}
return result;
}
@Override
public PageVO<MarkFlowEntity> getContendSearchList(MarkSearchDTO dto) {
try {
defaultMarkSearch(dto);
// 搜索es数据
Pair<SearchHits[], Map<String, Long>> hitsAndCounts = searchContendMarkHitsAndCount(dto, false);
// 总量
// long total = hitsAndCounts.getLeft()[0].getTotalHits().value > 10000 ? 10000 : hitsAndCounts.getLeft()[0].getTotalHits().value;
long total = hitsAndCounts.getLeft()[0].getTotalHits().value;
// 消息列表
List<MarkFlowEntity> flowEntityList = getMarkFlowEntity(dto, hitsAndCounts.getLeft()[0]);
// 返回分页结果并设置平台count
return PageVO.createPageVo(total, dto.getPage(), dto.getPageSize(), flowEntityList)
// 各平台计量
.setInfo(new JSONObject(ImmutableMap.of("platformCount", hitsAndCounts.getRight())));
} catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("es检索异常"));
}
return null;
}
private Pair<SearchHits[], Map<String, Long>> searchContendMarkHitsAndCount(MarkSearchDTO dto, boolean aggree) throws IOException {
EsClientDao.SearchHelper helper = createContendSearchHelper(dto);
if (aggree) {
helper.setSize(0);
// TODO 本地调试不滚动查询
// List<SearchResponse> searchResponses = esClientDao.searchScrollResponse(helper);
List<SearchResponse> searchResponses = Collections.singletonList(esClientDao.searchResponse(helper));
return Pair.of(searchResponses.stream().map(SearchResponse::getHits).toArray(SearchHits[]::new), null);
}
SearchHits searchHits = esClientDao.searchHits(helper);
//平台计量统计 仅第一页且无平台限制
Map<String, Long> counts = new HashMap<>();
if (1 == dto.getPage() && CollectionUtils.isEmpty(dto.getPlatforms())) {
for (MessagePlatform platform : GlobalPojo.PLATFORMS) {
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery().must(helper.getPostFilter()).must(EsQueryTools.assemblePlatformQuery(Collections.singletonList(platform)));
Long count = esClientDao.count(esClientDao.getIndexes(), queryBuilder, null);
counts.put(platform.getName(), count);
}
}
return Pair.of(new SearchHits[]{searchHits}, counts);
}
private EsClientDao.SearchHelper createContendSearchHelper(MarkSearchDTO dto){
EsClientDao.SearchHelper helper = EsClientDao.createSearchHelper();
// linkedGroupId、projectId、contendId
String projectId = dto.getProjectId();
String linkedGroupId = dto.getLinkedGroupId();
String contendId = dto.getContendId();
List<String> contendIdList = projectService.getProjectById(projectId).getContendList().stream().map(AbstractBaseMongo::getId).collect(Collectors.toList());
// PostFilter 后置过滤器 time
BoolQueryBuilder postFilter;
if ("-1".equals(contendId) && CollectionUtils.isNotEmpty(contendIdList)){
postFilter = QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery("brandkbs_cache_maps.key.keyword", Tools.concat(projectId, linkedGroupId, Constant.PRIMARY_CONTENDID)));
}else {
postFilter = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, contendId);
}
//time
postFilter.must(QueryBuilders.rangeQuery(dto.getTimeType()).gte(dto.getStartTime()).lt(dto.getEndTime()));
// platform
if (CollectionUtils.isNotEmpty(dto.getPlatforms())) {
postFilter.must(EsQueryTools.assemblePlatformQuery(Tools.getPlatformByIds(dto.getPlatforms())));
}
// mediaTypes
if (CollectionUtils.isNotEmpty(dto.getMediaTypes())) {
postFilter.must(EsQueryTools.assembleMediaTypeQuery(dto.getMediaTypes()));
}
// tags
if (CollectionUtils.isNotEmpty(dto.getTags())) {
postFilter.must(EsQueryTools.assembleTagQuery(dto.getTags()));
}
// customTags
if (CollectionUtils.isNotEmpty(dto.getCustomTags())) {
postFilter.must(EsQueryTools.assembleTagQuery(dto.getCustomTags()));
}
// Query 查询条件
// BoolQueryBuilder query = QueryBuilders.boolQuery();
// keyword
if (StringUtils.isNotEmpty(dto.getKeyword())) {
// Query
String[] fieldSearch = "标题".equals(dto.getSearchField()) ? new String[]{GenericAttribute.ES_IND_TITLE} : new String[]{GenericAttribute.ES_IND_FULL_TEXT};
postFilter.must(EsQueryTools.assembleNormalKeywordQuery(dto.getKeyword(), fieldSearch));
}
// sourceKeyword
if (StringUtils.isNotEmpty(dto.getSourceKeyword())) {
postFilter.must(EsQueryTools.assembleSourceQuery(dto.getSourceKeyword()));
}
helper.setPostFilter(postFilter);
// helper.setQuery(query);
// sort
FieldSortBuilder sort = null;
if (null != dto.getSorter()) {
for (Map.Entry<String, Object> entry : dto.getSorter().entrySet()) {
String key = entry.getKey();
if (key.equals("influence")) {
key = "channel_influence";
} else if (key.equals("followers")) {
key = "channel_followers";
}
if (null == entry.getValue() || entry.getValue().toString().contains("desc")) {
sort = SortBuilders.fieldSort(key).order(SortOrder.DESC);
} else {
sort = SortBuilders.fieldSort(key).order(SortOrder.ASC);
}
}
}
helper.setSort(sort);
// from size
helper.setFrom((dto.getPage() - 1) * dto.getPageSize());
helper.setSize(dto.getPageSize());
// HighlightBuilder ???
return helper;
}
@Override
public JSONObject getContendSummary(MarkSearchDTO dto, boolean cache) throws IOException {
// 为dto设置默认值
defaultContendDTO(dto);
ProjectVO project = projectService.getProjectVOById(dto.getProjectId());
// 缓存
SimpleDateFormat sdf = new SimpleDateFormat(Constant.DAY_PATTERN);
String startTimeStr = sdf.format(dto.getStartTime());
String endTimeStr = sdf.format(dto.getEndTime());
String redisKey = RedisKeyPrefix.INDEX_COMPARE_SUMMARY_PC + Tools.concat(dto.getProjectId(), dto.getContendId(), startTimeStr, endTimeStr);
if (cache){
String result = redisUtil.get(redisKey);
if (Objects.nonNull(result)) {
return JSON.parseObject(result);
}
}
JSONObject result = new JSONObject();
// 开始时间
result.put("startTime", dto.getStartTime());
// 结束时间
result.put("endTime", dto.getEndTime());
result.put("days", new Period(dto.getStartTime(), dto.getEndTime(), PeriodType.days()).getDays());
result.put("priName", project.getBrandName());
// 获取时间段主品牌在每个平台传播量占比
List<JSONObject> primaryPlatformsProportion =
getMarkPlatformProportion(dto.getStartTime(), dto.getEndTime(), dto.getProjectId(), dto.getLinkedGroupId(), Constant.PRIMARY_CONTENDID, true);
// 获取时间段主品牌总传播量
long primaryTotalCount = primaryPlatformsProportion.stream().mapToLong(platform -> platform.getLongValue("count")).sum();
result.put("priNorTotal", primaryTotalCount);
// 主品牌传播量最大的平台的数据
JSONObject primaryMaxPlatformProportion = primaryPlatformsProportion.stream().max(Comparator.comparing(o -> o.getLong("count"))).get();
result.put("priMaxPlatform", primaryMaxPlatformProportion.getString("platform"));
result.put("priMaxPlatformPro", primaryMaxPlatformProportion.getDoubleValue("proportion"));
// 竞品的名字
AbstractProject projectByContendId = projectService.getProjectByContendId(dto.getProjectId(), dto.getContendId());
String brandName = projectByContendId.getBrandName();
result.put("conName", brandName);
// 获取时间段竞品在每个平台传播量
List<JSONObject> contendPlatformsProportion =
getMarkPlatformProportion(dto.getStartTime(), dto.getEndTime(), dto.getProjectId(), dto.getLinkedGroupId(), dto.getContendId(), true);
// 获取时间段竞品总传播量
long contendTotalCount = contendPlatformsProportion.stream().mapToLong(platform -> platform.getLongValue("count")).sum();
result.put("conNorTotal", contendTotalCount);
// 竞品传播量最大的平台的数据
JSONObject contendMaxPlatformProportion = contendPlatformsProportion.stream().max(Comparator.comparing(o -> o.getLong("count"))).get();
result.put("conMaxPlatform", contendMaxPlatformProportion.getString("platform"));
result.put("conMaxPlatformPro", contendMaxPlatformProportion.getDoubleValue("proportion"));
result.put("compare", String.format("%.1f", primaryTotalCount * 1.0 / contendTotalCount));
List<JSONObject> platformPros = primaryPlatformsProportion.stream()
.peek(priJson -> {
JSONObject samePlatformJson = contendPlatformsProportion.stream()
.filter(conJson -> priJson.getString("platform").equals(conJson.getString("platform")))
.collect(Collectors.toList()).get(0);
priJson.put("conProportion", samePlatformJson.getDoubleValue("proportion"));
priJson.put("conNum", samePlatformJson.getLongValue("count"));
})
.filter(json -> json.getDoubleValue("proportion") > 0.001 || json.getDoubleValue("conProportion") > 0.001)
.collect(Collectors.toList());
result.put("platformPros", platformPros);
redisUtil.setExpire(redisKey, JSON.toJSONString(result), 1, TimeUnit.DAYS);
return result;
}
@Override
public JSONObject getContendSpreadAnalyze(MarkSearchDTO dto, int hotArticleSize, boolean cache) throws IOException {
defaultContendDTO(dto);
ProjectVO project = projectService.getProjectVOById(dto.getProjectId());
// 竞品的名字
AbstractProject projectByContendId = projectService.getProjectByContendId(dto.getProjectId(), dto.getContendId());
String brandName = projectByContendId.getBrandName();
// 缓存
SimpleDateFormat sdf = new SimpleDateFormat(Constant.DAY_PATTERN);
String startTimeStr = sdf.format(dto.getStartTime());
String endTimeStr = sdf.format(dto.getEndTime());
String redisKey = RedisKeyPrefix.INDEX_COMPARE_ANALYZE + Tools.concat(dto.getProjectId(), dto.getContendId(), startTimeStr, endTimeStr, hotArticleSize);
if (cache){
String result = redisUtil.get(redisKey);
if (Objects.nonNull(result)) {
return JSON.parseObject(result);
}
}
List<JSONObject> resList = new ArrayList<>(2);
JSONObject result = new JSONObject();
// 开始时间
result.put("startTime", dto.getStartTime());
// 结束时间
result.put("endTime", dto.getEndTime());
// 主品牌图谱
JSONObject primaryLine = new JSONObject();
primaryLine.put("id", Constant.PRIMARY_CONTENDID);
primaryLine.put("brand", project.getBrandName());
List<JSONObject> primarySpread =
getDayMarkArticleCount(dto.getStartTime(), dto.getEndTime(), dto.getProjectId(), dto.getLinkedGroupId(), Constant.PRIMARY_CONTENDID);
primaryLine.put("spread", primarySpread);
resList.add(primaryLine);
result.put("days", primarySpread.size());
// 竞品图谱
JSONObject contendLine = new JSONObject();
contendLine.put("id", dto.getContendId());
contendLine.put("brand", brandName);
List<JSONObject> contendSpread =
getDayMarkArticleCount(dto.getStartTime(), dto.getEndTime(), dto.getProjectId(), dto.getLinkedGroupId(), dto.getContendId());
contendLine.put("spread", contendSpread);
resList.add(contendLine);
result.put("spread", resList);
// 封装主品牌信息结果
result.put("priName", project.getBrandName());
// 获取主品牌稿件传播平均值
double avgPrimaryNormalCount = primarySpread.stream().mapToLong(line -> line.getLongValue("count")).average().orElse(0);
result.put("priAvg", (int) avgPrimaryNormalCount);
// 获取主品牌大于平均值的稿件天数
long priGreaterThanAvg = primarySpread.stream().filter(line -> line.getLongValue("count") > avgPrimaryNormalCount).count();
result.put("priGtAvg", priGreaterThanAvg);
// 获取主品牌传播峰值时信息
JSONObject primaryMax = primarySpread.stream().max(Comparator.comparing(line-> line.getLongValue("count"))).get();
result.put("priMaxTime", sdf.format(Long.parseLong(primaryMax.getString("date"))));
result.put("priMaxCount", primaryMax.getLongValue("count"));
long priMaxDayStartTime = Long.parseLong(primaryMax.getString("date"));
long priMaxDayEndTime = DateUtils.addDays(new Date(Long.parseLong(primaryMax.getString("date"))), 1).getTime();
// 获取时间段某情感数据最多的标题
List<Map.Entry<String, Integer>> priTopTitle =
getMarkTopTitle(priMaxDayStartTime, priMaxDayEndTime, EmotionEnum.ALL.getName(), dto.getProjectId(), dto.getLinkedGroupId(), Constant.PRIMARY_CONTENDID, 1);
if (CollectionUtils.isNotEmpty(priTopTitle)) {
// 首发稿件
BaseMap priFirstArticle =
getFirstArticle(priMaxDayStartTime, priMaxDayEndTime, priTopTitle.get(0).getKey(), dto.getProjectId(), dto.getLinkedGroupId(), Constant.PRIMARY_CONTENDID);
result.put("priTopTitle", priFirstArticle.getTitle());
result.put("priTopTitleUrl", priFirstArticle.getUrl());
}
// 获取主品牌热门媒体方向
List<Map.Entry<String, Integer>> priHotTitles =
getMarkTopTitle(priMaxDayStartTime, priMaxDayEndTime, EmotionEnum.ALL.getName(), dto.getProjectId(), dto.getLinkedGroupId(), Constant.PRIMARY_CONTENDID, hotArticleSize);
if (CollectionUtils.isNotEmpty(priHotTitles)) {
List<JSONObject> priHotArticle = priHotTitles.stream().map(map -> {
JSONObject jsonObject = new JSONObject();
try {
BaseMap priFirstArticle =
getFirstArticle(dto.getStartTime(), dto.getEndTime(), map.getKey(), dto.getProjectId(), dto.getLinkedGroupId(), Constant.PRIMARY_CONTENDID);
if (null != priFirstArticle) {
jsonObject.put("title", priFirstArticle.getTitle());
jsonObject.put("url", priFirstArticle.getUrl());
}
jsonObject.put("num", map.getValue());
} catch (IOException e) {
log.error("error getContendSpreadAnalyze-primary-getFirstArticle - ", e);
}
return jsonObject;
}).collect(Collectors.toList());
result.put("priHotArticle", priHotArticle);
}
// 封装竞品信息结果
// 竞品名
result.put("conName", brandName);
// 获取竞品稿件传播平均值
double avgContendSpreadNormalCount = contendSpread.stream().mapToLong(line -> line.getLongValue("count")).average().orElse(0);
result.put("conAvg", (int) avgContendSpreadNormalCount);
// 获取竞品大于平均值的稿件天数
long conGreaterThanAvg = contendSpread.stream().filter(line -> line.getLongValue("count") > avgContendSpreadNormalCount).count();
result.put("conGtAvg", conGreaterThanAvg);
//获取竞品传播峰值信息
JSONObject contendMax = contendSpread.stream().max(Comparator.comparing(line -> line.getLongValue("count"))).get();
result.put("conMaxTime", sdf.format(Long.parseLong(primaryMax.getString("date"))));
result.put("conMaxCount", contendMax.getLongValue("count"));
long conMaxDayStartTime = Long.parseLong(primaryMax.getString("date"));
long conMaxDayEndTime = DateUtils.addDays(new Date(Long.parseLong(primaryMax.getString("date"))), 1).getTime();
List<Map.Entry<String, Integer>> conTopTitle =
getMarkTopTitle(conMaxDayStartTime, conMaxDayEndTime, EmotionEnum.ALL.getName(), dto.getProjectId(), dto.getLinkedGroupId(), dto.getContendId(), 1);
if (CollectionUtils.isNotEmpty(conTopTitle)){
// 首发稿件
BaseMap conFirstArticle =
getFirstArticle(conMaxDayStartTime, conMaxDayEndTime, conTopTitle.get(0).getKey(), dto.getProjectId(), dto.getLinkedGroupId(), dto.getContendId());
result.put("conTopTitle", conFirstArticle.getTitle());
result.put("conTopTitleUrl", conFirstArticle.getUrl());
}
// 获取竞品热门媒体方向
List<Map.Entry<String, Integer>> conHotTitles =
getMarkTopTitle(conMaxDayStartTime, conMaxDayEndTime, EmotionEnum.ALL.getName(), dto.getProjectId(), dto.getLinkedGroupId(), dto.getContendId(), hotArticleSize);
if (CollectionUtils.isNotEmpty(conHotTitles)){
List<JSONObject> conHotArticle = conHotTitles.stream().map(map -> {
JSONObject jsonObject = new JSONObject();
try {
BaseMap conFirstArticle =
getFirstArticle(dto.getStartTime(), dto.getEndTime(), map.getKey(), dto.getProjectId(), dto.getLinkedGroupId(), dto.getContendId());
if (null != conFirstArticle) {
jsonObject.put("title", conFirstArticle.getTitle());
jsonObject.put("url", conFirstArticle.getUrl());
}
jsonObject.put("num", map.getValue());
} catch (IOException e) {
log.error("error getContendSpreadAnalyze-contend-getFirstArticle - ", e);
}
return jsonObject;
}).collect(Collectors.toList());
result.put("conHotArticle", conHotArticle);
}
redisUtil.setExpire(redisKey, JSON.toJSONString(result), 1, TimeUnit.DAYS);
return result;
}
/**
* 获取每日稿件数量
*
* @param startTime 开始时间
* @param endTime 结束时间
* @param projectId 项目ID
* @param linkedGroupId 关联组Id
* @param contendId 竞品ID
* @return 每日稿件数量
*/
private List<JSONObject> getDayMarkArticleCount(Long startTime, Long endTime, String projectId, String linkedGroupId, String contendId) throws IOException {
List<JSONObject> resultList = new ArrayList<>();
// index
String[] indexes = esClientDao.getIndexes();
// dayAgg
DateHistogramAggregationBuilder daysAggregation = AggregationBuilders.dateHistogram("dayAgg").field("time").calendarInterval(DateHistogramInterval.DAY);
BoolQueryBuilder query = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, contendId);
// time range
query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
// response
SearchResponse response = esClientDao.searchResponse(indexes, null, query, daysAggregation, null, null, 0, 0, null);
Map<String, Aggregation> aggMap = response.getAggregations().asMap();
ParsedDateHistogram dayAggTeam = (ParsedDateHistogram) aggMap.get("dayAgg");
List<? extends Histogram.Bucket> buckets = dayAggTeam.getBuckets();
buckets.forEach(bucket -> {
JSONObject result = new JSONObject();
result.put("date", bucket.getKeyAsString());
result.put("count", bucket.getDocCount());
resultList.add(result);
});
return resultList;
}
/**
* 为竞品库竞品对比 标注数据搜索传输类设置默认值
* @param dto 标注数据搜索传输类
*/
private void defaultContendDTO(MarkSearchDTO dto){
if (Objects.isNull(dto.getStartTime()) || Objects.isNull(dto.getEndTime())) {
Date now = new Date();
dto.setEndTime(now.getTime());
dto.setStartTime(DateUtils.addDays(now, -29).getTime());
}
String projectId = UserThreadLocal.getProjectId();
dto.setProjectId(projectId);
String linkedGroupId = dto.getLinkedGroupId();
if (null == linkedGroupId) {
linkedGroupId = projectService.getProjectVOById(projectId).getBrandLinkedGroupId();
dto.setLinkedGroupId(linkedGroupId);
}
}
/**
* 获取每日稿件倾向稿件数量信息
*
......@@ -936,14 +1358,14 @@ public class MarkDataServiceImpl implements MarkDataService {
});
}
private List<JSONObject> getMarkPlatformProportion(Long startTime, Long endTime, String projectId, String linkedGroupId, boolean cache) throws IOException {
private List<JSONObject> getMarkPlatformProportion(Long startTime, Long endTime, String projectId, String linkedGroupId, String contendId, boolean cache) throws IOException {
String redisKey = RedisKeyPrefix.MARK_PLATFORM_PROPORTION + Tools.concat(projectId, startTime, endTime);
String resultStr;
if (cache && StringUtils.isNotEmpty(resultStr = redisUtil.get(redisKey))) {
return JSON.parseArray(resultStr, JSONObject.class);
}
List<JSONObject> platformList = commonService.getQbjcPlatform("id", "name");
List<JSONObject> platformsCount = getPlatformsCount(startTime, endTime, null, null, projectId, linkedGroupId, platformList);
List<JSONObject> platformsCount = getPlatformsCount(startTime, endTime, null, null, projectId, linkedGroupId, contendId, platformList);
long articlesCount = platformsCount.stream().mapToLong(platform -> platform.getLongValue("count")).sum();
List<JSONObject> resultList = platformsCount.stream().peek(platform -> platform.put("proportion", 0 == articlesCount ? 0 : platform.getLongValue("count") * 1.0 / articlesCount)).collect(Collectors.toList());
redisUtil.setExpire(redisKey, JSON.toJSONString(resultList));
......@@ -961,13 +1383,13 @@ public class MarkDataServiceImpl implements MarkDataService {
* @param platformList 平台集合
* @return 所有平台稿件数量信息
*/
private List<JSONObject> getPlatformsCount(Long startTime, Long endTime, String keyword, String searchField, String projectId, String linedGroupId, List<JSONObject> platformList) throws IOException {
private List<JSONObject> getPlatformsCount(Long startTime, Long endTime, String keyword, String searchField, String projectId, String linkedGroupId, String contendId, List<JSONObject> platformList) throws IOException {
// 获取索引
String[] indexes = esClientDao.getIndexes();
// 聚合请求
TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("count").field("platform_id.keyword").order(BucketOrder.count(false)).size(100);
// query
BoolQueryBuilder query = projectLinkedGroupQuery(projectId, linedGroupId);
BoolQueryBuilder query = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, contendId);
// keyword
if (StringUtils.isNotEmpty(keyword)) {
String[] fieldSearch = "标题".equals(searchField) ? new String[]{GenericAttribute.ES_IND_TITLE} : new String[]{GenericAttribute.ES_IND_FULL_TEXT};
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment