Skip to content

B
brandkbs2
Commit 65dc384b by 陈健智
Options

查舆情-数据分析

parent fa2a2247
Showing with 471 additions and 234 deletions
src/main/java/com/zhiwei/brandkbs2/controller/app/AppArticleController.java
......@@ -168,6 +168,7 @@ public class AppArticleController extends BaseController {
@ApiOperation("舆情分析-舆情概览")
@ApiImplicitParams({@ApiImplicitParam(name = "startTime", value = "开始时间", paramType = "query", dataType = "long"), @ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long")})
@GetMapping("/analyze/summary")
@Deprecated
public ResponseResult getAnalyzeSummary(@RequestParam(value = "startTime", required = false) Long startTime, @RequestParam(value = "endTime", required = false) Long endTime) {
return ResponseResult.success(markDataService.getAnalyzeSummary(startTime, endTime, true));
}
......@@ -447,77 +448,58 @@ public class AppArticleController extends BaseController {
}
@ApiOperation("新-舆情分析-舆情总量")
@GetMapping("/analyze/amount")
@PostMapping("/analyze/amount")
@LogRecord(description = "舆情库-舆情分析")
public ResponseResult getYuqingAmount(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "planId", required = false) String planId) {
return ResponseResult.success(markDataService.getYuqingAmount(startTime, endTime, planId));
public ResponseResult getYuqingAmount(@RequestBody MarkSearchDTO markSearchDTO) {
return ResponseResult.success(markDataService.getYuqingAmount(markSearchDTO));
}
@ApiOperation("新-舆情分析-情感分布")
@GetMapping("/analyze/emotion")
public ResponseResult getYuqingEmotionDistribution(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "planId", required = false) String planId) {
return ResponseResult.success(markDataService.getYuqingEmotionDistribution(startTime, endTime, planId));
@PostMapping("/analyze/emotion")
public ResponseResult getYuqingEmotionDistribution(@RequestBody MarkSearchDTO markSearchDTO) {
return ResponseResult.success(markDataService.getYuqingEmotionDistribution(markSearchDTO));
}
@ApiOperation("新-舆情分析-重点平台")
@GetMapping("/analyze/important-platform")
public ResponseResult getImportantPlatformPercentage(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "planId", required = false) String planId) {
return ResponseResult.success(markDataService.getImportantPlatformPercentage(startTime, endTime, planId));
@PostMapping("/analyze/important-platform")
public ResponseResult getImportantPlatformPercentage(@RequestBody MarkSearchDTO markSearchDTO) {
return ResponseResult.success(markDataService.getImportantPlatformPercentage(markSearchDTO));
}
@ApiOperation("新-舆情分析-平台占比")
@GetMapping("/analyze/platform-percent")
public ResponseResult getPlatformPercentage(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "planId", required = false) String planId) {
return ResponseResult.success(markDataService.getPlatformPercentage(startTime, endTime, planId));
@PostMapping("/analyze/platform-percent")
public ResponseResult getPlatformPercentage(@RequestBody MarkSearchDTO markSearchDTO) {
return ResponseResult.success(markDataService.geAnalyzePlatformPercentage(markSearchDTO));
}
@ApiOperation("新-舆情分析-舆情走势图")
@GetMapping("/analyze/tendency")
public ResponseResult getSpreadTendency(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "planId", required = false) String planId) {
return ResponseResult.success(markDataService.getSpreadTendency(startTime, endTime, planId));
@PostMapping("/analyze/tendency")
public ResponseResult getSpreadTendency(@RequestBody MarkSearchDTO markSearchDTO) {
return ResponseResult.success(markDataService.getSpreadTendency(markSearchDTO));
}
@ApiOperation("新-舆情分析-活跃渠道")
@GetMapping("/analyze/active-channel")
public ResponseResult getActiveChannels(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "planId", required = false) String planId) {
return ResponseResult.success(markDataService.getActiveChannels(startTime, endTime, planId));
@PostMapping("/analyze/active-channel")
public ResponseResult getActiveChannels(@RequestBody MarkSearchDTO markSearchDTO) {
return ResponseResult.success(markDataService.getActiveChannels(markSearchDTO));
}
@ApiOperation("新-舆情分析-ip分布")
@GetMapping("/analyze/ip-located")
public ResponseResult getArticleIpLocated(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "planId", required = false) String planId,
@RequestParam(value = "size") int size) {
return ResponseResult.success(markDataService.getArticleIpLocated(startTime, endTime, planId, size));
@PostMapping("/analyze/ip-located")
public ResponseResult getArticleIpLocated(@RequestBody MarkSearchDTO markSearchDTO) {
return ResponseResult.success(markDataService.getArticleIpLocated(markSearchDTO));
}
@ApiOperation("新-舆情分析-词云")
@GetMapping("/analyze/high-word")
public ResponseResult getHighWord(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "planId", required = false) String planId) {
return ResponseResult.success(markDataService.getHighWord(startTime, endTime, planId, true));
@PostMapping("/analyze/high-word")
public ResponseResult getHighWord(@RequestBody MarkSearchDTO markSearchDTO) {
return ResponseResult.success(markDataService.getAnalyzeHighWord(markSearchDTO, true));
}
@ApiOperation("新-舆情分析-高频标题")
@GetMapping("/analyze/frequent-title")
public ResponseResult getLastNews(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "planId", required = false) String planId) {
return ResponseResult.success(markDataService.getLastNews(startTime, endTime, planId, 5, true));
@PostMapping("/analyze/frequent-title")
public ResponseResult getLastNews(@RequestBody MarkSearchDTO markSearchDTO) {
return ResponseResult.success(markDataService.getLastNews(markSearchDTO, 5));
}
@ApiOperation("新-舆情分析-活跃渠道、ip分布、词云详情页面,社媒平台发文")
......@@ -527,42 +509,33 @@ public class AppArticleController extends BaseController {
}
@ApiOperation("新-舆情分析-高频标题-基础信息")
@GetMapping("/analyze/frequent-title/info")
public ResponseResult getAggTitleBaseInfo(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "aggTitle") String aggTitle,
@RequestParam(value = "planId", required = false) String planId) {
return ResponseResult.success(markDataService.getAggTitleBaseInfo(aggTitle, startTime, endTime, planId));
@PostMapping("/analyze/frequent-title/info")
public ResponseResult getAggTitleBaseInfo(@RequestBody MarkSearchDTO markSearchDTO) {
return ResponseResult.success(markDataService.getAggTitleBaseInfo(markSearchDTO));
}
@ApiOperation("新-舆情分析-高频标题-发布节点")
@GetMapping("/analyze/frequent-title/article-point")
public ResponseResult getArticlePoints(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "aggTitle") String aggTitle,
@RequestParam(value = "planId", required = false) String planId) {
return ResponseResult.success(markDataService.getAggTitleArticlePoints(aggTitle, startTime, endTime, planId));
@PostMapping("/analyze/frequent-title/article-point")
public ResponseResult getArticlePoints(@RequestBody MarkSearchDTO markSearchDTO) {
return ResponseResult.success(markDataService.getAggTitleArticlePoints(markSearchDTO));
}
@ApiOperation("新-舆情分析-高频标题-平台分布")
@GetMapping("/analyze/frequent-title/platform-percent")
public ResponseResult getAggTitlePlatformPercentage(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "aggTitle") String aggTitle,
@RequestParam(value = "planId", required = false) String planId) {
return ResponseResult.success(markDataService.getAggTitlePlatformPercentage(aggTitle, startTime, endTime, planId));
@PostMapping("/analyze/frequent-title/platform-percent")
public ResponseResult getAggTitlePlatformPercentage(@RequestBody MarkSearchDTO markSearchDTO) {
return ResponseResult.success(markDataService.getAggTitlePlatformPercentage(markSearchDTO));
}
@ApiOperation("新-舆情分析-高频标题-发文列表")
@GetMapping("/analyze/frequent-title/articles")
public ResponseResult getArticleList(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "aggTitle") String aggTitle,
@RequestParam(value = "platform") String platform,
@RequestParam(value = "planId", required = false) String planId,
@RequestParam(value = "page", defaultValue = "1") Integer page,
@RequestParam(value = "pageSize", defaultValue = "10") Integer pageSize) {
return ResponseResult.success(markDataService.getArticleList(aggTitle, startTime, endTime, planId, platform, page, pageSize));
@PostMapping("/analyze/frequent-title/articles")
public ResponseResult getArticleList(@RequestBody MarkSearchDTO markSearchDTO) {
return ResponseResult.success(markDataService.getArticleList(markSearchDTO));
}
@ApiOperation("新-（查舆情）舆情分析-词云")
@PostMapping("/analyze/search/high-word")
public ResponseResult generateSearchAnalyzeHighWord(@RequestBody MarkSearchDTO markSearchDTO) {
return ResponseResult.success(markDataService.getSearchAnalyzeHighWord(markSearchDTO));
}
private boolean checkMTagIllegal(StringBuilder mtag) {
......
src/main/java/com/zhiwei/brandkbs2/controller/app/AppEventController.java
......@@ -9,6 +9,7 @@ import com.zhiwei.brandkbs2.enmus.RoleEnum;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.dto.EventSearchDTO;
import com.zhiwei.brandkbs2.service.EventService;
import com.zhiwei.brandkbs2.service.MarkDataService;
import com.zhiwei.middleware.event.pojo.dto.BrandkbsEventSearchDTO;
import com.zhiwei.middleware.event.pojo.dto.EventDTO;
import io.swagger.annotations.Api;
......@@ -20,6 +21,7 @@ import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.client.RestTemplate;
import javax.annotation.Resource;
import java.util.List;
/**
......@@ -64,6 +66,9 @@ public class AppEventController extends BaseController {
private final EventService eventService;
@Resource(name = "markDataServiceImpl")
MarkDataService markDataService;
public AppEventController(EventService eventService) {
this.eventService = eventService;
}
......@@ -355,4 +360,43 @@ public class AppEventController extends BaseController {
public ResponseResult aggTitleAnalyze(@RequestBody EventDTO dto){
return eventService.aggTitleAnalyze(dto);
}
@ApiOperation("新-舆情分析-高频标题-基础信息")
@GetMapping("/analyze/frequent-title/info")
public ResponseResult getAggTitleBaseInfo(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "aggTitle") String aggTitle,
@RequestParam(value = "planId", required = false) String planId) {
return ResponseResult.success(markDataService.getAggTitleBaseInfo(aggTitle, startTime, endTime, planId));
}
@ApiOperation("新-舆情分析-高频标题-发布节点")
@GetMapping("/analyze/frequent-title/article-point")
public ResponseResult getArticlePoints(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "aggTitle") String aggTitle,
@RequestParam(value = "planId", required = false) String planId) {
return ResponseResult.success(markDataService.getAggTitleArticlePoints(aggTitle, startTime, endTime, planId));
}
@ApiOperation("新-舆情分析-高频标题-平台分布")
@GetMapping("/analyze/frequent-title/platform-percent")
public ResponseResult getAggTitlePlatformPercentage(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "aggTitle") String aggTitle,
@RequestParam(value = "planId", required = false) String planId) {
return ResponseResult.success(markDataService.getAggTitlePlatformPercentage(aggTitle, startTime, endTime, planId));
}
@ApiOperation("新-舆情分析-高频标题-发文列表")
@GetMapping("/analyze/frequent-title/articles")
public ResponseResult getArticleList(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "aggTitle") String aggTitle,
@RequestParam(value = "platform") String platform,
@RequestParam(value = "planId", required = false) String planId,
@RequestParam(value = "page", defaultValue = "1") Integer page,
@RequestParam(value = "pageSize", defaultValue = "10") Integer pageSize) {
return ResponseResult.success(markDataService.getArticleList(aggTitle, startTime, endTime, planId, platform, page, pageSize));
}
}
src/main/java/com/zhiwei/brandkbs2/dao/MarkSearchAnalyzeTaskDao.java 0 → 100644
package com.zhiwei.brandkbs2.dao;
import com.zhiwei.brandkbs2.pojo.MarkSearchAnalyzeTask;
public interface MarkSearchAnalyzeTaskDao extends BaseMongoDao<MarkSearchAnalyzeTask>{
}
src/main/java/com/zhiwei/brandkbs2/dao/impl/MarkSearchAnalyzeTaskDaoImpl.java 0 → 100644
package com.zhiwei.brandkbs2.dao.impl;
import com.zhiwei.brandkbs2.dao.MarkSearchAnalyzeTaskDao;
import com.zhiwei.brandkbs2.pojo.MarkSearchAnalyzeTask;
import org.springframework.stereotype.Component;
@Component("markSearchAnalyzeTaskDao")
public class MarkSearchAnalyzeTaskDaoImpl extends BaseMongoDaoImpl<MarkSearchAnalyzeTask> implements MarkSearchAnalyzeTaskDao {
private static final String COLLECTION_NAME = "brandkbs_search_analyze_task";
public MarkSearchAnalyzeTaskDaoImpl() {
super(COLLECTION_NAME);
}
}
src/main/java/com/zhiwei/brandkbs2/pojo/MarkSearchAnalyzeTask.java 0 → 100644
package com.zhiwei.brandkbs2.pojo;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.Setter;
import java.util.List;
/**
* @ClassName: YuqingSearchAnalyzeTask
* @Description 查舆情数据分析任务
* @author: cjz
* @date: 2024-04-16 14:21
*/
@Getter
@Setter
@AllArgsConstructor
public class MarkSearchAnalyzeTask extends AbstractBaseMongo {
/**
* 查舆情搜索条件
*/
private MarkSearchDTO markSearchDTO;
/**
* 数据分析结果-词云
*/
List<JSONObject> highWord;
/**
* 词云计算是否已完成
*/
private Boolean finish;
/**
* 创建时间
*/
private Long cTime;
}
src/main/java/com/zhiwei/brandkbs2/pojo/dto/MarkSearchDTO.java
......@@ -195,4 +195,9 @@ public class MarkSearchDTO {
*/
@ApiModelProperty(value = "gid限制")
private Long pageGid;
/**
* 聚合标题
*/
@ApiModelProperty(value = "聚合标题")
private String aggTitle;
}
src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java
......@@ -5,11 +5,7 @@ import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.BaseMap;
import com.zhiwei.brandkbs2.pojo.Event;
import com.zhiwei.brandkbs2.pojo.MarkFlowEntity;
import com.zhiwei.brandkbs2.pojo.dto.ExportAppYuqingDTO;
import com.zhiwei.brandkbs2.pojo.dto.ExportSearchWholeDTO;
import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
import com.zhiwei.brandkbs2.pojo.dto.SearchFilterDTO;
import com.zhiwei.brandkbs2.pojo.dto.NonManualProjectPlanDTO;
import com.zhiwei.brandkbs2.pojo.dto.*;
import com.zhiwei.brandkbs2.pojo.vo.LineVO;
import com.zhiwei.brandkbs2.pojo.vo.PageVO;
import org.apache.commons.lang3.tuple.Pair;
......@@ -545,95 +541,88 @@ public interface MarkDataService {
/**
* 新-舆情分析-舆情总量
* @param startTime
* @param endTime
* @param planId
* @param dto 标注数据搜索传输类
* @return
* @throws IOException
*/
JSONObject getYuqingAmount(Long startTime, Long endTime, String planId);
JSONObject getYuqingAmount(MarkSearchDTO dto);
/**
* 新-舆情分析-情感分布
* @param startTime
* @param endTime
* @param planId
* @param dto 标注数据搜索传输类
* @return
* @throws IOException
*/
JSONObject getYuqingEmotionDistribution(Long startTime, Long endTime, String planId);
JSONObject getYuqingEmotionDistribution(MarkSearchDTO dto);
/**
* 新-舆情分析-重点平台
* @param startTime
* @param endTime
* @param planId
* @param dto 标注数据搜索传输类
* @return
*/
JSONObject getImportantPlatformPercentage(Long startTime, Long endTime, String planId);
JSONObject getImportantPlatformPercentage(MarkSearchDTO dto);
/**
* 新-舆情分析-平台占比
* @param startTime
* @param endTime
* @param planId
* @param dto 标注数据搜索传输类
* @return
*/
List<JSONObject> getPlatformPercentage(Long startTime, Long endTime, String planId);
List<JSONObject> geAnalyzePlatformPercentage(MarkSearchDTO dto);
/**
* 新-舆情分析-舆情走势图
* @param startTime
* @param endTime
* @param planId
* @param dto 标注数据搜索传输类
* @return
*/
JSONObject getSpreadTendency(Long startTime, Long endTime, String planId);
JSONObject getSpreadTendency(MarkSearchDTO dto);
/**
* 新-舆情分析-活跃渠道
* @param startTime
* @param endTime
* @param planId
* @param dto 标注数据搜索传输类
* @return
*/
List<JSONObject> getActiveChannels(Long startTime, Long endTime, String planId);
List<JSONObject> getActiveChannels(MarkSearchDTO dto);
/**
* 新-舆情分析-ip分布
* @param startTime
* @param endTime
* @param planId
* @param dto 标注数据搜索传输类
* @return
*/
List<JSONObject> getArticleIpLocated(Long startTime, Long endTime, String planId, int size);
List<JSONObject> getArticleIpLocated(MarkSearchDTO dto);
/**
* 新-舆情分析-活跃渠道、ip分布、词云详情页面
* @param dto
* @param dto 标注数据搜索传输类
* @return
*/
PageVO<MarkFlowEntity> getYuqingAnalyzeDetail(MarkSearchDTO dto);
void cacheHighWord(Long startTime, Long endTime, String planId, boolean cache);
/**
* 新-舆情分析-词云
* @param startTime
* @param endTime
* @param planId
* @param dto 标注数据搜索传输类
* @return
*/
List<JSONObject> getHighWord(Long startTime, Long endTime, String planId, boolean cache);
List<JSONObject> getAnalyzeHighWord(MarkSearchDTO dto, boolean cache);
/**
* 查舆情-数据分析-生成词云
* @param dto
* @return
*/
List<JSONObject> getSearchAnalyzeHighWord(MarkSearchDTO dto);
/**
* 新-舆情分析-高频标题
* @param startTime
* @param endTime
* @param planId
* @param size
* @param include
* @return
*/
List<JSONObject> getLastNews(Long startTime, Long endTime, String planId, int size, boolean include);
List<JSONObject> getLastNews(Long startTime, Long endTime, int size);
List<JSONObject> getLastNews(MarkSearchDTO dto, int size);
/**
* 舆情分析-高频标题详情页-基础信息
......@@ -645,6 +634,8 @@ public interface MarkDataService {
*/
JSONObject getAggTitleBaseInfo(String aggTitle, Long startTime, Long endTime, String planId);
JSONObject getAggTitleBaseInfo(MarkSearchDTO dto);
/**
* 舆情分析-高频标题详情页-发布节点
* @param aggTitle
......@@ -655,6 +646,8 @@ public interface MarkDataService {
*/
List<JSONObject> getAggTitleArticlePoints(String aggTitle, Long startTime, Long endTime, String planId);
List<JSONObject> getAggTitleArticlePoints(MarkSearchDTO dto);
/**
* 舆情分析-高频标题详情页-平台分布
* @param aggTitle
......@@ -665,6 +658,8 @@ public interface MarkDataService {
*/
List<JSONObject> getAggTitlePlatformPercentage(String aggTitle, Long startTime, Long endTime, String planId);
List<JSONObject> getAggTitlePlatformPercentage(MarkSearchDTO dto);
/**
* 舆情分析-高频标题详情页-发文列表
* @param aggTitle
......@@ -676,4 +671,6 @@ public interface MarkDataService {
* @return
*/
PageVO<JSONObject> getArticleList(String aggTitle, Long startTime, Long endTime, String planId, String platform, int page, int pageSize);
PageVO<JSONObject> getArticleList(MarkSearchDTO dto);
}
src/main/java/com/zhiwei/brandkbs2/service/impl/EventServiceImpl.java
......@@ -903,7 +903,7 @@ public class EventServiceImpl implements EventService {
if (cache && StringUtils.isNotEmpty(resultStr)) {
return JSON.parseArray(resultStr, JSONObject.class);
}
List<JSONObject> resultList = markDataService.getLastNews(startTime, endTime, null, 50, true);
List<JSONObject> resultList = markDataService.getLastNews(startTime, endTime, 50);
redisUtil.setExpire(redisKey, JSON.toJSONString(resultList));
return resultList;
}
......
src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java
......@@ -13,10 +13,7 @@ import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.common.GlobalPojo;
import com.zhiwei.brandkbs2.common.RedisKeyPrefix;
import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.dao.AggreeResultDao;
import com.zhiwei.brandkbs2.dao.ChannelDao;
import com.zhiwei.brandkbs2.dao.HighlightWordDao;
import com.zhiwei.brandkbs2.dao.NonManualProjectPlanDao;
import com.zhiwei.brandkbs2.dao.*;
import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportYuqingInteractionUpdateDTO;
import com.zhiwei.brandkbs2.enmus.ChannelEmotion;
......@@ -73,6 +70,7 @@ import org.joda.time.PeriodType;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
......@@ -173,6 +171,9 @@ public class MarkDataServiceImpl implements MarkDataService {
@Resource(name = "nonManualProjectPlanDao")
NonManualProjectPlanDao nonManualProjectPlanDao;
@Resource(name = "markSearchAnalyzeTaskDao")
MarkSearchAnalyzeTaskDao markSearchAnalyzeTaskDao;
@Resource(name = "toolsetServiceImpl")
private ToolsetService toolsetService;
......@@ -858,31 +859,22 @@ public class MarkDataServiceImpl implements MarkDataService {
return result.entrySet().stream().sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())).limit(size).collect(Collectors.toList());
}
private List<JSONObject> getMarkTopTitle(Long startTime, Long endTime, String emotion, String projectId, String contendId, String planId, int size) throws IOException {
private List<JSONObject> getMarkTopTitle(MarkSearchDTO dto, int size) throws IOException {
// 索引
String[] indexes = esClientDao.getIndexes();
TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("titles").field("agg_title.keyword").size(size + Tools.FILTER_TITLE.size());
TermsAggregationBuilder sourceAggregationBuilder = AggregationBuilders.terms("source").field("source").size(10000);
// query
BoolQueryBuilder query;
if (Objects.isNull(planId)) {
query = projectContendIdQuery(projectId, contendId);
}else {
query = EsQueryTools.assembleCacheMapsPlanQuery(projectId, planId);
}
query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime))
BoolQueryBuilder query = yuqingAnalyzeQuery(dto);
// 过滤微博
.mustNot(QueryBuilders.termQuery("platform_id", "5d02236e6395002a7c380b79"));
if (null != emotion && !Objects.equals(emotion, EmotionEnum.ALL.getName())) {
query.must(QueryBuilders.termQuery("brandkbs_mark_cache_maps.name.keyword", emotion));
}
query.mustNot(QueryBuilders.termQuery("platform_id", "5d02236e6395002a7c380b79"));
SearchResponse searchResponse;
// 单独处理buckets超过上限异常，出错时按时间分段查询
try {
searchResponse = esClientDao.searchResponse(indexes, null, query,
aggregationBuilder.subAggregation(sourceAggregationBuilder), null, null, 0, 0, null);
}catch (ElasticsearchStatusException e){
return getMarkTopTitleDivided(query, indexes, aggregationBuilder, startTime, endTime, size);
return getMarkTopTitleDivided(query, indexes, aggregationBuilder, dto.getStartTime(), dto.getEndTime(), size);
}
List<JSONObject> res = new ArrayList<>();
Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
......@@ -976,10 +968,6 @@ public class MarkDataServiceImpl implements MarkDataService {
return getArticleSort(startTime, endTime, aggTitle, projectId, contendId, planId, include, "{\"time\" : \"asc\"}");
}
private BaseMap getLastArticle(Long startTime, Long endTime, String aggTitle, String projectId, String contendId, String planId, boolean include) throws IOException {
return getArticleSort(startTime, endTime, aggTitle, projectId, contendId, planId, include, "{\"time\" : \"desc\"}");
}
private BaseMap getArticleSort(Long startTime, Long endTime, String aggTitle, String projectId, String contendId, String planId, boolean include, String sorter) throws IOException {
// 索引
String[] indexes = esClientDao.getIndexes();
......@@ -1014,6 +1002,36 @@ public class MarkDataServiceImpl implements MarkDataService {
return Tools.getBaseFromEsMap(hits.getAt(0).getSourceAsMap());
}
private BaseMap getAnalyzeFirstArticle(MarkSearchDTO dto) throws IOException{
return getArticleSort(dto, "{\"time\" : \"asc\"}");
}
private BaseMap getAnalyzeLastArticle(MarkSearchDTO dto) throws IOException {
return getArticleSort(dto, "{\"time\" : \"desc\"}");
}
private BaseMap getArticleSort(MarkSearchDTO dto, String sorter) throws IOException {
// 索引
String[] indexes = esClientDao.getIndexes();
BoolQueryBuilder query = yuqingAnalyzeQuery(dto);
//sort
FieldSortBuilder sort = new FieldSortBuilder("time").order(SortOrder.ASC);
JSONObject sortJson = JSONObject.parseObject(sorter);
for (Map.Entry<String, Object> entry : sortJson.entrySet()) {
if (entry.getValue().toString().contains("desc")) {
sort = SortBuilders.fieldSort(entry.getKey()).order(SortOrder.DESC);
} else {
sort = SortBuilders.fieldSort(entry.getKey()).order(SortOrder.ASC);
}
}
//hits
SearchHits hits = esClientDao.searchHits(indexes, query, null, null, sort, 0, 1, null);
if (0 == hits.getTotalHits().value) {
return new BaseMap();
}
return Tools.getBaseFromEsMap(hits.getAt(0).getSourceAsMap());
}
@Override
public List<JSONObject> searchMarkDataByEvent(Event event) {
Long endTime = null;
......@@ -2412,19 +2430,21 @@ public class MarkDataServiceImpl implements MarkDataService {
}
@Override
public JSONObject getYuqingAmount(Long startTime, Long endTime, String planId) {
public JSONObject getYuqingAmount(MarkSearchDTO dto) {
JSONObject jsonObject = new JSONObject();
try {
String projectId = UserThreadLocal.getProjectId();
String planId = dto.getPlanId();
dto.setProjectId(projectId);
// 舆情总量
long total = getYuqingAnalyzeCount(startTime, endTime, planId, EmotionEnum.ALL.getName());
long total = getYuqingAnalyzeCount(dto);
jsonObject.put("total", total);
// 项目日均
String projectYuqingCountAvgKey = RedisUtil.getYuqingAnalyzeProjectAvgCountKey(projectId, Constant.PRIMARY_CONTEND_ID, planId);
double projectYuqingAvgCount = Objects.isNull(redisUtil.get(projectYuqingCountAvgKey)) ? 0d : Double.parseDouble(redisUtil.get(projectYuqingCountAvgKey));
jsonObject.put("projectAvg", projectYuqingAvgCount);
// 本次日均
int days = new Period(startTime, endTime, PeriodType.days()).getDays();
int days = new Period(dto.getStartTime(), dto.getEndTime(), PeriodType.days()).getDays();
days = 0 == days ? 1 : days;
double yuqingAvgCount = total == 0 ? 0d : total / (double) days;
jsonObject.put("compare", compare(yuqingAvgCount, projectYuqingAvgCount));
......@@ -2435,18 +2455,20 @@ public class MarkDataServiceImpl implements MarkDataService {
}
@Override
public JSONObject getYuqingEmotionDistribution(Long startTime, Long endTime, String planId) {
public JSONObject getYuqingEmotionDistribution(MarkSearchDTO dto) {
JSONObject jsonObject = new JSONObject();
try {
String projectId = UserThreadLocal.getProjectId();
String planId = dto.getPlanId();
dto.setProjectId(projectId);
// 舆情总量
long total = getYuqingAnalyzeCount(startTime, endTime, planId, EmotionEnum.ALL.getName());
long total = getYuqingAnalyzeCount(dto);
// 正面舆情
long positiveCount = getYuqingAnalyzeCount(startTime, endTime, planId, EmotionEnum.POSITIVE.getName());
long positiveCount = getYuqingAnalyzeCount(dto, EmotionEnum.POSITIVE.getName());
// 中性舆情
long neutralCount = getYuqingAnalyzeCount(startTime, endTime, planId, EmotionEnum.NEUTRAL.getName());
long neutralCount = getYuqingAnalyzeCount(dto, EmotionEnum.NEUTRAL.getName());
// 负面舆情
long negativeCount = getYuqingAnalyzeCount(startTime, endTime, planId, EmotionEnum.NEGATIVE.getName());
long negativeCount = getYuqingAnalyzeCount(dto, EmotionEnum.NEGATIVE.getName());
// 各情感倾向占比
double positivePercentage = total == 0 ? 0d : positiveCount / (double) total;
double neutralPercentage = total == 0 ? 0d : neutralCount / (double) total;
......@@ -2480,15 +2502,20 @@ public class MarkDataServiceImpl implements MarkDataService {
}
@Override
public JSONObject getImportantPlatformPercentage(Long startTime, Long endTime, String planId) {
public JSONObject getImportantPlatformPercentage(MarkSearchDTO dto) {
JSONObject res = new JSONObject();
try {
List<JSONObject> list = new ArrayList<>();
List<String> importantPlatforms = Arrays.asList("微博", "微信", "抖音", "小红书", "今日头条", "网媒");
List<String> importantPlatforms = new ArrayList<>();
String projectId = UserThreadLocal.getProjectId();
Long total = getYuqingAnalyzeCount(startTime, endTime, planId, importantPlatforms);
dto.setProjectId(projectId);
for (String platformNames : Arrays.asList("微博", "微信", "抖音", "小红书", "今日头条", "网媒")) {
importantPlatforms.add(GlobalPojo.getPlatformIdByName(platformNames));
}
dto.setPlatforms(importantPlatforms);
Long total = getYuqingAnalyzeCount(dto);
// 平台聚合
SearchResponse searchResponse = platformAggSearchResponse(startTime, endTime, planId, importantPlatforms, null);
SearchResponse searchResponse = platformAggSearchResponse(dto);
Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
ParsedStringTerms teamAgg = (ParsedStringTerms) aggMap.get("platform_count");
List<? extends Terms.Bucket> buckets = teamAgg.getBuckets();
......@@ -2503,12 +2530,12 @@ public class MarkDataServiceImpl implements MarkDataService {
// 总量
overview.put("total", total);
// 日均
int days = new Period(startTime, endTime, PeriodType.days()).getDays();
int days = new Period(dto.getStartTime(), dto.getEndTime(), PeriodType.days()).getDays();
days = 0 == days ? 1 : days;
double avg = total == 0 ? 0d : total / (double) days;
overview.put("avg", avg);
// 项目均值
String key = RedisUtil.getYuqingAnalyzePlatformAvgCountKey(projectId, Constant.PRIMARY_CONTEND_ID, planId);
String key = RedisUtil.getYuqingAnalyzePlatformAvgCountKey(projectId, Constant.PRIMARY_CONTEND_ID, dto.getPlanId());
double projectAvg = Objects.isNull(redisUtil.get(key)) ? 0d : Double.parseDouble(redisUtil.get(key));
overview.put("compare", compare(avg, projectAvg));
res.put("overview", overview);
......@@ -2534,9 +2561,9 @@ public class MarkDataServiceImpl implements MarkDataService {
}
@Override
public List<JSONObject> getPlatformPercentage(Long startTime, Long endTime, String planId) {
public List<JSONObject> geAnalyzePlatformPercentage(MarkSearchDTO dto) {
try {
return getPlatformPercentage(null, startTime, endTime, planId);
return getPlatformPercentage(dto);
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "新舆情分析getPlatformPercentage异常-", e);
}
......@@ -2545,22 +2572,19 @@ public class MarkDataServiceImpl implements MarkDataService {
/**
* 获取舆情分析平台分布
* @param aggTitle
* @param startTime
* @param endTime
* @param planId
* @param dto 标注数据搜索传输类
* @return
* @throws IOException
*/
private List<JSONObject> getPlatformPercentage(String aggTitle, Long startTime, Long endTime, String planId) throws IOException {
List<String> platforms = commonService.getQbjcPlatformNames();
private List<JSONObject> getPlatformPercentage(MarkSearchDTO dto) throws IOException {
List<String> platforms = commonService.getQbjcPlatform().stream().map(jsonObject -> jsonObject.getString("id")).collect(Collectors.toList());
dto.setPlatforms(platforms);
List<JSONObject> list = new ArrayList<>();
// query
BoolQueryBuilder query = yuqingAnalyzeQuery(startTime, endTime, null, planId, null, null, aggTitle);
// total
Long total = esClientDao.count(query);
dto.setProjectId(UserThreadLocal.getProjectId());
Long total = getYuqingAnalyzeCount(dto);
// 平台聚合
SearchResponse searchResponse = platformAggSearchResponse(startTime, endTime, planId, platforms, aggTitle);
SearchResponse searchResponse = platformAggSearchResponse(dto);
Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
ParsedStringTerms teamAgg = (ParsedStringTerms) aggMap.get("platform_count");
List<? extends Terms.Bucket> buckets = teamAgg.getBuckets();
......@@ -2584,67 +2608,66 @@ public class MarkDataServiceImpl implements MarkDataService {
/**
* 平台聚合查询
* @param startTime
* @param endTime
* @param planId
* @param platforms
* @param dto 标注数据搜索传输类
* @return
* @throws IOException
*/
private SearchResponse platformAggSearchResponse(Long startTime, Long endTime, String planId, List<String> platforms, String aggTitle) throws IOException {
private SearchResponse platformAggSearchResponse(MarkSearchDTO dto) throws IOException {
// 索引
String[] indexes = esClientDao.getIndexes();
// 聚合请求
TermsAggregationBuilder platformAggregationBuilder = AggregationBuilders.terms("platform_count").field("platform_id").order(BucketOrder.count(false));
// query
BoolQueryBuilder query;
if (Objects.isNull(aggTitle)){
query = yuqingAnalyzeQuery(startTime, endTime, planId, null, platforms);
}else {
query = yuqingAnalyzeQuery(startTime, endTime, null, planId, null, null, aggTitle);
}
BoolQueryBuilder query = yuqingAnalyzeQuery(dto);
return esClientDao.searchResponse(indexes, null, query, platformAggregationBuilder, null, null, 0, 0, null);
}
@Override
public JSONObject getSpreadTendency(Long startTime, Long endTime, String planId) {
public JSONObject getSpreadTendency(MarkSearchDTO dto) {
JSONObject res = new JSONObject();
try {
String projectId = UserThreadLocal.getProjectId();
Long endTime = dto.getEndTime();
String planId = dto.getPlanId();
dto.setProjectId(projectId);
// 各平台趋势图
Map<String, List<LineVO>> platformSpreadTendency = getPlatformSpreadTendency(startTime, endTime, planId);
Map<String, List<LineVO>> platformSpreadTendency = getPlatformSpreadTendency(dto);
// 总趋势图
Pair<String, List<LineVO>> spreadTendency = getSpreadTendency(startTime, endTime, planId, null);
Pair<String, List<LineVO>> spreadTendency = getEmotionSpreadTendency(dto, null);
// 负面趋势图
Pair<String, List<LineVO>> negativeSpreadTendency = getSpreadTendency(startTime, endTime, planId, EmotionEnum.NEGATIVE.getName());
Pair<String, List<LineVO>> negativeSpreadTendency = getEmotionSpreadTendency(dto, EmotionEnum.NEGATIVE.getName());
Map<String, BaseMap> baseMaps = new HashMap<>(2);
BaseMap totalBaseMaps = null;
BaseMap negativeBaseMaps = null;
if (Objects.nonNull(negativeSpreadTendency.getLeft()) && Objects.nonNull(spreadTendency.getLeft())) {
CompletableFuture.allOf(Stream.of(spreadTendency.getLeft(), negativeSpreadTendency.getLeft()).map(aggTitle -> CompletableFuture.runAsync(() -> {
if (Objects.nonNull(negativeSpreadTendency.getLeft()) || Objects.nonNull(spreadTendency.getLeft())) {
CompletableFuture.allOf(Stream.of(spreadTendency.getLeft(), negativeSpreadTendency.getLeft()).filter(Objects::nonNull).map(aggTitle ->
CompletableFuture.runAsync(() -> {
try {
baseMaps.put(aggTitle, getFirstArticle(startTime, endTime, aggTitle, projectId, Constant.PRIMARY_CONTEND_ID, planId, true));
dto.setAggTitle(aggTitle);
baseMaps.put(aggTitle, getAnalyzeFirstArticle(dto));
} catch (IOException ignored) {
}
}, executor)).toArray(CompletableFuture[]::new)).join();
}
// 趋势图标题为空，此时说明最高点的时间段发文平台全为微博，此时取最高点时间段内时间最早的微博发文
if (Objects.isNull(spreadTendency.getLeft()) || Objects.isNull(negativeSpreadTendency.getLeft())){
// 标题为空，但趋势图不为空，此时说明最高点的时间段发文平台全为微博，此时取最高点时间段内时间最早的微博发文
if (Objects.isNull(spreadTendency.getLeft()) && CollectionUtils.isNotEmpty(spreadTendency.getRight())){
totalBaseMaps = getWeiboFirstArticle(endTime, spreadTendency.getRight(), planId, projectId, null);
}
if (Objects.isNull(negativeSpreadTendency.getLeft()) && CollectionUtils.isNotEmpty(negativeSpreadTendency.getRight())){
negativeBaseMaps = getWeiboFirstArticle(endTime, spreadTendency.getRight(), planId, projectId, EmotionEnum.NEGATIVE.getName());
}
// 最高点
BaseMap baseMap = Objects.isNull(baseMaps.get(spreadTendency.getLeft())) ? totalBaseMaps : baseMaps.get(spreadTendency.getLeft());
JSONObject highestJson = new JSONObject();
if (Objects.nonNull(baseMap)) {
highestJson.put("title", baseMap.getTitle());
highestJson.put("title", Tools.filterUselessTitle(baseMap.getTitle()) ? StringUtils.substring(baseMap.getContent(), 0, 15) : baseMap.getTitle());
highestJson.put("url", baseMap.getUrl());
}
// 负面最高点
BaseMap negativeBaseMap = Objects.isNull(baseMaps.get(negativeSpreadTendency.getLeft())) ? negativeBaseMaps : baseMaps.get(negativeSpreadTendency.getLeft());
JSONObject negativeHighestJson = new JSONObject();
if (Objects.nonNull(negativeBaseMap)) {
negativeHighestJson.put("title", negativeBaseMap.getTitle());
negativeHighestJson.put("title", Tools.filterUselessTitle(negativeBaseMap.getTitle()) ? StringUtils.substring(negativeBaseMap.getContent(), 0, 15) : negativeBaseMap.getTitle());
negativeHighestJson.put("url", negativeBaseMap.getUrl());
}
res.putAll(platformSpreadTendency);
......@@ -2707,13 +2730,14 @@ public class MarkDataServiceImpl implements MarkDataService {
}
@Override
public List<JSONObject> getActiveChannels(Long startTime, Long endTime, String planId) {
public List<JSONObject> getActiveChannels(MarkSearchDTO dto) {
List<JSONObject> list = new ArrayList<>();
try {
dto.setProjectId(UserThreadLocal.getProjectId());
// 索引
String[] indexes = esClientDao.getIndexes();
// query
BoolQueryBuilder query = yuqingAnalyzeQuery(startTime, endTime, planId, null, null);
BoolQueryBuilder query = yuqingAnalyzeQuery(dto);
// 渠道聚合，取前20
TermsAggregationBuilder sourceAggregationBuilder = AggregationBuilders.terms("source_count").field("source").order(BucketOrder.count(false)).size(20 + Tools.FILTER_SOURCE.size());
// 情感倾向子聚合
......@@ -2753,15 +2777,16 @@ public class MarkDataServiceImpl implements MarkDataService {
}
@Override
public List<JSONObject> getArticleIpLocated(Long startTime, Long endTime, String planId, int size) {
public List<JSONObject> getArticleIpLocated(MarkSearchDTO dto) {
List<JSONObject> list = new ArrayList<>();
try {
// 索引
String[] indexes = esClientDao.getIndexes();
dto.setProjectId(UserThreadLocal.getProjectId());
// query
BoolQueryBuilder query = yuqingAnalyzeQuery(startTime, endTime, planId, null, null);
BoolQueryBuilder query = yuqingAnalyzeQuery(dto);
// 聚合请求
TermsAggregationBuilder ipAggregationBuilder = AggregationBuilders.terms("ip_count").field("ip_location.keyword").order(BucketOrder.count(false)).size(size);
TermsAggregationBuilder ipAggregationBuilder = AggregationBuilders.terms("ip_count").field("ip_location.keyword").order(BucketOrder.count(false)).size(10);
// 情感倾向子聚合
TermsAggregationBuilder emotionAggregationBuilder = AggregationBuilders.terms("emotion_count").field("brandkbs_mark_cache_maps.name.keyword");
SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, query,
......@@ -2800,29 +2825,57 @@ public class MarkDataServiceImpl implements MarkDataService {
// 列表数据最小mgid
MarkFlowEntity markFlowEntity = yuqingMarkList.getList().stream()
.min(Comparator.comparingLong(entity -> JSONObject.parseObject(JSONObject.toJSONString(entity.getData())).getLongValue("mgid"))).orElse(null);
// yuqingMarkList.getInfo().put("stime", Objects.isNull(markFlowEntity) ? Long.MAX_VALUE : JSONObject.parseObject(JSONObject.toJSONString(markFlowEntity.getData())).getLongValue("stime"));
yuqingMarkList.getInfo().put("pageGid", Objects.isNull(markFlowEntity) ? Long.MAX_VALUE : JSONObject.parseObject(JSONObject.toJSONString(markFlowEntity.getData())).getLongValue("mgid"));
return yuqingMarkList;
}
@Override
public List<JSONObject> getHighWord(Long startTime, Long endTime, String planId, boolean cache) {
List<JSONObject> res = new ArrayList<>();
try {
public void cacheHighWord(Long startTime, Long endTime, String planId, boolean cache) {
getAnalyzeHighWord(convertedToMarkSearch(null, startTime, endTime, planId), cache);
}
@Override
public List<JSONObject> getAnalyzeHighWord(MarkSearchDTO dto, boolean cache) {
String projectId = UserThreadLocal.getProjectId();
String redisKey = RedisUtil.getYuqingAnalyzeHighWordKey(projectId, Constant.PRIMARY_CONTEND_ID, planId, startTime, endTime);
dto.setProjectId(projectId);
String redisKey = RedisUtil.getYuqingAnalyzeHighWordKey(projectId, Constant.PRIMARY_CONTEND_ID, dto.getPlanId(), dto.getStartTime(), dto.getEndTime());
String resultStr;
// 返回缓存
if (cache && StringUtils.isNotEmpty(resultStr = redisUtil.get(redisKey))) {
return JSONArray.parseArray(resultStr, JSONObject.class);
}
List<JSONObject> res = generateHighWord(dto);
redisUtil.setExpire(redisKey, JSONArray.toJSONString(res));
return res;
}
@Override
public List<JSONObject> getSearchAnalyzeHighWord(MarkSearchDTO dto) {
dto.setProjectId(UserThreadLocal.getProjectId());
MarkSearchAnalyzeTask existTask = markSearchAnalyzeTaskDao.findOne("markSearchDTO", dto);
if (Objects.nonNull(existTask)){
return existTask.getHighWord();
}
MarkSearchAnalyzeTask task = new MarkSearchAnalyzeTask(dto, null, false, System.currentTimeMillis());
markSearchAnalyzeTaskDao.insertOne(task);
List<JSONObject> highWord = generateHighWord(dto);
Update update = new Update();
update.set("finish", true);
update.set("highWord", highWord);
markSearchAnalyzeTaskDao.updateOneByIdWithField(task.getId(), update);
return highWord;
}
private List<JSONObject> generateHighWord(MarkSearchDTO dto){
List<JSONObject> res = new ArrayList<>();
try {
EsClientDao.SearchHelper searchHelper = EsClientDao.createSearchHelper();
// sort
searchHelper.setSort(SortBuilders.fieldSort("time").order(SortOrder.DESC));
// fetchSource
searchHelper.setFetchSource(new String[]{"ind_title", "ind_full_text", "c5", "foreign", "brandkbs_mark_cache_maps"});
// postFilter
BoolQueryBuilder postFilter = yuqingAnalyzeQuery(startTime, endTime, planId, null, null);
BoolQueryBuilder postFilter = yuqingAnalyzeQuery(dto);
searchHelper.setPostFilter(postFilter);
searchHelper.setSize(10000);
List<String> textList = new ArrayList<>();
......@@ -2837,7 +2890,6 @@ public class MarkDataServiceImpl implements MarkDataService {
}
}
res = textUtil.getHighWordsJsonDifferentFieldName(textList, 20);
redisUtil.setExpire(redisKey, JSONArray.toJSONString(res));
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "新舆情分析getHighWord异常-", e);
}
......@@ -2845,13 +2897,19 @@ public class MarkDataServiceImpl implements MarkDataService {
}
@Override
public List<JSONObject> getLastNews(Long startTime, Long endTime, String planId, int size, boolean include) {
public List<JSONObject> getLastNews(Long startTime, Long endTime, int size) {
return getLastNews(convertedToMarkSearch(null, startTime, endTime, null), size);
}
@Override
public List<JSONObject> getLastNews(MarkSearchDTO dto, int size) {
String projectId = UserThreadLocal.getProjectId();
try {
List<JSONObject> markTopTitleList = getMarkTopTitle(startTime, endTime, null, projectId, Constant.PRIMARY_CONTEND_ID, planId, size);
dto.setProjectId(projectId);
List<JSONObject> markTopTitleList = getMarkTopTitle(dto, size);
CompletableFuture.allOf(markTopTitleList.stream().map(json -> CompletableFuture.supplyAsync(() -> {
try {
BaseMap firstArticle = getFirstArticle(startTime, endTime, json.getString("title"), projectId, Constant.PRIMARY_CONTEND_ID, planId, include);
BaseMap firstArticle = getAnalyzeFirstArticle(dto);
json.put("content", firstArticle.getContent());
json.put("url", firstArticle.getUrl());
json.put("realSource", firstArticle.getRealSource());
......@@ -2870,14 +2928,20 @@ public class MarkDataServiceImpl implements MarkDataService {
@Override
public JSONObject getAggTitleBaseInfo(String aggTitle, Long startTime, Long endTime, String planId) {
return getAggTitleBaseInfo(convertedToMarkSearch(aggTitle, startTime, endTime, planId));
}
@Override
public JSONObject getAggTitleBaseInfo(MarkSearchDTO dto) {
JSONObject res = new JSONObject();
try {
String projectId = UserThreadLocal.getProjectId();
dto.setProjectId(projectId);
String linkedGroupId = projectService.getProjectById(projectId).getBrandLinkedGroupId();
CompletableFuture<JSONObject> firstArticleFuture = CompletableFuture.supplyAsync(() -> {
try {
// 最早发布
BaseMap firstArticle = getFirstArticle(startTime, endTime, aggTitle, projectId, Constant.PRIMARY_CONTEND_ID, planId, true);
BaseMap firstArticle = getAnalyzeFirstArticle(dto);
// 匹配重要渠道
JSONObject firstSensitiveChannel = channelService.matchYuQingSensitiveChannel(linkedGroupId, firstArticle.getSource());
JSONObject firstArticleJson = JSONObject.parseObject(JSONObject.toJSONString(firstArticle));
......@@ -2890,7 +2954,7 @@ public class MarkDataServiceImpl implements MarkDataService {
CompletableFuture<JSONObject> lastArticleFuture = CompletableFuture.supplyAsync(() -> {
try {
// 最新发布
BaseMap lastArticle = getLastArticle(startTime, endTime, aggTitle, projectId, Constant.PRIMARY_CONTEND_ID, planId, true);
BaseMap lastArticle = getAnalyzeLastArticle(dto);
// 匹配重要渠道
JSONObject lastSensitiveChannel = channelService.matchYuQingSensitiveChannel(linkedGroupId, lastArticle.getSource());
JSONObject lastArticleJson = JSONObject.parseObject(JSONObject.toJSONString(lastArticle));
......@@ -2916,11 +2980,17 @@ public class MarkDataServiceImpl implements MarkDataService {
@Override
public List<JSONObject> getAggTitleArticlePoints(String aggTitle, Long startTime, Long endTime, String planId) {
return getAggTitleArticlePoints(convertedToMarkSearch(aggTitle, startTime, endTime, planId));
}
@Override
public List<JSONObject> getAggTitleArticlePoints(MarkSearchDTO dto) {
List<JSONObject> jsonObject = new ArrayList<>();
try {
String[] indexes = esClientDao.getIndexes();
// query
BoolQueryBuilder query = yuqingAnalyzeQuery(startTime, endTime, null, planId, null, null, aggTitle);
dto.setProjectId(UserThreadLocal.getProjectId());
BoolQueryBuilder query = yuqingAnalyzeQuery(dto);
DateHistogramAggregationBuilder daysAggregationBuilder =
AggregationBuilders.dateHistogram("timeAgg").field("time").calendarInterval(DateHistogramInterval.DAY);
TermsAggregationBuilder sourceAggregationBuilder = AggregationBuilders.terms("sourceAgg").field("source").size(10000);
......@@ -2965,8 +3035,13 @@ public class MarkDataServiceImpl implements MarkDataService {
@Override
public List<JSONObject> getAggTitlePlatformPercentage(String aggTitle, Long startTime, Long endTime, String planId) {
return getAggTitlePlatformPercentage(convertedToMarkSearch(aggTitle, startTime, endTime, planId));
}
@Override
public List<JSONObject> getAggTitlePlatformPercentage(MarkSearchDTO dto) {
try {
return getPlatformPercentage(aggTitle, startTime, endTime, planId);
return getPlatformPercentage(dto);
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "新舆情分析getAggTitlePlatformPercentage异常-", e);
}
......@@ -2975,15 +3050,31 @@ public class MarkDataServiceImpl implements MarkDataService {
@Override
public PageVO<JSONObject> getArticleList(String aggTitle, Long startTime, Long endTime, String planId, String platform, int page, int pageSize) {
MarkSearchDTO dto = convertedToMarkSearch(aggTitle, startTime, endTime, planId);
dto.setPlatforms(Collections.singletonList(platform));
dto.setPage(page);
dto.setPageSize(pageSize);
return getArticleList(dto);
}
@Override
public PageVO<JSONObject> getArticleList(MarkSearchDTO dto) {
int page = Objects.isNull(dto.getPage()) ? 1 : dto.getPage();
int pageSize = Objects.isNull(dto.getPageSize()) ? 10 : dto.getPageSize();
try {
String[] indexes = esClientDao.getIndexes();
dto.setProjectId(UserThreadLocal.getProjectId());
// query
BoolQueryBuilder query = yuqingAnalyzeQuery(startTime, endTime, null, planId, null, Collections.singletonList(platform), aggTitle);
if (Objects.equals("其他", platform)){
List<String> platforms = getPlatformPercentage(aggTitle, startTime, endTime, planId).stream().map(json -> json.getString("platform")).collect(Collectors.toList());
if (Objects.nonNull(dto.getPlatforms()) && Objects.nonNull(dto.getPlatforms().get(0))){
dto.setPlatforms(Collections.singletonList(GlobalPojo.getPlatformIdByName(dto.getPlatforms().get(0))));
}
BoolQueryBuilder query = yuqingAnalyzeQuery(dto);
if (Objects.equals("其他", dto.getPlatforms().get(0))){
List<String> platforms = getPlatformPercentage(dto).stream().map(json -> json.getString("platform")).collect(Collectors.toList());
List<String> allPlatforms = GlobalPojo.PLATFORMS.stream().map(MessagePlatform::getName).collect(Collectors.toList());
allPlatforms.removeAll(platforms);
query = yuqingAnalyzeQuery(startTime, endTime, null, planId, null, allPlatforms, aggTitle);
dto.setPlatforms(allPlatforms.stream().map(GlobalPojo::getPlatformIdByName).collect(Collectors.toList()));
query = yuqingAnalyzeQuery(dto);
}
// sort
FieldSortBuilder sort = SortBuilders.fieldSort("time").order(SortOrder.DESC);
......@@ -3007,33 +3098,43 @@ public class MarkDataServiceImpl implements MarkDataService {
return PageVO.createPageVo(0, page, pageSize, Collections.emptyList());
}
private MarkSearchDTO convertedToMarkSearch(String aggTitle, Long startTime, Long endTime, String planId){
MarkSearchDTO dto = new MarkSearchDTO();
dto.setAggTitle(aggTitle);
dto.setStartTime(startTime);
dto.setEndTime(endTime);
dto.setPlanId(planId);
return dto;
}
/**
* 微博、微信、抖音、小红书平台趋势图
* @param startTime
* @param endTime
* @param planId
* @param dto
* @return
* @throws IOException
*/
private Map<String, List<LineVO>> getPlatformSpreadTendency(Long startTime, Long endTime, String planId) throws IOException{
private Map<String, List<LineVO>> getPlatformSpreadTendency(MarkSearchDTO dto) throws IOException{
Map<String, List<LineVO>> res = new HashMap<>(4);
// 平台
List<String> platforms = Arrays.asList("微博", "微信", "抖音", "小红书");
List<String> platforms = new ArrayList<>();
for (String platformNames : Arrays.asList("微博", "微信", "抖音", "小红书")) {
platforms.add(GlobalPojo.getPlatformIdByName(platformNames));
}
String[] indexes = esClientDao.getIndexes();
// 聚合请求
TermsAggregationBuilder platformAggregationBuilder = AggregationBuilders.terms("platformAgg").field("platform_id").order(BucketOrder.count(false));
DateHistogramAggregationBuilder daysAggregationBuilder;
// 天级以小时为颗粒度，其他以天作为颗粒度
if (endTime - startTime <= Constant.ONE_DAY){
if (dto.getEndTime() - dto.getStartTime() <= Constant.ONE_DAY){
daysAggregationBuilder = AggregationBuilders.dateHistogram("timeAgg").field("time").calendarInterval(DateHistogramInterval.HOUR);
}else {
daysAggregationBuilder = AggregationBuilders.dateHistogram("timeAgg").field("time").calendarInterval(DateHistogramInterval.DAY);
}
dto.setPlatforms(platforms);
// query
BoolQueryBuilder query = yuqingAnalyzeQuery(startTime, endTime, planId, null, platforms);
BoolQueryBuilder query = yuqingAnalyzeQuery(dto);
// response
SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, query,
platformAggregationBuilder.subAggregation(daysAggregationBuilder), null, null, 0, 0, null);
SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, query, platformAggregationBuilder.subAggregation(daysAggregationBuilder), null, null, 0, 0, null);
Map<String, Aggregation> map = searchResponse.getAggregations().asMap();
ParsedStringTerms countTeam = (ParsedStringTerms) map.get("platformAgg");
List<? extends Terms.Bucket> platformBuckets = countTeam.getBuckets();
......@@ -3055,26 +3156,24 @@ public class MarkDataServiceImpl implements MarkDataService {
/**
* 新舆情分析-舆情趋势图
* @param startTime
* @param endTime
* @param planId
* @param emotion
* @param dto 标注数据搜索传输类
* @return
* @throws IOException
*/
private Pair<String, List<LineVO>> getSpreadTendency(Long startTime, Long endTime, String planId, String emotion) throws IOException {
private Pair<String, List<LineVO>> getEmotionSpreadTendency(MarkSearchDTO dto, String emotion) throws IOException {
List<LineVO> res = new ArrayList<>();
String[] indexes = esClientDao.getIndexes();
DateHistogramAggregationBuilder daysAggregationBuilder;
// 天级以小时为颗粒度，其他以天作为颗粒度
if (endTime - startTime <= Constant.ONE_DAY){
if (dto.getEndTime() - dto.getStartTime() <= Constant.ONE_DAY){
daysAggregationBuilder = AggregationBuilders.dateHistogram("timeAgg").field("time").calendarInterval(DateHistogramInterval.HOUR);
}else {
daysAggregationBuilder = AggregationBuilders.dateHistogram("timeAgg").field("time").calendarInterval(DateHistogramInterval.DAY);
}
TermsAggregationBuilder titleAggregationBuilder = AggregationBuilders.terms("titleAgg").field("agg_title.keyword").order(BucketOrder.count(false));
dto.setPlatforms(null);
// query
BoolQueryBuilder query = yuqingAnalyzeQuery(startTime, endTime, planId, emotion, null);
BoolQueryBuilder query = yuqingAnalyzeQuery(dto, emotion);
// response
SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, query,
daysAggregationBuilder.subAggregation(titleAggregationBuilder), null, null, 0, 0, null);
......@@ -3100,13 +3199,13 @@ public class MarkDataServiceImpl implements MarkDataService {
return Pair.of(maxAggTitle, res);
}
private Long getYuqingAnalyzeCount(Long startTime, Long endTime, String planId, List<String> platforms) throws IOException {
BoolQueryBuilder query = yuqingAnalyzeQuery(startTime, endTime, planId, null, platforms);
private Long getYuqingAnalyzeCount(MarkSearchDTO dto) throws IOException {
BoolQueryBuilder query = yuqingAnalyzeQuery(dto, null);
return esClientDao.count(query);
}
private Long getYuqingAnalyzeCount(Long startTime, Long endTime, String planId, String emotion) throws IOException {
BoolQueryBuilder query = yuqingAnalyzeQuery(startTime, endTime, planId, emotion, null);
private Long getYuqingAnalyzeCount(MarkSearchDTO dto, String emotion) throws IOException {
BoolQueryBuilder query = yuqingAnalyzeQuery(dto, emotion);
return esClientDao.count(query);
}
......@@ -3115,10 +3214,6 @@ public class MarkDataServiceImpl implements MarkDataService {
return esClientDao.count(query);
}
private BoolQueryBuilder yuqingAnalyzeQuery(Long startTime, Long endTime, String planId, String emotion, List<String> platforms){
return yuqingAnalyzeQuery(startTime, endTime, null, planId, emotion, platforms, null);
}
private BoolQueryBuilder yuqingAnalyzeQuery(Long startTime, Long endTime, String projectId, String planId, String emotion, List<String> platforms, String aggTitle){
if (Objects.isNull(projectId)) {
projectId = UserThreadLocal.getProjectId();
......@@ -3155,6 +3250,71 @@ public class MarkDataServiceImpl implements MarkDataService {
}
return query;
}
private BoolQueryBuilder yuqingAnalyzeQuery(MarkSearchDTO dto){
return yuqingAnalyzeQuery(dto, null);
}
private BoolQueryBuilder yuqingAnalyzeQuery(MarkSearchDTO dto, String emotion){
// projectId、contendId
String projectId = dto.getProjectId();
String contendId = dto.getContendId();
// PostFilter 后置过滤器
// 新增channelEmotion
BoolQueryBuilder postFilter = EsQueryTools.assembleCacheMapsQuery(projectId, contendId, dto.getPoliticsLevel(), dto.getField(), dto.getRegion(), dto.getMainBodyType(), dto.getChannelEmotion());
if (Objects.nonNull(dto.getPlanId())){
postFilter = EsQueryTools.assembleCacheMapsPlanQuery(projectId, dto.getPlanId());
}
// time
postFilter.must(QueryBuilders.rangeQuery("time").gte(dto.getStartTime()).lt(dto.getEndTime()));
// platform
if (CollectionUtils.isNotEmpty(dto.getPlatforms())) {
postFilter.must(EsQueryTools.assemblePlatformQuery(Tools.getPlatformByIds(dto.getPlatforms())));
}
// tags
if (CollectionUtils.isNotEmpty(dto.getTags())) {
postFilter.must(EsQueryTools.assembleTagQuery(dto.getTags()));
}
// customTags
if (CollectionUtils.isNotEmpty(dto.getCustomTags())) {
postFilter.must(EsQueryTools.assembleTagQuery(dto.getCustomTags()));
}
// Query 查询条件
String[] fieldSearch = "仅标题".equals(dto.getSearchField()) ? new String[]{GenericAttribute.ES_IND_TITLE} : new String[]{GenericAttribute.ES_IND_FULL_TEXT};
// keyword
if (StringUtils.isNotEmpty(dto.getKeyword())) {
// Query
postFilter.must(EsQueryTools.assembleNormalKeywordQuery(dto.getKeyword(), fieldSearch));
}
// emotion
if (StringUtils.isNotEmpty(emotion) && !EmotionEnum.ALL.getName().equals(emotion)) {
postFilter.must(QueryBuilders.termQuery("brandkbs_mark_cache_maps.name.keyword", emotion));
}
// shieldWord
if (StringUtils.isNotEmpty(dto.getShieldWord())) {
postFilter.mustNot(EsQueryTools.assembleNormalKeywordQuery(dto.getShieldWord(), fieldSearch));
}
// sourceKeyword
if (StringUtils.isNotEmpty(dto.getSourceKeyword())) {
postFilter.must(EsQueryTools.assembleSourceQuery(dto.getSourceKeyword()));
}
// hostKeyword
if (StringUtils.isNotEmpty(dto.getHostKeyword())) {
postFilter.must(EsQueryTools.assembleFiledKeywordQuery("host", dto.getHostKeyword()));
}
// ip
if (StringUtils.isNotEmpty(dto.getIp())){
postFilter.must(EsQueryTools.assembleIpQuery(dto.getIp()));
}
// mgid限制 舆情分析页面滚动翻页用
if (Objects.nonNull(dto.getPageGid())){
postFilter.must(QueryBuilders.rangeQuery(GenericAttribute.ES_MGID).lt(dto.getPageGid()));
}
// 聚合标题
if (Objects.nonNull(dto.getAggTitle())){
postFilter.must(QueryBuilders.termQuery("agg_title.keyword", dto.getAggTitle()));
}
return postFilter;
}
/**
* 获取最大的gid
......
src/main/java/com/zhiwei/brandkbs2/service/impl/TaskServiceImpl.java
......@@ -384,14 +384,14 @@ public class TaskServiceImpl implements TaskService {
Long[] time = commonService.getTimeRangeMonth();
List<CompletableFuture<Object>> projectFutures = GlobalPojo.PROJECT_MAP.values().stream().map(project -> CompletableFuture.supplyAsync(() -> {
UserThreadLocal.set(new UserInfo().setProjectId(project.getId()));
markDataService.getHighWord(time[0], time[1], null, false);
markDataService.cacheHighWord(time[0], time[1], null, false);
log.info("项目:{}-{}-词云缓存已完成:{}个", project.getProjectName(), project.getId(), total.incrementAndGet());
return null;
}, cacheServiceExecutor)).collect(Collectors.toList());
AtomicInteger total2 = new AtomicInteger();
List<CompletableFuture<Object>> planFutures = nonManualProjectPlanDao.findList(new Query()).stream().map(plan -> CompletableFuture.supplyAsync(() -> {
UserThreadLocal.set(new UserInfo().setProjectId(plan.getProjectId()));
markDataService.getHighWord(time[0], time[1], plan.getId(), false);
markDataService.cacheHighWord(time[0], time[1], plan.getId(), false);
log.info("方案:{}-{}-词云缓存已完成:{}个", plan.getName(), plan.getId(), total2.incrementAndGet());
return null;
}, cacheServiceExecutor)).collect(Collectors.toList());
......
src/main/java/com/zhiwei/brandkbs2/util/Tools.java
......@@ -78,7 +78,7 @@ public class Tools {
/**
* 须过滤的标题
*/
public static final List<String> FILTER_TITLE = Arrays.asList("Notitle", "分享一篇文章", "暂无标题");
public static final List<String> FILTER_TITLE = Arrays.asList("Notitle", "分享一篇文章", "分享一篇文章。", "暂无标题");
/**
* 须过滤的渠道
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment