Skip to content

B
brandkbs2
Commit 053f2b38 by shenjunjie
Options

Merge branch 'feature' into 'dev' 

舆情库-舆情分析

See merge request !7
parents 54695e3d a5619849
Showing with 249 additions and 40 deletions
src/main/java/com/zhiwei/brandkbs2/common/RedisKeyPrefix.java
......@@ -15,8 +15,12 @@ public class RedisKeyPrefix {
private RedisKeyPrefix() {
}
public static final String MARK_ANALYZE_SUMMARY = "BRANDKBS:MARK:ANALYZE:SUMMARY:";
public static final String ARTICLE_SPREAD = "BRANDKBS:MARK:ANALYZE:SPREAD:";
public static final String MARK_ANALYZE_SUMMARY = "BRANDKBS:MARK:ANALYZE_SUMMARY:";
public static final String MARK_PLATFORM_PROPORTION = "BRANDKBS:MARK:PLATFORM_PROPORTION:";
public static final String MARK_SPREAD = "BRANDKBS:MARK:SPREAD:";
public static final String MARK_CHANNEL_PROPORTION = "BRANDKBS:MARK:CHANNEL_PROPORTION:";
public static final String MARK_EMOTION_PROPORTION = "BRANDKBS:MARK:EMOTION_PROPORTION:";
public static final String MARK_HIGH_WORD = "BRANDKBS:MARK:HIGH_WORD:";
private static final String SEPARATOR = ":";
......
src/main/java/com/zhiwei/brandkbs2/controller/AppArticleController.java
......@@ -19,38 +19,88 @@ import javax.annotation.Resource;
/**
* @author sjj
* @ClassName AppArticleController
* @Description 提供前台稿件相关信息展示
* @Description 提供前台舆情库相关信息展示
* @date 2022-06-20 09:27
*/
@RestController
@RequestMapping("/app/yuqing")
@Api(tags = "前台稿件展示接口", description = "提供前台稿件相关信息展示")
@Api(tags = "前台舆情库展示接口", description = "提供前台舆情库相关信息展示")
@Auth(role = RoleEnum.CUSTOMER)
public class AppArticleController extends BaseController {
@Resource(name = "markDataServiceImpl")
IMarkDataService markDataService;
@ApiOperation("舆情标注数据列表")
@GetMapping("/list/mark")
@ApiOperation("舆情列表")
@GetMapping("/mark/list")
public ResponseResult getYuqingMarkList(MarkSearchDTO markSearchDTO) {
return ResponseResult.success(markDataService.getYuqingMarkList(markSearchDTO));
}
@ApiOperation("舆情标注数据搜索条件")
@GetMapping("/list/mark/criteria")
@ApiOperation("舆情列表-搜索条件")
@GetMapping("/mark/list/criteria")
public ResponseResult getYuqingMark(@RequestParam(required = false) String linkedGroupId) {
return ResponseResult.success(markDataService.getYuqingMarkCriteria(linkedGroupId));
}
@ApiOperation("舆情标注数据提要")
@ApiOperation("舆情分析-提要")
@ApiImplicitParams({@ApiImplicitParam(name = "startTime", value = "开始时间", paramType = "query", dataType = "long"), @ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long")})
@GetMapping("/analyze/summary")
public ResponseResult getAnalyzeSummary(@RequestParam(value = "startTime", required = false) Long startTime, @RequestParam(value = "endTime", required = false) Long endTime) {
return ResponseResult.success(markDataService.getAnalyzeSummary(startTime, endTime, true));
}
@ApiOperation("舆情分析-传播趋势")
@ApiImplicitParams({
@ApiImplicitParam(name = "startTime", value = "开始时间", paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long")
})
@GetMapping("/analyze/summary")
public ResponseResult getArticleSummary(@RequestParam(value = "startTime", required = false) Long startTime,
@GetMapping("/spread")
public ResponseResult getMarkSpread(@RequestParam(value = "startTime", required = false) Long startTime,
@RequestParam(value = "endTime", required = false) Long endTime) {
return ResponseResult.success(markDataService.getMarkSpread(startTime, endTime, true));
}
@ApiOperation("舆情分析-情感分布")
@ApiImplicitParams({
@ApiImplicitParam(name = "startTime", value = "开始时间", paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long")
})
@GetMapping("/emotion/proportion")
public ResponseResult getMarkEmotionProportion(@RequestParam(value = "startTime", required = false) Long startTime,
@RequestParam(value = "endTime", required = false) Long endTime) {
return ResponseResult.success(markDataService.getMarkEmotionProportion(startTime, endTime, true));
}
@ApiOperation("舆情分析-平台分布")
@ApiImplicitParams({
@ApiImplicitParam(name = "startTime", value = "开始时间", paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long")
})
@GetMapping("/platform/proportion")
public ResponseResult getMarkPlatformProportion(@RequestParam(value = "startTime", required = false) Long startTime,
@RequestParam(value = "endTime", required = false) Long endTime) {
return ResponseResult.success(markDataService.getMarkPlatformProportion(startTime, endTime, true));
}
@ApiOperation("舆情分析-渠道分布")
@ApiImplicitParams({@ApiImplicitParam(name = "startTime", value = "开始时间", paramType = "query", dataType = "long"), @ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long")})
@GetMapping("/channel/proportion")
public ResponseResult getMarkChannelProportion(@RequestParam(value = "startTime", required = false) Long startTime, @RequestParam(value = "endTime", required = false) Long endTime) {
return ResponseResult.success(markDataService.getMarkChannelProportion(startTime, endTime, true));
}
@ApiOperation("舆情分析-高频词分布")
@ApiImplicitParams({
@ApiImplicitParam(name = "startTime", value = "开始时间", paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long")
})
@GetMapping("/highWord")
public ResponseResult getMarkHighWord(@RequestParam(value = "startTime", required = false) Long startTime,
@RequestParam(value = "endTime", required = false) Long endTime) {
return ResponseResult.success(markDataService.getAnalyzeSummary(startTime, endTime, true));
return ResponseResult.success(markDataService.getMarkHighWord(startTime, endTime));
}
}
src/main/java/com/zhiwei/brandkbs2/controller/CommonController.java
......@@ -4,6 +4,7 @@ import com.zhiwei.brandkbs2.auth.Auth;
import com.zhiwei.brandkbs2.enmus.RoleEnum;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.service.ICommonService;
import com.zhiwei.brandkbs2.service.IProjectService;
import com.zhiwei.middleware.mark.pojo.enums.TagField;
import com.zhiwei.middleware.mark.vo.MarkerTag;
import io.swagger.annotations.Api;
......@@ -34,6 +35,9 @@ public class CommonController extends BaseController {
@Resource(name = "commonServiceImpl")
ICommonService commonService;
@Resource(name = "projectServiceImpl")
IProjectService projectService;
@ApiOperation("获取情感倾向标签信息")
@ApiImplicitParam(name = "linkedGroupId", value = "关联项目ID", required = true, paramType = "path", dataType = "string")
@GetMapping("/get/tag/emotion/{linkedGroupId}")
......@@ -56,6 +60,13 @@ public class CommonController extends BaseController {
}
}
@ApiOperation("获取当前用户拥有的所有项目及品牌列表")
@GetMapping("/user/getUserAllProjects")
@Auth(role = RoleEnum.CUSTOMER)
public ResponseResult getUserAllProjects() {
return ResponseResult.success(projectService.getUserAllProjects());
}
@ApiOperation("测试接口")
@GetMapping("/test")
public ResponseResult test() {
......
src/main/java/com/zhiwei/brandkbs2/es/EsClientDao.java
......@@ -5,6 +5,8 @@ import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.pojo.ChannelIndex;
import com.zhiwei.brandkbs2.pojo.Event;
import com.zhiwei.brandkbs2.util.Tools;
import lombok.Getter;
import lombok.Setter;
import org.apache.commons.lang3.time.FastDateFormat;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
......@@ -226,12 +228,62 @@ public class EsClientDao {
// return builder;
// }
public SearchHits searchHits(String[] indexes, QueryBuilder postFilter, QueryBuilder query, FieldSortBuilder sort, int from, int size, HighlightBuilder highlighter) throws IOException {
return searchResponse(indexes, postFilter, query, null, sort, from, size, highlighter).getHits();
public SearchHits searchHits(String[] indexes, QueryBuilder postFilter, QueryBuilder query, String[] fetchSource, FieldSortBuilder sort, int from, int size, HighlightBuilder highlighter) throws IOException {
return searchResponse(indexes, postFilter, query, null, fetchSource, sort, from, size, highlighter).getHits();
}
public SearchHits searchHits(SearchHelper searchHelper) throws IOException {
return searchHits(searchRequest(searchHelper));
}
public SearchResponse searchResponse(String[] indexes, QueryBuilder postFilter, QueryBuilder query, AggregationBuilder aggregationBuilder,
FieldSortBuilder sort, int from, int size, HighlightBuilder highlighter) throws IOException {
String[] fetchSource, FieldSortBuilder sort, int from, int size, HighlightBuilder highlighter) throws IOException {
return searchResponse(searchRequest(indexes, postFilter, query, aggregationBuilder, fetchSource, sort, from, size, highlighter));
}
public SearchResponse searchResponse(SearchHelper searchHelper) throws IOException {
return searchResponse(searchRequest(searchHelper));
}
public List<SearchResponse> searchScrollResponse(SearchHelper searchHelper) throws IOException {
return retryTemplate.execute(context -> scrollSearch(searchRequest(searchHelper)));
}
public Long count(String[] indexes, QueryBuilder postFilter, QueryBuilder query) throws IOException {
CountRequest countRequest = new CountRequest();
BoolQueryBuilder countQuery = QueryBuilders.boolQuery();
if (!Objects.isNull(query)) {
countQuery.must(query);
}
if (!Objects.isNull(postFilter)) {
countQuery.must(postFilter);
}
countRequest.indices(indexes);
countRequest.query(countQuery);
CountResponse response = retryTemplate.execute(context -> esClient.count(countRequest, RequestOptions.DEFAULT));
return response.getCount();
}
public static SearchHelper createSearchHelper() {
return new SearchHelper();
}
private SearchHits searchHits(SearchRequest searchRequest) throws IOException {
return searchResponse(searchRequest).getHits();
}
private SearchResponse searchResponse(SearchRequest searchRequest) throws IOException {
return retryTemplate.execute(context -> esClient.search(searchRequest, RequestOptions.DEFAULT));
}
private SearchRequest searchRequest(SearchHelper searchHelper) {
defaultInit(searchHelper);
return searchRequest(searchHelper.getIndexes(), searchHelper.getPostFilter(), searchHelper.getQuery(), searchHelper.getAggregationBuilder(), searchHelper.getFetchSource(), searchHelper.getSort(), searchHelper.getFrom(), searchHelper.getSize(), searchHelper.getHighlighter());
}
private SearchRequest searchRequest(String[] indexes, QueryBuilder postFilter, QueryBuilder query, AggregationBuilder aggregationBuilder,
String[] fetchSource,
FieldSortBuilder sort, int from, int size, HighlightBuilder highlighter) {
SearchRequest searchRequest = new SearchRequest();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchRequest.indices(indexes);
......@@ -244,35 +296,69 @@ public class EsClientDao {
if (!Objects.isNull(aggregationBuilder)) {
searchSourceBuilder.aggregation(aggregationBuilder);
}
if (!Objects.isNull(fetchSource)) {
searchSourceBuilder.fetchSource(fetchSource, null);
}
if (!Objects.isNull(sort)) {
searchSourceBuilder.sort(sort);
}
if (from >= 0) {
searchSourceBuilder.from(from);
}
if (size >= 0) {
if (size == 0) {
searchSourceBuilder.size(10000);
} else if (size > 0) {
searchSourceBuilder.size(size);
}
if (!Objects.isNull(highlighter)) {
searchSourceBuilder.highlighter(highlighter);
}
searchRequest.source(searchSourceBuilder);
return retryTemplate.execute(context -> esClient.search(searchRequest, RequestOptions.DEFAULT));
return searchRequest;
}
public Long count(String[] indexes, QueryBuilder postFilter, QueryBuilder query) throws IOException {
CountRequest countRequest = new CountRequest();
BoolQueryBuilder countQuery = QueryBuilders.boolQuery();
if (!Objects.isNull(query)) {
countQuery.must(query);
private List<SearchResponse> scrollSearch(SearchRequest searchRequest) throws IOException {
List<SearchResponse> res = new ArrayList<>();
searchRequest.scroll(TIME_VALUE);
SearchResponse searchResponse = esClient.search(searchRequest, RequestOptions.DEFAULT);
String scrollId = searchResponse.getScrollId();
res.add(searchResponse);
while (Objects.nonNull(scrollId)) {
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(TIME_VALUE);
searchResponse = esClient.scroll(scrollRequest, RequestOptions.DEFAULT);
scrollId = searchResponse.getScrollId();
if (1 > searchResponse.getHits().getHits().length) {
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId);
esClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
scrollId = null;
} else {
res.add(searchResponse);
}
}
if (!Objects.isNull(postFilter)) {
countQuery.must(postFilter);
return res;
}
private void defaultInit(SearchHelper searchHelper) {
if (null == searchHelper.getIndexes()) {
searchHelper.setIndexes(EsClientDao.this.getIndexes());
}
countRequest.indices(indexes);
countRequest.query(countQuery);
CountResponse response = retryTemplate.execute(context -> esClient.count(countRequest, RequestOptions.DEFAULT));
return response.getCount();
}
@Setter
@Getter
public static class SearchHelper {
String[] indexes;
QueryBuilder postFilter;
QueryBuilder query;
AggregationBuilder aggregationBuilder;
String[] fetchSource;
FieldSortBuilder sort;
int from;
int size;
HighlightBuilder highlighter;
}
}
src/main/java/com/zhiwei/brandkbs2/pojo/BaseMap.java
......@@ -23,6 +23,8 @@ public class BaseMap {
*/
private String title;
private String incompleteContent;
/**
* content
*/
......@@ -58,4 +60,12 @@ public class BaseMap {
*/
private String emotion;
public String getTitleNullOptionalContent() {
if (null != title) {
return title;
} else {
return incompleteContent;
}
}
}
src/main/java/com/zhiwei/brandkbs2/pojo/Channel.java
......@@ -55,7 +55,7 @@ public class Channel extends ChannelIndex {
/**
* 渠道指数
*/
private Double EmotionIndex = 0d;
private Double emotionIndex = 0d;
/**
* 经验评级
......
src/main/java/com/zhiwei/brandkbs2/pojo/EventData.java
......@@ -91,7 +91,7 @@ public class EventData extends AbstractBaseMongo {
data.setRealSource(baseMap.getRealSource());
data.setSource(baseMap.getSource());
data.setUrl(baseMap.getUrl());
data.setTitle(baseMap.getTitle());
data.setTitle(baseMap.getTitleNullOptionalContent());
data.setAggTitle(Tools.filterSpecialCharacter(baseMap.getTitle()));
data.setContent(baseMap.getContent());
data.setEmotion(baseMap.getEmotion());
......
src/main/java/com/zhiwei/brandkbs2/pojo/dto/MarkSearchDTO.java
......@@ -95,7 +95,7 @@ public class MarkSearchDTO {
* 排序字段
*/
@ApiModelProperty(value = "排序字段")
private String sorter;
private String sorter = "{\"mtime\":\"descend\"}";
/**
* 是否聚合
......
src/main/java/com/zhiwei/brandkbs2/service/IMarkDataService.java
......@@ -5,6 +5,8 @@ import com.zhiwei.brandkbs2.pojo.MarkFlowEntity;
import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
import com.zhiwei.brandkbs2.pojo.vo.PageVO;
import java.util.List;
/**
* @ClassName: IMarkDataService
* @Description: 标注数据服务抽象类
......@@ -21,7 +23,6 @@ public interface IMarkDataService {
/**
* 舆情标注数据搜索条件
*
*/
JSONObject getYuqingMarkCriteria(String linkedGroup);
......@@ -30,8 +31,55 @@ public interface IMarkDataService {
*
* @param startTime 开始时间
* @param endTime 结束时间
* @param cache 是否启用缓存
* @return 舆情消息流提要信息
* @param cache 是否启用缓存
* @return 舆情标注数据提要信息
*/
JSONObject getAnalyzeSummary(Long startTime, Long endTime, boolean cache);
/**
* 获取舆情标注数据传播趋势
*
* @param startTime 开始时间
* @param endTime 结束时间
* @return 获取舆情标注数据传播趋势
*/
List<JSONObject> getMarkSpread(Long startTime, Long endTime, boolean cache);
/**
* 获取舆情标注数据舆情情感分布
*
* @param startTime 开始时间
* @param endTime 结束时间
* @return 获取舆情标注数据舆情情感分布
*/
JSONObject getMarkEmotionProportion(Long startTime, Long endTime, boolean cache);
/**
* 获取舆情标注数据舆情渠道分布
*
* @param startTime 开始时间
* @param endTime 结束时间
* @param cache 是否启用缓存
* @return 获取舆情标注数据舆情渠道分布
*/
List<JSONObject> getMarkChannelProportion(Long startTime, Long endTime, boolean cache);
/**
* 获取舆情标注数据舆情平台分布
*
* @param startTime 开始时间
* @param endTime 结束时间
* @return 获取舆情标注数据舆情平台分布
*/
List<JSONObject> getMarkPlatformProportion(Long startTime, Long endTime, boolean cache);
/**
* 获取舆情标注数据舆情高频词分布
*
* @param startTime 开始时间
* @param endTime 结束时间
* @return 获取舆情标注数据舆情高频词分布
*/
List<JSONObject> getMarkHighWord(Long startTime, Long endTime);
}
src/main/java/com/zhiwei/brandkbs2/service/impl/ChannelServiceImpl.java
......@@ -100,7 +100,7 @@ public class ChannelServiceImpl implements IChannelService {
JSONObject json = new JSONObject();
BaseMap baseMap = Tools.getBaseFromEsMap(sourceJson);
json.put("id", sourceJson.get("id"));
json.put("title", baseMap.getTitle());
json.put("title",baseMap.getTitleNullOptionalContent());
json.put("url", baseMap.getUrl());
json.put("time", new Date(baseMap.getTime()));
json.put("emotion", baseMap.getEmotion());
......
src/main/java/com/zhiwei/brandkbs2/util/Tools.java
......@@ -323,7 +323,7 @@ public class Tools {
case INCOMPLETE:
IncompleteText incompleteText = IncompleteText.restoreFromEs(map);
// 短文本使用content做title展示
res.setTitle(incompleteText.getContent());
res.setIncompleteContent(incompleteText.getContent());
res.setContent(incompleteText.getContent());
break;
case QA:
......
src/main/resources/application-dev.properties
......@@ -6,11 +6,11 @@ jwt.hour=120
brandkbs.file.url=/usr/local/sources/brandkbs2/files/
brandkbs.img.url=/usr/local/sources/brandkbs2/images/
brandkbs.head.url=/usr/local/sources/brandkbs2/head/
brandkbs.image.url=https://brandkbs2.test.zhiweidata.com/brandkbs2/images/
brandkbs.image.url=https://brandkbs2.test.zhiweidata.com/brandkbs/images/
#redis
spring.redis.host=192.168.0.39
spring.redis.database=7
spring.redis.database=12
spring.redis.port=7373
#mongo\u914D\u7F6E
......@@ -22,8 +22,8 @@ mongo.autoConnectRetry=true
mongo.socketKeepAlive=true
mongo.socketTimeout=120000
mongo.slaveOk=true
primary.uri=mongodb://brandkbs2:3vh65l$i6qQA@192.168.0.150:27017,192.168.0.151:27017,192.168.0.152:27017/qbjc?authSource=admin
secondary.uri=mongodb://qbjcuser:qbjc1q2w3e4r@192.168.0.150:27017,192.168.0.151:27017,192.168.0.152:27017/qbjc?authSource=admin
primary.uri=mongodb://brandkbs2:3vh65l$i6qQA@192.168.0.150:27017,192.168.0.151:27017,192.168.0.152:27017/brandkbs2?authSource=admin
secondary.uri=mongodb://qbjcuser:asSADf5ffs@192.168.0.150:27017,192.168.0.151:27017,192.168.0.152:27017/qbjc?authSource=admin
#es
es.clusterNodes=192.168.0.130:9200
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment