Merge branch 'feature' into 'dev'

舆情库-舆情分析

See merge request !7

src/main/java/com/zhiwei/brandkbs2/common/RedisKeyPrefix.java

@@ -15,8 +15,12 @@ public class RedisKeyPrefix {
     private RedisKeyPrefix() {
     }
 
-    public static final String MARK_ANALYZE_SUMMARY = "BRANDKBS:MARK:ANALYZE:SUMMARY:";
-    public static final String ARTICLE_SPREAD = "BRANDKBS:MARK:ANALYZE:SPREAD:";
+    public static final String MARK_ANALYZE_SUMMARY = "BRANDKBS:MARK:ANALYZE_SUMMARY:";
+    public static final String MARK_PLATFORM_PROPORTION = "BRANDKBS:MARK:PLATFORM_PROPORTION:";
+    public static final String MARK_SPREAD = "BRANDKBS:MARK:SPREAD:";
+    public static final String MARK_CHANNEL_PROPORTION = "BRANDKBS:MARK:CHANNEL_PROPORTION:";
+    public static final String MARK_EMOTION_PROPORTION = "BRANDKBS:MARK:EMOTION_PROPORTION:";
+    public static final String MARK_HIGH_WORD = "BRANDKBS:MARK:HIGH_WORD:";
 
     private static final String SEPARATOR = ":";
 

src/main/java/com/zhiwei/brandkbs2/controller/AppArticleController.java

@@ -19,38 +19,88 @@ import javax.annotation.Resource;
 /**
  * @author sjj
  * @ClassName AppArticleController
- * @Description 提供前台稿件相关信息展示
+ * @Description 提供前台舆情库相关信息展示
  * @date 2022-06-20 09:27
  */
 @RestController
 @RequestMapping("/app/yuqing")
-@Api(tags = "前台稿件展示接口", description = "提供前台稿件相关信息展示")
+@Api(tags = "前台舆情库展示接口", description = "提供前台舆情库相关信息展示")
 @Auth(role = RoleEnum.CUSTOMER)
 public class AppArticleController extends BaseController {
 
     @Resource(name = "markDataServiceImpl")
     IMarkDataService markDataService;
 
-    @ApiOperation("舆情标注数据列表")
-    @GetMapping("/list/mark")
+    @ApiOperation("舆情列表")
+    @GetMapping("/mark/list")
     public ResponseResult getYuqingMarkList(MarkSearchDTO markSearchDTO) {
         return ResponseResult.success(markDataService.getYuqingMarkList(markSearchDTO));
     }
 
-    @ApiOperation("舆情标注数据搜索条件")
-    @GetMapping("/list/mark/criteria")
+    @ApiOperation("舆情列表-搜索条件")
+    @GetMapping("/mark/list/criteria")
     public ResponseResult getYuqingMark(@RequestParam(required = false) String linkedGroupId) {
         return ResponseResult.success(markDataService.getYuqingMarkCriteria(linkedGroupId));
     }
 
-    @ApiOperation("舆情标注数据提要")
+    @ApiOperation("舆情分析-提要")
+    @ApiImplicitParams({@ApiImplicitParam(name = "startTime", value = "开始时间", paramType = "query", dataType = "long"), @ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long")})
+    @GetMapping("/analyze/summary")
+    public ResponseResult getAnalyzeSummary(@RequestParam(value = "startTime", required = false) Long startTime, @RequestParam(value = "endTime", required = false) Long endTime) {
+        return ResponseResult.success(markDataService.getAnalyzeSummary(startTime, endTime, true));
+    }
+
+    @ApiOperation("舆情分析-传播趋势")
     @ApiImplicitParams({
             @ApiImplicitParam(name = "startTime", value = "开始时间", paramType = "query", dataType = "long"),
             @ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long")
     })
-    @GetMapping("/analyze/summary")
-    public ResponseResult getArticleSummary(@RequestParam(value = "startTime", required = false) Long startTime,
+    @GetMapping("/spread")
+    public ResponseResult getMarkSpread(@RequestParam(value = "startTime", required = false) Long startTime,
+                                        @RequestParam(value = "endTime", required = false) Long endTime) {
+        return ResponseResult.success(markDataService.getMarkSpread(startTime, endTime, true));
+    }
+
+    @ApiOperation("舆情分析-情感分布")
+    @ApiImplicitParams({
+            @ApiImplicitParam(name = "startTime", value = "开始时间", paramType = "query", dataType = "long"),
+            @ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long")
+    })
+    @GetMapping("/emotion/proportion")
+    public ResponseResult getMarkEmotionProportion(@RequestParam(value = "startTime", required = false) Long startTime,
+                                                   @RequestParam(value = "endTime", required = false) Long endTime) {
+        return ResponseResult.success(markDataService.getMarkEmotionProportion(startTime, endTime, true));
+    }
+
+    @ApiOperation("舆情分析-平台分布")
+    @ApiImplicitParams({
+            @ApiImplicitParam(name = "startTime", value = "开始时间", paramType = "query", dataType = "long"),
+            @ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long")
+    })
+    @GetMapping("/platform/proportion")
+    public ResponseResult getMarkPlatformProportion(@RequestParam(value = "startTime", required = false) Long startTime,
+                                                    @RequestParam(value = "endTime", required = false) Long endTime) {
+        return ResponseResult.success(markDataService.getMarkPlatformProportion(startTime, endTime, true));
+    }
+
+    @ApiOperation("舆情分析-渠道分布")
+    @ApiImplicitParams({@ApiImplicitParam(name = "startTime", value = "开始时间", paramType = "query", dataType = "long"), @ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long")})
+    @GetMapping("/channel/proportion")
+    public ResponseResult getMarkChannelProportion(@RequestParam(value = "startTime", required = false) Long startTime, @RequestParam(value = "endTime", required = false) Long endTime) {
+        return ResponseResult.success(markDataService.getMarkChannelProportion(startTime, endTime, true));
+    }
+
+    @ApiOperation("舆情分析-高频词分布")
+    @ApiImplicitParams({
+            @ApiImplicitParam(name = "startTime", value = "开始时间", paramType = "query", dataType = "long"),
+            @ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long")
+    })
+    @GetMapping("/highWord")
+    public ResponseResult getMarkHighWord(@RequestParam(value = "startTime", required = false) Long startTime,
                                             @RequestParam(value = "endTime", required = false) Long endTime) {
-        return ResponseResult.success(markDataService.getAnalyzeSummary(startTime, endTime, true));
+        return ResponseResult.success(markDataService.getMarkHighWord(startTime, endTime));
     }
+
+
+
 }

src/main/java/com/zhiwei/brandkbs2/controller/CommonController.java

@@ -4,6 +4,7 @@ import com.zhiwei.brandkbs2.auth.Auth;
 import com.zhiwei.brandkbs2.enmus.RoleEnum;
 import com.zhiwei.brandkbs2.model.ResponseResult;
 import com.zhiwei.brandkbs2.service.ICommonService;
+import com.zhiwei.brandkbs2.service.IProjectService;
 import com.zhiwei.middleware.mark.pojo.enums.TagField;
 import com.zhiwei.middleware.mark.vo.MarkerTag;
 import io.swagger.annotations.Api;
@@ -34,6 +35,9 @@ public class CommonController extends BaseController {
     @Resource(name = "commonServiceImpl")
     ICommonService commonService;
 
+    @Resource(name = "projectServiceImpl")
+    IProjectService projectService;
+
     @ApiOperation("获取情感倾向标签信息")
     @ApiImplicitParam(name = "linkedGroupId", value = "关联项目ID", required = true, paramType = "path", dataType = "string")
     @GetMapping("/get/tag/emotion/{linkedGroupId}")
@@ -56,6 +60,13 @@ public class CommonController extends BaseController {
         }
     }
 
+    @ApiOperation("获取当前用户拥有的所有项目及品牌列表")
+    @GetMapping("/user/getUserAllProjects")
+    @Auth(role = RoleEnum.CUSTOMER)
+    public ResponseResult getUserAllProjects() {
+        return ResponseResult.success(projectService.getUserAllProjects());
+    }
+
     @ApiOperation("测试接口")
     @GetMapping("/test")
     public ResponseResult test() {

src/main/java/com/zhiwei/brandkbs2/es/EsClientDao.java

@@ -5,6 +5,8 @@ import com.zhiwei.brandkbs2.common.GenericAttribute;
 import com.zhiwei.brandkbs2.pojo.ChannelIndex;
 import com.zhiwei.brandkbs2.pojo.Event;
 import com.zhiwei.brandkbs2.util.Tools;
+import lombok.Getter;
+import lombok.Setter;
 import org.apache.commons.lang3.time.FastDateFormat;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -226,12 +228,62 @@ public class EsClientDao {
 //        return builder;
 //    }
 
-    public SearchHits searchHits(String[] indexes, QueryBuilder postFilter, QueryBuilder query, FieldSortBuilder sort, int from, int size, HighlightBuilder highlighter) throws IOException {
-        return searchResponse(indexes, postFilter, query, null, sort, from, size, highlighter).getHits();
+    public SearchHits searchHits(String[] indexes, QueryBuilder postFilter, QueryBuilder query, String[] fetchSource, FieldSortBuilder sort, int from, int size, HighlightBuilder highlighter) throws IOException {
+        return searchResponse(indexes, postFilter, query, null, fetchSource, sort, from, size, highlighter).getHits();
+    }
+
+    public SearchHits searchHits(SearchHelper searchHelper) throws IOException {
+        return searchHits(searchRequest(searchHelper));
     }
 
     public SearchResponse searchResponse(String[] indexes, QueryBuilder postFilter, QueryBuilder query, AggregationBuilder aggregationBuilder,
-                                         FieldSortBuilder sort, int from, int size, HighlightBuilder highlighter) throws IOException {
+                                         String[] fetchSource, FieldSortBuilder sort, int from, int size, HighlightBuilder highlighter) throws IOException {
+        return searchResponse(searchRequest(indexes, postFilter, query, aggregationBuilder, fetchSource, sort, from, size, highlighter));
+    }
+
+    public SearchResponse searchResponse(SearchHelper searchHelper) throws IOException {
+        return searchResponse(searchRequest(searchHelper));
+    }
+
+    public List<SearchResponse> searchScrollResponse(SearchHelper searchHelper) throws IOException {
+        return retryTemplate.execute(context -> scrollSearch(searchRequest(searchHelper)));
+    }
+
+    public Long count(String[] indexes, QueryBuilder postFilter, QueryBuilder query) throws IOException {
+        CountRequest countRequest = new CountRequest();
+        BoolQueryBuilder countQuery = QueryBuilders.boolQuery();
+        if (!Objects.isNull(query)) {
+            countQuery.must(query);
+        }
+        if (!Objects.isNull(postFilter)) {
+            countQuery.must(postFilter);
+        }
+        countRequest.indices(indexes);
+        countRequest.query(countQuery);
+        CountResponse response = retryTemplate.execute(context -> esClient.count(countRequest, RequestOptions.DEFAULT));
+        return response.getCount();
+    }
+
+    public static SearchHelper createSearchHelper() {
+        return new SearchHelper();
+    }
+
+    private SearchHits searchHits(SearchRequest searchRequest) throws IOException {
+        return searchResponse(searchRequest).getHits();
+    }
+
+    private SearchResponse searchResponse(SearchRequest searchRequest) throws IOException {
+        return retryTemplate.execute(context -> esClient.search(searchRequest, RequestOptions.DEFAULT));
+    }
+
+    private SearchRequest searchRequest(SearchHelper searchHelper) {
+        defaultInit(searchHelper);
+        return searchRequest(searchHelper.getIndexes(), searchHelper.getPostFilter(), searchHelper.getQuery(), searchHelper.getAggregationBuilder(), searchHelper.getFetchSource(), searchHelper.getSort(), searchHelper.getFrom(), searchHelper.getSize(), searchHelper.getHighlighter());
+    }
+
+    private SearchRequest searchRequest(String[] indexes, QueryBuilder postFilter, QueryBuilder query, AggregationBuilder aggregationBuilder,
+                                        String[] fetchSource,
+                                        FieldSortBuilder sort, int from, int size, HighlightBuilder highlighter) {
         SearchRequest searchRequest = new SearchRequest();
         SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
         searchRequest.indices(indexes);
@@ -244,35 +296,69 @@ public class EsClientDao {
         if (!Objects.isNull(aggregationBuilder)) {
             searchSourceBuilder.aggregation(aggregationBuilder);
         }
+        if (!Objects.isNull(fetchSource)) {
+            searchSourceBuilder.fetchSource(fetchSource, null);
+        }
         if (!Objects.isNull(sort)) {
             searchSourceBuilder.sort(sort);
         }
         if (from >= 0) {
             searchSourceBuilder.from(from);
         }
-        if (size >= 0) {
+        if (size == 0) {
+            searchSourceBuilder.size(10000);
+        } else if (size > 0) {
             searchSourceBuilder.size(size);
         }
         if (!Objects.isNull(highlighter)) {
             searchSourceBuilder.highlighter(highlighter);
         }
         searchRequest.source(searchSourceBuilder);
-        return retryTemplate.execute(context -> esClient.search(searchRequest, RequestOptions.DEFAULT));
+        return searchRequest;
     }
 
-    public Long count(String[] indexes, QueryBuilder postFilter, QueryBuilder query) throws IOException {
-        CountRequest countRequest = new CountRequest();
-        BoolQueryBuilder countQuery = QueryBuilders.boolQuery();
-        if (!Objects.isNull(query)) {
-            countQuery.must(query);
+    private List<SearchResponse> scrollSearch(SearchRequest searchRequest) throws IOException {
+        List<SearchResponse> res = new ArrayList<>();
+        searchRequest.scroll(TIME_VALUE);
+        SearchResponse searchResponse = esClient.search(searchRequest, RequestOptions.DEFAULT);
+        String scrollId = searchResponse.getScrollId();
+        res.add(searchResponse);
+        while (Objects.nonNull(scrollId)) {
+            SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
+            scrollRequest.scroll(TIME_VALUE);
+            searchResponse = esClient.scroll(scrollRequest, RequestOptions.DEFAULT);
+            scrollId = searchResponse.getScrollId();
+            if (1 > searchResponse.getHits().getHits().length) {
+                ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
+                clearScrollRequest.addScrollId(scrollId);
+                esClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
+                scrollId = null;
+            } else {
+                res.add(searchResponse);
+            }
         }
-        if (!Objects.isNull(postFilter)) {
-            countQuery.must(postFilter);
+        return res;
+    }
+
+    private void defaultInit(SearchHelper searchHelper) {
+        if (null == searchHelper.getIndexes()) {
+            searchHelper.setIndexes(EsClientDao.this.getIndexes());
         }
-        countRequest.indices(indexes);
-        countRequest.query(countQuery);
-        CountResponse response = retryTemplate.execute(context -> esClient.count(countRequest, RequestOptions.DEFAULT));
-        return response.getCount();
+    }
+
+
+    @Setter
+    @Getter
+    public static class SearchHelper {
+        String[] indexes;
+        QueryBuilder postFilter;
+        QueryBuilder query;
+        AggregationBuilder aggregationBuilder;
+        String[] fetchSource;
+        FieldSortBuilder sort;
+        int from;
+        int size;
+        HighlightBuilder highlighter;
     }
 
 }

src/main/java/com/zhiwei/brandkbs2/pojo/BaseMap.java

@@ -23,6 +23,8 @@ public class BaseMap {
      */
     private String title;
 
+    private String incompleteContent;
+
     /**
      * content
      */
@@ -58,4 +60,12 @@ public class BaseMap {
      */
     private String emotion;
 
+    public String getTitleNullOptionalContent() {
+        if (null != title) {
+            return title;
+        } else {
+            return incompleteContent;
+        }
+    }
+
 }

src/main/java/com/zhiwei/brandkbs2/pojo/Channel.java

@@ -55,7 +55,7 @@ public class Channel extends ChannelIndex {
     /**
      * 渠道指数
      */
-    private Double EmotionIndex = 0d;
+    private Double emotionIndex = 0d;
 
     /**
      * 经验评级

src/main/java/com/zhiwei/brandkbs2/pojo/EventData.java

@@ -91,7 +91,7 @@ public class EventData extends AbstractBaseMongo {
         data.setRealSource(baseMap.getRealSource());
         data.setSource(baseMap.getSource());
         data.setUrl(baseMap.getUrl());
-        data.setTitle(baseMap.getTitle());
+        data.setTitle(baseMap.getTitleNullOptionalContent());
         data.setAggTitle(Tools.filterSpecialCharacter(baseMap.getTitle()));
         data.setContent(baseMap.getContent());
         data.setEmotion(baseMap.getEmotion());

src/main/java/com/zhiwei/brandkbs2/pojo/dto/MarkSearchDTO.java

@@ -95,7 +95,7 @@ public class MarkSearchDTO {
      * 排序字段
      */
     @ApiModelProperty(value = "排序字段")
-    private String sorter;
+    private String sorter = "{\"mtime\":\"descend\"}";
 
     /**
      * 是否聚合

src/main/java/com/zhiwei/brandkbs2/service/IMarkDataService.java

@@ -5,6 +5,8 @@ import com.zhiwei.brandkbs2.pojo.MarkFlowEntity;
 import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
 import com.zhiwei.brandkbs2.pojo.vo.PageVO;
 
+import java.util.List;
+
 /**
  * @ClassName: IMarkDataService
  * @Description: 标注数据服务抽象类
@@ -21,7 +23,6 @@ public interface IMarkDataService {
 
     /**
      * 舆情标注数据搜索条件
-     *
      */
     JSONObject getYuqingMarkCriteria(String linkedGroup);
 
@@ -30,8 +31,55 @@ public interface IMarkDataService {
      *
      * @param startTime 开始时间
      * @param endTime   结束时间
-     * @param cache 是否启用缓存
-     * @return 舆情消息流提要信息
+     * @param cache     是否启用缓存
+     * @return 舆情标注数据提要信息
      */
     JSONObject getAnalyzeSummary(Long startTime, Long endTime, boolean cache);
+
+    /**
+     * 获取舆情标注数据传播趋势
+     *
+     * @param startTime 开始时间
+     * @param endTime   结束时间
+     * @return 获取舆情标注数据传播趋势
+     */
+    List<JSONObject> getMarkSpread(Long startTime, Long endTime, boolean cache);
+
+    /**
+     * 获取舆情标注数据舆情情感分布
+     *
+     * @param startTime 开始时间
+     * @param endTime   结束时间
+     * @return 获取舆情标注数据舆情情感分布
+     */
+    JSONObject getMarkEmotionProportion(Long startTime, Long endTime, boolean cache);
+
+    /**
+     * 获取舆情标注数据舆情渠道分布
+     *
+     * @param startTime 开始时间
+     * @param endTime   结束时间
+     * @param cache     是否启用缓存
+     * @return 获取舆情标注数据舆情渠道分布
+     */
+    List<JSONObject> getMarkChannelProportion(Long startTime, Long endTime, boolean cache);
+
+    /**
+     * 获取舆情标注数据舆情平台分布
+     *
+     * @param startTime 开始时间
+     * @param endTime   结束时间
+     * @return 获取舆情标注数据舆情平台分布
+     */
+    List<JSONObject> getMarkPlatformProportion(Long startTime, Long endTime, boolean cache);
+
+    /**
+     * 获取舆情标注数据舆情高频词分布
+     *
+     * @param startTime 开始时间
+     * @param endTime   结束时间
+     * @return 获取舆情标注数据舆情高频词分布
+     */
+    List<JSONObject> getMarkHighWord(Long startTime, Long endTime);
+
 }

src/main/java/com/zhiwei/brandkbs2/service/impl/ChannelServiceImpl.java

@@ -100,7 +100,7 @@ public class ChannelServiceImpl implements IChannelService {
                 JSONObject json = new JSONObject();
                 BaseMap baseMap = Tools.getBaseFromEsMap(sourceJson);
                 json.put("id", sourceJson.get("id"));
-                json.put("title", baseMap.getTitle());
+                json.put("title",baseMap.getTitleNullOptionalContent());
                 json.put("url", baseMap.getUrl());
                 json.put("time", new Date(baseMap.getTime()));
                 json.put("emotion", baseMap.getEmotion());

src/main/java/com/zhiwei/brandkbs2/util/Tools.java

@@ -323,7 +323,7 @@ public class Tools {
             case INCOMPLETE:
                 IncompleteText incompleteText = IncompleteText.restoreFromEs(map);
                 // 短文本使用content做title展示
-                res.setTitle(incompleteText.getContent());
+                res.setIncompleteContent(incompleteText.getContent());
                 res.setContent(incompleteText.getContent());
                 break;
             case QA:

src/main/resources/application-dev.properties

@@ -6,11 +6,11 @@ jwt.hour=120
 brandkbs.file.url=/usr/local/sources/brandkbs2/files/
 brandkbs.img.url=/usr/local/sources/brandkbs2/images/
 brandkbs.head.url=/usr/local/sources/brandkbs2/head/
-brandkbs.image.url=https://brandkbs2.test.zhiweidata.com/brandkbs2/images/
+brandkbs.image.url=https://brandkbs2.test.zhiweidata.com/brandkbs/images/
 
 #redis
 spring.redis.host=192.168.0.39
-spring.redis.database=7
+spring.redis.database=12
 spring.redis.port=7373
 
 #mongo\u914D\u7F6E
@@ -22,8 +22,8 @@ mongo.autoConnectRetry=true
 mongo.socketKeepAlive=true
 mongo.socketTimeout=120000
 mongo.slaveOk=true
-primary.uri=mongodb://brandkbs2:3vh65l$i6qQA@192.168.0.150:27017,192.168.0.151:27017,192.168.0.152:27017/qbjc?authSource=admin
-secondary.uri=mongodb://qbjcuser:qbjc1q2w3e4r@192.168.0.150:27017,192.168.0.151:27017,192.168.0.152:27017/qbjc?authSource=admin
+primary.uri=mongodb://brandkbs2:3vh65l$i6qQA@192.168.0.150:27017,192.168.0.151:27017,192.168.0.152:27017/brandkbs2?authSource=admin
+secondary.uri=mongodb://qbjcuser:asSADf5ffs@192.168.0.150:27017,192.168.0.151:27017,192.168.0.152:27017/qbjc?authSource=admin
 
 #es
 es.clusterNodes=192.168.0.130:9200