Merge branch 'release' into 'master'

Release

See merge request !479

src/main/java/com/zhiwei/brandkbs2/easyexcel/listener/BytedanceEventListener.java

@@ -36,10 +36,10 @@ public class BytedanceEventListener extends AnalysisEventListener<UploadBytedanc
         taskData.setEventName(dto.getEventName());
         // 填 "/" 当作0处理
         taskData.setReading(Objects.isNull(dto.getReading()) || Objects.equals("/", dto.getReading()) ? 0 : Integer.parseInt(dto.getReading()));
-        taskData.setRead(Objects.isNull(dto.getRead()) || Objects.equals("/", dto.getRead())  ? 0 : Integer.parseInt(dto.getRead()));
-        taskData.setLike(Objects.isNull(dto.getLike()) || Objects.equals("/", dto.getLike())  ? 0 : Integer.parseInt(dto.getLike()));
-        taskData.setComment(Objects.isNull(dto.getComment()) || Objects.equals("/", dto.getComment())  ? 0 : Integer.parseInt(dto.getComment()));
-        taskData.setForward(Objects.isNull(dto.getForward()) || Objects.equals("/", dto.getForward())  ? 0 : Integer.parseInt(dto.getForward()));
+        taskData.setRead(Objects.isNull(dto.getRead()) || Objects.equals("/", dto.getRead()) ? 0 : Integer.parseInt(dto.getRead()));
+        taskData.setLike(Objects.isNull(dto.getLike()) || Objects.equals("/", dto.getLike()) ? 0 : Integer.parseInt(dto.getLike()));
+        taskData.setComment(Objects.isNull(dto.getComment()) || Objects.equals("/", dto.getComment()) ? 0 : Integer.parseInt(dto.getComment()));
+        taskData.setForward(Objects.isNull(dto.getForward()) || Objects.equals("/", dto.getForward()) ? 0 : Integer.parseInt(dto.getForward()));
         data.add(taskData);
     }
 

src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java

@@ -828,7 +828,7 @@ public class MarkDataServiceImpl implements MarkDataService {
     public List<Map.Entry<String, Integer>> getMarkTopTitle(Long startTime, Long endTime, String emotion, String projectId, String contendId, int size, boolean include) throws IOException {
         // 索引
         String[] indexes = esClientDao.getIndexes();
-        TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("titles").field("agg_title.keyword").size(size + 1);
+        TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("titles").field("agg_title.keyword").size(size + Tools.FILTER_TITLE.size());
         // query
         BoolQueryBuilder query;
         if (include) {
@@ -848,8 +848,8 @@ public class MarkDataServiceImpl implements MarkDataService {
         ParsedStringTerms teamAgg = (ParsedStringTerms) aggMap.get("titles");
         for (Terms.Bucket bucket : teamAgg.getBuckets()) {
             String title = bucket.getKeyAsString();
-            // 过滤 “分享一篇文章” 的标题
-            if ("分享一篇文章".equals(title)) {
+            // 过滤 “分享一篇文章”，”暂无标题“，”Notitle“ 的标题
+            if (Tools.filterUselessTitle(title)) {
                 continue;
             }
             int num = (int) bucket.getDocCount();
@@ -861,7 +861,7 @@ public class MarkDataServiceImpl implements MarkDataService {
     private List<JSONObject> getMarkTopTitle(Long startTime, Long endTime, String emotion, String projectId, String contendId, String planId, int size) throws IOException {
         // 索引
         String[] indexes = esClientDao.getIndexes();
-        TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("titles").field("agg_title.keyword").size(size + 1);
+        TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("titles").field("agg_title.keyword").size(size + Tools.FILTER_TITLE.size());
         TermsAggregationBuilder sourceAggregationBuilder = AggregationBuilders.terms("source").field("source").size(10000);
         // query
         BoolQueryBuilder query;
@@ -890,8 +890,8 @@ public class MarkDataServiceImpl implements MarkDataService {
         for (Terms.Bucket bucket : teamAgg.getBuckets()) {
             JSONObject jsonObject = new JSONObject();
             String title = bucket.getKeyAsString();
-            // 过滤 “分享一篇文章” 的标题
-            if ("分享一篇文章".equals(title)) {
+            // 过滤 “分享一篇文章”，”暂无标题“，”Notitle“ 的标题
+            if (Tools.filterUselessTitle(title)) {
                 continue;
             }
             jsonObject.put("title", title);
@@ -1720,7 +1720,9 @@ public class MarkDataServiceImpl implements MarkDataService {
                     result.put("source", channel.getSource());
                     result.put("platform", channel.getPlatform());
                     result.put("num", bucket.getDocCount());
-                    channelResultList.add(result);
+                    if (!Tools.filterUselessSource(channel.getSource())){
+                        channelResultList.add(result);
+                    }
                 }
             }
         });
@@ -2713,7 +2715,7 @@ public class MarkDataServiceImpl implements MarkDataService {
             // query
             BoolQueryBuilder query = yuqingAnalyzeQuery(startTime, endTime, planId, null, null);
             // 渠道聚合，取前20
-            TermsAggregationBuilder sourceAggregationBuilder = AggregationBuilders.terms("source_count").field("source").order(BucketOrder.count(false)).size(20);
+            TermsAggregationBuilder sourceAggregationBuilder = AggregationBuilders.terms("source_count").field("source").order(BucketOrder.count(false)).size(20 + Tools.FILTER_SOURCE.size());
             // 情感倾向子聚合
             TermsAggregationBuilder emotionAggregationBuilder = AggregationBuilders.terms("emotion_count").field("brandkbs_mark_cache_maps.name.keyword");
             SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, query,
@@ -2721,15 +2723,20 @@ public class MarkDataServiceImpl implements MarkDataService {
             Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
             ParsedStringTerms sourceCountTeam = (ParsedStringTerms) aggMap.get("source_count");
             List<? extends Terms.Bucket> buckets = sourceCountTeam.getBuckets();
-            buckets.forEach(bucket -> {
+            for (Terms.Bucket bucket : buckets) {
                 JSONObject jsonObject = new JSONObject();
                 Map<String, Aggregation> map = bucket.getAggregations().asMap();
-                ParsedStringTerms countTeam = (ParsedStringTerms) map.get("emotion_count");
-                List<? extends Terms.Bucket> bucketList = countTeam.getBuckets();
-                jsonObject.put("source", bucket.getKeyAsString());
+                String source = bucket.getKeyAsString();
+                // 过滤渠道名：”匿名“
+                if (Tools.filterUselessSource(source)){
+                    continue;
+                }
+                jsonObject.put("source", source);
                 // 发文次数
                 jsonObject.put("count", bucket.getDocCount());
                 // 负面发文次数
+                ParsedStringTerms countTeam = (ParsedStringTerms) map.get("emotion_count");
+                List<? extends Terms.Bucket> bucketList = countTeam.getBuckets();
                 AtomicLong negativeCount = new AtomicLong();
                 bucketList.forEach(data -> {
                     if (Objects.equals(data.getKeyAsString(), EmotionEnum.NEGATIVE.getName())) {
@@ -2738,7 +2745,7 @@ public class MarkDataServiceImpl implements MarkDataService {
                 });
                 jsonObject.put("negativeCount", negativeCount.get());
                 list.add(jsonObject);
-            });
+            }
         }catch (Exception e){
             ExceptionCast.cast(CommonCodeEnum.FAIL, "新舆情分析getActiveChannels异常-", e);
         }

src/main/java/com/zhiwei/brandkbs2/util/Tools.java

@@ -52,7 +52,7 @@ import java.util.stream.Stream;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
 
-import static com.zhiwei.brandkbs2.common.GenericAttribute.ES_MARK_CACHE_MAPS;
+import static com.zhiwei.brandkbs2.common.GenericAttribute.*;
 import static com.zhiwei.brandkbs2.config.Constant.*;
 import static java.util.Objects.nonNull;
 
@@ -75,6 +75,16 @@ public class Tools {
 
     private static final DozerBeanMapper DOZER_BEAN_MAPPER = new DozerBeanMapper();
 
+    /**
+     * 须过滤的标题
+     */
+    public static final List<String> FILTER_TITLE = Arrays.asList("Notitle", "分享一篇文章", "暂无标题");
+
+    /**
+     * 须过滤的渠道
+     */
+    public static final List<String> FILTER_SOURCE = Arrays.asList("匿名");
+
 
     /**
      * 上传文件
@@ -1345,4 +1355,22 @@ public class Tools {
         }
         return Arrays.stream(str.split("[\\u007c]+")).filter(StringUtils::isNotBlank).collect(Collectors.joining("|"));
     }
+
+    /**
+     * 过滤 “分享一篇文章”，”暂无标题“，”Notitle“ 的无用标题
+     * @param title
+     * @return
+     */
+    public static Boolean filterUselessTitle(String title){
+        return FILTER_TITLE.contains(title);
+    }
+
+    /**
+     * 过滤 “匿名“ 的无用渠道
+     * @param source
+     * @return
+     */
+    public static Boolean filterUselessSource(String source){
+        return FILTER_SOURCE.contains(source);
+    }
 }
\ No newline at end of file