原发追溯

src/main/java/com/zhiwei/brandkbs2/common/GenericAttribute.java

@@ -22,6 +22,11 @@ public class GenericAttribute {
 //    public static final String ES_INDEX_TEST = "brandkbs2_2022";
     public static final String ES_CHANNEL_INDEX_TEST = "brandkbs2_channel_record_test";
     public static final String ES_CHANNEL_INDEX_PREFIX = "brandkbs2_channel_record_";
+
+    public static final String ES_COMPLETE_TEXT_INDEX_PRE = "complete_text_";
+    public static final String ES_INCOMPLETE_TEXT_INDEX_PRE = "incomplete_text_";
+    public static final String ES_QA_TEXT_INDEX_PRE = "qa_text_";
+    public static final String ES_VIDEO_INDEX_PRE = "video_";
     /**
      * es ind_title
      **/

src/main/java/com/zhiwei/brandkbs2/config/Constant.java

@@ -34,6 +34,8 @@ public class Constant {
     public static final FastDateFormat ONLY_MONTH_FORMAT = FastDateFormat.getInstance("M月");
     public static final FastDateFormat DF_yyyyMMdd = FastDateFormat.getInstance("yyyyMMdd");
 
+    public static final FastDateFormat DF_yyyyMM = FastDateFormat.getInstance("yyyyMM");
+
     /**
      * 自定义fid分隔符号
      */

src/main/java/com/zhiwei/brandkbs2/controller/app/AppArticleController.java

@@ -553,6 +553,36 @@ public class AppArticleController extends BaseController {
         return ResponseResult.success(markDataService.getDailyReportDetail(id));
     }
 
+    @ApiOperation("原发溯源-首发信息")
+    @PostMapping("/search-whole/origin/first-article")
+    public ResponseResult getWholeSearchFirstArticle(@RequestBody MarkSearchDTO dto) {
+        return ResponseResult.success(markDataService.getWholeSearchFirstArticle(dto));
+    }
+
+    @ApiOperation("原发溯源-媒体参与")
+    @PostMapping("/search-whole/origin/media-info")
+    public ResponseResult getWholeSearchMediaParticipation(@RequestBody MarkSearchDTO dto) {
+        return ResponseResult.success(markDataService.getWholeSearchMediaParticipation(dto));
+    }
+
+    @ApiOperation("原发溯源-发布节点")
+    @PostMapping("/search-whole/origin/article-point")
+    public ResponseResult getWholeSearchArticlePoint(@RequestBody MarkSearchDTO dto) {
+        return ResponseResult.success(markDataService.getWholeSearchArticlePoint(dto));
+    }
+
+    @ApiOperation("原发溯源-平台分布")
+    @PostMapping("/search-whole/origin/platform-percent")
+    public ResponseResult getWholeSearchPlatformPercentage(@RequestBody MarkSearchDTO dto) {
+        return ResponseResult.success(markDataService.getWholeSearchPlatformPercentage(dto));
+    }
+
+    @ApiOperation("原发溯源-发文列表")
+    @PostMapping("/search-whole/origin/articles")
+    public ResponseResult getWholeSearchArticleList(@RequestBody MarkSearchDTO dto) {
+        return ResponseResult.success(markDataService.getWholeSearchArticleList(dto));
+    }
+
     private boolean checkMTagIllegal(StringBuilder mtag) {
         List<MarkerTag> hitTags = projectService.getProjectById(UserThreadLocal.getProjectId()).getHitTags();
         if (!Tools.isEmpty(hitTags)) {

src/main/java/com/zhiwei/brandkbs2/es/EsClientDao.java

@@ -7,6 +7,7 @@ import com.zhiwei.brandkbs2.pojo.ChannelIndex;
 import com.zhiwei.brandkbs2.util.Tools;
 import lombok.Getter;
 import lombok.Setter;
+import org.apache.commons.lang3.time.DateUtils;
 import org.apache.commons.lang3.time.FastDateFormat;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.logging.log4j.LogManager;
@@ -29,6 +30,8 @@ import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.sort.FieldSortBuilder;
+import org.joda.time.Period;
+import org.joda.time.PeriodType;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.retry.support.RetryTemplate;
 import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
@@ -195,6 +198,10 @@ public class EsClientDao {
         return getIndexList().toArray(new String[0]);
     }
 
+    public String[] getAllIndexes(Long startTime, Long endTime) {
+        return getAllIndexList(startTime, endTime).toArray(new String[0]);
+    }
+
     protected RestHighLevelClient getEsClient() {
         return esClient;
     }
@@ -214,6 +221,25 @@ public class EsClientDao {
         return res;
     }
 
+    public List<String> getAllIndexList(Long start, Long end) {
+        List<String> list = new ArrayList<>();
+        Date endTime = new Date(end);
+        Date startTime = new Date(start);
+        String pattern = "yyyyMM";
+        startTime = Tools.truncDate(startTime, pattern);
+        endTime = Tools.truncDate(endTime, pattern);
+        Period period = new Period(startTime.getTime(), endTime.getTime(), PeriodType.months());
+        int mouths = period.getMonths();
+        for (int i = 0; i <= mouths; i++) {
+            String suffix = Constant.DF_yyyyMM.format(DateUtils.addMonths(startTime, i));
+            list.add(GenericAttribute.ES_COMPLETE_TEXT_INDEX_PRE + suffix);
+            list.add(GenericAttribute.ES_INCOMPLETE_TEXT_INDEX_PRE + suffix);
+            list.add(GenericAttribute.ES_QA_TEXT_INDEX_PRE + suffix);
+            list.add(GenericAttribute.ES_VIDEO_INDEX_PRE + suffix);
+        }
+        return list;
+    }
+
 //    private SearchSourceBuilder addSort(SearchSourceBuilder builder, String sorter) {
 //        if (StringUtils.isNotEmpty(sorter)) {
 //            for (Map.Entry<String, Object> entry : JSONObject.parseObject(sorter).entrySet()) {

src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java

@@ -715,4 +715,39 @@ public interface MarkDataService {
      * @return
      */
     DailyReport getDailyReportDetail(String id);
+
+    /**
+     * 获取原发溯源-首发信息
+     * @param dto 全网搜数据搜索传输类
+     * @return
+     */
+    MarkFlowEntity getWholeSearchFirstArticle(MarkSearchDTO dto);
+
+    /**
+     * 获取原发溯源-平台分布
+     * @param dto 全网搜数据搜索传输类
+     * @return
+     */
+    List<JSONObject> getWholeSearchMediaParticipation(MarkSearchDTO dto);
+
+    /**
+     * 获取原发溯源-发布节点
+     * @param dto 全网搜数据搜索传输类
+     * @return
+     */
+    List<JSONObject> getWholeSearchArticlePoint(MarkSearchDTO dto);
+
+    /**
+     * 获取原发溯源-平台分布
+     * @param dto 全网搜数据搜索传输类
+     * @return
+     */
+    List<JSONObject> getWholeSearchPlatformPercentage(MarkSearchDTO dto);
+
+    /**
+     * 获取原发溯源-发文列表
+     * @param dto 全网搜数据搜索传输类
+     * @return
+     */
+    PageVO<JSONObject> getWholeSearchArticleList(MarkSearchDTO dto);
 }

src/main/java/com/zhiwei/brandkbs2/service/MarkFlowService.java

@@ -3,6 +3,7 @@ package com.zhiwei.brandkbs2.service;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.base.entity.subclass.mark.MarkInfo;
 import com.zhiwei.brandkbs2.pojo.MarkFlowEntity;
+import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
 import com.zhiwei.brandkbs2.pojo.dto.TagFilterDTO;
 import lombok.AllArgsConstructor;
 import lombok.Data;
@@ -74,6 +75,14 @@ public interface MarkFlowService {
      */
     MarkFlowEntity getShotPageFromCache(String id);
 
+    /**
+     * 获取大库数据实体
+     * @param json
+     * @param dto
+     * @return
+     */
+    MarkFlowEntity createAllIndexFlowInfo(JSONObject json, MarkSearchDTO dto);
+
     @Data
     @AllArgsConstructor
     class MarkInfoSource {

src/main/java/com/zhiwei/brandkbs2/service/impl/MarkFlowServiceImpl.java

@@ -10,14 +10,14 @@ import com.zhiwei.brandkbs2.auth.UserThreadLocal;
 import com.zhiwei.brandkbs2.common.GenericAttribute;
 import com.zhiwei.brandkbs2.common.GlobalPojo;
 import com.zhiwei.brandkbs2.config.Constant;
-import com.zhiwei.brandkbs2.dao.HighlightWordDao;
 import com.zhiwei.brandkbs2.enmus.ChannelEmotion;
 import com.zhiwei.brandkbs2.exception.ExceptionCast;
 import com.zhiwei.brandkbs2.model.CommonCodeEnum;
 import com.zhiwei.brandkbs2.pojo.MarkFlowEntity;
+import com.zhiwei.brandkbs2.pojo.SensitiveChannel;
+import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
 import com.zhiwei.brandkbs2.pojo.dto.TagFilterDTO;
 import com.zhiwei.brandkbs2.service.MarkFlowService;
-import com.zhiwei.brandkbs2.service.TagFilterService;
 import com.zhiwei.brandkbs2.util.RedisUtil;
 import com.zhiwei.brandkbs2.util.Tools;
 import com.zhiwei.middleware.automaticmark.vo.Keyword;
@@ -27,6 +27,7 @@ import com.zhiwei.middleware.mark.vo.QueryResult;
 import com.zhiwei.qbjc.bean.pojo.common.Tag;
 import com.zhiwei.qbjc.bean.tools.BeanTools;
 import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.springframework.stereotype.Service;
@@ -197,6 +198,63 @@ public class MarkFlowServiceImpl implements MarkFlowService {
         return JSON.parseObject(data, MarkFlowEntity.class);
     }
 
+    @Override
+    public MarkFlowEntity createAllIndexFlowInfo(JSONObject json, MarkSearchDTO dto) {
+        MarkFlowEntity markFlowEntity = new MarkFlowEntity(json);
+        JSONObject info = new JSONObject();
+        info.put("sourceDetails", getSourceDetails(json));
+        info.put("startTime", dto.getStartTime());
+        info.put("endTime", dto.getEndTime());
+        info.put("highlightWordMap", getHighlightWordMap(UserThreadLocal.getProjectId(), json));
+        markFlowEntity.setInfo(info);
+        return markFlowEntity;
+    }
+
+    /**
+     * 底层大库数据SourceDetails提取
+     * @param tJson
+     * @return
+     */
+    private JSONObject getSourceDetails(JSONObject tJson){
+        JSONObject sourceDetails = new JSONObject();
+        String source = tJson.getString(GenericAttribute.ES_SOURCE);
+        String rootSource = tJson.getString(GenericAttribute.ES_ROOT_SOURCE);
+        // 是否原创
+        if (tJson.containsKey(GenericAttribute.ES_ROOT_SOURCE) && (rootSource.contains(source) || source.contains(rootSource))) {
+            sourceDetails.put("rootPublish", "原创");
+        }
+        // C4,realSource提取展示
+        sourceDetails.put("clientFrom", Tools.isEmpty(tJson.getString(GenericAttribute.ES_REAL_SOURCE)) ? ClassD.TypeD.fromEncode(tJson.getIntValue(GenericAttribute.ES_C4)).title() : tJson.getString(GenericAttribute.ES_REAL_SOURCE));
+        // platform
+        sourceDetails.put("platform", BeanTools.filterPlatform(GlobalPojo.PLATFORMS, tJson.getIntValue(GenericAttribute.ES_C5), tJson.getIntValue(GenericAttribute.ES_FOREIGN)).getName());
+        // source
+        sourceDetails.put("source", source);
+        // 粉丝量提取
+        long followersNum = tJson.getLongValue("followers_num");
+        if (followersNum > 0) {
+            sourceDetails.put("followersNum", followersNum);
+        }
+        // 渠道标签
+        SensitiveChannel sensitiveChannel = GlobalPojo.COMMON_SENSITIVE_CHANNEL.get(source);
+        if (Objects.nonNull(sensitiveChannel)) {
+            List<String> channelTagStr = new ArrayList<>(4);
+            if (StringUtils.isNotBlank(sensitiveChannel.getPoliticsLevel())){
+                channelTagStr.add(sensitiveChannel.getPoliticsLevel());
+            }
+            if (StringUtils.isNotBlank(sensitiveChannel.getMainBodyType())){
+                channelTagStr.add(sensitiveChannel.getMainBodyType());
+            }
+            if (StringUtils.isNotBlank(sensitiveChannel.getField())){
+                channelTagStr.add(sensitiveChannel.getField());
+            }
+            if (StringUtils.isNotBlank(sensitiveChannel.getRegion())){
+                channelTagStr.add(sensitiveChannel.getRegion());
+            }
+            sourceDetails.put("channelTag", StringUtils.join(channelTagStr, "|"));
+        }
+        return sourceDetails;
+    }
+
     /**
      * 获取自定义标签
      * @param markInfoSource