Merge branch 'feature' into 'dev'

Feature

See merge request !509

src/main/java/com/zhiwei/brandkbs2/common/GenericAttribute.java

 package com.zhiwei.brandkbs2.common;
 
+import java.util.Arrays;
+import java.util.List;
+
 /**
  * @ClassName: GenericAttribute
  * @Description 一般属性值
@@ -131,6 +134,10 @@ public class GenericAttribute {
      * 地域
      */
     public static final String ES_FIELD = "field";
+    /**
+     * video_urls
+     */
+    public static final String ES_VIDEO_URLS = "video_urls";
 
 //    public static final String ES_CACHE_MAP_PROJECT = "brandkbs_cache_maps.project_id.keyword";
 //    public static final String ES_CACHE_MAP_LINKED_GROUP_ID = "brandkbs_cache_maps.linked_group_id.keyword";
@@ -166,5 +173,6 @@ public class GenericAttribute {
 //            return value;
 //        }
 //    }
-
+    /**视频real_source*/
+    public static final List<String> VIDEO_PLATFORM = Arrays.asList("抖音", "快手", "小红书", "bilibili", "新浪微博视频", "今日头条", "微信视频号", "西瓜视频", "其他");
 }

src/main/java/com/zhiwei/brandkbs2/controller/app/AppArticleController.java

@@ -542,7 +542,7 @@ public class AppArticleController extends BaseController {
     @GetMapping("/daily-report/list")
     @LogRecord(description = "舆情简报-每日一报")
     public ResponseResult getDailyReportList(@RequestParam(value = "page", defaultValue = "1") Integer page,
-                                             @RequestParam(value = "pageSize", defaultValue = "10")Integer pageSize) {
+                                             @RequestParam(value = "pageSize", defaultValue = "10") Integer pageSize) {
         return ResponseResult.success(markDataService.getDailyReportList(page, pageSize));
     }
 

src/main/java/com/zhiwei/brandkbs2/controller/app/AppDownloadController.java

@@ -18,6 +18,7 @@ import com.zhiwei.brandkbs2.pojo.vo.ProjectVO;
 import com.zhiwei.brandkbs2.service.*;
 import com.zhiwei.brandkbs2.util.Tools;
 import io.swagger.annotations.*;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.time.DateUtils;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.logging.log4j.LogManager;
@@ -395,6 +396,9 @@ public class AppDownloadController extends BaseController {
     @PostMapping(value = "/contend/mark")
     @DownloadTask(taskName = "竞品库竞品舆情下载", description = "竞品库竞品舆情")
     public ResponseResult exportContendMarkList(@RequestBody MarkSearchDTO markSearchDTO) {
+        if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
+            return ResponseResult.failure("不支持特殊符号字段查询");
+        }
         Pair<String, List<ExportAppYuqingDTO>> stringListPair = markDataService.downloadContendMarkList(markSearchDTO);
         // excel写入至指定路径
         String projectName = projectService.getProjectById(UserThreadLocal.getProjectId()).getProjectName();
@@ -408,6 +412,9 @@ public class AppDownloadController extends BaseController {
     @LogRecord(description = "全网搜-舆情导出", values = {"startTime", "endTime", "fans", "filterType", "filterWords", "search", "keyword", "platforms", "sensitiveChannels", "sourceKeyword"}, entity = true, arguments = true)
     @DownloadTask(taskName = "全网搜舆情下载", description = "全网搜舆情")
     public ResponseResult exportSearchWhole(@RequestBody SearchFilterDTO dto) {
+        if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
+            return ResponseResult.failure("不支持特殊符号字段查询");
+        }
         // 针对商业数据库做限制
         if (dto.isExternalDataSource()) {
             long time = DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), -89).getTime();
@@ -430,7 +437,7 @@ public class AppDownloadController extends BaseController {
         String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), "全网搜舆情列表数据");
         EasyExcelUtil.write(filePath, "sheet1", ExportSearchWholeDTO.class, exportList);
         if (dto.isExternalDataSource()) {
-            extraService.decreaseRecord(dto.getSearch(), WholeSearchRecord.UsedType.output, exportList.size());
+            extraService.decreaseRecord(dto.getKeyword(), WholeSearchRecord.UsedType.output, exportList.size());
         }
         return ResponseResult.success(filePath);
     }

src/main/java/com/zhiwei/brandkbs2/controller/app/AppSearchController.java

@@ -21,6 +21,7 @@ import com.zhiwei.brandkbs2.util.Tools;
 import com.zhiwei.middleware.event.pojo.dto.BrandkbsEventSearchDTO;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.time.DateUtils;
 import org.apache.commons.lang3.tuple.Pair;
 import org.joda.time.Period;
@@ -147,6 +148,9 @@ public class AppSearchController extends BaseController {
     @LogRecord(values = {"fans", "sensitiveChannels:father,son", "keyword", "search"}, description = "全网搜", arguments = true, entity = true)
     @PostMapping("/searchWhole")
     public ResponseResult searchWholeNetwork(@RequestBody SearchFilterDTO dto) {
+        if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
+            return ResponseResult.failure("不支持特殊符号字段查询");
+        }
         // 针对商业数据库做限制
         if (dto.isExternalDataSource()) {
             long time = DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), -89).getTime();
@@ -168,14 +172,23 @@ public class AppSearchController extends BaseController {
             List<JSONObject> list = pair.getLeft().getJSONArray("list").toJavaList(JSONObject.class);
             commonService.sentimentAnalysis(list);
         }
-        extraService.decreaseRecord(dto.getSearch(), WholeSearchRecord.UsedType.query, pair.getRight());
+        extraService.decreaseRecord(dto.getKeyword(), WholeSearchRecord.UsedType.query, pair.getRight());
         return ResponseResult.success(pair.getLeft());
     }
 
+    @ApiOperation("搜索-获取全网搜余额")
+    @PostMapping("/searchWhole/balance")
+    public ResponseResult getWholeSearchBalance() {
+        return ResponseResult.success(extraService.getWholeSearchBalance());
+    }
+
     @ApiOperation("搜索-全网搜-舆情导出")
     @PostMapping("/exportSearchWhole")
     @LogRecord(description = "全网搜-舆情导出", values = {"startTime", "endTime", "fans", "filterType", "filterWords", "search", "keyword", "platforms", "sensitiveChannels", "sourceKeyword"}, entity = true, arguments = true)
     public ResponseResult exportSearchWhole(@RequestBody SearchFilterDTO dto) {
+        if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
+            return ResponseResult.failure("不支持特殊符号字段查询");
+        }
         // 针对商业数据库做限制
         if (dto.isExternalDataSource()) {
             long time = DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), -89).getTime();
@@ -195,20 +208,27 @@ public class AppSearchController extends BaseController {
         List<ExportSearchWholeDTO> exportList = markDataService.exportSearchWhole(dto);
         EasyExcelUtil.download("全网搜舆情列表数据", "sheet1", ExportSearchWholeDTO.class, exportList, response);
         if (dto.isExternalDataSource()) {
-            extraService.decreaseRecord(dto.getSearch(), WholeSearchRecord.UsedType.output, exportList.size());
+            extraService.decreaseRecord(dto.getKeyword(), WholeSearchRecord.UsedType.output, exportList.size());
         }
         return ResponseResult.success();
     }
 
     @ApiOperation("舆情列表")
-    @LogRecord(values = "keyword", description = "查舆情", arguments = true, entity = true)
+    @LogRecord(values = {"searchType", "keyword"}, description = "查舆情", arguments = true, entity = true)
     @PostMapping("/mark/list")
     public ResponseResult getYuqingMarkList(@RequestBody MarkSearchDTO markSearchDTO) {
+        if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
+            return ResponseResult.failure("不支持特殊符号字段查询");
+        }
         PageVO<MarkFlowEntity> yuqingMarkList = markDataService.getYuqingMarkList(markSearchDTO);
         // 仅第一页增加平台进量（声量）统计
         if (1 == markSearchDTO.getPage()) {
             JSONObject info = yuqingMarkList.getInfo();
-            info.put("counts", markDataService.countMarkPlatform(markSearchDTO));
+            if (Objects.equals("视频", markSearchDTO.getSearchType())) {
+                info.put("counts", markDataService.countMarkVideoPlatform(markSearchDTO));
+            }else {
+                info.put("counts", markDataService.countMarkPlatform(markSearchDTO));
+            }
             yuqingMarkList.setInfo(info);
         }
         return ResponseResult.success(yuqingMarkList);
@@ -220,7 +240,7 @@ public class AppSearchController extends BaseController {
     public ResponseResult getOriginList(@RequestBody MarkSearchDTO markSearchDTO) {
         PageVO<MarkFlowEntity> originList = markDataService.getOriginList(markSearchDTO);
         // 仅第一页增加平台进量（声量）统计
-        if (1 == markSearchDTO.getPage()) {
+        if (1 == markSearchDTO.getPage() && !Objects.equals("视频", markSearchDTO.getSearchType())) {
             JSONObject info = originList.getInfo();
             info.put("counts", markDataService.countOriginPlatform(markSearchDTO));
             originList.setInfo(info);
@@ -301,6 +321,9 @@ public class AppSearchController extends BaseController {
     @LogRecord(values = "keyword", description = "查竞品",arguments = true, entity = true)
     @PostMapping("/contend/list")
     public ResponseResult getContendSearchList(@RequestBody MarkSearchDTO markSearchDTO) {
+        if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
+            return ResponseResult.failure("不支持特殊符号字段查询");
+        }
         return ResponseResult.success(markDataService.getContendSearchList(markSearchDTO));
     }
 

src/main/java/com/zhiwei/brandkbs2/controller/app/AppToolsetController.java

@@ -77,8 +77,7 @@ public class AppToolsetController {
     @LogRecord(description = "工具库-本品事件影响力更新")
     public ResponseResult primaryEventUpdate(@RequestParam MultipartFile file,
                                              @RequestParam Boolean extraCompute){
-        toolsetService.primaryEventUpdate(file, extraCompute);
-        return ResponseResult.success();
+        return toolsetService.primaryEventUpdate(file, extraCompute);
     }
 
     @ApiOperation("工具库-事件影响力更新-竞品更新")

src/main/java/com/zhiwei/brandkbs2/easyexcel/listener/BytedanceEventListener.java

@@ -4,8 +4,10 @@ import com.alibaba.excel.context.AnalysisContext;
 import com.alibaba.excel.event.AnalysisEventListener;
 import com.zhiwei.brandkbs2.easyexcel.dto.UploadBytedanceEventDTO;
 import com.zhiwei.brandkbs2.pojo.BytedanceCustomEventUpdateTaskData;
+import org.apache.commons.collections4.CollectionUtils;
 
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 
 /**
@@ -16,20 +18,39 @@ import java.util.Objects;
  */
 public class BytedanceEventListener extends AnalysisEventListener<UploadBytedanceEventDTO> {
 
-    private String taskId;
-
     private List<BytedanceCustomEventUpdateTaskData> data;
 
+    private List<Boolean> correctHead;
+
 
-    public BytedanceEventListener(String taskId, List<BytedanceCustomEventUpdateTaskData> data){
-        this.taskId = taskId;
+    public BytedanceEventListener(List<BytedanceCustomEventUpdateTaskData> data, List<Boolean> correctHead){
         this.data = data;
+        this.correctHead = correctHead;
+    }
+
+    /**
+     * 表头格式校验
+     * @param headMap
+     * @param context
+     */
+    @Override
+    public void invokeHeadMap(Map<Integer, String> headMap, AnalysisContext context) {
+        if (!"序号".equals(headMap.get(0)) || !"平台类型".equals(headMap.get(1)) || !"发布平台".equals(headMap.get(2))
+                || !"渠道".equals(headMap.get(3)) || !"时间".equals(headMap.get(4)) || !"标题".equals(headMap.get(5))
+                || !"文本".equals(headMap.get(6)) || !"地址".equals(headMap.get(7)) || !"命中词".equals(headMap.get(8))
+                || !"情感倾向".equals(headMap.get(9)) || !"事件标签".equals(headMap.get(10)) || !"在看".equals(headMap.get(11))
+                || !"阅读".equals(headMap.get(12)) || !"点赞".equals(headMap.get(13)) || !"评论".equals(headMap.get(14))
+                || !"转发".equals(headMap.get(15))) {
+            correctHead.add(false);
+        }
     }
 
     @Override
     public void invoke(UploadBytedanceEventDTO dto, AnalysisContext context) {
+        if (CollectionUtils.isNotEmpty(correctHead) && !correctHead.get(0)){
+            return;
+        }
         BytedanceCustomEventUpdateTaskData taskData = new BytedanceCustomEventUpdateTaskData();
-        taskData.setTaskId(taskId);
         taskData.setPlatformType(dto.getPlatformType());
         taskData.setPlatform(dto.getPlatform());
         taskData.setChannel(dto.getChannel());

src/main/java/com/zhiwei/brandkbs2/es/EsQueryTools.java

@@ -291,6 +291,27 @@ public class EsQueryTools {
     }
 
     /**
+     * 视频平台查询
+     * @param platforms
+     * @return
+     */
+    public static BoolQueryBuilder assembleVideoPlatformQuery(List<String> platforms){
+        BoolQueryBuilder realSourceQuery = QueryBuilders.boolQuery();
+        // realSource
+        // 其他 特殊处理
+        if (platforms.contains("其他")) {
+            BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
+            for (int i = 0; i < GenericAttribute.VIDEO_PLATFORM.size() - 1; i++) {
+                queryBuilder.mustNot(QueryBuilders.termQuery(GenericAttribute.ES_REAL_SOURCE, GenericAttribute.VIDEO_PLATFORM.get(i)));
+            }
+            realSourceQuery.should(queryBuilder);
+            platforms.remove("其他");
+        }
+        platforms.forEach(pt -> realSourceQuery.should(QueryBuilders.termQuery(GenericAttribute.ES_REAL_SOURCE, pt)));
+        return realSourceQuery;
+    }
+
+    /**
      * 针对微博平台转发 查询语句
      *
      * @param isForward

src/main/java/com/zhiwei/brandkbs2/pojo/BytedanceCustomEventUpdateTaskData.java

@@ -14,8 +14,6 @@ import lombok.Setter;
 @Setter
 @NoArgsConstructor
 public class BytedanceCustomEventUpdateTaskData extends AbstractBaseMongo {
-    private String taskId;
-
     private String platformType;
 
     private String platform;

src/main/java/com/zhiwei/brandkbs2/pojo/MarkFlowEntity.java

@@ -15,6 +15,7 @@ import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
 import org.apache.commons.lang3.ObjectUtils;
+import org.apache.commons.lang3.StringUtils;
 
 import java.io.Serializable;
 import java.util.List;
@@ -163,6 +164,10 @@ public class MarkFlowEntity implements Serializable {
                     videoMark.setMgroup(GlobalPojo.getYuQingProjectNameById(linkedGroupId));
                     this.filterInfo = videoMark.filterInfo();
                 }
+                // 视频时长处理
+                if (StringUtils.isNotBlank(videoMark.getLength())){
+                    videoMark.setLength(Tools.formatVideoLength(videoMark.getLength()));
+                }
                 this.type = FlowEntityUtil.getVideoType(videoMark);
                 this.data = videoMark;
                 break;

src/main/java/com/zhiwei/brandkbs2/pojo/dto/MarkSearchDTO.java

@@ -64,12 +64,24 @@ public class MarkSearchDTO {
     private Long endTime;
 
     /**
+     * 数据类型 全量数据 | 视频
+     */
+    @ApiModelProperty("搜索类型:全量数据|视频")
+    private String searchType;
+
+    /**
      * 平台
      */
     @ApiModelProperty(value = "平台")
     private List<String> platforms;
 
     /**
+     * 视频平台
+     */
+    @ApiModelProperty(value = "视频平台")
+    private List<String> videoPlatforms;
+
+    /**
      * 媒体类型
      */
     @ApiModelProperty(value = "重要发声方")
@@ -119,7 +131,7 @@ public class MarkSearchDTO {
 
     // query部分
     /**
-     * 检索字段
+     * 关键字检索字段
      */
     @ApiModelProperty(value = "检索字段")
     private String searchField;
@@ -155,10 +167,16 @@ public class MarkSearchDTO {
     private boolean fold;
 
     /**
-     *
+     * 屏蔽词
      */
     @ApiModelProperty(value = "屏蔽词")
-    private String shieldWord;
+    private String filterWords;
+
+    /**
+     * 屏蔽词检索字段 全文|标题
+     */
+    @ApiModelProperty(value = "屏蔽词检索字段：全文|标题")
+    private String filterType;
 
     /**
      * 方案id,仅非人工项目

src/main/java/com/zhiwei/brandkbs2/pojo/dto/SearchFilterDTO.java

 package com.zhiwei.brandkbs2.pojo.dto;
 
 import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.brandkbs2.common.GenericAttribute;
+import com.zhiwei.brandkbs2.common.GlobalPojo;
+import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform;
 import io.swagger.annotations.ApiModel;
 import io.swagger.annotations.ApiModelProperty;
 import lombok.Data;
+import org.apache.commons.collections4.CollectionUtils;
 
 import java.util.*;
+import java.util.stream.Collectors;
 
 @Data
 @ApiModel("全网搜搜索传输类")
@@ -42,10 +47,16 @@ public class SearchFilterDTO {
     private Long crawlerTime;
 
     /**
+     * 数据类型 全量数据 | 视频
+     */
+    @ApiModelProperty("搜索类型:全量数据 | 视频")
+    private String searchType;
+
+    /**
      * 关键字
      */
     @ApiModelProperty(value = "关键字")
-    private String search;
+    private String keyword;
 
     /**
      * 平台
@@ -54,6 +65,12 @@ public class SearchFilterDTO {
     private List<String> platforms;
 
     /**
+     * 视频平台
+     */
+    @ApiModelProperty(value = "视频平台")
+    private List<String> videoPlatforms;
+
+    /**
      * 排序字段
      */
     @ApiModelProperty(value = "排序字段")
@@ -146,8 +163,9 @@ public class SearchFilterDTO {
         Map<String, Object> map = new HashMap<>();
         map.put("timeType", "time");
         map.put("pt", getPlatforms());
+        map.put("dataType", searchType);
         map.put("matchType", matchType);
-        map.put("search", search);
+        map.put("search", keyword);
         map.put("startTime", startTime);
         map.put("endTime", endTime);
         map.put("source", sourceKeyword);
@@ -173,7 +191,10 @@ public class SearchFilterDTO {
     }
 
     public List<String> getPlatforms() {
-        return Objects.nonNull(platforms) ? this.platforms : Collections.emptyList();
+        if (Objects.equals("视频", this.searchType)){
+            return CollectionUtils.isNotEmpty(this.videoPlatforms) ? this.videoPlatforms : new ArrayList<>(GenericAttribute.VIDEO_PLATFORM);
+        }
+        return CollectionUtils.isNotEmpty(this.platforms) ? this.platforms : GlobalPojo.PLATFORMS.stream().map(MessagePlatform::getId).collect(Collectors.toList());
     }
 
     public String getMatchType() {

src/main/java/com/zhiwei/brandkbs2/service/EsSearchService.java

@@ -75,4 +75,12 @@ public interface EsSearchService {
      * @throws IOException
      */
     JSONObject countNonManualPlatform(MarkSearchDTO dto) throws IOException;
+
+    /**
+     * 查舆情视频中心进量（声量）统计
+     * @param dto
+     * @return
+     * @throws IOException
+     */
+    JSONObject countMarkVideoPlatform(MarkSearchDTO dto) throws IOException;
 }

src/main/java/com/zhiwei/brandkbs2/service/ExtraService.java

@@ -56,6 +56,12 @@ public interface ExtraService {
     JSONObject overview();
 
     /**
+     * 获取全网搜余额
+     * @return
+     */
+    Integer getWholeSearchBalance();
+
+    /**
      * @param keyword 关键词
      * @param usedType 使用类别
      * @param usedCount 使用量

src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java

@@ -393,6 +393,13 @@ public interface MarkDataService {
     JSONObject countOriginPlatform(MarkSearchDTO dto);
 
     /**
+     * 查舆情视频中心-平台进量（声量）统计
+     * @param dto   标注数据搜索传输类
+     * @return
+     */
+    JSONObject countMarkVideoPlatform(MarkSearchDTO dto);
+
+    /**
      * 新建非人工项目方案
      * @param dto
      * @return

src/main/java/com/zhiwei/brandkbs2/service/ToolsetService.java

@@ -76,7 +76,7 @@ public interface ToolsetService {
      * @param file          excel文件
      * @param extraCompute  是否计算实时传播力与传播影响力
      */
-    void primaryEventUpdate(MultipartFile file, Boolean extraCompute);
+    ResponseResult primaryEventUpdate(MultipartFile file, Boolean extraCompute);
 
     /**
      * 竞品事件影响力计算

src/main/java/com/zhiwei/brandkbs2/service/impl/EsSearchServiceImpl.java

@@ -77,6 +77,11 @@ public class EsSearchServiceImpl implements EsSearchService {
         return platformCount(dto, this::createNonManualSearchHelper, esClientDao);
     }
 
+    @Override
+    public JSONObject countMarkVideoPlatform(MarkSearchDTO dto) throws IOException {
+        return videoPlatformCount(dto, this::createSearchHelper, esClientDao);
+    }
+
     private Pair<SearchHits[], Map<String, Long>> searchHitsAndCount(MarkSearchDTO dto, ArticleFunction.CreateSearchHelper createSearchHelper,
                                                                      EsClientDao clientDao, boolean scroll) throws IOException {
         EsClientDao.SearchHelper helper = createSearchHelper.create(dto);
@@ -99,10 +104,18 @@ public class EsSearchServiceImpl implements EsSearchService {
         BoolQueryBuilder postFilter = EsQueryTools.assembleCacheMapsQuery(projectId, contendId, dto.getPoliticsLevel(), dto.getField(), dto.getRegion(), dto.getMainBodyType(), dto.getChannelEmotion());
         // time
         postFilter.must(QueryBuilders.rangeQuery(dto.getTimeType()).gte(dto.getStartTime()).lt(dto.getEndTime()));
+        // searchType
+        if (Objects.equals("视频", dto.getSearchType())){
+            postFilter.must(QueryBuilders.existsQuery(GenericAttribute.ES_VIDEO_URLS));
+        }
         // platform
         if (CollectionUtils.isNotEmpty(dto.getPlatforms())) {
             postFilter.must(EsQueryTools.assemblePlatformQuery(Tools.getPlatformByIds(dto.getPlatforms())));
         }
+        // videoPlatforms
+        if (CollectionUtils.isNotEmpty(dto.getVideoPlatforms())){
+            postFilter.must(EsQueryTools.assembleVideoPlatformQuery(dto.getVideoPlatforms()));
+        }
         // tags
         if (CollectionUtils.isNotEmpty(dto.getTags())) {
             postFilter.must(EsQueryTools.assembleTagQuery(dto.getTags()));
@@ -119,9 +132,10 @@ public class EsSearchServiceImpl implements EsSearchService {
             // Query
             postFilter.must(EsQueryTools.assembleNormalKeywordQuery(dto.getKeyword(), fieldSearch));
         }
-        // shieldWord
-        if (StringUtils.isNotEmpty(dto.getShieldWord())) {
-            postFilter.mustNot(EsQueryTools.assembleNormalKeywordQuery(dto.getShieldWord(), fieldSearch));
+        // filterWords
+        String[] filterFieldSearch = "标题".equals(dto.getFilterType()) ? new String[]{GenericAttribute.ES_IND_TITLE} : new String[]{GenericAttribute.ES_IND_FULL_TEXT};
+        if (StringUtils.isNotEmpty(dto.getFilterWords())) {
+            postFilter.mustNot(EsQueryTools.assembleNormalKeywordQuery(dto.getFilterWords(), filterFieldSearch));
         }
         // sourceKeyword
         if (StringUtils.isNotEmpty(dto.getSourceKeyword())) {
@@ -383,4 +397,31 @@ public class EsSearchServiceImpl implements EsSearchService {
         res.put("total", total);
         return res;
     }
+
+    /**
+     * 视频realSource进量统计
+     * @param dto
+     * @param createSearchHelper
+     * @param esClientDao
+     * @return
+     * @throws IOException
+     */
+    private JSONObject videoPlatformCount(MarkSearchDTO dto, ArticleFunction.CreateSearchHelper createSearchHelper, EsClientDao esClientDao) throws IOException {
+        List<JSONObject> list = new ArrayList<>();
+        for (String realSource : GenericAttribute.VIDEO_PLATFORM) {
+            List<String> oneRealSource = new ArrayList<>(1);
+            oneRealSource.add(realSource);
+            dto.setVideoPlatforms(oneRealSource);
+            EsClientDao.SearchHelper helper = createSearchHelper.create(dto);
+            JSONObject jsonObject = new JSONObject();
+            jsonObject.put("count", esClientDao.count(helper.getPostFilter()));
+            jsonObject.put("name", realSource);
+            list.add(jsonObject);
+        }
+        Long total = list.stream().map(jsonObject -> jsonObject.getLongValue("count")).reduce(Long::sum).orElse(0L);
+        JSONObject res = new JSONObject();
+        res.put("count", list);
+        res.put("total", total);
+        return res;
+    }
 }

src/main/java/com/zhiwei/brandkbs2/service/impl/ExtraServiceImpl.java

@@ -109,6 +109,11 @@ public class ExtraServiceImpl implements ExtraService {
     }
 
     @Override
+    public Integer getWholeSearchBalance() {
+        return projectDao.findOneById(UserThreadLocal.getProjectId()).getWholeSearchBalance();
+    }
+
+    @Override
     public void decreaseRecord(String keyword, WholeSearchRecord.UsedType usedType, Integer usedCount) {
         if (null == usedCount) {
             return;

src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java

@@ -1438,6 +1438,12 @@ public class MarkDataServiceImpl implements MarkDataService {
             result.put("followersNum", list.get("followers_num"));
             result.put("realSource", list.get("real_source"));
             result.put("ipLocation", list.get("ip_location"));
+            result.put("picUrls", list.get("pic_urls"));
+            result.put("videoUrls", list.getJSONArray("video_urls"));
+            // 视频时长处理
+            if (StringUtils.isNotBlank(list.getString("length"))){
+                result.put("length", Tools.formatVideoLength(list.getString("length")));
+            }
             resultList.add(result);
         }
         // 仅第一页返回平台进量（声量）统计
@@ -1643,7 +1649,7 @@ public class MarkDataServiceImpl implements MarkDataService {
      */
     private JSONObject externalTransform(SearchFilterDTO dto) {
         JSONObject params = new JSONObject();
-        params.put("crawlerWord", dto.getSearch());
+        params.put("crawlerWord", dto.getKeyword());
         params.put("startTime", dto.getStartTime());
         params.put("endTime", dto.getEndTime());
         String platforms;
@@ -1662,6 +1668,12 @@ public class MarkDataServiceImpl implements MarkDataService {
         // matchFields
         if (dto.getMatchType().equals("标题")) {
             params.put("matchFields", Collections.singletonList("title"));
+        }else if (dto.getMatchType().equals("全文")){
+            params.put("matchFields", Collections.singletonList("content"));
+        }
+        // sourceKeyword
+        if (StringUtils.isNotBlank(dto.getSourceKeyword())){
+            params.put("userList", Arrays.asList(dto.getSourceKeyword().trim().split("\\|")));
         }
 //        // 历史部分
 //        if (Tools.isEmpty(dto.getPlatforms())) {
@@ -1883,28 +1895,50 @@ public class MarkDataServiceImpl implements MarkDataService {
     }
 
     @Override
+    public JSONObject countMarkVideoPlatform(MarkSearchDTO dto) {
+        defaultMarkSearch(dto);
+        try {
+            return esSearchService.countMarkVideoPlatform(dto);
+        } catch (Exception e) {
+            ExceptionCast.cast(CommonCodeEnum.FAIL, "es查询异常", e);
+        }
+        return null;
+    }
+
+    @Override
     public ResponseResult addNonManualProjectPlan(NonManualProjectPlanDTO dto) {
-        if (nonManualProjectPlanDao.count(new Query(Criteria.where("projectId").is(UserThreadLocal.getProjectId()))) >= 5) {
+        String projectId = UserThreadLocal.getProjectId();
+        if (nonManualProjectPlanDao.count(new Query(Criteria.where("projectId").is(projectId))) >= 5) {
             return ResponseResult.failure("超过方案数量上限5");
         }
-        // 同时添加快搜订阅任务
-        String iStarShineSubjectTaskId = addIStarShineSubjectTask(dto);
-        if (Objects.isNull(iStarShineSubjectTaskId)){
-            return ResponseResult.failure("添加方案失败");
-        }
         NonManualProjectPlan plan = new NonManualProjectPlan(dto);
-        plan.setIStarShineSubjectTaskId(iStarShineSubjectTaskId);
+        Project project = projectService.getProjectById(projectId);
+        if (project.getExternalDataSource()) {
+            // 同时添加快搜订阅任务
+            String iStarShineSubjectTaskId = addIStarShineSubjectTask(dto);
+            if (Objects.isNull(iStarShineSubjectTaskId)) {
+                return ResponseResult.failure("添加方案失败");
+            }
+            plan.setIStarShineSubjectTaskId(iStarShineSubjectTaskId);
+        }
         nonManualProjectPlanDao.insertOne(plan);
         return ResponseResult.success();
     }
 
     @Override
     public ResponseResult deleteNonManualProjectPlan(String id) {
-        ResponseResult responseResult = deleteIStarShineSubjectTask(id);
-        if (responseResult.isStatus()){
+        Project project = projectService.getProjectById(UserThreadLocal.getProjectId());
+        // 商业数据源同时删除快搜订阅任务
+        if (project.getExternalDataSource()){
+            ResponseResult responseResult = deleteIStarShineSubjectTask(id);
+            if (responseResult.isStatus()){
+                nonManualProjectPlanDao.deleteOneById(id);
+            }
+            return responseResult;
+        }else {
             nonManualProjectPlanDao.deleteOneById(id);
+            return ResponseResult.success();
         }
-        return responseResult;
     }
 
     @Override
@@ -1929,12 +1963,16 @@ public class MarkDataServiceImpl implements MarkDataService {
         NonManualProjectPlan plan = new NonManualProjectPlan(dto);
         plan.setCTime(existPlan.getCTime());
         plan.setIStarShineSubjectTaskId(existPlan.getIStarShineSubjectTaskId());
-        ResponseResult responseResult = updateIStarShineSubjectTask(dto);
-        if (!responseResult.isStatus()){
-            return responseResult;
+        Project project = projectService.getProjectById(UserThreadLocal.getProjectId());
+        // 商业数据源同时修改快搜订阅任务
+        if (project.getExternalDataSource()){
+            ResponseResult responseResult = updateIStarShineSubjectTask(dto);
+            if (!responseResult.isStatus()){
+                return responseResult;
+            }
         }
         nonManualProjectPlanDao.updateOne(plan);
-        return responseResult;
+        return ResponseResult.success();
     }
 
     /**
@@ -2001,7 +2039,7 @@ public class MarkDataServiceImpl implements MarkDataService {
         HttpEntity<String> request = new HttpEntity<>(param.toJSONString(), httpHeaders);
         JSONObject body = restTemplate.postForEntity(updateIStarShineSubjectTaskUrl, request, JSONObject.class).getBody();
         if (Objects.isNull(body) || body.isEmpty() || Boolean.FALSE.equals(body.getBooleanValue("status"))){
-            log.info(JSONObject.toJSONString(plan) + "-修改快搜订阅任务失败-" + body);
+            log.error(JSONObject.toJSONString(plan) + "-修改快搜订阅任务失败-" + body);
             return ResponseResult.failure("修改失败");
         }
         return ResponseResult.success();
@@ -2009,14 +2047,35 @@ public class MarkDataServiceImpl implements MarkDataService {
 
     @Override
     public Integer getPlanDataPrediction(String keyword, String filterWord) {
-        JSONObject data = getSearchWholeOpinionJson(createSearchFilterDTO(keyword, filterWord));
-        JSONObject task = data.getJSONObject("task");
-        return task.getInteger("searchCount");
+        try {
+            Project project = projectService.getProjectById(UserThreadLocal.getProjectId());
+            if (project.getExternalDataSource()) {
+                JSONObject data = getSearchWholeOpinionJson(createSearchFilterDTO(keyword, filterWord));
+                JSONObject task = data.getJSONObject("task");
+                return task.getInteger("searchCount");
+            } else {
+                long endTime = System.currentTimeMillis();
+                long startTime = endTime - Constant.ONE_DAY;
+                BoolQueryBuilder postFilter = QueryBuilders.boolQuery();
+                // time
+                postFilter.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
+                // keyword
+                postFilter.must(EsQueryTools.assembleNormalKeywordQuery(keyword, new String[]{GenericAttribute.ES_IND_FULL_TEXT}));
+                // filterWord
+                if (StringUtils.isNotBlank(filterWord)) {
+                    postFilter.mustNot(EsQueryTools.assembleNormalKeywordQuery(filterWord, new String[]{GenericAttribute.ES_IND_FULL_TEXT}));
+                }
+                return hotEsDao.count(postFilter).intValue();
+            }
+        }catch (Exception e){
+            ExceptionCast.cast(CommonCodeEnum.FAIL, "方案预估失败", e);
+        }
+        return null;
     }
 
     private SearchFilterDTO createSearchFilterDTO(String keyword, String filterWord){
         SearchFilterDTO dto = new SearchFilterDTO();
-        dto.setSearch(keyword);
+        dto.setKeyword(keyword);
         dto.setFilterWords(filterWord);
         // 预估一天
         dto.setEndTime(System.currentTimeMillis());
@@ -2379,15 +2438,15 @@ public class MarkDataServiceImpl implements MarkDataService {
             List<JSONObject> res = new ArrayList<>();
             String projectId = UserThreadLocal.getProjectId();
             List<NonManualProjectPlan> planList = nonManualProjectPlanDao.findList(new Query(Criteria.where("projectId").is(projectId)));
-            long startTime = Constant.DF_yyyyMMdd.parse(Constant.DF_yyyyMMdd.format(new Date())).getTime();
-            long endTime = System.currentTimeMillis();
+//            long startTime = Constant.DF_yyyyMMdd.parse(Constant.DF_yyyyMMdd.format(new Date())).getTime();
+//            long endTime = System.currentTimeMillis();
             for (NonManualProjectPlan plan : planList) {
                 JSONObject jsonObject = new JSONObject();
                 jsonObject.put("id", plan.getId());
                 jsonObject.put("name", plan.getName());
-                BoolQueryBuilder query = EsQueryTools.assembleCacheMapsPlanQuery(projectId, plan.getId());
-                query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
-                jsonObject.put("count", esClientDao.count(query));
+//                BoolQueryBuilder query = EsQueryTools.assembleCacheMapsPlanQuery(projectId, plan.getId());
+//                query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
+//                jsonObject.put("count", esClientDao.count(query));
                 res.add(jsonObject);
             }
             return res;
@@ -2914,10 +2973,11 @@ public class MarkDataServiceImpl implements MarkDataService {
             // postFilter
             BoolQueryBuilder postFilter = yuqingAnalyzeQuery(dto);
             searchHelper.setPostFilter(postFilter);
-            searchHelper.setSize(10000);
+            searchHelper.setSize(1000);
             List<String> textList = new ArrayList<>();
-            List<SearchResponse> searchResponses = Collections.singletonList(esClientDao.searchResponse(searchHelper));
-            for (SearchResponse searchResponse : searchResponses) {
+            for (int i = 1; i <= 10; i++) {
+                searchHelper.setFrom((i - 1) * 1000);
+                SearchResponse searchResponse = esClientDao.searchResponse(searchHelper);
                 for (SearchHit searchHit : searchResponse.getHits().getHits()) {
                     Map<String, Object> source = searchHit.getSourceAsMap();
                     BaseMap baseMap = Tools.getBaseFromEsMap(source);
@@ -3224,11 +3284,6 @@ public class MarkDataServiceImpl implements MarkDataService {
      */
     private Map<String, List<LineVO>> getPlatformSpreadTendency(MarkSearchDTO dto) throws IOException{
         Map<String, List<LineVO>> res = new HashMap<>(4);
-        // 平台
-//        List<String> platforms = new ArrayList<>();
-//        for (String platformNames : Arrays.asList("微博", "微信", "抖音", "小红书")) {
-//            platforms.add(GlobalPojo.getPlatformIdByName(platformNames));
-//        }
         if (CollectionUtils.isEmpty(dto.getPlatforms())){
             dto.setPlatforms(Stream.of("微博", "微信", "抖音", "小红书").map(GlobalPojo::getPlatformIdByName).collect(Collectors.toList()));
         }
@@ -3373,12 +3428,20 @@ public class MarkDataServiceImpl implements MarkDataService {
         if (Objects.nonNull(dto.getPlanId())){
             postFilter = EsQueryTools.assembleCacheMapsPlanQuery(projectId, dto.getPlanId(), dto.getPoliticsLevel());
         }
+        // searchType
+        if (Objects.equals("视频", dto.getSearchType())){
+            postFilter.must(QueryBuilders.existsQuery(GenericAttribute.ES_VIDEO_URLS));
+        }
         // time
         postFilter.must(QueryBuilders.rangeQuery("time").gte(dto.getStartTime()).lt(dto.getEndTime()));
         // platform
         if (CollectionUtils.isNotEmpty(dto.getPlatforms())) {
             postFilter.must(EsQueryTools.assemblePlatformQuery(Tools.getPlatformByIds(dto.getPlatforms())));
         }
+        // videoPlatforms
+        if (CollectionUtils.isNotEmpty(dto.getVideoPlatforms())){
+            postFilter.must(EsQueryTools.assembleVideoPlatformQuery(dto.getVideoPlatforms()));
+        }
         // tags
         if (CollectionUtils.isNotEmpty(dto.getTags())) {
             postFilter.must(EsQueryTools.assembleTagQuery(dto.getTags()));
@@ -3394,14 +3457,15 @@ public class MarkDataServiceImpl implements MarkDataService {
             // Query
             postFilter.must(EsQueryTools.assembleNormalKeywordQuery(dto.getKeyword(), fieldSearch));
         }
+        // filterWords
+        String[] filterFieldSearch = "标题".equals(dto.getFilterType()) ? new String[]{GenericAttribute.ES_IND_TITLE} : new String[]{GenericAttribute.ES_IND_FULL_TEXT};
+        if (StringUtils.isNotEmpty(dto.getFilterWords())) {
+            postFilter.mustNot(EsQueryTools.assembleNormalKeywordQuery(dto.getFilterWords(), filterFieldSearch));
+        }
         // emotion
         if (StringUtils.isNotEmpty(emotion) && !EmotionEnum.ALL.getName().equals(emotion)) {
             postFilter.must(QueryBuilders.termQuery("brandkbs_mark_cache_maps.name.keyword", emotion));
         }
-        // shieldWord
-        if (StringUtils.isNotEmpty(dto.getShieldWord())) {
-            postFilter.mustNot(EsQueryTools.assembleNormalKeywordQuery(dto.getShieldWord(), fieldSearch));
-        }
         // sourceKeyword
         if (StringUtils.isNotEmpty(dto.getSourceKeyword())) {
             postFilter.must(EsQueryTools.assembleSourceKeywordQuery(dto.getSourceKeyword()));

src/main/java/com/zhiwei/brandkbs2/service/impl/ProjectServiceImpl.java

@@ -252,6 +252,13 @@ public class ProjectServiceImpl implements ProjectService {
         // 删除非人工项目维护记录和方案
         if (!project.isManual()){
             Query query = new Query(Criteria.where("projectId").is(pid));
+            List<NonManualProjectPlan> planList = nonManualProjectPlanDao.findList(query);
+            // 删除非人工项目数据订阅任务
+            if (project.getExternalDataSource()){
+                for (NonManualProjectPlan plan : planList) {
+                    markDataService.deleteIStarShineSubjectTask(plan.getId());
+                }
+            }
             nonManualProjectPlanDao.deleteOneByQuery(query);
             nonManualProjectDataUpdateRecordDao.deleteOneByQuery(query);
         }
@@ -531,6 +538,7 @@ public class ProjectServiceImpl implements ProjectService {
         project.setShow(existsProject.isShow());
         project.setDataBalance(existsProject.getDataBalance());
         project.setStart(existsProject.isStart());
+        project.setExternalDataSource(existsProject.getExternalDataSource());
         projectDao.updateOne(project);
         return ResponseResult.success();
     }

src/main/java/com/zhiwei/brandkbs2/service/impl/ToolsetServiceImpl.java

@@ -368,19 +368,23 @@ public class ToolsetServiceImpl implements ToolsetService {
     }
 
     @Override
-    public void primaryEventUpdate(MultipartFile file, Boolean extraCompute) {
-        BytedanceCustomEventUpdateTask task = BytedanceCustomEventUpdateTask.createTaskPrimary(extraCompute);
-        bytedanceCustomEventUpdateTaskDao.insertOne(task);
-
+    public ResponseResult primaryEventUpdate(MultipartFile file, Boolean extraCompute) {
         List<BytedanceCustomEventUpdateTaskData> data = new ArrayList<>();
+        List<Boolean> correctHead = new ArrayList<>(1);
         // excel信息提取
         ReadExcelDTO<UploadBytedanceEventDTO> readExcel = new ReadExcelDTO<>();
         readExcel.setClazz(UploadBytedanceEventDTO.class);
-        readExcel.setAnalysisEventListener(new BytedanceEventListener(task.getId(), data));
+        readExcel.setAnalysisEventListener(new BytedanceEventListener(data, correctHead));
         EasyExcelUtil.read(file, readExcel);
+        if (CollectionUtils.isNotEmpty(correctHead) && !correctHead.get(0)){
+            return ResponseResult.failure("表格模板错误,计算失败");
+        }
+        BytedanceCustomEventUpdateTask task = BytedanceCustomEventUpdateTask.createTaskPrimary(extraCompute);
+        bytedanceCustomEventUpdateTaskDao.insertOne(task);
         // 更新处理进度
         resetProcessSchedule(task.getId(), data.size());
         ApplicationProjectListener.getThreadPool().execute(() -> processEventUpdate(data, task, extraCompute));
+        return ResponseResult.success();
     }
 
     @Override
@@ -399,7 +403,7 @@ public class ToolsetServiceImpl implements ToolsetService {
         // excel信息提取
         ReadExcelDTO<UploadBytedanceEventDTO> readExcel = new ReadExcelDTO<>();
         readExcel.setClazz(UploadBytedanceEventDTO.class);
-        readExcel.setAnalysisEventListener(new BytedanceEventListener(taskId, data));
+        readExcel.setAnalysisEventListener(new BytedanceEventListener(data, null));
         EasyExcelUtil.read(file, readExcel);
         // 将旧数据与新数据合并
         List<BytedanceCustomEventUpdateTaskData> taskData = bytedanceCustomEventUpdateTaskDataDao.findList(new Query(Criteria.where("taskId").is(taskId)));
@@ -688,6 +692,7 @@ public class ToolsetServiceImpl implements ToolsetService {
 
     private void updateStatus(String taskId, BytedanceCustomEventUpdateTask.TaskStatus taskStatus){
         BytedanceCustomEventUpdateTask task = bytedanceCustomEventUpdateTaskDao.findOneById(taskId);
+        task.setRead(true);
         updateStatus(task, taskStatus, null);
     }
 
@@ -724,6 +729,10 @@ public class ToolsetServiceImpl implements ToolsetService {
                             ? data.getPlatformType() : "网媒", data.getChannel()));
             // 1.分别将等级为S,A,B,C的渠道各自加和
             if (Objects.nonNull(channelInfluence)){
+                // 2024/5/21修改：无论表中是否匹配，当类型为D，取值=1
+                if (Objects.equals("D", channelInfluence.getLevel())){
+                    channelInfluence.setInfluence(1d);
+                }
                 levelSumH.compute(channelInfluence.getLevel(), (k, v) -> Objects.isNull(v)
                         ? channelInfluence.getInfluence()
                         : BigDecimal.valueOf(v).add(BigDecimal.valueOf(channelInfluence.getInfluence())).doubleValue());

src/main/java/com/zhiwei/brandkbs2/util/Tools.java

@@ -37,6 +37,8 @@ import org.springframework.web.multipart.MultipartFile;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import java.io.*;
+import java.math.BigDecimal;
+import java.math.RoundingMode;
 import java.net.URLEncoder;
 import java.nio.ByteBuffer;
 import java.nio.channels.Channels;
@@ -1356,6 +1358,10 @@ public class Tools {
         return Arrays.stream(str.split("[\\u007c]+")).filter(StringUtils::isNotBlank).collect(Collectors.joining("|"));
     }
 
+    public static boolean checkUniteString(String str){
+        return str.split("[\\u4e28]+").length > 1 || str.split("[\\uff5c]+").length > 1 || str.split("[\\u007c]+").length > 1;
+    }
+
     /**
      * 过滤 “分享一篇文章”，”暂无标题“，”Notitle“ 的无用标题
      * @param title
@@ -1373,4 +1379,19 @@ public class Tools {
     public static Boolean filterUselessSource(String source){
         return FILTER_SOURCE.contains(source);
     }
+
+    /**
+     * 格式化视频时长秒
+     * @param videoLength
+     * @return
+     */
+    public static String formatVideoLength(String videoLength){
+        if (videoLength.contains(":")){
+            return videoLength;
+        }
+        double temp = Double.parseDouble(videoLength);
+        long mm = BigDecimal.valueOf(temp).divide(BigDecimal.valueOf(60), 0, RoundingMode.HALF_UP).longValue();
+        long ss = BigDecimal.valueOf(temp).divideAndRemainder(BigDecimal.valueOf(60))[1].setScale(0, RoundingMode.HALF_UP).longValue();
+        return ((mm < 10 ? ("0" + mm) : mm) + ":" + (ss < 10 ? ("0" + ss) : ss));
+    }
 }
\ No newline at end of file

src/main/resources/application-dev.properties

@@ -102,7 +102,7 @@ ef.checkCaptcha.url=https://ef.zhiweidata.com/captchaApi/checkeCaptcha.do.do?id=
 #\u5168\u7F51\u641C\u5916\u90E8\u63A5\u53E3
 istarshine.addIStarShineKSData.url=https://istarshine-service.zhiweidata.com/istarshine/ks/addIStarShineKSData
 istarshine.getIStarShineKSInfoData.url=https://istarshine-service.zhiweidata.com/istarshine/ks/getIStarShineKSInfoData?taskId={1}&appId=6183571e0d710000f6003a12
-whole.search.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/search/yuqing
+whole.search.url=https://yuqing.test.zhiweidata.com/qbjcbackPhoenix/interface/middleware/search/yuqing
 whole.searchCriteria.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/search/pt
 whole.extraParam.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/search/extra/param
 brief.report.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/brief-report-web