Merge branch 'feature' into 'release'

全网搜调整上线

See merge request !364

src/main/java/com/zhiwei/brandkbs2/controller/admin/ExtraController.java

+package com.zhiwei.brandkbs2.controller.admin;
+
+import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.brandkbs2.auth.Auth;
+import com.zhiwei.brandkbs2.controller.BaseController;
+import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
+import com.zhiwei.brandkbs2.easyexcel.dto.ExportLineDTO;
+import com.zhiwei.brandkbs2.easyexcel.dto.ExportWholeSearchRecordDTO;
+import com.zhiwei.brandkbs2.enmus.RoleEnum;
+import com.zhiwei.brandkbs2.model.ResponseResult;
+import com.zhiwei.brandkbs2.service.WholeSearchService;
+import io.swagger.annotations.*;
+import org.springframework.web.bind.annotation.*;
+
+import javax.annotation.Resource;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * @ClassName: ExtraController
+ * @Description 后台增值服务
+ * @author: sjj
+ * @date: 2023-07-18 17:24
+ */
+@RestController
+@RequestMapping("/admin/extra")
+@Api(tags = "增值服务")
+@Auth(role = RoleEnum.COMMON_ADMIN)
+public class ExtraController extends BaseController {
+
+    @Resource(name = "wholeSearchServiceImpl")
+    private WholeSearchService wholeSearchService;
+
+    @ApiOperation("全网搜-使用记录-查询")
+    @ApiImplicitParams({@ApiImplicitParam(name = "page", value = "页码", defaultValue = "1", paramType = "query", dataType = "int"),
+            @ApiImplicitParam(name = "pageSize", value = "每页记录数", defaultValue = "5", paramType = "query", dataType = "int"),
+            @ApiImplicitParam(name = "sorter", defaultValue = "{\"usedCount\":\"ascend\"}", value = "排序字段", paramType = "query", dataType = "string")}
+    )
+    @GetMapping("/used/list")
+    public ResponseResult findUseList(@RequestParam(value = "page", defaultValue = "1") int page,
+                                      @RequestParam(value = "size", defaultValue = "5") int pageSize,
+                                      @RequestParam(value = "sorter", defaultValue = "{\"usedCount\":\"ascend\"}") String sorter) {
+        return ResponseResult.success(wholeSearchService.findUsedList(page, pageSize, sorter));
+    }
+
+    @ApiOperation("全网搜-维护记录-查询")
+    @ApiImplicitParams({@ApiImplicitParam(name = "page", value = "页码", defaultValue = "1", paramType = "query", dataType = "int"),
+            @ApiImplicitParam(name = "pageSize", value = "每页记录数", defaultValue = "5", paramType = "query", dataType = "int")})
+    @GetMapping("/maintain/list")
+    public ResponseResult findMaintainList(@RequestParam(value = "page", defaultValue = "1") int page,
+                                           @RequestParam(value = "size", defaultValue = "5") int pageSize) {
+        return ResponseResult.success(wholeSearchService.findMaintainList(page, pageSize));
+    }
+
+    @ApiOperation("全网搜-数据总览")
+    @GetMapping("/overview")
+    public ResponseResult overview() {
+        return ResponseResult.success(wholeSearchService.overview());
+    }
+
+
+    @ApiOperation("全网搜-使用记录-导出")
+    @PostMapping("/used/download")
+    @Auth(role = RoleEnum.ADMIN)
+    public ResponseResult downloadUseList(@ApiParam(name = "json:{personal:个人明细=true,startTime:起始时间,endTime:结束时间,day:颗粒度天级=true}") @RequestBody JSONObject json) {
+        boolean personal = json.getBooleanValue("personal");
+        Long startTime = json.getLong("startTime");
+        Long endTime = json.getLong("endTime");
+        boolean day = json.getBooleanValue("day");
+        List<JSONObject> collect = wholeSearchService.outputUsedList(personal, startTime, endTime, day);
+        String fileName = startTime + "_" + endTime + "使用记录";
+        if (personal) {
+            EasyExcelUtil.download(fileName, "sheet", ExportWholeSearchRecordDTO.class, collect.stream().map(ExportWholeSearchRecordDTO::createFromJSONObject).collect(Collectors.toList()), response);
+        } else {
+            EasyExcelUtil.download(fileName, "sheet", ExportLineDTO.class, collect.stream().map(ExportLineDTO::createFromJSONObject).collect(Collectors.toList()), response);
+        }
+        return ResponseResult.success();
+    }
+
+    @ApiOperation("全网搜-维护记录-调整")
+    @PutMapping("/maintain/adjust")
+    @Auth(role = RoleEnum.ADMIN)
+    public ResponseResult adjustRecord(@ApiParam(name = "json:{count:调整数值)") @RequestBody JSONObject json) {
+        wholeSearchService.adjustRecord(json.getInteger("count"));
+        return ResponseResult.success();
+    }
+
+}

src/main/java/com/zhiwei/brandkbs2/controller/app/AppArticleController.java

@@ -12,10 +12,7 @@ import com.zhiwei.brandkbs2.pojo.dto.ExportAppYuqingDTO;
 import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
 import com.zhiwei.brandkbs2.pojo.dto.ReportDTO;
 import com.zhiwei.brandkbs2.pojo.dto.ReportSearchDTO;
-import com.zhiwei.brandkbs2.service.CommonService;
-import com.zhiwei.brandkbs2.service.MarkDataService;
-import com.zhiwei.brandkbs2.service.ProjectService;
-import com.zhiwei.brandkbs2.service.ReportService;
+import com.zhiwei.brandkbs2.service.*;
 import com.zhiwei.brandkbs2.util.Tools;
 import com.zhiwei.middleware.mark.vo.MarkerTag;
 import io.swagger.annotations.Api;
@@ -160,8 +157,8 @@ public class AppArticleController extends BaseController {
 
     @ApiOperation("舆情列表-搜索条件")
     @GetMapping("/mark/list/criteria")
-    public ResponseResult getYuqingMarkCriteria(@RequestParam(required = false) String linkedGroupId) {
-        return ResponseResult.success(markDataService.getYuqingMarkCriteria(linkedGroupId));
+    public ResponseResult getYuqingMarkCriteria() {
+        return ResponseResult.success(markDataService.getYuqingMarkCriteria());
     }
 
     @ApiOperation("舆情导出")
@@ -313,9 +310,12 @@ public class AppArticleController extends BaseController {
         List<MarkerTag> hitTags = projectService.getProjectById(UserThreadLocal.getProjectId()).getHitTags();
         if (!Tools.isEmpty(hitTags)) {
             for (String mustTag : hitTags.stream().map(MarkerTag::getUniqueId).collect(Collectors.toList())) {
+                // 不包含必要标签则添加
+                if (!mtag.toString().contains(mustTag)) {
                     mtag.append(mustTag);
                 }
             }
+        }
         return Tools.isIllegalTag(mtag.toString());
     }
 

src/main/java/com/zhiwei/brandkbs2/controller/app/AppHotController.java

@@ -292,8 +292,9 @@ public class AppHotController extends BaseController {
     @ApiOperation("热点库-品牌热点-热点总览")
     @GetMapping("/overview")
     public ResponseResult hotOverview() {
-        Long[] timeMonth = commonService.getTimeRangeMonth();
-        JSONObject result = searchHotHandler(10000, 1, allPlatform, "endTime", timeMonth[0], timeMonth[1]);
+        long endTime = System.currentTimeMillis();
+        long startTime = endTime - Constant.ONE_MONTH;
+        JSONObject result = searchHotHandler(10000, 1, allPlatform, "endTime", startTime, endTime);
         if (Objects.isNull(result)) {
             return ResponseResult.success();
         }

src/main/java/com/zhiwei/brandkbs2/controller/app/AppSearchController.java

@@ -11,6 +11,7 @@ import com.zhiwei.brandkbs2.controller.BaseController;
 import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
 import com.zhiwei.brandkbs2.enmus.RoleEnum;
 import com.zhiwei.brandkbs2.model.ResponseResult;
+import com.zhiwei.brandkbs2.pojo.WholeSearchRecord;
 import com.zhiwei.brandkbs2.pojo.dto.*;
 import com.zhiwei.brandkbs2.pojo.vo.ChannelListVO;
 import com.zhiwei.brandkbs2.service.*;
@@ -19,6 +20,7 @@ import com.zhiwei.middleware.event.pojo.dto.BrandkbsEventSearchDTO;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
 import org.apache.commons.lang3.time.DateUtils;
+import org.apache.commons.lang3.tuple.Pair;
 import org.joda.time.Period;
 import org.joda.time.PeriodType;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -76,6 +78,9 @@ public class AppSearchController extends BaseController {
     @Resource(name = "projectServiceImpl")
     ProjectService projectService;
 
+    @Resource(name = "wholeSearchServiceImpl")
+    WholeSearchService wholeSearchService;
+
     @ApiOperation("搜索-查热点")
     @LogRecord(values = "keyword", description = "查热点", arguments = true, entity = false)
     @GetMapping("/hot/list")
@@ -139,7 +144,7 @@ public class AppSearchController extends BaseController {
     @PostMapping("/searchWhole")
     public ResponseResult searchWholeNetwork(@RequestBody SearchFilterDTO dto) {
         // 针对商业数据库做限制
-        if (2 == projectService.getProjectById(UserThreadLocal.getProjectId()).getWholeSearchDataSource()) {
+        if (dto.isExternalDataSource()) {
             long time = DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), -89).getTime();
             if (time > dto.getStartTime()) {
                 // 仅对查商业数据库时限制时间，查舆情库时本质上无时间限制
@@ -150,8 +155,17 @@ public class AppSearchController extends BaseController {
                 // 仅对查商业数据库时限制时间，查舆情库时本质上无时间限制
                 return ResponseResult.failure("时间跨度不能超过30天");
             }
+            if (dto.getPageSize() >= projectService.getProjectById(UserThreadLocal.getProjectId()).getWholeSearchBalance()) {
+                return ResponseResult.failure("实时采集余额不足");
+            }
+        }
+        Pair<JSONObject, Integer> pair = markDataService.searchWholeNetworkWithBalance(dto);
+        if(dto.isSentiment()){
+            List<JSONObject> list = pair.getLeft().getJSONArray("list").toJavaList(JSONObject.class);
+            commonService.sentimentAnalysis(list);
         }
-        return ResponseResult.success(markDataService.searchWholeNetwork(dto));
+        wholeSearchService.decreaseRecord(dto.getSearch(), WholeSearchRecord.UsedType.query, pair.getRight());
+        return ResponseResult.success(pair.getLeft());
     }
 
     @ApiOperation("搜索-全网搜-舆情导出")
@@ -159,7 +173,7 @@ public class AppSearchController extends BaseController {
     @LogRecord(description = "全网搜-舆情导出", values = {"startTime", "endTime", "fans", "filterType", "filterWords", "search", "keyword", "platforms", "sensitiveChannels", "sourceKeyword"}, entity = true, arguments = true)
     public ResponseResult exportSearchWhole(@RequestBody SearchFilterDTO dto) {
         // 针对商业数据库做限制
-        if (2 == projectService.getProjectById(UserThreadLocal.getProjectId()).getWholeSearchDataSource()) {
+        if (dto.isExternalDataSource()) {
             long time = DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), -89).getTime();
             if (time > dto.getStartTime()) {
                 // 仅对查商业数据库时限制时间，查舆情库时本质上无时间限制
@@ -170,9 +184,15 @@ public class AppSearchController extends BaseController {
                 // 仅对查商业数据库时限制时间，查舆情库时本质上无时间限制
                 return ResponseResult.failure("时间跨度不能超过30天");
             }
+            if (dto.getOutputCount() >= projectService.getProjectById(UserThreadLocal.getProjectId()).getWholeSearchBalance()) {
+                return ResponseResult.failure("实时采集余额不足");
+            }
         }
         List<ExportSearchWholeDTO> exportList = markDataService.exportSearchWhole(dto);
         EasyExcelUtil.download("全网搜舆情列表数据", "sheet1", ExportSearchWholeDTO.class, exportList, response);
+        if (dto.isExternalDataSource()) {
+            wholeSearchService.decreaseRecord(dto.getSearch(), WholeSearchRecord.UsedType.output, exportList.size());
+        }
         return ResponseResult.success();
     }
 
@@ -192,8 +212,8 @@ public class AppSearchController extends BaseController {
 
     @ApiOperation("搜索-舆情列表-搜索条件")
     @GetMapping("/mark/list/criteria")
-    public ResponseResult getYuqingMarkCriteria(@RequestParam(required = false) String linkedGroupId) {
-        return ResponseResult.success(markDataService.getYuqingMarkCriteria(linkedGroupId));
+    public ResponseResult getYuqingMarkCriteria() {
+        return ResponseResult.success(markDataService.getYuqingMarkCriteria());
     }
 
     @ApiOperation("搜索-活跃渠道")

src/main/java/com/zhiwei/brandkbs2/dao/WholeSearchRecordDao.java

+package com.zhiwei.brandkbs2.dao;
+
+import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.brandkbs2.pojo.WholeSearchRecord;
+import com.zhiwei.brandkbs2.pojo.vo.LineVO;
+
+import java.util.List;
+
+/**
+ * @ClassName: WholeSearchRecordDao
+ * @Description WholeSearchRecordDao
+ * @author: sjj
+ * @date: 2023-07-19 11:57
+ */
+public interface WholeSearchRecordDao extends BaseMongoDao<WholeSearchRecord>, ShardingMongo{
+
+    /**
+     *
+     * @param startTime 起始时间
+     * @param endTime 结束时间
+     * @param day 颗粒度
+     * @return list
+     */
+    List<JSONObject> aggregateProjectUsedRecord(long startTime, long endTime, boolean day);
+
+    /**
+     *
+     * @param startTime 起始时间
+     * @param endTime 结束时间
+     * @return list
+     */
+    List<LineVO> aggregateDayLastRecord(long startTime, long endTime);
+
+}

src/main/java/com/zhiwei/brandkbs2/dao/impl/ProjectDaoImlp.java → src/main/java/com/zhiwei/brandkbs2/dao/impl/ProjectDaoImpl.java

@@ -5,17 +5,17 @@ import com.zhiwei.brandkbs2.pojo.Project;
 import org.springframework.stereotype.Component;
 
 /**
- * @ClassName: ProjectDaoImlp
+ * @ClassName: ProjectDaoImpl
  * @Description 项目业务接口实现类
  * @author: sjj
  * @date: 2022-04-21 13:51
  */
 @Component("projectDao")
-public class ProjectDaoImlp extends BaseMongoDaoImpl<Project> implements ProjectDao {
+public class ProjectDaoImpl extends BaseMongoDaoImpl<Project> implements ProjectDao {
 
     private static final String COLLECTION_NAME = "brandkbs_project";
 
-    public ProjectDaoImlp() {
+    public ProjectDaoImpl() {
         super(COLLECTION_NAME);
     }
 

src/main/java/com/zhiwei/brandkbs2/dao/impl/WholeSearchRecordDaoImpl.java

+package com.zhiwei.brandkbs2.dao.impl;
+
+import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.brandkbs2.auth.UserThreadLocal;
+import com.zhiwei.brandkbs2.dao.WholeSearchRecordDao;
+import com.zhiwei.brandkbs2.pojo.WholeSearchRecord;
+import com.zhiwei.brandkbs2.pojo.vo.LineVO;
+import org.springframework.data.domain.Sort;
+import org.springframework.data.mongodb.core.aggregation.Aggregation;
+import org.springframework.data.mongodb.core.aggregation.AggregationOperation;
+import org.springframework.data.mongodb.core.aggregation.AggregationResults;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.stereotype.Component;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+/**
+ * @ClassName: WholeSearchRecordDaoImpl
+ * @Description WholeSearchRecordDaoImpl
+ * @author: sjj
+ * @date: 2023-07-19 12:02
+ */
+@Component("wholeSearchRecordDao")
+public class WholeSearchRecordDaoImpl extends BaseMongoDaoImpl<WholeSearchRecord> implements WholeSearchRecordDao{
+    private static final String COLLECTION_PREFIX = "brandkbs_whole_search_record";
+
+    private static final String TIME_PATTERN = "yyyy";
+
+    public WholeSearchRecordDaoImpl() {
+        super(null);
+    }
+
+    @Override
+    public String collectionPrefix() {
+        return COLLECTION_PREFIX;
+    }
+
+    @Override
+    public String timePattern() {
+        return TIME_PATTERN;
+    }
+
+    @Override
+    public List<JSONObject> aggregateProjectUsedRecord(long startTime, long endTime, boolean day) {
+        List<JSONObject> res = new ArrayList<>();
+        String[] collectionNames = generateCollectionNames(new Date(startTime), new Date(endTime));
+        Criteria criteria = Criteria.where("isUsed").is(true).and("projectId").is(UserThreadLocal.getProjectId());
+        int nrOfChars = day ? 10 : 7;
+        List<AggregationOperation> operations = new ArrayList<>();
+        // operations区分先后顺序
+        operations.add(Aggregation.match(criteria));
+        operations.add(Aggregation.project( "usedCount", "cTime").and("cTime").substring(0, nrOfChars).as("cTime"));
+        operations.add(Aggregation.group("cTime").sum("usedCount").as("count"));
+        operations.add(Aggregation.sort(Sort.by(Sort.Order.desc("_id"))));
+        Aggregation aggregation = Aggregation.newAggregation(operations);
+        for (String collectionName : collectionNames) {
+            AggregationResults<JSONObject> aggregateResult = mongoTemplate.aggregate(aggregation, collectionName, JSONObject.class);
+            res.addAll(aggregateResult.getMappedResults());
+        }
+        return res;
+    }
+
+    @Override
+    public List<LineVO> aggregateDayLastRecord(long startTime, long endTime) {
+        List<LineVO> res = new ArrayList<>();
+        String[] collectionNames = generateCollectionNames(new Date(startTime), new Date(endTime));
+        Criteria criteria = Criteria.where("projectId").is(UserThreadLocal.getProjectId());
+        List<AggregationOperation> operations = new ArrayList<>();
+        // operations区分先后顺序
+        operations.add(Aggregation.match(criteria));
+        operations.add(Aggregation.project("currentCount").and("cTime").substring(0, 10).as("cTime"));
+        operations.add(Aggregation.group("cTime").last("currentCount").as("count"));
+        operations.add(Aggregation.sort(Sort.by(Sort.Order.asc("_id"))));
+        Aggregation aggregation = Aggregation.newAggregation(operations);
+        // 优先小的年份
+        for (int i = collectionNames.length - 1; i >= 0; i--) {
+            AggregationResults<JSONObject> aggregateResult = mongoTemplate.aggregate(aggregation, collectionNames[i], JSONObject.class);
+            List<JSONObject> mappedResults = aggregateResult.getMappedResults();
+            for (JSONObject mappedResult : mappedResults) {
+                res.add(new LineVO(mappedResult.getLong("count"), mappedResult.getDate("_id").getTime()));
+            }
+        }
+        return res;
+    }
+}

src/main/java/com/zhiwei/brandkbs2/easyexcel/dto/ExportLineDTO.java

+package com.zhiwei.brandkbs2.easyexcel.dto;
+
+import com.alibaba.excel.annotation.ExcelProperty;
+import com.alibaba.fastjson.JSONObject;
+import lombok.Data;
+import lombok.ToString;
+
+import java.util.Date;
+
+/**
+ * @ClassName: ExportLineDTO
+ * @Description ExportLineDTO
+ * @author: sjj
+ * @date: 2023-07-19 18:07
+ */
+@Data
+@ToString
+public class ExportLineDTO {
+
+    @ExcelProperty("时间")
+    private Date time;
+
+    @ExcelProperty("数据量")
+    private Long count;
+
+    public static ExportLineDTO createFromJSONObject(JSONObject json) {
+        ExportLineDTO dto = new ExportLineDTO();
+        dto.setTime(json.getDate("_id"));
+        dto.setCount(json.getLong("count"));
+        return dto;
+    }
+
+}

src/main/java/com/zhiwei/brandkbs2/easyexcel/dto/ExportWholeSearchRecordDTO.java

+package com.zhiwei.brandkbs2.easyexcel.dto;
+
+import com.alibaba.excel.annotation.ExcelProperty;
+import com.alibaba.fastjson.JSONObject;
+import lombok.Data;
+import lombok.ToString;
+
+import java.util.Date;
+
+/**
+ * @ClassName: ExportWholeSearchRecordDTO
+ * @Description ExportWholeSearchRecordDTO
+ * @author: sjj
+ * @date: 2023-07-19 18:02
+ */
+@Data
+@ToString
+public class ExportWholeSearchRecordDTO {
+
+    @ExcelProperty("关键词")
+    private String keyword;
+
+    @ExcelProperty("类型")
+    private String usedType;
+
+    @ExcelProperty("数据量")
+    private Integer usedCount;
+
+    @ExcelProperty("操作时间")
+    private Date cTime;
+
+    @ExcelProperty("操作人")
+    private String submitter;
+
+    public static ExportWholeSearchRecordDTO createFromJSONObject(JSONObject json) {
+        ExportWholeSearchRecordDTO dto = new ExportWholeSearchRecordDTO();
+        dto.setKeyword(json.getString("keyword"));
+        dto.setUsedType(json.getString("usedType"));
+        dto.setUsedCount(json.getInteger("usedCount"));
+        ;
+        dto.setCTime(new Date(json.getLong("cTime")));
+        dto.setSubmitter(json.getString("submitter"));
+        return dto;
+    }
+}

src/main/java/com/zhiwei/brandkbs2/enmus/HotPlatformEnum.java

@@ -20,7 +20,8 @@ public enum HotPlatformEnum {
     ZHIHU("知乎", "zhihu", "知乎热搜"),
     JINRITOUTIAO("今日头条", "toutiao", "今日头条热搜"),
     DOUYIN("抖音", "douyin", "抖音热搜"),
-    BILIBILI("B站", "bilibili-ranking", "B站热搜"),
+    BILIBILI("B站", "bilibili", "B站热搜"),
+    BILIBILI_RANKING("B站", "bilibili-ranking", "B站排行榜"),
     KUAISHOU("快手", "kuaishou", "快手热榜"),
     BAIDU("百度", "baidu", "百度热搜");
 

src/main/java/com/zhiwei/brandkbs2/pojo/Project.java

@@ -92,6 +92,11 @@ public class Project extends AbstractProject {
     private List<String> highlightKeyword;
 
     /**
+     * 全网搜余额
+     */
+    private int wholeSearchBalance;
+
+    /**
      * 项目主品牌配置信息转换
      *
      * @return 项目对象

src/main/java/com/zhiwei/brandkbs2/pojo/WholeSearchRecord.java

+package com.zhiwei.brandkbs2.pojo;
+
+import lombok.Getter;
+import lombok.Setter;
+
+import java.util.Date;
+
+/**
+ * @ClassName: WholeSearchRecord
+ * @Description 全网搜记录实体类
+ * @author: sjj
+ * @date: 2023-07-19 11:21
+ */
+@Getter
+@Setter
+public class WholeSearchRecord extends AbstractBaseMongo {
+
+    /**
+     * 项目id
+     */
+    private String projectId;
+
+    /**
+     * 搜索关键词
+     */
+    private String keyword;
+
+    /**
+     * 使用记录
+     */
+    private boolean isUsed;
+
+    /**
+     * 使用类型(导出/查询)
+     */
+    private String usedType;
+
+    /**
+     * 上一次记录数
+     */
+    private int lastCount;
+
+    /**
+     * 该次操作使用(减少)数值
+     */
+    private int usedCount;
+
+    /**
+     * 本次记录数
+     */
+    private int currentCount;
+
+    /**
+     * 提交人
+     */
+    private String submitter;
+
+    /**
+     * 创建时间(Date类型方便聚合查询)
+     */
+    private Date cTime;
+
+
+    private WholeSearchRecord() {
+    }
+
+    public static WholeSearchRecord createUsedRecord(String projectId, String keyword, String usedType, int lastCount, int usedCount, String submitter) {
+        WholeSearchRecord record = new WholeSearchRecord();
+        record.projectId = projectId;
+        record.keyword = keyword;
+        record.isUsed = true;
+        record.usedType = usedType;
+        record.lastCount = lastCount;
+        record.usedCount = usedCount;
+        record.currentCount = lastCount - usedCount;
+        record.submitter = submitter;
+        record.cTime = new Date();
+        return record;
+    }
+
+    public static WholeSearchRecord createMaintainRecord(String projectId, int lastCount, int currentCount, String submitter) {
+        WholeSearchRecord record = new WholeSearchRecord();
+        record.projectId = projectId;
+        record.isUsed = false;
+        record.lastCount = lastCount;
+        record.currentCount = currentCount;
+        record.submitter = submitter;
+        record.cTime = new Date();
+        return record;
+    }
+
+    public enum UsedType {
+        query("查询"),
+        output("导出");
+        @Getter
+        private final String value;
+
+        UsedType(String value) {
+            this.value = value;
+        }
+    }
+}

src/main/java/com/zhiwei/brandkbs2/pojo/dto/MarkSearchDTO.java

@@ -154,4 +154,10 @@ public class MarkSearchDTO {
     @ApiModelProperty(value = "是否去重")
     private boolean fold;
 
+    /**
+     *
+     */
+    @ApiModelProperty(value = "屏蔽词")
+    private String shieldWord;
+
 }

src/main/java/com/zhiwei/brandkbs2/pojo/dto/SearchFilterDTO.java

@@ -12,12 +12,6 @@ import java.util.*;
 public class SearchFilterDTO {
 
     /**
-     * 项目Id
-     */
-    @ApiModelProperty(value = "项目id")
-    private String projectId;
-
-    /**
      * 开始时间戳
      */
     @ApiModelProperty(value = "开始时间")
@@ -42,6 +36,12 @@ public class SearchFilterDTO {
     private int pageSize;
 
     /**
+     * "采集时间(商业数据翻页用)"
+     */
+    @ApiModelProperty("采集时间(商业数据翻页用)")
+    private Long crawlerTime;
+
+    /**
      * 关键字
      */
     @ApiModelProperty(value = "关键字")
@@ -54,54 +54,24 @@ public class SearchFilterDTO {
     private List<String> platforms;
 
     /**
-     * 平台(查商业数据库用)
-     */
-    @ApiModelProperty(value = "平台(查商业数据库用)")
-    private String platformsStr;
-
-    /**
-     * 数据类型 1:原创, 2:转发, 7:评论 (查商业数据库用)
-     */
-    @ApiModelProperty(value = "数据类型(查商业数据库用)")
-    private String filterForward = "1,2";
-
-    /**
      * 排序字段
      */
     @ApiModelProperty(value = "排序字段")
-    private JSONObject sorter;
-
-    /**
-     * 排序字段(查商业数据库用)
-     */
-    @ApiModelProperty(value = "排序字段(查商业数据库用)")
-    private String sortOf;
+    private JSONObject sorter = JSONObject.parseObject("{\"time\":\"descend\"}");
 
     /**
-     * 来源
+     * 渠道关键字
      */
-    @ApiModelProperty(value = "来源")
+    @ApiModelProperty(value = "渠道关键字")
     private String sourceKeyword;
 
     /**
-     * 域名
+     * 域名关键字
      */
-    @ApiModelProperty(value = "域名")
+    @ApiModelProperty(value = "域名关键字")
     private String hostKeyword;
 
     /**
-     * 过滤关键字
-     */
-    @ApiModelProperty(value = "过滤关键字")
-    private String filterWords;
-
-    /**
-     * 匹配域名 默认标题+内容(查商业数据库用)
-     */
-    @ApiModelProperty(value = "匹配域名(查商业数据库用)")
-    private String matchFields = null;
-
-    /**
      * 查询类型 标题/全文
      */
     @ApiModelProperty(value = "查询类型")
@@ -116,7 +86,7 @@ public class SearchFilterDTO {
     /**
      * 粉丝数
      */
-    @ApiModelProperty(value = "粉丝数")
+    @ApiModelProperty(value = "粉丝数 千万/百万")
     private String fans;
 
     /**
@@ -125,6 +95,53 @@ public class SearchFilterDTO {
     @ApiModelProperty(value = "过滤类型 标题/全文")
     private String filterType;
 
+    /**
+     * 过滤关键字
+     */
+    @ApiModelProperty(value = "过滤关键字")
+    private String filterWords;
+
+    /**
+     * 外部数据源
+     */
+    @ApiModelProperty(value = "是否为外部数据源,默认为false")
+    private boolean externalDataSource = false;
+
+    /**
+     * 情感倾向分析
+     */
+    @ApiModelProperty(value = "情感倾向分析,默认为false")
+    private boolean sentiment = false;
+
+    /**
+     * 导出数据量
+     */
+    @ApiModelProperty(value = "导出数据量")
+    private int outputCount;
+
+//    /**
+//     * 这部分字段无需前端传参
+//     */
+//    /**
+//     * 平台(查商业数据库用)
+//     */
+//    private String platformsStr;
+//
+//    /**
+//     * 数据类型 1:原创, 2:转发, 7:评论 (查商业数据库用)
+//     */
+//    private String filterForward = "1,2";
+//
+//    /**
+//     * 匹配域名 默认标题+内容(查商业数据库用)
+//     */
+//    private String matchFields = null;
+//
+//    /**
+//     * 排序字段(查商业数据库用)
+//     */
+//    private String sortOf;
+
     public Map<String, Object> toMap() {
         Map<String, Object> map = new HashMap<>();
         map.put("timeType", "time");

src/main/java/com/zhiwei/brandkbs2/pojo/vo/LineVO.java

 package com.zhiwei.brandkbs2.pojo.vo;
 
+import lombok.AllArgsConstructor;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 import lombok.ToString;
 
-import java.util.Date;
-
 /**
  * @author lxj
  * @version 1.0
@@ -13,6 +13,8 @@ import java.util.Date;
  */
 @Data
 @ToString
+@NoArgsConstructor
+@AllArgsConstructor
 public class LineVO {
     /**
      * 个数

src/main/java/com/zhiwei/brandkbs2/service/CommonService.java

@@ -59,6 +59,13 @@ public interface CommonService {
      */
     void addMtagBatch(List<MarkInfo> markInfos);
 
+    /**
+     * 情感倾向分析
+     * @param texts
+     * @return
+     */
+    List<JSONObject> sentimentAnalysis(List<JSONObject> texts);
+
     Long[] getTimeRangeYear();
 
     Long[] getTimeRangeMonth();

src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java

@@ -108,7 +108,7 @@ public interface MarkDataService {
     /**
      * 舆情标注数据搜索条件
      */
-    JSONObject getYuqingMarkCriteria(String linkedGroup);
+    JSONObject getYuqingMarkCriteria();
 
     /**
      * 舆情标注数据搜索条件
@@ -304,7 +304,7 @@ public interface MarkDataService {
      * @param dto 全网搜搜索传输类
      * @return JSONObject
      */
-    JSONObject searchWholeNetwork(SearchFilterDTO dto);
+    Pair<JSONObject, Integer> searchWholeNetworkWithBalance(SearchFilterDTO dto);
 
     /**
      * 搜索-全网搜-舆情导出

src/main/java/com/zhiwei/brandkbs2/service/WholeSearchService.java

+package com.zhiwei.brandkbs2.service;
+
+import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.brandkbs2.pojo.WholeSearchRecord;
+import com.zhiwei.brandkbs2.pojo.vo.PageVO;
+
+import java.util.List;
+
+/**
+ * @ClassName: WholeSearchService
+ * @Description 全网搜服务类
+ * @author: sjj
+ * @date: 2023-07-19 11:43
+ */
+public interface WholeSearchService {
+
+    /**
+     * 查询使用列表
+     *
+     * @param page 页码
+     * @param pageSize 页码大小
+     * @param sorter 排序字段
+     * @return PageVO
+     */
+    PageVO<JSONObject> findUsedList(int page,int pageSize,String sorter);
+
+    /**
+     * 查询维护列表
+     *
+     * @param page 页码
+     * @param pageSize 页码大小
+     * @return PageVO
+     */
+    PageVO<JSONObject> findMaintainList(int page,int pageSize);
+
+    /**
+     * 导出使用列表
+     * @param personal 个人明细/项目明细
+     * @param startTime 开始时间
+     * @param endTime 结束时间
+     * @param day 颗粒度(天级/月级)
+     * @return List
+     */
+    List<JSONObject> outputUsedList(boolean personal,long startTime,long endTime,boolean day);
+
+    /**
+     * @param currentCount 调整后数值
+     */
+    void adjustRecord(Integer currentCount);
+
+    /**
+     * 数据总览
+     * @return json
+     */
+    JSONObject overview();
+
+    /**
+     * @param keyword 关键词
+     * @param usedType 使用类别
+     * @param usedCount 使用量
+     */
+    void decreaseRecord(String keyword, WholeSearchRecord.UsedType usedType, Integer usedCount);
+
+}

src/main/java/com/zhiwei/brandkbs2/service/impl/CommonServiceImpl.java

@@ -15,6 +15,7 @@ import com.zhiwei.middleware.mark.vo.TagSearch;
 import org.apache.commons.lang3.time.DateUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Service;
 import org.springframework.web.client.RestTemplate;
 
@@ -132,6 +133,26 @@ public class CommonServiceImpl implements CommonService {
     }
 
     @Override
+    public List<JSONObject> sentimentAnalysis(List<JSONObject> texts) {
+        ResponseEntity<JSONObject> responseEntity = restTemplate.postForEntity("https://sentiment.zhiweidata.com/long/multi/longText", texts.stream().map(json ->
+                {
+                    String title = json.getString("title");
+                    return null == title ? json.getString("content") : title;
+                }
+        ).collect(Collectors.toList()), JSONObject.class);
+        if (Objects.requireNonNull(responseEntity.getBody()).getBooleanValue("status")) {
+            if (Objects.nonNull(responseEntity.getBody().get("result"))) {
+                List<JSONObject> result = responseEntity.getBody().getJSONArray("result").toJavaList(JSONObject.class);
+                for (int i = 0; i < result.size(); i++) {
+                    String emotion = result.get(i).getString("emotion");
+                    texts.get(i).put("emotion", emotion);
+                }
+            }
+        }
+        return texts;
+    }
+
+    @Override
     public Long[] getTimeRangeYear() {
         long endTime = Tools.truncDate(new Date(), Constant.DAY_PATTERN).getTime();
         long startTime = DateUtils.addYears(new Date(endTime), -1).getTime();

src/main/java/com/zhiwei/brandkbs2/service/impl/EsSearchServiceImpl.java

@@ -91,12 +91,16 @@ public class EsSearchServiceImpl implements EsSearchService {
         }
         // Query 查询条件
 //        BoolQueryBuilder query = QueryBuilders.boolQuery();
+        String[] fieldSearch = "仅标题".equals(dto.getSearchField()) ? new String[]{GenericAttribute.ES_IND_TITLE} : new String[]{GenericAttribute.ES_IND_FULL_TEXT};
         // keyword
         if (StringUtils.isNotEmpty(dto.getKeyword())) {
             // Query
-            String[] fieldSearch = "仅标题".equals(dto.getSearchField()) ? new String[]{GenericAttribute.ES_IND_TITLE} : new String[]{GenericAttribute.ES_IND_FULL_TEXT};
             postFilter.must(EsQueryTools.assembleNormalKeywordQuery(dto.getKeyword(), fieldSearch));
         }
+        // shieldWord
+        if (StringUtils.isNotEmpty(dto.getShieldWord())) {
+            postFilter.mustNot(EsQueryTools.assembleNormalKeywordQuery(dto.getShieldWord(), fieldSearch));
+        }
         // sourceKeyword
         if (StringUtils.isNotEmpty(dto.getSourceKeyword())) {
             postFilter.must(EsQueryTools.assembleSourceQuery(dto.getSourceKeyword()));

src/main/java/com/zhiwei/brandkbs2/service/impl/ProjectServiceImpl.java

@@ -462,7 +462,7 @@ public class ProjectServiceImpl implements ProjectService {
                 }
             });
         }
-        return lists.stream().filter(Project::isShow).map(project -> {
+        return lists.stream().sorted(Comparator.comparingLong(Project::getCTime)).filter(Project::isShow).map(project -> {
             JSONObject json = new JSONObject();
             json.put("id", project.getId());
             json.put("brands", getBrands(project.getId(), true));

src/main/java/com/zhiwei/brandkbs2/service/impl/WholeSearchServiceImpl.java

+package com.zhiwei.brandkbs2.service.impl;
+
+import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.brandkbs2.auth.UserThreadLocal;
+import com.zhiwei.brandkbs2.config.Constant;
+import com.zhiwei.brandkbs2.dao.ProjectDao;
+import com.zhiwei.brandkbs2.dao.WholeSearchRecordDao;
+import com.zhiwei.brandkbs2.pojo.Project;
+import com.zhiwei.brandkbs2.pojo.WholeSearchRecord;
+import com.zhiwei.brandkbs2.pojo.vo.PageVO;
+import com.zhiwei.brandkbs2.service.WholeSearchService;
+import com.zhiwei.brandkbs2.util.MongoUtil;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.data.mongodb.core.query.Query;
+import org.springframework.data.mongodb.core.query.Update;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.Resource;
+import java.util.Date;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * @ClassName: WholeSearchServiceImpl
+ * @Description WholeSearchServiceImpl
+ * @author: sjj
+ * @date: 2023-07-19 11:55
+ */
+@Service("wholeSearchServiceImpl")
+public class WholeSearchServiceImpl implements WholeSearchService {
+
+    @Resource(name = "wholeSearchRecordDao")
+    private WholeSearchRecordDao wholeSearchRecordDao;
+
+    @Resource(name = "projectDao")
+    private ProjectDao projectDao;
+
+    @Resource(name = "mongoUtil")
+    private MongoUtil mongoUtil;
+
+    @Override
+    public PageVO<JSONObject> findUsedList(int page, int pageSize, String sorter) {
+        String collectionName = wholeSearchRecordDao.generateCollectionName();
+        Query query = Query.query(Criteria.where("projectId").is(UserThreadLocal.getProjectId()).and("isUsed").is(true));
+        wholeSearchRecordDao.addSort(query, sorter);
+        long total = wholeSearchRecordDao.count(query, collectionName);
+        mongoUtil.start(page, pageSize, query);
+        return PageVO.createPageVo(total, page, pageSize, usedList(query, collectionName));
+    }
+
+    @Override
+    public PageVO<JSONObject> findMaintainList(int page, int pageSize) {
+        String collectionName = wholeSearchRecordDao.generateCollectionName();
+        Query query = Query.query(Criteria.where("projectId").is(UserThreadLocal.getProjectId()).and("isUsed").is(false));
+        wholeSearchRecordDao.addSort(query, "{\"cTime\":\"descend\"}");
+        long total = wholeSearchRecordDao.count(query, collectionName);
+        mongoUtil.start(page, pageSize, query);
+        List<JSONObject> collect = wholeSearchRecordDao.findList(query, collectionName).stream().map(record -> {
+            JSONObject json = new JSONObject();
+            json.put("lastCount", record.getLastCount());
+            json.put("currentCount", record.getCurrentCount());
+            json.put("cTime", record.getCTime().getTime());
+            json.put("submitter", record.getSubmitter());
+            return json;
+        }).collect(Collectors.toList());
+        return PageVO.createPageVo(total, page, pageSize, collect);
+    }
+
+    @Override
+    public List<JSONObject> outputUsedList(boolean personal, long startTime, long endTime, boolean day) {
+        // 个人明细
+        if (personal) {
+            Criteria criteria = Criteria.where("isUsed").is(true);
+            criteria.and("submitter").is(UserThreadLocal.getNickname());
+            Query query = Query.query(criteria);
+            wholeSearchRecordDao.addSort(query, "{\"cTime\":\"descend\"}");
+            return usedList(query, wholeSearchRecordDao.generateCollectionNames(new Date(startTime), new Date(endTime)));
+        } else {// 项目明细,需要细分颗粒度
+            return wholeSearchRecordDao.aggregateProjectUsedRecord(startTime, endTime, day);
+        }
+    }
+
+    @Override
+    public void adjustRecord(Integer currentCount) {
+        if (null == currentCount) {
+            return;
+        }
+        String projectId = UserThreadLocal.getProjectId();
+        Update update = Update.update("wholeSearchBalance", currentCount);
+        Project project = projectDao.findOneById(projectId);
+        projectDao.updateOneByIdWithField(projectId, update);
+        wholeSearchRecordDao.insertOne(WholeSearchRecord.createMaintainRecord(projectId, project.getWholeSearchBalance(), currentCount, UserThreadLocal.getNickname()), wholeSearchRecordDao.generateCollectionName());
+    }
+
+    @Override
+    public JSONObject overview() {
+        JSONObject json = new JSONObject();
+        long endTime = System.currentTimeMillis();
+        long startTime = endTime - Constant.ONE_MONTH;
+        json.put("balance", projectDao.findOneById(UserThreadLocal.getProjectId()).getWholeSearchBalance());
+        json.put("trendList", wholeSearchRecordDao.aggregateDayLastRecord(startTime, endTime));
+        return json;
+    }
+
+    @Override
+    public void decreaseRecord(String keyword, WholeSearchRecord.UsedType usedType, Integer usedCount) {
+        if (null == usedCount) {
+            return;
+        }
+        String projectId = UserThreadLocal.getProjectId();
+        Update update = new Update();
+        update.inc("wholeSearchBalance", -usedCount);
+        Project project = projectDao.findOneById(projectId);
+        projectDao.updateOneByIdWithField(projectId, update);
+        wholeSearchRecordDao.insertOne(WholeSearchRecord.createUsedRecord(projectId, keyword, usedType.getValue(), project.getWholeSearchBalance(), usedCount, UserThreadLocal.getNickname()), wholeSearchRecordDao.generateCollectionName());
+    }
+
+    private List<JSONObject> usedList(Query query, String... collectionNames) {
+        return wholeSearchRecordDao.findList(query, collectionNames).stream().map(record -> {
+            JSONObject json = new JSONObject();
+            json.put("keyword", record.getKeyword());
+            json.put("usedType", record.getUsedType());
+            json.put("usedCount", record.getUsedCount());
+            json.put("cTime", record.getCTime().getTime());
+            json.put("submitter", record.getSubmitter());
+            return json;
+        }).collect(Collectors.toList());
+    }
+}