Merge branch 'release' into 'master'

Release

See merge request !365

src/main/java/com/zhiwei/brandkbs2/controller/admin/ExtraController.java

+package com.zhiwei.brandkbs2.controller.admin;
+
+import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.brandkbs2.auth.Auth;
+import com.zhiwei.brandkbs2.controller.BaseController;
+import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
+import com.zhiwei.brandkbs2.easyexcel.dto.ExportLineDTO;
+import com.zhiwei.brandkbs2.easyexcel.dto.ExportWholeSearchRecordDTO;
+import com.zhiwei.brandkbs2.enmus.RoleEnum;
+import com.zhiwei.brandkbs2.model.ResponseResult;
+import com.zhiwei.brandkbs2.service.WholeSearchService;
+import io.swagger.annotations.*;
+import org.springframework.web.bind.annotation.*;
+
+import javax.annotation.Resource;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * @ClassName: ExtraController
+ * @Description 后台增值服务
+ * @author: sjj
+ * @date: 2023-07-18 17:24
+ */
+@RestController
+@RequestMapping("/admin/extra")
+@Api(tags = "增值服务")
+@Auth(role = RoleEnum.COMMON_ADMIN)
+public class ExtraController extends BaseController {
+
+    @Resource(name = "wholeSearchServiceImpl")
+    private WholeSearchService wholeSearchService;
+
+    @ApiOperation("全网搜-使用记录-查询")
+    @ApiImplicitParams({@ApiImplicitParam(name = "page", value = "页码", defaultValue = "1", paramType = "query", dataType = "int"),
+            @ApiImplicitParam(name = "pageSize", value = "每页记录数", defaultValue = "5", paramType = "query", dataType = "int"),
+            @ApiImplicitParam(name = "sorter", defaultValue = "{\"usedCount\":\"ascend\"}", value = "排序字段", paramType = "query", dataType = "string")}
+    )
+    @GetMapping("/used/list")
+    public ResponseResult findUseList(@RequestParam(value = "page", defaultValue = "1") int page,
+                                      @RequestParam(value = "size", defaultValue = "5") int pageSize,
+                                      @RequestParam(value = "sorter", defaultValue = "{\"usedCount\":\"ascend\"}") String sorter) {
+        return ResponseResult.success(wholeSearchService.findUsedList(page, pageSize, sorter));
+    }
+
+    @ApiOperation("全网搜-维护记录-查询")
+    @ApiImplicitParams({@ApiImplicitParam(name = "page", value = "页码", defaultValue = "1", paramType = "query", dataType = "int"),
+            @ApiImplicitParam(name = "pageSize", value = "每页记录数", defaultValue = "5", paramType = "query", dataType = "int")})
+    @GetMapping("/maintain/list")
+    public ResponseResult findMaintainList(@RequestParam(value = "page", defaultValue = "1") int page,
+                                           @RequestParam(value = "size", defaultValue = "5") int pageSize) {
+        return ResponseResult.success(wholeSearchService.findMaintainList(page, pageSize));
+    }
+
+    @ApiOperation("全网搜-数据总览")
+    @GetMapping("/overview")
+    public ResponseResult overview() {
+        return ResponseResult.success(wholeSearchService.overview());
+    }
+
+
+    @ApiOperation("全网搜-使用记录-导出")
+    @PostMapping("/used/download")
+    @Auth(role = RoleEnum.ADMIN)
+    public ResponseResult downloadUseList(@ApiParam(name = "json:{personal:个人明细=true,startTime:起始时间,endTime:结束时间,day:颗粒度天级=true}") @RequestBody JSONObject json) {
+        boolean personal = json.getBooleanValue("personal");
+        Long startTime = json.getLong("startTime");
+        Long endTime = json.getLong("endTime");
+        boolean day = json.getBooleanValue("day");
+        List<JSONObject> collect = wholeSearchService.outputUsedList(personal, startTime, endTime, day);
+        String fileName = startTime + "_" + endTime + "使用记录";
+        if (personal) {
+            EasyExcelUtil.download(fileName, "sheet", ExportWholeSearchRecordDTO.class, collect.stream().map(ExportWholeSearchRecordDTO::createFromJSONObject).collect(Collectors.toList()), response);
+        } else {
+            EasyExcelUtil.download(fileName, "sheet", ExportLineDTO.class, collect.stream().map(ExportLineDTO::createFromJSONObject).collect(Collectors.toList()), response);
+        }
+        return ResponseResult.success();
+    }
+
+    @ApiOperation("全网搜-维护记录-调整")
+    @PutMapping("/maintain/adjust")
+    @Auth(role = RoleEnum.ADMIN)
+    public ResponseResult adjustRecord(@ApiParam(name = "json:{count:调整数值)") @RequestBody JSONObject json) {
+        wholeSearchService.adjustRecord(json.getInteger("count"));
+        return ResponseResult.success();
+    }
+
+}

src/main/java/com/zhiwei/brandkbs2/controller/app/AppArticleController.java

@@ -12,10 +12,7 @@ import com.zhiwei.brandkbs2.pojo.dto.ExportAppYuqingDTO;
 import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
 import com.zhiwei.brandkbs2.pojo.dto.ReportDTO;
 import com.zhiwei.brandkbs2.pojo.dto.ReportSearchDTO;
-import com.zhiwei.brandkbs2.service.CommonService;
-import com.zhiwei.brandkbs2.service.MarkDataService;
-import com.zhiwei.brandkbs2.service.ProjectService;
-import com.zhiwei.brandkbs2.service.ReportService;
+import com.zhiwei.brandkbs2.service.*;
 import com.zhiwei.brandkbs2.util.Tools;
 import com.zhiwei.middleware.mark.vo.MarkerTag;
 import io.swagger.annotations.Api;
@@ -160,8 +157,8 @@ public class AppArticleController extends BaseController {
 
     @ApiOperation("舆情列表-搜索条件")
     @GetMapping("/mark/list/criteria")
-    public ResponseResult getYuqingMarkCriteria(@RequestParam(required = false) String linkedGroupId) {
-        return ResponseResult.success(markDataService.getYuqingMarkCriteria(linkedGroupId));
+    public ResponseResult getYuqingMarkCriteria() {
+        return ResponseResult.success(markDataService.getYuqingMarkCriteria());
     }
 
     @ApiOperation("舆情导出")
@@ -313,9 +310,12 @@ public class AppArticleController extends BaseController {
         List<MarkerTag> hitTags = projectService.getProjectById(UserThreadLocal.getProjectId()).getHitTags();
         if (!Tools.isEmpty(hitTags)) {
             for (String mustTag : hitTags.stream().map(MarkerTag::getUniqueId).collect(Collectors.toList())) {
+                // 不包含必要标签则添加
+                if (!mtag.toString().contains(mustTag)) {
                     mtag.append(mustTag);
                 }
             }
+        }
         return Tools.isIllegalTag(mtag.toString());
     }
 

src/main/java/com/zhiwei/brandkbs2/controller/app/AppHotController.java

@@ -292,8 +292,9 @@ public class AppHotController extends BaseController {
     @ApiOperation("热点库-品牌热点-热点总览")
     @GetMapping("/overview")
     public ResponseResult hotOverview() {
-        Long[] timeMonth = commonService.getTimeRangeMonth();
-        JSONObject result = searchHotHandler(10000, 1, allPlatform, "endTime", timeMonth[0], timeMonth[1]);
+        long endTime = System.currentTimeMillis();
+        long startTime = endTime - Constant.ONE_MONTH;
+        JSONObject result = searchHotHandler(10000, 1, allPlatform, "endTime", startTime, endTime);
         if (Objects.isNull(result)) {
             return ResponseResult.success();
         }

src/main/java/com/zhiwei/brandkbs2/controller/app/AppSearchController.java

@@ -11,6 +11,7 @@ import com.zhiwei.brandkbs2.controller.BaseController;
 import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
 import com.zhiwei.brandkbs2.enmus.RoleEnum;
 import com.zhiwei.brandkbs2.model.ResponseResult;
+import com.zhiwei.brandkbs2.pojo.WholeSearchRecord;
 import com.zhiwei.brandkbs2.pojo.dto.*;
 import com.zhiwei.brandkbs2.pojo.vo.ChannelListVO;
 import com.zhiwei.brandkbs2.service.*;
@@ -19,6 +20,7 @@ import com.zhiwei.middleware.event.pojo.dto.BrandkbsEventSearchDTO;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
 import org.apache.commons.lang3.time.DateUtils;
+import org.apache.commons.lang3.tuple.Pair;
 import org.joda.time.Period;
 import org.joda.time.PeriodType;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -76,6 +78,9 @@ public class AppSearchController extends BaseController {
     @Resource(name = "projectServiceImpl")
     ProjectService projectService;
 
+    @Resource(name = "wholeSearchServiceImpl")
+    WholeSearchService wholeSearchService;
+
     @ApiOperation("搜索-查热点")
     @LogRecord(values = "keyword", description = "查热点", arguments = true, entity = false)
     @GetMapping("/hot/list")
@@ -139,7 +144,7 @@ public class AppSearchController extends BaseController {
     @PostMapping("/searchWhole")
     public ResponseResult searchWholeNetwork(@RequestBody SearchFilterDTO dto) {
         // 针对商业数据库做限制
-        if (2 == projectService.getProjectById(UserThreadLocal.getProjectId()).getWholeSearchDataSource()) {
+        if (dto.isExternalDataSource()) {
             long time = DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), -89).getTime();
             if (time > dto.getStartTime()) {
                 // 仅对查商业数据库时限制时间，查舆情库时本质上无时间限制
@@ -150,8 +155,17 @@ public class AppSearchController extends BaseController {
                 // 仅对查商业数据库时限制时间，查舆情库时本质上无时间限制
                 return ResponseResult.failure("时间跨度不能超过30天");
             }
+            if (dto.getPageSize() >= projectService.getProjectById(UserThreadLocal.getProjectId()).getWholeSearchBalance()) {
+                return ResponseResult.failure("实时采集余额不足");
+            }
+        }
+        Pair<JSONObject, Integer> pair = markDataService.searchWholeNetworkWithBalance(dto);
+        if(dto.isSentiment()){
+            List<JSONObject> list = pair.getLeft().getJSONArray("list").toJavaList(JSONObject.class);
+            commonService.sentimentAnalysis(list);
         }
-        return ResponseResult.success(markDataService.searchWholeNetwork(dto));
+        wholeSearchService.decreaseRecord(dto.getSearch(), WholeSearchRecord.UsedType.query, pair.getRight());
+        return ResponseResult.success(pair.getLeft());
     }
 
     @ApiOperation("搜索-全网搜-舆情导出")
@@ -159,7 +173,7 @@ public class AppSearchController extends BaseController {
     @LogRecord(description = "全网搜-舆情导出", values = {"startTime", "endTime", "fans", "filterType", "filterWords", "search", "keyword", "platforms", "sensitiveChannels", "sourceKeyword"}, entity = true, arguments = true)
     public ResponseResult exportSearchWhole(@RequestBody SearchFilterDTO dto) {
         // 针对商业数据库做限制
-        if (2 == projectService.getProjectById(UserThreadLocal.getProjectId()).getWholeSearchDataSource()) {
+        if (dto.isExternalDataSource()) {
             long time = DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), -89).getTime();
             if (time > dto.getStartTime()) {
                 // 仅对查商业数据库时限制时间，查舆情库时本质上无时间限制
@@ -170,9 +184,15 @@ public class AppSearchController extends BaseController {
                 // 仅对查商业数据库时限制时间，查舆情库时本质上无时间限制
                 return ResponseResult.failure("时间跨度不能超过30天");
             }
+            if (dto.getOutputCount() >= projectService.getProjectById(UserThreadLocal.getProjectId()).getWholeSearchBalance()) {
+                return ResponseResult.failure("实时采集余额不足");
+            }
         }
         List<ExportSearchWholeDTO> exportList = markDataService.exportSearchWhole(dto);
         EasyExcelUtil.download("全网搜舆情列表数据", "sheet1", ExportSearchWholeDTO.class, exportList, response);
+        if (dto.isExternalDataSource()) {
+            wholeSearchService.decreaseRecord(dto.getSearch(), WholeSearchRecord.UsedType.output, exportList.size());
+        }
         return ResponseResult.success();
     }
 
@@ -192,8 +212,8 @@ public class AppSearchController extends BaseController {
 
     @ApiOperation("搜索-舆情列表-搜索条件")
     @GetMapping("/mark/list/criteria")
-    public ResponseResult getYuqingMarkCriteria(@RequestParam(required = false) String linkedGroupId) {
-        return ResponseResult.success(markDataService.getYuqingMarkCriteria(linkedGroupId));
+    public ResponseResult getYuqingMarkCriteria() {
+        return ResponseResult.success(markDataService.getYuqingMarkCriteria());
     }
 
     @ApiOperation("搜索-活跃渠道")

src/main/java/com/zhiwei/brandkbs2/dao/WholeSearchRecordDao.java

+package com.zhiwei.brandkbs2.dao;
+
+import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.brandkbs2.pojo.WholeSearchRecord;
+import com.zhiwei.brandkbs2.pojo.vo.LineVO;
+
+import java.util.List;
+
+/**
+ * @ClassName: WholeSearchRecordDao
+ * @Description WholeSearchRecordDao
+ * @author: sjj
+ * @date: 2023-07-19 11:57
+ */
+public interface WholeSearchRecordDao extends BaseMongoDao<WholeSearchRecord>, ShardingMongo{
+
+    /**
+     *
+     * @param startTime 起始时间
+     * @param endTime 结束时间
+     * @param day 颗粒度
+     * @return list
+     */
+    List<JSONObject> aggregateProjectUsedRecord(long startTime, long endTime, boolean day);
+
+    /**
+     *
+     * @param startTime 起始时间
+     * @param endTime 结束时间
+     * @return list
+     */
+    List<LineVO> aggregateDayLastRecord(long startTime, long endTime);
+
+}

src/main/java/com/zhiwei/brandkbs2/dao/impl/ProjectDaoImlp.java → src/main/java/com/zhiwei/brandkbs2/dao/impl/ProjectDaoImpl.java

@@ -5,17 +5,17 @@ import com.zhiwei.brandkbs2.pojo.Project;
 import org.springframework.stereotype.Component;
 
 /**
- * @ClassName: ProjectDaoImlp
+ * @ClassName: ProjectDaoImpl
  * @Description 项目业务接口实现类
  * @author: sjj
  * @date: 2022-04-21 13:51
  */
 @Component("projectDao")
-public class ProjectDaoImlp extends BaseMongoDaoImpl<Project> implements ProjectDao {
+public class ProjectDaoImpl extends BaseMongoDaoImpl<Project> implements ProjectDao {
 
     private static final String COLLECTION_NAME = "brandkbs_project";
 
-    public ProjectDaoImlp() {
+    public ProjectDaoImpl() {
         super(COLLECTION_NAME);
     }
 

src/main/java/com/zhiwei/brandkbs2/dao/impl/WholeSearchRecordDaoImpl.java

+package com.zhiwei.brandkbs2.dao.impl;
+
+import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.brandkbs2.auth.UserThreadLocal;
+import com.zhiwei.brandkbs2.dao.WholeSearchRecordDao;
+import com.zhiwei.brandkbs2.pojo.WholeSearchRecord;
+import com.zhiwei.brandkbs2.pojo.vo.LineVO;
+import org.springframework.data.domain.Sort;
+import org.springframework.data.mongodb.core.aggregation.Aggregation;
+import org.springframework.data.mongodb.core.aggregation.AggregationOperation;
+import org.springframework.data.mongodb.core.aggregation.AggregationResults;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.stereotype.Component;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+/**
+ * @ClassName: WholeSearchRecordDaoImpl
+ * @Description WholeSearchRecordDaoImpl
+ * @author: sjj
+ * @date: 2023-07-19 12:02
+ */
+@Component("wholeSearchRecordDao")
+public class WholeSearchRecordDaoImpl extends BaseMongoDaoImpl<WholeSearchRecord> implements WholeSearchRecordDao{
+    private static final String COLLECTION_PREFIX = "brandkbs_whole_search_record";
+
+    private static final String TIME_PATTERN = "yyyy";
+
+    public WholeSearchRecordDaoImpl() {
+        super(null);
+    }
+
+    @Override
+    public String collectionPrefix() {
+        return COLLECTION_PREFIX;
+    }
+
+    @Override
+    public String timePattern() {
+        return TIME_PATTERN;
+    }
+
+    @Override
+    public List<JSONObject> aggregateProjectUsedRecord(long startTime, long endTime, boolean day) {
+        List<JSONObject> res = new ArrayList<>();
+        String[] collectionNames = generateCollectionNames(new Date(startTime), new Date(endTime));
+        Criteria criteria = Criteria.where("isUsed").is(true).and("projectId").is(UserThreadLocal.getProjectId());
+        int nrOfChars = day ? 10 : 7;
+        List<AggregationOperation> operations = new ArrayList<>();
+        // operations区分先后顺序
+        operations.add(Aggregation.match(criteria));
+        operations.add(Aggregation.project( "usedCount", "cTime").and("cTime").substring(0, nrOfChars).as("cTime"));
+        operations.add(Aggregation.group("cTime").sum("usedCount").as("count"));
+        operations.add(Aggregation.sort(Sort.by(Sort.Order.desc("_id"))));
+        Aggregation aggregation = Aggregation.newAggregation(operations);
+        for (String collectionName : collectionNames) {
+            AggregationResults<JSONObject> aggregateResult = mongoTemplate.aggregate(aggregation, collectionName, JSONObject.class);
+            res.addAll(aggregateResult.getMappedResults());
+        }
+        return res;
+    }
+
+    @Override
+    public List<LineVO> aggregateDayLastRecord(long startTime, long endTime) {
+        List<LineVO> res = new ArrayList<>();
+        String[] collectionNames = generateCollectionNames(new Date(startTime), new Date(endTime));
+        Criteria criteria = Criteria.where("projectId").is(UserThreadLocal.getProjectId());
+        List<AggregationOperation> operations = new ArrayList<>();
+        // operations区分先后顺序
+        operations.add(Aggregation.match(criteria));
+        operations.add(Aggregation.project("currentCount").and("cTime").substring(0, 10).as("cTime"));
+        operations.add(Aggregation.group("cTime").last("currentCount").as("count"));
+        operations.add(Aggregation.sort(Sort.by(Sort.Order.asc("_id"))));
+        Aggregation aggregation = Aggregation.newAggregation(operations);
+        // 优先小的年份
+        for (int i = collectionNames.length - 1; i >= 0; i--) {
+            AggregationResults<JSONObject> aggregateResult = mongoTemplate.aggregate(aggregation, collectionNames[i], JSONObject.class);
+            List<JSONObject> mappedResults = aggregateResult.getMappedResults();
+            for (JSONObject mappedResult : mappedResults) {
+                res.add(new LineVO(mappedResult.getLong("count"), mappedResult.getDate("_id").getTime()));
+            }
+        }
+        return res;
+    }
+}

src/main/java/com/zhiwei/brandkbs2/easyexcel/dto/ExportLineDTO.java

+package com.zhiwei.brandkbs2.easyexcel.dto;
+
+import com.alibaba.excel.annotation.ExcelProperty;
+import com.alibaba.fastjson.JSONObject;
+import lombok.Data;
+import lombok.ToString;
+
+import java.util.Date;
+
+/**
+ * @ClassName: ExportLineDTO
+ * @Description ExportLineDTO
+ * @author: sjj
+ * @date: 2023-07-19 18:07
+ */
+@Data
+@ToString
+public class ExportLineDTO {
+
+    @ExcelProperty("时间")
+    private Date time;
+
+    @ExcelProperty("数据量")
+    private Long count;
+
+    public static ExportLineDTO createFromJSONObject(JSONObject json) {
+        ExportLineDTO dto = new ExportLineDTO();
+        dto.setTime(json.getDate("_id"));
+        dto.setCount(json.getLong("count"));
+        return dto;
+    }
+
+}

src/main/java/com/zhiwei/brandkbs2/easyexcel/dto/ExportWholeSearchRecordDTO.java

+package com.zhiwei.brandkbs2.easyexcel.dto;
+
+import com.alibaba.excel.annotation.ExcelProperty;
+import com.alibaba.fastjson.JSONObject;
+import lombok.Data;
+import lombok.ToString;
+
+import java.util.Date;
+
+/**
+ * @ClassName: ExportWholeSearchRecordDTO
+ * @Description ExportWholeSearchRecordDTO
+ * @author: sjj
+ * @date: 2023-07-19 18:02
+ */
+@Data
+@ToString
+public class ExportWholeSearchRecordDTO {
+
+    @ExcelProperty("关键词")
+    private String keyword;
+
+    @ExcelProperty("类型")
+    private String usedType;
+
+    @ExcelProperty("数据量")
+    private Integer usedCount;
+
+    @ExcelProperty("操作时间")
+    private Date cTime;
+
+    @ExcelProperty("操作人")
+    private String submitter;
+
+    public static ExportWholeSearchRecordDTO createFromJSONObject(JSONObject json) {
+        ExportWholeSearchRecordDTO dto = new ExportWholeSearchRecordDTO();
+        dto.setKeyword(json.getString("keyword"));
+        dto.setUsedType(json.getString("usedType"));
+        dto.setUsedCount(json.getInteger("usedCount"));
+        ;
+        dto.setCTime(new Date(json.getLong("cTime")));
+        dto.setSubmitter(json.getString("submitter"));
+        return dto;
+    }
+}

src/main/java/com/zhiwei/brandkbs2/enmus/HotPlatformEnum.java

@@ -20,7 +20,8 @@ public enum HotPlatformEnum {
     ZHIHU("知乎", "zhihu", "知乎热搜"),
     JINRITOUTIAO("今日头条", "toutiao", "今日头条热搜"),
     DOUYIN("抖音", "douyin", "抖音热搜"),
-    BILIBILI("B站", "bilibili-ranking", "B站热搜"),
+    BILIBILI("B站", "bilibili", "B站热搜"),
+    BILIBILI_RANKING("B站", "bilibili-ranking", "B站排行榜"),
     KUAISHOU("快手", "kuaishou", "快手热榜"),
     BAIDU("百度", "baidu", "百度热搜");
 

src/main/java/com/zhiwei/brandkbs2/pojo/Project.java

@@ -92,6 +92,11 @@ public class Project extends AbstractProject {
     private List<String> highlightKeyword;
 
     /**
+     * 全网搜余额
+     */
+    private int wholeSearchBalance;
+
+    /**
      * 项目主品牌配置信息转换
      *
      * @return 项目对象

src/main/java/com/zhiwei/brandkbs2/pojo/WholeSearchRecord.java

+package com.zhiwei.brandkbs2.pojo;
+
+import lombok.Getter;
+import lombok.Setter;
+
+import java.util.Date;
+
+/**
+ * @ClassName: WholeSearchRecord
+ * @Description 全网搜记录实体类
+ * @author: sjj
+ * @date: 2023-07-19 11:21
+ */
+@Getter
+@Setter
+public class WholeSearchRecord extends AbstractBaseMongo {
+
+    /**
+     * 项目id
+     */
+    private String projectId;
+
+    /**
+     * 搜索关键词
+     */
+    private String keyword;
+
+    /**
+     * 使用记录
+     */
+    private boolean isUsed;
+
+    /**
+     * 使用类型(导出/查询)
+     */
+    private String usedType;
+
+    /**
+     * 上一次记录数
+     */
+    private int lastCount;
+
+    /**
+     * 该次操作使用(减少)数值
+     */
+    private int usedCount;
+
+    /**
+     * 本次记录数
+     */
+    private int currentCount;
+
+    /**
+     * 提交人
+     */
+    private String submitter;
+
+    /**
+     * 创建时间(Date类型方便聚合查询)
+     */
+    private Date cTime;
+
+
+    private WholeSearchRecord() {
+    }
+
+    public static WholeSearchRecord createUsedRecord(String projectId, String keyword, String usedType, int lastCount, int usedCount, String submitter) {
+        WholeSearchRecord record = new WholeSearchRecord();
+        record.projectId = projectId;
+        record.keyword = keyword;
+        record.isUsed = true;
+        record.usedType = usedType;
+        record.lastCount = lastCount;
+        record.usedCount = usedCount;
+        record.currentCount = lastCount - usedCount;
+        record.submitter = submitter;
+        record.cTime = new Date();
+        return record;
+    }
+
+    public static WholeSearchRecord createMaintainRecord(String projectId, int lastCount, int currentCount, String submitter) {
+        WholeSearchRecord record = new WholeSearchRecord();
+        record.projectId = projectId;
+        record.isUsed = false;
+        record.lastCount = lastCount;
+        record.currentCount = currentCount;
+        record.submitter = submitter;
+        record.cTime = new Date();
+        return record;
+    }
+
+    public enum UsedType {
+        query("查询"),
+        output("导出");
+        @Getter
+        private final String value;
+
+        UsedType(String value) {
+            this.value = value;
+        }
+    }
+}

src/main/java/com/zhiwei/brandkbs2/pojo/dto/MarkSearchDTO.java

@@ -154,4 +154,10 @@ public class MarkSearchDTO {
     @ApiModelProperty(value = "是否去重")
     private boolean fold;
 
+    /**
+     *
+     */
+    @ApiModelProperty(value = "屏蔽词")
+    private String shieldWord;
+
 }

src/main/java/com/zhiwei/brandkbs2/pojo/dto/SearchFilterDTO.java

@@ -12,12 +12,6 @@ import java.util.*;
 public class SearchFilterDTO {
 
     /**
-     * 项目Id
-     */
-    @ApiModelProperty(value = "项目id")
-    private String projectId;
-
-    /**
      * 开始时间戳
      */
     @ApiModelProperty(value = "开始时间")
@@ -42,6 +36,12 @@ public class SearchFilterDTO {
     private int pageSize;
 
     /**
+     * "采集时间(商业数据翻页用)"
+     */
+    @ApiModelProperty("采集时间(商业数据翻页用)")
+    private Long crawlerTime;
+
+    /**
      * 关键字
      */
     @ApiModelProperty(value = "关键字")
@@ -54,54 +54,24 @@ public class SearchFilterDTO {
     private List<String> platforms;
 
     /**
-     * 平台(查商业数据库用)
-     */
-    @ApiModelProperty(value = "平台(查商业数据库用)")
-    private String platformsStr;
-
-    /**
-     * 数据类型 1:原创, 2:转发, 7:评论 (查商业数据库用)
-     */
-    @ApiModelProperty(value = "数据类型(查商业数据库用)")
-    private String filterForward = "1,2";
-
-    /**
      * 排序字段
      */
     @ApiModelProperty(value = "排序字段")
-    private JSONObject sorter;
-
-    /**
-     * 排序字段(查商业数据库用)
-     */
-    @ApiModelProperty(value = "排序字段(查商业数据库用)")
-    private String sortOf;
+    private JSONObject sorter = JSONObject.parseObject("{\"time\":\"descend\"}");
 
     /**
-     * 来源
+     * 渠道关键字
      */
-    @ApiModelProperty(value = "来源")
+    @ApiModelProperty(value = "渠道关键字")
     private String sourceKeyword;
 
     /**
-     * 域名
+     * 域名关键字
      */
-    @ApiModelProperty(value = "域名")
+    @ApiModelProperty(value = "域名关键字")
     private String hostKeyword;
 
     /**
-     * 过滤关键字
-     */
-    @ApiModelProperty(value = "过滤关键字")
-    private String filterWords;
-
-    /**
-     * 匹配域名 默认标题+内容(查商业数据库用)
-     */
-    @ApiModelProperty(value = "匹配域名(查商业数据库用)")
-    private String matchFields = null;
-
-    /**
      * 查询类型 标题/全文
      */
     @ApiModelProperty(value = "查询类型")
@@ -116,7 +86,7 @@ public class SearchFilterDTO {
     /**
      * 粉丝数
      */
-    @ApiModelProperty(value = "粉丝数")
+    @ApiModelProperty(value = "粉丝数 千万/百万")
     private String fans;
 
     /**
@@ -125,6 +95,53 @@ public class SearchFilterDTO {
     @ApiModelProperty(value = "过滤类型 标题/全文")
     private String filterType;
 
+    /**
+     * 过滤关键字
+     */
+    @ApiModelProperty(value = "过滤关键字")
+    private String filterWords;
+
+    /**
+     * 外部数据源
+     */
+    @ApiModelProperty(value = "是否为外部数据源,默认为false")
+    private boolean externalDataSource = false;
+
+    /**
+     * 情感倾向分析
+     */
+    @ApiModelProperty(value = "情感倾向分析,默认为false")
+    private boolean sentiment = false;
+
+    /**
+     * 导出数据量
+     */
+    @ApiModelProperty(value = "导出数据量")
+    private int outputCount;
+
+//    /**
+//     * 这部分字段无需前端传参
+//     */
+//    /**
+//     * 平台(查商业数据库用)
+//     */
+//    private String platformsStr;
+//
+//    /**
+//     * 数据类型 1:原创, 2:转发, 7:评论 (查商业数据库用)
+//     */
+//    private String filterForward = "1,2";
+//
+//    /**
+//     * 匹配域名 默认标题+内容(查商业数据库用)
+//     */
+//    private String matchFields = null;
+//
+//    /**
+//     * 排序字段(查商业数据库用)
+//     */
+//    private String sortOf;
+
     public Map<String, Object> toMap() {
         Map<String, Object> map = new HashMap<>();
         map.put("timeType", "time");

src/main/java/com/zhiwei/brandkbs2/pojo/vo/LineVO.java

 package com.zhiwei.brandkbs2.pojo.vo;
 
+import lombok.AllArgsConstructor;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 import lombok.ToString;
 
-import java.util.Date;
-
 /**
  * @author lxj
  * @version 1.0
@@ -13,6 +13,8 @@ import java.util.Date;
  */
 @Data
 @ToString
+@NoArgsConstructor
+@AllArgsConstructor
 public class LineVO {
     /**
      * 个数

src/main/java/com/zhiwei/brandkbs2/service/CommonService.java

@@ -59,6 +59,13 @@ public interface CommonService {
      */
     void addMtagBatch(List<MarkInfo> markInfos);
 
+    /**
+     * 情感倾向分析
+     * @param texts
+     * @return
+     */
+    List<JSONObject> sentimentAnalysis(List<JSONObject> texts);
+
     Long[] getTimeRangeYear();
 
     Long[] getTimeRangeMonth();

src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java

@@ -108,7 +108,7 @@ public interface MarkDataService {
     /**
      * 舆情标注数据搜索条件
      */
-    JSONObject getYuqingMarkCriteria(String linkedGroup);
+    JSONObject getYuqingMarkCriteria();
 
     /**
      * 舆情标注数据搜索条件
@@ -304,7 +304,7 @@ public interface MarkDataService {
      * @param dto 全网搜搜索传输类
      * @return JSONObject
      */
-    JSONObject searchWholeNetwork(SearchFilterDTO dto);
+    Pair<JSONObject, Integer> searchWholeNetworkWithBalance(SearchFilterDTO dto);
 
     /**
      * 搜索-全网搜-舆情导出

src/main/java/com/zhiwei/brandkbs2/service/WholeSearchService.java

+package com.zhiwei.brandkbs2.service;
+
+import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.brandkbs2.pojo.WholeSearchRecord;
+import com.zhiwei.brandkbs2.pojo.vo.PageVO;
+
+import java.util.List;
+
+/**
+ * @ClassName: WholeSearchService
+ * @Description 全网搜服务类
+ * @author: sjj
+ * @date: 2023-07-19 11:43
+ */
+public interface WholeSearchService {
+
+    /**
+     * 查询使用列表
+     *
+     * @param page 页码
+     * @param pageSize 页码大小
+     * @param sorter 排序字段
+     * @return PageVO
+     */
+    PageVO<JSONObject> findUsedList(int page,int pageSize,String sorter);
+
+    /**
+     * 查询维护列表
+     *
+     * @param page 页码
+     * @param pageSize 页码大小
+     * @return PageVO
+     */
+    PageVO<JSONObject> findMaintainList(int page,int pageSize);
+
+    /**
+     * 导出使用列表
+     * @param personal 个人明细/项目明细
+     * @param startTime 开始时间
+     * @param endTime 结束时间
+     * @param day 颗粒度(天级/月级)
+     * @return List
+     */
+    List<JSONObject> outputUsedList(boolean personal,long startTime,long endTime,boolean day);
+
+    /**
+     * @param currentCount 调整后数值
+     */
+    void adjustRecord(Integer currentCount);
+
+    /**
+     * 数据总览
+     * @return json
+     */
+    JSONObject overview();
+
+    /**
+     * @param keyword 关键词
+     * @param usedType 使用类别
+     * @param usedCount 使用量
+     */
+    void decreaseRecord(String keyword, WholeSearchRecord.UsedType usedType, Integer usedCount);
+
+}

src/main/java/com/zhiwei/brandkbs2/service/impl/CommonServiceImpl.java

@@ -15,6 +15,7 @@ import com.zhiwei.middleware.mark.vo.TagSearch;
 import org.apache.commons.lang3.time.DateUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Service;
 import org.springframework.web.client.RestTemplate;
 
@@ -132,6 +133,26 @@ public class CommonServiceImpl implements CommonService {
     }
 
     @Override
+    public List<JSONObject> sentimentAnalysis(List<JSONObject> texts) {
+        ResponseEntity<JSONObject> responseEntity = restTemplate.postForEntity("https://sentiment.zhiweidata.com/long/multi/longText", texts.stream().map(json ->
+                {
+                    String title = json.getString("title");
+                    return null == title ? json.getString("content") : title;
+                }
+        ).collect(Collectors.toList()), JSONObject.class);
+        if (Objects.requireNonNull(responseEntity.getBody()).getBooleanValue("status")) {
+            if (Objects.nonNull(responseEntity.getBody().get("result"))) {
+                List<JSONObject> result = responseEntity.getBody().getJSONArray("result").toJavaList(JSONObject.class);
+                for (int i = 0; i < result.size(); i++) {
+                    String emotion = result.get(i).getString("emotion");
+                    texts.get(i).put("emotion", emotion);
+                }
+            }
+        }
+        return texts;
+    }
+
+    @Override
     public Long[] getTimeRangeYear() {
         long endTime = Tools.truncDate(new Date(), Constant.DAY_PATTERN).getTime();
         long startTime = DateUtils.addYears(new Date(endTime), -1).getTime();

src/main/java/com/zhiwei/brandkbs2/service/impl/EsSearchServiceImpl.java

@@ -91,12 +91,16 @@ public class EsSearchServiceImpl implements EsSearchService {
         }
         // Query 查询条件
 //        BoolQueryBuilder query = QueryBuilders.boolQuery();
+        String[] fieldSearch = "仅标题".equals(dto.getSearchField()) ? new String[]{GenericAttribute.ES_IND_TITLE} : new String[]{GenericAttribute.ES_IND_FULL_TEXT};
         // keyword
         if (StringUtils.isNotEmpty(dto.getKeyword())) {
             // Query
-            String[] fieldSearch = "仅标题".equals(dto.getSearchField()) ? new String[]{GenericAttribute.ES_IND_TITLE} : new String[]{GenericAttribute.ES_IND_FULL_TEXT};
             postFilter.must(EsQueryTools.assembleNormalKeywordQuery(dto.getKeyword(), fieldSearch));
         }
+        // shieldWord
+        if (StringUtils.isNotEmpty(dto.getShieldWord())) {
+            postFilter.mustNot(EsQueryTools.assembleNormalKeywordQuery(dto.getShieldWord(), fieldSearch));
+        }
         // sourceKeyword
         if (StringUtils.isNotEmpty(dto.getSourceKeyword())) {
             postFilter.must(EsQueryTools.assembleSourceQuery(dto.getSourceKeyword()));

src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java

@@ -6,6 +6,7 @@ import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
+import com.zhiwei.base.category.ClassB;
 import com.zhiwei.base.entity.subclass.mark.MarkInfo;
 import com.zhiwei.brandkbs2.auth.UserThreadLocal;
 import com.zhiwei.brandkbs2.common.ChannelType;
@@ -40,7 +41,6 @@ import com.zhiwei.brandkbs2.util.MongoUtil;
 import com.zhiwei.brandkbs2.util.RedisUtil;
 import com.zhiwei.brandkbs2.util.TextUtil;
 import com.zhiwei.brandkbs2.util.Tools;
-import com.zhiwei.tools.istarshine.IstarShineUtil;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.time.DateUtils;
@@ -283,11 +283,9 @@ public class MarkDataServiceImpl implements MarkDataService {
     }
 
     @Override
-    public JSONObject getYuqingMarkCriteria(String linkedGroupId) {
+    public JSONObject getYuqingMarkCriteria() {
         String projectId = UserThreadLocal.getProjectId();
-        if (null == linkedGroupId) {
-            linkedGroupId = projectService.getProjectVOById(projectId).getBrandLinkedGroupId();
-        }
+        String linkedGroupId = projectService.getProjectVOById(projectId).getBrandLinkedGroupId();
         JSONObject result = new JSONObject();
         // 搜索时间
         result.put("times", Arrays.asList("今天", "24小时", "三天", "七天", "近30天"));
@@ -317,32 +315,32 @@ public class MarkDataServiceImpl implements MarkDataService {
     private List<JSONObject> getFilterRegionList() {
         List<JSONObject> res = new ArrayList<>();
         JSONObject beijing = new JSONObject();
-        beijing.put("name","北京");
-        beijing.put("id","北京市");
+        beijing.put("name", "北京");
+        beijing.put("id", "北京市");
         res.add(beijing);
         JSONObject shanghai = new JSONObject();
-        shanghai.put("name","上海");
-        shanghai.put("id","上海市");
+        shanghai.put("name", "上海");
+        shanghai.put("id", "上海市");
         res.add(shanghai);
         JSONObject guangzhou = new JSONObject();
-        guangzhou.put("name","广州");
-        guangzhou.put("id","广东省|广州市");
+        guangzhou.put("name", "广州");
+        guangzhou.put("id", "广东省|广州市");
         res.add(guangzhou);
         JSONObject shenzhen = new JSONObject();
-        shenzhen.put("name","深圳");
-        shenzhen.put("id","广东省|深圳市");
+        shenzhen.put("name", "深圳");
+        shenzhen.put("id", "广东省|深圳市");
         res.add(shenzhen);
         JSONObject zhejiang = new JSONObject();
-        zhejiang.put("name","浙江");
-        zhejiang.put("id","浙江省");
+        zhejiang.put("name", "浙江");
+        zhejiang.put("id", "浙江省");
         res.add(zhejiang);
         JSONObject sichuan = new JSONObject();
-        sichuan.put("name","四川");
-        sichuan.put("id","四川省");
+        sichuan.put("name", "四川");
+        sichuan.put("id", "四川省");
         res.add(sichuan);
         JSONObject chongqing = new JSONObject();
-        chongqing.put("name","重庆");
-        chongqing.put("id","重庆市");
+        chongqing.put("name", "重庆");
+        chongqing.put("id", "重庆市");
         res.add(chongqing);
         return res;
     }
@@ -562,10 +560,13 @@ public class MarkDataServiceImpl implements MarkDataService {
         }
         String projectId = UserThreadLocal.getProjectId();
         markSearchDTO.setProjectId(projectId);
-        String linkedGroupId = markSearchDTO.getLinkedGroupId();
-        if (null == linkedGroupId) {
-            linkedGroupId = projectService.getProjectVOById(projectId).getBrandLinkedGroupId();
-            markSearchDTO.setLinkedGroupId(linkedGroupId);
+        // 主品牌
+        if (Constant.PRIMARY_CONTEND_ID.equals(markSearchDTO.getContendId())) {
+            markSearchDTO.setLinkedGroupId(projectService.getProjectVOById(projectId).getBrandLinkedGroupId());
+        } else {
+            Optional<Contend> contendOptional =
+                    projectService.getProjectVOById(projectId).getContendList().stream().filter(contend -> contend.getId().equals(markSearchDTO.getContendId())).findFirst();
+            contendOptional.ifPresent(contend -> markSearchDTO.setLinkedGroupId(contend.getBrandLinkedGroupId()));
         }
         markSearchDTO.setPage(1);
         markSearchDTO.setPageSize(10000);
@@ -594,10 +595,13 @@ public class MarkDataServiceImpl implements MarkDataService {
         }
         String projectId = UserThreadLocal.getProjectId();
         markSearchDTO.setProjectId(projectId);
-        String linkedGroupId = markSearchDTO.getLinkedGroupId();
-        if (null == linkedGroupId) {
-            linkedGroupId = projectService.getProjectVOById(projectId).getBrandLinkedGroupId();
-            markSearchDTO.setLinkedGroupId(linkedGroupId);
+        // 主品牌
+        if (Constant.PRIMARY_CONTEND_ID.equals(markSearchDTO.getContendId())) {
+            markSearchDTO.setLinkedGroupId(projectService.getProjectVOById(projectId).getBrandLinkedGroupId());
+        } else {
+            Optional<Contend> contendOptional =
+                    projectService.getProjectVOById(projectId).getContendList().stream().filter(contend -> contend.getId().equals(markSearchDTO.getContendId())).findFirst();
+            contendOptional.ifPresent(contend -> markSearchDTO.setLinkedGroupId(contend.getBrandLinkedGroupId()));
         }
     }
 
@@ -885,8 +889,10 @@ public class MarkDataServiceImpl implements MarkDataService {
             return jsonObject;
         }).collect(Collectors.toList());
         result.put("contendNameList", contendNameList);
-        // 其他标签组(筛选器)
-        result.put("tagFilterList", tagFilterService.getTagFilter());
+        // 重要发声方
+        result.put("channelTypeList", ChannelType.channelOption);
+        // 地域
+        result.put("regionList", getFilterRegionList());
         // 自定义标签列表
         List<CustomTagVo> customTagList = customTagService.findCustomTagList(1, 1).getList();
         if (CollectionUtils.isNotEmpty(customTagList)) {
@@ -1138,7 +1144,6 @@ public class MarkDataServiceImpl implements MarkDataService {
 
     private JSONObject getBackUpPlatform() {
         JSONObject jsonObject = new JSONObject();
-        jsonObject.put("全部", "");
         jsonObject.put("微博", "微博");
         jsonObject.put("微信", "微信");
         jsonObject.put("网媒", "新闻");
@@ -1151,32 +1156,33 @@ public class MarkDataServiceImpl implements MarkDataService {
     }
 
     @Override
-    public JSONObject searchWholeNetwork(SearchFilterDTO dto) {
-        Project project = projectService.getProjectById(UserThreadLocal.getProjectId());
-        JSONObject jsonObject = new JSONObject();
-        if (2 == project.getWholeSearchDataSource()) { // 查商业数据库
-            defaultSearchFilter(dto);
+    public Pair<JSONObject, Integer> searchWholeNetworkWithBalance(SearchFilterDTO dto) {
+        if (dto.isExternalDataSource()) { // 查商业数据库
             try {
-                JSONObject params = IstarShineUtil.getKSSearchParam(dto.getSearch(), null, dto.getStartTime(), dto.getEndTime(), dto.getPlatformsStr(), dto.getFilterForward(), dto.getSortOf(), dto.getPage(), dto.getPageSize(), dto.getMatchFields(), null, null);
+                JSONObject params = externalTransform(dto);
+//                JSONObject params = IstarShineUtil.getKSSearchParam(dto.getSearch(), dto.getFilterWords(), dto.getStartTime(), dto.getEndTime(), dto.getPlatformsStr(),
+//                        dto.getFilterForward(), dto.getSortOf(), dto.getPage(), dto.getPageSize(), dto.getMatchFields(), null, null);
 
                 JSONObject taskEntity = new JSONObject();
                 taskEntity.put("appId", "6183571e0d710000f6003a12");    // 应用id, 由张志伟提供给使用者
-                taskEntity.put("taskType", "");                         // 可为null, 任务类型
-                taskEntity.put("taskDescription", "");                  // 可为null, 任务描述
+                taskEntity.put("taskType", null);                         // 可为null, 任务类型
+                taskEntity.put("taskDescription", null);                  // 可为null, 任务描述
+                taskEntity.put("taskName", UserThreadLocal.getProjectId() + "_" + UserThreadLocal.getUserId());
                 taskEntity.put("userName", "品见");                      // 提交任务的人
                 taskEntity.put("searchParams", params);                 // 任务详情参数
 
                 ResponseEntity<JSONObject> responseEntity = restTemplate.postForEntity(addIStarShineKSDataUrl, taskEntity, JSONObject.class);
                 JSONObject result = Objects.requireNonNull(responseEntity.getBody()).getJSONObject("data");
-                return searchWholeBackUp(result.get("id").toString());
+                JSONObject json = searchWholeBackUp(result.get("id").toString());
+                return Pair.of(json, json.getInteger("pageLimit"));
             } catch (Exception e) {
                 log.error("error searchWholeNetwork - ", e);
+                return Pair.of(null, null);
             }
         } else { // 查舆情库
             ResponseEntity<JSONObject> responseEntity = restTemplate.postForEntity(wholeSearchUrl, dto.toMap(), JSONObject.class);
-            return searchWholeOpinion(responseEntity, jsonObject);
+            return Pair.of(searchWholeOpinion(responseEntity), null);
         }
-        return jsonObject;
     }
 
     @Override
@@ -1191,7 +1197,7 @@ public class MarkDataServiceImpl implements MarkDataService {
                 break;
             }
             // 获取当页数据
-            JSONObject jsonObject = searchWholeNetwork(dto);
+            JSONObject jsonObject = searchWholeNetworkWithBalance(dto).getLeft();
             JSONArray array = jsonObject.getJSONArray("list");
             if (Objects.isNull(array) || 0 == array.size()) {
                 break;
@@ -1202,7 +1208,8 @@ public class MarkDataServiceImpl implements MarkDataService {
         return jsonArray.stream().map(json -> ExportSearchWholeDTO.creatExportSearchWholeDTO((JSONObject) json)).collect(Collectors.toList());
     }
 
-    private JSONObject searchWholeOpinion(ResponseEntity<JSONObject> responseEntity, JSONObject jsonObject) {
+    private JSONObject searchWholeOpinion(ResponseEntity<JSONObject> responseEntity) {
+        JSONObject jsonObject = new JSONObject();
         JSONObject bodyData = Objects.requireNonNull(responseEntity.getBody()).getJSONObject("data");
         JSONObject data = bodyData.getJSONObject("data");
         JSONArray listArray = data.getJSONArray("list");
@@ -1250,22 +1257,27 @@ public class MarkDataServiceImpl implements MarkDataService {
             ResponseEntity<JSONObject> responseEntity = restTemplate.getForEntity(getIStarShineKSInfoDataUrl, JSONObject.class, taskId);
             JSONObject data = Objects.requireNonNull(responseEntity.getBody()).getJSONObject("data");
             JSONObject task = data.getJSONObject("task");
+            if (2 == task.getInteger("crawlerFinish")) {
+                throw new IllegalArgumentException(task.getString("searchResultMsg"));
+            }
             if (1 == task.getInteger("crawlerFinish")) {
+
                 //pageSize
                 Integer receiveCount = task.getInteger("receiveCount");
                 Integer searchCount = task.getInteger("searchCount");
                 int totalPage = searchCount % receiveCount == 0 ? searchCount / receiveCount : (searchCount / receiveCount + 1);
 
-                JSONArray listArray = data.getJSONArray("data");
-                for (Object o : listArray) {
-                    JSONObject list = (JSONObject) o;
+                List<JSONObject> listArray = data.getJSONArray("data").toJavaList(JSONObject.class);
+                for (JSONObject json : listArray) {
                     JSONObject result = new JSONObject();
-                    result.put("platform", list.getJSONObject("publisher").get("platform"));
-                    result.put("title", list.get("title"));
-                    result.put("content", list.get("content"));
-                    result.put("channel", list.get("channel"));
-                    result.put("url", getUrl(list));
-                    result.put("time", list.get("ctime"));
+                    String platform = getPlatformExternal(json.getJSONObject("gather"));
+                    result.put("platform", platform);
+                    result.put("title", json.get("title"));
+                    result.put("content", json.get("content"));
+                    // 渠道获取分平台
+                    result.put("source", getSourceExternal(json, getExternalType(json, platform)));
+                    result.put("url", getUrl(json));
+                    result.put("time", json.getString("ctime").length() == 10 ? json.getLong("ctime") * 1000L : json.getLong("ctime"));
                     resultList.add(result);
                 }
                 jsonObject.put("list", resultList);
@@ -1273,6 +1285,7 @@ public class MarkDataServiceImpl implements MarkDataService {
                 jsonObject.put("pageLimit", receiveCount);
                 jsonObject.put("total", searchCount);
                 jsonObject.put("totalPage", totalPage);
+                jsonObject.put("balance", projectService.getProjectById(UserThreadLocal.getProjectId()).getWholeSearchBalance() - receiveCount);
                 jsonObject.put("isBackUp", true);
                 return jsonObject;
             }
@@ -1281,6 +1294,71 @@ public class MarkDataServiceImpl implements MarkDataService {
         return jsonObject;
     }
 
+    private ClassB.TypeB getExternalType(JSONObject tJson, String platform) {
+        int wtype = tJson.getIntValue("wtype");
+        if (platform.equals("微博") && 2 == wtype) {
+            return ClassB.TypeB.INCOMPLETE;
+        } else if (platform.equals("微博") && 1 == wtype) {
+            return ClassB.TypeB.INCOMPLETE;
+        } else if (platform.equals("视频") || platform.equals("短视频")) {
+            return ClassB.TypeB.VIDEO;
+        }
+        return ClassB.TypeB.COMPLETE;
+    }
+
+    private String getSourceExternal(JSONObject tJson, ClassB.TypeB typeB){
+        String source = null;
+        switch (typeB) {
+            case COMPLETE:
+            case INCOMPLETE:
+            case VIDEO:
+                if (!StringUtils.isEmpty(tJson.getString("user"))) {
+                    source = tJson.getJSONObject("user").getString("name");
+                } else {
+                    source = tJson.getString("repostSource");
+                }
+                break;
+            default:
+                break;
+        }
+        return source;
+    }
+
+    private String getPlatformExternal(JSONObject gather) {
+        String siteDomain = gather.getString("site_domain");
+        String infoFlag = JSON.toJSONString(gather.get("info_flag"));
+        if (Objects.isNull(siteDomain) && gather.getString("site_name").contains("今日头条")) {
+            return "今日头条";
+        }
+        if (Objects.nonNull(siteDomain) && siteDomain.equals("toutiao.com")) {
+            return "今日头条";
+        }
+        if (infoFlag.contains("06")) {
+            return "微信";
+        }
+        if (infoFlag.contains("04") || infoFlag.contains("08")) {
+            return "微博";
+        }
+        if (infoFlag.contains("02")) {
+            return "贴吧论坛";
+        }
+        if (infoFlag.contains("0105")) {
+            return "平媒";
+        }
+        if (infoFlag.contains("0109")) {
+            return "自媒体";
+        }
+        if (infoFlag.contains("07")) {
+            return "视频";
+        }
+        if (infoFlag.contains("11")) {
+            return "短视频";
+        }
+        // '01'：新闻, '0105'：平媒,0109'：APP, '02'：论坛,  '03'：博客,  '04'：微博,'0401':新浪微博, '06'：微信, '07'： 视频,
+        // '08'：长微博,' '11'： 小视频, '12'：境外资讯,'1201': 境外,'1202'：外文, '13': 境外社交媒体,'1301': Twitter',1302': Facebook, '99':搜索
+        return "网媒";
+    }
+
     /**
      * 获取全网搜接口信息url
      *
@@ -1299,26 +1377,38 @@ public class MarkDataServiceImpl implements MarkDataService {
         return url;
     }
 
-    private void defaultSearchFilter(SearchFilterDTO dto) {
-        String projectId = UserThreadLocal.getProjectId();
-        dto.setProjectId(projectId);
-        if (Objects.isNull(dto.getStartTime()) || Objects.isNull(dto.getEndTime())) {
-            Date now = new Date();
-            dto.setEndTime(now.getTime());
-            dto.setStartTime(DateUtils.addDays(now, -1).getTime());
-        }
+    /**
+     * 外部数据源字段转换
+     *
+     * @param dto
+     */
+    private JSONObject externalTransform(SearchFilterDTO dto) {
+        JSONObject params = new JSONObject();
+        params.put("crawlerWord", dto.getSearch());
+        params.put("startTime", dto.getStartTime());
+        params.put("endTime", dto.getEndTime());
+        String platforms;
         // 平台为空时，传全部
-        if (dto.getPlatforms().size() == 0) {
-            dto.setPlatforms(Arrays.asList("微博", "微信", "新闻", "平媒", "自媒体", "论坛", "视频", "短视频"));
-        }
-        // 平台
-        dto.setPlatformsStr(StringUtils.join(dto.getPlatforms(), ","));
-        // sort
-        dto.setSortOf("ctime+" + dto.getSorter());
+        if (Tools.isEmpty(dto.getPlatforms())) {
+            platforms = "App,博客,平媒,新闻,微博,微信,论坛,视频,短视频";
+        } else {
+            platforms = StringUtils.join(dto.getPlatforms(), ",");
+        }
+        params.put("platforms", platforms);
+        params.put("sortOf", "ctime+" + (dto.getSorter().toJSONString().contains("asc") ? "asc" : "desc"));
+        params.put("page", dto.getPage());
+        params.put("pageSize", dto.getPageSize());
+        params.put("crawlerTime", dto.getCrawlerTime());
+        params.put("filterWord", dto.getFilterWords());
         // matchFields
         if (dto.getMatchType().equals("标题")) {
-            dto.setMatchFields("title");
+            params.put("matchFields", Collections.singletonList("title"));
         }
+//        // 历史部分
+//        if (Tools.isEmpty(dto.getPlatforms())) {
+//            dto.setPlatformsStr("App,博客,平媒,新闻,微博,微信,论坛,视频,短视频");
+//        }
+        return params;
     }
 
     @Override
@@ -1784,6 +1874,7 @@ public class MarkDataServiceImpl implements MarkDataService {
 
     /**
      * 组装单条信息
+     *
      * @param map
      * @param linkedGroupId
      * @return
@@ -1849,7 +1940,7 @@ public class MarkDataServiceImpl implements MarkDataService {
     }
 
 
-    private Pair<String, List<ExportAppYuqingDTO>> downloadList(MarkSearchDTO markSearchDTO,ArticleFunction.SearchHitsAndCount searchHitsAndCount) {
+    private Pair<String, List<ExportAppYuqingDTO>> downloadList(MarkSearchDTO markSearchDTO, ArticleFunction.SearchHitsAndCount searchHitsAndCount) {
         try {
             List<ExportAppYuqingDTO> returnList = new ArrayList<>();
             Project project = projectService.getProjectById(UserThreadLocal.getProjectId());

src/main/java/com/zhiwei/brandkbs2/service/impl/ProjectServiceImpl.java

@@ -462,7 +462,7 @@ public class ProjectServiceImpl implements ProjectService {
                 }
             });
         }
-        return lists.stream().filter(Project::isShow).map(project -> {
+        return lists.stream().sorted(Comparator.comparingLong(Project::getCTime)).filter(Project::isShow).map(project -> {
             JSONObject json = new JSONObject();
             json.put("id", project.getId());
             json.put("brands", getBrands(project.getId(), true));

src/main/java/com/zhiwei/brandkbs2/service/impl/WholeSearchServiceImpl.java

+package com.zhiwei.brandkbs2.service.impl;
+
+import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.brandkbs2.auth.UserThreadLocal;
+import com.zhiwei.brandkbs2.config.Constant;
+import com.zhiwei.brandkbs2.dao.ProjectDao;
+import com.zhiwei.brandkbs2.dao.WholeSearchRecordDao;
+import com.zhiwei.brandkbs2.pojo.Project;
+import com.zhiwei.brandkbs2.pojo.WholeSearchRecord;
+import com.zhiwei.brandkbs2.pojo.vo.PageVO;
+import com.zhiwei.brandkbs2.service.WholeSearchService;
+import com.zhiwei.brandkbs2.util.MongoUtil;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.data.mongodb.core.query.Query;
+import org.springframework.data.mongodb.core.query.Update;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.Resource;
+import java.util.Date;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * @ClassName: WholeSearchServiceImpl
+ * @Description WholeSearchServiceImpl
+ * @author: sjj
+ * @date: 2023-07-19 11:55
+ */
+@Service("wholeSearchServiceImpl")
+public class WholeSearchServiceImpl implements WholeSearchService {
+
+    @Resource(name = "wholeSearchRecordDao")
+    private WholeSearchRecordDao wholeSearchRecordDao;
+
+    @Resource(name = "projectDao")
+    private ProjectDao projectDao;
+
+    @Resource(name = "mongoUtil")
+    private MongoUtil mongoUtil;
+
+    @Override
+    public PageVO<JSONObject> findUsedList(int page, int pageSize, String sorter) {
+        String collectionName = wholeSearchRecordDao.generateCollectionName();
+        Query query = Query.query(Criteria.where("projectId").is(UserThreadLocal.getProjectId()).and("isUsed").is(true));
+        wholeSearchRecordDao.addSort(query, sorter);
+        long total = wholeSearchRecordDao.count(query, collectionName);
+        mongoUtil.start(page, pageSize, query);
+        return PageVO.createPageVo(total, page, pageSize, usedList(query, collectionName));
+    }
+
+    @Override
+    public PageVO<JSONObject> findMaintainList(int page, int pageSize) {
+        String collectionName = wholeSearchRecordDao.generateCollectionName();
+        Query query = Query.query(Criteria.where("projectId").is(UserThreadLocal.getProjectId()).and("isUsed").is(false));
+        wholeSearchRecordDao.addSort(query, "{\"cTime\":\"descend\"}");
+        long total = wholeSearchRecordDao.count(query, collectionName);
+        mongoUtil.start(page, pageSize, query);
+        List<JSONObject> collect = wholeSearchRecordDao.findList(query, collectionName).stream().map(record -> {
+            JSONObject json = new JSONObject();
+            json.put("lastCount", record.getLastCount());
+            json.put("currentCount", record.getCurrentCount());
+            json.put("cTime", record.getCTime().getTime());
+            json.put("submitter", record.getSubmitter());
+            return json;
+        }).collect(Collectors.toList());
+        return PageVO.createPageVo(total, page, pageSize, collect);
+    }
+
+    @Override
+    public List<JSONObject> outputUsedList(boolean personal, long startTime, long endTime, boolean day) {
+        // 个人明细
+        if (personal) {
+            Criteria criteria = Criteria.where("isUsed").is(true);
+            criteria.and("submitter").is(UserThreadLocal.getNickname());
+            Query query = Query.query(criteria);
+            wholeSearchRecordDao.addSort(query, "{\"cTime\":\"descend\"}");
+            return usedList(query, wholeSearchRecordDao.generateCollectionNames(new Date(startTime), new Date(endTime)));
+        } else {// 项目明细,需要细分颗粒度
+            return wholeSearchRecordDao.aggregateProjectUsedRecord(startTime, endTime, day);
+        }
+    }
+
+    @Override
+    public void adjustRecord(Integer currentCount) {
+        if (null == currentCount) {
+            return;
+        }
+        String projectId = UserThreadLocal.getProjectId();
+        Update update = Update.update("wholeSearchBalance", currentCount);
+        Project project = projectDao.findOneById(projectId);
+        projectDao.updateOneByIdWithField(projectId, update);
+        wholeSearchRecordDao.insertOne(WholeSearchRecord.createMaintainRecord(projectId, project.getWholeSearchBalance(), currentCount, UserThreadLocal.getNickname()), wholeSearchRecordDao.generateCollectionName());
+    }
+
+    @Override
+    public JSONObject overview() {
+        JSONObject json = new JSONObject();
+        long endTime = System.currentTimeMillis();
+        long startTime = endTime - Constant.ONE_MONTH;
+        json.put("balance", projectDao.findOneById(UserThreadLocal.getProjectId()).getWholeSearchBalance());
+        json.put("trendList", wholeSearchRecordDao.aggregateDayLastRecord(startTime, endTime));
+        return json;
+    }
+
+    @Override
+    public void decreaseRecord(String keyword, WholeSearchRecord.UsedType usedType, Integer usedCount) {
+        if (null == usedCount) {
+            return;
+        }
+        String projectId = UserThreadLocal.getProjectId();
+        Update update = new Update();
+        update.inc("wholeSearchBalance", -usedCount);
+        Project project = projectDao.findOneById(projectId);
+        projectDao.updateOneByIdWithField(projectId, update);
+        wholeSearchRecordDao.insertOne(WholeSearchRecord.createUsedRecord(projectId, keyword, usedType.getValue(), project.getWholeSearchBalance(), usedCount, UserThreadLocal.getNickname()), wholeSearchRecordDao.generateCollectionName());
+    }
+
+    private List<JSONObject> usedList(Query query, String... collectionNames) {
+        return wholeSearchRecordDao.findList(query, collectionNames).stream().map(record -> {
+            JSONObject json = new JSONObject();
+            json.put("keyword", record.getKeyword());
+            json.put("usedType", record.getUsedType());
+            json.put("usedCount", record.getUsedCount());
+            json.put("cTime", record.getCTime().getTime());
+            json.put("submitter", record.getSubmitter());
+            return json;
+        }).collect(Collectors.toList());
+    }
+}