非人工项目-舆情库

src/main/java/com/zhiwei/brandkbs2/auth/AuthAspect.java

@@ -88,7 +88,8 @@ public class AuthAspect {
                     }
                     UserThreadLocal.set(userInfo);
                 }
-            } catch (Exception ignore) {
+            } catch (Exception e) {
+                log.info("around-通过token设置userInfo出错,", e);
             } finally {
                 proceed = joinPoint.proceed();
                 UserThreadLocal.clear();

src/main/java/com/zhiwei/brandkbs2/common/GenericAttribute.java

@@ -15,8 +15,8 @@ public class GenericAttribute {
      * es index
      */
     public static final String ES_INDEX_PRE = "brandkbs2_";
-//  public static final String ES_INDEX_TEST = "brandkbs2_test";
-    public static final String ES_INDEX_TEST = "brandkbs2_2022";
+  public static final String ES_INDEX_TEST = "brandkbs2_test";
+//    public static final String ES_INDEX_TEST = "brandkbs2_2022";
     public static final String ES_CHANNEL_INDEX_TEST = "brandkbs2_channel_record_test";
     public static final String ES_CHANNEL_INDEX_PREFIX = "brandkbs2_channel_record_";
     /**
@@ -145,6 +145,7 @@ public class GenericAttribute {
     public static final String EXPORT_AMOUNT = "exportAmount";
     public static final String EXPIRED_TIME = "expiredTime";
     public static final String PHONE_NUMBER = "phoneNumber";
+    public static final String IS_MANUAL = "isManual";
 
 //    public enum ChannelParam{
 //        负面稿件数("negativeArticles"),

src/main/java/com/zhiwei/brandkbs2/common/RedisKeyPrefix.java

@@ -100,6 +100,8 @@ public class RedisKeyPrefix {
      */
     public static final String TOOLSET_ARTICLE_SUMMARY_LIMIT = "BRANDKBS:TOOLSET:ARTICLE_SUMMARY:LIMIT:";
 
+    public static final String NON_MANUAL_PROJECT_MARK_MAX_GID = "BRANDKBS:NON_MANUAL:PROJECT:MARK:MAX_GID:";
+
     public static String projectWarnHotTopKeyAll(String projectId, String type) {
         return RedisKeyPrefix.generateRedisKey(RedisKeyPrefix.PROJECT_WARN_HOT_TOP, projectId, Tools.concat(type, "*"));
     }

src/main/java/com/zhiwei/brandkbs2/config/Constant.java

@@ -13,6 +13,7 @@ import java.text.ParseException;
 public class Constant {
 
     public static final Long EIGHT_HOUR = 8 * 60 * 60 * 1000L;
+    public static final Long ONE_HOUR = 1 * 60 * 60 * 1000L;
     public static final Long ONE_DAY = 24 * 60 * 60 * 1000L;
     public static final Long ONE_WEEK = 7 * 24 * 60 * 60 * 1000L;
     public static final Long ONE_MONTH = 30 * 24 * 60 * 60 * 1000L;

src/main/java/com/zhiwei/brandkbs2/controller/admin/ProjectController.java

@@ -262,4 +262,50 @@ public class ProjectController extends BaseController {
         noticeInfoService.deleteNoticeInfo(id);
         return ResponseResult.success();
     }
+
+    @ApiOperation("添加非人工项目")
+    @PostMapping("/non-manual/add")
+    public ResponseResult addNonManualProject(@RequestBody ProjectVO projectVO) {
+        return ProjectService.addNonManualProject(projectVO);
+    }
+
+    @ApiOperation("修改非人工项目")
+    @PutMapping("/non-manual/update")
+    public ResponseResult updateNonManualProject(@RequestBody ProjectVO projectVO) {
+        return ProjectService.updateNonManualProject(projectVO);
+    }
+
+    @ApiOperation("查询所有非人工项目")
+    @ApiImplicitParams({@ApiImplicitParam(name = "page", value = "页码", required = false, defaultValue = "1", paramType = "query", dataType = "int"),
+                        @ApiImplicitParam(name = "size", value = "每页记录数", required = false, defaultValue = "10", paramType = "query", dataType = "int"),
+                        @ApiImplicitParam(name = "sorter", value = "排序", required = false, defaultValue = "{\"cTime\":\"descend\"}", paramType = "query", dataType = "string"),
+                        @ApiImplicitParam(name = "keyword", value = "搜索关键字", required = false, defaultValue = "", paramType = "query", dataType = "string")})
+    @GetMapping("/non-manual/list")
+    public ResponseResult findNonManualProjectList(@RequestParam(value = "page", defaultValue = "1") int page,
+                                                   @RequestParam(value = "size", defaultValue = "10") int size,
+                                                   @RequestParam(value = "sorter", defaultValue = "{\"cTime\":\"descend\"}") String sorter,
+                                                   @RequestParam(value = "keyword", defaultValue = "") String keyword) {
+        return ResponseResult.success(ProjectService.findNonManualProjectList(page, size, sorter, keyword));
+    }
+
+    @ApiOperation("非人工项目-维护记录列表")
+    @GetMapping("/non-manual/maintain/list")
+    public ResponseResult findNonManualProjectMaintainList(@RequestParam(value = "page", defaultValue = "1") int page,
+                                                           @RequestParam(value = "size", defaultValue = "10") int size,
+                                                           @RequestParam(value = "projectId") String projectId) {
+        return ResponseResult.success(ProjectService.findNonManualProjectMaintainList(projectId, page, size));
+    }
+
+    @ApiOperation("非人工项目-维护记录数据总览")
+    @GetMapping("/non-manual/maintain/overview")
+    public ResponseResult interactionOverview(@RequestParam(value = "projectId") String projectId) {
+        return ResponseResult.success(ProjectService.nonManualProjectOverview(projectId));
+    }
+
+    @ApiOperation("非人工项目-维护记录-剩余数据量调整")
+    @PutMapping("/non-manual/maintain/adjust")
+    public ResponseResult adjustInteractionRecord(@ApiParam(name = "json:{count:调整数值, projectId:项目id)") @RequestBody JSONObject json) {
+        ProjectService.adjustNonManualProjectDataBalance(json.getString("projectId"), json.getInteger("count"));
+        return ResponseResult.success();
+    }
 }

src/main/java/com/zhiwei/brandkbs2/controller/app/AppArticleController.java

@@ -9,17 +9,11 @@ import com.zhiwei.brandkbs2.controller.BaseController;
 import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
 import com.zhiwei.brandkbs2.enmus.RoleEnum;
 import com.zhiwei.brandkbs2.model.ResponseResult;
-import com.zhiwei.brandkbs2.pojo.dto.ExportAppYuqingDTO;
-import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
-import com.zhiwei.brandkbs2.pojo.dto.ReportDTO;
-import com.zhiwei.brandkbs2.pojo.dto.ReportSearchDTO;
+import com.zhiwei.brandkbs2.pojo.dto.*;
 import com.zhiwei.brandkbs2.service.*;
 import com.zhiwei.brandkbs2.util.Tools;
 import com.zhiwei.middleware.mark.vo.MarkerTag;
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiImplicitParam;
-import io.swagger.annotations.ApiImplicitParams;
-import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.*;
 import org.apache.commons.lang3.tuple.Pair;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
@@ -321,6 +315,132 @@ public class AppArticleController extends BaseController {
         return ResponseResult.success(markDataService.interactionUpdatePrediction(markSearchDTO));
     }
 
+    @ApiOperation("非人工项目-方案设置-新建方案")
+    @PostMapping("/non-manual/plan/add")
+    public ResponseResult addNonManualProjectPlan(@RequestBody NonManualProjectPlanDTO dto){
+        return markDataService.addNonManualProjectPlan(dto);
+    }
+
+    @ApiOperation("非人工项目-方案设置-删除方案")
+    @DeleteMapping("/non-manual/plan/delete/{id}")
+    public ResponseResult deleteNonManualProjectPlan(@PathVariable String id) {
+        markDataService.deleteNonManualProjectPlan(id);
+        return ResponseResult.success();
+    }
+
+    @ApiOperation("非人工项目-方案设置-获取方案详情")
+    @GetMapping("/non-manual/plan/{id}")
+    public ResponseResult getNonManualProjectPlanInfo(@PathVariable String id) {
+        return ResponseResult.success(markDataService.getNonManualProjectPlanInfo(id));
+    }
+
+    @ApiOperation("非人工项目-方案设置-编辑方案")
+    @PostMapping("/non-manual/plan/update")
+    public ResponseResult updateNonManualProjectPlan(@RequestBody NonManualProjectPlanDTO dto){
+        return markDataService.updateNonManualProjectPlan(dto);
+    }
+
+    @ApiOperation("非人工项目-方案设置-数据量预估")
+    @PutMapping("/non-manual/plan/prediction")
+    public ResponseResult getPlanDataPrediction(@ApiParam(name = "json:{keyword:关键词, filterWord:去噪词)") @RequestBody JSONObject json){
+        return ResponseResult.success(markDataService.getPlanDataPrediction(json.getString("keyword"), json.getString("filterWord")));
+    }
+
+    @ApiOperation("非人工项目-方案设置-方案列表")
+    @GetMapping("/non-manual/plan/list")
+    public ResponseResult getNonManualProjectPlanList(){
+        return ResponseResult.success(markDataService.getNonManualProjectPlanList());
+    }
+
+    @ApiOperation("非人工项目-方案设置-昨日，今日数据消耗量")
+    @GetMapping("/non-manual/plan/count")
+    public ResponseResult getProjectPlanConsume(){
+        return ResponseResult.success(markDataService.getProjectPlanConsume());
+    }
+
+    @ApiOperation("非人工项目-方案设置-获取总数据消耗图谱")
+    @GetMapping("/non-manual/plan/total-graph")
+    public ResponseResult getTotalDataConsumeGraph(@RequestParam(value = "startTime") Long startTime,
+                                                   @RequestParam(value = "endTime") Long endTime){
+        return ResponseResult.success(markDataService.getTotalDataConsumeGraph(startTime, endTime));
+    }
+
+    @ApiOperation("非人工项目-方案设置-获取时间范围数据消耗图谱")
+    @GetMapping("/non-manual/plan/time-graph")
+    public ResponseResult getTimeDataConsumeGraph(@RequestParam(value = "planId") String planId,
+                                                  @RequestParam(value = "startTime") Long startTime,
+                                                  @RequestParam(value = "endTime") Long endTime){
+        return ResponseResult.success(markDataService.getTimeDataConsumeGraph(planId, startTime, endTime));
+    }
+
+    @ApiOperation("非人工项目-舆情数据-舆情数据筛选条件")
+    @GetMapping("/non-manual/mark/criteria")
+    public ResponseResult getNonManualMarkCriteria(){
+        return ResponseResult.success(markDataService.getNonManualMarkCriteria());
+    }
+
+    @ApiOperation("非人工项目-舆情数据-舆情数据列表")
+    @PostMapping("/non-manual/mark/list")
+    public ResponseResult getNonManualMarkList(@RequestBody MarkSearchDTO markSearchDTO){
+        return ResponseResult.success(markDataService.getNonManualMarkList(markSearchDTO));
+    }
+
+    @ApiOperation("非人工项目-舆情数据-单条纠错")
+    @PostMapping("/non-manual/mark/tag")
+    public ResponseResult updateNonManualMarkTag(@RequestBody JSONObject json) {
+        String zip = json.getString("zip");
+        String linkedGroupId = json.getString("linkedGroupId");
+        StringBuilder mtag = new StringBuilder(json.getString("mtag"));
+        // 带上项目配置的必要标注标签
+        if (checkMTagIllegal(mtag)){
+            return ResponseResult.failure("标注标签出现同组标签，请勿连点标注");
+        }
+        markDataService.addMtag(zip, linkedGroupId, mtag.toString());
+        return ResponseResult.success();
+    }
+
+    @ApiOperation("非人工项目-舆情数据-互动量更新")
+    @PostMapping("/non-manual/mark/interaction-update")
+    @LogRecord(description = "非人工项目舆情库-有效舆情互动量更新")
+    @DownloadTask(taskName = "舆情库互动量更新结果下载", description = "舆情库互动量更新结果", entity = true)
+    public ResponseResult nonManualMarkInteractionUpdate(@RequestBody MarkSearchDTO markSearchDTO){
+        return markDataService.nonManualMarkInteractionUpdate(markSearchDTO);
+    }
+
+    @ApiOperation("非人工项目-舆情库互动量更新-数据量预估")
+    @PostMapping("/non-manual/mark/interaction-prediction")
+    public ResponseResult nonManualInteractionUpdatePrediction(@RequestBody MarkSearchDTO markSearchDTO){
+        return ResponseResult.success(markDataService.nonManualInteractionUpdatePrediction(markSearchDTO));
+    }
+
+    @ApiOperation("非人工项目-舆情数据-统计各方案数据量")
+    @GetMapping("/non-manual/mark/count")
+    public ResponseResult countNonManualMarkData(){
+        return ResponseResult.success(markDataService.getNonManualMarkCountList());
+    }
+
+    @ApiOperation("非人工项目-舆情数据-记录gid")
+    @PutMapping("/non-manual/mark/gid")
+    public ResponseResult adjustMaxGid(@ApiParam(name = "json:{planId:方案id, maxGid:值)") @RequestBody JSONObject json) {
+        markDataService.adjustMaxGid(json.getString("planId"), json.getLong("maxGid"));
+        return ResponseResult.success();
+    }
+
+    @ApiOperation("非人工项目-舆情数据-生成聚合")
+    @PostMapping("/non-manual/mark/aggree")
+    public ResponseResult generateNonManualMarkAggreeList(@ApiParam(name = "info:{planId:方案id, startTime:开始时间, endTime:结束时间)") @RequestBody JSONObject info) {
+        String planId = info.getString("planId");
+        Long startTime = info.getLong("startTime");
+        Long endTime = info.getLong("endTime");
+        return ResponseResult.success(markDataService.generateNonManualMarkAggreeList(planId, startTime, endTime));
+    }
+
+    @ApiOperation("非人工项目-舆情数据-获取聚合结果")
+    @PostMapping("/non-manual/mark/aggree/list")
+    public ResponseResult getNonManualMarkAggreeList(@RequestBody MarkSearchDTO markSearchDTO) {
+        return ResponseResult.success(markDataService.getNonManualMarkAggreeList(markSearchDTO));
+    }
+
     private boolean checkMTagIllegal(StringBuilder mtag) {
         List<MarkerTag> hitTags = projectService.getProjectById(UserThreadLocal.getProjectId()).getHitTags();
         if (!Tools.isEmpty(hitTags)) {

src/main/java/com/zhiwei/brandkbs2/controller/app/AppDownloadController.java

@@ -336,6 +336,19 @@ public class AppDownloadController extends BaseController {
         return ResponseResult.success(filePath);
     }
 
+    @ApiOperation("非人工项目-舆情库-舆情数据下载")
+    @PostMapping(value = "/yuqing/non-manual/mark")
+    @LogRecord(description = "非人工项目-舆情库-舆情导出", values = {"startTime", "endTime", "customTags", "field", "keyword", "politicsLevel", "mainBodyType", "platforms", "region", "tags"}, entity = true, arguments = true)
+    @DownloadTask(taskName = "舆情库舆情数据下载", description = "舆情库舆情数据")
+    public ResponseResult downloadNonManualMarkList(@RequestBody MarkSearchDTO markSearchDTO) {
+        Pair<String, List<ExportAppYuqingDTO>> stringListPair = markDataService.downloadNonManualMarkList(markSearchDTO);
+        // excel写入至指定路径
+        String projectName = projectService.getProjectById(UserThreadLocal.getProjectId()).getProjectName();
+        String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), stringListPair.getLeft() + "_舆情列表数据");
+        EasyExcelUtil.write(filePath, "sheet1", ExportAppYuqingDTO.class, stringListPair.getRight());
+        return ResponseResult.success(filePath);
+    }
+
     @ApiOperation("渠道库文章列表")
     @ApiImplicitParams({
             @ApiImplicitParam(name = "startTime", value = "开始时间", required = true, paramType = "query", dataType = "long"),

src/main/java/com/zhiwei/brandkbs2/dao/NonManualProjectDataUpdateRecordDao.java

+package com.zhiwei.brandkbs2.dao;
+
+import com.zhiwei.brandkbs2.pojo.NonManualProjectDataUpdateRecord;
+import com.zhiwei.brandkbs2.pojo.vo.LineVO;
+
+import java.util.List;
+
+/**
+ * @ClassName: BytedanceCustomEventUpdateTaskDataDao
+ * @Description BytedanceCustomEventUpdateTaskDataDao
+ * @author: cjz
+ * @date: 2023-11-10 16:30
+ */
+public interface NonManualProjectDataUpdateRecordDao extends BaseMongoDao<NonManualProjectDataUpdateRecord>{
+    /**
+     * 数据总览
+     * @return list
+     */
+    List<LineVO> aggregateDayLastRecord(String projectId);
+}

src/main/java/com/zhiwei/brandkbs2/dao/NonManualProjectPlanDao.java

+package com.zhiwei.brandkbs2.dao;
+
+import com.zhiwei.brandkbs2.pojo.NonManualProjectPlan;
+
+/**
+ * @ClassName: NonManualProjectPlanDao
+ * @Description NonManualProjectPlanDao
+ * @author: cjz
+ * @date: 2023-11-16 10:30
+ */
+public interface NonManualProjectPlanDao extends BaseMongoDao<NonManualProjectPlan> {
+}

src/main/java/com/zhiwei/brandkbs2/dao/impl/NonManualProjectDataUpdateRecordDaoImpl.java

+package com.zhiwei.brandkbs2.dao.impl;
+
+import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.brandkbs2.dao.NonManualProjectDataUpdateRecordDao;
+import com.zhiwei.brandkbs2.pojo.NonManualProjectDataUpdateRecord;
+import com.zhiwei.brandkbs2.pojo.vo.LineVO;
+import org.springframework.data.domain.Sort;
+import org.springframework.data.mongodb.core.aggregation.Aggregation;
+import org.springframework.data.mongodb.core.aggregation.AggregationOperation;
+import org.springframework.data.mongodb.core.aggregation.AggregationResults;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.stereotype.Component;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @ClassName: BytedanceCustomEventUpdateTaskDataDaoImpl
+ * @Description BytedanceCustomEventUpdateTaskDataDaoImpl
+ * @author: cjz
+ * @date: 2023-11-10 16:31
+ */
+@Component("nonManualProjectDataUpdateRecordDao")
+public class NonManualProjectDataUpdateRecordDaoImpl extends BaseMongoDaoImpl<NonManualProjectDataUpdateRecord> implements NonManualProjectDataUpdateRecordDao {
+    private static final String COLLECTION_NAME = "brandkbs_project_data_update_record";
+
+    public NonManualProjectDataUpdateRecordDaoImpl() {
+        super(COLLECTION_NAME);
+    }
+
+    @Override
+    public List<LineVO> aggregateDayLastRecord(String projectId) {
+        List<LineVO> res = new ArrayList<>();
+        Criteria criteria = Criteria.where("projectId").is(projectId);
+        List<AggregationOperation> operations = new ArrayList<>();
+        // operations区分先后顺序
+        operations.add(Aggregation.match(criteria));
+        operations.add(Aggregation.project("currentCount").and("cTime").substring(0, 10).as("cTime"));
+        operations.add(Aggregation.group("cTime").last("currentCount").as("count"));
+        operations.add(Aggregation.sort(Sort.by(Sort.Order.asc("_id"))));
+        Aggregation aggregation = Aggregation.newAggregation(operations);
+        // 优先小的年份
+        AggregationResults<JSONObject> aggregateResult = mongoTemplate.aggregate(aggregation, COLLECTION_NAME, JSONObject.class);
+        List<JSONObject> mappedResults = aggregateResult.getMappedResults();
+        for (JSONObject mappedResult : mappedResults) {
+            res.add(new LineVO(mappedResult.getLong("count"), mappedResult.getDate("_id").getTime()));
+        }
+        return res;
+    }
+}

src/main/java/com/zhiwei/brandkbs2/dao/impl/NonManualProjectPlanDaoImpl.java

+package com.zhiwei.brandkbs2.dao.impl;
+
+import com.zhiwei.brandkbs2.dao.NonManualProjectPlanDao;
+import com.zhiwei.brandkbs2.pojo.NonManualProjectPlan;
+import org.springframework.stereotype.Component;
+
+/**
+ * @ClassName: NonManualProjectPlanDaoImpl
+ * @Description NonManualProjectPlanDaoImpl
+ * @author: cjz
+ * @date: 2023-11-16 10:31
+ */
+@Component("nonManualProjectPlanDao")
+public class NonManualProjectPlanDaoImpl extends BaseMongoDaoImpl<NonManualProjectPlan> implements NonManualProjectPlanDao {
+
+    private static final String COLLECTION_NAME = "brandkbs_non_manual_project_plan";
+
+    public NonManualProjectPlanDaoImpl() {
+        super(COLLECTION_NAME);
+    }
+}

src/main/java/com/zhiwei/brandkbs2/es/EsQueryTools.java

@@ -104,8 +104,12 @@ public class EsQueryTools {
         return assembleCacheMapsQuery(projectId, contendId, politicsLevel, field, region, mainBodyType, null);
     }
 
-    public static BoolQueryBuilder assembleCacheMapsQuery(String projectId,String contendId, String politicsLevel, String field, String region,
-                                                          String mainBodyType,Integer channelEmotion) {
+    public static BoolQueryBuilder assembleCacheMapsQuery(String projectId, String planId, String politicsLevel) {
+        return assembleCacheMapsQuery(projectId, planId, politicsLevel, null, null, null, null);
+    }
+
+    public static BoolQueryBuilder assembleCacheMapsQuery(String projectId, String contendId, String politicsLevel, String field, String region,
+                                                          String mainBodyType, Integer channelEmotion) {
         BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
         String key = concat(projectId, contendId);
         BoolQueryBuilder nestedBoolBuilder = QueryBuilders.boolQuery();
@@ -121,6 +125,22 @@ public class EsQueryTools {
         return boolQueryBuilder;
     }
 
+    public static BoolQueryBuilder assembleCacheMapsPlanQuery(String projectId, String planId) {
+        return assembleCacheMapsPlanQuery(projectId, planId, null);
+    }
+
+    public static BoolQueryBuilder assembleCacheMapsPlanQuery(String projectId, String planId, String politicsLevel) {
+        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+        String key = concat(projectId, planId);
+        BoolQueryBuilder nestedBoolBuilder = QueryBuilders.boolQuery();
+        // 必要条件
+        nestedBoolBuilder.must(QueryBuilders.termQuery("brandkbs_cache_maps.plan_key.keyword", key));
+        boolQueryBuilder.must(cacheMapsNestedQuery(nestedBoolBuilder));
+        // sensitiveChannel
+        addSensitiveChannel(politicsLevel, null, null, null, boolQueryBuilder);
+        return boolQueryBuilder;
+    }
+
     public static BoolQueryBuilder assembleCacheMapsQueryWithProject(String projectId, BrandkbsYuQingConfig config) {
         BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
         BoolQueryBuilder nestedBoolBuilder = QueryBuilders.boolQuery();
@@ -254,6 +274,31 @@ public class EsQueryTools {
         return queryBuilder;
     }
 
+    public static BoolQueryBuilder assembleForward2Query(boolean isForward) {
+        BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
+        // 需要转发数据，平台必须为微博
+        if (isForward) {
+            BoolQueryBuilder mustWeiboCondition = QueryBuilders.boolQuery();
+            queryBuilder.should(mustWeiboCondition.must(QueryBuilders.termQuery("platform_id", "5d02236e6395002a7c380b79")));
+            queryBuilder.should(QueryBuilders.termQuery("is_forward", true));
+        }else {
+            queryBuilder.mustNot(QueryBuilders.termQuery("is_forward", true));
+        }
+        return queryBuilder;
+    }
+
+    /**
+     * c2查询
+     *
+     * @param dataType 数据类型（1:长文本, 2:短文本, 3:问答, 5:视频）
+     * @return
+     */
+    public static BoolQueryBuilder assembleC2Query(Integer dataType) {
+        BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
+        queryBuilder.should(QueryBuilders.termQuery("c2", dataType << 23));
+        return queryBuilder;
+    }
+
     /**
      * 字段不拆封,多字段 同关键词
      *
@@ -349,8 +394,7 @@ public class EsQueryTools {
 
     public static void addSensitiveChannel(String politicsLevel, String field, String region, String mainBodyType, BoolQueryBuilder boolQueryBuilder) {
         if (null != politicsLevel) {
-            boolQueryBuilder.must(QueryBuilders.termQuery(GenericAttribute.ES_SENSITIVE_CHANNEL + "." + GenericAttribute.ES_POLITICS_LEVEL + ".keyword",
-                    politicsLevel));
+            boolQueryBuilder.must(QueryBuilders.termQuery(GenericAttribute.ES_SENSITIVE_CHANNEL + "." + GenericAttribute.ES_POLITICS_LEVEL + ".keyword", politicsLevel));
         }
         if (null != mainBodyType) {
             boolQueryBuilder.must(QueryBuilders.matchPhraseQuery(GenericAttribute.ES_SENSITIVE_CHANNEL + "." + GenericAttribute.ES_MAIN_BODY_TYPE + ".keyword", mainBodyType));

src/main/java/com/zhiwei/brandkbs2/pojo/ChannelIndex.java

@@ -103,9 +103,14 @@ public class ChannelIndex extends AbstractBaseMongo {
             return res;
         }
         List<Map<String, Object>> cacheMaps = (List<Map<String, Object>>) sourceAsMap.get(GenericAttribute.ES_BRANDKBS_CACHE_MAPS);
+        List<String> manualProjectIds = GlobalPojo.PROJECT_MAP.entrySet().stream().filter(entry -> entry.getValue().isManual()).map(Map.Entry::getKey).collect(Collectors.toList());
         for (Map<String, Object> cacheMap : cacheMaps) {
             String projectId = String.valueOf(cacheMap.get("project_id"));
             String contendId = String.valueOf(cacheMap.get("contend_id"));
+            // 剔除非人工项目
+            if(manualProjectIds.contains(projectId)){
+                continue;
+            }
             ChannelIndex channelIndex = new ChannelIndex(projectId, contendId, messagePlatform.getName(), realSource, source);
             // 默认主品牌
             channelIndex.setContendId(String.valueOf(0));

src/main/java/com/zhiwei/brandkbs2/pojo/MarkFlowEntity.java

@@ -100,6 +100,11 @@ public class MarkFlowEntity implements Serializable {
             tJson.put(GenericAttribute.ES_CNAME, "采集来源缺失");
         }
         int c2 = tJson.getIntValue(GenericAttribute.ES_C2);
+        // 微博平台头像url拼接
+        String weiboHeadUrl = "https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/weibo-avatar-url?url=";
+        if (1020 == tJson.getIntValue(GenericAttribute.ES_C4) && !tJson.getString("avatar_url").contains(weiboHeadUrl)){
+            tJson.put("avatar_url", weiboHeadUrl + tJson.getString("avatar_url"));
+        }
         switch (ClassB.TypeB.fromEncode(c2)) {
             case COMPLETE:
                 if (!Tools.isEmpty(tJson.get("ind_full_text"))) {

src/main/java/com/zhiwei/brandkbs2/pojo/NonManualProjectDataUpdateRecord.java

+package com.zhiwei.brandkbs2.pojo;
+
+import lombok.Getter;
+import lombok.Setter;
+
+import java.util.Date;
+
+/**
+ * @ClassName: NonManualProjectDataUpdateRecord
+ * @Description 非人工项目剩余数据量调整记录
+ * @author: cjz
+ * @date: 2023-11-10 15:53
+ */
+@Getter
+@Setter
+public class NonManualProjectDataUpdateRecord extends AbstractBaseMongo {
+    /**
+     * 项目id
+     */
+    private String projectId;
+    /**
+     * 上一次记录数(本次调整前剩余次数)
+     */
+    private int lastCount;
+
+    /**
+     * 本次记录数(本次调整后剩余次数)
+     */
+    private int currentCount;
+
+    /**
+     * 提交人
+     */
+    private String submitter;
+
+    /**
+     * 创建时间(Date类型方便聚合查询)
+     */
+    private Date cTime;
+
+    public NonManualProjectDataUpdateRecord(String projectId, int lastCount, int currentCount, String submitter){
+        this.projectId = projectId;
+        this.lastCount = lastCount;
+        this.currentCount = currentCount;
+        this.submitter = submitter;
+        this.cTime = new Date();
+    }
+}

src/main/java/com/zhiwei/brandkbs2/pojo/NonManualProjectPlan.java

+package com.zhiwei.brandkbs2.pojo;
+
+import com.zhiwei.brandkbs2.auth.UserThreadLocal;
+import com.zhiwei.brandkbs2.pojo.dto.NonManualProjectPlanDTO;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+import java.util.List;
+
+/**
+ * @ClassName: NonManualProjectPlan
+ * @Description 非人工项目方案
+ * @author: cjz
+ * @date: 2023-11-15 14:53
+ */
+@Getter
+@Setter
+@NoArgsConstructor
+public class NonManualProjectPlan extends AbstractBaseMongo{
+    /**
+     * 项目id
+     */
+    private String projectId;
+    /**
+     * 方案名
+     */
+    private String name;
+    /**
+     * 关键词
+     */
+    private String keyword;
+    /**
+     * 去噪词
+     */
+    private String filterWord;
+    /**
+     * 方案类型 （永久/临时）
+     */
+    private Boolean lasting;
+    /**
+     * 方案结束时间
+     */
+    private Long endTime;
+    /**
+     * 数据范围 （原创，转发，ocr）
+     */
+    private List<String> type;
+    /**
+     * 创建时间
+     */
+    private Long cTime;
+    /**
+     * 修改时间
+     */
+    private Long uTime;
+    /**
+     * 提交人
+     */
+    private String submitter;
+
+    public NonManualProjectPlan(NonManualProjectPlanDTO dto){
+        this.projectId = UserThreadLocal.getProjectId();
+        this.name = dto.getName();
+        this.keyword = dto.getKeyword();
+        this.filterWord = dto.getFilterWord();
+        this.lasting = dto.getLasting();
+        this.endTime = dto.getEndTime();
+        this.type = dto.getType();
+        this.cTime = System.currentTimeMillis();
+        this.uTime = System.currentTimeMillis();
+        this.submitter = UserThreadLocal.getNickname();
+    }
+}

src/main/java/com/zhiwei/brandkbs2/pojo/Project.java

@@ -75,6 +75,11 @@ public class Project extends AbstractProject {
     private boolean isStart;
 
     /**
+     * 是否为人工项目
+     */
+    private boolean isManual;
+
+    /**
      * 创建时间
      */
     private Long cTime;
@@ -100,6 +105,11 @@ public class Project extends AbstractProject {
     private int interactionBalance;
 
     /**
+     * 剩余数据量（仅非人工项目）
+     */
+    private int dataBalance;
+
+    /**
      * 项目主品牌配置信息转换
      *
      * @return 项目对象

src/main/java/com/zhiwei/brandkbs2/pojo/dto/MarkSearchDTO.java

@@ -160,4 +160,28 @@ public class MarkSearchDTO {
     @ApiModelProperty(value = "屏蔽词")
     private String shieldWord;
 
+    /**
+     * 方案id,仅非人工项目
+     */
+    @ApiModelProperty(value = "方案id")
+    private String planId;
+    /**
+     * 内容类型（原创/转发）,仅非人工项目
+     */
+    @ApiModelProperty(value = "内容类型（原创/转发）")
+    private String forward;
+    /**
+     * 信息浏览（已读/未读）,仅非人工项目
+     */
+    @ApiModelProperty(value = "信息浏览（已读/未读）")
+    private String read;
+    /**
+     * 数据类型（长文本/短文本/问答/视频）,仅非人工项目
+     */
+    @ApiModelProperty(value = "数据类型（1:长文本, 2:短文本, 3:问答, 5:视频）")
+    private Integer dataType;
+    /**
+     * gid用于未读已读筛选,仅非人工项目
+     */
+    private Long gid;
 }

src/main/java/com/zhiwei/brandkbs2/pojo/dto/NonManualProjectPlanDTO.java

+package com.zhiwei.brandkbs2.pojo.dto;
+
+import com.zhiwei.brandkbs2.pojo.NonManualProjectPlan;
+import io.swagger.annotations.ApiModelProperty;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+import java.util.List;
+
+/**
+ * @ClassName: NonManualProjectPlan
+ * @Description 非人工项目方案传输
+ * @author: cjz
+ * @date: 2023-11-16 9:53
+ */
+@Getter
+@Setter
+@AllArgsConstructor
+@NoArgsConstructor
+public class NonManualProjectPlanDTO {
+    /**
+     * 主键ID
+     */
+    @ApiModelProperty("项目主键ID")
+    private String id;
+    /**
+     * 方案名
+     */
+    private String name;
+    /**
+     * 关键词
+     */
+    private String keyword;
+    /**
+     * 去噪词
+     */
+    private String filterWord;
+    /**
+     * 方案类型 （永久/临时）
+     */
+    private Boolean lasting;
+    /**
+     * 方案结束时间
+     */
+    private Long endTime;
+    /**
+     * 数据范围 （原创，转发，ocr）
+     */
+    private List<String> type;
+
+    public NonManualProjectPlanDTO(NonManualProjectPlan plan){
+        this.id = plan.getId();
+        this.name = plan.getName();
+        this.keyword = plan.getKeyword();
+        this.filterWord = plan.getFilterWord();
+        this.lasting = plan.getLasting();
+        this.endTime = plan.getEndTime();
+        this.type = plan.getType();
+    }
+}

src/main/java/com/zhiwei/brandkbs2/pojo/vo/ProjectVO.java

@@ -9,12 +9,10 @@ import io.swagger.annotations.ApiModel;
 import io.swagger.annotations.ApiModelProperty;
 import lombok.Data;
 import lombok.ToString;
+import org.apache.commons.collections4.CollectionUtils;
 import org.bson.types.ObjectId;
 
-import java.util.Arrays;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.stream.Collectors;
 
 /**
@@ -166,11 +164,38 @@ public class ProjectVO {
         project.setBlackChannelGroup(this.getBlackChannelGroup());
         project.setShow(true);
         project.setStart(false);
+        project.setManual(true);
         project.setCTime(time.getTime());
         project.setUTime(time.getTime());
         return project;
     }
 
+    public static Project createNonManualProject(ProjectVO vo){
+        Project project = new Project();
+        project.setId(vo.getId());
+        project.setProjectName(vo.getProjectName());
+        project.setBrandName(vo.getBrandName());
+        project.setBrandLinkedGroup(vo.getBrandLinkedGroup());
+        project.setBrandLinkedGroupId(vo.getBrandLinkedGroupId());
+        project.setImportTime(vo.getImportTime().getTime());
+        if (null == vo.getAvatarUrl()) {
+            project.setAvatarUrl(Constant.DEFAULT_PROJECT_URL);
+        } else {
+            project.setAvatarUrl(vo.getAvatarUrl());
+        }
+        project.setModuleShowList(Collections.emptyList());
+        project.setToolsetShowList(Arrays.asList("articleSummary", "interactionUpdate", "highWord"));
+        // 新建项目时默认数据量50w
+        project.setDataBalance(500000);
+        // 非人工项目
+        project.setManual(false);
+        project.setShow(true);
+        project.setStart(false);
+        project.setCTime(System.currentTimeMillis());
+        project.setUTime(System.currentTimeMillis());
+        return project;
+    }
+
 //    private static Map<String, Double> parse2NegativeChannelParmasMap(List<Double> negativeChannel) {
 //        if (4 != negativeChannel.size()) {
 //            ExceptionCast.cast(CommonCodeEnum.INVALID_PARAM.message("不友好渠道传参异常"));

src/main/java/com/zhiwei/brandkbs2/service/EsSearchService.java

@@ -44,6 +44,15 @@ public interface EsSearchService {
     Pair<SearchHits[], Map<String, Long>> searchOriginHitsAndCount(MarkSearchDTO dto, boolean scroll) throws IOException;
 
     /**
+     * 搜索非人工项目数据
+     *
+     * @param dto    搜索实体类
+     * @param scroll 滚动查询
+     * @return 搜索结果
+     */
+    Pair<SearchHits[], Map<String, Long>> searchNonManualHitsAndCount(MarkSearchDTO dto, boolean scroll) throws IOException;
+
+    /**
      * 查舆情舆情库-平台进量（声量）统计
      * @param dto   搜索实体类
      * @return
@@ -58,4 +67,12 @@ public interface EsSearchService {
      * @throws IOException
      */
     JSONObject countOriginPlatform(MarkSearchDTO dto) throws IOException;
+
+    /**
+     * 非人工项目舆情库方案-平台进量（声量）统计
+     * @param dto   搜索实体类
+     * @return
+     * @throws IOException
+     */
+    JSONObject countNonManualPlatform(MarkSearchDTO dto) throws IOException;
 }

src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java

@@ -9,6 +9,8 @@ import com.zhiwei.brandkbs2.pojo.dto.ExportAppYuqingDTO;
 import com.zhiwei.brandkbs2.pojo.dto.ExportSearchWholeDTO;
 import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
 import com.zhiwei.brandkbs2.pojo.dto.SearchFilterDTO;
+import com.zhiwei.brandkbs2.pojo.dto.NonManualProjectPlanDTO;
+import com.zhiwei.brandkbs2.pojo.vo.LineVO;
 import com.zhiwei.brandkbs2.pojo.vo.PageVO;
 import org.apache.commons.lang3.tuple.Pair;
 import org.elasticsearch.index.query.BoolQueryBuilder;
@@ -392,4 +394,121 @@ public interface MarkDataService {
      * @return
      */
     JSONObject countOriginPlatform(MarkSearchDTO dto);
+
+    /**
+     * 新建非人工项目方案
+     * @param dto
+     * @return
+     */
+    ResponseResult addNonManualProjectPlan(NonManualProjectPlanDTO dto);
+
+    /**
+     * 删除非人工项目方案
+     * @param id
+     */
+    void deleteNonManualProjectPlan(String id);
+
+    /**
+     * 获取非人工项目方案详情
+     * @param id
+     */
+    NonManualProjectPlanDTO getNonManualProjectPlanInfo(String id);
+
+    /**
+     * 编辑非人工项目方案
+     * @param dto
+     * @return
+     */
+    ResponseResult updateNonManualProjectPlan(NonManualProjectPlanDTO dto);
+
+    /**
+     * 非人工项目方案-预估数据量
+     * @param keyword       关键词
+     * @param filterWord    去噪词
+     * @return
+     */
+    Integer getPlanDataPrediction(String keyword, String filterWord);
+
+    /**
+     * 获取方案设置-方案列表
+     * @return
+     */
+    PageVO<JSONObject> getNonManualProjectPlanList();
+
+    /**
+     * 获取方案设置-方案列表-昨日，今日数据消耗量
+     * @return
+     */
+    JSONObject getProjectPlanConsume();
+
+    /**
+     * 获取时间范围数据消耗图谱
+     * @return
+     */
+    List<LineVO> getTimeDataConsumeGraph(String planId, Long startTime, Long endTime);
+
+    /**
+     * 获取总数据消耗图谱
+     * @return
+     */
+    List<LineVO> getTotalDataConsumeGraph(Long startTime, Long endTime);
+
+    /**
+     * 获取非人工项目舆情数据筛选条件
+     * @return
+     */
+    JSONObject getNonManualMarkCriteria();
+
+    /**
+     * 获取非人工项目舆情数据列表
+     * @param markSearchDTO 标注数据搜索传输类
+     * @return
+     */
+    PageVO<MarkFlowEntity> getNonManualMarkList(MarkSearchDTO markSearchDTO);
+
+    /**
+     * 非人工项目舆情数据列表-互动量更新
+     * @param dto 标注数据搜索传输类
+     * @return
+     */
+    ResponseResult nonManualMarkInteractionUpdate(MarkSearchDTO dto);
+
+    /**
+     * 非人工项目工具库-舆情库互动量更新预估
+     * @param dto 标注数据搜索传输类
+     * @return
+     */
+    Long nonManualInteractionUpdatePrediction(MarkSearchDTO dto);
+
+    /**
+     * @param markSearchDTO 标注数据搜索传输类
+     * @return 返回结果
+     */
+    Pair<String, List<ExportAppYuqingDTO>> downloadNonManualMarkList(MarkSearchDTO markSearchDTO);
+
+    /**
+     * 保存/更新gid
+     * @param gid
+     */
+    void adjustMaxGid(String planId, Long gid);
+
+    /**
+     * 生成聚合列表并返回id
+     *
+     * @return 聚合id
+     */
+    String generateNonManualMarkAggreeList(String planId, Long startTime, Long endTime);
+
+    /**
+     * 获取非人工项目聚合结果
+     * @param dto
+     * @return
+     */
+    PageVO<MarkFlowEntity> getNonManualMarkAggreeList(MarkSearchDTO dto);
+
+    /**
+     * 非人工项目舆情数据列表-方案列表与各方案数据量统计
+     * @return
+     */
+    List<JSONObject> getNonManualMarkCountList();
 }

src/main/java/com/zhiwei/brandkbs2/service/MarkFlowService.java

@@ -33,6 +33,14 @@ public interface MarkFlowService {
     JSONObject createMarkFlowInfo4Origin(MarkInfoSource markInfoSource);
 
     /**
+     * 创建标注消息流实体
+     *
+     * @param markInfoSource
+     * @return
+     */
+    JSONObject createNonManualMarkFlowInfo(MarkInfoSource markInfoSource);
+
+    /**
      * 匹配标注标签值
      * @param mes
      * @param linkedGroupId
@@ -72,6 +80,7 @@ public interface MarkFlowService {
         private String projectId;
         private String contendId;
         private String linkedGroupId;
+        private String planId;
 
         public MarkInfoSource(JSONObject json, String projectId, String contendId) {
             this.json = json;
@@ -83,6 +92,13 @@ public interface MarkFlowService {
             this.json = json;
             this.linkedGroupId = linkedGroupId;
         }
+
+        public MarkInfoSource(JSONObject json, String projectId, String contendId, String planId) {
+            this.json = json;
+            this.projectId = projectId;
+            this.contendId = contendId;
+            this.planId = planId;
+        }
     }
 
 }

src/main/java/com/zhiwei/brandkbs2/service/ProjectService.java

@@ -149,4 +149,48 @@ public interface ProjectService {
      * @return
      */
     PageVO<JSONObject> getToolsetBytedanceCustomInfo(int page, int pageSize, String type);
+
+    /**
+     * 添加非人工项目
+     *
+     * @param projectVO 项目配置展示对象
+     */
+    ResponseResult addNonManualProject(ProjectVO projectVO);
+
+    /**
+     * 修改非人工项目信息
+     * @param projectVO 项目配置展示对象
+     * @return
+     */
+    ResponseResult updateNonManualProject(ProjectVO projectVO);
+
+    /**
+     * 查询非人工项目列表
+     * @param page 页码
+     * @param size 页码大小
+     * @param keyword   关键词
+     * @return
+     */
+    PageVO<JSONObject> findNonManualProjectList(int page, int size, String sorter, String keyword);
+
+    /**
+     * 非人工项目剩余数据量调整记录列表
+     *
+     * @param page 页码
+     * @param pageSize 页码大小
+     * @return PageVO
+     */
+    PageVO<JSONObject> findNonManualProjectMaintainList(String projectId, int page, int pageSize);
+
+    /**
+     * 非人工项目剩余数据量调整
+     * @param currentCount 调整后数值
+     */
+    void adjustNonManualProjectDataBalance(String projectId, Integer currentCount);
+
+    /**
+     * 非人工项目剩余数据调整记录总览
+     * @return json
+     */
+    JSONObject nonManualProjectOverview(String project);
 }

src/main/java/com/zhiwei/brandkbs2/service/impl/EsSearchServiceImpl.java

@@ -58,15 +58,23 @@ public class EsSearchServiceImpl implements EsSearchService {
     }
 
     @Override
+    public Pair<SearchHits[], Map<String, Long>> searchNonManualHitsAndCount(MarkSearchDTO dto, boolean scroll) throws IOException {
+        return searchHitsAndCount(dto, this::createNonManualSearchHelper, esClientDao, scroll);
+    }
+
+    @Override
     public JSONObject countMarkPlatform(MarkSearchDTO dto) throws IOException {
-        EsClientDao.SearchHelper helper = createSearchHelper(dto);
-        return platformCount(helper.getPostFilter(), esClientDao);
+        return platformCount(dto, this::createSearchHelper, esClientDao);
     }
 
     @Override
     public JSONObject countOriginPlatform(MarkSearchDTO dto) throws IOException {
-        EsClientDao.SearchHelper helper = createOriginSearchHelper(dto);
-        return platformCount(helper.getPostFilter(), hotEsDao);
+        return platformCount(dto, this::createOriginSearchHelper, hotEsDao);
+    }
+
+    @Override
+    public JSONObject countNonManualPlatform(MarkSearchDTO dto) throws IOException {
+        return platformCount(dto, this::createNonManualSearchHelper, esClientDao);
     }
 
     private Pair<SearchHits[], Map<String, Long>> searchHitsAndCount(MarkSearchDTO dto, ArticleFunction.CreateSearchHelper createSearchHelper,
@@ -259,16 +267,93 @@ public class EsSearchServiceImpl implements EsSearchService {
         return helper;
     }
 
+    private EsClientDao.SearchHelper createNonManualSearchHelper(MarkSearchDTO dto) {
+        EsClientDao.SearchHelper helper = EsClientDao.createSearchHelper();
+        // projectId、contendId
+        String projectId = dto.getProjectId();
+        String planId = dto.getPlanId();
+        // PostFilter 后置过滤器
+        BoolQueryBuilder postFilter = EsQueryTools.assembleCacheMapsPlanQuery(projectId, planId, dto.getPoliticsLevel());
+        // time
+        postFilter.must(QueryBuilders.rangeQuery(dto.getTimeType()).gte(dto.getStartTime()).lt(dto.getEndTime()));
+        // platform
+        if (CollectionUtils.isNotEmpty(dto.getPlatforms())) {
+            postFilter.must(EsQueryTools.assemblePlatformQuery(Tools.getPlatformByIds(dto.getPlatforms())));
+        }
+        // tags
+        if (CollectionUtils.isNotEmpty(dto.getTags())) {
+            postFilter.must(EsQueryTools.assembleTagQuery(dto.getTags()));
+        }
+        // Query 查询条件
+        String[] fieldSearch = "仅标题".equals(dto.getSearchField()) ? new String[]{GenericAttribute.ES_IND_TITLE} : new String[]{GenericAttribute.ES_IND_FULL_TEXT};
+        // keyword
+        if (StringUtils.isNotEmpty(dto.getKeyword())) {
+            // Query
+            postFilter.must(EsQueryTools.assembleNormalKeywordQuery(dto.getKeyword(), fieldSearch));
+        }
+        // sourceKeyword
+        if (StringUtils.isNotEmpty(dto.getSourceKeyword())) {
+            postFilter.must(EsQueryTools.assembleSourceQuery(dto.getSourceKeyword()));
+        }
+        // hostKeyword
+        if (StringUtils.isNotEmpty(dto.getHostKeyword())) {
+            postFilter.must(EsQueryTools.assembleFiledKeywordQuery("host", dto.getHostKeyword()));
+        }
+        // forward
+        if (Objects.nonNull(dto.getForward())){
+            postFilter.must(EsQueryTools.assembleForward2Query(Objects.equals(dto.getForward(), "转发")));
+        }
+        // gid 已读 未读
+        if (Objects.nonNull(dto.getRead()) && Objects.nonNull(dto.getGid())){
+            if (Objects.equals(dto.getRead(), "已读")){
+                postFilter.must(QueryBuilders.rangeQuery(GenericAttribute.ES_GID).lte(dto.getGid()));
+            }else {
+                postFilter.must(QueryBuilders.rangeQuery(GenericAttribute.ES_GID).gt(dto.getGid()));
+            }
+        }
+        // dataType
+        if (Objects.nonNull(dto.getDataType())){
+            postFilter.must(EsQueryTools.assembleC2Query(dto.getDataType()));
+        }
+        helper.setPostFilter(postFilter);
+        // sort
+        FieldSortBuilder sort = null;
+        if (null != dto.getSorter()) {
+            for (Map.Entry<String, Object> entry : dto.getSorter().entrySet()) {
+                String key = entry.getKey();
+                if (key.equals("influence")) {
+                    key = "channel_influence";
+                } else if (key.equals("followers")) {
+                    key = "channel_followers";
+                }
+                if (null == entry.getValue() || entry.getValue().toString().contains("desc")) {
+                    sort = SortBuilders.fieldSort(key).order(SortOrder.DESC);
+                } else {
+                    sort = SortBuilders.fieldSort(key).order(SortOrder.ASC);
+                }
+            }
+        }
+        helper.setSort(sort);
+        // from size
+        if (null != dto.getPageSize()) {
+            helper.setFrom((dto.getPage() - 1) * dto.getPageSize());
+            helper.setSize(dto.getPageSize());
+        }
+        // HighlightBuilder ???
+        return helper;
+    }
+
     /**
      * 平台进量统计
      *
      * @return
      */
-    private JSONObject platformCount(BoolQueryBuilder postFilter, EsClientDao esClientDao) throws IOException {
+    private JSONObject platformCount(MarkSearchDTO dto, ArticleFunction.CreateSearchHelper createSearchHelper, EsClientDao esClientDao) throws IOException {
+        EsClientDao.SearchHelper helper = createSearchHelper.create(dto);
         List<JSONObject> list = new ArrayList<>();
         //平台计量统计 仅第一页且无平台限制
         for (MessagePlatform platform : GlobalPojo.PLATFORMS.stream().filter(messagePlatform -> !"外媒".equals(messagePlatform.getName())).collect(Collectors.toList())) {
-            BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery().must(postFilter).must(EsQueryTools.assemblePlatformQuery(Collections.singletonList(platform)));
+            BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery().must(helper.getPostFilter()).must(EsQueryTools.assemblePlatformQuery(Collections.singletonList(platform)));
             Long count = esClientDao.count(esClientDao.getIndexes(), queryBuilder, null);
             JSONObject jsonObject = new JSONObject();
             jsonObject.put("id", platform.getId());
@@ -282,5 +367,4 @@ public class EsSearchServiceImpl implements EsSearchService {
         res.put("total", total);
         return res;
     }
-
 }

src/main/java/com/zhiwei/brandkbs2/service/impl/MarkFlowServiceImpl.java

@@ -80,6 +80,20 @@ public class MarkFlowServiceImpl implements MarkFlowService {
         return resultInfo;
     }
 
+    @Override
+    public JSONObject createNonManualMarkFlowInfo(MarkInfoSource markInfoSource) {
+        JSONObject json = markInfoSource.getJson();
+        JSONObject resultInfo = createInfoWithEmotion(json);
+        resultInfo.put("sourceDetails", getSourceDetails(markInfoSource));
+        String zip = Tools.gzip(JSON.toJSONString(json));
+        resultInfo.put("zip", zip);
+        // 每条数据高亮词
+        resultInfo.put("highLightWord", getHighLightWords(json, markInfoSource.getProjectId(), markInfoSource.getPlanId()));
+        // 词频
+        resultInfo.put("keywordCount", getKeywordCount(json, markInfoSource.getProjectId(), markInfoSource.getPlanId()));
+        return resultInfo;
+    }
+
     /**
      * 消息实体批量处理标注信息
      *
@@ -193,6 +207,49 @@ public class MarkFlowServiceImpl implements MarkFlowService {
         return info;
     }
 
+    private Map<String, Integer> getKeywordCount(JSONObject json, String projectId, String planId){
+        List<String> keywords = getKeywords(json, projectId, planId);
+        Map<String, Integer> res = new HashMap<>();
+        for (String keyword : keywords) {
+            if (Objects.isNull(keyword)){
+                continue;
+            }
+            String[] keywordAndFrequency = keyword.split(":");
+            String word = keywordAndFrequency[0];
+            Integer frequency = Integer.valueOf(keywordAndFrequency[1]);
+            res.put(word, frequency);
+        }
+        return res;
+    }
+
+    private List<String> getHighLightWords(JSONObject json, String projectId, String planId){
+        List<String> keywords = getKeywords(json, projectId, planId);
+        List<String> res = new ArrayList<>();
+        for (String keyword : keywords) {
+            if (Objects.isNull(keyword)){
+                continue;
+            }
+            String[] keywordAndFrequency = keyword.split(":");
+            String words = keywordAndFrequency[0];
+            if (Objects.nonNull(words)){
+                String[] word = words.split(" ");
+                res.addAll(new ArrayList<>(Arrays.asList(word)));
+            }
+        }
+        return res.stream().distinct().collect(Collectors.toList());
+    }
+
+    private List<String> getKeywords(JSONObject json, String projectId, String planId){
+        List<String> keywords = new ArrayList<>();
+        List<JSONObject> brandkbsCacheMaps = json.getJSONArray("brandkbs_cache_maps").toJavaList(JSONObject.class);
+        for (JSONObject brandkbsCacheMap : brandkbsCacheMaps) {
+            if (Objects.equals(brandkbsCacheMap.getString("plan_key"), Tools.concat(projectId, planId))){
+                keywords.add(brandkbsCacheMap.getString("keyword"));
+            }
+        }
+        return keywords;
+    }
+
     private JSONObject getSourceDetails(MarkInfoSource markInfoSource) {
         JSONObject sourceDetails = new JSONObject();
         JSONObject tJson = markInfoSource.getJson();

src/main/java/com/zhiwei/brandkbs2/service/impl/ProjectServiceImpl.java

@@ -10,10 +10,7 @@ import com.zhiwei.brandkbs2.enmus.response.ProjectCodeEnum;
 import com.zhiwei.brandkbs2.exception.ExceptionCast;
 import com.zhiwei.brandkbs2.model.CommonCodeEnum;
 import com.zhiwei.brandkbs2.model.ResponseResult;
-import com.zhiwei.brandkbs2.pojo.AbstractProject;
-import com.zhiwei.brandkbs2.pojo.Contend;
-import com.zhiwei.brandkbs2.pojo.Project;
-import com.zhiwei.brandkbs2.pojo.User;
+import com.zhiwei.brandkbs2.pojo.*;
 import com.zhiwei.brandkbs2.pojo.vo.PageVO;
 import com.zhiwei.brandkbs2.pojo.vo.ProjectVO;
 import com.zhiwei.brandkbs2.service.ProjectService;
@@ -77,6 +74,12 @@ public class ProjectServiceImpl implements ProjectService {
     @Resource(name = "BytedanceCustomPlatformWeightDao")
     private BytedanceCustomPlatformWeightDao bytedanceCustomPlatformWeightDao;
 
+    @Resource(name = "nonManualProjectDataUpdateRecordDao")
+    private NonManualProjectDataUpdateRecordDao nonManualProjectDataUpdateRecordDao;
+
+    @Resource(name = "nonManualProjectPlanDao")
+    NonManualProjectPlanDao nonManualProjectPlanDao;
+
     @Value("${jwt.key}")
     private String jwtKey;
 
@@ -88,13 +91,13 @@ public class ProjectServiceImpl implements ProjectService {
 
     @Override
     public ResponseResult addProject(ProjectVO projectVO) {
-        if (!Tools.isEmpty(projectVO.getHitKeywords()) && projectVO.getModuleShowList().contains(2)) {
-            return ResponseResult.failure("关键字和原始数据配置异常");
-        }
         if (Objects.isNull(projectVO)) {
             //抛出非法参数异常
             ExceptionCast.cast(CommonCodeEnum.INVALID_PARAM);
         }
+        if (!Tools.isEmpty(projectVO.getHitKeywords()) && projectVO.getModuleShowList().contains(2)) {
+            return ResponseResult.failure("关键字和原始数据配置异常");
+        }
         Project existsProject = projectDao.findOne("projectName", projectVO.getProjectName());
         if (Objects.nonNull(existsProject)) {
             //抛出项目名称已存在异常
@@ -127,6 +130,7 @@ public class ProjectServiceImpl implements ProjectService {
     @Override
     public PageVO<JSONObject> findProjectList(int page, int size, String keyword) {
         Query query = new Query();
+        query.addCriteria(Criteria.where("isManual").is(true));
         projectDao.addKeywordFuzz(query, keyword, "projectName");
         long count = projectDao.count(query);
         projectDao.addSort(query, "{\"cTime\":\"descend\"}");
@@ -238,6 +242,12 @@ public class ProjectServiceImpl implements ProjectService {
 //        });
         //删除项目
         projectDao.deleteOneById(pid);
+        // 删除非人工项目维护记录和方案
+        if (!project.isManual()){
+            Query query = new Query(Criteria.where("projectId").is(pid));
+            nonManualProjectPlanDao.deleteOneByQuery(query);
+            nonManualProjectDataUpdateRecordDao.deleteOneByQuery(query);
+        }
     }
 
     @Override
@@ -475,6 +485,106 @@ public class ProjectServiceImpl implements ProjectService {
         }
     }
 
+    @Override
+    public ResponseResult addNonManualProject(ProjectVO projectVO) {
+        if (Objects.isNull(projectVO)) {
+            //抛出非法参数异常
+            ExceptionCast.cast(CommonCodeEnum.INVALID_PARAM);
+        }
+        Project existsProject = projectDao.findOne("projectName", projectVO.getProjectName());
+        if (Objects.nonNull(existsProject)) {
+            //抛出项目名称已存在异常
+            return ResponseResult.failure(ProjectCodeEnum.PROJECT_EXISTSNAME_ERROR.message());
+        }
+        Project project = ProjectVO.createNonManualProject(projectVO);
+        projectDao.insertOneWithoutId(project);
+        return ResponseResult.success();
+    }
+
+    @Override
+    public ResponseResult updateNonManualProject(ProjectVO projectVO) {
+        Project existsProject = projectDao.findOneById(projectVO.getId());
+        if (Objects.isNull(existsProject)) {
+            //抛出非法参数异常
+            ExceptionCast.cast(CommonCodeEnum.INVALID_PARAM);
+        }
+        Project existsNameProject = projectDao.findOne("projectName", projectVO.getProjectName());
+        //如果该项目的名字已存在且不是本身
+        if (Objects.nonNull(existsNameProject) && !projectVO.getId().equals(existsNameProject.getId())) {
+            //抛出项目名已存在异常
+            ExceptionCast.cast(ProjectCodeEnum.PROJECT_EXISTSNAME_ERROR);
+        }
+        Project project = ProjectVO.createNonManualProject(projectVO);
+        project.setCTime(existsProject.getCTime());
+        project.setShow(existsProject.isShow());
+        project.setDataBalance(existsProject.getDataBalance());
+        project.setStart(existsProject.isStart());
+        projectDao.updateOne(project);
+        return ResponseResult.success();
+    }
+
+    @Override
+    public PageVO<JSONObject> findNonManualProjectList(int page, int size, String sorter, String keyword) {
+        Query query = new Query();
+        query.addCriteria(Criteria.where("isManual").is(false));
+        projectDao.addKeywordFuzz(query, keyword, "projectName");
+        long count = projectDao.count(query);
+        projectDao.addSort(query, sorter);
+        mongoUtil.start(page, size, query);
+        List<Project> projectList = projectDao.findList(query);
+        List<JSONObject> resList = projectList.stream().map(project -> {
+            JSONObject result = new JSONObject();
+            result.put("id", project.getId());
+            result.put("importTime", project.getImportTime());
+            result.put("cTime", project.getCTime());
+            result.put("projectName", project.getProjectName());
+            result.put("brandName", project.getBrandName());
+            result.put("isStart", project.isStart());
+            result.put("isShow", project.isShow());
+            result.put("dataBalance", project.getDataBalance());
+            return result;
+        }).collect(Collectors.toList());
+        MongoUtil.PageHelper<JSONObject> pageHelper = mongoUtil.pageHelper(count, resList);
+        return PageVO.createPageVo(pageHelper, resList);
+    }
+
+    @Override
+    public PageVO<JSONObject> findNonManualProjectMaintainList(String projectId, int page, int pageSize) {
+        Query query = Query.query(Criteria.where("projectId").is(projectId));
+        nonManualProjectDataUpdateRecordDao.addSort(query, "{\"cTime\":\"descend\"}");
+        long total = nonManualProjectDataUpdateRecordDao.count(query);
+        mongoUtil.start(page, pageSize, query);
+        List<JSONObject> collect = nonManualProjectDataUpdateRecordDao.findList(query).stream().map(record -> {
+            JSONObject json = new JSONObject();
+            json.put("lastCount", record.getLastCount());
+            json.put("currentCount", record.getCurrentCount());
+            json.put("cTime", record.getCTime().getTime());
+            json.put("submitter", record.getSubmitter());
+            return json;
+        }).collect(Collectors.toList());
+        return PageVO.createPageVo(total, page, pageSize, collect);
+    }
+
+    @Override
+    public void adjustNonManualProjectDataBalance(String projectId, Integer currentCount) {
+        if (null == currentCount) {
+            return;
+        }
+        Update update = Update.update("dataBalance", currentCount);
+        Project project = projectDao.findOneById(projectId);
+        projectDao.updateOneByIdWithField(projectId, update);
+        NonManualProjectDataUpdateRecord record = new NonManualProjectDataUpdateRecord(projectId, project.getDataBalance(), currentCount, UserThreadLocal.getNickname());
+        nonManualProjectDataUpdateRecordDao.insertOne(record);
+    }
+
+    @Override
+    public JSONObject nonManualProjectOverview(String projectId) {
+        JSONObject json = new JSONObject();
+        json.put("balance", projectDao.findOneById(projectId).getDataBalance());
+        json.put("trendList", nonManualProjectDataUpdateRecordDao.aggregateDayLastRecord(projectId));
+        return json;
+    }
+
     /**
      * 获取舆情对应项目的情感标签
      * @param brandName
@@ -512,6 +622,7 @@ public class ProjectServiceImpl implements ProjectService {
             json.put("group", project.getBrandLinkedGroup());
             json.put("avatarUrl", project.getAvatarUrl());
             json.put("permission", getPermissionList(project));
+            json.put("isManual", project.isManual());
             Date expiredTime = roleExpired.get(project.getId());
             if (null == expiredTime || expiredTime.after(new Date())) {
                 json.put("expired", false);
@@ -549,7 +660,7 @@ public class ProjectServiceImpl implements ProjectService {
     private List<Map<String, Object>> getPermissionList(Project project) {
         List<Map<String, Object>> permissionList = new ArrayList<>();
         Map<String, Object> originPermission = new HashMap<>();
-        originPermission.put("origin", project.getModuleShowList().contains(2));
+        originPermission.put("origin", Objects.nonNull(project.getModuleShowList()) && project.getModuleShowList().contains(2));
         permissionList.add(originPermission);
         return permissionList;
     }

src/main/java/com/zhiwei/brandkbs2/util/RedisUtil.java

@@ -78,6 +78,10 @@ public class RedisUtil {
         return RedisKeyPrefix.TOOLSET_ARTICLE_SUMMARY_LIMIT + projectId;
     }
 
+    public static String getNonManualProjectMarkMaxGidKey(String projectId, String planId, String userId){
+        return RedisKeyPrefix.NON_MANUAL_PROJECT_MARK_MAX_GID + Tools.concat(projectId, planId, userId);
+    }
+
     public void setExpire(String key, String value, long timeout, TimeUnit unit) {
         stringRedisTemplate.opsForValue().set(key, value, timeout, unit);
     }