Merge branch 'feature' into 'dev'

Feature

See merge request !425

pom.xml

@@ -29,7 +29,7 @@
         <fastjson.version>1.2.47</fastjson.version>
         <easyexcel.version>3.0.5</easyexcel.version>
         <marker.client.version>1.1.9-SNAPSHOT</marker.client.version>
-        <base-objects.version>2.1.4-SNAPSHOT</base-objects.version>
+        <base-objects.version>2.1.6.1-SNAPSHOT</base-objects.version>
         <es.version>7.9.2</es.version>
         <dozer.version>5.5.1</dozer.version>
         <dubbo.version>2.7.4.1</dubbo.version>
@@ -204,7 +204,7 @@
         <!-- 基础包 -->
         <dependency>
             <groupId>com.zhiwei.base</groupId>
-            <artifactId>base-objects</artifactId>
+            <artifactId>base-objects-application</artifactId>
             <version>${base-objects.version}</version>
         </dependency>
         <!-- es -->

src/main/java/com/zhiwei/brandkbs2/aop/AopDownloadTask.java

@@ -56,13 +56,18 @@ public class AopDownloadTask {
             downloadTaskService.updateDownloadTask(taskId, 100, DownloadTask.Status.FAILED.getName(), null);
             ExceptionCast.cast(CommonCodeEnum.FAIL, "下载异常", e);
         }
+        ResponseResult responseResult = (ResponseResult) proceed;
         // 更新下载任务
-        if (method.getAnnotation(com.zhiwei.brandkbs2.aop.DownloadTask.class).entity()){
-            fileAddress = JSONObject.parseObject(((ResponseResult) proceed).getData().toString()).getString("filePath");
+        if (responseResult.isStatus()) {
+            if (method.getAnnotation(com.zhiwei.brandkbs2.aop.DownloadTask.class).entity()) {
+                fileAddress = JSONObject.parseObject(responseResult.getData().toString()).getString("filePath");
+            } else {
+                fileAddress = ((ResponseResult) proceed).getData().toString();
+            }
+            downloadTaskService.updateDownloadTask(taskId, 100, DownloadTask.Status.FINISH.getName(), fileAddress);
         }else {
-            fileAddress = ((ResponseResult) proceed).getData().toString();
+            downloadTaskService.updateDownloadTask(taskId, 100, DownloadTask.Status.FAILED.getName(), null);
         }
-        downloadTaskService.updateDownloadTask(taskId, 100, DownloadTask.Status.FINISH.getName(), fileAddress);
         return proceed;
     }
 }

src/main/java/com/zhiwei/brandkbs2/auth/AuthAspect.java

@@ -88,7 +88,8 @@ public class AuthAspect {
                     }
                     UserThreadLocal.set(userInfo);
                 }
-            } catch (Exception ignore) {
+            } catch (Exception e) {
+                log.info("around-通过token设置userInfo出错,", e);
             } finally {
                 proceed = joinPoint.proceed();
                 UserThreadLocal.clear();

src/main/java/com/zhiwei/brandkbs2/common/CommonConfig.java

@@ -50,7 +50,7 @@ public class CommonConfig {
     public RestTemplate restTemplate() throws Exception {
         final OkHttpClient client = new OkHttpClient().newBuilder()
                 .connectTimeout(120, TimeUnit.SECONDS)
-                .readTimeout(30, TimeUnit.SECONDS)
+                .readTimeout(120, TimeUnit.SECONDS)
                 // 忽略SSL证书验证
                 .sslSocketFactory(createSslSocketFactory(), new SkipX509TrustManager())
                 .hostnameVerifier((s, sslSession) -> true)

src/main/java/com/zhiwei/brandkbs2/common/GenericAttribute.java

@@ -15,8 +15,8 @@ public class GenericAttribute {
      * es index
      */
     public static final String ES_INDEX_PRE = "brandkbs2_";
-//  public static final String ES_INDEX_TEST = "brandkbs2_test";
-    public static final String ES_INDEX_TEST = "brandkbs2_2022";
+  public static final String ES_INDEX_TEST = "brandkbs2_test";
+//    public static final String ES_INDEX_TEST = "brandkbs2_2022";
     public static final String ES_CHANNEL_INDEX_TEST = "brandkbs2_channel_record_test";
     public static final String ES_CHANNEL_INDEX_PREFIX = "brandkbs2_channel_record_";
     /**
@@ -145,6 +145,7 @@ public class GenericAttribute {
     public static final String EXPORT_AMOUNT = "exportAmount";
     public static final String EXPIRED_TIME = "expiredTime";
     public static final String PHONE_NUMBER = "phoneNumber";
+    public static final String IS_MANUAL = "isManual";
 
 //    public enum ChannelParam{
 //        负面稿件数("negativeArticles"),

src/main/java/com/zhiwei/brandkbs2/common/RedisKeyPrefix.java

@@ -100,6 +100,8 @@ public class RedisKeyPrefix {
      */
     public static final String TOOLSET_ARTICLE_SUMMARY_LIMIT = "BRANDKBS:TOOLSET:ARTICLE_SUMMARY:LIMIT:";
 
+    public static final String NON_MANUAL_PROJECT_MARK_MAX_GID = "BRANDKBS:NON_MANUAL:PROJECT:MARK:MAX_GID:";
+
     public static String projectWarnHotTopKeyAll(String projectId, String type) {
         return RedisKeyPrefix.generateRedisKey(RedisKeyPrefix.PROJECT_WARN_HOT_TOP, projectId, Tools.concat(type, "*"));
     }

src/main/java/com/zhiwei/brandkbs2/config/Constant.java

@@ -13,6 +13,7 @@ import java.text.ParseException;
 public class Constant {
 
     public static final Long EIGHT_HOUR = 8 * 60 * 60 * 1000L;
+    public static final Long ONE_HOUR = 1 * 60 * 60 * 1000L;
     public static final Long ONE_DAY = 24 * 60 * 60 * 1000L;
     public static final Long ONE_WEEK = 7 * 24 * 60 * 60 * 1000L;
     public static final Long ONE_MONTH = 30 * 24 * 60 * 60 * 1000L;

src/main/java/com/zhiwei/brandkbs2/controller/admin/ArticleController.java

@@ -43,6 +43,7 @@ import java.util.Map;
 @RequestMapping("/admin/article")
 @Api(tags = "稿件上传", description = "提供稿件的上传功能")
 @Auth(role = RoleEnum.COMMON_ADMIN)
+@Deprecated
 public class ArticleController extends BaseController {
     public static final Logger log = LogManager.getLogger(ArticleController.class);
 

src/main/java/com/zhiwei/brandkbs2/controller/admin/ProjectController.java

@@ -262,4 +262,50 @@ public class ProjectController extends BaseController {
         noticeInfoService.deleteNoticeInfo(id);
         return ResponseResult.success();
     }
+
+    @ApiOperation("添加非人工项目")
+    @PostMapping("/non-manual/add")
+    public ResponseResult addNonManualProject(@RequestBody ProjectVO projectVO) {
+        return ProjectService.addNonManualProject(projectVO);
+    }
+
+    @ApiOperation("修改非人工项目")
+    @PutMapping("/non-manual/update")
+    public ResponseResult updateNonManualProject(@RequestBody ProjectVO projectVO) {
+        return ProjectService.updateNonManualProject(projectVO);
+    }
+
+    @ApiOperation("查询所有非人工项目")
+    @ApiImplicitParams({@ApiImplicitParam(name = "page", value = "页码", required = false, defaultValue = "1", paramType = "query", dataType = "int"),
+                        @ApiImplicitParam(name = "size", value = "每页记录数", required = false, defaultValue = "10", paramType = "query", dataType = "int"),
+                        @ApiImplicitParam(name = "sorter", value = "排序", required = false, defaultValue = "{\"cTime\":\"descend\"}", paramType = "query", dataType = "string"),
+                        @ApiImplicitParam(name = "keyword", value = "搜索关键字", required = false, defaultValue = "", paramType = "query", dataType = "string")})
+    @GetMapping("/non-manual/list")
+    public ResponseResult findNonManualProjectList(@RequestParam(value = "page", defaultValue = "1") int page,
+                                                   @RequestParam(value = "size", defaultValue = "10") int size,
+                                                   @RequestParam(value = "sorter", defaultValue = "{\"cTime\":\"descend\"}") String sorter,
+                                                   @RequestParam(value = "keyword", defaultValue = "") String keyword) {
+        return ResponseResult.success(ProjectService.findNonManualProjectList(page, size, sorter, keyword));
+    }
+
+    @ApiOperation("非人工项目-维护记录列表")
+    @GetMapping("/non-manual/maintain/list")
+    public ResponseResult findNonManualProjectMaintainList(@RequestParam(value = "page", defaultValue = "1") int page,
+                                                           @RequestParam(value = "size", defaultValue = "10") int size,
+                                                           @RequestParam(value = "projectId") String projectId) {
+        return ResponseResult.success(ProjectService.findNonManualProjectMaintainList(projectId, page, size));
+    }
+
+    @ApiOperation("非人工项目-维护记录数据总览")
+    @GetMapping("/non-manual/maintain/overview")
+    public ResponseResult interactionOverview(@RequestParam(value = "projectId") String projectId) {
+        return ResponseResult.success(ProjectService.nonManualProjectOverview(projectId));
+    }
+
+    @ApiOperation("非人工项目-维护记录-剩余数据量调整")
+    @PutMapping("/non-manual/maintain/adjust")
+    public ResponseResult adjustInteractionRecord(@ApiParam(name = "json:{count:调整数值, projectId:项目id)") @RequestBody JSONObject json) {
+        ProjectService.adjustNonManualProjectDataBalance(json.getString("projectId"), json.getInteger("count"));
+        return ResponseResult.success();
+    }
 }

src/main/java/com/zhiwei/brandkbs2/controller/app/AppArticleController.java

@@ -9,17 +9,11 @@ import com.zhiwei.brandkbs2.controller.BaseController;
 import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
 import com.zhiwei.brandkbs2.enmus.RoleEnum;
 import com.zhiwei.brandkbs2.model.ResponseResult;
-import com.zhiwei.brandkbs2.pojo.dto.ExportAppYuqingDTO;
-import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
-import com.zhiwei.brandkbs2.pojo.dto.ReportDTO;
-import com.zhiwei.brandkbs2.pojo.dto.ReportSearchDTO;
+import com.zhiwei.brandkbs2.pojo.dto.*;
 import com.zhiwei.brandkbs2.service.*;
 import com.zhiwei.brandkbs2.util.Tools;
 import com.zhiwei.middleware.mark.vo.MarkerTag;
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiImplicitParam;
-import io.swagger.annotations.ApiImplicitParams;
-import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.*;
 import org.apache.commons.lang3.tuple.Pair;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
@@ -242,7 +236,7 @@ public class AppArticleController extends BaseController {
                                          @RequestParam(value = "endTime", required = false) Long endTime,
                                          @RequestParam(value = "page", defaultValue = "1") Integer page,
                                          @RequestParam(value = "pageSize", defaultValue = "10")Integer pageSize,
-                                                 @RequestParam(value = "keyword", required = false) String keyword) {
+                                         @RequestParam(value = "keyword", required = false) String keyword) {
         String linkedGroupId = projectService.getProjectById(UserThreadLocal.getProjectId()).getBrandLinkedGroupId();
         ResponseEntity<JSONObject> responseEntity = restTemplate.getForEntity(briefReportUrl + "?startTime={1}&endTime={2}&page={3}&pageSize={4}&projectId={5" +
                 "}&keyword={6}", JSONObject.class, startTime, endTime, page, pageSize, linkedGroupId, keyword);
@@ -321,6 +315,132 @@ public class AppArticleController extends BaseController {
         return ResponseResult.success(markDataService.interactionUpdatePrediction(markSearchDTO));
     }
 
+    @ApiOperation("非人工项目-方案设置-新建方案")
+    @PostMapping("/non-manual/plan/add")
+    public ResponseResult addNonManualProjectPlan(@RequestBody NonManualProjectPlanDTO dto){
+        return markDataService.addNonManualProjectPlan(dto);
+    }
+
+    @ApiOperation("非人工项目-方案设置-删除方案")
+    @DeleteMapping("/non-manual/plan/delete/{id}")
+    public ResponseResult deleteNonManualProjectPlan(@PathVariable String id) {
+        markDataService.deleteNonManualProjectPlan(id);
+        return ResponseResult.success();
+    }
+
+    @ApiOperation("非人工项目-方案设置-获取方案详情")
+    @GetMapping("/non-manual/plan/{id}")
+    public ResponseResult getNonManualProjectPlanInfo(@PathVariable String id) {
+        return ResponseResult.success(markDataService.getNonManualProjectPlanInfo(id));
+    }
+
+    @ApiOperation("非人工项目-方案设置-编辑方案")
+    @PostMapping("/non-manual/plan/update")
+    public ResponseResult updateNonManualProjectPlan(@RequestBody NonManualProjectPlanDTO dto){
+        return markDataService.updateNonManualProjectPlan(dto);
+    }
+
+    @ApiOperation("非人工项目-方案设置-数据量预估")
+    @PutMapping("/non-manual/plan/prediction")
+    public ResponseResult getPlanDataPrediction(@ApiParam(name = "json:{keyword:关键词, filterWord:去噪词)") @RequestBody JSONObject json){
+        return ResponseResult.success(markDataService.getPlanDataPrediction(json.getString("keyword"), json.getString("filterWord")));
+    }
+
+    @ApiOperation("非人工项目-方案设置-方案列表")
+    @GetMapping("/non-manual/plan/list")
+    public ResponseResult getNonManualProjectPlanList(){
+        return ResponseResult.success(markDataService.getNonManualProjectPlanList());
+    }
+
+    @ApiOperation("非人工项目-方案设置-昨日，今日数据消耗量")
+    @GetMapping("/non-manual/plan/count")
+    public ResponseResult getProjectPlanConsume(){
+        return ResponseResult.success(markDataService.getProjectPlanConsume());
+    }
+
+    @ApiOperation("非人工项目-方案设置-获取总数据消耗图谱")
+    @GetMapping("/non-manual/plan/total-graph")
+    public ResponseResult getTotalDataConsumeGraph(@RequestParam(value = "startTime") Long startTime,
+                                                   @RequestParam(value = "endTime") Long endTime){
+        return ResponseResult.success(markDataService.getTotalDataConsumeGraph(startTime, endTime));
+    }
+
+    @ApiOperation("非人工项目-方案设置-获取时间范围数据消耗图谱")
+    @GetMapping("/non-manual/plan/time-graph")
+    public ResponseResult getTimeDataConsumeGraph(@RequestParam(value = "planId") String planId,
+                                                  @RequestParam(value = "startTime") Long startTime,
+                                                  @RequestParam(value = "endTime") Long endTime){
+        return ResponseResult.success(markDataService.getTimeDataConsumeGraph(planId, startTime, endTime));
+    }
+
+    @ApiOperation("非人工项目-舆情数据-舆情数据筛选条件")
+    @GetMapping("/non-manual/mark/criteria")
+    public ResponseResult getNonManualMarkCriteria(){
+        return ResponseResult.success(markDataService.getNonManualMarkCriteria());
+    }
+
+    @ApiOperation("非人工项目-舆情数据-舆情数据列表")
+    @PostMapping("/non-manual/mark/list")
+    public ResponseResult getNonManualMarkList(@RequestBody MarkSearchDTO markSearchDTO){
+        return ResponseResult.success(markDataService.getNonManualMarkList(markSearchDTO));
+    }
+
+    @ApiOperation("非人工项目-舆情数据-单条纠错")
+    @PostMapping("/non-manual/mark/tag")
+    public ResponseResult updateNonManualMarkTag(@RequestBody JSONObject json) {
+        String zip = json.getString("zip");
+        String linkedGroupId = json.getString("linkedGroupId");
+        StringBuilder mtag = new StringBuilder(json.getString("mtag"));
+        // 带上项目配置的必要标注标签
+        if (checkMTagIllegal(mtag)){
+            return ResponseResult.failure("标注标签出现同组标签，请勿连点标注");
+        }
+        markDataService.addMtag(zip, linkedGroupId, mtag.toString());
+        return ResponseResult.success();
+    }
+
+    @ApiOperation("非人工项目-舆情数据-互动量更新")
+    @PostMapping("/non-manual/mark/interaction-update")
+    @LogRecord(description = "非人工项目舆情库-有效舆情互动量更新")
+    @DownloadTask(taskName = "舆情库互动量更新结果下载", description = "舆情库互动量更新结果", entity = true)
+    public ResponseResult nonManualMarkInteractionUpdate(@RequestBody MarkSearchDTO markSearchDTO){
+        return markDataService.nonManualMarkInteractionUpdate(markSearchDTO);
+    }
+
+    @ApiOperation("非人工项目-舆情库互动量更新-数据量预估")
+    @PostMapping("/non-manual/mark/interaction-prediction")
+    public ResponseResult nonManualInteractionUpdatePrediction(@RequestBody MarkSearchDTO markSearchDTO){
+        return ResponseResult.success(markDataService.nonManualInteractionUpdatePrediction(markSearchDTO));
+    }
+
+    @ApiOperation("非人工项目-舆情数据-统计各方案数据量")
+    @GetMapping("/non-manual/mark/count")
+    public ResponseResult countNonManualMarkData(){
+        return ResponseResult.success(markDataService.getNonManualMarkCountList());
+    }
+
+    @ApiOperation("非人工项目-舆情数据-记录gid")
+    @PutMapping("/non-manual/mark/gid")
+    public ResponseResult adjustMaxGid(@ApiParam(name = "json:{planId:方案id, maxGid:值)") @RequestBody JSONObject json) {
+        markDataService.adjustMaxGid(json.getString("planId"), json.getLong("maxGid"));
+        return ResponseResult.success();
+    }
+
+    @ApiOperation("非人工项目-舆情数据-生成聚合")
+    @PostMapping("/non-manual/mark/aggree")
+    public ResponseResult generateNonManualMarkAggreeList(@ApiParam(name = "info:{planId:方案id, startTime:开始时间, endTime:结束时间)") @RequestBody JSONObject info) {
+        String planId = info.getString("planId");
+        Long startTime = info.getLong("startTime");
+        Long endTime = info.getLong("endTime");
+        return ResponseResult.success(markDataService.generateNonManualMarkAggreeList(planId, startTime, endTime));
+    }
+
+    @ApiOperation("非人工项目-舆情数据-获取聚合结果")
+    @PostMapping("/non-manual/mark/aggree/list")
+    public ResponseResult getNonManualMarkAggreeList(@RequestBody MarkSearchDTO markSearchDTO) {
+        return ResponseResult.success(markDataService.getNonManualMarkAggreeList(markSearchDTO));
+    }
+
     private boolean checkMTagIllegal(StringBuilder mtag) {
         List<MarkerTag> hitTags = projectService.getProjectById(UserThreadLocal.getProjectId()).getHitTags();
         if (!Tools.isEmpty(hitTags)) {

src/main/java/com/zhiwei/brandkbs2/controller/app/AppDownloadController.java

@@ -336,6 +336,19 @@ public class AppDownloadController extends BaseController {
         return ResponseResult.success(filePath);
     }
 
+    @ApiOperation("非人工项目-舆情库-舆情数据下载")
+    @PostMapping(value = "/yuqing/non-manual/mark")
+    @LogRecord(description = "非人工项目-舆情库-舆情导出", values = {"startTime", "endTime", "customTags", "field", "keyword", "politicsLevel", "mainBodyType", "platforms", "region", "tags"}, entity = true, arguments = true)
+    @DownloadTask(taskName = "舆情库舆情数据下载", description = "舆情库舆情数据")
+    public ResponseResult downloadNonManualMarkList(@RequestBody MarkSearchDTO markSearchDTO) {
+        Pair<String, List<ExportAppYuqingDTO>> stringListPair = markDataService.downloadNonManualMarkList(markSearchDTO);
+        // excel写入至指定路径
+        String projectName = projectService.getProjectById(UserThreadLocal.getProjectId()).getProjectName();
+        String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), stringListPair.getLeft() + "_舆情列表数据");
+        EasyExcelUtil.write(filePath, "sheet1", ExportAppYuqingDTO.class, stringListPair.getRight());
+        return ResponseResult.success(filePath);
+    }
+
     @ApiOperation("渠道库文章列表")
     @ApiImplicitParams({
             @ApiImplicitParam(name = "startTime", value = "开始时间", required = true, paramType = "query", dataType = "long"),

src/main/java/com/zhiwei/brandkbs2/controller/app/AppSearchController.java

@@ -11,9 +11,11 @@ import com.zhiwei.brandkbs2.controller.BaseController;
 import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
 import com.zhiwei.brandkbs2.enmus.RoleEnum;
 import com.zhiwei.brandkbs2.model.ResponseResult;
+import com.zhiwei.brandkbs2.pojo.MarkFlowEntity;
 import com.zhiwei.brandkbs2.pojo.WholeSearchRecord;
 import com.zhiwei.brandkbs2.pojo.dto.*;
 import com.zhiwei.brandkbs2.pojo.vo.ChannelListVO;
+import com.zhiwei.brandkbs2.pojo.vo.PageVO;
 import com.zhiwei.brandkbs2.service.*;
 import com.zhiwei.brandkbs2.util.Tools;
 import com.zhiwei.middleware.event.pojo.dto.BrandkbsEventSearchDTO;
@@ -199,14 +201,28 @@ public class AppSearchController extends BaseController {
     @LogRecord(values = "keyword", description = "查舆情", arguments = true, entity = true)
     @PostMapping("/mark/list")
     public ResponseResult getYuqingMarkList(@RequestBody MarkSearchDTO markSearchDTO) {
-        return ResponseResult.success(markDataService.getYuqingMarkList(markSearchDTO));
+        PageVO<MarkFlowEntity> yuqingMarkList = markDataService.getYuqingMarkList(markSearchDTO);
+        // 仅第一页增加平台进量（声量）统计
+        if (1 == markSearchDTO.getPage()) {
+            JSONObject info = yuqingMarkList.getInfo();
+            info.put("counts", markDataService.countMarkPlatform(markSearchDTO));
+            yuqingMarkList.setInfo(info);
+        }
+        return ResponseResult.success(yuqingMarkList);
     }
 
     @ApiOperation("原始数据列表")
     @LogRecord(values = "keyword", description = "查原始数据", arguments = true, entity = true)
     @PostMapping("/origin/list")
     public ResponseResult getOriginList(@RequestBody MarkSearchDTO markSearchDTO) {
-        return ResponseResult.success(markDataService.getOriginList(markSearchDTO));
+        PageVO<MarkFlowEntity> originList = markDataService.getOriginList(markSearchDTO);
+        // 仅第一页增加平台进量（声量）统计
+        if (1 == markSearchDTO.getPage()) {
+            JSONObject info = originList.getInfo();
+            info.put("counts", markDataService.countOriginPlatform(markSearchDTO));
+            originList.setInfo(info);
+        }
+        return ResponseResult.success(originList);
     }
 
     @ApiOperation("搜索-舆情列表-搜索条件")

src/main/java/com/zhiwei/brandkbs2/controller/app/AppToolsetController.java

@@ -138,4 +138,10 @@ public class AppToolsetController {
     public ResponseResult getIfUnread(){
         return ResponseResult.success(toolsetService.getIfUnread());
     }
+
+    @ApiOperation("工具库-获取项目工具库功能显示列表")
+    @GetMapping("/show-list")
+    public ResponseResult getProjectToolsetShowList(){
+        return ResponseResult.success(toolsetService.getProjectToolsetShowList());
+    }
 }

src/main/java/com/zhiwei/brandkbs2/dao/BytedanceCustomEventUpdateTaskDao.java

@@ -9,7 +9,7 @@ import org.springframework.data.mongodb.core.query.Query;
  * @author: cjz
  * @date: 2023-10-17 09:30
  */
-public interface BytedanceCustomEventUpdateTaskDao  extends BaseMongoDao<BytedanceCustomEventUpdateTask>{
+public interface BytedanceCustomEventUpdateTaskDao extends BaseMongoDao<BytedanceCustomEventUpdateTask>{
 
     BytedanceCustomEventUpdateTask findOne(Query query);
 }

src/main/java/com/zhiwei/brandkbs2/dao/BytedanceCustomEventUpdateTaskDataDao.java

+package com.zhiwei.brandkbs2.dao;
+
+import com.zhiwei.brandkbs2.pojo.BytedanceCustomEventUpdateTaskData;
+
+/**
+ * @ClassName: BytedanceCustomEventUpdateTaskDataDao
+ * @Description BytedanceCustomEventUpdateTaskDataDao
+ * @author: cjz
+ * @date: 2023-10-24 13:30
+ */
+public interface BytedanceCustomEventUpdateTaskDataDao extends BaseMongoDao<BytedanceCustomEventUpdateTaskData> {
+}

src/main/java/com/zhiwei/brandkbs2/dao/NonManualProjectDataUpdateRecordDao.java

+package com.zhiwei.brandkbs2.dao;
+
+import com.zhiwei.brandkbs2.pojo.NonManualProjectDataUpdateRecord;
+import com.zhiwei.brandkbs2.pojo.vo.LineVO;
+
+import java.util.List;
+
+/**
+ * @ClassName: BytedanceCustomEventUpdateTaskDataDao
+ * @Description BytedanceCustomEventUpdateTaskDataDao
+ * @author: cjz
+ * @date: 2023-11-10 16:30
+ */
+public interface NonManualProjectDataUpdateRecordDao extends BaseMongoDao<NonManualProjectDataUpdateRecord>{
+    /**
+     * 数据总览
+     * @return list
+     */
+    List<LineVO> aggregateDayLastRecord(String projectId);
+}

src/main/java/com/zhiwei/brandkbs2/dao/NonManualProjectPlanDao.java

+package com.zhiwei.brandkbs2.dao;
+
+import com.zhiwei.brandkbs2.pojo.NonManualProjectPlan;
+
+/**
+ * @ClassName: NonManualProjectPlanDao
+ * @Description NonManualProjectPlanDao
+ * @author: cjz
+ * @date: 2023-11-16 10:30
+ */
+public interface NonManualProjectPlanDao extends BaseMongoDao<NonManualProjectPlan> {
+}

src/main/java/com/zhiwei/brandkbs2/dao/impl/BytedanceCustomEventUpdateTaskDataDaoImpl.java

+package com.zhiwei.brandkbs2.dao.impl;
+
+import com.zhiwei.brandkbs2.dao.BytedanceCustomEventUpdateTaskDataDao;
+import com.zhiwei.brandkbs2.pojo.BytedanceCustomEventUpdateTaskData;
+import org.springframework.stereotype.Component;
+
+/**
+ * @ClassName: BytedanceCustomEventUpdateTaskDataDaoImpl
+ * @Description BytedanceCustomEventUpdateTaskDataDaoImpl
+ * @author: cjz
+ * @date: 2023-10-24 13:31
+ */
+@Component("bytedanceCustomEventUpdateTaskDataDao")
+public class BytedanceCustomEventUpdateTaskDataDaoImpl extends BaseMongoDaoImpl<BytedanceCustomEventUpdateTaskData> implements BytedanceCustomEventUpdateTaskDataDao {
+
+    private static final String COLLECTION_NAME = "brandkbs_bytedance_custom_event_task_data";
+
+    public BytedanceCustomEventUpdateTaskDataDaoImpl() {
+        super(COLLECTION_NAME);
+    }
+}

src/main/java/com/zhiwei/brandkbs2/dao/impl/NonManualProjectDataUpdateRecordDaoImpl.java

+package com.zhiwei.brandkbs2.dao.impl;
+
+import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.brandkbs2.dao.NonManualProjectDataUpdateRecordDao;
+import com.zhiwei.brandkbs2.pojo.NonManualProjectDataUpdateRecord;
+import com.zhiwei.brandkbs2.pojo.vo.LineVO;
+import org.springframework.data.domain.Sort;
+import org.springframework.data.mongodb.core.aggregation.Aggregation;
+import org.springframework.data.mongodb.core.aggregation.AggregationOperation;
+import org.springframework.data.mongodb.core.aggregation.AggregationResults;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.stereotype.Component;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @ClassName: BytedanceCustomEventUpdateTaskDataDaoImpl
+ * @Description BytedanceCustomEventUpdateTaskDataDaoImpl
+ * @author: cjz
+ * @date: 2023-11-10 16:31
+ */
+@Component("nonManualProjectDataUpdateRecordDao")
+public class NonManualProjectDataUpdateRecordDaoImpl extends BaseMongoDaoImpl<NonManualProjectDataUpdateRecord> implements NonManualProjectDataUpdateRecordDao {
+    private static final String COLLECTION_NAME = "brandkbs_project_data_update_record";
+
+    public NonManualProjectDataUpdateRecordDaoImpl() {
+        super(COLLECTION_NAME);
+    }
+
+    @Override
+    public List<LineVO> aggregateDayLastRecord(String projectId) {
+        List<LineVO> res = new ArrayList<>();
+        Criteria criteria = Criteria.where("projectId").is(projectId);
+        List<AggregationOperation> operations = new ArrayList<>();
+        // operations区分先后顺序
+        operations.add(Aggregation.match(criteria));
+        operations.add(Aggregation.project("currentCount").and("cTime").substring(0, 10).as("cTime"));
+        operations.add(Aggregation.group("cTime").last("currentCount").as("count"));
+        operations.add(Aggregation.sort(Sort.by(Sort.Order.asc("_id"))));
+        Aggregation aggregation = Aggregation.newAggregation(operations);
+        // 优先小的年份
+        AggregationResults<JSONObject> aggregateResult = mongoTemplate.aggregate(aggregation, COLLECTION_NAME, JSONObject.class);
+        List<JSONObject> mappedResults = aggregateResult.getMappedResults();
+        for (JSONObject mappedResult : mappedResults) {
+            res.add(new LineVO(mappedResult.getLong("count"), mappedResult.getDate("_id").getTime()));
+        }
+        return res;
+    }
+}

src/main/java/com/zhiwei/brandkbs2/dao/impl/NonManualProjectPlanDaoImpl.java

+package com.zhiwei.brandkbs2.dao.impl;
+
+import com.zhiwei.brandkbs2.dao.NonManualProjectPlanDao;
+import com.zhiwei.brandkbs2.pojo.NonManualProjectPlan;
+import org.springframework.stereotype.Component;
+
+/**
+ * @ClassName: NonManualProjectPlanDaoImpl
+ * @Description NonManualProjectPlanDaoImpl
+ * @author: cjz
+ * @date: 2023-11-16 10:31
+ */
+@Component("nonManualProjectPlanDao")
+public class NonManualProjectPlanDaoImpl extends BaseMongoDaoImpl<NonManualProjectPlan> implements NonManualProjectPlanDao {
+
+    private static final String COLLECTION_NAME = "brandkbs_non_manual_project_plan";
+
+    public NonManualProjectPlanDaoImpl() {
+        super(COLLECTION_NAME);
+    }
+}

src/main/java/com/zhiwei/brandkbs2/easyexcel/dto/ExportInteractionUpdateDTO.java

@@ -37,16 +37,16 @@ public class ExportInteractionUpdateDTO {
     private Date time;
 
     @ExcelProperty("评论数")
-    private int commentCount;
+    private Integer commentCount;
 
     @ExcelProperty("点赞数")
-    private int likeCount;
+    private Integer likeCount;
 
     @ExcelProperty("转发数")
-    private int repostCount;
+    private Integer repostCount;
 
     @ExcelProperty("阅读数")
-    private int readCount;
+    private Integer readCount;
 
     public ExportInteractionUpdateDTO(String id, Map<Object, JSONObject> urlMap, String url){
         this.id = id;
@@ -54,27 +54,11 @@ public class ExportInteractionUpdateDTO {
         this.url = url;
         if (Objects.nonNull(urlMap.get(url))){
             JSONObject jsonObject = urlMap.get(url);
-            this.commentCount = jsonObject.getIntValue("commentCount");
-            this.likeCount = jsonObject.getIntValue("likeCount");
-            this.repostCount = jsonObject.getIntValue("repostCount");
-            this.readCount = jsonObject.getIntValue("readCount");
+            this.commentCount = Objects.nonNull(jsonObject.getInteger("commentCount")) ? jsonObject.getIntValue("commentCount") : 0;
+            this.likeCount = Objects.nonNull(jsonObject.getInteger("likeCount")) ? jsonObject.getIntValue("likeCount") : 0;
+            this.repostCount = Objects.nonNull(jsonObject.getInteger("repostCount")) ? jsonObject.getIntValue("repostCount") : 0;
+            this.readCount = Objects.nonNull(jsonObject.getInteger("readCount")) ? jsonObject.getIntValue("readCount") : 0;
         }
         this.time = new Date();
     }
-
-    public static ExportInteractionUpdateDTO createWithPlatform(String id, Map<String, String> urlPlatformMap, Map<Object, JSONObject> urlMap, String url){
-        ExportInteractionUpdateDTO dto = new ExportInteractionUpdateDTO();
-        dto.setId(id);
-        dto.setPlatform(urlPlatformMap.get(url));
-        dto.setUrl(url);
-        if (Objects.nonNull(urlMap.get(url))){
-            JSONObject jsonObject = urlMap.get(url);
-            dto.setCommentCount(jsonObject.getIntValue("commentCount"));
-            dto.setLikeCount(jsonObject.getIntValue("likeCount"));
-            dto.setRepostCount(jsonObject.getIntValue("repostCount"));
-            dto.setReadCount(jsonObject.getIntValue("readCount"));
-        }
-        dto.setTime(new Date());
-        return dto;
-    }
 }

src/main/java/com/zhiwei/brandkbs2/easyexcel/dto/ExportYuqingInteractionUpdateDTO.java

+package com.zhiwei.brandkbs2.easyexcel.dto;
+
+import com.alibaba.excel.annotation.ExcelProperty;
+import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.brandkbs2.pojo.BaseMap;
+import lombok.Data;
+import lombok.ToString;
+
+import java.util.Date;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * @author cjz
+ * @version 1.0
+ * @description 导出舆情库互动量更新结果数据实体类
+ * @date 2023年10月23日 15:44:08
+ */
+@Data
+@ToString
+public class ExportYuqingInteractionUpdateDTO {
+    @ExcelProperty("序号")
+    private String id;
+
+    @ExcelProperty("平台")
+    private String platform;
+
+    @ExcelProperty("渠道")
+    private String channel;
+
+    @ExcelProperty("标题")
+    private String title;
+
+    @ExcelProperty("文本")
+    private String content;
+
+    @ExcelProperty("发文时间")
+    private Date time;
+
+    @ExcelProperty("情感倾向")
+    private String emotion;
+
+    @ExcelProperty("地址")
+    private String url;
+
+    @ExcelProperty("更新时间")
+    private Date updateTime;
+
+    @ExcelProperty("评论数")
+    private Integer commentCount;
+
+    @ExcelProperty("点赞数")
+    private Integer likeCount;
+
+    @ExcelProperty("转发数")
+    private Integer repostCount;
+
+    @ExcelProperty("阅读数")
+    private Integer readCount;
+
+    @ExcelProperty("在看数")
+    private Integer shareCount;
+
+    @ExcelProperty("收藏数")
+    private Integer collectCount;
+
+    public ExportYuqingInteractionUpdateDTO(int id, Map<String, BaseMap> baseMap, Map<Object, JSONObject> resultMap, String url){
+        BaseMap baseMapEntity = baseMap.get(url);
+        this.id = String.valueOf(id);
+        this.platform = baseMapEntity.getPlatform();
+        this.channel = baseMapEntity.getSource();
+        this.title = baseMapEntity.getTitle();
+        this.content = baseMapEntity.getContent();
+        this.time = new Date(baseMapEntity.getTime());
+        this.emotion = baseMapEntity.getEmotion();
+        if (Objects.nonNull(resultMap.get(url))){
+            JSONObject jsonObject = resultMap.get(url);
+            this.commentCount = Objects.nonNull(jsonObject.getInteger("commentCount")) ? jsonObject.getIntValue("commentCount") : 0;
+            this.likeCount = Objects.nonNull(jsonObject.getInteger("likeCount")) ? jsonObject.getIntValue("likeCount") : 0;
+            this.repostCount = Objects.nonNull(jsonObject.getInteger("repostCount")) ? jsonObject.getIntValue("repostCount") : 0;
+            this.readCount = Objects.nonNull(jsonObject.getInteger("readCount")) ? jsonObject.getIntValue("readCount") : 0;
+            this.shareCount = Objects.nonNull(jsonObject.getInteger("shareCount")) ? jsonObject.getIntValue("shareCount") : 0;
+            this.collectCount = Objects.nonNull(jsonObject.getInteger("collectCount")) ? jsonObject.getIntValue("collectCount") : 0;
+        }
+        this.url = url;
+        this.updateTime = new Date();
+    }
+}

src/main/java/com/zhiwei/brandkbs2/easyexcel/listener/BytedanceEventListener.java

@@ -3,6 +3,8 @@ package com.zhiwei.brandkbs2.easyexcel.listener;
 import com.alibaba.excel.context.AnalysisContext;
 import com.alibaba.excel.event.AnalysisEventListener;
 import com.zhiwei.brandkbs2.easyexcel.dto.UploadBytedanceEventDTO;
+import com.zhiwei.brandkbs2.pojo.BytedanceCustomEventUpdateTaskData;
+import com.zhiwei.brandkbs2.util.Tools;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -17,22 +19,30 @@ import java.util.Objects;
  */
 public class BytedanceEventListener extends AnalysisEventListener<UploadBytedanceEventDTO> {
 
-    private Map<String, List<UploadBytedanceEventDTO>> map;
+    private String taskId;
 
+    private List<BytedanceCustomEventUpdateTaskData> data;
 
-    public BytedanceEventListener(Map<String, List<UploadBytedanceEventDTO>> map){
-        this.map = map;
+
+    public BytedanceEventListener(String taskId, List<BytedanceCustomEventUpdateTaskData> data){
+        this.taskId = taskId;
+        this.data = data;
     }
 
     @Override
-    public void invoke(UploadBytedanceEventDTO data, AnalysisContext context) {
-        if (map.containsKey(data.getEventName())){
-            map.get(data.getEventName()).add(data);
-        }else {
-            List<UploadBytedanceEventDTO> list = new ArrayList<>();
-            list.add(data);
-            map.put(data.getEventName(), list);
-        }
+    public void invoke(UploadBytedanceEventDTO dto, AnalysisContext context) {
+        BytedanceCustomEventUpdateTaskData taskData = new BytedanceCustomEventUpdateTaskData();
+        taskData.setTaskId(taskId);
+        taskData.setEventName(dto.getEventName());
+        taskData.setChannel(dto.getChannel());
+        taskData.setPlatform(dto.getPlatform());
+        taskData.setUrl(dto.getUrl());
+        taskData.setWeiboComment(dto.getWeiboComment());
+        taskData.setWeiboForward(dto.getWeiboForward());
+        taskData.setWeiboLike(dto.getWeiboLike());
+        taskData.setWechatRead(dto.getWechatRead());
+        taskData.setWechatReading(dto.getWechatReading());
+        data.add(taskData);
     }
 
     @Override

src/main/java/com/zhiwei/brandkbs2/es/EsQueryTools.java

@@ -104,8 +104,12 @@ public class EsQueryTools {
         return assembleCacheMapsQuery(projectId, contendId, politicsLevel, field, region, mainBodyType, null);
     }
 
-    public static BoolQueryBuilder assembleCacheMapsQuery(String projectId,String contendId, String politicsLevel, String field, String region,
-                                                          String mainBodyType,Integer channelEmotion) {
+    public static BoolQueryBuilder assembleCacheMapsQuery(String projectId, String planId, String politicsLevel) {
+        return assembleCacheMapsQuery(projectId, planId, politicsLevel, null, null, null, null);
+    }
+
+    public static BoolQueryBuilder assembleCacheMapsQuery(String projectId, String contendId, String politicsLevel, String field, String region,
+                                                          String mainBodyType, Integer channelEmotion) {
         BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
         String key = concat(projectId, contendId);
         BoolQueryBuilder nestedBoolBuilder = QueryBuilders.boolQuery();
@@ -121,6 +125,22 @@ public class EsQueryTools {
         return boolQueryBuilder;
     }
 
+    public static BoolQueryBuilder assembleCacheMapsPlanQuery(String projectId, String planId) {
+        return assembleCacheMapsPlanQuery(projectId, planId, null);
+    }
+
+    public static BoolQueryBuilder assembleCacheMapsPlanQuery(String projectId, String planId, String politicsLevel) {
+        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+        String key = concat(projectId, planId);
+        BoolQueryBuilder nestedBoolBuilder = QueryBuilders.boolQuery();
+        // 必要条件
+        nestedBoolBuilder.must(QueryBuilders.termQuery("brandkbs_cache_maps.plan_key.keyword", key));
+        boolQueryBuilder.must(cacheMapsNestedQuery(nestedBoolBuilder));
+        // sensitiveChannel
+        addSensitiveChannel(politicsLevel, null, null, null, boolQueryBuilder);
+        return boolQueryBuilder;
+    }
+
     public static BoolQueryBuilder assembleCacheMapsQueryWithProject(String projectId, BrandkbsYuQingConfig config) {
         BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
         BoolQueryBuilder nestedBoolBuilder = QueryBuilders.boolQuery();
@@ -254,6 +274,31 @@ public class EsQueryTools {
         return queryBuilder;
     }
 
+    public static BoolQueryBuilder assembleForward2Query(boolean isForward) {
+        BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
+        // 需要转发数据，平台必须为微博
+        if (isForward) {
+            BoolQueryBuilder mustWeiboCondition = QueryBuilders.boolQuery();
+            queryBuilder.should(mustWeiboCondition.must(QueryBuilders.termQuery("platform_id", "5d02236e6395002a7c380b79")));
+            queryBuilder.should(QueryBuilders.termQuery("is_forward", true));
+        }else {
+            queryBuilder.mustNot(QueryBuilders.termQuery("is_forward", true));
+        }
+        return queryBuilder;
+    }
+
+    /**
+     * c2查询
+     *
+     * @param dataType 数据类型（1:长文本, 2:短文本, 3:问答, 5:视频）
+     * @return
+     */
+    public static BoolQueryBuilder assembleC2Query(Integer dataType) {
+        BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
+        queryBuilder.should(QueryBuilders.termQuery("c2", dataType << 23));
+        return queryBuilder;
+    }
+
     /**
      * 字段不拆封,多字段 同关键词
      *
@@ -349,8 +394,7 @@ public class EsQueryTools {
 
     public static void addSensitiveChannel(String politicsLevel, String field, String region, String mainBodyType, BoolQueryBuilder boolQueryBuilder) {
         if (null != politicsLevel) {
-            boolQueryBuilder.must(QueryBuilders.termQuery(GenericAttribute.ES_SENSITIVE_CHANNEL + "." + GenericAttribute.ES_POLITICS_LEVEL + ".keyword",
-                    politicsLevel));
+            boolQueryBuilder.must(QueryBuilders.termQuery(GenericAttribute.ES_SENSITIVE_CHANNEL + "." + GenericAttribute.ES_POLITICS_LEVEL + ".keyword", politicsLevel));
         }
         if (null != mainBodyType) {
             boolQueryBuilder.must(QueryBuilders.matchPhraseQuery(GenericAttribute.ES_SENSITIVE_CHANNEL + "." + GenericAttribute.ES_MAIN_BODY_TYPE + ".keyword", mainBodyType));

src/main/java/com/zhiwei/brandkbs2/pojo/BytedanceCustomEventUpdateTask.java

@@ -42,11 +42,11 @@ public class BytedanceCustomEventUpdateTask extends AbstractBaseMongo{
     /**
      * 事件数据
      */
-    private Map<String, List<UploadBytedanceEventDTO>> eventData;
+//    private Map<String, List<UploadBytedanceEventDTO>> eventData;
     /**
      * 报错数据
      */
-    private List<UploadBytedanceEventDTO> errorData;
+    private List<BytedanceCustomEventUpdateTaskData> errorData;
     /**
      * 竞品影响力指数
      */
@@ -60,6 +60,14 @@ public class BytedanceCustomEventUpdateTask extends AbstractBaseMongo{
      */
     private Boolean cancel;
     /**
+     * 已处理的数据量
+     */
+    private int schedule;
+    /**
+     * 待处理的数据总量
+     */
+    private int totalSchedule;
+    /**
      * 项目id
      */
     private String projectId;
@@ -136,9 +144,11 @@ public class BytedanceCustomEventUpdateTask extends AbstractBaseMongo{
         task.setTaskStatus(TaskStatus.CALCULATING.getStatus());
         task.setProcessStatus(ProcessStatus.CHANNEL_MATCH.getStatus());
         task.setExtraCompute(extraCompute);
-        task.setEventInfo(null);
+        task.setEventInfo(new ArrayList<>());
         task.setErrorData(new ArrayList<>());
         task.setCancel(false);
+        task.setSchedule(0);
+        task.setTotalSchedule(0);
         task.setProjectId(UserThreadLocal.getProjectId());
         task.setUserId(UserThreadLocal.getUserId());
         task.setNickName(UserThreadLocal.getNickname());
@@ -157,6 +167,8 @@ public class BytedanceCustomEventUpdateTask extends AbstractBaseMongo{
         task.setEventInfo(null);
         task.setErrorData(null);
         task.setCancel(false);
+        task.setSchedule(0);
+        task.setTotalSchedule(0);
         task.setProjectId(UserThreadLocal.getProjectId());
         task.setUserId(UserThreadLocal.getUserId());
         task.setNickName(UserThreadLocal.getNickname());

src/main/java/com/zhiwei/brandkbs2/pojo/BytedanceCustomEventUpdateTaskData.java

+package com.zhiwei.brandkbs2.pojo;
+
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+/**
+ * @ClassName: BytedanceCustomEventInfluenceTask
+ * @Description 字节定制事件影响力计算rw
+ * @author: cjz
+ * @date: 2023-10-16 15:12
+ */
+@Getter
+@Setter
+@NoArgsConstructor
+public class BytedanceCustomEventUpdateTaskData extends AbstractBaseMongo {
+    private String taskId;
+
+    private String platform;
+
+    private String channel;
+
+    private String url;
+
+    private String eventName;
+
+    private Integer weiboForward;
+
+    private Integer weiboComment;
+
+    private Integer weiboLike;
+
+    private Integer wechatRead;
+
+    private Integer wechatReading;
+}

src/main/java/com/zhiwei/brandkbs2/pojo/BytedanceCustomPlatformWeight.java

@@ -21,4 +21,8 @@ public class BytedanceCustomPlatformWeight extends AbstractBaseMongo{
      * 权重
      */
     private double weight;
+    /**
+     * 常量
+     */
+    private double constant;
 }

src/main/java/com/zhiwei/brandkbs2/pojo/ChannelIndex.java

@@ -103,9 +103,14 @@ public class ChannelIndex extends AbstractBaseMongo {
             return res;
         }
         List<Map<String, Object>> cacheMaps = (List<Map<String, Object>>) sourceAsMap.get(GenericAttribute.ES_BRANDKBS_CACHE_MAPS);
+        List<String> manualProjectIds = GlobalPojo.PROJECT_MAP.entrySet().stream().filter(entry -> entry.getValue().isManual()).map(Map.Entry::getKey).collect(Collectors.toList());
         for (Map<String, Object> cacheMap : cacheMaps) {
             String projectId = String.valueOf(cacheMap.get("project_id"));
             String contendId = String.valueOf(cacheMap.get("contend_id"));
+            // 剔除非人工项目
+            if(manualProjectIds.contains(projectId)){
+                continue;
+            }
             ChannelIndex channelIndex = new ChannelIndex(projectId, contendId, messagePlatform.getName(), realSource, source);
             // 默认主品牌
             channelIndex.setContendId(String.valueOf(0));

src/main/java/com/zhiwei/brandkbs2/pojo/MarkFlowEntity.java

@@ -100,6 +100,11 @@ public class MarkFlowEntity implements Serializable {
             tJson.put(GenericAttribute.ES_CNAME, "采集来源缺失");
         }
         int c2 = tJson.getIntValue(GenericAttribute.ES_C2);
+        // 微博平台头像url拼接
+        String weiboHeadUrl = "https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/weibo-avatar-url?url=";
+        if (1020 == tJson.getIntValue(GenericAttribute.ES_C4) && !tJson.getString("avatar_url").contains(weiboHeadUrl)){
+            tJson.put("avatar_url", weiboHeadUrl + tJson.getString("avatar_url"));
+        }
         switch (ClassB.TypeB.fromEncode(c2)) {
             case COMPLETE:
                 if (!Tools.isEmpty(tJson.get("ind_full_text"))) {

src/main/java/com/zhiwei/brandkbs2/pojo/NonManualProjectDataUpdateRecord.java

+package com.zhiwei.brandkbs2.pojo;
+
+import lombok.Getter;
+import lombok.Setter;
+
+import java.util.Date;
+
+/**
+ * @ClassName: NonManualProjectDataUpdateRecord
+ * @Description 非人工项目剩余数据量调整记录
+ * @author: cjz
+ * @date: 2023-11-10 15:53
+ */
+@Getter
+@Setter
+public class NonManualProjectDataUpdateRecord extends AbstractBaseMongo {
+    /**
+     * 项目id
+     */
+    private String projectId;
+    /**
+     * 上一次记录数(本次调整前剩余次数)
+     */
+    private int lastCount;
+
+    /**
+     * 本次记录数(本次调整后剩余次数)
+     */
+    private int currentCount;
+
+    /**
+     * 提交人
+     */
+    private String submitter;
+
+    /**
+     * 创建时间(Date类型方便聚合查询)
+     */
+    private Date cTime;
+
+    public NonManualProjectDataUpdateRecord(String projectId, int lastCount, int currentCount, String submitter){
+        this.projectId = projectId;
+        this.lastCount = lastCount;
+        this.currentCount = currentCount;
+        this.submitter = submitter;
+        this.cTime = new Date();
+    }
+}

src/main/java/com/zhiwei/brandkbs2/pojo/NonManualProjectPlan.java

+package com.zhiwei.brandkbs2.pojo;
+
+import com.zhiwei.brandkbs2.auth.UserThreadLocal;
+import com.zhiwei.brandkbs2.pojo.dto.NonManualProjectPlanDTO;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+import java.util.List;
+
+/**
+ * @ClassName: NonManualProjectPlan
+ * @Description 非人工项目方案
+ * @author: cjz
+ * @date: 2023-11-15 14:53
+ */
+@Getter
+@Setter
+@NoArgsConstructor
+public class NonManualProjectPlan extends AbstractBaseMongo{
+    /**
+     * 项目id
+     */
+    private String projectId;
+    /**
+     * 方案名
+     */
+    private String name;
+    /**
+     * 关键词
+     */
+    private String keyword;
+    /**
+     * 去噪词
+     */
+    private String filterWord;
+    /**
+     * 方案类型 （永久/临时）
+     */
+    private Boolean lasting;
+    /**
+     * 方案结束时间
+     */
+    private Long endTime;
+    /**
+     * 数据范围 （原创，转发，ocr）
+     */
+    private List<String> type;
+    /**
+     * 创建时间
+     */
+    private Long cTime;
+    /**
+     * 修改时间
+     */
+    private Long uTime;
+    /**
+     * 提交人
+     */
+    private String submitter;
+
+    public NonManualProjectPlan(NonManualProjectPlanDTO dto){
+        this.projectId = UserThreadLocal.getProjectId();
+        this.name = dto.getName();
+        this.keyword = dto.getKeyword();
+        this.filterWord = dto.getFilterWord();
+        this.lasting = dto.getLasting();
+        this.endTime = dto.getEndTime();
+        this.type = dto.getType();
+        this.cTime = System.currentTimeMillis();
+        this.uTime = System.currentTimeMillis();
+        this.submitter = UserThreadLocal.getNickname();
+    }
+}

src/main/java/com/zhiwei/brandkbs2/pojo/Project.java

@@ -75,6 +75,11 @@ public class Project extends AbstractProject {
     private boolean isStart;
 
     /**
+     * 是否为人工项目
+     */
+    private boolean isManual;
+
+    /**
      * 创建时间
      */
     private Long cTime;
@@ -100,6 +105,11 @@ public class Project extends AbstractProject {
     private int interactionBalance;
 
     /**
+     * 剩余数据量（仅非人工项目）
+     */
+    private int dataBalance;
+
+    /**
      * 项目主品牌配置信息转换
      *
      * @return 项目对象
@@ -123,9 +133,7 @@ public class Project extends AbstractProject {
         projectVO.setContendList(this.getContendList());
         // 模块配置
         projectVO.setModuleShowList(this.getModuleShowList());
-        if (Objects.nonNull(this.getToolsetShowList())){
-            projectVO.setToolsetShowList(this.getToolsetShowList());
-        }
+        projectVO.setToolsetShowList(this.getToolsetShowList());
         projectVO.setChannelFileUrl(this.getChannelFileUrl());
         projectVO.setNegativeChannelParams(this.getNegativeChannelParams());
         projectVO.setPositiveChannelParams(this.getPositiveChannelParams());

src/main/java/com/zhiwei/brandkbs2/pojo/dto/MarkSearchDTO.java

@@ -160,4 +160,28 @@ public class MarkSearchDTO {
     @ApiModelProperty(value = "屏蔽词")
     private String shieldWord;
 
+    /**
+     * 方案id,仅非人工项目
+     */
+    @ApiModelProperty(value = "方案id")
+    private String planId;
+    /**
+     * 内容类型（原创/转发）,仅非人工项目
+     */
+    @ApiModelProperty(value = "内容类型（原创/转发）")
+    private String forward;
+    /**
+     * 信息浏览（已读/未读）,仅非人工项目
+     */
+    @ApiModelProperty(value = "信息浏览（已读/未读）")
+    private String read;
+    /**
+     * 数据类型（长文本/短文本/问答/视频）,仅非人工项目
+     */
+    @ApiModelProperty(value = "数据类型（1:长文本, 2:短文本, 3:问答, 5:视频）")
+    private Integer dataType;
+    /**
+     * gid用于未读已读筛选,仅非人工项目
+     */
+    private Long gid;
 }

src/main/java/com/zhiwei/brandkbs2/pojo/dto/NonManualProjectPlanDTO.java

+package com.zhiwei.brandkbs2.pojo.dto;
+
+import com.zhiwei.brandkbs2.pojo.NonManualProjectPlan;
+import io.swagger.annotations.ApiModelProperty;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+import java.util.List;
+
+/**
+ * @ClassName: NonManualProjectPlan
+ * @Description 非人工项目方案传输
+ * @author: cjz
+ * @date: 2023-11-16 9:53
+ */
+@Getter
+@Setter
+@AllArgsConstructor
+@NoArgsConstructor
+public class NonManualProjectPlanDTO {
+    /**
+     * 主键ID
+     */
+    @ApiModelProperty("项目主键ID")
+    private String id;
+    /**
+     * 方案名
+     */
+    private String name;
+    /**
+     * 关键词
+     */
+    private String keyword;
+    /**
+     * 去噪词
+     */
+    private String filterWord;
+    /**
+     * 方案类型 （永久/临时）
+     */
+    private Boolean lasting;
+    /**
+     * 方案结束时间
+     */
+    private Long endTime;
+    /**
+     * 数据范围 （原创，转发，ocr）
+     */
+    private List<String> type;
+
+    public NonManualProjectPlanDTO(NonManualProjectPlan plan){
+        this.id = plan.getId();
+        this.name = plan.getName();
+        this.keyword = plan.getKeyword();
+        this.filterWord = plan.getFilterWord();
+        this.lasting = plan.getLasting();
+        this.endTime = plan.getEndTime();
+        this.type = plan.getType();
+    }
+}

src/main/java/com/zhiwei/brandkbs2/pojo/vo/ProjectVO.java

@@ -9,12 +9,10 @@ import io.swagger.annotations.ApiModel;
 import io.swagger.annotations.ApiModelProperty;
 import lombok.Data;
 import lombok.ToString;
+import org.apache.commons.collections4.CollectionUtils;
 import org.bson.types.ObjectId;
 
-import java.util.Arrays;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.stream.Collectors;
 
 /**
@@ -100,7 +98,7 @@ public class ProjectVO {
      * 工具库功能入口展示列表（目前有：摘要提取，互动量更新，词云，字节影响力计算，其中前三者默认开启）
      */
     @ApiModelProperty("工具库功能入口展示列表（目前有：摘要提取，互动量更新，词云，字节影响力计算，其中前三者默认开启）")
-    private List<String> toolsetShowList = Arrays.asList("articleSummary", "interactionUpdate", "highWord");
+    private List<String> toolsetShowList;
     /**
      * 黑渠道对应组
      */
@@ -166,11 +164,38 @@ public class ProjectVO {
         project.setBlackChannelGroup(this.getBlackChannelGroup());
         project.setShow(true);
         project.setStart(false);
+        project.setManual(true);
         project.setCTime(time.getTime());
         project.setUTime(time.getTime());
         return project;
     }
 
+    public static Project createNonManualProject(ProjectVO vo){
+        Project project = new Project();
+        project.setId(vo.getId());
+        project.setProjectName(vo.getProjectName());
+        project.setBrandName(vo.getBrandName());
+        project.setBrandLinkedGroup(vo.getBrandLinkedGroup());
+        project.setBrandLinkedGroupId(vo.getBrandLinkedGroupId());
+        project.setImportTime(vo.getImportTime().getTime());
+        if (null == vo.getAvatarUrl()) {
+            project.setAvatarUrl(Constant.DEFAULT_PROJECT_URL);
+        } else {
+            project.setAvatarUrl(vo.getAvatarUrl());
+        }
+        project.setModuleShowList(Collections.emptyList());
+        project.setToolsetShowList(Arrays.asList("articleSummary", "interactionUpdate", "highWord"));
+        // 新建项目时默认数据量50w
+        project.setDataBalance(500000);
+        // 非人工项目
+        project.setManual(false);
+        project.setShow(true);
+        project.setStart(false);
+        project.setCTime(System.currentTimeMillis());
+        project.setUTime(System.currentTimeMillis());
+        return project;
+    }
+
 //    private static Map<String, Double> parse2NegativeChannelParmasMap(List<Double> negativeChannel) {
 //        if (4 != negativeChannel.size()) {
 //            ExceptionCast.cast(CommonCodeEnum.INVALID_PARAM.message("不友好渠道传参异常"));

src/main/java/com/zhiwei/brandkbs2/service/EsSearchService.java

 package com.zhiwei.brandkbs2.service;
 
+import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
 import org.apache.commons.lang3.tuple.Pair;
 import org.elasticsearch.search.SearchHits;
@@ -18,7 +19,7 @@ public interface EsSearchService {
     /**
      * 搜索标注数据
      *
-     * @param dto    搜素实体类
+     * @param dto    搜索实体类
      * @param scroll 滚动查询
      * @return 搜索结果
      */
@@ -27,7 +28,7 @@ public interface EsSearchService {
     /**
      * 搜索竞品标注数据
      *
-     * @param dto    搜素实体类
+     * @param dto    搜索实体类
      * @param scroll 滚动查询
      * @return 搜索结果
      */
@@ -36,10 +37,42 @@ public interface EsSearchService {
     /**
      * 搜索原始数据
      *
-     * @param dto    搜素实体类
+     * @param dto    搜索实体类
      * @param scroll 滚动查询
      * @return 搜索结果
      */
     Pair<SearchHits[], Map<String, Long>> searchOriginHitsAndCount(MarkSearchDTO dto, boolean scroll) throws IOException;
 
+    /**
+     * 搜索非人工项目数据
+     *
+     * @param dto    搜索实体类
+     * @param scroll 滚动查询
+     * @return 搜索结果
+     */
+    Pair<SearchHits[], Map<String, Long>> searchNonManualHitsAndCount(MarkSearchDTO dto, boolean scroll) throws IOException;
+
+    /**
+     * 查舆情舆情库-平台进量（声量）统计
+     * @param dto   搜索实体类
+     * @return
+     * @throws IOException
+     */
+    JSONObject countMarkPlatform(MarkSearchDTO dto) throws IOException;
+
+    /**
+     * 查舆情原始数据-平台进量（声量）统计
+     * @param dto   搜索实体类
+     * @return
+     * @throws IOException
+     */
+    JSONObject countOriginPlatform(MarkSearchDTO dto) throws IOException;
+
+    /**
+     * 非人工项目舆情库方案-平台进量（声量）统计
+     * @param dto   搜索实体类
+     * @return
+     * @throws IOException
+     */
+    JSONObject countNonManualPlatform(MarkSearchDTO dto) throws IOException;
 }

src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java

@@ -9,6 +9,8 @@ import com.zhiwei.brandkbs2.pojo.dto.ExportAppYuqingDTO;
 import com.zhiwei.brandkbs2.pojo.dto.ExportSearchWholeDTO;
 import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
 import com.zhiwei.brandkbs2.pojo.dto.SearchFilterDTO;
+import com.zhiwei.brandkbs2.pojo.dto.NonManualProjectPlanDTO;
+import com.zhiwei.brandkbs2.pojo.vo.LineVO;
 import com.zhiwei.brandkbs2.pojo.vo.PageVO;
 import org.apache.commons.lang3.tuple.Pair;
 import org.elasticsearch.index.query.BoolQueryBuilder;
@@ -378,4 +380,135 @@ public interface MarkDataService {
      * @return
      */
     Long interactionUpdatePrediction(MarkSearchDTO dto);
+
+    /**
+     * 查舆情舆情库-平台进量（声量）统计
+     * @param dto   标注数据搜索传输类
+     * @return
+     */
+    JSONObject countMarkPlatform(MarkSearchDTO dto);
+
+    /**
+     * 查舆情原始数据-平台进量（声量）统计
+     * @param dto   标注数据搜索传输类
+     * @return
+     */
+    JSONObject countOriginPlatform(MarkSearchDTO dto);
+
+    /**
+     * 新建非人工项目方案
+     * @param dto
+     * @return
+     */
+    ResponseResult addNonManualProjectPlan(NonManualProjectPlanDTO dto);
+
+    /**
+     * 删除非人工项目方案
+     * @param id
+     */
+    void deleteNonManualProjectPlan(String id);
+
+    /**
+     * 获取非人工项目方案详情
+     * @param id
+     */
+    NonManualProjectPlanDTO getNonManualProjectPlanInfo(String id);
+
+    /**
+     * 编辑非人工项目方案
+     * @param dto
+     * @return
+     */
+    ResponseResult updateNonManualProjectPlan(NonManualProjectPlanDTO dto);
+
+    /**
+     * 非人工项目方案-预估数据量
+     * @param keyword       关键词
+     * @param filterWord    去噪词
+     * @return
+     */
+    Integer getPlanDataPrediction(String keyword, String filterWord);
+
+    /**
+     * 获取方案设置-方案列表
+     * @return
+     */
+    PageVO<JSONObject> getNonManualProjectPlanList();
+
+    /**
+     * 获取方案设置-方案列表-昨日，今日数据消耗量
+     * @return
+     */
+    JSONObject getProjectPlanConsume();
+
+    /**
+     * 获取时间范围数据消耗图谱
+     * @return
+     */
+    List<LineVO> getTimeDataConsumeGraph(String planId, Long startTime, Long endTime);
+
+    /**
+     * 获取总数据消耗图谱
+     * @return
+     */
+    List<LineVO> getTotalDataConsumeGraph(Long startTime, Long endTime);
+
+    /**
+     * 获取非人工项目舆情数据筛选条件
+     * @return
+     */
+    JSONObject getNonManualMarkCriteria();
+
+    /**
+     * 获取非人工项目舆情数据列表
+     * @param markSearchDTO 标注数据搜索传输类
+     * @return
+     */
+    PageVO<MarkFlowEntity> getNonManualMarkList(MarkSearchDTO markSearchDTO);
+
+    /**
+     * 非人工项目舆情数据列表-互动量更新
+     * @param dto 标注数据搜索传输类
+     * @return
+     */
+    ResponseResult nonManualMarkInteractionUpdate(MarkSearchDTO dto);
+
+    /**
+     * 非人工项目工具库-舆情库互动量更新预估
+     * @param dto 标注数据搜索传输类
+     * @return
+     */
+    Long nonManualInteractionUpdatePrediction(MarkSearchDTO dto);
+
+    /**
+     * @param markSearchDTO 标注数据搜索传输类
+     * @return 返回结果
+     */
+    Pair<String, List<ExportAppYuqingDTO>> downloadNonManualMarkList(MarkSearchDTO markSearchDTO);
+
+    /**
+     * 保存/更新gid
+     * @param gid
+     */
+    void adjustMaxGid(String planId, Long gid);
+
+    /**
+     * 生成聚合列表并返回id
+     *
+     * @return 聚合id
+     */
+    String generateNonManualMarkAggreeList(String planId, Long startTime, Long endTime);
+
+    /**
+     * 获取非人工项目聚合结果
+     * @param dto
+     * @return
+     */
+    PageVO<MarkFlowEntity> getNonManualMarkAggreeList(MarkSearchDTO dto);
+
+    /**
+     * 非人工项目舆情数据列表-方案列表与各方案数据量统计
+     * @return
+     */
+    List<JSONObject> getNonManualMarkCountList();
 }

src/main/java/com/zhiwei/brandkbs2/service/MarkFlowService.java

@@ -33,6 +33,14 @@ public interface MarkFlowService {
     JSONObject createMarkFlowInfo4Origin(MarkInfoSource markInfoSource);
 
     /**
+     * 创建标注消息流实体
+     *
+     * @param markInfoSource
+     * @return
+     */
+    JSONObject createNonManualMarkFlowInfo(MarkInfoSource markInfoSource);
+
+    /**
      * 匹配标注标签值
      * @param mes
      * @param linkedGroupId
@@ -72,6 +80,7 @@ public interface MarkFlowService {
         private String projectId;
         private String contendId;
         private String linkedGroupId;
+        private String planId;
 
         public MarkInfoSource(JSONObject json, String projectId, String contendId) {
             this.json = json;
@@ -83,6 +92,13 @@ public interface MarkFlowService {
             this.json = json;
             this.linkedGroupId = linkedGroupId;
         }
+
+        public MarkInfoSource(JSONObject json, String projectId, String contendId, String planId) {
+            this.json = json;
+            this.projectId = projectId;
+            this.contendId = contendId;
+            this.planId = planId;
+        }
     }
 
 }

src/main/java/com/zhiwei/brandkbs2/service/ProjectService.java

@@ -149,4 +149,48 @@ public interface ProjectService {
      * @return
      */
     PageVO<JSONObject> getToolsetBytedanceCustomInfo(int page, int pageSize, String type);
+
+    /**
+     * 添加非人工项目
+     *
+     * @param projectVO 项目配置展示对象
+     */
+    ResponseResult addNonManualProject(ProjectVO projectVO);
+
+    /**
+     * 修改非人工项目信息
+     * @param projectVO 项目配置展示对象
+     * @return
+     */
+    ResponseResult updateNonManualProject(ProjectVO projectVO);
+
+    /**
+     * 查询非人工项目列表
+     * @param page 页码
+     * @param size 页码大小
+     * @param keyword   关键词
+     * @return
+     */
+    PageVO<JSONObject> findNonManualProjectList(int page, int size, String sorter, String keyword);
+
+    /**
+     * 非人工项目剩余数据量调整记录列表
+     *
+     * @param page 页码
+     * @param pageSize 页码大小
+     * @return PageVO
+     */
+    PageVO<JSONObject> findNonManualProjectMaintainList(String projectId, int page, int pageSize);
+
+    /**
+     * 非人工项目剩余数据量调整
+     * @param currentCount 调整后数值
+     */
+    void adjustNonManualProjectDataBalance(String projectId, Integer currentCount);
+
+    /**
+     * 非人工项目剩余数据调整记录总览
+     * @return json
+     */
+    JSONObject nonManualProjectOverview(String project);
 }

src/main/java/com/zhiwei/brandkbs2/service/ToolsetService.java

@@ -148,4 +148,10 @@ public interface ToolsetService {
      * @param taskId    任务id
      */
     void cancelTask(String taskId);
+
+    /**
+     * 获取项目工具库功能显示列表
+     * @return
+     */
+    List<String> getProjectToolsetShowList();
 }

src/main/java/com/zhiwei/brandkbs2/service/impl/EsSearchServiceImpl.java

 package com.zhiwei.brandkbs2.service.impl;
 
+import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.brandkbs2.common.GenericAttribute;
+import com.zhiwei.brandkbs2.common.GlobalPojo;
 import com.zhiwei.brandkbs2.es.EsClientDao;
 import com.zhiwei.brandkbs2.es.EsQueryTools;
 import com.zhiwei.brandkbs2.function.ArticleFunction;
 import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
 import com.zhiwei.brandkbs2.service.EsSearchService;
 import com.zhiwei.brandkbs2.util.Tools;
+import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.tuple.Pair;
@@ -21,9 +24,8 @@ import org.springframework.stereotype.Service;
 
 import javax.annotation.Resource;
 import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
+import java.util.stream.Collectors;
 
 /**
  * @ClassName: EsSearchServiceImpl
@@ -55,6 +57,26 @@ public class EsSearchServiceImpl implements EsSearchService {
         return searchHitsAndCount(dto, this::createOriginSearchHelper, hotEsDao, scroll);
     }
 
+    @Override
+    public Pair<SearchHits[], Map<String, Long>> searchNonManualHitsAndCount(MarkSearchDTO dto, boolean scroll) throws IOException {
+        return searchHitsAndCount(dto, this::createNonManualSearchHelper, esClientDao, scroll);
+    }
+
+    @Override
+    public JSONObject countMarkPlatform(MarkSearchDTO dto) throws IOException {
+        return platformCount(dto, this::createSearchHelper, esClientDao);
+    }
+
+    @Override
+    public JSONObject countOriginPlatform(MarkSearchDTO dto) throws IOException {
+        return platformCount(dto, this::createOriginSearchHelper, hotEsDao);
+    }
+
+    @Override
+    public JSONObject countNonManualPlatform(MarkSearchDTO dto) throws IOException {
+        return platformCount(dto, this::createNonManualSearchHelper, esClientDao);
+    }
+
     private Pair<SearchHits[], Map<String, Long>> searchHitsAndCount(MarkSearchDTO dto, ArticleFunction.CreateSearchHelper createSearchHelper,
                                                                      EsClientDao clientDao, boolean scroll) throws IOException {
         EsClientDao.SearchHelper helper = createSearchHelper.create(dto);
@@ -63,8 +85,8 @@ public class EsSearchServiceImpl implements EsSearchService {
             return Pair.of(clientDao.searchScrollResponse(helper).stream().map(SearchResponse::getHits).toArray(SearchHits[]::new), null);
         }
         SearchHits searchHits = clientDao.searchHits(helper);
-        Map<String, Long> platformCount = platformCount(dto.getPage(), dto.getPlatforms(), helper.getPostFilter(), clientDao);
-        return Pair.of(new SearchHits[]{searchHits}, platformCount);
+//        Map<String, Long> platformCount = platformCount(dto.getPage(), dto.getPlatforms(), helper.getPostFilter(), clientDao);
+        return Pair.of(new SearchHits[]{searchHits}, new HashMap<>());
     }
 
     private EsClientDao.SearchHelper createSearchHelper(MarkSearchDTO dto) {
@@ -245,23 +267,104 @@ public class EsSearchServiceImpl implements EsSearchService {
         return helper;
     }
 
+    private EsClientDao.SearchHelper createNonManualSearchHelper(MarkSearchDTO dto) {
+        EsClientDao.SearchHelper helper = EsClientDao.createSearchHelper();
+        // projectId、contendId
+        String projectId = dto.getProjectId();
+        String planId = dto.getPlanId();
+        // PostFilter 后置过滤器
+        BoolQueryBuilder postFilter = EsQueryTools.assembleCacheMapsPlanQuery(projectId, planId, dto.getPoliticsLevel());
+        // time
+        postFilter.must(QueryBuilders.rangeQuery(dto.getTimeType()).gte(dto.getStartTime()).lt(dto.getEndTime()));
+        // platform
+        if (CollectionUtils.isNotEmpty(dto.getPlatforms())) {
+            postFilter.must(EsQueryTools.assemblePlatformQuery(Tools.getPlatformByIds(dto.getPlatforms())));
+        }
+        // tags
+        if (CollectionUtils.isNotEmpty(dto.getTags())) {
+            postFilter.must(EsQueryTools.assembleTagQuery(dto.getTags()));
+        }
+        // Query 查询条件
+        String[] fieldSearch = "仅标题".equals(dto.getSearchField()) ? new String[]{GenericAttribute.ES_IND_TITLE} : new String[]{GenericAttribute.ES_IND_FULL_TEXT};
+        // keyword
+        if (StringUtils.isNotEmpty(dto.getKeyword())) {
+            // Query
+            postFilter.must(EsQueryTools.assembleNormalKeywordQuery(dto.getKeyword(), fieldSearch));
+        }
+        // sourceKeyword
+        if (StringUtils.isNotEmpty(dto.getSourceKeyword())) {
+            postFilter.must(EsQueryTools.assembleSourceQuery(dto.getSourceKeyword()));
+        }
+        // hostKeyword
+        if (StringUtils.isNotEmpty(dto.getHostKeyword())) {
+            postFilter.must(EsQueryTools.assembleFiledKeywordQuery("host", dto.getHostKeyword()));
+        }
+        // forward
+        if (Objects.nonNull(dto.getForward())){
+            postFilter.must(EsQueryTools.assembleForward2Query(Objects.equals(dto.getForward(), "转发")));
+        }
+        // gid 已读 未读
+        if (Objects.nonNull(dto.getRead()) && Objects.nonNull(dto.getGid())){
+            if (Objects.equals(dto.getRead(), "已读")){
+                postFilter.must(QueryBuilders.rangeQuery(GenericAttribute.ES_GID).lte(dto.getGid()));
+            }else {
+                postFilter.must(QueryBuilders.rangeQuery(GenericAttribute.ES_GID).gt(dto.getGid()));
+            }
+        }
+        // dataType
+        if (Objects.nonNull(dto.getDataType())){
+            postFilter.must(EsQueryTools.assembleC2Query(dto.getDataType()));
+        }
+        helper.setPostFilter(postFilter);
+        // sort
+        FieldSortBuilder sort = null;
+        if (null != dto.getSorter()) {
+            for (Map.Entry<String, Object> entry : dto.getSorter().entrySet()) {
+                String key = entry.getKey();
+                if (key.equals("influence")) {
+                    key = "channel_influence";
+                } else if (key.equals("followers")) {
+                    key = "channel_followers";
+                }
+                if (null == entry.getValue() || entry.getValue().toString().contains("desc")) {
+                    sort = SortBuilders.fieldSort(key).order(SortOrder.DESC);
+                } else {
+                    sort = SortBuilders.fieldSort(key).order(SortOrder.ASC);
+                }
+            }
+        }
+        helper.setSort(sort);
+        // from size
+        if (null != dto.getPageSize()) {
+            helper.setFrom((dto.getPage() - 1) * dto.getPageSize());
+            helper.setSize(dto.getPageSize());
+        }
+        // HighlightBuilder ???
+        return helper;
+    }
+
     /**
      * 平台进量统计
      *
      * @return
      */
-    private Map<String, Long> platformCount(int page, List<String> platforms, BoolQueryBuilder postFilter, EsClientDao esClientDao) throws IOException {
-        Map<String, Long> counts = new HashMap<>();
-        // TODO 平台进量暂不统计
+    private JSONObject platformCount(MarkSearchDTO dto, ArticleFunction.CreateSearchHelper createSearchHelper, EsClientDao esClientDao) throws IOException {
+        EsClientDao.SearchHelper helper = createSearchHelper.create(dto);
+        List<JSONObject> list = new ArrayList<>();
         //平台计量统计 仅第一页且无平台限制
-//        if (1 == page && CollectionUtils.isEmpty(platforms)) {
-//            for (MessagePlatform platform : GlobalPojo.PLATFORMS) {
-//                BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery().must(postFilter).must(EsQueryTools.assemblePlatformQuery(Collections.singletonList(platform)));
-//                Long count = esClientDao.count(esClientDao.getIndexes(), queryBuilder, null);
-//                counts.put(platform.getName(), count);
-//            }
-//        }
-        return counts;
+        for (MessagePlatform platform : GlobalPojo.PLATFORMS.stream().filter(messagePlatform -> !"外媒".equals(messagePlatform.getName())).collect(Collectors.toList())) {
+            BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery().must(helper.getPostFilter()).must(EsQueryTools.assemblePlatformQuery(Collections.singletonList(platform)));
+            Long count = esClientDao.count(esClientDao.getIndexes(), queryBuilder, null);
+            JSONObject jsonObject = new JSONObject();
+            jsonObject.put("id", platform.getId());
+            jsonObject.put("name", platform.getName());
+            jsonObject.put("count", count);
+            list.add(jsonObject);
+        }
+        Long total = list.stream().map(jsonObject -> jsonObject.getLongValue("count")).reduce(Long::sum).orElse(0L);
+        JSONObject res = new JSONObject();
+        res.put("count", list);
+        res.put("total", total);
+        return res;
     }
-
 }

src/main/java/com/zhiwei/brandkbs2/service/impl/IndexServiceImpl.java

@@ -131,6 +131,9 @@ public class IndexServiceImpl implements IndexService {
             //获取总正面稿件数
             long totalPositiveCount = markDataService.getYuqingMarkCount
                     (null, null, EmotionEnum.POSITIVE.getName(), projectId, Constant.PRIMARY_CONTEND_ID);
+            //获取总中性稿件数
+            long totalNeutralCount = markDataService.getYuqingMarkCount
+                    (null, null, EmotionEnum.NEUTRAL.getName(), projectId, Constant.PRIMARY_CONTEND_ID);
             //获取时间范围内总稿件数
             long normalCount = markDataService.getYuqingMarkCount
                     (startTime, endTime, EmotionEnum.ALL.getName(), projectId, Constant.PRIMARY_CONTEND_ID);
@@ -149,15 +152,20 @@ public class IndexServiceImpl implements IndexService {
             //获取上个周期时间范围内总正面稿件数
             long oldPositiveCount = markDataService.getYuqingMarkCount
                     (oldStartTime, startTime, EmotionEnum.POSITIVE.getName(), projectId, Constant.PRIMARY_CONTEND_ID);
+            //获取上个周期时间范围内总中性稿件数
+            long oldNeutralCount = markDataService.getYuqingMarkCount
+                    (oldStartTime, startTime, EmotionEnum.NEUTRAL.getName(), projectId, Constant.PRIMARY_CONTEND_ID);
+            double reputation = normalCount == 0 ? 0d : (positiveCount + neutralCount) / (double) normalCount;
             double posPro = normalCount == 0 ? 0d : positiveCount / (double) normalCount;
             double neuPro = normalCount == 0 ? 0d : neutralCount / (double) normalCount;
             double negPro = normalCount == 0 ? 0d : negativeCount / (double) normalCount;
-            double oldPosPro = oldCount == 0 ? 0d : oldPositiveCount / (double) oldCount;
+            double oldReputation = oldCount == 0 ? 0d : (oldPositiveCount + oldNeutralCount) / (double) oldCount;
             jsonObject.put("posPro", posPro);
             jsonObject.put("neuPro", neuPro);
             jsonObject.put("negPro", negPro);
-            jsonObject.put("compare", oldPosPro == 0 ? 0d : (posPro - oldPosPro) / oldPosPro);
-            double avgPosPro = totalNormalCount == 0 ? 0d : totalPositiveCount / (double) totalNormalCount;
+            jsonObject.put("reputation", reputation);
+            jsonObject.put("compare", oldReputation == 0 ? 0d : (reputation - oldReputation) / oldReputation);
+            double avgPosPro = totalNormalCount == 0 ? 0d : (totalPositiveCount + totalNeutralCount) / (double) totalNormalCount;
             jsonObject.put("avgPosPro", avgPosPro);
             redisUtil.setExpire(redisKey, JSON.toJSONString(jsonObject));
         } catch (IOException e) {
@@ -310,7 +318,7 @@ public class IndexServiceImpl implements IndexService {
         JSONObject yuQing = new JSONObject();
         yuQing.put("avgTotal", yuQingResult.getIntValue("avgTotal"));
         yuQing.put("compare", yuQingResult.getDoubleValue("compare"));
-        yuQing.put("yuqingTotal", yuQingResult.getIntValue("yuqingTotal"));
+        yuQing.put("yuqingTotal", yuQingResult.getIntValue("rangeCount"));
         result.put("yuqing", yuQing);
         JSONObject reputation = new JSONObject();
         reputation.put("avgReputation", reputationResult.getDoubleValue("avgPosPro"));

src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java

 package com.zhiwei.brandkbs2.service.impl;
 
-
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
@@ -17,8 +16,9 @@ import com.zhiwei.brandkbs2.config.Constant;
 import com.zhiwei.brandkbs2.dao.AggreeResultDao;
 import com.zhiwei.brandkbs2.dao.ChannelDao;
 import com.zhiwei.brandkbs2.dao.HighlightWordDao;
+import com.zhiwei.brandkbs2.dao.NonManualProjectPlanDao;
 import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
-import com.zhiwei.brandkbs2.easyexcel.dto.ExportInteractionUpdateDTO;
+import com.zhiwei.brandkbs2.easyexcel.dto.ExportYuqingInteractionUpdateDTO;
 import com.zhiwei.brandkbs2.enmus.ChannelEmotion;
 import com.zhiwei.brandkbs2.enmus.EmotionEnum;
 import com.zhiwei.brandkbs2.enmus.ImportantChannelEnum;
@@ -31,14 +31,8 @@ import com.zhiwei.brandkbs2.listener.ApplicationProjectListener;
 import com.zhiwei.brandkbs2.model.CommonCodeEnum;
 import com.zhiwei.brandkbs2.model.ResponseResult;
 import com.zhiwei.brandkbs2.pojo.*;
-import com.zhiwei.brandkbs2.pojo.dto.ExportAppYuqingDTO;
-import com.zhiwei.brandkbs2.pojo.dto.ExportSearchWholeDTO;
-import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
-import com.zhiwei.brandkbs2.pojo.dto.SearchFilterDTO;
-import com.zhiwei.brandkbs2.pojo.vo.CustomTagVo;
-import com.zhiwei.brandkbs2.pojo.vo.LineVO;
-import com.zhiwei.brandkbs2.pojo.vo.PageVO;
-import com.zhiwei.brandkbs2.pojo.vo.ProjectVO;
+import com.zhiwei.brandkbs2.pojo.dto.*;
+import com.zhiwei.brandkbs2.pojo.vo.*;
 import com.zhiwei.brandkbs2.service.*;
 import com.zhiwei.brandkbs2.util.MongoUtil;
 import com.zhiwei.brandkbs2.util.RedisUtil;
@@ -157,6 +151,9 @@ public class MarkDataServiceImpl implements MarkDataService {
     @Resource(name = "highlightWordDao")
     HighlightWordDao highlightWordDao;
 
+    @Resource(name = "nonManualProjectPlanDao")
+    NonManualProjectPlanDao nonManualProjectPlanDao;
+
     @Resource(name = "toolsetServiceImpl")
     private ToolsetService toolsetService;
 
@@ -185,10 +182,12 @@ public class MarkDataServiceImpl implements MarkDataService {
             long total = hitsAndCounts.getLeft()[0].getTotalHits().value;
             // 消息列表
             List<MarkFlowEntity> flowEntityList = getMarkFlowEntity4Origin(markSearchDTO, hitsAndCounts.getLeft()[0]);
+            JSONObject jsonObject = new JSONObject();
+            jsonObject.put("highlightWord", highlightWordDao.findList(Query.query(Criteria.where("projectId").is(UserThreadLocal.getProjectId()))).stream().map(HighlightWord::getKeyword).collect(Collectors.toList()));
             // 返回分页结果并设置平台count
             return PageVO.createPageVo(total, markSearchDTO.getPage(), markSearchDTO.getPageSize(), flowEntityList)
                     // 各平台计量
-                    .setInfo(new JSONObject(ImmutableMap.of("highlightWord", highlightWordDao.findList(Query.query(Criteria.where("projectId").is(UserThreadLocal.getProjectId()))).stream().map(HighlightWord::getKeyword).collect(Collectors.toList()))));
+                    .setInfo(jsonObject);
         } catch (IOException e) {
             ExceptionCast.cast(CommonCodeEnum.FAIL, "es查询异常");
         }
@@ -267,10 +266,12 @@ public class MarkDataServiceImpl implements MarkDataService {
             long total = hitsAndCounts.getLeft()[0].getTotalHits().value;
             // 消息列表
             List<MarkFlowEntity> flowEntityList = getMarkFlowEntity(markSearchDTO, hitsAndCounts.getLeft()[0]);
+            JSONObject jsonObject = new JSONObject();
+            // 高亮关键词
+            jsonObject.put("highlightWord", GlobalPojo.HIGHLIGHT_MAP.get(UserThreadLocal.getProjectId()));
             // 返回分页结果并返回高亮关键词
             return PageVO.createPageVo(total, markSearchDTO.getPage(), markSearchDTO.getPageSize(), flowEntityList)
-                    // 高亮关键词
-                    .setInfo(new JSONObject(ImmutableMap.of("highlightWord", GlobalPojo.HIGHLIGHT_MAP.get(UserThreadLocal.getProjectId()))));
+                    .setInfo(jsonObject);
         } catch (IOException e) {
             ExceptionCast.cast(CommonCodeEnum.FAIL, "es查询异常");
         }
@@ -971,7 +972,6 @@ public class MarkDataServiceImpl implements MarkDataService {
                 primaryPlatformsProportion.stream().max(Comparator.comparing(o -> o.getLong("count"))).orElseGet(JSONObject::new);
         result.put("priMaxPlatform", primaryMaxPlatformProportion.getString("platform"));
         result.put("priMaxPlatformPro", primaryMaxPlatformProportion.getDoubleValue("proportion"));
-        // 竞品的名字
         String brandName = projectService.getProjectByContendId(dto.getProjectId(), dto.getContendId()).getBrandName();
         result.put("conName", brandName);
         // 获取时间段竞品在每个平台传播量
@@ -998,7 +998,6 @@ public class MarkDataServiceImpl implements MarkDataService {
     @Override
     public JSONObject getContendSpreadAnalyze(Long startTime, Long endTime, String projectId, String contendId, int hotArticleSize, boolean cache) throws IOException {
         ProjectVO project = projectService.getProjectVOById(projectId);
-        // 竞品的名字
         AbstractProject projectByContendId = projectService.getProjectByContendId(projectId, contendId);
         String brandName = projectByContendId.getBrandName();
         // 缓存
@@ -1080,7 +1079,6 @@ public class MarkDataServiceImpl implements MarkDataService {
         }
 
         // 封装竞品信息结果
-        // 竞品名
         result.put("conName", brandName);
         // 获取竞品稿件传播平均值
         double avgContendSpreadNormalCount = contendSpread.stream().mapToLong(LineVO::getCount).average().orElse(0);
@@ -1151,17 +1149,12 @@ public class MarkDataServiceImpl implements MarkDataService {
     public Pair<JSONObject, Integer> searchWholeNetworkWithBalance(SearchFilterDTO dto) {
         if (dto.isExternalDataSource()) { // 查商业数据库
             try {
-                JSONObject params = externalTransform(dto);
-                JSONObject taskEntity = new JSONObject();
-                taskEntity.put("appId", "6183571e0d710000f6003a12");   // 应用id, 由张志伟提供给使用者
-                taskEntity.put("taskType", null);                      // 可为null, 任务类型
-                taskEntity.put("taskDescription", null);               // 可为null, 任务描述
-                taskEntity.put("taskName", UserThreadLocal.getProjectId() + "_" + UserThreadLocal.getUserId());
-                taskEntity.put("userName", "品见");                     // 提交任务的人
-                taskEntity.put("searchParams", params);                // 任务详情参数
-                ResponseEntity<JSONObject> responseEntity = restTemplate.postForEntity(addIStarShineKSDataUrl, taskEntity, JSONObject.class);
-                JSONObject result = Objects.requireNonNull(responseEntity.getBody()).getJSONObject("data");
-                JSONObject json = searchWholeBackUp(result.get("id").toString());
+                JSONObject data = getSearchWholeOpinionJson(dto);
+                JSONObject json = packageExternalDataSourceResult(data);
+                // 仅当第一页时统计平台进量（声量）
+                if (1 == dto.getPage()) {
+                    json.put("info", getExternalDataSourcePlatformCount(dto));
+                }
                 return Pair.of(json, json.getInteger("pageLimit"));
             } catch (Exception e) {
                 log.error("error searchWholeNetwork - ", e);
@@ -1173,6 +1166,38 @@ public class MarkDataServiceImpl implements MarkDataService {
         }
     }
 
+    /**
+     * 获取外部数据源搜索结果json
+     *
+     * @param dto
+     * @return
+     */
+    private JSONObject getSearchWholeOpinionJson(SearchFilterDTO dto) {
+        // 组装请求参数
+        JSONObject taskEntity = packageExternalDataSourceParams(dto);
+        ResponseEntity<JSONObject> responseEntity = restTemplate.postForEntity(addIStarShineKSDataUrl, taskEntity, JSONObject.class);
+        JSONObject result = Objects.requireNonNull(responseEntity.getBody()).getJSONObject("data");
+        return searchWholeBackUp(result.get("id").toString());
+    }
+
+    /**
+     * 外部数据源组装请求参数
+     *
+     * @param dto
+     * @return
+     */
+    private JSONObject packageExternalDataSourceParams(SearchFilterDTO dto) {
+        JSONObject params = externalTransform(dto);
+        JSONObject taskEntity = new JSONObject();
+        taskEntity.put("appId", "6183571e0d710000f6003a12");   // 应用id, 由张志伟提供给使用者
+        taskEntity.put("taskType", null);                      // 可为null, 任务类型
+        taskEntity.put("taskDescription", null);               // 可为null, 任务描述
+        taskEntity.put("taskName", UserThreadLocal.getProjectId() + "_" + UserThreadLocal.getUserId());
+        taskEntity.put("userName", "品见");                     // 提交任务的人
+        taskEntity.put("searchParams", params);                // 任务详情参数
+        return taskEntity;
+    }
+
     @Override
     public List<ExportSearchWholeDTO> exportSearchWhole(SearchFilterDTO dto) {
         Integer exportAmount = userService.queryUserInfo(UserThreadLocal.getUserId(), UserThreadLocal.getProjectId()).getExportAmount();
@@ -1196,6 +1221,12 @@ public class MarkDataServiceImpl implements MarkDataService {
         return jsonArray.stream().map(json -> ExportSearchWholeDTO.creatExportSearchWholeDTO((JSONObject) json)).collect(Collectors.toList());
     }
 
+    /**
+     * 舆情库数据结果组装成标准分页返回格式
+     *
+     * @param responseEntity
+     * @return
+     */
     private JSONObject searchWholeOpinion(ResponseEntity<JSONObject> responseEntity) {
         JSONObject jsonObject = new JSONObject();
         JSONObject bodyData = Objects.requireNonNull(responseEntity.getBody()).getJSONObject("data");
@@ -1228,6 +1259,20 @@ public class MarkDataServiceImpl implements MarkDataService {
             result.put("ipLocation", list.get("ip_location"));
             resultList.add(result);
         }
+        // 仅第一页返回平台进量（声量）统计
+        if (1 == data.getInteger("page")) {
+            JSONObject count = bodyData.getJSONObject("count");
+            List<JSONObject> platformCounts = count.getJSONArray("count").toJavaList(JSONObject.class);
+            // 去除外媒和脉脉两个平台
+            List<JSONObject> list = platformCounts.stream().filter(json -> !Objects.equals(json.getString("name"), "脉脉") &&
+                    !Objects.equals(json.getString("name"), "外媒")).collect(Collectors.toList());
+            // 重新计算总数
+            Long total = list.stream().map(json -> json.getLongValue("count")).reduce(Long::sum).orElse(0L);
+            JSONObject counts = new JSONObject();
+            counts.put("count", list);
+            counts.put("total", total);
+            jsonObject.put("info", counts);
+        }
         jsonObject.put("list", resultList);
         jsonObject.put("page", data.get("page"));
         jsonObject.put("pageLimit", data.get("pageLimit"));
@@ -1237,10 +1282,14 @@ public class MarkDataServiceImpl implements MarkDataService {
         return jsonObject;
     }
 
+    /**
+     * 获取外部数据源搜索结果JSONObject
+     *
+     * @param taskId
+     * @return
+     */
     private JSONObject searchWholeBackUp(String taskId) {
         JSONObject jsonObject = new JSONObject();
-        List<JSONObject> resultList = new ArrayList<>();
-
         for (int i = 0; i < 30; i++) {
             ResponseEntity<JSONObject> responseEntity = restTemplate.getForEntity(getIStarShineKSInfoDataUrl, JSONObject.class, taskId);
             JSONObject data = Objects.requireNonNull(responseEntity.getBody()).getJSONObject("data");
@@ -1249,43 +1298,80 @@ public class MarkDataServiceImpl implements MarkDataService {
                 throw new IllegalArgumentException(task.getString("searchResultMsg"));
             }
             if (1 == task.getInteger("crawlerFinish")) {
-
-                //pageSize
-                Integer receiveCount = task.getInteger("receiveCount");
-                Integer searchCount = task.getInteger("searchCount");
-                int totalPage;
-                if (0 == receiveCount || 0 == searchCount){
-                    totalPage = 0;
-                }else {
-                    totalPage = searchCount % receiveCount == 0 ? searchCount / receiveCount : (searchCount / receiveCount + 1);
-                }
-                List<JSONObject> listArray = data.getJSONArray("data").toJavaList(JSONObject.class);
-                for (JSONObject json : listArray) {
-                    JSONObject result = new JSONObject();
-                    String platform = getPlatformExternal(json.getJSONObject("gather"));
-                    result.put("platform", platform);
-                    result.put("title", json.get("title"));
-                    result.put("content", json.get("content"));
-                    // 渠道获取分平台
-                    result.put("source", getSourceExternal(json, getExternalType(json, platform)));
-                    result.put("url", getUrl(json));
-                    result.put("time", json.getString("ctime").length() == 10 ? json.getLong("ctime") * 1000L : json.getLong("ctime"));
-                    resultList.add(result);
-                }
-                jsonObject.put("list", resultList);
-                //jsonObject.put("page",data1.get("page"));
-                jsonObject.put("pageLimit", receiveCount);
-                jsonObject.put("total", searchCount);
-                jsonObject.put("totalPage", totalPage);
-                jsonObject.put("balance", projectService.getProjectById(UserThreadLocal.getProjectId()).getWholeSearchBalance() - receiveCount);
-                jsonObject.put("isBackUp", true);
-                return jsonObject;
+                return data;
             }
             Tools.sleep(500L);
         }
         return jsonObject;
     }
 
+    /**
+     * 将外部数据源JSONObject组装成标准的返回结果
+     *
+     * @param data
+     * @return
+     */
+    private JSONObject packageExternalDataSourceResult(JSONObject data) {
+        JSONObject jsonObject = new JSONObject();
+        List<JSONObject> resultList = new ArrayList<>();
+        JSONObject task = data.getJSONObject("task");
+        Integer receiveCount = task.getInteger("receiveCount");
+        Integer searchCount = task.getInteger("searchCount");
+        int totalPage;
+        if (0 == receiveCount || 0 == searchCount) {
+            totalPage = 0;
+        } else {
+            totalPage = searchCount % receiveCount == 0 ? searchCount / receiveCount : (searchCount / receiveCount + 1);
+        }
+        List<JSONObject> listArray = data.getJSONArray("data").toJavaList(JSONObject.class);
+        for (JSONObject json : listArray) {
+            JSONObject result = new JSONObject();
+            String platform = getPlatformExternal(json.getJSONObject("gather"));
+            result.put("platform", platform);
+            result.put("title", json.get("title"));
+            result.put("content", json.get("content"));
+            // 渠道获取分平台
+            result.put("source", getSourceExternal(json, getExternalType(json, platform)));
+            result.put("url", getUrl(json));
+            result.put("time", json.getString("ctime").length() == 10 ? json.getLong("ctime") * 1000L : json.getLong("ctime"));
+            resultList.add(result);
+        }
+        jsonObject.put("list", resultList);
+        //jsonObject.put("page",data1.get("page"));
+        jsonObject.put("pageLimit", receiveCount);
+        jsonObject.put("total", searchCount);
+        jsonObject.put("totalPage", totalPage);
+        jsonObject.put("balance", projectService.getProjectById(UserThreadLocal.getProjectId()).getWholeSearchBalance() - receiveCount);
+        jsonObject.put("isBackUp", true);
+        return jsonObject;
+    }
+
+    /**
+     * 获取实时采集（外部数据源）平台进量（声量）
+     *
+     * @param dto
+     * @return
+     */
+    private JSONObject getExternalDataSourcePlatformCount(SearchFilterDTO dto) {
+        JSONObject res = new JSONObject();
+        List<JSONObject> list = new ArrayList<>();
+        List<String> platforms = Arrays.asList("App", "博客", "平媒", "新闻", "微博", "微信", "论坛", "视频", "短视频");
+        for (String platform : platforms) {
+            // 重置平台条件
+            dto.setPlatforms(Collections.singletonList(platform));
+            JSONObject data = getSearchWholeOpinionJson(dto);
+            JSONObject task = data.getJSONObject("task");
+            JSONObject jsonObject = new JSONObject();
+            jsonObject.put("count", task.getLongValue("searchCount"));
+            jsonObject.put("name", platform);
+            list.add(jsonObject);
+        }
+        Long total = list.stream().map(jsonObject -> jsonObject.getLongValue("count")).reduce(Long::sum).orElse(0L);
+        res.put("count", list);
+        res.put("total", total);
+        return res;
+    }
+
     private ClassB.TypeB getExternalType(JSONObject tJson, String platform) {
         int wtype = tJson.getIntValue("wtype");
         if (platform.equals("微博") && 2 == wtype) {
@@ -1298,7 +1384,7 @@ public class MarkDataServiceImpl implements MarkDataService {
         return ClassB.TypeB.COMPLETE;
     }
 
-    private String getSourceExternal(JSONObject tJson, ClassB.TypeB typeB){
+    private String getSourceExternal(JSONObject tJson, ClassB.TypeB typeB) {
         String source = null;
         switch (typeB) {
             case COMPLETE:
@@ -1510,15 +1596,38 @@ public class MarkDataServiceImpl implements MarkDataService {
 
     @Override
     public ResponseResult markInteractionUpdate(MarkSearchDTO dto) {
-        Pair<String, List<ExportAppYuqingDTO>> pair = downloadList(dto, esSearchService::searchMarkHitsAndCount);
-        assert pair != null;
-        List<ExportAppYuqingDTO> list = pair.getRight();
+        List<BaseMap> list = new ArrayList<>();
+        dto.setProjectId(UserThreadLocal.getProjectId());
+        try {
+            SearchHits[] searchHits = esSearchService.searchMarkHitsAndCount(dto, true).getLeft();
+            for (SearchHits searchHit : searchHits) {
+                List<BaseMap> baseMaps = Arrays.stream(searchHit.getHits()).map(SearchHit::getSourceAsMap).map(Tools::getBaseFromEsMap).collect(Collectors.toList());
+                list.addAll(baseMaps);
+            }
+        } catch (IOException e) {
+            ExceptionCast.cast(CommonCodeEnum.FAIL, "es查询异常");
+        }
         // 互动量更新剩余可用次数
         int interactionBalance = projectService.getProjectById(UserThreadLocal.getProjectId()).getInteractionBalance();
-        if (interactionBalance - list.size() < 0){
+        if (interactionBalance - list.size() < 0) {
             return ResponseResult.failure("剩余可用次数不足");
         }
-        Map<String, String> map = list.stream().collect(Collectors.toMap(ExportAppYuqingDTO::getUrl, ExportAppYuqingDTO::getPlatform, (v1, v2) -> v1));
+        List<ExportYuqingInteractionUpdateDTO> exportList = markInteractionUpdate(list);
+        // excel输出到指定路径
+        String projectName = projectService.getProjectById(UserThreadLocal.getProjectId()).getProjectName();
+        String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), "舆情库互动量更新结果");
+        EasyExcelUtil.write(filePath, "sheet1", ExportYuqingInteractionUpdateDTO.class, exportList);
+        JSONObject res = new JSONObject();
+        // 记录使用情况
+        extraService.decreaseInteractionRecord(InteractionUpdateRecord.UsedType.yuqing, list.size());
+        res.put("filePath", filePath);
+        res.put("remainingCount", interactionBalance - list.size());
+        // 记录使用
+        return ResponseResult.success(res);
+    }
+
+    private List<ExportYuqingInteractionUpdateDTO> markInteractionUpdate(List<BaseMap> list){
+        Map<String, String> map = list.stream().collect(Collectors.toMap(BaseMap::getUrl, BaseMap::getPlatform, (v1, v2) -> v1));
         // 分割map，list里每个map.size为100，目前品见最大链接限制为100，见知微tapd-wiki《互动量更新中间件使用》说明
         List<Map<String, String>> mapList = Tools.splitMap(map, 100);
         AtomicInteger id = new AtomicInteger(0);
@@ -1536,36 +1645,523 @@ public class MarkDataServiceImpl implements MarkDataService {
                 result.addAll(toolsetService.getInteractionResult(ids));
             }
         }
-        Map<Object, JSONObject> urlMap = result.stream().collect(Collectors.toMap(jsonObject -> jsonObject.get("url"), o -> o));
-        List<ExportInteractionUpdateDTO> exportList = list.stream().map(ExportAppYuqingDTO::getUrl)
-                .map(url -> ExportInteractionUpdateDTO.createWithPlatform(String.valueOf(id.incrementAndGet()), map, urlMap, url))
+        Map<Object, JSONObject> resultMap = result.stream().collect(Collectors.toMap(jsonObject -> jsonObject.get("url"), o -> o));
+        Map<String, BaseMap> baseMap = list.stream().collect(Collectors.toMap(BaseMap::getUrl, o -> o, (v1, v2) -> v2));
+        return list.stream().map(BaseMap::getUrl)
+                .map(url -> new ExportYuqingInteractionUpdateDTO(id.incrementAndGet(), baseMap, resultMap, url))
                 .collect(Collectors.toList());
+    }
+
+    @Override
+    public Long interactionUpdatePrediction(MarkSearchDTO dto) {
+        try {
+            String projectId = UserThreadLocal.getProjectId();
+            String contendId = Constant.PRIMARY_CONTEND_ID;
+            BoolQueryBuilder postFilter = EsQueryTools.assembleCacheMapsQuery(projectId, contendId, dto.getPoliticsLevel(), dto.getField(), dto.getRegion(), dto.getMainBodyType(), dto.getChannelEmotion());
+            // time
+            postFilter.must(QueryBuilders.rangeQuery(dto.getTimeType()).gte(dto.getStartTime()).lt(dto.getEndTime()));
+            // platform
+            if (CollectionUtils.isNotEmpty(dto.getPlatforms())) {
+                postFilter.must(EsQueryTools.assemblePlatformQuery(Tools.getPlatformByIds(dto.getPlatforms())));
+            }
+            return esClientDao.count(postFilter);
+        } catch (Exception e) {
+            ExceptionCast.cast(CommonCodeEnum.FAIL, "es查询异常");
+        }
+        return null;
+    }
+
+    @Override
+    public JSONObject countMarkPlatform(MarkSearchDTO dto) {
+        defaultMarkSearch(dto);
+        try {
+            return esSearchService.countMarkPlatform(dto);
+        } catch (Exception e) {
+            ExceptionCast.cast(CommonCodeEnum.FAIL, "es查询异常", e);
+        }
+        return null;
+    }
+
+    @Override
+    public JSONObject countOriginPlatform(MarkSearchDTO dto) {
+        // 时间范围默认当天
+        if (Objects.isNull(dto.getStartTime()) || Objects.isNull(dto.getEndTime())) {
+            Date now = new Date();
+            dto.setEndTime(now.getTime());
+            dto.setStartTime(DateUtils.addDays(now, -1).getTime());
+        }
+        defaultMarkSearch(dto);
+        try {
+            return esSearchService.countOriginPlatform(dto);
+        } catch (Exception e) {
+            ExceptionCast.cast(CommonCodeEnum.FAIL, "es查询异常", e);
+        }
+        return null;
+    }
+
+    @Override
+    public ResponseResult addNonManualProjectPlan(NonManualProjectPlanDTO dto) {
+        if (nonManualProjectPlanDao.count(new Query(Criteria.where("projectId").is(UserThreadLocal.getProjectId()))) >= 5) {
+            return ResponseResult.failure("超过方案数量上限5");
+        }
+        // TODO 数据预估超限返回
+        NonManualProjectPlan plan = new NonManualProjectPlan(dto);
+        nonManualProjectPlanDao.insertOne(plan);
+        return ResponseResult.success();
+    }
+
+    @Override
+    public void deleteNonManualProjectPlan(String id) {
+        nonManualProjectPlanDao.deleteOneById(id);
+    }
+
+    @Override
+    public NonManualProjectPlanDTO getNonManualProjectPlanInfo(String id) {
+        return new NonManualProjectPlanDTO(nonManualProjectPlanDao.findOneById(id));
+    }
+
+    @Override
+    public ResponseResult updateNonManualProjectPlan(NonManualProjectPlanDTO dto) {
+        NonManualProjectPlan existPlan = nonManualProjectPlanDao.findOneById(dto.getId());
+        NonManualProjectPlan plan = new NonManualProjectPlan(dto);
+        plan.setCTime(existPlan.getCTime());
+        nonManualProjectPlanDao.updateOne(existPlan);
+        return ResponseResult.success();
+    }
+
+    @Override
+    public Integer getPlanDataPrediction(String keyword, String filterWord) {
+        JSONObject data = getSearchWholeOpinionJson(createSearchFilterDTO(keyword, filterWord));
+        JSONObject task = data.getJSONObject("task");
+        return task.getInteger("searchCount");
+    }
+
+    private SearchFilterDTO createSearchFilterDTO(String keyword, String filterWord){
+        SearchFilterDTO dto = new SearchFilterDTO();
+        dto.setSearch(keyword);
+        dto.setFilterWords(filterWord);
+        // 预估一天
+        dto.setEndTime(System.currentTimeMillis());
+        dto.setStartTime(dto.getEndTime() - Constant.ONE_DAY);
+        dto.setPage(1);
+        dto.setPageSize(20);
+        return dto;
+    }
+
+    @Override
+    public PageVO<JSONObject> getNonManualProjectPlanList() {
+        Query query = new Query(Criteria.where("projectId").is(UserThreadLocal.getProjectId()));
+        List<NonManualProjectPlan> planList = nonManualProjectPlanDao.findList(query);
+        long total = nonManualProjectPlanDao.count(query);
+        List<JSONObject> collect = planList.stream().map(plan -> {
+            JSONObject jsonObject = new JSONObject();
+            jsonObject.put("id", plan.getId());
+            jsonObject.put("name", plan.getName());
+            jsonObject.put("keyword", plan.getKeyword());
+            jsonObject.put("keywordCount", getWordCount(plan.getKeyword()));
+            jsonObject.put("filterWord", plan.getFilterWord());
+            jsonObject.put("filterWordCount", getWordCount(plan.getFilterWord()));
+            jsonObject.put("lasting", plan.getLasting());
+            jsonObject.put("endTime", plan.getEndTime());
+            jsonObject.put("isEnd", !plan.getLasting() && Objects.nonNull(plan.getEndTime()) && System.currentTimeMillis() > plan.getEndTime());
+            return jsonObject;
+        }).collect(Collectors.toList());
+        return PageVO.createPageVo(total, 1, 5, collect);
+    }
+
+    /**
+     * 获取关键词/去噪词数量
+     * @param word
+     * @return
+     */
+    private int getWordCount(String word){
+        if (Objects.isNull(word)){
+            return 0;
+        }
+        int count = 0;
+        for (String s : word.split("\\|")) {
+            count += s.split(" ").length;
+        }
+        return count;
+    }
+
+    @Override
+    public JSONObject getProjectPlanConsume() {
+        Calendar calendar = Calendar.getInstance();
+        calendar.add(Calendar.DAY_OF_YEAR, -1);
+        calendar.set(Calendar.HOUR_OF_DAY, 0);
+        calendar.set(Calendar.SECOND, 0);
+        calendar.set(Calendar.MINUTE, 0);
+        calendar.set(Calendar.MILLISECOND, 0);
+        long yesterdayStartTime = calendar.getTimeInMillis();
+        calendar.add(Calendar.DAY_OF_YEAR, 1);
+        long yesterdayEndTime = calendar.getTimeInMillis();
+        JSONObject jsonObject = new JSONObject();
+        try {
+            jsonObject.put("yesterdayCount", getProjectPlanConsume(yesterdayStartTime, yesterdayEndTime));
+            jsonObject.put("todayCount", getProjectPlanConsume(yesterdayEndTime, System.currentTimeMillis()));
+            jsonObject.put("dataBalance", projectService.getProjectById(UserThreadLocal.getProjectId()).getDataBalance());
+        }catch (Exception e){
+            log.error("获取昨日今日数据消耗量出错", e);
+        }
+        return jsonObject;
+    }
+
+    private long getProjectPlanConsume(Long startTime, Long endTime) throws IOException {
+        String projectId = UserThreadLocal.getProjectId();
+        BoolQueryBuilder query = projectContendIdQuery(projectId, Constant.PRIMARY_CONTEND_ID);
+        // stime
+        query.must(QueryBuilders.rangeQuery("stime").gte(startTime).lt(endTime));
+        return esClientDao.count(query);
+    }
+
+    @Override
+    public JSONObject getNonManualMarkCriteria() {
+        String projectId = UserThreadLocal.getProjectId();
+        String linkedGroupId = projectService.getProjectVOById(projectId).getBrandLinkedGroupId();
+        JSONObject result = new JSONObject();
+        // 搜索时间
+        result.put("times", Arrays.asList("今天", "24小时", "三天"));
+        // 平台
+        result.put("platformList", commonService.getQbjcPlatform("id", "name"));
+        // 情感标签
+        result.put("emotionList", commonService.getEmotionTagsWithSort(projectId, linkedGroupId).stream().map(markerTag -> {
+            JSONObject json = new JSONObject();
+            json.put("uniqueId", markerTag.getUniqueId());
+            json.put("name", markerTag.getName());
+            return json;
+        }).collect(Collectors.toList()));
+        // 行政级别
+        result.put("politicsLevels", ChannelType.channelOption.getJSONArray("politicsLevels"));
+        // 内容类型
+        result.put("forward", Arrays.asList("原创", "转发"));
+        // 数据类型
+        List<JSONObject> dataType = new ArrayList<>();
+        List<String> list = Arrays.asList("长文本", "短文本", "问答", "图片", "视频");
+        for (int i = 0; i < list.size(); i++) {
+            JSONObject jsonObject = new JSONObject();
+            jsonObject.put("id", i + 1);
+            jsonObject.put("name", list.get(i));
+            dataType.add(jsonObject);
+        }
+        result.put("dataType", dataType.stream().filter(jsonObject -> !jsonObject.getString("name").equals("图片")));
+        // 信息浏览
+        result.put("read", Arrays.asList("已读", "未读"));
+        // 其他标签组(筛选器)
+//        result.put("tagFilterList", tagFilterService.getTagFilter());
+        // 地域
+//        result.put("regionList", getFilterRegionList());
+        // 自定义标签列表
+        List<CustomTagVo> customTagList = customTagService.findCustomTagList(1, 1).getList();
+        if (CollectionUtils.isNotEmpty(customTagList)) {
+            result.put("customTag", customTagList.get(0).generateSearchSonTagMaps());
+        }
+        return result;
+    }
+
+    @Override
+    public PageVO<MarkFlowEntity> getNonManualMarkList(MarkSearchDTO markSearchDTO) {
+        try {
+            //设置默认的时间、页码、排序方式等
+            defaultMarkSearch(markSearchDTO);
+            // gid
+            String gid = redisUtil.get(RedisUtil.getNonManualProjectMarkMaxGidKey(UserThreadLocal.getProjectId(), markSearchDTO.getPlanId(), UserThreadLocal.getUserId()));
+            if (Objects.nonNull(gid)) {
+                markSearchDTO.setGid(Long.parseLong(gid));
+            }
+            // 搜索es数据
+            Pair<SearchHits[], Map<String, Long>> hitsAndCounts = esSearchService.searchNonManualHitsAndCount(markSearchDTO, false);
+            // 总量
+            long total = hitsAndCounts.getLeft()[0].getTotalHits().value;
+            // 消息列表
+            List<MarkFlowEntity> flowEntityList = getNonManualMarkFlowEntity(markSearchDTO, hitsAndCounts.getLeft()[0]);
+            JSONObject jsonObject = new JSONObject();
+            // 统计平台进量（声量）
+            if (1 == markSearchDTO.getPage()) {
+                jsonObject.put("counts", esSearchService.countNonManualPlatform(markSearchDTO));
+            }
+            // 获取最大的gid
+            jsonObject.put("maxGid", getMaxGid(markSearchDTO));
+            // 返回分页结果
+            return PageVO.createPageVo(total, markSearchDTO.getPage(), markSearchDTO.getPageSize(), flowEntityList).setInfo(jsonObject);
+        } catch (IOException e) {
+            ExceptionCast.cast(CommonCodeEnum.FAIL, "es查询异常");
+        }
+        return null;
+    }
+
+    @Override
+    public List<LineVO> getTimeDataConsumeGraph(String planId, Long startTime, Long endTime) {
+        try {
+            return getDataConsumeGraph(planId, startTime, endTime);
+        }catch (Exception e){
+            log.error("获取时间数据消耗量图谱出错", e);
+            return Collections.singletonList(new LineVO(0L, 0L));
+        }
+    }
+
+    @Override
+    public List<LineVO> getTotalDataConsumeGraph(Long startTime, Long endTime) {
+        try {
+            return getDataConsumeGraph(null, startTime, endTime);
+        }catch (Exception e){
+            log.error("获取总数据消耗量图谱出错", e);
+            return Collections.singletonList(new LineVO(0L, 0L));
+        }
+    }
+
+    private LineVO getDataConsumeGraph2(String planId, Long startTime, Long endTime) throws IOException {
+        // project plan query
+        String projectId = UserThreadLocal.getProjectId();
+        BoolQueryBuilder query = EsQueryTools.assembleCacheMapsPlanQuery(projectId, planId);
+        if (Objects.isNull(planId)){
+            query = projectContendIdQuery(projectId, Constant.PRIMARY_CONTEND_ID);
+        }
+        // stime
+        query.must(QueryBuilders.rangeQuery("stime").gte(startTime).lt(endTime));
+        return new LineVO(esClientDao.count(query), startTime);
+    }
+
+    /**
+     * 获取非人工项目时间段内舆情数据图
+     * @param planId
+     * @param startTime
+     * @param endTime
+     * @return
+     * @throws IOException
+     */
+    private List<LineVO> getDataConsumeGraph(String planId, Long startTime, Long endTime) throws IOException{
+        List<LineVO> res = new ArrayList<>();
+        String[] indexes = esClientDao.getIndexes();
+        DateHistogramAggregationBuilder daysAggregationBuilder;
+        // 天级以小时为颗粒度，周、月级以天作为颗粒度
+        if (endTime - startTime <= Constant.ONE_DAY){
+            daysAggregationBuilder = AggregationBuilders.dateHistogram("timeAgg").field("stime").calendarInterval(DateHistogramInterval.HOUR);
+        }else {
+            daysAggregationBuilder = AggregationBuilders.dateHistogram("timeAgg").field("stime").calendarInterval(DateHistogramInterval.DAY);
+        }
+        String projectId = UserThreadLocal.getProjectId();
+        // project plan query
+        BoolQueryBuilder query = EsQueryTools.assembleCacheMapsPlanQuery(projectId, planId);
+        if (Objects.isNull(planId)){
+            query = projectContendIdQuery(projectId, Constant.PRIMARY_CONTEND_ID);
+        }
+        // stime
+        query.must(QueryBuilders.rangeQuery("stime").gte(startTime).lt(endTime));
+        // response
+        SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, query, daysAggregationBuilder, null, null, 0, 0, null);
+        Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
+        ParsedDateHistogram teamAgg = (ParsedDateHistogram) aggMap.get("timeAgg");
+        for (Histogram.Bucket bucket : teamAgg.getBuckets()) {
+            long time = Long.parseLong(bucket.getKeyAsString());
+            long count = bucket.getDocCount();
+            res.add(new LineVO(count, time));
+        }
+        return CompleteLine(res, startTime, endTime);
+    }
+
+    /**
+     * 补全图谱
+     * @param lines
+     * @param startTime
+     * @param endTime
+     * @return
+     */
+    private List<LineVO> CompleteLine(List<LineVO> lines, Long startTime, Long endTime){
+        int size = endTime - startTime > 7 * Constant.ONE_DAY ? 30 : (endTime - startTime <= Constant.ONE_DAY ? 24 : 7);
+        long timeRange = 7 == size || 30 == size ? Constant.ONE_DAY : Constant.ONE_HOUR;
+        if (lines.size() < size){
+            // 无数据，按时间段全部补为0
+            if (CollectionUtils.isEmpty(lines)){
+                for (int i = 0; i < size; i++) {
+                    lines.add(new LineVO(0L, startTime));
+                    startTime = startTime + timeRange;
+                    if (startTime >= endTime || lines.size() >= size){
+                        break;
+                    }
+                }
+                return lines;
+            }else {
+                // 往前补0
+                LineVO startLine = lines.get(0);
+                Long startLineTime = startLine.getDate();
+                for (int i = 0; i < size; i++) {
+                    startLineTime = startLineTime - timeRange;
+                    if (startLineTime <= startTime || lines.size() >= size){
+                        break;
+                    }
+                    lines.add(0, new LineVO(0L, startLineTime));
+                }
+                // 往后补0
+                LineVO endLine = lines.get(lines.size() - 1);
+                Long endLineTime = endLine.getDate();
+                for (int i = 0; i < size; i++) {
+                    endLineTime = endLineTime + timeRange;
+                    if (endLineTime >= endTime || lines.size() >= size){
+                        break;
+                    }
+                    lines.add(new LineVO(0L, endLineTime));
+                }
+            }
+        }
+        return lines;
+    }
+
+    @Override
+    public ResponseResult nonManualMarkInteractionUpdate(MarkSearchDTO dto) {
+        List<BaseMap> list = new ArrayList<>();
+        dto.setProjectId(UserThreadLocal.getProjectId());
+        try {
+            SearchHits[] searchHits = esSearchService.searchNonManualHitsAndCount(dto, true).getLeft();
+            for (SearchHits searchHit : searchHits) {
+                List<BaseMap> baseMaps = Arrays.stream(searchHit.getHits()).map(SearchHit::getSourceAsMap).map(Tools::getBaseFromEsMap).collect(Collectors.toList());
+                list.addAll(baseMaps);
+            }
+        } catch (IOException e) {
+            ExceptionCast.cast(CommonCodeEnum.FAIL, "es查询异常");
+        }
+        // 互动量更新剩余可用次数
+        int interactionBalance = projectService.getProjectById(UserThreadLocal.getProjectId()).getInteractionBalance();
+        if (interactionBalance - list.size() < 0) {
+            return ResponseResult.failure("剩余可用次数不足");
+        }
+        List<ExportYuqingInteractionUpdateDTO> exportList = markInteractionUpdate(list);
         // excel输出到指定路径
         String projectName = projectService.getProjectById(UserThreadLocal.getProjectId()).getProjectName();
         String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), "舆情库互动量更新结果");
-        EasyExcelUtil.write(filePath, "sheet1", ExportInteractionUpdateDTO.class, exportList);
+        EasyExcelUtil.write(filePath, "sheet1", ExportYuqingInteractionUpdateDTO.class, exportList);
         JSONObject res = new JSONObject();
         // 记录使用情况
         extraService.decreaseInteractionRecord(InteractionUpdateRecord.UsedType.yuqing, list.size());
         res.put("filePath", filePath);
         res.put("remainingCount", interactionBalance - list.size());
-        // 记录使用
         return ResponseResult.success(res);
     }
 
     @Override
-    public Long interactionUpdatePrediction(MarkSearchDTO dto) {
+    public Long nonManualInteractionUpdatePrediction(MarkSearchDTO dto) {
         try {
-            defaultMarkSearch(dto);
-            dto.setPageSize(null);
-            Pair<SearchHits[], Map<String, Long>> hitsAndCounts = esSearchService.searchMarkHitsAndCount(dto, false);
-            return hitsAndCounts.getLeft()[0].getTotalHits().value;
-        }catch (Exception e){
+            String projectId = UserThreadLocal.getProjectId();
+            String planId = dto.getPlanId();
+            BoolQueryBuilder postFilter = EsQueryTools.assembleCacheMapsPlanQuery(projectId, planId);
+            // time
+            postFilter.must(QueryBuilders.rangeQuery(dto.getTimeType()).gte(dto.getStartTime()).lt(dto.getEndTime()));
+            // platform
+            if (CollectionUtils.isNotEmpty(dto.getPlatforms())) {
+                postFilter.must(EsQueryTools.assemblePlatformQuery(Tools.getPlatformByIds(dto.getPlatforms())));
+            }
+            return esClientDao.count(postFilter);
+        } catch (Exception e) {
             ExceptionCast.cast(CommonCodeEnum.FAIL, "es查询异常");
         }
         return null;
     }
 
+    @Override
+    public Pair<String, List<ExportAppYuqingDTO>> downloadNonManualMarkList(MarkSearchDTO markSearchDTO) {
+        return downloadList(markSearchDTO, esSearchService::searchNonManualHitsAndCount);
+    }
+
+    @Override
+    public void adjustMaxGid(String planId, Long gid) {
+        if (Objects.isNull(gid) || -1 == gid){
+            return;
+        }
+        String key = RedisUtil.getNonManualProjectMarkMaxGidKey(UserThreadLocal.getProjectId(), planId, UserThreadLocal.getUserId());
+        redisUtil.set(key, String.valueOf(gid));
+    }
+
+    @Override
+    public String generateNonManualMarkAggreeList(String planId, Long startTime, Long endTime) {
+        return generateAggreeList(UserThreadLocal.getProjectId(), planId, startTime, endTime, esSearchService::searchNonManualHitsAndCount);
+    }
+
+    @Override
+    public PageVO<MarkFlowEntity> getNonManualMarkAggreeList(MarkSearchDTO dto) {
+        try {
+            PageVO<MarkFlowEntity> aggreeList = getAggreeList(dto, markFlowService::createNonManualMarkFlowInfo, false);
+            JSONObject jsonObject = new JSONObject();
+            // 统计平台进量（声量）
+            if (1 == dto.getPage()) {
+                jsonObject.put("counts", esSearchService.countNonManualPlatform(dto));
+            }
+            // 获取最大的gid
+            jsonObject.put("maxGid", getMaxGid(dto));
+            aggreeList.setInfo(jsonObject);
+            return aggreeList;
+        }catch (Exception e){
+            log.error("获取非人工项目舆情列表聚合结果出错", e);
+            return new PageVO<>();
+        }
+    }
+
+    @Override
+    public List<JSONObject> getNonManualMarkCountList() {
+        try {
+            List<JSONObject> res = new ArrayList<>();
+            String projectId = UserThreadLocal.getProjectId();
+            List<NonManualProjectPlan> planList = nonManualProjectPlanDao.findList(new Query(Criteria.where("projectId").is(projectId)));
+            for (NonManualProjectPlan plan : planList) {
+                JSONObject jsonObject = new JSONObject();
+                jsonObject.put("id", plan.getId());
+                jsonObject.put("name", plan.getName());
+                jsonObject.put("count", esClientDao.count(EsQueryTools.assembleCacheMapsPlanQuery(projectId, plan.getId())));
+                res.add(jsonObject);
+            }
+            return res;
+        }catch (Exception e){
+            log.error("获取非人工项目舆情列表数据量统计出错", e);
+            return Collections.emptyList();
+        }
+    }
+
+    /**
+     * 获取最大的gid
+     * @param dto
+     * @return
+     * @throws IOException
+     */
+    private Long getMaxGid(MarkSearchDTO dto) throws IOException {
+        EsClientDao.SearchHelper helper = EsClientDao.createSearchHelper();
+        BoolQueryBuilder postFilter = EsQueryTools.assembleCacheMapsPlanQuery(dto.getProjectId(), dto.getPlanId());
+        helper.setPostFilter(postFilter);
+        helper.setSort(SortBuilders.fieldSort(GenericAttribute.ES_GID).order(SortOrder.DESC));
+        helper.setFrom(null);
+        helper.setSize(1);
+        SearchResponse response = esClientDao.searchResponse(helper);
+        SearchHits hits = response.getHits();
+        if (0 == hits.getTotalHits().value) {
+            return -1L;
+        }
+        return Long.parseLong(hits.getAt(0).getSourceAsMap().get(GenericAttribute.ES_GID) + "");
+    }
+
+    private String generateAggreeList(String projectId, String planId, Long startTime, Long endTime, ArticleFunction.SearchHitsAndCount searchHitsAndCount) {
+        String uuid = Tools.getUUID();
+        MarkSearchDTO markSearchDTO = defaultMarkSearch4Aggree(startTime, endTime);
+        markSearchDTO.setPlanId(planId);
+        ApplicationProjectListener.getThreadPool().execute(() -> {
+            String redisKey = RedisUtil.getAggreeCacheKey(uuid, projectId);
+            try {
+                redisUtil.setExpire(redisKey, "0");
+                log.info("taskId:{},聚合任务开始", uuid);
+                // 搜索es数据
+                Pair<SearchHits[], Map<String, Long>> hitsAndCounts = searchHitsAndCount.search(markSearchDTO, true);
+                log.info("taskId:{},聚合前数据:{}条", uuid, hitsAndCounts.getLeft()[0].getTotalHits().value);
+                // 聚合分组后数据
+                List<List<JSONObject>> groupList = aggregation(hitsAndCounts.getLeft());
+                log.info("taskId:{},聚合完成,聚合组:{}个", uuid, groupList.size());
+                // 入mongo缓存
+                cacheAggreeResult(groupList, uuid, projectId);
+                redisUtil.setExpire(redisKey, "100");
+            } catch (Exception e) {
+                log.error("generateAggreeList-", e);
+                redisUtil.remove(redisKey);
+            }
+        });
+        return uuid;
+    }
+
     private static JSONObject getTitleAndUrl(Map<String, Object> map) {
         JSONObject json = new JSONObject();
         BaseMap baseMap = Tools.getBaseFromEsMap(map);
@@ -1868,6 +2464,37 @@ public class MarkDataServiceImpl implements MarkDataService {
         return textList;
     }
 
+    private List<MarkFlowEntity> getNonManualMarkFlowEntity(MarkSearchDTO markSearchDTO, SearchHits searchHits) {
+        String projectId = markSearchDTO.getProjectId();
+        String contendId = markSearchDTO.getContendId();
+        String planId = markSearchDTO.getPlanId();
+        // 重复消息折叠
+        if (markSearchDTO.isFold()) {
+            // 根据旧顺序排序
+            List<String> sortList = Arrays.stream(searchHits.getHits()).map(hit -> hit.getSourceAsMap().get("id") + "").collect(Collectors.toList());
+            Map<String, List<Map<String, Object>>> groupMap = Arrays.stream(searchHits.getHits()).map(SearchHit::getSourceAsMap).collect(Collectors.groupingBy(map -> {
+                BaseMap baseMap = Tools.getBaseFromEsMap(map);
+                String title = baseMap.getTitleNullOptionalContent();
+                return Tools.filterSpecialCharacter(title) + baseMap.getTypeB().encode();
+            }));
+            Map<String, List<Map<String, Object>>> collect = groupMap.values().stream().map(list -> list.stream().sorted(Comparator.comparingLong(o -> Long.parseLong(o.get("time") + ""))).collect(Collectors.toList())).collect(Collectors.toMap(list -> list.get(0).get("id") + "", list -> list));
+            return sortList.stream().map(id -> {
+                List<Map<String, Object>> maps = collect.get(id);
+                if (null != maps) {
+                    return MarkFlowEntity.getFoldInstance(maps.stream().map(map -> getNonManualMarkFlowEntity(map, projectId, contendId, planId)).collect(Collectors.toList()));
+                }
+                return null;
+            }).filter(Objects::nonNull).collect(Collectors.toList());
+        }
+        return Arrays.stream(searchHits.getHits()).map(hit -> getNonManualMarkFlowEntity(hit.getSourceAsMap(), projectId, contendId, planId)).collect(Collectors.toList());
+    }
+
+    private MarkFlowEntity getNonManualMarkFlowEntity(Map<String, Object> map, String projectId, String contendId, String planId) {
+        MarkFlowEntity instance = new MarkFlowEntity(new JSONObject(map));
+        instance.setInfo(markFlowService.createNonManualMarkFlowInfo(new MarkFlowService.MarkInfoSource(new JSONObject(map), projectId, contendId, planId)));
+        return instance;
+    }
+
     private List<MarkFlowEntity> getMarkFlowEntity(MarkSearchDTO markSearchDTO, SearchHits searchHits) {
         String projectId = markSearchDTO.getProjectId();
         String contendId = markSearchDTO.getContendId();
@@ -1978,7 +2605,7 @@ public class MarkDataServiceImpl implements MarkDataService {
         for (AggreeResult aggreeResult : fatherList) {
             List<MarkFlowEntity> markFlowEntities = new ArrayList<>();
             MarkFlowEntity instance = new MarkFlowEntity(aggreeResult.getData(), dto.getLinkedGroupId());
-            instance.setInfo(createMarkFlowInfo.create(new MarkFlowService.MarkInfoSource(aggreeResult.getData(), dto.getProjectId(), dto.getContendId(), dto.getLinkedGroupId())));
+            instance.setInfo(createMarkFlowInfo.create(new MarkFlowService.MarkInfoSource(aggreeResult.getData(), dto.getProjectId(), dto.getContendId(), dto.getPlanId(), dto.getLinkedGroupId())));
             markFlowEntities.add(instance);
             // 添加父标题集下的子标题集
             resList.add(MarkFlowEntity.getAggreeInstance(markFlowEntities, aggreeResult, sonDetails));

src/main/java/com/zhiwei/brandkbs2/service/impl/MarkFlowServiceImpl.java

@@ -80,6 +80,20 @@ public class MarkFlowServiceImpl implements MarkFlowService {
         return resultInfo;
     }
 
+    @Override
+    public JSONObject createNonManualMarkFlowInfo(MarkInfoSource markInfoSource) {
+        JSONObject json = markInfoSource.getJson();
+        JSONObject resultInfo = createInfoWithEmotion(json);
+        resultInfo.put("sourceDetails", getSourceDetails(markInfoSource));
+        String zip = Tools.gzip(JSON.toJSONString(json));
+        resultInfo.put("zip", zip);
+        // 每条数据高亮词
+        resultInfo.put("highLightWord", getHighLightWords(json, markInfoSource.getProjectId(), markInfoSource.getPlanId()));
+        // 词频
+        resultInfo.put("keywordCount", getKeywordCount(json, markInfoSource.getProjectId(), markInfoSource.getPlanId()));
+        return resultInfo;
+    }
+
     /**
      * 消息实体批量处理标注信息
      *
@@ -193,6 +207,49 @@ public class MarkFlowServiceImpl implements MarkFlowService {
         return info;
     }
 
+    private Map<String, Integer> getKeywordCount(JSONObject json, String projectId, String planId){
+        List<String> keywords = getKeywords(json, projectId, planId);
+        Map<String, Integer> res = new HashMap<>();
+        for (String keyword : keywords) {
+            if (Objects.isNull(keyword)){
+                continue;
+            }
+            String[] keywordAndFrequency = keyword.split(":");
+            String word = keywordAndFrequency[0];
+            Integer frequency = Integer.valueOf(keywordAndFrequency[1]);
+            res.put(word, frequency);
+        }
+        return res;
+    }
+
+    private List<String> getHighLightWords(JSONObject json, String projectId, String planId){
+        List<String> keywords = getKeywords(json, projectId, planId);
+        List<String> res = new ArrayList<>();
+        for (String keyword : keywords) {
+            if (Objects.isNull(keyword)){
+                continue;
+            }
+            String[] keywordAndFrequency = keyword.split(":");
+            String words = keywordAndFrequency[0];
+            if (Objects.nonNull(words)){
+                String[] word = words.split(" ");
+                res.addAll(new ArrayList<>(Arrays.asList(word)));
+            }
+        }
+        return res.stream().distinct().collect(Collectors.toList());
+    }
+
+    private List<String> getKeywords(JSONObject json, String projectId, String planId){
+        List<String> keywords = new ArrayList<>();
+        List<JSONObject> brandkbsCacheMaps = json.getJSONArray("brandkbs_cache_maps").toJavaList(JSONObject.class);
+        for (JSONObject brandkbsCacheMap : brandkbsCacheMaps) {
+            if (Objects.equals(brandkbsCacheMap.getString("plan_key"), Tools.concat(projectId, planId))){
+                keywords.add(brandkbsCacheMap.getString("keyword"));
+            }
+        }
+        return keywords;
+    }
+
     private JSONObject getSourceDetails(MarkInfoSource markInfoSource) {
         JSONObject sourceDetails = new JSONObject();
         JSONObject tJson = markInfoSource.getJson();

src/main/java/com/zhiwei/brandkbs2/service/impl/ProjectServiceImpl.java

@@ -10,10 +10,7 @@ import com.zhiwei.brandkbs2.enmus.response.ProjectCodeEnum;
 import com.zhiwei.brandkbs2.exception.ExceptionCast;
 import com.zhiwei.brandkbs2.model.CommonCodeEnum;
 import com.zhiwei.brandkbs2.model.ResponseResult;
-import com.zhiwei.brandkbs2.pojo.AbstractProject;
-import com.zhiwei.brandkbs2.pojo.Contend;
-import com.zhiwei.brandkbs2.pojo.Project;
-import com.zhiwei.brandkbs2.pojo.User;
+import com.zhiwei.brandkbs2.pojo.*;
 import com.zhiwei.brandkbs2.pojo.vo.PageVO;
 import com.zhiwei.brandkbs2.pojo.vo.ProjectVO;
 import com.zhiwei.brandkbs2.service.ProjectService;
@@ -77,6 +74,12 @@ public class ProjectServiceImpl implements ProjectService {
     @Resource(name = "BytedanceCustomPlatformWeightDao")
     private BytedanceCustomPlatformWeightDao bytedanceCustomPlatformWeightDao;
 
+    @Resource(name = "nonManualProjectDataUpdateRecordDao")
+    private NonManualProjectDataUpdateRecordDao nonManualProjectDataUpdateRecordDao;
+
+    @Resource(name = "nonManualProjectPlanDao")
+    NonManualProjectPlanDao nonManualProjectPlanDao;
+
     @Value("${jwt.key}")
     private String jwtKey;
 
@@ -88,13 +91,13 @@ public class ProjectServiceImpl implements ProjectService {
 
     @Override
     public ResponseResult addProject(ProjectVO projectVO) {
-        if (!Tools.isEmpty(projectVO.getHitKeywords()) && projectVO.getModuleShowList().contains(2)) {
-            return ResponseResult.failure("关键字和原始数据配置异常");
-        }
         if (Objects.isNull(projectVO)) {
             //抛出非法参数异常
             ExceptionCast.cast(CommonCodeEnum.INVALID_PARAM);
         }
+        if (!Tools.isEmpty(projectVO.getHitKeywords()) && projectVO.getModuleShowList().contains(2)) {
+            return ResponseResult.failure("关键字和原始数据配置异常");
+        }
         Project existsProject = projectDao.findOne("projectName", projectVO.getProjectName());
         if (Objects.nonNull(existsProject)) {
             //抛出项目名称已存在异常
@@ -127,6 +130,7 @@ public class ProjectServiceImpl implements ProjectService {
     @Override
     public PageVO<JSONObject> findProjectList(int page, int size, String keyword) {
         Query query = new Query();
+        query.addCriteria(Criteria.where("isManual").is(true));
         projectDao.addKeywordFuzz(query, keyword, "projectName");
         long count = projectDao.count(query);
         projectDao.addSort(query, "{\"cTime\":\"descend\"}");
@@ -238,6 +242,12 @@ public class ProjectServiceImpl implements ProjectService {
 //        });
         //删除项目
         projectDao.deleteOneById(pid);
+        // 删除非人工项目维护记录和方案
+        if (!project.isManual()){
+            Query query = new Query(Criteria.where("projectId").is(pid));
+            nonManualProjectPlanDao.deleteOneByQuery(query);
+            nonManualProjectDataUpdateRecordDao.deleteOneByQuery(query);
+        }
     }
 
     @Override
@@ -443,27 +453,27 @@ public class ProjectServiceImpl implements ProjectService {
     @Override
     public PageVO<JSONObject> getToolsetBytedanceCustomInfo(int page, int size, String type) {
         Query query = new Query();
+        long channelTotal = bytedanceCustomChannelDao.count(query);
+        long channelWeightTotal = bytedanceCustomPlatformWeightDao.count(query);
+        long channelInfluenceTotal = bytedanceCustomChannelInfluenceDao.count(query);
         query.limit(size);
         query.skip((long) (page - 1) * size);
         switch (type){
             case "重要媒体":
-                long total = bytedanceCustomChannelDao.count(query);
-                return PageVO.createPageVo(total, page, size, bytedanceCustomChannelDao.findList(query).stream().map(data ->{
+                return PageVO.createPageVo(channelTotal, page, size, bytedanceCustomChannelDao.findList(query).stream().map(data ->{
                     JSONObject jsonObject = new JSONObject();
                     jsonObject.put("channel", data.getChannel());
                     return jsonObject;
                 }).collect(Collectors.toList()));
             case "平台权重":
-                long total2 = bytedanceCustomPlatformWeightDao.count(query);
-                return PageVO.createPageVo(total2, page, size, bytedanceCustomPlatformWeightDao.findList(query).stream().map(data ->{
+                return PageVO.createPageVo(channelWeightTotal, page, size, bytedanceCustomPlatformWeightDao.findList(query).stream().map(data ->{
                     JSONObject jsonObject = new JSONObject();
                     jsonObject.put("type", data.getType());
                     jsonObject.put("weight", data.getWeight());
                     return jsonObject;
                 }).collect(Collectors.toList()));
             case "渠道影响力":
-                long total3 = bytedanceCustomChannelInfluenceDao.count(query);
-                return PageVO.createPageVo(total3, page, size, bytedanceCustomChannelInfluenceDao.findList(query).stream().map(data ->{
+                return PageVO.createPageVo(channelInfluenceTotal, page, size, bytedanceCustomChannelInfluenceDao.findList(query).stream().map(data ->{
                     JSONObject jsonObject = new JSONObject();
                     jsonObject.put("platform", data.getPlatform());
                     jsonObject.put("channel", data.getChannel());
@@ -475,6 +485,106 @@ public class ProjectServiceImpl implements ProjectService {
         }
     }
 
+    @Override
+    public ResponseResult addNonManualProject(ProjectVO projectVO) {
+        if (Objects.isNull(projectVO)) {
+            //抛出非法参数异常
+            ExceptionCast.cast(CommonCodeEnum.INVALID_PARAM);
+        }
+        Project existsProject = projectDao.findOne("projectName", projectVO.getProjectName());
+        if (Objects.nonNull(existsProject)) {
+            //抛出项目名称已存在异常
+            return ResponseResult.failure(ProjectCodeEnum.PROJECT_EXISTSNAME_ERROR.message());
+        }
+        Project project = ProjectVO.createNonManualProject(projectVO);
+        projectDao.insertOneWithoutId(project);
+        return ResponseResult.success();
+    }
+
+    @Override
+    public ResponseResult updateNonManualProject(ProjectVO projectVO) {
+        Project existsProject = projectDao.findOneById(projectVO.getId());
+        if (Objects.isNull(existsProject)) {
+            //抛出非法参数异常
+            ExceptionCast.cast(CommonCodeEnum.INVALID_PARAM);
+        }
+        Project existsNameProject = projectDao.findOne("projectName", projectVO.getProjectName());
+        //如果该项目的名字已存在且不是本身
+        if (Objects.nonNull(existsNameProject) && !projectVO.getId().equals(existsNameProject.getId())) {
+            //抛出项目名已存在异常
+            ExceptionCast.cast(ProjectCodeEnum.PROJECT_EXISTSNAME_ERROR);
+        }
+        Project project = ProjectVO.createNonManualProject(projectVO);
+        project.setCTime(existsProject.getCTime());
+        project.setShow(existsProject.isShow());
+        project.setDataBalance(existsProject.getDataBalance());
+        project.setStart(existsProject.isStart());
+        projectDao.updateOne(project);
+        return ResponseResult.success();
+    }
+
+    @Override
+    public PageVO<JSONObject> findNonManualProjectList(int page, int size, String sorter, String keyword) {
+        Query query = new Query();
+        query.addCriteria(Criteria.where("isManual").is(false));
+        projectDao.addKeywordFuzz(query, keyword, "projectName");
+        long count = projectDao.count(query);
+        projectDao.addSort(query, sorter);
+        mongoUtil.start(page, size, query);
+        List<Project> projectList = projectDao.findList(query);
+        List<JSONObject> resList = projectList.stream().map(project -> {
+            JSONObject result = new JSONObject();
+            result.put("id", project.getId());
+            result.put("importTime", project.getImportTime());
+            result.put("cTime", project.getCTime());
+            result.put("projectName", project.getProjectName());
+            result.put("brandName", project.getBrandName());
+            result.put("isStart", project.isStart());
+            result.put("isShow", project.isShow());
+            result.put("dataBalance", project.getDataBalance());
+            return result;
+        }).collect(Collectors.toList());
+        MongoUtil.PageHelper<JSONObject> pageHelper = mongoUtil.pageHelper(count, resList);
+        return PageVO.createPageVo(pageHelper, resList);
+    }
+
+    @Override
+    public PageVO<JSONObject> findNonManualProjectMaintainList(String projectId, int page, int pageSize) {
+        Query query = Query.query(Criteria.where("projectId").is(projectId));
+        nonManualProjectDataUpdateRecordDao.addSort(query, "{\"cTime\":\"descend\"}");
+        long total = nonManualProjectDataUpdateRecordDao.count(query);
+        mongoUtil.start(page, pageSize, query);
+        List<JSONObject> collect = nonManualProjectDataUpdateRecordDao.findList(query).stream().map(record -> {
+            JSONObject json = new JSONObject();
+            json.put("lastCount", record.getLastCount());
+            json.put("currentCount", record.getCurrentCount());
+            json.put("cTime", record.getCTime().getTime());
+            json.put("submitter", record.getSubmitter());
+            return json;
+        }).collect(Collectors.toList());
+        return PageVO.createPageVo(total, page, pageSize, collect);
+    }
+
+    @Override
+    public void adjustNonManualProjectDataBalance(String projectId, Integer currentCount) {
+        if (null == currentCount) {
+            return;
+        }
+        Update update = Update.update("dataBalance", currentCount);
+        Project project = projectDao.findOneById(projectId);
+        projectDao.updateOneByIdWithField(projectId, update);
+        NonManualProjectDataUpdateRecord record = new NonManualProjectDataUpdateRecord(projectId, project.getDataBalance(), currentCount, UserThreadLocal.getNickname());
+        nonManualProjectDataUpdateRecordDao.insertOne(record);
+    }
+
+    @Override
+    public JSONObject nonManualProjectOverview(String projectId) {
+        JSONObject json = new JSONObject();
+        json.put("balance", projectDao.findOneById(projectId).getDataBalance());
+        json.put("trendList", nonManualProjectDataUpdateRecordDao.aggregateDayLastRecord(projectId));
+        return json;
+    }
+
     /**
      * 获取舆情对应项目的情感标签
      * @param brandName
@@ -512,6 +622,7 @@ public class ProjectServiceImpl implements ProjectService {
             json.put("group", project.getBrandLinkedGroup());
             json.put("avatarUrl", project.getAvatarUrl());
             json.put("permission", getPermissionList(project));
+            json.put("isManual", project.isManual());
             Date expiredTime = roleExpired.get(project.getId());
             if (null == expiredTime || expiredTime.after(new Date())) {
                 json.put("expired", false);
@@ -549,7 +660,7 @@ public class ProjectServiceImpl implements ProjectService {
     private List<Map<String, Object>> getPermissionList(Project project) {
         List<Map<String, Object>> permissionList = new ArrayList<>();
         Map<String, Object> originPermission = new HashMap<>();
-        originPermission.put("origin", project.getModuleShowList().contains(2));
+        originPermission.put("origin", Objects.nonNull(project.getModuleShowList()) && project.getModuleShowList().contains(2));
         permissionList.add(originPermission);
         return permissionList;
     }

src/main/java/com/zhiwei/brandkbs2/service/impl/ToolsetServiceImpl.java

@@ -5,10 +5,7 @@ import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.brandkbs2.auth.UserThreadLocal;
 import com.zhiwei.brandkbs2.common.GenericAttribute;
 import com.zhiwei.brandkbs2.controller.app.AppToolsetController;
-import com.zhiwei.brandkbs2.dao.BytedanceCustomChannelDao;
-import com.zhiwei.brandkbs2.dao.BytedanceCustomChannelInfluenceDao;
-import com.zhiwei.brandkbs2.dao.BytedanceCustomEventUpdateTaskDao;
-import com.zhiwei.brandkbs2.dao.BytedanceCustomPlatformWeightDao;
+import com.zhiwei.brandkbs2.dao.*;
 import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
 import com.zhiwei.brandkbs2.easyexcel.config.ReadExcelDTO;
 import com.zhiwei.brandkbs2.easyexcel.dto.*;
@@ -79,9 +76,6 @@ public class ToolsetServiceImpl implements ToolsetService {
     @Resource(name = "projectServiceImpl")
     private ProjectService projectService;
 
-    @Resource(name = "markDataServiceImpl")
-    MarkDataService markDataService;
-
     @Autowired
     private RestTemplate restTemplate;
 
@@ -118,6 +112,9 @@ public class ToolsetServiceImpl implements ToolsetService {
     @Resource(name = "bytedanceCustomEventUpdateTaskDao")
     private BytedanceCustomEventUpdateTaskDao bytedanceCustomEventUpdateTaskDao;
 
+    @Resource(name = "bytedanceCustomEventUpdateTaskDataDao")
+    private BytedanceCustomEventUpdateTaskDataDao bytedanceCustomEventUpdateTaskDataDao;
+
     @Resource(name = "esClientDao")
     private EsClientDao esClientDao;
 
@@ -235,8 +232,8 @@ public class ToolsetServiceImpl implements ToolsetService {
         readExcel.setClazz(UploadInteractionUpdateDTO.class);
         readExcel.setAnalysisEventListener(new InteractionUpdateListener(urls));
         EasyExcelUtil.read(file, readExcel);
-        if (urls.size() > 300){
-            return ResponseResult.failure("超过每次更新链接数量上限300");
+        if (urls.size() > 500){
+            return ResponseResult.failure("超过每次更新链接数量上限500");
         }
         // 互动量更新剩余可用次数
         int interactionBalance = projectService.getProjectById(UserThreadLocal.getProjectId()).getInteractionBalance();
@@ -267,6 +264,10 @@ public class ToolsetServiceImpl implements ToolsetService {
      * @return
      */
     private List<JSONObject> urlInteractionUpdate(List<String> urls){
+        return urlInteractionUpdate(urls, null);
+    }
+
+    private List<JSONObject> urlInteractionUpdate(List<String> urls, String taskId){
         // 通过url获取域名进而获取任务类型
         Map<String, String> map = new HashMap<>();
         urls.forEach(url -> map.compute(url, (key, value) -> {
@@ -290,6 +291,10 @@ public class ToolsetServiceImpl implements ToolsetService {
             for (List<String> ids : ListUtils.partition(taskIdList, 50)) {
                 result.addAll(getInteractionResult(ids));
             }
+            if (Objects.nonNull(taskId)) {
+                // 更新处理进度
+                addProcessSchedule(taskId, taskIdList.size());
+            }
         }
         return result;
     }
@@ -332,13 +337,18 @@ public class ToolsetServiceImpl implements ToolsetService {
             try {
                 ResponseEntity<JSONObject> response = restTemplate.postForEntity(interactionResultUrl, request, JSONObject.class);
                 JSONObject body = response.getBody();
-                if (Objects.nonNull(body) && 200 == body.getIntValue("code") && !body.isEmpty() && !body.getJSONArray("data").toJavaList(JSONObject.class).isEmpty() &&
-                        body.getJSONArray("data").toJavaList(JSONObject.class).stream().map(m -> m.getIntValue("code")).allMatch(code -> 200 == code)) {
+                if (Objects.nonNull(body) && !body.isEmpty() && 200 == body.getIntValue("code") && CollectionUtils.isNotEmpty(body.getJSONArray("data").toJavaList(JSONObject.class)) &&
+                        body.getJSONArray("data").toJavaList(JSONObject.class).stream().map(m -> m.getIntValue("code")).allMatch(code -> 200 == code || 403 == code) &&
+                        taskIds.size() == body.getJSONArray("data").toJavaList(JSONObject.class).size()) {
                     res = body.getJSONArray("data").toJavaList(JSONObject.class);
                     break;
                 } else {
                     Thread.sleep(10000L);
                 }
+                // 最后一轮即使不满足条件也该返回
+                if (29 == i && Objects.nonNull(body) && !body.isEmpty() && 200 == body.getIntValue("code") && CollectionUtils.isNotEmpty(body.getJSONArray("data").toJavaList(JSONObject.class))){
+                    res = body.getJSONArray("data").toJavaList(JSONObject.class);
+                }
             }catch (Exception e){
                 ExceptionCast.cast(CommonCodeEnum.FAIL, "轮询互动量更新任务异常,taskId:" + taskIds, e);
             }
@@ -358,14 +368,16 @@ public class ToolsetServiceImpl implements ToolsetService {
     public void primaryEventUpdate(MultipartFile file, Boolean extraCompute) {
         BytedanceCustomEventUpdateTask task = BytedanceCustomEventUpdateTask.createTaskPrimary(extraCompute);
         bytedanceCustomEventUpdateTaskDao.insertOne(task);
-        Map<String, List<UploadBytedanceEventDTO>> map = new HashMap<>();
+
+        List<BytedanceCustomEventUpdateTaskData> data = new ArrayList<>();
         // excel信息提取
         ReadExcelDTO<UploadBytedanceEventDTO> readExcel = new ReadExcelDTO<>();
         readExcel.setClazz(UploadBytedanceEventDTO.class);
-        readExcel.setAnalysisEventListener(new BytedanceEventListener(map));
+        readExcel.setAnalysisEventListener(new BytedanceEventListener(task.getId(), data));
         EasyExcelUtil.read(file, readExcel);
-        task.setEventData(map);
-        ApplicationProjectListener.getThreadPool().execute(() -> processEventUpdate(task, extraCompute));
+        // 更新处理进度
+        resetProcessSchedule(task.getId(), data.size());
+        ApplicationProjectListener.getThreadPool().execute(() -> processEventUpdate(data, task, extraCompute));
     }
 
     @Override
@@ -379,27 +391,20 @@ public class ToolsetServiceImpl implements ToolsetService {
     public void eventAdditionalUpdate(MultipartFile file, String taskId) {
         BytedanceCustomEventUpdateTask task = bytedanceCustomEventUpdateTaskDao.findOneById(taskId);
         updateStatus(taskId, BytedanceCustomEventUpdateTask.TaskStatus.CALCULATING, BytedanceCustomEventUpdateTask.ProcessStatus.CHANNEL_MATCH);
+        List<BytedanceCustomEventUpdateTaskData> data = new ArrayList<>();
         // excel信息提取
-        Map<String, List<UploadBytedanceEventDTO>> map = new HashMap<>();
         ReadExcelDTO<UploadBytedanceEventDTO> readExcel = new ReadExcelDTO<>();
         readExcel.setClazz(UploadBytedanceEventDTO.class);
-        readExcel.setAnalysisEventListener(new BytedanceEventListener(map));
+        readExcel.setAnalysisEventListener(new BytedanceEventListener(taskId, data));
         EasyExcelUtil.read(file, readExcel);
         // 将旧数据与新数据合并
-        Map<String, List<UploadBytedanceEventDTO>> eventData = task.getEventData();
-        for (Map.Entry<String, List<UploadBytedanceEventDTO>> entry : map.entrySet()) {
-            String key = entry.getKey();
-            List<UploadBytedanceEventDTO> value = entry.getValue();
-            eventData.compute(key, (k, v) -> {
-                if (Objects.nonNull(v)) {
-                    v.addAll(value);
-                    return v;
-                }
-                return value;
-            });
-        }
-        task.setEventData(eventData);
-        ApplicationProjectListener.getThreadPool().execute(() -> processEventUpdate(task, task.getExtraCompute()));
+        List<BytedanceCustomEventUpdateTaskData> taskData = bytedanceCustomEventUpdateTaskDataDao.findList(new Query(Criteria.where("taskId").is(taskId)));
+        taskData.addAll(data);
+        // 清空错误旧数据
+        task.getErrorData().clear();
+        // 更新进度
+        resetProcessSchedule(task.getId(), taskData.size());
+        ApplicationProjectListener.getThreadPool().execute(() -> processEventUpdate(taskData, task, task.getExtraCompute()));
     }
 
     @Override
@@ -408,12 +413,18 @@ public class ToolsetServiceImpl implements ToolsetService {
                 .with(Sort.by(Sort.Direction.DESC, "uTime"));
         BytedanceCustomEventUpdateTask task = bytedanceCustomEventUpdateTaskDao.findOne(query);
         JSONObject jsonObject = new JSONObject();
+        if (Objects.isNull(task)){
+            jsonObject.put("stopSchedule", true);
+            return jsonObject;
+        }
         jsonObject.put("type", task.getType());
         jsonObject.put("taskStatus", task.getTaskStatus());
         jsonObject.put("processStatus", task.getProcessStatus());
         jsonObject.put("taskId", task.getId());
         jsonObject.put("extraCompute", task.getExtraCompute());
         jsonObject.put("cTime", task.getCTime());
+        jsonObject.put("schedule", task.getSchedule());
+        jsonObject.put("totalSchedule", task.getTotalSchedule());
         if ((Objects.equals(task.getTaskStatus(), BytedanceCustomEventUpdateTask.TaskStatus.FINISH.getStatus()) &&
         Objects.equals(task.getProcessStatus(), BytedanceCustomEventUpdateTask.ProcessStatus.FINISH.getStatus())) ||
         Objects.equals(task.getTaskStatus(), BytedanceCustomEventUpdateTask.TaskStatus.ERROR.getStatus()) ||
@@ -479,7 +490,8 @@ public class ToolsetServiceImpl implements ToolsetService {
         // excel写入至指定路径
         String projectName = projectService.getProjectById(UserThreadLocal.getProjectId()).getProjectName();
         String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), "报错数据");
-        EasyExcelUtil.write(filePath, "sheet1", UploadBytedanceEventDTO.class, task.getErrorData());
+        List<UploadBytedanceEventDTO> collect = task.getErrorData().stream().map(errorData -> Tools.convertMap(errorData, UploadBytedanceEventDTO.class)).collect(Collectors.toList());
+        EasyExcelUtil.write(filePath, "sheet1", UploadBytedanceEventDTO.class, collect);
         return filePath;
     }
 
@@ -520,30 +532,42 @@ public class ToolsetServiceImpl implements ToolsetService {
         bytedanceCustomEventUpdateTaskDao.updateOne(task);
     }
 
+    @Override
+    public List<String> getProjectToolsetShowList() {
+        Project project = projectService.getProjectById(UserThreadLocal.getProjectId());
+        return project.getToolsetShowList();
+    }
+
     /**
      * 处理字节定制本品事件影响力更新
      * @param task
      * @param extraCompute
      */
-    private void processEventUpdate(BytedanceCustomEventUpdateTask task, Boolean extraCompute){
+    private void processEventUpdate(List<BytedanceCustomEventUpdateTaskData> taskData, BytedanceCustomEventUpdateTask task, Boolean extraCompute){
         String taskId = task.getId();
         try {
             List<BytedanceCustomEventUpdateTask.EventInfo> eventInfos = new ArrayList<>();
-            Map<String, List<UploadBytedanceEventDTO>> map = task.getEventData();
-            for (Map.Entry<String, List<UploadBytedanceEventDTO>> entry : map.entrySet()) {
-                List<UploadBytedanceEventDTO> data = entry.getValue();
+            Map<String, List<BytedanceCustomEventUpdateTaskData>> map = taskData.stream().collect(Collectors.groupingBy(BytedanceCustomEventUpdateTaskData::getEventName));
+            for (Map.Entry<String, List<BytedanceCustomEventUpdateTaskData>> entry : map.entrySet()) {
+                List<BytedanceCustomEventUpdateTaskData> data = entry.getValue();
                 BytedanceCustomEventUpdateTask.EventInfo eventInfo = new BytedanceCustomEventUpdateTask.EventInfo();
                 eventInfo.setName(entry.getKey());
                 // 计算影响力指数
                 eventInfo.setInfluence(computeInf(data));
                 eventInfos.add(eventInfo);
+                // 更新渠道匹配处理进度
+                addProcessSchedule(taskId, data.size());
             }
             // 若需要计算实时传播力与传播影响力
             if (extraCompute) {
                 // 更新为互动量更新处理状态
                 updateStatus(taskId, BytedanceCustomEventUpdateTask.TaskStatus.CALCULATING, BytedanceCustomEventUpdateTask.ProcessStatus.INTERACTION_UPDATE);
+                long count = taskData.stream().filter(data -> Objects.isNull(data.getWechatRead()) && Objects.isNull(data.getWechatReading()) && Objects.isNull(data.getWeiboForward())
+                        && Objects.isNull(data.getWeiboComment()) && Objects.isNull(data.getWeiboLike())).count();
+                // 更新互动量更新处理进度
+                resetProcessSchedule(taskId, (int) count);
                 for (BytedanceCustomEventUpdateTask.EventInfo eventInfo : eventInfos) {
-                    List<UploadBytedanceEventDTO> data = map.get(eventInfo.getName());
+                    List<BytedanceCustomEventUpdateTaskData> data = map.get(eventInfo.getName());
                     // 计算实时传播力
                     eventInfo.setRMI(computeRMI(data, task));
                     // 计算传播影响力
@@ -551,18 +575,12 @@ public class ToolsetServiceImpl implements ToolsetService {
                 }
                 updateStatus(taskId, BytedanceCustomEventUpdateTask.TaskStatus.CALCULATING, BytedanceCustomEventUpdateTask.ProcessStatus.INFLUENCE_COMPUTE);
             }
-            // 移除数据中的错误数据
-            for (List<UploadBytedanceEventDTO> value : task.getEventData().values()) {
-                if (CollectionUtils.isNotEmpty(task.getErrorData())) {
-                    value.removeAll(task.getErrorData());
-                }
-            }
             task.setEventInfo(eventInfos);
             updateStatus(task, BytedanceCustomEventUpdateTask.TaskStatus.FINISH, BytedanceCustomEventUpdateTask.ProcessStatus.FINISH);
         }catch (Exception e){
             log.info("本品字节事件影响力补充计算出错-taskId：{}", taskId, e);
             if (!Objects.equals(e.getMessage(), "手动取消")){
-                updateStatus(task, BytedanceCustomEventUpdateTask.TaskStatus.ERROR, null);
+                updateStatus(taskId, BytedanceCustomEventUpdateTask.TaskStatus.ERROR);
             }
         }
     }
@@ -599,24 +617,26 @@ public class ToolsetServiceImpl implements ToolsetService {
             searchHelper.setIndexes(indexes);
             searchHelper.setPostFilter(postFilter);
             List<SearchResponse> searchResponses = esClientDao.searchScrollResponse(searchHelper);
-            List<UploadBytedanceEventDTO> data = new ArrayList<>();
+            List<BytedanceCustomEventUpdateTaskData> data = new ArrayList<>();
             for (SearchResponse searchResponse : searchResponses) {
                 for (SearchHit hit : searchResponse.getHits().getHits()) {
                     BaseMap baseMap = Tools.getBaseFromEsMap(hit.getSourceAsMap());
-                    UploadBytedanceEventDTO dto = new UploadBytedanceEventDTO();
+                    BytedanceCustomEventUpdateTaskData dto = new BytedanceCustomEventUpdateTaskData();
                     dto.setPlatform(baseMap.getPlatform());
                     dto.setChannel(baseMap.getSource());
                     data.add(dto);
                 }
             }
             updateStatus(taskId, null, BytedanceCustomEventUpdateTask.ProcessStatus.INFLUENCE_COMPUTE);
+            resetProcessSchedule(taskId, data.size());
             // 计算影响力指数
             task.setContendInfluence(computeInf(data));
+            addProcessSchedule(taskId, data.size());
             updateStatus(task, BytedanceCustomEventUpdateTask.TaskStatus.FINISH, BytedanceCustomEventUpdateTask.ProcessStatus.FINISH);
         }catch (Exception e){
             log.info("竞品字节事件影响力补充计算出错-taskId：{}", taskId, e);
             if (!Objects.equals(e.getMessage(), "手动取消")){
-                updateStatus(task, BytedanceCustomEventUpdateTask.TaskStatus.ERROR, null);
+                updateStatus(taskId, BytedanceCustomEventUpdateTask.TaskStatus.ERROR);
             }
         }
     }
@@ -655,17 +675,38 @@ public class ToolsetServiceImpl implements ToolsetService {
         bytedanceCustomEventUpdateTaskDao.updateOne(task);
     }
 
+    private void updateStatus(String taskId, BytedanceCustomEventUpdateTask.TaskStatus taskStatus){
+        BytedanceCustomEventUpdateTask task = bytedanceCustomEventUpdateTaskDao.findOneById(taskId);
+        updateStatus(task, taskStatus, null);
+    }
+
+    private void addProcessSchedule(String taskId, Integer schedule){
+        BytedanceCustomEventUpdateTask task = bytedanceCustomEventUpdateTaskDao.findOneById(taskId);
+        task.setUTime(System.currentTimeMillis());
+        int taskSchedule = task.getSchedule();
+        task.setSchedule(taskSchedule + schedule);
+        bytedanceCustomEventUpdateTaskDao.updateOne(task);
+    }
+
+    private void resetProcessSchedule(String taskId, Integer total){
+        BytedanceCustomEventUpdateTask task = bytedanceCustomEventUpdateTaskDao.findOneById(taskId);
+        task.setUTime(System.currentTimeMillis());
+        task.setSchedule(0);
+        task.setTotalSchedule(total);
+        bytedanceCustomEventUpdateTaskDao.updateOne(task);
+    }
+
     /**
      * 计算影响力指数（Inf）
      * @param data
      * @return
      */
-    private double computeInf(List<UploadBytedanceEventDTO> data){
+    private double computeInf(List<BytedanceCustomEventUpdateTaskData> data){
         double sumH = 0;
         if (CollectionUtils.isEmpty(data)){
             return sumH;
         }
-        for (UploadBytedanceEventDTO datum : data) {
+        for (BytedanceCustomEventUpdateTaskData datum : data) {
             // 网媒、平媒、外媒和自媒体四个平台类型的渠道，匹配重要渠道表，如果匹配成功，则按照“网媒”算影响力值。
             if (COMMON_MEDIA.contains(datum.getPlatform()) && bytedanceCustomChannelDao.exist(new Query(Criteria.where("channel").is(datum.getChannel())))){
                 datum.setPlatform("网媒");
@@ -707,8 +748,8 @@ public class ToolsetServiceImpl implements ToolsetService {
      * @param data
      * @return
      */
-    private double computeRMI(List<UploadBytedanceEventDTO> data, BytedanceCustomEventUpdateTask task){
-        List<UploadBytedanceEventDTO> willBeInteractionUpdateData = new ArrayList<>();
+    private double computeRMI(List<BytedanceCustomEventUpdateTaskData> data, BytedanceCustomEventUpdateTask task){
+        List<BytedanceCustomEventUpdateTaskData> willBeInteractionUpdateData = new ArrayList<>();
         int weiboForward = 0;
         int weiboComment = 0;
         int weiboLike = 0;
@@ -719,11 +760,11 @@ public class ToolsetServiceImpl implements ToolsetService {
         // 微信文章数
         int wechatArticle = (int) data.stream().filter(article -> Objects.equals("微信", article.getPlatform())).count();
         // 网媒文章数
-        int media = (int) data.stream().filter(article -> Objects.equals("网媒", article.getPlatform())).count();
+        int normalMediaArticle = (int) data.stream().filter(article -> Objects.equals("网媒", article.getPlatform())).count();
         // 其他平台文章数
-        int otherPlatform = (int) data.stream().filter(article -> !Objects.equals("网媒", article.getPlatform())
+        int otherPlatformArticle = (int) data.stream().filter(article -> !Objects.equals("网媒", article.getPlatform())
                 && !Objects.equals("微博", article.getPlatform()) && !Objects.equals("微信", article.getPlatform())).count();
-        for (UploadBytedanceEventDTO datum : data) {
+        for (BytedanceCustomEventUpdateTaskData datum : data) {
             // 选填字段全部未填，后续走互动量更新
             if (Objects.isNull(datum.getWechatRead()) && Objects.isNull(datum.getWechatReading()) && Objects.isNull(datum.getWeiboForward())
             && Objects.isNull(datum.getWeiboComment()) && Objects.isNull(datum.getWeiboLike())){
@@ -736,48 +777,94 @@ public class ToolsetServiceImpl implements ToolsetService {
             wechatRead = wechatRead + datum.getWechatRead();
             wechatReading = wechatReading + datum.getWechatReading();
         }
+        data.removeAll(willBeInteractionUpdateData);
         // 选填字段全部未填的链接互动量更新结果
         try {
-            List<String> urls = willBeInteractionUpdateData.stream().map(UploadBytedanceEventDTO::getUrl).collect(Collectors.toList());
-            List<JSONObject> jsonObjects = urlInteractionUpdate(urls);
+            List<String> successUrl = new ArrayList<>();
+            List<String> urls = willBeInteractionUpdateData.stream().map(BytedanceCustomEventUpdateTaskData::getUrl).collect(Collectors.toList());
+            Map<String, BytedanceCustomEventUpdateTaskData> urlMap = willBeInteractionUpdateData.stream().collect(Collectors.toMap(BytedanceCustomEventUpdateTaskData::getUrl, o -> o, (v1, v2) -> v1));
+            // 互动量更新
+            List<JSONObject> jsonObjects = urlInteractionUpdate(urls, task.getId());
             for (JSONObject jsonObject : jsonObjects) {
-                // 转发数
-                int repostCount = Objects.isNull(jsonObject.getInteger("repostCount")) ? 0 : jsonObject.getIntValue("repostCount");
-                // 评论数
-                int commentCount = Objects.isNull(jsonObject.getInteger("commentCount")) ? 0 : jsonObject.getIntValue("commentCount");
-                // 点赞数
-                int likeCount = Objects.isNull(jsonObject.getInteger("likeCount")) ? 0 : jsonObject.getIntValue("likeCount");
-                // 阅读数
-                int readCount = Objects.isNull(jsonObject.getInteger("readCount")) ? 0 : jsonObject.getIntValue("readCount");
-                // 在看数
-                int kanCount = Objects.isNull(jsonObject.getInteger("kanCount")) ? 0 : jsonObject.getIntValue("kanCount");
-                weiboForward = weiboForward + repostCount;
-                weiboComment = weiboComment + commentCount;
-                weiboLike = weiboLike + likeCount;
-                wechatRead = wechatRead + readCount;
-                wechatReading = wechatReading + kanCount;
+                if (200 == jsonObject.getInteger("code")) {
+                    // 成功的链接
+                    String url = jsonObject.getString("url");
+                    successUrl.add(url);
+                    // 转发数
+                    int repostCount = Objects.isNull(jsonObject.getInteger("repostCount")) ? 0 : jsonObject.getIntValue("repostCount");
+                    urlMap.get(url).setWeiboForward(repostCount);
+                    // 评论数
+                    int commentCount = Objects.isNull(jsonObject.getInteger("commentCount")) ? 0 : jsonObject.getIntValue("commentCount");
+                    urlMap.get(url).setWeiboComment(commentCount);
+                    // 点赞数
+                    int likeCount = Objects.isNull(jsonObject.getInteger("likeCount")) ? 0 : jsonObject.getIntValue("likeCount");
+                    urlMap.get(url).setWeiboLike(likeCount);
+                    // 阅读数
+                    int readCount = Objects.isNull(jsonObject.getInteger("readCount")) ? 0 : jsonObject.getIntValue("readCount");
+                    urlMap.get(url).setWechatRead(readCount);
+                    // 在看数，实际使用的是返回结果中的分享数
+                    int shareCount = Objects.isNull(jsonObject.getInteger("shareCount")) ? 0 : jsonObject.getIntValue("shareCount");
+                    urlMap.get(url).setWechatReading(shareCount);
+                    // 累加
+                    weiboForward = weiboForward + repostCount;
+                    weiboComment = weiboComment + commentCount;
+                    weiboLike = weiboLike + likeCount;
+                    wechatRead = wechatRead + readCount;
+                    wechatReading = wechatReading + shareCount;
+                }
             }
+            // 报错数据
+            urls.removeAll(successUrl); // 去除成功数据，剩余即为失败数据
+            List<BytedanceCustomEventUpdateTaskData> errorData = urls.stream().map(urlMap::get).collect(Collectors.toList());
+            List<BytedanceCustomEventUpdateTaskData> error = task.getErrorData();
+            error.addAll(errorData);
+            task.setErrorData(error);
+            // 互动量更新成功数据
+            List<BytedanceCustomEventUpdateTaskData> successData = successUrl.stream().map(urlMap::get).collect(Collectors.toList());
+            data.addAll(successData);
         }catch (Exception e){
-            // 报错数据：为互动量更新失败数据，更新失败数据不参与计算
             log.info("字节事件影响力更新-互动量更新出错-id:{}", task.getId(), e);
-            List<UploadBytedanceEventDTO> errorData = task.getErrorData();
+            List<BytedanceCustomEventUpdateTaskData> errorData = task.getErrorData();
             errorData.addAll(willBeInteractionUpdateData);
             task.setErrorData(errorData);
         }
-        // 平台指标权重
-        Map<String, Double> weightMap = bytedanceCustomPlatformWeightDao.findList(new Query())
+        // 存储数据以用于补充计算
+        bytedanceCustomEventUpdateTaskDataDao.deleteOneByQuery(new Query(Criteria.where("taskId").is(task.getId())));
+        bytedanceCustomEventUpdateTaskDataDao.insertMany(data);
+        // 平台指标权重，常量
+        Map<String, BytedanceCustomPlatformWeight> weightMap = bytedanceCustomPlatformWeightDao.findList(new Query())
                 .stream()
-                .collect(Collectors.toMap(BytedanceCustomPlatformWeight::getType, BytedanceCustomPlatformWeight::getWeight));
-        // 各平台数据指标与权重相乘求和
-        return BigDecimal.valueOf(media).multiply(BigDecimal.valueOf(weightMap.get("网媒文章数")))
-                .add(BigDecimal.valueOf(otherPlatform).multiply(BigDecimal.valueOf(weightMap.get("其他平台文章数"))))
-                .add(BigDecimal.valueOf(wechatArticle).multiply(BigDecimal.valueOf(weightMap.get("微信文章数"))))
-                .add(BigDecimal.valueOf(wechatRead).multiply(BigDecimal.valueOf(weightMap.get("微信阅读数"))))
-                .add(BigDecimal.valueOf(wechatReading).multiply(BigDecimal.valueOf(weightMap.get("微信在看数"))))
-                .add(BigDecimal.valueOf(weiboArticle).multiply(BigDecimal.valueOf(weightMap.get("微博消息数"))))
-                .add(BigDecimal.valueOf(weiboForward).multiply(BigDecimal.valueOf(weightMap.get("微博转发数"))))
-                .add(BigDecimal.valueOf(weiboComment).multiply(BigDecimal.valueOf(weightMap.get("微博评论数"))))
-                .add(BigDecimal.valueOf(weiboLike).multiply(BigDecimal.valueOf(weightMap.get("微博点赞数"))))
+                .collect(Collectors.toMap(BytedanceCustomPlatformWeight::getType, o -> o));
+        BytedanceCustomPlatformWeight normalMediaArticleWeight = weightMap.get("网媒文章数");
+        BytedanceCustomPlatformWeight otherPlatformArticleWeight = weightMap.get("其他平台文章数");
+        BytedanceCustomPlatformWeight wechatArticleWeight = weightMap.get("微信文章数");
+        BytedanceCustomPlatformWeight wechatReadWeight = weightMap.get("微信阅读数");
+        BytedanceCustomPlatformWeight wechatReadingWeight = weightMap.get("微信在看数");
+        BytedanceCustomPlatformWeight weiboArticleWeight = weightMap.get("微博消息数");
+        BytedanceCustomPlatformWeight weiboForwardWeight = weightMap.get("微博转发数");
+        BytedanceCustomPlatformWeight weiboCommentWeight = weightMap.get("微博评论数");
+        BytedanceCustomPlatformWeight weiboLikeWeight = weightMap.get("微博点赞数");
+        // 标准值计算
+        BigDecimal bigDecimal100 = BigDecimal.valueOf(100);
+        BigDecimal NM = BigDecimal.valueOf(normalMediaArticle).divide(BigDecimal.valueOf(normalMediaArticleWeight.getConstant()), 9, RoundingMode.HALF_UP).multiply(bigDecimal100);
+        BigDecimal OP = BigDecimal.valueOf(otherPlatformArticle).divide(BigDecimal.valueOf(otherPlatformArticleWeight.getConstant()), 9, RoundingMode.HALF_UP).multiply(bigDecimal100);
+        BigDecimal WC = BigDecimal.valueOf(wechatArticle).divide(BigDecimal.valueOf(wechatArticleWeight.getConstant()), 9, RoundingMode.HALF_UP).multiply(bigDecimal100);
+        BigDecimal WCR = BigDecimal.valueOf(Math.log(wechatRead + 1)).divide(BigDecimal.valueOf(Math.log(wechatReadWeight.getConstant() + 1)), 9, RoundingMode.HALF_UP).multiply(bigDecimal100);
+        BigDecimal WCL = BigDecimal.valueOf(Math.log(wechatReading + 1)).divide(BigDecimal.valueOf(Math.log(wechatReadingWeight.getConstant() + 1)), 9, RoundingMode.HALF_UP).multiply(bigDecimal100);
+        BigDecimal WB = BigDecimal.valueOf(weiboArticle).divide(BigDecimal.valueOf(weiboArticleWeight.getConstant()), 9, RoundingMode.HALF_UP).multiply(bigDecimal100);
+        BigDecimal WBF = BigDecimal.valueOf(Math.log(weiboForward + 1)).divide(BigDecimal.valueOf(Math.log(weiboForwardWeight.getConstant() + 1)), 9, RoundingMode.HALF_UP).multiply(bigDecimal100);
+        BigDecimal WBC = BigDecimal.valueOf(Math.log(weiboComment + 1)).divide(BigDecimal.valueOf(Math.log(weiboCommentWeight.getConstant() + 1)), 9, RoundingMode.HALF_UP).multiply(bigDecimal100);
+        BigDecimal WBL = BigDecimal.valueOf(Math.log(weiboLike + 1)).divide(BigDecimal.valueOf(Math.log(weiboLikeWeight.getConstant() + 1)), 9, RoundingMode.HALF_UP).multiply(bigDecimal100);
+        // 标准值与权重计算得RMI
+        return NM.multiply(BigDecimal.valueOf(normalMediaArticleWeight.getWeight()))
+                .add(OP.multiply(BigDecimal.valueOf(otherPlatformArticleWeight.getWeight())))
+                .add(WC.multiply(BigDecimal.valueOf(wechatArticleWeight.getWeight())))
+                .add(WCR.multiply(BigDecimal.valueOf(wechatReadWeight.getWeight())))
+                .add(WCL.multiply(BigDecimal.valueOf(wechatReadingWeight.getWeight())))
+                .add(WB.multiply(BigDecimal.valueOf(weiboArticleWeight.getWeight())))
+                .add(WBF.multiply(BigDecimal.valueOf(weiboForwardWeight.getWeight())))
+                .add(WBC.multiply(BigDecimal.valueOf(weiboCommentWeight.getWeight())))
+                .add(WBL.multiply(BigDecimal.valueOf(weiboLikeWeight.getWeight())))
                 .setScale(7, RoundingMode.HALF_UP)
                 .doubleValue();
     }

src/main/java/com/zhiwei/brandkbs2/service/impl/UserServiceImpl.java

@@ -164,6 +164,10 @@ public class UserServiceImpl implements UserService {
             }
         }
         User user = userDao.findOneById(userDTO.getId());
+        // 通过手机号再查询一次
+        if (null == user) {
+            user = userDao.findOne("phoneNumber", userDTO.getPhoneNumber());
+        }
         // 新用户
         if (null == user) {
             User newUser;

src/main/java/com/zhiwei/brandkbs2/util/RedisUtil.java

@@ -78,6 +78,10 @@ public class RedisUtil {
         return RedisKeyPrefix.TOOLSET_ARTICLE_SUMMARY_LIMIT + projectId;
     }
 
+    public static String getNonManualProjectMarkMaxGidKey(String projectId, String planId, String userId){
+        return RedisKeyPrefix.NON_MANUAL_PROJECT_MARK_MAX_GID + Tools.concat(projectId, planId, userId);
+    }
+
     public void setExpire(String key, String value, long timeout, TimeUnit unit) {
         stringRedisTemplate.opsForValue().set(key, value, timeout, unit);
     }

src/main/resources/application-dev.properties

 application.name = brandkbs2
 server.port=8888
 spring.flyway.encoding=UTF-8
+#\u5355\u4E2A\u4E0A\u4F20\u6587\u4EF6\u5927\u5C0F\u8BBE\u7F6E
+spring.servlet.multipart.max-file-size=10MB
+#\u603B\u4E0A\u4F20\u6587\u4EF6\u5927\u5C0F\u8BBE\u7F6E
+spring.servlet.multipart.max-request-size=20MB
 server.servlet.context-path=/brandkbs
 #jwt
 jwt.key=Token
@@ -73,7 +77,7 @@ qbjc.platform.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/platfo
 qbjc.userCenter.url=https://login.zhiweidata.com/plogin/center
 qbjc.userCenter.token=AoJ0ooy3HV1EElWnvQw9YTS9b5Y+fmtkbM6DdpPgDO6D/OhNqH4qrJKarzMr
 qbjc.channel.application.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/channel-application
-qbjc.channel.influence=https://yuqing.test.zhiweidata.com/qbjcbackPhoenix/interface/middleware/channel-influence?platform={1}&channel={2}
+qbjc.channel.influence=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/channel-influence?platform={1}&channel={2}
 #\u5371\u673A\u5E93\u5916\u90E8\u63A5\u53E3
 crisis.search.url=https://crisis.zhiweidata.com/app/brandkbs/crisisSearch?page={1}&size={2}&keyword={3}
 crisis.searchTags.url=https://crisis.zhiweidata.com/app/brandkbs/searchCrisisByTags?page={1}&size={2}&brand={3}&category={4}
@@ -81,7 +85,7 @@ crisis.top3.url=https://crisis.zhiweidata.com/app/brandkbs/event/online-top3
 crisis.searchCriteria.url=https://crisis.zhiweidata.com/app/brandkbs/condition
 crisis.list.url=https://crisis.zhiweidata.com/app/brandkbs/crisisList?page={1}&size={2}&startTime={3}&endTime={4}&category={5}
 crisis.share.url=https://crisis.zhiweidata.com/app/brandkbs/share/{1}
-crisis.event.url=https://crisis.zhiweidata.com/event/{1}/general?share={2}
+crisis.event.url=https://crisis.zhiweidata.com/share/crisisDetails?id={1}&share={2}
 #\u70ED\u70B9\u5E93\u5916\u90E8\u63A5\u53E3
 trends.longTimeInListSearchByInner.url=https://trends.zhiweidata.com/hotSearchTrend/inner/longTimeInListSearchByInner?sortType={1}&type={2}&day={3}
 trends.findHotSearchESDataInTimeByInner.url=https://hotsearch-manage.zhiweidata.com/hotsearch/hotSearch/findHotSearchESDataInTimeByInner?limit={1}&page={2}&type={3}&word={4}

src/main/resources/application-local.properties

 application.name = brandkbs2
 server.port=8888
 spring.flyway.encoding=UTF-8
+#\u5355\u4E2A\u4E0A\u4F20\u6587\u4EF6\u5927\u5C0F\u8BBE\u7F6E
+spring.servlet.multipart.max-file-size=10MB
+#\u603B\u4E0A\u4F20\u6587\u4EF6\u5927\u5C0F\u8BBE\u7F6E
+spring.servlet.multipart.max-request-size=20MB
 server.servlet.context-path=/brandkbs
 #jwt
 jwt.key=Token
 jwt.hour=120
 #\u8DEF\u5F84\u5B58\u653E
-brandkbs.file.url=E:\\work3\\brandkbs3\\
-brandkbs.img.url=E:\\work\\brandkbs2\\img\\
-brandkbs.head.url=E:\\work\\brandkbs2\\head\\
+brandkbs.file.url=D:\\ExcelTest\\
+brandkbs.img.url=D:\\ExcelTest\\
+brandkbs.head.url=D:\\ExcelTest\\
 brandkbs.image.url=https://brandkbs.zhiweidata.com/brandkbs/images/
 #\u56FE\u7247\u8D44\u6E90\u8DEF\u5F84
 cbs.imagesPath=file:${brandkbs.img.url},file:${brandkbs.head.url}
@@ -76,7 +80,7 @@ qbjc.platform.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/platfo
 qbjc.userCenter.url=http://login.zhiweidata.top/plogin/center
 qbjc.userCenter.token=AoJ0ooy3HV1EElWnvQw9YTS9b5Y+fmtkbM6DdpPgDO6D/OhNqH4qrJKarzMr
 qbjc.channel.application.url=http://192.168.0.79:11000/qbjcbackPhoenix/interface/middleware/channel-application
-qbjc.channel.influence=https://yuqing.test.zhiweidata.com/qbjcbackPhoenix/interface/middleware/channel-influence?platform={1}&channel={2}
+qbjc.channel.influence=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/channel-influence?platform={1}&channel={2}
 #\u5371\u673A\u5E93\u5916\u90E8\u63A5\u53E3
 crisis.search.url=https://crisis.zhiweidata.com/app/brandkbs/crisisSearch?page={1}&size={2}&keyword={3}
 crisis.searchTags.url=https://crisis.zhiweidata.com/app/brandkbs/searchCrisisByTags?page={1}&size={2}&brand={3}&category={4}
@@ -84,7 +88,7 @@ crisis.top3.url=https://crisis.zhiweidata.com/app/brandkbs/event/online-top3
 crisis.searchCriteria.url=https://crisis.zhiweidata.com/app/brandkbs/condition
 crisis.list.url=https://crisis.zhiweidata.com/app/brandkbs/crisisList?page={1}&size={2}&startTime={3}&endTime={4}&category={5}
 crisis.share.url=https://crisis.zhiweidata.com/app/brandkbs/share/{1}
-crisis.event.url=https://crisis.zhiweidata.com/event/{1}/general?share={2}
+crisis.event.url=https://crisis.zhiweidata.com/share/crisisDetails?id={1}&share={2}
 #\u70ED\u70B9\u5E93\u5916\u90E8\u63A5\u53E3
 trends.longTimeInListSearchByInner.url=https://trends.zhiweidata.com/hotSearchTrend/inner/longTimeInListSearchByInner?sortType={1}&type={2}&day={3}
 trends.findHotSearchESDataInTimeByInner.url=https://hotsearch-manage.zhiweidata.com/hotsearch/hotSearch/findHotSearchESDataInTimeByInner?limit={1}&page={2}&type={3}&word={4}

src/main/resources/application-prod.properties

 application.name = brandkbs2
 server.port=8888
 spring.flyway.encoding=UTF-8
+#\u5355\u4E2A\u4E0A\u4F20\u6587\u4EF6\u5927\u5C0F\u8BBE\u7F6E
+spring.servlet.multipart.max-file-size=10MB
+#\u603B\u4E0A\u4F20\u6587\u4EF6\u5927\u5C0F\u8BBE\u7F6E
+spring.servlet.multipart.max-request-size=20MB
 server.servlet.context-path=/brandkbs
 #jwt
 jwt.key=Token
@@ -73,7 +77,7 @@ qbjc.platform.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/platfo
 qbjc.userCenter.url=https://login.zhiweidata.com/plogin/center
 qbjc.userCenter.token=AoJ0ooy3HV1EElWnvQw9YTS9b5Y+fmtkbM6DdpPgDO6D/OhNqH4qrJKarzMr
 qbjc.channel.application.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/channel-application
-qbjc.channel.influence=https://yuqing.test.zhiweidata.com/qbjcbackPhoenix/interface/middleware/channel-influence?platform={1}&channel={2}
+qbjc.channel.influence=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/channel-influence?platform={1}&channel={2}
 #\u5371\u673A\u5E93\u5916\u90E8\u63A5\u53E3
 crisis.search.url=https://crisis.zhiweidata.com/app/brandkbs/crisisSearch?page={1}&size={2}&keyword={3}
 crisis.searchTags.url=https://crisis.zhiweidata.com/app/brandkbs/searchCrisisByTags?page={1}&size={2}&brand={3}&category={4}
@@ -81,7 +85,7 @@ crisis.top3.url=https://crisis.zhiweidata.com/app/brandkbs/event/online-top3
 crisis.searchCriteria.url=https://crisis.zhiweidata.com/app/brandkbs/condition
 crisis.list.url=https://crisis.zhiweidata.com/app/brandkbs/crisisList?page={1}&size={2}&startTime={3}&endTime={4}&category={5}
 crisis.share.url=https://crisis.zhiweidata.com/app/brandkbs/share/{1}
-crisis.event.url=https://crisis.zhiweidata.com/event/{1}/general?share={2}
+crisis.event.url=https://crisis.zhiweidata.com/share/crisisDetails?id={1}&share={2}
 #\u70ED\u70B9\u5E93\u5916\u90E8\u63A5\u53E3
 trends.longTimeInListSearchByInner.url=https://trends.zhiweidata.com/hotSearchTrend/inner/longTimeInListSearchByInner?sortType={1}&type={2}&day={3}
 trends.findHotSearchESDataInTimeByInner.url=https://hotsearch-manage.zhiweidata.com/hotsearch/hotSearch/findHotSearchESDataInTimeByInner?limit={1}&page={2}&type={3}&word={4}