Merge branch 'feature' into 'dev'

Feature

See merge request !454

src/main/java/com/zhiwei/brandkbs2/common/GenericAttribute.java

@@ -47,6 +47,7 @@ public class GenericAttribute {
      * es gid
      **/
     public static final String ES_GID = "gid";
+    public static final String ES_MGID = "mgid";
     public static final String ES_PROJECT_ID = "project_id";
     public static final String ES_CONTEND_ID = "contend_id";
     public static final String ES_CHANNEL_FID = "fid";
@@ -88,6 +89,7 @@ public class GenericAttribute {
      * es mtime
      **/
     public static final String ES_MTIME = "mtime";
+    public static final String ES_STIME = "stime";
     /**
      * es mtag
      **/

src/main/java/com/zhiwei/brandkbs2/common/GlobalPojo.java

@@ -5,6 +5,7 @@ import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.brandkbs2.pojo.ChannelTag;
 import com.zhiwei.brandkbs2.pojo.HighlightWord;
 import com.zhiwei.brandkbs2.pojo.Project;
+import com.zhiwei.brandkbs2.service.ChannelService;
 import com.zhiwei.brandkbs2.service.SystemInfoService;
 import com.zhiwei.brandkbs2.util.Tools;
 import com.zhiwei.middleware.automaticmark.graphs.Graphs;
@@ -46,6 +47,9 @@ public class GlobalPojo {
     @Resource(name = "highlightWordDao")
     private HighlightWordDao highlightWordDao;
 
+    @Resource(name = "channelServiceImpl")
+    private ChannelService channelService;
+
     /**
      * 监测系统平台
      **/
@@ -82,6 +86,8 @@ public class GlobalPojo {
 
     public static Map<String, Graphs> PROJECT_GRAPHS = new HashMap<>();
 
+    public static Map<String, JSONObject> PROJECT_EMOTION_CHANNEL_DATA = new HashMap<>();
+
     public static final List<String> PERMANENT_PLATFORM_NAMES = Arrays.asList("网媒", "微博", "微信", "今日头条");
 
     public static final String ELSE_PLATFORM_NAME = "其他自媒体";
@@ -119,9 +125,10 @@ public class GlobalPojo {
             MEDIA_TYPE = systemInfoService.getMediaTypes();
             PROJECT_MAP = systemInfoService.getProjects();
             YU_QING_PROJECTS = systemInfoService.getYuQingProjects();
+            PROJECT_EMOTION_CHANNEL_DATA = channelService.getProjectEmotionChannelListData();
             updateHighlightGraphs();
-            log.info("{}-获取PLATFORMS-size:{},TAGS-size:{},LINKED_GROUP_ID_TAGS:{},CHANNEL_TAGS:{},MEDIA_TYPE:{},PROJECT_MAP:{},YUQING-PROJECTS-size:{}", logMsg, PLATFORMS.size(), TAGS.size(),
-                    LINKED_GROUP_ID_TAGS.size(), CHANNEL_TAGS.size(), MEDIA_TYPE.size(), PROJECT_MAP.size(), YU_QING_PROJECTS.size());
+            log.info("{}-获取PLATFORMS-size:{},TAGS-size:{},LINKED_GROUP_ID_TAGS:{},CHANNEL_TAGS:{},MEDIA_TYPE:{},PROJECT_MAP:{},YUQING-PROJECTS-size:{},PROJECT_EMOTION_CHANNEL_DATA-size:{}", logMsg, PLATFORMS.size(), TAGS.size(),
+                    LINKED_GROUP_ID_TAGS.size(), CHANNEL_TAGS.size(), MEDIA_TYPE.size(), PROJECT_MAP.size(), YU_QING_PROJECTS.size(), PROJECT_EMOTION_CHANNEL_DATA.size());
         } catch (Exception e) {
             log.info("{}-获取缓存值异常", logMsg, e);
         }

src/main/java/com/zhiwei/brandkbs2/common/RedisKeyPrefix.java

@@ -102,6 +102,17 @@ public class RedisKeyPrefix {
 
     public static final String NON_MANUAL_PROJECT_MARK_MAX_GID = "BRANDKBS:NON_MANUAL:PROJECT:MARK:MAX_GID:";
 
+    /**
+     * 新舆情分析页面相关缓存
+     */
+    public static final String YUQING_ANALYZE_PROJECT_AVG_COUNT = "BRANDKBS:YUQING:ANALYZE:AMOUNT:AVG:";
+
+    public static final String YUQING_ANALYZE_EMOTION_DISTRIBUTION_AVG = "BRANDKBS:YUQING:ANALYZE:EMOTION:AVG:";
+
+    public static final String YUQING_ANALYZE_PLATFORM_AVG_COUNT = "BRANDKBS:YUQING:ANALYZE:PLATFORM:AVG:";
+
+    public static final String YUQING_ANALYZE_HIGH_WORD = "BRANDKBS:YUQING:ANALYZE:HIGH:WORD:";
+
     public static String projectWarnHotTopKeyAll(String projectId, String type) {
         return RedisKeyPrefix.generateRedisKey(RedisKeyPrefix.PROJECT_WARN_HOT_TOP, projectId, Tools.concat(type, "*"));
     }

src/main/java/com/zhiwei/brandkbs2/controller/CommonController.java

@@ -54,6 +54,16 @@ public class CommonController extends BaseController {
         }
     }
 
+    @ApiOperation("获取平台类型及id")
+    @GetMapping("/get/platform-id")
+    public ResponseResult getPlatformWithId() {
+        try {
+            return ResponseResult.success(commonService.getQbjcPlatform("id", "name"));
+        } catch (Exception e) {
+            return ResponseResult.failure(e.getMessage());
+        }
+    }
+
     @ApiOperation("获取当前用户拥有的所有项目及品牌列表")
     @GetMapping("/user/getUserAllProjects")
     @Auth(role = RoleEnum.CUSTOMER)

src/main/java/com/zhiwei/brandkbs2/controller/InterfaceController.java

@@ -2,10 +2,7 @@ package com.zhiwei.brandkbs2.controller;
 
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.brandkbs2.model.ResponseResult;
-import com.zhiwei.brandkbs2.pojo.external.BrandkbsChannelConfig;
-import com.zhiwei.brandkbs2.pojo.external.BrandkbsHotEventConfig;
-import com.zhiwei.brandkbs2.pojo.external.BrandkbsHotTopConfig;
-import com.zhiwei.brandkbs2.pojo.external.BrandkbsYuQingConfig;
+import com.zhiwei.brandkbs2.pojo.external.*;
 import com.zhiwei.brandkbs2.pojo.vo.CrisisCaseWarnVO;
 import com.zhiwei.brandkbs2.service.ProjectService;
 import com.zhiwei.brandkbs2.service.ProjectWarnService;
@@ -58,12 +55,21 @@ public class InterfaceController {
 
     @ApiOperation("获取舆情动态-预警结果")
     @PostMapping("/warn/yuqing")
+    @Deprecated
     public ResponseResult getYuqingWaring(@RequestBody JSONObject json) {
         String projectId = json.getString("projectId");
         BrandkbsYuQingConfig config = json.getObject("config", BrandkbsYuQingConfig.class);
         return projectWarnService.getYuqingWaring(projectId, config);
     }
 
+    @ApiOperation("获取舆情动态-预警结果")
+    @PostMapping("/warn/yuqingNew")
+    public ResponseResult getYuqingWaringNew(@RequestBody JSONObject json) {
+        String projectId = json.getString("projectId");
+        BrandkbsYuQingConfigNew config = json.getObject("config", BrandkbsYuQingConfigNew.class);
+        return projectWarnService.getYuqingWaringNew(projectId, config);
+    }
+
     @ApiOperation("获取渠道参与-预警结果")
     @PostMapping("/warn/channel")
     public ResponseResult getChannelWaring(@RequestBody JSONObject json) {
@@ -113,4 +119,19 @@ public class InterfaceController {
         List<String> list = JSONObject.parseObject(texts).getJSONArray("texts").toJavaList(String.class);
         return ResponseResult.success(textUtil.getHighWordsJson(list, 30));
     }
+
+    @ApiOperation("获取用户项目列表分页")
+    @GetMapping("/middleware/user-project-page-data")
+    public ResponseResult getUserProjectPageData(@RequestParam(value = "current") int current,
+                                                 @RequestParam(value = "pageSize") int pageSize,
+                                                 @RequestParam(value = "userId", required = false) String userId,
+                                                 @RequestParam(value = "keyword", required = false) String keyword) {
+        return projectService.getUserProjectPageData(userId, keyword, current, pageSize);
+    }
+
+    @ApiOperation("获取用户项目列表")
+    @GetMapping("/middleware/user-project-list")
+    public ResponseResult getUserProjectList(String userId) {
+        return projectService.getUserProject(userId);
+    }
 }

src/main/java/com/zhiwei/brandkbs2/controller/app/AppArticleController.java

@@ -440,6 +440,91 @@ public class AppArticleController extends BaseController {
         return ResponseResult.success(markDataService.getNonManualMarkAggreeList(markSearchDTO));
     }
 
+    @ApiOperation("获取方案列表")
+    @GetMapping("/analyze/plan/list")
+    public ResponseResult getNonManualPlanList(){
+        return ResponseResult.success(markDataService.getNonManualPlanList());
+    }
+
+    @ApiOperation("新-舆情分析-舆情总量")
+    @GetMapping("/analyze/amount")
+    public ResponseResult getYuqingAmount(@RequestParam(value = "startTime") Long startTime,
+                                          @RequestParam(value = "endTime") Long endTime,
+                                          @RequestParam(value = "planId", required = false) String planId) {
+        return ResponseResult.success(markDataService.getYuqingAmount(startTime, endTime, planId));
+    }
+
+    @ApiOperation("新-舆情分析-情感分布")
+    @GetMapping("/analyze/emotion")
+    public ResponseResult getYuqingEmotionDistribution(@RequestParam(value = "startTime") Long startTime,
+                                                       @RequestParam(value = "endTime") Long endTime,
+                                                       @RequestParam(value = "planId", required = false) String planId) {
+        return ResponseResult.success(markDataService.getYuqingEmotionDistribution(startTime, endTime, planId));
+    }
+
+    @ApiOperation("新-舆情分析-重点平台")
+    @GetMapping("/analyze/important-platform")
+    public ResponseResult getImportantPlatformPercentage(@RequestParam(value = "startTime") Long startTime,
+                                                         @RequestParam(value = "endTime") Long endTime,
+                                                         @RequestParam(value = "planId", required = false) String planId) {
+        return ResponseResult.success(markDataService.getImportantPlatformPercentage(startTime, endTime, planId));
+    }
+
+    @ApiOperation("新-舆情分析-平台占比")
+    @GetMapping("/analyze/platform-percent")
+    public ResponseResult getPlatformPercentage(@RequestParam(value = "startTime") Long startTime,
+                                                @RequestParam(value = "endTime") Long endTime,
+                                                @RequestParam(value = "planId", required = false) String planId) {
+        return ResponseResult.success(markDataService.getPlatformPercentage(startTime, endTime, planId));
+    }
+
+    @ApiOperation("新-舆情分析-舆情走势图")
+    @GetMapping("/analyze/tendency")
+    public ResponseResult getSpreadTendency(@RequestParam(value = "startTime") Long startTime,
+                                            @RequestParam(value = "endTime") Long endTime,
+                                            @RequestParam(value = "planId", required = false) String planId) {
+        return ResponseResult.success(markDataService.getSpreadTendency(startTime, endTime, planId));
+    }
+
+    @ApiOperation("新-舆情分析-活跃渠道")
+    @GetMapping("/analyze/active-channel")
+    public ResponseResult getActiveChannels(@RequestParam(value = "startTime") Long startTime,
+                                            @RequestParam(value = "endTime") Long endTime,
+                                            @RequestParam(value = "planId", required = false) String planId) {
+        return ResponseResult.success(markDataService.getActiveChannels(startTime, endTime, planId));
+    }
+
+    @ApiOperation("新-舆情分析-ip分布")
+    @GetMapping("/analyze/ip-located")
+    public ResponseResult getArticleIpLocated(@RequestParam(value = "startTime") Long startTime,
+                                              @RequestParam(value = "endTime") Long endTime,
+                                              @RequestParam(value = "planId", required = false) String planId,
+                                              @RequestParam(value = "size") int size) {
+        return ResponseResult.success(markDataService.getArticleIpLocated(startTime, endTime, planId, size));
+    }
+
+    @ApiOperation("新-舆情分析-词云")
+    @GetMapping("/analyze/high-word")
+    public ResponseResult getHighWord(@RequestParam(value = "startTime") Long startTime,
+                                      @RequestParam(value = "endTime") Long endTime,
+                                      @RequestParam(value = "planId", required = false) String planId) {
+        return ResponseResult.success(markDataService.getHighWord(startTime, endTime, planId, true));
+    }
+
+    @ApiOperation("新-舆情分析-高频标题")
+    @GetMapping("/analyze/frequent-title")
+    public ResponseResult getLastNews(@RequestParam(value = "startTime") Long startTime,
+                                      @RequestParam(value = "endTime") Long endTime,
+                                      @RequestParam(value = "planId", required = false) String planId) {
+        return ResponseResult.success(markDataService.getLastNews(startTime, endTime, planId, 5, true));
+    }
+
+    @ApiOperation("新-舆情分析-活跃渠道、ip分布、词云详情页面,社媒平台发文")
+    @PostMapping("/analyze/mark-data")
+    public ResponseResult getYuqingAnalyzeDetail(@RequestBody MarkSearchDTO markSearchDTO) {
+        return ResponseResult.success(markDataService.getYuqingAnalyzeDetail(markSearchDTO));
+    }
+
     private boolean checkMTagIllegal(StringBuilder mtag) {
         List<MarkerTag> hitTags = projectService.getProjectById(UserThreadLocal.getProjectId()).getHitTags();
         if (!Tools.isEmpty(hitTags)) {

src/main/java/com/zhiwei/brandkbs2/controller/app/AppChannelController.java

@@ -277,7 +277,7 @@ public class AppChannelController extends BaseController {
                                                  @RequestParam(value = "endTime", required = false) Long endTime,
                                                  @RequestParam(value = "sorter", defaultValue = "{\"index\":\"descend\"}") String sorter,
                                                  @RequestParam(value = "pageSize", defaultValue = "50") int size) {
-        return ResponseResult.success(channelService.getPositiveChannelList(contendId, startTime, endTime, size, sorter, true));
+        return ResponseResult.success(channelService.getPositiveChannelList(contendId, startTime, endTime, size, null, true));
     }
 
     @ApiImplicitParams({@ApiImplicitParam(name = "contendId", value = "品牌id", defaultValue = "0", paramType = "query", dataType = "string"),
@@ -294,7 +294,7 @@ public class AppChannelController extends BaseController {
                                                  @RequestParam(value = "endTime", required = false) Long endTime,
                                                  @RequestParam(value = "sorter", defaultValue = "{\"index\":\"descend\"}") String sorter,
                                                  @RequestParam(value = "pageSize", defaultValue = "50") int size) {
-        return ResponseResult.success(channelService.getNegativeChannelList(contendId, startTime, endTime, size, sorter, true));
+        return ResponseResult.success(channelService.getNegativeChannelList(contendId, startTime, endTime, size, null, true));
     }
 
     @ApiOperation("渠道库-渠道申请")

src/main/java/com/zhiwei/brandkbs2/controller/app/AppWarnController.java

@@ -31,6 +31,7 @@ import javax.annotation.Resource;
 import java.io.UnsupportedEncodingException;
 import java.net.URLEncoder;
 import java.util.Map;
+import java.util.Objects;
 
 
 /**
@@ -93,6 +94,9 @@ public class AppWarnController extends BaseController {
     @Value("${warn.taskSwitch.url}")
     private String warnTaskSwitchUrl;
 
+    @Value("${warn.push.ticket.url}")
+    private String warnPushTicketUrl;
+
     @ApiOperation("情报预警-推送任务获取")
     @GetMapping("/project")
     public ResponseResult getPushTaskId() {
@@ -108,7 +112,13 @@ public class AppWarnController extends BaseController {
     @PostMapping("/project")
     public ResponseResult postPushTaskId(@RequestBody JSONObject json) {
         try {
-            return pushTaskId(null, json.getString("planName"), HttpMethod.POST);
+            ResponseResult responseResult = pushTaskId(null, json.getString("planName"), HttpMethod.POST);
+            JSONObject body = (JSONObject) responseResult.getData();
+            if (!body.getBoolean("status") && Objects.isNull(body.getJSONObject("data"))){
+                return ResponseResult.failure("超过方案数量上限，新增失败");
+            }
+            JSONObject data = body.getJSONObject("data");
+            return ResponseResult.success(data.getString("id"));
         } catch (Exception e) {
             log.error("情报预警-推送任务新增失败", e);
             return ResponseResult.failure("情报预警-推送任务新增失败");
@@ -119,7 +129,8 @@ public class AppWarnController extends BaseController {
     @PutMapping("/project")
     public ResponseResult putPushTaskId(@RequestBody JSONObject json) {
         try {
-            return pushTaskId(json.getString("taskId"), json.getString("planName"), HttpMethod.PUT);
+            pushTaskId(json.getString("taskId"), json.getString("planName"), HttpMethod.PUT);
+            return ResponseResult.success();
         } catch (Exception e) {
             log.error("情报预警-推送任务修改失败", e);
             return ResponseResult.failure("情报预警-推送任务修改失败");
@@ -305,6 +316,27 @@ public class AppWarnController extends BaseController {
         }
     }
 
+
+    @ApiOperation("情报预警-获取自动化ticket")
+    @GetMapping("/push/ticket")
+    public ResponseResult getTicket() {
+        try {
+            String token = request.getHeader(jwtKey);
+            // 请求头参数设置
+            HttpHeaders headers = new HttpHeaders();
+            headers.setContentType(MediaType.APPLICATION_JSON);
+            headers.add("Push-Token", token);
+            HttpEntity<String> request = new HttpEntity<>(headers);
+            String ticket = restTemplate.exchange(warnPushTicketUrl, HttpMethod.GET, request, JSONObject.class)
+                    .getBody().getJSONObject("data").getString("ticket");
+            return ResponseResult.success(ticket);
+        } catch (Exception e) {
+            log.error("情报预警-获取自动化ticket", e);
+            return ResponseResult.failure("情报预警-获取自动化ticket");
+        }
+    }
+
+
     private ResponseResult pushTaskId(String taskId, String planName, HttpMethod httpMethod) throws UnsupportedEncodingException {
         String projectId = UserThreadLocal.getProjectId();
         String projectName = projectService.getProjectById(projectId).getProjectName();
@@ -323,8 +355,7 @@ public class AppWarnController extends BaseController {
         }
         if (HttpMethod.DELETE == httpMethod) {
             request = new HttpEntity<>(headers);
-            return ResponseResult.success(restTemplate.exchange(warnProjectUrl + "/"+taskId, httpMethod, request, JSONObject.class).getBody().getJSONArray(
-                    "data"));
+            return ResponseResult.success(restTemplate.exchange(warnProjectUrl + "/"+taskId, httpMethod, request, JSONObject.class).getBody().getJSONArray("data"));
         }
        JSONObject paramMap = new JSONObject();
         paramMap.put("planName", planName);
@@ -335,7 +366,8 @@ public class AppWarnController extends BaseController {
             paramMap.put("projectName", projectName);
         }
         request = new HttpEntity<>(paramMap, headers);
-        return ResponseResult.success(restTemplate.exchange(warnProjectUrl, httpMethod, request, JSONObject.class).getBody().getJSONArray("data"));
+        JSONObject body = restTemplate.exchange(warnProjectUrl, httpMethod, request, JSONObject.class).getBody();
+        return ResponseResult.success(body);
     }
 
 }

src/main/java/com/zhiwei/brandkbs2/es/ChannelEsDao.java

@@ -26,6 +26,8 @@ import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
 
 /**
@@ -74,22 +76,30 @@ public class ChannelEsDao extends EsClientDao {
 
     public void upsertChannelRecord(List<ChannelRecord> channelRecords) {
         String index = getChannelRecordIndexes().get(0);
-        BulkRequest bulkRequest = new BulkRequest();
-        Long startTime = null;
-        Long endTime = null;
         for (List<ChannelRecord> records : ListUtils.partition(channelRecords, 100)) {
+            // 打印统计时间
+            AtomicLong atomicStartTime = new AtomicLong(-1);
+            AtomicLong atomicEndTime = new AtomicLong(-1);
+            BulkRequest bulkRequest = new BulkRequest();
             for (ChannelRecord record : records) {
-                startTime = null == startTime ? record.getRangeStartTime() : Math.min(startTime, record.getRangeStartTime());
-                endTime = null == endTime ? record.getRangeEndTime() : Math.max(endTime, record.getRangeEndTime());
-                bulkRequest.add(new IndexRequest(index).id(record.getEsId()).source(record.toEsMap()));
+                bulkRequest.add(createChannelRecordIndexRequest(record, index, atomicStartTime, atomicEndTime));
             }
-            BulkResponse bulkResponse = retryTemplate.execute(context -> {
+            BulkResponse bulkResponse;
+            try {
+                bulkResponse = retryTemplate.execute(context -> {
                     try {
                         return channelEsClient.bulk(bulkRequest, RequestOptions.DEFAULT);
-                } catch (Exception ignored) {
+                    } catch (Exception e) {
+                        log.info("upsertRecord批量操作失败,尝试重试第{}次-", context.getRetryCount() + 1, e);
                         return null;
                     }
                 });
+            } catch (Exception e) {
+                // 重试三次后失败拆分channelRecords
+                bulkResponse = upsertChannelRecordLimit(records, index, 10);
+            }
+            Long startTime = atomicStartTime.get();
+            Long endTime = atomicEndTime.get();
             if (null == bulkResponse) {
                 log.error("upsertRecord批量操作重试后失败,index:{},rangeTime:{}", index, startTime + "-" + endTime);
             } else if (bulkResponse.hasFailures()) {
@@ -101,6 +111,46 @@ public class ChannelEsDao extends EsClientDao {
         }
     }
 
+    private BulkResponse upsertChannelRecordLimit(List<ChannelRecord> records, String index, int limit) {
+        AtomicBoolean res = new AtomicBoolean(true);
+        BulkResponse bulkResponse = null;
+        // 重试三次后失败拆分channelRecords
+        for (List<ChannelRecord> minRecords : ListUtils.partition(records, limit)) {
+            BulkRequest minBulkRequest = new BulkRequest();
+            for (ChannelRecord minRecord : minRecords) {
+                minBulkRequest.add(createChannelRecordIndexRequest(minRecord, index, null, null));
+            }
+            bulkResponse = retryTemplate.execute(context -> {
+                try {
+                    return channelEsClient.bulk(minBulkRequest, RequestOptions.DEFAULT);
+                } catch (Exception ex) {
+                    res.set(false);
+                    log.info("upsertRecord批量操作失败后二次重试,尝试重试第{}次-", context.getRetryCount() + 1, ex);
+                    return null;
+                }
+            });
+        }
+        return !res.get() ? null : bulkResponse;
+    }
+
+    private IndexRequest createChannelRecordIndexRequest(ChannelRecord record, String index, AtomicLong startTime, AtomicLong endTime) {
+        if (null != startTime) {
+            if (-1 == startTime.get()) {
+                startTime.set(record.getRangeStartTime());
+            } else {
+                startTime.set(Math.min(startTime.get(), record.getRangeStartTime()));
+            }
+        }
+        if (null != endTime) {
+            if (-1 == endTime.get()) {
+                endTime.set(record.getRangeEndTime());
+            } else {
+                endTime.set(Math.max(endTime.get(), record.getRangeEndTime()));
+            }
+        }
+        return new IndexRequest(index).id(record.getEsId()).source(record.toEsMap());
+    }
+
     public void batchInsert(List<Map<String, Object>> insertList) {
         retryTemplate.execute(context -> {
             try {

src/main/java/com/zhiwei/brandkbs2/es/EsQueryTools.java

@@ -6,6 +6,7 @@ import com.zhiwei.brandkbs2.common.GlobalPojo;
 import com.zhiwei.brandkbs2.config.Constant;
 import com.zhiwei.brandkbs2.enmus.ImportantChannelEnum;
 import com.zhiwei.brandkbs2.pojo.external.BrandkbsYuQingConfig;
+import com.zhiwei.brandkbs2.pojo.external.BrandkbsYuQingConfigNew;
 import com.zhiwei.pushlog.tools.Tools;
 import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform;
 import org.apache.commons.collections4.CollectionUtils;
@@ -161,6 +162,19 @@ public class EsQueryTools {
         return boolQueryBuilder;
     }
 
+    public static BoolQueryBuilder assembleCacheMapsQueryWithProject(String projectId, BrandkbsYuQingConfigNew config) {
+        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+        String key = concat(projectId, Constant.PRIMARY_CONTEND_ID);
+        BoolQueryBuilder nestedBoolBuilder = QueryBuilders.boolQuery();
+        // 必要条件
+        nestedBoolBuilder.must(QueryBuilders.termQuery("brandkbs_cache_maps.key.keyword", key));
+        // 或需要添加其余nested字段
+        boolQueryBuilder.must(cacheMapsNestedQuery(nestedBoolBuilder));
+        // sensitiveChannel
+        addSensitiveChannel(config, boolQueryBuilder);
+        return boolQueryBuilder;
+    }
+
     public static BoolQueryBuilder assembleCacheMapsQueryExcludePrimaryId(String projectId){
         BoolQueryBuilder query = QueryBuilders.boolQuery();
         query.must(cacheMapsNestedQuery(QueryBuilders.termQuery("brandkbs_cache_maps.project_id.keyword", projectId)));
@@ -276,11 +290,8 @@ public class EsQueryTools {
 
     public static BoolQueryBuilder assembleForward2Query(boolean isForward) {
         BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
-        // 需要转发数据，平台必须为微博
         if (isForward) {
-            BoolQueryBuilder mustWeiboCondition = QueryBuilders.boolQuery();
-            queryBuilder.should(mustWeiboCondition.must(QueryBuilders.termQuery("platform_id", "5d02236e6395002a7c380b79")));
-            queryBuilder.should(QueryBuilders.termQuery("is_forward", true));
+            queryBuilder.must(QueryBuilders.termQuery("is_forward", true));
         }else {
             queryBuilder.mustNot(QueryBuilders.termQuery("is_forward", true));
         }
@@ -295,7 +306,19 @@ public class EsQueryTools {
      */
     public static BoolQueryBuilder assembleC2Query(Integer dataType) {
         BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
-        queryBuilder.should(QueryBuilders.termQuery("c2", dataType << 23));
+        queryBuilder.should(QueryBuilders.termQuery(GenericAttribute.ES_C2, dataType << 23));
+        return queryBuilder;
+    }
+
+    /**
+     * ip查询
+     *
+     * @param ip ip地址
+     * @return
+     */
+    public static BoolQueryBuilder assembleIpQuery(String ip) {
+        BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
+        queryBuilder.should(QueryBuilders.termQuery("ip_location.keyword", ip));
         return queryBuilder;
     }
 
@@ -392,6 +415,25 @@ public class EsQueryTools {
         boolQueryBuilder.must(hitBoolQuery);
     }
 
+    public static void addSensitiveChannel(BrandkbsYuQingConfigNew config, BoolQueryBuilder boolQueryBuilder) {
+        BoolQueryBuilder hitBoolQuery = QueryBuilders.boolQuery();
+        if (!Tools.isEmpty(config.getPoliticsLevels())) {
+            List<String> hitList = config.getPoliticsLevels();
+            if (config.getPoliticsLevels().contains("全部")) {
+                hitList = ChannelType.POLITICS_LEVELS;
+            }
+            hitList.forEach(politicsLevel -> hitBoolQuery.should(QueryBuilders.termQuery(GenericAttribute.ES_SENSITIVE_CHANNEL + "." + GenericAttribute.ES_POLITICS_LEVEL + ".keyword", politicsLevel)));
+        }
+        if (!Tools.isEmpty(config.getFields())) {
+            List<String> hitList = config.getFields();
+            if (config.getFields().contains("全部")) {
+                hitList = ChannelType.COMMON_FIELDS;
+            }
+            hitList.forEach(field -> hitBoolQuery.should(QueryBuilders.termQuery(GenericAttribute.ES_SENSITIVE_CHANNEL + "." + GenericAttribute.ES_FIELD + ".keyword", field)));
+        }
+        boolQueryBuilder.must(hitBoolQuery);
+    }
+
     public static void addSensitiveChannel(String politicsLevel, String field, String region, String mainBodyType, BoolQueryBuilder boolQueryBuilder) {
         if (null != politicsLevel) {
             boolQueryBuilder.must(QueryBuilders.termQuery(GenericAttribute.ES_SENSITIVE_CHANNEL + "." + GenericAttribute.ES_POLITICS_LEVEL + ".keyword", politicsLevel));

src/main/java/com/zhiwei/brandkbs2/pojo/ChannelIndex.java

@@ -103,12 +103,14 @@ public class ChannelIndex extends AbstractBaseMongo {
             return res;
         }
         List<Map<String, Object>> cacheMaps = (List<Map<String, Object>>) sourceAsMap.get(GenericAttribute.ES_BRANDKBS_CACHE_MAPS);
-        List<String> manualProjectIds = GlobalPojo.PROJECT_MAP.entrySet().stream().filter(entry -> entry.getValue().isManual()).map(Map.Entry::getKey).collect(Collectors.toList());
+        // 非人工项目
+        List<String> nonManualProjectIds =
+                GlobalPojo.PROJECT_MAP.entrySet().stream().filter(entry ->!entry.getValue().isManual()).map(Map.Entry::getKey).collect(Collectors.toList());
         for (Map<String, Object> cacheMap : cacheMaps) {
             String projectId = String.valueOf(cacheMap.get("project_id"));
             String contendId = String.valueOf(cacheMap.get("contend_id"));
             // 剔除非人工项目
-            if(manualProjectIds.contains(projectId)){
+            if(nonManualProjectIds.contains(projectId)){
                 continue;
             }
             ChannelIndex channelIndex = new ChannelIndex(projectId, contendId, messagePlatform.getName(), realSource, source);
@@ -198,11 +200,21 @@ public class ChannelIndex extends AbstractBaseMongo {
         }
 
         public static List<Article> filterArticles(Long startTime, Long endTime, List<Article> articles) {
+            return filterArticles(startTime, endTime, articles, null);
+        }
+
+        public static List<Article> filterArticles(Long startTime, Long endTime, List<Article> articles, Integer emotion) {
             // 去除不符合时间段数据
             articles = articles.stream().filter(article -> Tools.hitTimeRange(startTime, endTime, article.getTime())).collect(Collectors.toList());
             // 去重并保留最近标注时间
             Map<String, ChannelIndex.Article> setMap = new HashMap<>();
             for (ChannelIndex.Article article : articles) {
+                if (null != emotion) {
+                    // 文章类型与其不一致
+                    if (emotion != EmotionEnum.ALL.getState() && emotion != article.getEmotion()) {
+                        continue;
+                    }
+                }
                 setMap.compute(article.getId(), (k, v) -> {
                     // 旧值为null或标注时间更新
                     if (null == v || article.getMtime() > v.getMtime()) {

src/main/java/com/zhiwei/brandkbs2/pojo/MarkFlowEntity.java

@@ -102,7 +102,7 @@ public class MarkFlowEntity implements Serializable {
         int c2 = tJson.getIntValue(GenericAttribute.ES_C2);
         // 微博平台头像url拼接
         String weiboHeadUrl = "https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/weibo-avatar-url?url=";
-        if (1020 == tJson.getIntValue(GenericAttribute.ES_C4) && !tJson.getString("avatar_url").contains(weiboHeadUrl)){
+        if (1020 == tJson.getIntValue(GenericAttribute.ES_C4) && Objects.nonNull(tJson.getString("avatar_url")) && !tJson.getString("avatar_url").contains(weiboHeadUrl)){
             tJson.put("avatar_url", weiboHeadUrl + tJson.getString("avatar_url"));
         }
         switch (ClassB.TypeB.fromEncode(c2)) {

src/main/java/com/zhiwei/brandkbs2/pojo/dto/MarkSearchDTO.java

@@ -181,7 +181,18 @@ public class MarkSearchDTO {
     @ApiModelProperty(value = "数据类型（1:长文本, 2:短文本, 3:问答, 5:视频）")
     private Integer dataType;
     /**
-     * gid用于未读已读筛选,仅非人工项目
+     * ip地址
      */
+    @ApiModelProperty(value = "ip地址")
+    private String ip;
+    /**
+     * gid用于未读已读筛选,非人工项目舆情数据用
+     */
+    @ApiModelProperty(value = "gid")
     private Long gid;
+    /**
+     * gid限制 舆情分析页面滚动翻页用
+     */
+    @ApiModelProperty(value = "gid限制")
+    private Long pageGid;
 }

src/main/java/com/zhiwei/brandkbs2/pojo/external/BrandkbsYuQingConfig.java

@@ -5,6 +5,7 @@ import lombok.Data;
 import java.util.List;
 
 @Data
+@Deprecated
 public class BrandkbsYuQingConfig {
 
     private String configName;

src/main/java/com/zhiwei/brandkbs2/pojo/external/BrandkbsYuQingConfigNew.java

+package com.zhiwei.brandkbs2.pojo.external;
+
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+public class BrandkbsYuQingConfigNew {
+    private String configName;
+
+    /**
+     * 关键字命中
+     */
+    private String keyword;
+
+    /**
+     * 自定义渠道
+     */
+    private String channel;
+
+    /**
+     * 行政级别
+     */
+    private List<String> politicsLevels;
+
+    /**
+     * 是否转发
+     */
+    private List<Boolean> primary;
+
+    /**
+     * 领域
+     */
+    private List<String> fields;
+
+    /**
+     * 本品舆情
+     */
+    private List<String> markerTags;
+
+    /**
+     * pushInterval 推送间隔
+     */
+    private Long pushInterval;
+
+    /**
+     * distinctFilter 去重过滤
+     */
+    private boolean distinctFilter;
+}

src/main/java/com/zhiwei/brandkbs2/pojo/vo/PageVO.java

@@ -45,6 +45,7 @@ public class PageVO<T> {
         pageVO.setPages(pages);
         pageVO.setSize(size);
         pageVO.setHasNext(pageNum < pages);
+        pageVO.setPageSize(size);
         return pageVO;
     }
 

src/main/java/com/zhiwei/brandkbs2/pojo/vo/ProjectVO.java

@@ -163,7 +163,7 @@ public class ProjectVO {
         project.setPositiveChannelParams((this.getPositiveChannelParams()));
         project.setBlackChannelGroup(this.getBlackChannelGroup());
         project.setShow(true);
-        project.setStart(false);
+        project.setStart(true);
         project.setManual(true);
         project.setCTime(time.getTime());
         project.setUTime(time.getTime());
@@ -189,7 +189,7 @@ public class ProjectVO {
         // 非人工项目
         project.setManual(false);
         project.setShow(true);
-        project.setStart(false);
+        project.setStart(true);
         project.setCTime(System.currentTimeMillis());
         project.setUTime(System.currentTimeMillis());
         return project;

src/main/java/com/zhiwei/brandkbs2/service/ChannelService.java

@@ -12,7 +12,9 @@ import com.zhiwei.brandkbs2.pojo.vo.ChannelListVO;
 import com.zhiwei.brandkbs2.pojo.vo.ChannelVO;
 import com.zhiwei.brandkbs2.pojo.vo.PageVO;
 
+import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 
 /**
  * @ClassName: ChannelService
@@ -300,4 +302,10 @@ public interface ChannelService {
 
     JSONObject getMobileSpreadingTend(String channelId,String type);
 
+    /**
+     * 获取项目正面、负面渠道数据
+     * @return
+     * @throws IOException
+     */
+    Map<String, JSONObject> getProjectEmotionChannelListData() throws IOException;
 }

src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java

@@ -436,6 +436,12 @@ public interface MarkDataService {
     PageVO<JSONObject> getNonManualProjectPlanList();
 
     /**
+     * 获取方案列表
+     * @return
+     */
+    List<JSONObject> getNonManualPlanList();
+
+    /**
      * 获取方案设置-方案列表-昨日，今日数据消耗量
      * @return
      */
@@ -511,4 +517,121 @@ public interface MarkDataService {
      * @return
      */
     List<JSONObject> getNonManualMarkCountList();
+
+    /**
+     * 计算近一年舆情总量项目日均
+     * @param projectId
+     * @param planId
+     * @return
+     * @throws IOException
+     */
+    void countYuqingAmountAvg(Long startTime, Long endTime, String projectId, String planId) throws IOException;
+
+    /**
+     * 计算近一年情感分布均值
+     * @param projectId
+     * @param planId
+     * @throws IOException
+     */
+    void countEmotionDistributionAvg(Long startTime, Long endTime, String projectId, String planId) throws IOException;
+
+    /**
+     * 计算项目近一年重点平台均值
+     * @param projectId
+     * @param planId
+     * @throws IOException
+     */
+    void countImportantPlatformPercentageAvg(Long startTime, Long endTime, String projectId, String planId) throws IOException;
+
+    /**
+     * 新-舆情分析-舆情总量
+     * @param startTime
+     * @param endTime
+     * @param planId
+     * @return
+     * @throws IOException
+     */
+    JSONObject getYuqingAmount(Long startTime, Long endTime, String planId);
+
+    /**
+     * 新-舆情分析-情感分布
+     * @param startTime
+     * @param endTime
+     * @param planId
+     * @return
+     * @throws IOException
+     */
+    JSONObject getYuqingEmotionDistribution(Long startTime, Long endTime, String planId);
+
+    /**
+     * 新-舆情分析-重点平台
+     * @param startTime
+     * @param endTime
+     * @param planId
+     * @return
+     */
+    JSONObject getImportantPlatformPercentage(Long startTime, Long endTime, String planId);
+
+    /**
+     * 新-舆情分析-平台占比
+     * @param startTime
+     * @param endTime
+     * @param planId
+     * @return
+     */
+    List<JSONObject> getPlatformPercentage(Long startTime, Long endTime, String planId);
+
+    /**
+     * 新-舆情分析-舆情走势图
+     * @param startTime
+     * @param endTime
+     * @param planId
+     * @return
+     */
+    JSONObject getSpreadTendency(Long startTime, Long endTime, String planId);
+
+    /**
+     * 新-舆情分析-活跃渠道
+     * @param startTime
+     * @param endTime
+     * @param planId
+     * @return
+     */
+    List<JSONObject> getActiveChannels(Long startTime, Long endTime, String planId);
+
+    /**
+     * 新-舆情分析-ip分布
+     * @param startTime
+     * @param endTime
+     * @param planId
+     * @return
+     */
+    List<JSONObject> getArticleIpLocated(Long startTime, Long endTime, String planId, int size);
+
+    /**
+     * 新-舆情分析-活跃渠道、ip分布、词云详情页面
+     * @param dto
+     * @return
+     */
+    PageVO<MarkFlowEntity> getYuqingAnalyzeDetail(MarkSearchDTO dto);
+
+    /**
+     * 新-舆情分析-词云
+     * @param startTime
+     * @param endTime
+     * @param planId
+     * @return
+     */
+    List<JSONObject> getHighWord(Long startTime, Long endTime, String planId, boolean cache);
+
+    /**
+     * 新-舆情分析-高频标题
+     * @param startTime
+     * @param endTime
+     * @param planId
+     * @param size
+     * @param include
+     * @return
+     */
+    List<JSONObject> getLastNews(Long startTime, Long endTime, String planId, int size, boolean include);
 }

src/main/java/com/zhiwei/brandkbs2/service/ProjectService.java

@@ -193,4 +193,21 @@ public interface ProjectService {
      * @return json
      */
     JSONObject nonManualProjectOverview(String project);
+
+    /**
+     * 获取用户拥有权限的项目
+     * @param userId
+     * @return
+     */
+    ResponseResult getUserProject(String userId);
+
+    /**
+     * 分页获取用户拥有权限的项目，支持项目名关键词模糊
+     * @param userId
+     * @param keyword 项目名模糊查询关键词
+     * @param page
+     * @param pageSize
+     * @return
+     */
+    ResponseResult getUserProjectPageData(String userId, String keyword, int page, int pageSize);
 }

src/main/java/com/zhiwei/brandkbs2/service/ProjectWarnService.java

 package com.zhiwei.brandkbs2.service;
 
 import com.zhiwei.brandkbs2.model.ResponseResult;
-import com.zhiwei.brandkbs2.pojo.external.BrandkbsChannelConfig;
-import com.zhiwei.brandkbs2.pojo.external.BrandkbsHotEventConfig;
-import com.zhiwei.brandkbs2.pojo.external.BrandkbsHotTopConfig;
-import com.zhiwei.brandkbs2.pojo.external.BrandkbsYuQingConfig;
+import com.zhiwei.brandkbs2.pojo.external.*;
 import com.zhiwei.brandkbs2.pojo.vo.CrisisCaseWarnVO;
 
 public interface ProjectWarnService {
@@ -34,6 +31,14 @@ public interface ProjectWarnService {
     ResponseResult getYuqingWaring(String projectId, BrandkbsYuQingConfig brandkbsYuQingConfig);
 
     /**
+     * 获取舆情动态预警-新
+     * @param projectId
+     * @param brandkbsYuQingConfigNew
+     * @return
+     */
+    ResponseResult getYuqingWaringNew(String projectId, BrandkbsYuQingConfigNew brandkbsYuQingConfigNew);
+
+    /**
      * 获取渠道参与预警
      * @param projectId
      * @param brandkbsChannelConfig

src/main/java/com/zhiwei/brandkbs2/service/TaskService.java

@@ -42,4 +42,14 @@ public interface TaskService{
      * 事件相关更新
      */
     void eventUpdate();
+
+    /**
+     * 计算项目舆情总量、情感分布、重点平台均值
+     */
+    void calculateProjectAvg();
+
+    /**
+     * 生成舆情分析词云缓存
+     */
+    void yuqingAnalyzeHighWordCache();
 }

src/main/java/com/zhiwei/brandkbs2/service/impl/ChannelServiceImpl.java

@@ -45,8 +45,11 @@ import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.RangeQueryBuilder;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.aggregations.Aggregation;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.BucketOrder;
+import org.elasticsearch.search.aggregations.bucket.terms.ParsedLongTerms;
+import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms;
 import org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.Sum;
@@ -99,6 +102,9 @@ public class ChannelServiceImpl implements ChannelService {
 	@Resource(name = "channelTagDao")
 	ChannelTagDao channelTagDao;
 
+	@Resource(name = "projectDao")
+	private ProjectDao projectDao;
+
 	@Resource(name = "qbjcPojoDao")
 	private QbjcPojoDao qbjcPojoDao;
 
@@ -510,15 +516,14 @@ public class ChannelServiceImpl implements ChannelService {
 		// 过滤掉不符合时间条件的数据并排序
 		Map<String, List<ChannelRecord>> channelRecords = keyMap.values().stream().map(pair -> {
 			ChannelRecord channelRecord = pair.getRight();
-			// 情感过滤
-			if (emotion == EmotionEnum.ALL.getState() || emotion == channelRecord.getEmotion()) {
 			List<ChannelIndex.Article> articles = ChannelIndex.Record.filterArticles(startTime, endTime,
-						channelRecord.getRecord().getArticles());
+					channelRecord.getRecord().getArticles(), emotion);
+			if (0 == articles.size()) {
+				return null;
+			}
 			articles.sort(Comparator.comparingLong(ChannelIndex.Article::getTime).reversed());
 			channelRecord.getRecord().setArticles(articles);
 			return channelRecord;
-			}
-			return null;
 		}).filter(Objects::nonNull).collect(Collectors.groupingBy(ChannelRecord::getPlatform));
 		for (String platformName : PLATFORMS) {
 			List<ChannelRecord> channelRecordList = channelRecords.getOrDefault(platformName, Collections.emptyList()).stream().limit(size).collect(Collectors.toList());
@@ -569,6 +574,7 @@ public class ChannelServiceImpl implements ChannelService {
 		return resList;
 	}
 
+	@Deprecated
 	private List<ChannelListVO> getEmotionList(String projectId, String contendId, String platform, String keyword, String sorter,
 											   Long startTime, Long endTime, int size, int emotion, boolean cache) throws IOException {
 		List<ChannelListVO> resList = new ArrayList<>();
@@ -1216,6 +1222,39 @@ public class ChannelServiceImpl implements ChannelService {
 		return result;
 	}
 
+	@Override
+	public Map<String, JSONObject> getProjectEmotionChannelListData() throws IOException {
+		int size = projectDao.findList(new Query()).size();
+		Map<String, JSONObject> res = new HashMap<>();
+		// query
+		BoolQueryBuilder query = QueryBuilders.boolQuery();
+		// contendId
+		query.must(QueryBuilders.termQuery("contend_id.keyword", Constant.PRIMARY_CONTEND_ID));
+		// emotion
+		query.mustNot(QueryBuilders.termQuery("emotion", EmotionEnum.NEUTRAL.getState()));
+		query.mustNot(QueryBuilders.termQuery("emotion", EmotionEnum.UNDEFINED.getState()));
+		// agg
+		TermsAggregationBuilder projectAgg = AggregationBuilders.terms("projectAgg").field("project_id.keyword").size(size);
+		TermsAggregationBuilder emotionAgg = AggregationBuilders.terms("emotionAgg").field("emotion");
+		// response
+		SearchResponse searchResponse = esClientDao.searchResponse(new String[]{"brandkbs2_channel_copy"}, null, query,
+				projectAgg.subAggregation(emotionAgg), null, null, 0, 0, null);
+		Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
+		ParsedStringTerms projectTeamAgg = (ParsedStringTerms) aggMap.get("projectAgg");
+		List<? extends Terms.Bucket> buckets = projectTeamAgg.getBuckets();
+		buckets.forEach(bucket -> {
+			JSONObject jsonObject = new JSONObject();
+			Map<String, Aggregation> map = bucket.getAggregations().asMap();
+			ParsedLongTerms emotionTeamAgg = (ParsedLongTerms) map.get("emotionAgg");
+			List<? extends Terms.Bucket> list = emotionTeamAgg.getBuckets();
+			for (Terms.Bucket eBucket : list) {
+				jsonObject.put(EmotionEnum.state2Name(eBucket.getKeyAsNumber().intValue()), eBucket.getDocCount());
+			}
+			res.put(bucket.getKeyAsString(), jsonObject);
+		});
+		return res;
+	}
+
 	private BoolQueryBuilder getChannelListQuery(String projectId, String contendId, String keyword,
 												 List<String> platforms, List<Integer> emotions, List<String> mediaTypes, Integer[] articlesCount) {
 		BoolQueryBuilder postFilter = QueryBuilders.boolQuery();

src/main/java/com/zhiwei/brandkbs2/service/impl/EsSearchServiceImpl.java

@@ -131,6 +131,14 @@ public class EsSearchServiceImpl implements EsSearchService {
         if (StringUtils.isNotEmpty(dto.getHostKeyword())) {
             postFilter.must(EsQueryTools.assembleFiledKeywordQuery("host", dto.getHostKeyword()));
         }
+        // ip
+        if (StringUtils.isNotEmpty(dto.getIp())){
+            postFilter.must(EsQueryTools.assembleIpQuery(dto.getIp()));
+        }
+        // mgid限制 舆情分析页面滚动翻页用
+        if (Objects.nonNull(dto.getPageGid())){
+            postFilter.must(QueryBuilders.rangeQuery(GenericAttribute.ES_MGID).lt(dto.getPageGid()));
+        }
         helper.setPostFilter(postFilter);
 //        helper.setQuery(query);
         // sort
@@ -315,6 +323,14 @@ public class EsSearchServiceImpl implements EsSearchService {
         if (Objects.nonNull(dto.getDataType())){
             postFilter.must(EsQueryTools.assembleC2Query(dto.getDataType()));
         }
+        // ip
+        if (StringUtils.isNotEmpty(dto.getIp())){
+            postFilter.must(EsQueryTools.assembleIpQuery(dto.getIp()));
+        }
+        // mgid限制 舆情分析页面滚动翻页用
+        if (Objects.nonNull(dto.getPageGid())){
+            postFilter.must(QueryBuilders.rangeQuery(GenericAttribute.ES_MGID).lt(dto.getPageGid()));
+        }
         helper.setPostFilter(postFilter);
         // sort
         FieldSortBuilder sort = null;

src/main/java/com/zhiwei/brandkbs2/service/impl/IndexServiceImpl.java

@@ -378,8 +378,10 @@ public class IndexServiceImpl implements IndexService {
         long normalCount = markDataService.getYuqingMarkCount(startTime, endTime, EmotionEnum.ALL.getName(), projectId, contendId);
         //获取时间范围内总正面稿件数
         long positiveCount = markDataService.getYuqingMarkCount(startTime, endTime, EmotionEnum.POSITIVE.getName(), projectId, contendId);
+        //获取时间范围内总中性稿件数
+        long neutralCount = markDataService.getYuqingMarkCount(startTime, endTime, EmotionEnum.NEUTRAL.getName(), projectId, contendId);
         double reputation = normalCount == 0 ? 0d : 100 * positiveCount / (double) normalCount;
-
+        double neutralPercent = normalCount == 0 ? 0d : 100 * neutralCount / (double) normalCount;
         SimpleDateFormat sdf;
         if ("year".equals(type)) {
             type = "年";
@@ -393,9 +395,11 @@ public class IndexServiceImpl implements IndexService {
         }
         String date = spreadResult.stream().max(Comparator.comparing(json -> json.getIntValue("normalCount"))).map(json -> sdf.format(json.getDate("time"))).get();
         if (!flag) {
-            return "近一月，传播高峰为" + date + "，全网品牌相关新闻传播总量达" + normalCount + "篇，正面内容占比为" + String.format("%.1f", reputation) + "%。";
+            return "近一月，传播高峰为" + date + "，全网品牌相关新闻传播总量达" + normalCount + "篇，正面内容占比为" + String.format("%.1f", reputation) + "%，中性内容占比为" + String.format("%.1f"
+                    , neutralPercent) + "%。";
         } else {
-            return "近" + spreadResult.size() + type + "内，传播高峰为" + date + "，全网品牌相关新闻传播总量达" + normalCount + "篇，正面内容占比为" + String.format("%.1f", reputation) + "%。";
+            return "近" + spreadResult.size() + type + "内，传播高峰为" + date + "，全网品牌相关新闻传播总量达" + normalCount + "篇，正面内容占比为" + String.format("%.1f", reputation) +
+                    "%，中性内容占比为" + String.format("%.1f", neutralPercent) + "%。";
         }
     }
 

src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java

@@ -55,6 +55,7 @@ import org.elasticsearch.search.SearchHits;
 import org.elasticsearch.search.aggregations.Aggregation;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.BucketOrder;
+import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
@@ -74,6 +75,7 @@ import org.springframework.http.HttpEntity;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.MediaType;
 import org.springframework.http.ResponseEntity;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
 import org.springframework.stereotype.Service;
 import org.springframework.web.client.RestTemplate;
 
@@ -81,9 +83,12 @@ import javax.annotation.Resource;
 import java.io.IOException;
 import java.text.SimpleDateFormat;
 import java.util.*;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /**
  * @ClassName: MarkDataServiceImpl
@@ -178,6 +183,9 @@ public class MarkDataServiceImpl implements MarkDataService {
     @Value("${brandkbs.file.url}")
     private String brandkbsFilePath;
 
+    @Resource(name = "esSearchExecutor")
+    ThreadPoolTaskExecutor executor;
+
     @Override
     public PageVO<MarkFlowEntity> getOriginList(MarkSearchDTO markSearchDTO) {
         try {
@@ -845,6 +853,52 @@ public class MarkDataServiceImpl implements MarkDataService {
         return result.entrySet().stream().sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())).limit(size).collect(Collectors.toList());
     }
 
+    private List<JSONObject> getMarkTopTitle(Long startTime, Long endTime, String emotion, String projectId, String contendId, String planId, int size) throws IOException {
+        // 索引
+        String[] indexes = esClientDao.getIndexes();
+        TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("titles").field("agg_title.keyword").size(size + 1);
+        TermsAggregationBuilder sourceAggregationBuilder = AggregationBuilders.terms("source").field("source").size(10000);
+        // query
+        BoolQueryBuilder query;
+        if (Objects.isNull(planId)) {
+            query = projectContendIdQuery(projectId, contendId);
+        }else {
+            query = EsQueryTools.assembleCacheMapsPlanQuery(projectId, planId);
+        }
+        query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime))
+                // 过滤微博
+                .mustNot(QueryBuilders.termQuery("platform_id", "5d02236e6395002a7c380b79"));
+        if (null != emotion && !Objects.equals(emotion, EmotionEnum.ALL.getName())) {
+            query.must(QueryBuilders.termQuery("brandkbs_mark_cache_maps.name.keyword", emotion));
+        }
+        SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, query,
+                aggregationBuilder.subAggregation(sourceAggregationBuilder), null, null, 0, 0, null);
+        List<JSONObject> res = new ArrayList<>();
+        Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
+        ParsedStringTerms teamAgg = (ParsedStringTerms) aggMap.get("titles");
+        for (Terms.Bucket bucket : teamAgg.getBuckets()) {
+            JSONObject jsonObject = new JSONObject();
+            String title = bucket.getKeyAsString();
+            // 过滤 “分享一篇文章” 的标题
+            if ("分享一篇文章".equals(title)) {
+                continue;
+            }
+//            result.merge(title, num, Integer::sum);
+            jsonObject.put("title", title);
+            jsonObject.put("num", bucket.getDocCount());
+            Map<String, Aggregation> aggregationMap = bucket.getAggregations().asMap();
+            ParsedStringTerms sourceAgg = (ParsedStringTerms) aggregationMap.get("source");
+            jsonObject.put("sourceCount", sourceAgg.getBuckets().size());
+            List<String> sources = new ArrayList<>();
+            for (Terms.Bucket sourceBucket : sourceAgg.getBuckets()) {
+                sources.add(sourceBucket.getKeyAsString());
+            }
+            jsonObject.put("sources", sources);
+            res.add(jsonObject);
+        }
+        return res;
+    }
+
     @Override
     public BaseMap getFirstArticle(Long startTime, Long endTime, String aggTitle, String projectId, String contendId) throws IOException {
         return getFirstArticle(startTime, endTime, aggTitle, projectId, contendId, true);
@@ -852,16 +906,28 @@ public class MarkDataServiceImpl implements MarkDataService {
 
     @Override
     public BaseMap getFirstArticle(Long startTime, Long endTime, String aggTitle, String projectId, String contendId, boolean include) throws IOException {
+        return getFirstArticle(startTime, endTime, aggTitle, projectId, contendId, null, include);
+    }
+
+    private BaseMap getFirstArticle(Long startTime, Long endTime, String aggTitle, String projectId, String contendId, String planId, boolean include) throws IOException{
+        if (StringUtils.isBlank(aggTitle)){
+            return null;
+        }
         // 索引
         String[] indexes = esClientDao.getIndexes();
         // postFilter
         BoolQueryBuilder postFilter;
+        if (Objects.isNull(planId)) {
             if (include) {
                 postFilter = projectContendIdQuery(projectId, contendId);
             } else {
                 postFilter = EsQueryTools.assembleCacheMapsQueryExcludePrimaryId(projectId);
             }
-        postFilter.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime)).must(QueryBuilders.termQuery("agg_title.keyword", aggTitle));
+        }else {
+            postFilter = EsQueryTools.assembleCacheMapsPlanQuery(projectId, planId);
+        }
+        postFilter.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
+        postFilter.must(QueryBuilders.termQuery("agg_title.keyword", aggTitle));
         //sort
         FieldSortBuilder sort = new FieldSortBuilder("time").order(SortOrder.ASC);
         //hits
@@ -872,7 +938,6 @@ public class MarkDataServiceImpl implements MarkDataService {
         return Tools.getBaseFromEsMap(hits.getAt(0).getSourceAsMap());
     }
 
-
     @Override
     public List<JSONObject> searchMarkDataByEvent(Event event) {
         Long endTime = null;
@@ -1874,6 +1939,18 @@ public class MarkDataServiceImpl implements MarkDataService {
         return PageVO.createPageVo(total, 1, 5, collect);
     }
 
+    @Override
+    public List<JSONObject> getNonManualPlanList() {
+        Query query = new Query(Criteria.where("projectId").is(UserThreadLocal.getProjectId()));
+        List<NonManualProjectPlan> planList = nonManualProjectPlanDao.findList(query);
+        return planList.stream().map(plan -> {
+            JSONObject jsonObject = new JSONObject();
+            jsonObject.put("id", plan.getId());
+            jsonObject.put("name", plan.getName());
+            return jsonObject;
+        }).collect(Collectors.toList());
+    }
+
     /**
      * 获取关键词/去噪词数量
      * @param word
@@ -2183,9 +2260,9 @@ public class MarkDataServiceImpl implements MarkDataService {
             aggreeList.setInfo(jsonObject);
             return aggreeList;
         }catch (Exception e){
-            log.error("获取非人工项目舆情列表聚合结果出错", e);
-            return new PageVO<>();
+            ExceptionCast.cast(CommonCodeEnum.FAIL, "getNonManualMarkAggreeList异常-", e);
         }
+        return new PageVO<>();
     }
 
     @Override
@@ -2207,9 +2284,573 @@ public class MarkDataServiceImpl implements MarkDataService {
             }
             return res;
         }catch (Exception e){
-            log.error("获取非人工项目舆情列表数据量统计出错", e);
+            ExceptionCast.cast(CommonCodeEnum.FAIL, "getNonManualMarkAggreeList异常-", e);
+        }
+        return Collections.emptyList();
+    }
+
+    @Override
+    public void countYuqingAmountAvg(Long startTime, Long endTime, String projectId, String planId) throws IOException {
+        Long total = getYuqingAnalyzeCount(startTime, endTime, projectId, planId, null, null);
+        int days = new Period(startTime, endTime, PeriodType.days()).getDays();
+        days = 0 == days ? 1 : days;
+        // avg
+        double avg = Objects.isNull(total) || 0 == total ? 0d : total / (double) days;
+        String projectYuqingCountAvgKey = RedisUtil.getYuqingAnalyzeProjectAvgCountKey(projectId, Constant.PRIMARY_CONTEND_ID, planId);
+        redisUtil.set(projectYuqingCountAvgKey, String.valueOf(avg));
+    }
+
+    @Override
+    public void countEmotionDistributionAvg(Long startTime, Long endTime, String projectId, String planId) throws IOException {
+        // 舆情总量
+        Long total = getYuqingAnalyzeCount(startTime, endTime, projectId, planId, null, null);
+        // 正面舆情
+        long positiveCount = getYuqingAnalyzeCount(startTime, endTime, projectId, planId, EmotionEnum.POSITIVE.getName(), null);
+        // 中性舆情
+        long neutralCount = getYuqingAnalyzeCount(startTime, endTime, projectId, planId, EmotionEnum.NEUTRAL.getName(), null);
+        // 负面舆情
+        long negativeCount = getYuqingAnalyzeCount(startTime, endTime, projectId, planId, EmotionEnum.NEGATIVE.getName(), null);
+        // key
+        String positiveKey = RedisUtil.getYuqingAnalyzeEmotionDistributionAvgKey(projectId, Constant.PRIMARY_CONTEND_ID, EmotionEnum.POSITIVE.getName(), planId);
+        String neutralKey = RedisUtil.getYuqingAnalyzeEmotionDistributionAvgKey(projectId, Constant.PRIMARY_CONTEND_ID, EmotionEnum.NEUTRAL.getName(), planId);
+        String negativeKey = RedisUtil.getYuqingAnalyzeEmotionDistributionAvgKey(projectId, Constant.PRIMARY_CONTEND_ID, EmotionEnum.NEGATIVE.getName(), planId);
+        // avg
+        double positiveAvg = Objects.isNull(total) || 0 == total ? 0d : positiveCount / (double) total;
+        double neutralAvg = Objects.isNull(total) || 0 == total ? 0d : neutralCount / (double) total;
+        double negativeAvg = Objects.isNull(total) || 0 == total ? 0d : negativeCount / (double) total;
+        redisUtil.set(positiveKey, String.valueOf(positiveAvg));
+        redisUtil.set(neutralKey, String.valueOf(neutralAvg));
+        redisUtil.set(negativeKey, String.valueOf(negativeAvg));
+    }
+
+    @Override
+    public void countImportantPlatformPercentageAvg(Long startTime, Long endTime, String projectId, String planId) throws IOException {
+        Long total = getYuqingAnalyzeCount(startTime, endTime, projectId, planId, null, Arrays.asList("微博", "微信", "抖音", "小红书", "今日头条", "网媒"));
+        String key = RedisUtil.getYuqingAnalyzePlatformAvgCountKey(projectId, Constant.PRIMARY_CONTEND_ID, planId);
+        int days = new Period(startTime, endTime, PeriodType.days()).getDays();
+        days = 0 == days ? 1 : days;
+        double avg = Objects.isNull(total) || 0 == total ? 0d : total / (double) days;
+        redisUtil.set(key, String.valueOf(avg));
+    }
+
+    @Override
+    public JSONObject getYuqingAmount(Long startTime, Long endTime, String planId) {
+        JSONObject jsonObject = new JSONObject();
+        try {
+            String projectId = UserThreadLocal.getProjectId();
+            // 舆情总量
+            long total = getYuqingAnalyzeCount(startTime, endTime, planId, EmotionEnum.ALL.getName());
+            jsonObject.put("total", total);
+            // 项目日均
+            String projectYuqingCountAvgKey = RedisUtil.getYuqingAnalyzeProjectAvgCountKey(projectId, Constant.PRIMARY_CONTEND_ID, planId);
+            double projectYuqingAvgCount = Objects.isNull(redisUtil.get(projectYuqingCountAvgKey)) ? 0d : Double.parseDouble(redisUtil.get(projectYuqingCountAvgKey));
+            jsonObject.put("projectAvg", projectYuqingAvgCount);
+            // 本次日均
+            int days = new Period(startTime, endTime, PeriodType.days()).getDays();
+            days = 0 == days ? 1 : days;
+            double yuqingAvgCount = total == 0 ? 0d : total / (double) days;
+            jsonObject.put("compare", compare(yuqingAvgCount, projectYuqingAvgCount));
+        }catch (Exception e){
+            ExceptionCast.cast(CommonCodeEnum.FAIL, "getYuqingAmount异常-", e);
+        }
+        return jsonObject;
+    }
+
+    @Override
+    public JSONObject getYuqingEmotionDistribution(Long startTime, Long endTime, String planId) {
+        JSONObject jsonObject = new JSONObject();
+        try {
+            String projectId = UserThreadLocal.getProjectId();
+            // 舆情总量
+            long total = getYuqingAnalyzeCount(startTime, endTime, planId, EmotionEnum.ALL.getName());
+            // 正面舆情
+            long positiveCount = getYuqingAnalyzeCount(startTime, endTime, planId, EmotionEnum.POSITIVE.getName());
+            // 中性舆情
+            long neutralCount = getYuqingAnalyzeCount(startTime, endTime, planId, EmotionEnum.NEUTRAL.getName());
+            // 负面舆情
+            long negativeCount = getYuqingAnalyzeCount(startTime, endTime, planId, EmotionEnum.NEGATIVE.getName());
+            // 各情感倾向占比
+            double positivePercentage = total == 0 ? 0d : positiveCount / (double) total;
+            double neutralPercentage = total == 0 ? 0d : neutralCount / (double) total;
+            double negativePercentage = total == 0 ? 0d : negativeCount / (double) total;
+            // 项目情感倾向占比均值
+            String positiveKey = RedisUtil.getYuqingAnalyzeEmotionDistributionAvgKey(projectId, Constant.PRIMARY_CONTEND_ID, EmotionEnum.POSITIVE.getName(), planId);
+            String neutralKey = RedisUtil.getYuqingAnalyzeEmotionDistributionAvgKey(projectId, Constant.PRIMARY_CONTEND_ID, EmotionEnum.NEUTRAL.getName(), planId);
+            String negativeKey = RedisUtil.getYuqingAnalyzeEmotionDistributionAvgKey(projectId, Constant.PRIMARY_CONTEND_ID, EmotionEnum.NEGATIVE.getName(), planId);
+            double positiveAvg = Objects.isNull(redisUtil.get(positiveKey)) ? 0d : Double.parseDouble(redisUtil.get(positiveKey));
+            double neutralAvg = Objects.isNull(redisUtil.get(neutralKey)) ? 0d : Double.parseDouble(redisUtil.get(neutralKey));
+            double negativeAvg = Objects.isNull(redisUtil.get(negativeKey)) ? 0d : Double.parseDouble(redisUtil.get(negativeKey));
+            JSONObject positive = new JSONObject();
+            JSONObject neutral = new JSONObject();
+            JSONObject negative = new JSONObject();
+            positive.put("positiveCount", positiveCount);
+            neutral.put("neutralCount", neutralCount);
+            negative.put("negativeCount", negativeCount);
+            positive.put("positivePercent", positivePercentage);
+            neutral.put("neutralPercent", neutralPercentage);
+            negative.put("negativePercent", negativePercentage);
+            positive.put("positiveAvg", positiveAvg);
+            neutral.put("neutralAvg", neutralAvg);
+            negative.put("negativeAvg", negativeAvg);
+            jsonObject.put("positive", positive);
+            jsonObject.put("neutral", neutral);
+            jsonObject.put("negative", negative);
+        }catch (Exception e){
+            ExceptionCast.cast(CommonCodeEnum.FAIL, "getYuqingEmotionDistribution异常-", e);
+        }
+        return jsonObject;
+    }
+
+    @Override
+    public JSONObject getImportantPlatformPercentage(Long startTime, Long endTime, String planId) {
+        JSONObject res = new JSONObject();
+        try {
+            List<JSONObject> list = new ArrayList<>();
+            List<String> importantPlatforms = Arrays.asList("微博", "微信", "抖音", "小红书", "今日头条", "网媒");
+            String projectId = UserThreadLocal.getProjectId();
+            Long total = getYuqingAnalyzeCount(startTime, endTime, planId, importantPlatforms);
+            // 平台聚合
+            SearchResponse searchResponse = platformAggSearchResponse(startTime, endTime, planId, importantPlatforms);
+            Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
+            ParsedStringTerms teamAgg = (ParsedStringTerms) aggMap.get("platform_count");
+            List<? extends Terms.Bucket> buckets = teamAgg.getBuckets();
+            buckets.forEach(bucket -> {
+                JSONObject platformResult = new JSONObject();
+                platformResult.put("platform", GlobalPojo.getPlatformNameById(bucket.getKeyAsString()));
+                platformResult.put("count", bucket.getDocCount());
+                list.add(platformResult);
+            });
+            res.put("platformCount", list.stream().sorted(Comparator.comparingLong((JSONObject json) -> json.getLongValue("count")).reversed()));
+            JSONObject overview = new JSONObject();
+            // 总量
+            overview.put("total", total);
+            // 日均
+            int days = new Period(startTime, endTime, PeriodType.days()).getDays();
+            days = 0 == days ? 1 : days;
+            double avg = total == 0 ? 0d : total / (double) days;
+            overview.put("avg", avg);
+            // 项目均值
+            String key = RedisUtil.getYuqingAnalyzePlatformAvgCountKey(projectId, Constant.PRIMARY_CONTEND_ID, planId);
+            double projectAvg = Objects.isNull(redisUtil.get(key)) ? 0d : Double.parseDouble(redisUtil.get(key));
+            overview.put("compare", compare(avg, projectAvg));
+            res.put("overview", overview);
+            return res;
+        }catch (Exception e){
+            ExceptionCast.cast(CommonCodeEnum.FAIL, "新舆情分析getImportantPlatformPercentage异常-", e);
+        }
+        return res;
+    }
+
+    private String compare(double avg, double projectAvg){
+        // 本次查询日均数据 > 项目日均1.2倍为【本次偏高】，项目日均0.8倍<本次查询日均数据<项目日均1.2倍为【数据量正常】，本次查询日均数据 < 项目日均0.8倍为【本次偏低】
+        String compare = "normal";
+        double highSide = 1.2 * projectAvg;
+        double lowSize = 0.8 * projectAvg;
+        if (avg > highSide){
+            compare = "higher";
+        }
+        if (lowSize > avg){
+            compare =  "lower";
+        }
+        return compare;
+    }
+
+    @Override
+    public List<JSONObject> getPlatformPercentage(Long startTime, Long endTime, String planId) {
+        try {
+            List<String> platforms = commonService.getQbjcPlatformNames();
+            List<JSONObject> list = new ArrayList<>();
+            // query
+            BoolQueryBuilder query = yuqingAnalyzeQuery(startTime, endTime, planId, null, platforms);
+            // total
+            Long total = esClientDao.count(query);
+            // 平台聚合
+            SearchResponse searchResponse = platformAggSearchResponse(startTime, endTime, planId, platforms);
+            Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
+            ParsedStringTerms teamAgg = (ParsedStringTerms) aggMap.get("platform_count");
+            List<? extends Terms.Bucket> buckets = teamAgg.getBuckets();
+            buckets.forEach(bucket -> {
+                JSONObject platformResult = new JSONObject();
+                platformResult.put("platform", GlobalPojo.getPlatformNameById(bucket.getKeyAsString()));
+                platformResult.put("percent", 0 == total ? 0d : bucket.getDocCount() / (double) total);
+                list.add(platformResult);
+            });
+            // 取占比最高的前9
+            List<JSONObject> res = list.stream()
+                    .sorted(Comparator.comparingDouble((JSONObject jsonObject) -> jsonObject.getDoubleValue("percent"))
+                            .reversed()).limit(9).collect(Collectors.toList());
+            // 第10个算做其他平台
+            JSONObject other = new JSONObject();
+            other.put("platform", "其他");
+            other.put("percent", 1 - res.stream().mapToDouble(count -> count.getDoubleValue("percent")).sum());
+            res.add(other);
+            return res;
+        }catch (Exception e){
+            ExceptionCast.cast(CommonCodeEnum.FAIL, "新舆情分析getPlatformPercentage异常-", e);
+        }
         return Collections.emptyList();
     }
+
+    /**
+     * 平台聚合查询
+     * @param startTime
+     * @param endTime
+     * @param planId
+     * @param platforms
+     * @return
+     * @throws IOException
+     */
+    private SearchResponse platformAggSearchResponse(Long startTime, Long endTime, String planId, List<String> platforms) throws IOException {
+        // 索引
+        String[] indexes = esClientDao.getIndexes();
+        // 聚合请求
+        TermsAggregationBuilder platformAggregationBuilder = AggregationBuilders.terms("platform_count").field("platform_id").order(BucketOrder.count(false));
+        // query
+        BoolQueryBuilder query = yuqingAnalyzeQuery(startTime, endTime, planId, null, platforms);
+        return esClientDao.searchResponse(indexes, null, query, platformAggregationBuilder, null, null, 0, 0, null);
+    }
+
+    @Override
+    public JSONObject getSpreadTendency(Long startTime, Long endTime, String planId) {
+        JSONObject res = new JSONObject();
+        try {
+            String projectId = UserThreadLocal.getProjectId();
+            // 各平台趋势图
+            Map<String, List<LineVO>> platformSpreadTendency = getPlatformSpreadTendency(startTime, endTime, planId);
+            // 总趋势图
+            Pair<String, List<LineVO>> spreadTendency = getSpreadTendency(startTime, endTime, planId, null);
+            // 负面趋势图
+            Pair<String, List<LineVO>> negativeSpreadTendency = getSpreadTendency(startTime, endTime, planId, EmotionEnum.NEGATIVE.getName());
+            Map<String, BaseMap> baseMaps = new HashMap<>(2);
+            CompletableFuture.allOf(Stream.of(spreadTendency.getLeft(), negativeSpreadTendency.getLeft()).map(aggTitle -> CompletableFuture.runAsync(() -> {
+                try {
+                    baseMaps.put(aggTitle, getFirstArticle(startTime, endTime, aggTitle, projectId, Constant.PRIMARY_CONTEND_ID, planId, true));
+                } catch (IOException ignored) {
+                }
+            }, executor)).toArray(CompletableFuture[]::new)).join();
+            // 最高点
+            BaseMap baseMap = baseMaps.get(spreadTendency.getLeft());
+            JSONObject highestJson = new JSONObject();
+            highestJson.put("title", Objects.isNull(baseMap) ? null : baseMap.getTitle());
+            highestJson.put("url", Objects.isNull(baseMap) ? null : baseMap.getUrl());
+            BaseMap negativeBaseMap = baseMaps.get(negativeSpreadTendency.getLeft());
+            JSONObject negativeHighestJson = new JSONObject();
+            negativeHighestJson.put("title", Objects.isNull(negativeBaseMap) ? null : negativeBaseMap.getTitle());
+            negativeHighestJson.put("url", Objects.isNull(negativeBaseMap) ? null : negativeBaseMap.getUrl());
+            res.putAll(platformSpreadTendency);
+            res.put("总量", spreadTendency.getRight());
+            res.put("负面", negativeSpreadTendency.getRight());
+            res.put("highest", highestJson);
+            res.put("negativeHighest", negativeHighestJson);
+        }catch (Exception e){
+            ExceptionCast.cast(CommonCodeEnum.FAIL, "新舆情分析getSpreadTendency异常-", e);
+        }
+        return res;
+    }
+
+    @Override
+    public List<JSONObject> getActiveChannels(Long startTime, Long endTime, String planId) {
+        List<JSONObject> list = new ArrayList<>();
+        try {
+            // 索引
+            String[] indexes = esClientDao.getIndexes();
+            // query
+            BoolQueryBuilder query = yuqingAnalyzeQuery(startTime, endTime, planId, null, null);
+            // 渠道聚合，取前20
+            TermsAggregationBuilder sourceAggregationBuilder = AggregationBuilders.terms("source_count").field("source").order(BucketOrder.count(false)).size(20);
+            // 情感倾向子聚合
+            TermsAggregationBuilder emotionAggregationBuilder = AggregationBuilders.terms("emotion_count").field("brandkbs_mark_cache_maps.name.keyword");
+            SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, query,
+                    sourceAggregationBuilder.subAggregation(emotionAggregationBuilder), null, null, 0, 0, null);
+            Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
+            ParsedStringTerms sourceCountTeam = (ParsedStringTerms) aggMap.get("source_count");
+            List<? extends Terms.Bucket> buckets = sourceCountTeam.getBuckets();
+            buckets.forEach(bucket -> {
+                JSONObject jsonObject = new JSONObject();
+                Map<String, Aggregation> map = bucket.getAggregations().asMap();
+                ParsedStringTerms countTeam = (ParsedStringTerms) map.get("emotion_count");
+                List<? extends Terms.Bucket> bucketList = countTeam.getBuckets();
+                jsonObject.put("source", bucket.getKeyAsString());
+                // 发文次数
+                jsonObject.put("count", bucket.getDocCount());
+                // 负面发文次数
+                AtomicLong negativeCount = new AtomicLong();
+                bucketList.forEach(data -> {
+                    if (Objects.equals(data.getKeyAsString(), EmotionEnum.NEGATIVE.getName())) {
+                        negativeCount.set(data.getDocCount());
+                    }
+                });
+                jsonObject.put("negativeCount", negativeCount.get());
+                list.add(jsonObject);
+            });
+        }catch (Exception e){
+            ExceptionCast.cast(CommonCodeEnum.FAIL, "新舆情分析getActiveChannels异常-", e);
+        }
+        return list;
+    }
+
+    @Override
+    public List<JSONObject> getArticleIpLocated(Long startTime, Long endTime, String planId, int size) {
+        List<JSONObject> list = new ArrayList<>();
+        try {
+            // 索引
+            String[] indexes = esClientDao.getIndexes();
+            // query
+            BoolQueryBuilder query = yuqingAnalyzeQuery(startTime, endTime, planId, null, null);
+            // 聚合请求
+            TermsAggregationBuilder ipAggregationBuilder = AggregationBuilders.terms("ip_count").field("ip_location.keyword").order(BucketOrder.count(false)).size(size);
+            // 情感倾向子聚合
+            TermsAggregationBuilder emotionAggregationBuilder = AggregationBuilders.terms("emotion_count").field("brandkbs_mark_cache_maps.name.keyword");
+            SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, query,
+                    ipAggregationBuilder.subAggregation(emotionAggregationBuilder), null, null, 0, 0, null);
+            Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
+            ParsedStringTerms sourceCountTeam = (ParsedStringTerms) aggMap.get("ip_count");
+            List<? extends Terms.Bucket> buckets = sourceCountTeam.getBuckets();
+            buckets.forEach(bucket -> {
+                JSONObject jsonObject = new JSONObject();
+                jsonObject.put("ip", bucket.getKeyAsString());
+                jsonObject.put("count", bucket.getDocCount());
+                // 取负面
+                Map<String, Aggregation> emotionAgg = bucket.getAggregations().asMap();
+                ParsedStringTerms emotionCountTeam = (ParsedStringTerms) emotionAgg.get("emotion_count");
+                List<? extends Terms.Bucket> emotionBuckets = emotionCountTeam.getBuckets();
+                Optional<? extends Terms.Bucket> negative = emotionBuckets.stream().filter(b -> Objects.equals(EmotionEnum.NEGATIVE.getName(), b.getKeyAsString())).findFirst();
+                jsonObject.put("negativeCount", negative.map(MultiBucketsAggregation.Bucket::getDocCount).orElse(0L));
+                list.add(jsonObject);
+            });
+        }catch (Exception e){
+            ExceptionCast.cast(CommonCodeEnum.FAIL, "新舆情分析getArticleIpLocated异常-", e);
+        }
+        return list;
+    }
+
+    @Override
+    public PageVO<MarkFlowEntity> getYuqingAnalyzeDetail(MarkSearchDTO dto) {
+        //设置默认的时间、页码、排序方式等
+        defaultMarkSearch(dto);
+        PageVO<MarkFlowEntity> yuqingMarkList;
+        if (Objects.isNull(dto.getPlanId())){
+            yuqingMarkList = getYuqingMarkList(dto);
+        }else {
+            yuqingMarkList = getNonManualMarkList(dto);
+        }
+        // 列表数据最小mgid
+        MarkFlowEntity markFlowEntity = yuqingMarkList.getList().stream()
+                .min(Comparator.comparingLong(entity -> JSONObject.parseObject(JSONObject.toJSONString(entity.getData())).getLongValue("mgid"))).orElse(null);
+//        yuqingMarkList.getInfo().put("stime", Objects.isNull(markFlowEntity) ? Long.MAX_VALUE : JSONObject.parseObject(JSONObject.toJSONString(markFlowEntity.getData())).getLongValue("stime"));
+        yuqingMarkList.getInfo().put("pageGid", Objects.isNull(markFlowEntity) ? Long.MAX_VALUE : JSONObject.parseObject(JSONObject.toJSONString(markFlowEntity.getData())).getLongValue("mgid"));
+        return yuqingMarkList;
+    }
+
+    @Override
+    public List<JSONObject> getHighWord(Long startTime, Long endTime, String planId, boolean cache) {
+        List<JSONObject> res = new ArrayList<>();
+        try {
+            String projectId = UserThreadLocal.getProjectId();
+            String redisKey = RedisUtil.getYuqingAnalyzeHighWordKey(projectId, Constant.PRIMARY_CONTEND_ID, planId, startTime, endTime);
+            String resultStr;
+            // 返回缓存
+            if (cache && StringUtils.isNotEmpty(resultStr = redisUtil.get(redisKey))) {
+                return JSONArray.parseArray(resultStr, JSONObject.class);
+            }
+            EsClientDao.SearchHelper searchHelper = EsClientDao.createSearchHelper();
+            // sort
+            searchHelper.setSort(SortBuilders.fieldSort("time").order(SortOrder.DESC));
+            // fetchSource
+            searchHelper.setFetchSource(new String[]{"ind_title", "ind_full_text", "c5", "foreign", "brandkbs_mark_cache_maps"});
+            // postFilter
+            BoolQueryBuilder postFilter = yuqingAnalyzeQuery(startTime, endTime, planId, null, null);
+            searchHelper.setPostFilter(postFilter);
+            searchHelper.setSize(10000);
+            List<String> textList = new ArrayList<>();
+            List<SearchResponse> searchResponses = Collections.singletonList(esClientDao.searchResponse(searchHelper));
+            for (SearchResponse searchResponse : searchResponses) {
+                for (SearchHit searchHit : searchResponse.getHits().getHits()) {
+                    Map<String, Object> source = searchHit.getSourceAsMap();
+                    BaseMap baseMap = Tools.getBaseFromEsMap(source);
+                    String title = baseMap.getTitle();
+                    String content = baseMap.getContent();
+                    textList.add(title + content);
+                }
+            }
+            res = textUtil.getHighWordsJsonDifferentFieldName(textList, 20);
+            redisUtil.setExpire(redisKey, JSONArray.toJSONString(res));
+        }catch (Exception e){
+            ExceptionCast.cast(CommonCodeEnum.FAIL, "新舆情分析getHighWord异常-", e);
+        }
+        return res;
+    }
+
+    @Override
+    public List<JSONObject> getLastNews(Long startTime, Long endTime, String planId, int size, boolean include) {
+        try {
+            String projectId = UserThreadLocal.getProjectId();
+            List<JSONObject> markTopTitleList = getMarkTopTitle(startTime, endTime, null, projectId, Constant.PRIMARY_CONTEND_ID, planId, size);
+            CompletableFuture.allOf(markTopTitleList.stream().map(json -> CompletableFuture.supplyAsync(() -> {
+                try {
+                    BaseMap firstArticle = getFirstArticle(startTime, endTime, json.getString("title"), projectId, Constant.PRIMARY_CONTEND_ID, planId, include);
+                    if (Objects.isNull(firstArticle)){
+                        return null;
+                    }
+                    json.put("content", firstArticle.getContent());
+                    json.put("url", firstArticle.getUrl());
+                    json.put("realSource", firstArticle.getRealSource());
+                    json.put("emotion", firstArticle.getEmotion());
+                    json.put("time", firstArticle.getTime());
+                } catch (IOException ignored) {
+                }
+                return null;
+            }, executor)).toArray(CompletableFuture[]::new)).join();
+            return markTopTitleList.stream().filter(Objects::nonNull).limit(size).collect(Collectors.toList());
+        }catch (Exception e){
+            ExceptionCast.cast(CommonCodeEnum.FAIL, "新舆情分析getLastNews异常-", e);
+        }
+        return Collections.emptyList();
+    }
+
+    /**
+     * 微博、微信、抖音、小红书平台趋势图
+     * @param startTime
+     * @param endTime
+     * @param planId
+     * @return
+     * @throws IOException
+     */
+    private Map<String, List<LineVO>> getPlatformSpreadTendency(Long startTime, Long endTime, String planId) throws IOException{
+        Map<String, List<LineVO>> res = new HashMap<>(4);
+        // 平台
+        List<String> platforms = Arrays.asList("微博", "微信", "抖音", "小红书");
+        String[] indexes = esClientDao.getIndexes();
+        // 聚合请求
+        TermsAggregationBuilder platformAggregationBuilder = AggregationBuilders.terms("platformAgg").field("platform_id").order(BucketOrder.count(false));
+        DateHistogramAggregationBuilder daysAggregationBuilder;
+        // 天级以小时为颗粒度，其他以天作为颗粒度
+        if (endTime - startTime <= Constant.ONE_DAY){
+            daysAggregationBuilder = AggregationBuilders.dateHistogram("timeAgg").field("time").calendarInterval(DateHistogramInterval.HOUR);
+        }else {
+            daysAggregationBuilder = AggregationBuilders.dateHistogram("timeAgg").field("time").calendarInterval(DateHistogramInterval.DAY);
+        }
+        // query
+        BoolQueryBuilder query = yuqingAnalyzeQuery(startTime, endTime, planId, null, platforms);
+        // response
+        SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, query,
+                platformAggregationBuilder.subAggregation(daysAggregationBuilder), null, null, 0, 0, null);
+        Map<String, Aggregation> map = searchResponse.getAggregations().asMap();
+        ParsedStringTerms countTeam = (ParsedStringTerms) map.get("platformAgg");
+        List<? extends Terms.Bucket> platformBuckets = countTeam.getBuckets();
+        platformBuckets.forEach(bucket -> {
+            List<LineVO> line = new ArrayList<>();
+            String platformId = bucket.getKeyAsString();
+            Map<String, Aggregation> aggMap = bucket.getAggregations().asMap();
+            ParsedDateHistogram teamAgg = (ParsedDateHistogram) aggMap.get("timeAgg");
+            List<? extends Histogram.Bucket> buckets = teamAgg.getBuckets();
+            for (Histogram.Bucket timeBucket : buckets) {
+                long time = Long.parseLong(timeBucket.getKeyAsString());
+                long count = timeBucket.getDocCount();
+                line.add(new LineVO(count, time));
+            }
+            res.put(GlobalPojo.getPlatformNameById(platformId), line);
+        });
+        return res;
+    }
+
+    /**
+     * 新舆情分析-舆情趋势图
+     * @param startTime
+     * @param endTime
+     * @param planId
+     * @param emotion
+     * @return
+     * @throws IOException
+     */
+    private Pair<String, List<LineVO>> getSpreadTendency(Long startTime, Long endTime, String planId, String emotion) throws IOException {
+        List<LineVO> res = new ArrayList<>();
+        String[] indexes = esClientDao.getIndexes();
+        DateHistogramAggregationBuilder daysAggregationBuilder;
+        // 天级以小时为颗粒度，其他以天作为颗粒度
+        if (endTime - startTime <= Constant.ONE_DAY){
+            daysAggregationBuilder = AggregationBuilders.dateHistogram("timeAgg").field("time").calendarInterval(DateHistogramInterval.HOUR);
+        }else {
+            daysAggregationBuilder = AggregationBuilders.dateHistogram("timeAgg").field("time").calendarInterval(DateHistogramInterval.DAY);
+        }
+        TermsAggregationBuilder titleAggregationBuilder = AggregationBuilders.terms("titleAgg").field("agg_title.keyword").order(BucketOrder.count(false));
+        // query
+        BoolQueryBuilder query = yuqingAnalyzeQuery(startTime, endTime, planId, emotion, null);
+        // response
+        SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, query,
+                daysAggregationBuilder.subAggregation(titleAggregationBuilder), null, null, 0, 0, null);
+        Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
+        ParsedDateHistogram teamAgg = (ParsedDateHistogram) aggMap.get("timeAgg");
+        List<? extends Histogram.Bucket> buckets = teamAgg.getBuckets();
+        // 走势图
+        buckets.forEach(bucket -> {
+            long time = Long.parseLong(bucket.getKeyAsString());
+            long count = bucket.getDocCount();
+            res.add(new LineVO(count, time));
+        });
+        // 走势图最高点的聚合标题
+        String maxAggTitle = "";
+        Optional<? extends Histogram.Bucket> maxBucket = buckets.stream().max(Comparator.comparingLong(MultiBucketsAggregation.Bucket::getDocCount));
+        if (maxBucket.isPresent()){
+            Map<String, Aggregation> map = maxBucket.get().getAggregations().asMap();
+            ParsedStringTerms countTeam = (ParsedStringTerms) map.get("titleAgg");
+            List<? extends Terms.Bucket> titleBuckets = countTeam.getBuckets();
+            // 没有拿到聚合标题，说明此时发文的平台全为微博
+            maxAggTitle = CollectionUtils.isEmpty(titleBuckets) ? "" : titleBuckets.get(0).getKeyAsString();
+        }
+        return Pair.of(maxAggTitle, res);
+    }
+
+    private Long getYuqingAnalyzeCount(Long startTime, Long endTime, String planId, List<String> platforms) throws IOException {
+        BoolQueryBuilder query = yuqingAnalyzeQuery(startTime, endTime, planId, null, platforms);
+        return esClientDao.count(query);
+    }
+
+    private Long getYuqingAnalyzeCount(Long startTime, Long endTime, String planId, String emotion) throws IOException {
+        BoolQueryBuilder query = yuqingAnalyzeQuery(startTime, endTime, planId, emotion, null);
+        return esClientDao.count(query);
+    }
+
+    private Long getYuqingAnalyzeCount(Long startTime, Long endTime, String projectId, String planId, String emotion, List<String> platforms) throws IOException {
+        BoolQueryBuilder query = yuqingAnalyzeQuery(startTime, endTime, projectId, planId, emotion, platforms);
+        return esClientDao.count(query);
+    }
+
+    private BoolQueryBuilder yuqingAnalyzeQuery(Long startTime, Long endTime, String planId, String emotion, List<String> platforms){
+        return yuqingAnalyzeQuery(startTime, endTime, null, planId, emotion, platforms);
+    }
+
+    private BoolQueryBuilder yuqingAnalyzeQuery(Long startTime, Long endTime, String projectId, String planId, String emotion, List<String> platforms){
+        if (Objects.isNull(projectId)) {
+            projectId = UserThreadLocal.getProjectId();
+        }
+        // project plan query
+        BoolQueryBuilder query = EsQueryTools.assembleCacheMapsPlanQuery(projectId, planId);
+        if (Objects.isNull(planId)){
+            query = projectContendIdQuery(projectId, Constant.PRIMARY_CONTEND_ID);
+        }
+        // emotion
+        if (StringUtils.isNotEmpty(emotion) && !EmotionEnum.ALL.getName().equals(emotion)) {
+            query.must(QueryBuilders.termQuery("brandkbs_mark_cache_maps.name.keyword", emotion));
+        }
+        // platform
+        if (CollectionUtils.isNotEmpty(platforms) && Objects.nonNull(platforms.get(0))){
+            List<String> platformIds = new ArrayList<>(6);
+            for (String platform : platforms) {
+                String platformId = GlobalPojo.getPlatformIdByName(platform);
+                platformIds.add(platformId);
+            }
+            query.must(QueryBuilders.termsQuery("platform_id", platformIds));
+        }
+        // startTime
+        if (Objects.nonNull(startTime)) {
+            query.must(QueryBuilders.rangeQuery("time").gte(startTime));
+        }
+        // endTime
+        if (Objects.nonNull(endTime)) {
+            query.must(QueryBuilders.rangeQuery("time").lt(endTime));
+        }
+        return query;
     }
 
     /**

src/main/java/com/zhiwei/brandkbs2/service/impl/ProjectServiceImpl.java

@@ -6,6 +6,7 @@ import com.zhiwei.brandkbs2.common.GenericAttribute;
 import com.zhiwei.brandkbs2.common.GlobalPojo;
 import com.zhiwei.brandkbs2.config.Constant;
 import com.zhiwei.brandkbs2.dao.*;
+import com.zhiwei.brandkbs2.enmus.EmotionEnum;
 import com.zhiwei.brandkbs2.enmus.response.ProjectCodeEnum;
 import com.zhiwei.brandkbs2.exception.ExceptionCast;
 import com.zhiwei.brandkbs2.model.CommonCodeEnum;
@@ -22,6 +23,7 @@ import com.zhiwei.middleware.event.pojo.dto.EventTagRelatedDTO;
 import com.zhiwei.middleware.event.pojo.entity.BrandkbsBasicInfo;
 import com.zhiwei.middleware.mark.vo.MarkerTag;
 import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.springframework.beans.factory.annotation.Value;
@@ -498,6 +500,9 @@ public class ProjectServiceImpl implements ProjectService {
         }
         Project project = ProjectVO.createNonManualProject(projectVO);
         projectDao.insertOneWithoutId(project);
+        // 绑定事件标签，默认已绑定情感标签
+        eventMiddlewareDao.bindBrandkbs(project.getBrandLinkedGroup(), project.getBrandLinkedGroupId(), Collections.emptyList(),
+                project.getProjectName(), project.getId(), project.getBrandName(), project.getId(), null, null);
         return ResponseResult.success();
     }
 
@@ -585,6 +590,51 @@ public class ProjectServiceImpl implements ProjectService {
         return json;
     }
 
+    @Override
+    public ResponseResult getUserProject(String userId) {
+        User user = userDao.findOneById(userId);
+        Query query = new Query();
+        if (!user.isSuperAdmin()){
+            List<String> projectIds = user.getRoles().stream().map(UserRole::getProjectId).collect(Collectors.toList());
+            query.addCriteria(Criteria.where("_id").in(projectIds));
+        }
+        List<Project> projectList = projectDao.findList(query);
+        List<JSONObject> res = projectList.stream().map(project -> {
+            JSONObject jsonObject = new JSONObject();
+            jsonObject.put("id", project.getId());
+            jsonObject.put("name", project.getProjectName());
+            return jsonObject;
+        }).collect(Collectors.toList());
+        return ResponseResult.success(res);
+    }
+
+    @Override
+    public ResponseResult getUserProjectPageData(String userId, String keyword, int page, int pageSize) {
+        Query query = new Query();
+        projectDao.addSort(query, "{\"_id\":\"asc\"}");
+        // 项目名关键词模糊
+        if (StringUtils.isNotBlank(keyword)){
+            projectDao.addKeywordFuzz(query, keyword, "projectName");
+        }
+        // 用户拥有权限的项目
+        if (StringUtils.isNotBlank(userId) && !userDao.findOneById(userId).isSuperAdmin()){
+            List<String> projectIds = userDao.findOneById(userId).getRoles().stream().map(UserRole::getProjectId).collect(Collectors.toList());
+            query.addCriteria(Criteria.where("_id").in(projectIds));
+        }
+        long total = projectDao.count(query);
+        int skipCount = (page - 1) * pageSize;
+        query.limit(pageSize);
+        query.skip(skipCount);
+        List<Project> projectList = projectDao.findList(query);
+        List<JSONObject> resList = projectList.stream().map(project -> {
+            JSONObject result = new JSONObject();
+            result.put("id", project.getId());
+            result.put("projectName", project.getProjectName());
+            return result;
+        }).collect(Collectors.toList());
+        return ResponseResult.success(PageVO.createPageVo(total, page, pageSize, resList));
+    }
+
     /**
      * 获取舆情对应项目的情感标签
      * @param brandName
@@ -662,6 +712,12 @@ public class ProjectServiceImpl implements ProjectService {
         Map<String, Object> originPermission = new HashMap<>();
         originPermission.put("origin", Objects.nonNull(project.getModuleShowList()) && project.getModuleShowList().contains(2));
         permissionList.add(originPermission);
+        // 友好渠道榜、敏感渠道榜是否有数据
+        Map<String, Object> channelPermission = new HashMap<>();
+        JSONObject jsonObject = GlobalPojo.PROJECT_EMOTION_CHANNEL_DATA.get(project.getId());
+        channelPermission.put("positiveChannel", Objects.nonNull(jsonObject) && Objects.nonNull(jsonObject.getLong(EmotionEnum.POSITIVE.getName())) && 0 != jsonObject.getLong(EmotionEnum.POSITIVE.getName()));
+        channelPermission.put("negativeChannel", Objects.nonNull(jsonObject) && Objects.nonNull(jsonObject.getLong(EmotionEnum.NEGATIVE.getName())) && 0 != jsonObject.getLong(EmotionEnum.NEGATIVE.getName()));
+        permissionList.add(channelPermission);
         return permissionList;
     }
 

src/main/java/com/zhiwei/brandkbs2/service/impl/ProjectWarnServiceImpl.java

@@ -170,12 +170,19 @@ public class ProjectWarnServiceImpl implements ProjectWarnService {
     public ResponseResult getProjectWarnCriteriaDefault(String projectId, String type) {
         switch (type) {
             case "舆情动态":
-                BrandkbsYuQingConfig config = new BrandkbsYuQingConfig();
+//                BrandkbsYuQingConfig config = new BrandkbsYuQingConfig();
+//                config.setPoliticsLevels(Collections.singletonList("全部"));
+//                config.setFields(Collections.singletonList("全部"));
+//                config.setPrimary(Collections.singletonList(Boolean.TRUE));
+//                config.setMarkerTags(Arrays.asList("正面", "负面", "中性"));
+//                config.setDuplicate(Boolean.FALSE);
+//                config.setPushInterval(10 * ONE_MINUTE);
+                BrandkbsYuQingConfigNew config = new BrandkbsYuQingConfigNew();
                 config.setPoliticsLevels(Collections.singletonList("央级"));
-                config.setFields(Collections.singletonList("财经"));
-                config.setPrimary(Collections.singletonList(Boolean.TRUE));
-                config.setMarkerTags(Collections.singletonList("负面"));
-                config.setDuplicate(Boolean.FALSE);
+                config.setFields(Collections.singletonList("全部"));
+                config.setPrimary(Collections.singletonList(Boolean.FALSE));
+                config.setMarkerTags(Arrays.asList("正面", "负面", "中性"));
+                config.setDistinctFilter(true);
                 config.setPushInterval(10 * ONE_MINUTE);
                 return ResponseResult.success(config);
             case "渠道参与":
@@ -213,6 +220,89 @@ public class ProjectWarnServiceImpl implements ProjectWarnService {
         return ResponseResult.success();
     }
 
+    private BrandkbsWarnTemplate brandkbsWarnTemplate4Yuqing(BrandkbsYuQingConfigNew config, List<BaseMap> datas, long start, long end, AbstractProject project) {
+        if (datas.isEmpty()) {
+            return null;
+        }
+        int firstCount = datas.size();
+        // key1
+        String key1 = "【品见】舆情动态-" + project.getProjectName();
+        // key2
+//        String key2Header = "全部";
+        String key2Header = "";
+        StringBuilder mediaChannel = new StringBuilder();
+        List<String> politicsLevels = config.getPoliticsLevels();
+        List<String> fields = config.getFields();
+        if(Tools.isEmpty(politicsLevels)){
+            key2Header += "【全部级别】";
+        }else if(politicsLevels.contains("全部")){
+            key2Header += "【重点级别】";
+        }else{
+            politicsLevels.forEach(politicsLevel -> mediaChannel.append(politicsLevel).append("、"));
+        }
+        if (Tools.isEmpty(fields)) {
+            if (0 != key2Header.length()) {
+                key2Header += "、";
+                key2Header += "【全部领域】,";
+            } else {
+                key2Header += "【全部领域】";
+            }
+        } else if (fields.contains("全部")) {
+            if (0 != key2Header.length()) {
+                key2Header += "、";
+                key2Header += "【重点领域】,";
+            } else {
+                key2Header += "【重点领域】";
+            }
+        } else {
+            fields.forEach(field -> mediaChannel.append(field).append("、"));
+        }
+        if (0 != mediaChannel.length()) {
+            if (0 != key2Header.length()) {
+                key2Header += "、";
+            }
+            // 替换“全部” 并去掉末位的、
+            key2Header += mediaChannel.deleteCharAt(mediaChannel.length() - 1).toString();
+            key2Header += "媒体,";
+        }
+        StringBuilder dataType = new StringBuilder();
+        config.getMarkerTags().forEach(emotion -> dataType.append(emotion).append(" "));
+//        dataType.append("(本品)");
+        // 如果有竞品
+//        if (CollectionUtils.isNotEmpty(config.getContends())) {
+//            dataType.append(",");
+//            config.getContends().forEach(contend -> dataType.append(projectService.getProjectByContendId(project.getId(), contend).getBrandName()).append(" "));
+//            dataType.append("(竞品)");
+//        }
+        String key2 = key2Header + dataType;
+        // key3
+        String key3 = Constant.SPEC_MINUTE_FORMAT.format(start) + " ~ " + Constant.SPEC_MINUTE_FORMAT.format(end);
+        // key4
+        String key4;
+        StringBuilder keyBuilder4 = new StringBuilder();
+        for (int i = 0; i < Math.min(datas.size(), 2); i++) {
+            BaseMap baseMap = datas.get(i);
+            String platform = baseMap.getPlatform();
+            String source = baseMap.getSource();
+            String title = baseMap.getTitle();
+            keyBuilder4.append(platform).append("-").append(source).append("\r\n");
+            // 最后一条
+            if (datas.size() == i + 1) {
+                keyBuilder4.append(i + 1).append("、").append(title);
+            } else {
+                keyBuilder4.append(i + 1).append("、").append(title).append("\r\n");
+            }
+        }
+        if (datas.size() > 2) {
+            key4 = keyBuilder4.append("...").toString();
+        } else {
+            key4 = keyBuilder4.toString();
+        }
+        List<BaseMapCompound> collect = datas.stream().map(baseMap -> BaseMapCompound.createFromBaseMap(baseMap, project.getId())).collect(Collectors.toList());
+        return new BrandkbsWarnTemplate(firstCount, key1, key2, key3, key4, new BrandkbsYuQingWarn(collect));
+    }
+
+    @Deprecated
     private BrandkbsWarnTemplate brandkbsWarnTemplate4Yuqing(BrandkbsYuQingConfig config, List<BaseMap> datas, long start, long end, AbstractProject project) {
         if (datas.isEmpty()) {
             return null;
@@ -613,6 +703,7 @@ public class ProjectWarnServiceImpl implements ProjectWarnService {
     }
 
     @Override
+    @Deprecated
     public ResponseResult getYuqingWaring(String projectId, BrandkbsYuQingConfig config) {
         Project project = null;
         long end = System.currentTimeMillis();
@@ -671,6 +762,60 @@ public class ProjectWarnServiceImpl implements ProjectWarnService {
     }
 
     @Override
+    public ResponseResult getYuqingWaringNew(String projectId, BrandkbsYuQingConfigNew config) {
+        Project project;
+        long end = System.currentTimeMillis();
+        long start = end - config.getPushInterval();
+        try {
+            project = projectService.getProjectById(projectId);
+            log.info("获取新舆情动态预警开始,project:{}", project.getProjectName());
+            EsClientDao.SearchHelper helper = EsClientDao.createSearchHelper();
+            // 发声渠道
+            BoolQueryBuilder postFilter = EsQueryTools.assembleCacheMapsQueryWithProject(projectId, config);
+            // 时间(标注时间)
+            postFilter.must(QueryBuilders.rangeQuery("mtime").gte(start).lt(end));
+            // 文章类型
+            if (CollectionUtils.isNotEmpty(config.getPrimary()) && config.getPrimary().size() == 1) {
+                postFilter.must(EsQueryTools.assembleForward2Query(config.getPrimary().get(0)));
+            }
+            // 舆情调性
+            BoolQueryBuilder sourceBuilder = QueryBuilders.boolQuery();
+            if (CollectionUtils.isNotEmpty(config.getMarkerTags())) {
+                config.getMarkerTags().forEach(tagName -> sourceBuilder.should(QueryBuilders.termQuery("brandkbs_mark_cache_maps.name.keyword", tagName)));
+            }
+            postFilter.must(sourceBuilder);
+            // 自定义渠道
+            if (StringUtils.isNotBlank(config.getChannel())){
+                postFilter.must(EsQueryTools.assembleSourceQuery(config.getChannel()));
+            }
+            helper.setPostFilter(postFilter);
+            SearchHits searchHits = esClientDao.searchHits(helper);
+            log.info("获取新舆情动态预警-es搜索到数据:{}条,剩余关键词筛选未进行,project:{}", searchHits.getHits().length, project.getProjectName());
+            List<BaseMap> datas = Arrays.stream(searchHits.getHits()).map(hit -> {
+                Map<String, Object> sourceMap = hit.getSourceAsMap();
+                BaseMap baseMap = Tools.getBaseFromEsMap(sourceMap);
+                // 关键词命中
+                if (!Tools.isContains(config.getKeyword(), baseMap.getTitle() + baseMap.getContent())) {
+                    return null;
+                }
+                baseMap.setTitle(baseMap.getTitleNullOptionalContent());
+                baseMap.subStringContentTop();
+                return baseMap;
+            }).filter(Objects::nonNull).collect(Collectors.toList());
+            // 开启去重
+            if (Boolean.TRUE.equals(config.isDistinctFilter())) {
+                datas = duplicateBaseMapList(datas);
+            }
+            log.info("获取舆情动态预警结束,project:{}", project.getProjectName());
+            // 封装返回值
+            return ResponseResult.success(brandkbsWarnTemplate4Yuqing(config, datas, start, end, project));
+        } catch (Exception e) {
+            log.info("获取舆情动态预警失败,projectId:{},start:{},end:{}", projectId, start, end, e);
+        }
+        return ResponseResult.failure("获取舆情动态预警失败");
+    }
+
+    @Override
     public ResponseResult getChannelWaring(String projectId, BrandkbsChannelConfig config) {
         Project project = null;
         long end = System.currentTimeMillis();

src/main/java/com/zhiwei/brandkbs2/service/impl/TaskServiceImpl.java

@@ -3,14 +3,13 @@ package com.zhiwei.brandkbs2.service.impl;
 import com.zhiwei.brandkbs2.auth.UserThreadLocal;
 import com.zhiwei.brandkbs2.common.GlobalPojo;
 import com.zhiwei.brandkbs2.config.Constant;
-import com.zhiwei.brandkbs2.dao.AggreeResultDao;
-import com.zhiwei.brandkbs2.dao.BrandkbsTaskDao;
-import com.zhiwei.brandkbs2.dao.ChannelDao;
-import com.zhiwei.brandkbs2.dao.ReportSettingsDao;
+import com.zhiwei.brandkbs2.dao.*;
 import com.zhiwei.brandkbs2.enmus.ReportTypeEnum;
 import com.zhiwei.brandkbs2.es.ChannelEsDao;
 import com.zhiwei.brandkbs2.es.EsClientDao;
+import com.zhiwei.brandkbs2.exception.ExceptionCast;
 import com.zhiwei.brandkbs2.listener.ApplicationProjectListener;
+import com.zhiwei.brandkbs2.model.CommonCodeEnum;
 import com.zhiwei.brandkbs2.pojo.*;
 import com.zhiwei.brandkbs2.service.*;
 import com.zhiwei.brandkbs2.util.Tools;
@@ -19,6 +18,8 @@ import org.apache.commons.lang3.tuple.Pair;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchStatusException;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.data.mongodb.core.query.Query;
 import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
 import org.springframework.stereotype.Service;
 
@@ -28,6 +29,7 @@ import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
 
 /**
@@ -61,6 +63,9 @@ public class TaskServiceImpl implements TaskService {
     @Resource(name = "aggreeResultDaoImpl")
     AggreeResultDao aggreeResultDao;
 
+    @Resource(name = "nonManualProjectPlanDao")
+    NonManualProjectPlanDao nonManualProjectPlanDao;
+
     @Resource(name = "brandkbsTaskServiceImpl")
     BrandkbsTaskService brandkbsTaskService;
 
@@ -82,6 +87,9 @@ public class TaskServiceImpl implements TaskService {
     @Resource(name = "customEventServiceImpl")
     CustomEventService customEventService;
 
+    @Resource(name = "markDataServiceImpl")
+    MarkDataService markDataService;
+
     @Resource(name = "taskServiceExecutor")
     ThreadPoolTaskExecutor taskServiceExecutor;
 
@@ -155,10 +163,10 @@ public class TaskServiceImpl implements TaskService {
                 channelService.getActiveChannelList(Constant.PRIMARY_CONTEND_ID,  times[0], times[1], 50, false);
                 // 友好渠道榜
                 channelService.getPositiveList(Constant.PRIMARY_CONTEND_ID, null, null, sorter, times[0], times[1], 50, false);
-                channelService.getPositiveChannelList(Constant.PRIMARY_CONTEND_ID,  times[0], times[1], 50, sorter, false);
+                channelService.getPositiveChannelList(Constant.PRIMARY_CONTEND_ID,  times[0], times[1], 50, null, false);
                 // 敏感渠道榜
                 channelService.getNegativeList(Constant.PRIMARY_CONTEND_ID, null, null, sorter, times[0], times[1], 50, false);
-                channelService.getNegativeChannelList(Constant.PRIMARY_CONTEND_ID,  times[0], times[1], 50, sorter, false);
+                channelService.getNegativeChannelList(Constant.PRIMARY_CONTEND_ID,  times[0], times[1], 50, null, false);
             });
             log.info("项目:{}-渠道榜单缓存已完成:{}个", project.getProjectName(), total.incrementAndGet());
             return null;
@@ -324,6 +332,56 @@ public class TaskServiceImpl implements TaskService {
         }
     }
 
+    @Override
+    public void calculateProjectAvg() {
+        AtomicLong total = new AtomicLong();
+        GlobalPojo.PROJECT_MAP.forEach((projectId, project) ->{
+            try {
+                // 取近一年
+                long endTime = System.currentTimeMillis();
+                long startTime = endTime - Constant.ONE_YEAR;
+                // 项目创建时间在近一年内则取项目创建时间
+                startTime = project.getCTime() > startTime ? project.getCTime() : startTime;
+                // 同时计算非人工项目各方案均值
+                if (!project.isManual()) {
+                    List<NonManualProjectPlan> plans = nonManualProjectPlanDao.findList(new Query(Criteria.where("projectId").is(projectId)));
+                    for (NonManualProjectPlan plan : plans) {
+                        markDataService.countYuqingAmountAvg(startTime, endTime, projectId, plan.getId());
+                        markDataService.countEmotionDistributionAvg(startTime, endTime, projectId, plan.getId());
+                        markDataService.countImportantPlatformPercentageAvg(startTime, endTime, projectId, plan.getId());
+                    }
+                }
+                markDataService.countYuqingAmountAvg(startTime, endTime, projectId, null);
+                markDataService.countEmotionDistributionAvg(startTime, endTime, projectId, null);
+                markDataService.countImportantPlatformPercentageAvg(startTime, endTime, projectId, null);
+                log.info("项目:{}-均值计算已完成:{}个", project.getProjectName(), total.incrementAndGet());
+            }catch (Exception e){
+                ExceptionCast.cast(CommonCodeEnum.FAIL, "calculateProjectAvg异常-projectId:" + projectId, e);
+            }
+        });
+    }
+
+    @Override
+    public void yuqingAnalyzeHighWordCache() {
+        AtomicInteger total = new AtomicInteger();
+        Long[] time = commonService.getTimeRangeMonth();
+        List<CompletableFuture<Object>> projectFutures = GlobalPojo.PROJECT_MAP.values().stream().map(project -> CompletableFuture.supplyAsync(() -> {
+            UserThreadLocal.set(new UserInfo().setProjectId(project.getId()));
+            markDataService.getHighWord(time[0], time[1], null, false);
+            log.info("项目:{}-{}-词云缓存已完成:{}个", project.getProjectName(), project.getId(), total.incrementAndGet());
+            return null;
+        }, cacheServiceExecutor)).collect(Collectors.toList());
+        AtomicInteger total2 = new AtomicInteger();
+        List<CompletableFuture<Object>> planFutures = nonManualProjectPlanDao.findList(new Query()).stream().map(plan -> CompletableFuture.supplyAsync(() -> {
+            UserThreadLocal.set(new UserInfo().setProjectId(plan.getProjectId()));
+            markDataService.getHighWord(time[0], time[1], plan.getId(), false);
+            log.info("方案:{}-{}-词云缓存已完成:{}个", plan.getName(), plan.getId(), total2.incrementAndGet());
+            return null;
+        }, cacheServiceExecutor)).collect(Collectors.toList());
+        projectFutures.addAll(planFutures);
+        CompletableFuture.allOf(projectFutures.toArray(new CompletableFuture[0])).join();
+    }
+
     private boolean reportSendByProject(Project project) {
         boolean flag = false;
         // 扫描setting信息并生成对应报告

src/main/java/com/zhiwei/brandkbs2/task/ControlCenter.java

@@ -45,6 +45,7 @@ public class ControlCenter {
         try {
             taskService.messageFlowCache();
             taskService.customEventCache();
+            taskService.yuqingAnalyzeHighWordCache();
         } catch (Exception e) {
             log.error("定时按天缓存数据-出错", e);
         } finally {
@@ -91,4 +92,16 @@ public class ControlCenter {
         }
     }
 
+    @Async("scheduledExecutor")
+    @Scheduled(cron = "0 0 3 ? * SUN")
+    public void calculateProjectAvg() {
+        log.info("定时每周计算项目均值-启动");
+        try {
+            taskService.calculateProjectAvg();
+        } catch (Exception e) {
+            log.error("定时按周计算项目均值-出错", e);
+        } finally {
+            log.info("定时按周计算项目均值-结束");
+        }
+    }
 }

src/main/java/com/zhiwei/brandkbs2/util/IndexUtil.java

@@ -58,16 +58,17 @@ public class IndexUtil {
         start = truncDate(start, pattern);
         end = truncDate(end, pattern);
         List<String> list = Lists.newArrayList();
-        Period period = new Period(start.getTime(), end.getTime(), PeriodType.months());
+        Period monthPeriod = new Period(start.getTime(), end.getTime(), PeriodType.months());
+        Period yearPeriod = new Period(start.getTime(), end.getTime(), PeriodType.years());
         switch (pattern) {
             case "yyyy": {
-                for (int i = 0; i <= period.getYears(); i++) {
+                for (int i = 0; i <= yearPeriod.getYears(); i++) {
                     list.add(index + "_" + DF_yyyy.format(DateUtils.addYears(start, i)));
                 }
                 break;
             }
             case "yyyyMM": {
-                for (int i = 0; i <= period.getMonths(); i++) {
+                for (int i = 0; i <= monthPeriod.getMonths(); i++) {
                     list.add(index + "_" + DF_yyyyMM.format(DateUtils.addMonths(start, i)));
                 }
                 break;

src/main/java/com/zhiwei/brandkbs2/util/RedisUtil.java

@@ -7,6 +7,7 @@ import org.springframework.data.redis.core.StringRedisTemplate;
 import org.springframework.stereotype.Component;
 
 import java.util.List;
+import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
@@ -82,6 +83,34 @@ public class RedisUtil {
         return RedisKeyPrefix.NON_MANUAL_PROJECT_MARK_MAX_GID + Tools.concat(projectId, planId, userId);
     }
 
+    public static String getYuqingAnalyzeProjectAvgCountKey(String projectId, String contendId, String planId){
+        if (Objects.isNull(planId)){
+            return RedisKeyPrefix.YUQING_ANALYZE_PROJECT_AVG_COUNT + Tools.concat(projectId, contendId);
+        }
+        return RedisKeyPrefix.YUQING_ANALYZE_PROJECT_AVG_COUNT + Tools.concat(projectId, contendId, planId);
+    }
+
+    public static String getYuqingAnalyzeEmotionDistributionAvgKey(String projectId, String contendId, String emotion, String planId){
+        if (Objects.isNull(planId)){
+            return RedisKeyPrefix.YUQING_ANALYZE_EMOTION_DISTRIBUTION_AVG + Tools.concat(projectId, contendId, emotion);
+        }
+        return RedisKeyPrefix.YUQING_ANALYZE_EMOTION_DISTRIBUTION_AVG + Tools.concat(projectId, contendId, emotion, planId);
+    }
+
+    public static String getYuqingAnalyzePlatformAvgCountKey(String projectId, String contendId, String planId){
+        if (Objects.isNull(planId)){
+            return RedisKeyPrefix.YUQING_ANALYZE_PLATFORM_AVG_COUNT + Tools.concat(projectId, contendId);
+        }
+        return RedisKeyPrefix.YUQING_ANALYZE_PLATFORM_AVG_COUNT + Tools.concat(projectId, contendId, planId);
+    }
+
+    public static String getYuqingAnalyzeHighWordKey(String projectId, String contendId, String planId, Long startTime, Long endTime){
+        if (Objects.isNull(planId)){
+            return RedisKeyPrefix.YUQING_ANALYZE_HIGH_WORD + Tools.concat(projectId, contendId, startTime, endTime);
+        }
+        return RedisKeyPrefix.YUQING_ANALYZE_HIGH_WORD + Tools.concat(projectId, contendId, planId, startTime, endTime);
+    }
+
     public void setExpire(String key, String value, long timeout, TimeUnit unit) {
         stringRedisTemplate.opsForValue().set(key, value, timeout, unit);
     }

src/main/resources/application-dev.properties

@@ -117,6 +117,7 @@ warn.hotEvent.url=https://auto-push.zhiweidata.com/qbjc/brandkbsPush/interface/b
 warn.hotTop.url=https://auto-push.zhiweidata.com/qbjc/brandkbsPush/interface/brandkbs/config/hotTop/{1}
 warn.yuQing.url=https://auto-push.zhiweidata.com/qbjc/brandkbsPush/interface/brandkbs/config/yuQing/{1}
 warn.taskSwitch.url=https://auto-push.zhiweidata.com/qbjc/brandkbsPush/interface/brandkbs/enable/used
+warn.push.ticket.url=https://auto-push.zhiweidata.com/qbjc/pushTask/interface/task/token/ticket
 #\u9884\u8B66\u5916\u90E8\u63A5\u53E3
 ef.external.filterNew.url=https://ef.zhiweidata.com/external/filterNew.do?firstTypes={1}&start={2}&end={3}
 hot.search.url=https://hotsearch-manage.zhiweidata.com/hotsearch/hotSearch/findNewHotSearch?type={1}

src/main/resources/application-local.properties

@@ -123,6 +123,7 @@ warn.hotEvent.url=http://192.168.0.225:11003/qbjc/brandkbsPush/interface/brandkb
 warn.hotTop.url=http://192.168.0.225:11003/qbjc/brandkbsPush/interface/brandkbs/config/hotTop/{1}
 warn.yuQing.url=http://192.168.0.225:11003/qbjc/brandkbsPush/interface/brandkbs/config/yuQing/{1}
 warn.taskSwitch.url=http://192.168.0.225:11003/qbjc/brandkbsPush/interface/brandkbs/enable/used
+warn.push.ticket.url=http://192.168.0.225:11003/qbjc/pushTask/interface/task/token/ticket
 #\u9884\u8B66\u5916\u90E8\u63A5\u53E3
 ef.external.filterNew.url=https://ef.zhiweidata.com/external/filterNew.do?firstTypes={1}&start={2}&end={3}
 hot.search.url=https://hotsearch-manage.zhiweidata.com/hotsearch/hotSearch/findNewHotSearch?type={1}

src/main/resources/application-prod.properties

@@ -117,6 +117,7 @@ warn.hotEvent.url=https://auto-push.zhiweidata.com/qbjc/brandkbsPush/interface/b
 warn.hotTop.url=https://auto-push.zhiweidata.com/qbjc/brandkbsPush/interface/brandkbs/config/hotTop/{1}
 warn.yuQing.url=https://auto-push.zhiweidata.com/qbjc/brandkbsPush/interface/brandkbs/config/yuQing/{1}
 warn.taskSwitch.url=https://auto-push.zhiweidata.com/qbjc/brandkbsPush/interface/brandkbs/enable/used
+warn.push.ticket.url=https://auto-push.zhiweidata.com/qbjc/pushTask/interface/task/token/ticket
 #\u9884\u8B66\u5916\u90E8\u63A5\u53E3
 ef.external.filterNew.url=https://ef.zhiweidata.com/external/filterNew.do?firstTypes={1}&start={2}&end={3}
 hot.search.url=https://hotsearch-manage.zhiweidata.com/hotsearch/hotSearch/findNewHotSearch?type={1}