Merge branch 'feature' into 'dev'

2022/7/12

See merge request !10

src/main/java/com/zhiwei/brandkbs2/common/GenericAttribute.java

@@ -69,16 +69,6 @@ public class GenericAttribute {
     public static final String ROLE_ID = "roleId";
     public static final String PROJECT_ID = "projectId";
     public static final String AVATAR_URL= "avatarUrl";
-    /**
-     * 标签相关
-     */
-    public static final String EMOTION_LABEL_KEY = "情感倾向";
-    public static final String BRAND_LABEL_KEY = "品牌归属";
-    /**
-     * 报告相关
-     */
-    public static final String MONTH_REPORT = "月报";
-    public static final String WEEK_REPORT = "周报";
 
 //    public enum ChannelParam{
 //        负面稿件数("negativeArticles"),

src/main/java/com/zhiwei/brandkbs2/common/GlobalPojo.java

@@ -60,6 +60,7 @@ public class GlobalPojo {
     private void updatePojo(String logMsg) {
         PLATFORMS = systemInfoService.getPlatforms();
         TAGS = systemInfoService.getTags().stream().collect(Collectors.groupingBy(Tag::getGroupName));
+        log.info("{}-获取PLATFORMS-size:{},TAGS-size:{}", logMsg, PLATFORMS.size(), TAGS.size());
     }
 
 }

src/main/java/com/zhiwei/brandkbs2/config/Constant.java

@@ -14,4 +14,16 @@ public class Constant {
      * 自定义fid分隔符号
      */
     public static final String DEFAULT_SEPARATOR = "_";
+
+    /**
+     * 标签相关
+     */
+    public static final String EMOTION_LABEL_KEY = "情感倾向";
+    public static final String BRAND_LABEL_KEY = "品牌归属";
+
+    /**
+     * 任务名分类
+     */
+    public static final String TASK_REPORT_JOB = "定时生成项目简报数据";
+    public static final int INIT_VERSION = 0;
 }

src/main/java/com/zhiwei/brandkbs2/controller/AppArticleController.java

@@ -5,14 +5,12 @@ import com.zhiwei.brandkbs2.enmus.RoleEnum;
 import com.zhiwei.brandkbs2.model.ResponseResult;
 import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
 import com.zhiwei.brandkbs2.service.IMarkDataService;
+import com.zhiwei.brandkbs2.service.IReportService;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiImplicitParam;
 import io.swagger.annotations.ApiImplicitParams;
 import io.swagger.annotations.ApiOperation;
-import org.springframework.web.bind.annotation.GetMapping;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestParam;
-import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.bind.annotation.*;
 
 import javax.annotation.Resource;
 
@@ -31,9 +29,12 @@ public class AppArticleController extends BaseController {
     @Resource(name = "markDataServiceImpl")
     IMarkDataService markDataService;
 
+    @Resource(name = "reportServiceImpl")
+    IReportService reportService;
+
     @ApiOperation("舆情列表")
-    @GetMapping("/mark/list")
-    public ResponseResult getYuqingMarkList(MarkSearchDTO markSearchDTO) {
+    @PostMapping("/mark/list")
+    public ResponseResult getYuqingMarkList(@RequestBody MarkSearchDTO markSearchDTO) {
         return ResponseResult.success(markDataService.getYuqingMarkList(markSearchDTO));
     }
 
@@ -43,14 +44,14 @@ public class AppArticleController extends BaseController {
         return ResponseResult.success(markDataService.getYuqingMarkCriteria(linkedGroupId));
     }
 
-    @ApiOperation("舆情分析-提要")
+    @ApiOperation("舆情分析-舆情概览")
     @ApiImplicitParams({@ApiImplicitParam(name = "startTime", value = "开始时间", paramType = "query", dataType = "long"), @ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long")})
     @GetMapping("/analyze/summary")
     public ResponseResult getAnalyzeSummary(@RequestParam(value = "startTime", required = false) Long startTime, @RequestParam(value = "endTime", required = false) Long endTime) {
         return ResponseResult.success(markDataService.getAnalyzeSummary(startTime, endTime, true));
     }
 
-    @ApiOperation("舆情分析-传播趋势")
+    @ApiOperation("舆情分析-舆情趋势分布")
     @ApiImplicitParams({
             @ApiImplicitParam(name = "startTime", value = "开始时间", paramType = "query", dataType = "long"),
             @ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long")
@@ -61,7 +62,7 @@ public class AppArticleController extends BaseController {
         return ResponseResult.success(markDataService.getMarkSpread(startTime, endTime, true));
     }
 
-    @ApiOperation("舆情分析-情感分布")
+    @ApiOperation("舆情分析-平台曝光对比")
     @ApiImplicitParams({
             @ApiImplicitParam(name = "startTime", value = "开始时间", paramType = "query", dataType = "long"),
             @ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long")
@@ -83,14 +84,14 @@ public class AppArticleController extends BaseController {
         return ResponseResult.success(markDataService.getMarkPlatformProportion(startTime, endTime, true));
     }
 
-    @ApiOperation("舆情分析-渠道分布")
+    @ApiOperation("舆情分析-渠道参与")
     @ApiImplicitParams({@ApiImplicitParam(name = "startTime", value = "开始时间", paramType = "query", dataType = "long"), @ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long")})
     @GetMapping("/channel/proportion")
     public ResponseResult getMarkChannelProportion(@RequestParam(value = "startTime", required = false) Long startTime, @RequestParam(value = "endTime", required = false) Long endTime) {
         return ResponseResult.success(markDataService.getMarkChannelProportion(startTime, endTime, true));
     }
 
-    @ApiOperation("舆情分析-高频词分布")
+    @ApiOperation("舆情分析-舆情高频词")
     @ApiImplicitParams({
             @ApiImplicitParam(name = "startTime", value = "开始时间", paramType = "query", dataType = "long"),
             @ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long")
@@ -101,6 +102,12 @@ public class AppArticleController extends BaseController {
         return ResponseResult.success(markDataService.getMarkHighWord(startTime, endTime));
     }
 
+    @ApiOperation("舆情简报-分类统计")
+    @GetMapping("/aggCount")
+    public ResponseResult getReportsAggCount() {
+        return ResponseResult.success(reportService.getReportsAggCount());
+    }
+
 
 
 }

src/main/java/com/zhiwei/brandkbs2/dao/IBrandkbsTaskDao.java

+package com.zhiwei.brandkbs2.dao;
+
+import com.zhiwei.brandkbs2.pojo.BrandkbsTask;
+
+/**
+ * @ClassName: IBrandkbsTaskDao
+ * @Description IBrandkbsTaskDao
+ * @author: sjj
+ * @date: 2022-07-12 17:25
+ */
+public interface IBrandkbsTaskDao extends IBaseMongoDao<BrandkbsTask> {
+
+    /**
+     * 根据任务名称获取任务
+     *
+     * @param taskName 任务名称
+     * @param version  版本号
+     * @return 任务对象
+     */
+    BrandkbsTask findTaskByTaskNameAndVersion(String taskName, int version);
+
+    /**
+     * 修改定时任务版本
+     *
+     * @param task 任务对象
+     * @return 改变的行数
+     */
+    long updateTaskVersion(BrandkbsTask task);
+
+}

src/main/java/com/zhiwei/brandkbs2/dao/impl/BrandkbsTaskDaoImpl.java

+package com.zhiwei.brandkbs2.dao.impl;
+
+import com.zhiwei.brandkbs2.dao.IBrandkbsTaskDao;
+import com.zhiwei.brandkbs2.pojo.BrandkbsTask;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.data.mongodb.core.query.Query;
+import org.springframework.data.mongodb.core.query.Update;
+import org.springframework.stereotype.Component;
+
+/**
+ * @ClassName: BrandkbsTaskDaoImpl
+ * @Description BrandkbsTaskDaoImpl
+ * @author: sjj
+ * @date: 2022-07-12 17:27
+ */
+@Component("brandkbsTaskDao")
+public class BrandkbsTaskDaoImpl extends BaseMongoDaoImpl<BrandkbsTask> implements IBrandkbsTaskDao {
+    private static final String COLLECTION_PREFIX = "brandkbs_task";
+
+    public BrandkbsTaskDaoImpl() {
+        super(COLLECTION_PREFIX);
+    }
+
+    @Override
+    public BrandkbsTask findTaskByTaskNameAndVersion(String taskName, int version) {
+        Query query = Query.query(Criteria.where("taskName").is(taskName).and("version").is(version));
+        return mongoTemplate.findOne(query, clazz, COLLECTION_PREFIX);
+    }
+
+    @Override
+    public long updateTaskVersion(BrandkbsTask task) {
+        int version = task.getVersion();
+        Query query =
+                Query.query(Criteria.where("_id").is(task.getId()).and("version").is(version).and("executableTime").is(task.getExecutableTime()));
+        return mongoTemplate.updateFirst(query, Update.update("version", ++version), COLLECTION_PREFIX).getModifiedCount();
+    }
+}

src/main/java/com/zhiwei/brandkbs2/dao/impl/ChannelDaoImpl.java

@@ -24,9 +24,12 @@ public class ChannelDaoImpl extends BaseMongoDaoImpl<Channel> implements IChanne
 
     @Override
     public Channel queryUnique(ChannelIndex channelIndex) {
-        Query query = Query.query(Criteria.where("linkedGroupId").is(channelIndex.getLinkedGroupId()).and("platform").is(channelIndex.getPlatform()).and(
-                "realSource").is(channelIndex.getRealSource()).and("source").is(channelIndex.getSource()));
-        return mongoTemplate.findOne(query, clazz);
+        Query query = Query.query(Criteria.where("linkedGroupId").is(channelIndex.getLinkedGroupId()).
+                and("projectId").is(channelIndex.getProjectId()).
+                and("platform").is(channelIndex.getPlatform()).
+                and("realSource").is(channelIndex.getRealSource()).
+                and("source").is(channelIndex.getSource()));
+        return mongoTemplate.findOne(query, clazz, COLLECTION_PREFIX);
     }
 
 }

src/main/java/com/zhiwei/brandkbs2/easyexcel/dto/UploadEventDataDTO.java

@@ -4,6 +4,7 @@ import com.alibaba.excel.annotation.ExcelProperty;
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.brandkbs2.common.GenericAttribute;
+import com.zhiwei.brandkbs2.config.Constant;
 import com.zhiwei.brandkbs2.pojo.Event;
 import com.zhiwei.brandkbs2.pojo.EventData;
 import com.zhiwei.brandkbs2.util.Tools;
@@ -80,8 +81,8 @@ public class UploadEventDataDTO {
         eventData.setContent(content);
         eventData.setTime(this.getTime().getTime());
         JSONObject tagInfo = new JSONObject();
-        Optional.ofNullable(this.getEmotion()).ifPresent(emotion -> tagInfo.put(GenericAttribute.EMOTION_LABEL_KEY, emotion));
-        Optional.ofNullable(this.getBrand()).ifPresent(brand -> tagInfo.put(GenericAttribute.BRAND_LABEL_KEY, brand));
+        Optional.ofNullable(this.getEmotion()).ifPresent(emotion -> tagInfo.put(Constant.EMOTION_LABEL_KEY, emotion));
+        Optional.ofNullable(this.getBrand()).ifPresent(brand -> tagInfo.put(Constant.BRAND_LABEL_KEY, brand));
         eventData.setTagInfo(JSON.toJSONString(tagInfo));
         eventData.setEmotion(this.getEmotion());
         eventData.setForward(this.isForward);

src/main/java/com/zhiwei/brandkbs2/es/EsClientDao.java

@@ -50,7 +50,7 @@ import java.util.stream.Collectors;
 public class EsClientDao {
     private static final Logger log = LogManager.getLogger(EsClientDao.class);
     private static final FastDateFormat DF = FastDateFormat.getInstance("yyyy-MM-dd HH:mm:ss");
-    private static final String[] CHANNEL_RECORD_FETCH_SOURCE = new String[]{"c5", "foreign", "real_source", "source", "mtime", "time", "brandkbs_cache_maps"};
+    private static final String[] CHANNEL_RECORD_FETCH_SOURCE = new String[]{"id", "c5", "foreign", "real_source", "source", "mtime", "time", "brandkbs_cache_maps"};
     private static final String[] EVENT_FETCH_SOURCE = new String[]{"ind_full_text", "c5", "real_source", "source", "mtime", "time", "url", "mtag"};
 
     private static final Long ONE_HOUR = 60 * 60 * 1000L;
@@ -185,7 +185,12 @@ public class EsClientDao {
                         if (null == v) {
                             v = new ChannelIndex.ChannelRecord();
                         }
+                        try {
                             return v.mergeRecord(new ChannelIndex.ChannelRecord((long) result.get(GenericAttribute.ES_TIME), String.valueOf(result.get("id"))));
+                        } catch (Exception e) {
+                            log.error("searchChannelRecord-error-id:{}", result.get("id"), e);
+                        }
+                        return v;
                     });
                 }
             }

src/main/java/com/zhiwei/brandkbs2/pojo/BrandkbsTask.java

+package com.zhiwei.brandkbs2.pojo;
+
+import lombok.Getter;
+import lombok.Setter;
+
+/**
+ * @author sjj
+ * @version 1.0
+ * @description 分布式任务控制表类
+ * @date 2022年7月12日17:24:16
+ */
+@Setter
+@Getter
+public class BrandkbsTask extends AbstractBaseMongo{
+    /**
+     * 任务描述
+     */
+    private String taskName;
+    /**
+     * 版本号
+     */
+    private int version;
+    /**
+     * 可执行时间
+     */
+    private Long executableTime;
+}

src/main/java/com/zhiwei/brandkbs2/pojo/Channel.java

@@ -93,6 +93,7 @@ public class Channel extends ChannelIndex {
     public static Channel createFromChannelIndexRecord(ChannelIndex channelIndex, ChannelRecord channelRecord) {
         Channel channel = new Channel();
         channel.setCTime(new Date().getTime());
+        channel.setProjectId(channelIndex.getProjectId());
         channel.setLinkedGroupId(channelIndex.getLinkedGroupId());
         channel.setPlatform(channelIndex.getPlatform());
         channel.setRealSource(channelIndex.getRealSource());

src/main/java/com/zhiwei/brandkbs2/pojo/ChannelIndex.java

@@ -22,6 +22,8 @@ import java.util.*;
 @NoArgsConstructor
 public class ChannelIndex extends AbstractBaseMongo {
 
+    private String projectId;
+
     /**
      * 关联项目组ID
      */
@@ -50,9 +52,10 @@ public class ChannelIndex extends AbstractBaseMongo {
         Integer foreign = Integer.parseInt(String.valueOf(sourceAsMap.get(GenericAttribute.ES_FOREIGN)));
         List<Map<String, Object>> cacheMaps = (List<Map<String, Object>>) sourceAsMap.get(GenericAttribute.ES_BRANDKBS_CACHE_MAPS);
         for (Map<String, Object> cacheMap : cacheMaps) {
+            String projectId = String.valueOf(cacheMap.get(GenericAttribute.PROJECT_ID));
             String linkedGroupId = String.valueOf(cacheMap.get(GenericAttribute.LINKED_GROUP_ID));
             String platform = BeanTools.filterPlatform(GlobalPojo.PLATFORMS, c5, foreign).getName();
-            res.add(new ChannelIndex(linkedGroupId, platform, realSource, source));
+            res.add(new ChannelIndex(projectId,linkedGroupId, platform, realSource, source));
         }
         return res;
     }

src/main/java/com/zhiwei/brandkbs2/pojo/dto/MarkSearchDTO.java

 package com.zhiwei.brandkbs2.pojo.dto;
 
+import com.alibaba.fastjson.JSONObject;
 import io.swagger.annotations.ApiModel;
 import io.swagger.annotations.ApiModelProperty;
 import lombok.Data;
@@ -27,14 +28,14 @@ public class MarkSearchDTO {
     /**
      * 关联项目组id
      */
-    @ApiModelProperty(value = "关联项目组id", required = true)
+    @ApiModelProperty(value = "关联项目组id")
     private String linkedGroupId;
 
     /**
      * 时间类型
      */
     @ApiModelProperty("时间类型")
-    private String timeType = "mtime";
+    private String timeType = "time";
 
     /**
      * 开始时间戳
@@ -95,7 +96,7 @@ public class MarkSearchDTO {
      * 排序字段
      */
     @ApiModelProperty(value = "排序字段")
-    private String sorter = "{\"mtime\":\"descend\"}";
+    private JSONObject sorter;
 
     /**
      * 是否聚合

src/main/java/com/zhiwei/brandkbs2/service/IBrandkbsTaskService.java

+package com.zhiwei.brandkbs2.service;
+
+
+import com.zhiwei.brandkbs2.pojo.BrandkbsTask;
+
+/**
+ * @author lxj
+ * @version 1.0
+ * @description 定时任务业务接口
+ * @date 2019/12/12 13:58
+ */
+public interface IBrandkbsTaskService {
+    /**
+     * 根据任务名称获取任务
+     *
+     * @param taskName 任务名称
+     * @param version  版本号
+     * @return 任务对象
+     */
+    BrandkbsTask findTaskByTaskNameAndVersion(String taskName, int version);
+
+    /**
+     * 获取任务
+     *
+     * @param task 任务对象
+     * @param now  当前时间
+     * @return 是否获取到任务
+     */
+    boolean getTaskJob(BrandkbsTask task, Long now);
+
+    /**
+     * 根据主键更新定时任务表
+     *
+     * @param task     任务对象
+     * @param nextTime 下次可执行时间
+     */
+    void resetTask(BrandkbsTask task, Long nextTime);
+}

src/main/java/com/zhiwei/brandkbs2/service/IReportService.java

 package com.zhiwei.brandkbs2.service;
 
+import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.brandkbs2.pojo.dto.ReportSettingsDTO;
 
 import java.util.Map;
@@ -25,4 +26,11 @@ public interface IReportService {
      * @param reportSettingsDTO 报告配置传输类
      */
     void upsertReportSettings(ReportSettingsDTO reportSettingsDTO);
+
+    /**
+     * 获取简报分类统计
+     *
+     * @return 获取简报分类统计
+     */
+    JSONObject getReportsAggCount();
 }

src/main/java/com/zhiwei/brandkbs2/service/ITaskService.java

@@ -11,6 +11,11 @@ public interface ITaskService{
     /**
      * 渠道记录
      */
-    void channelRecordFromEs(int day);
+    void messageFlowCount(int day);
+
+    /**
+     * 简报推送
+     */
+    void reportSend();
 
 }

src/main/java/com/zhiwei/brandkbs2/service/impl/BrandkbsTaskServiceImpl.java

+package com.zhiwei.brandkbs2.service.impl;
+
+import com.zhiwei.brandkbs2.config.Constant;
+import com.zhiwei.brandkbs2.dao.IBrandkbsTaskDao;
+import com.zhiwei.brandkbs2.pojo.BrandkbsTask;
+import com.zhiwei.brandkbs2.service.IBrandkbsTaskService;
+import org.springframework.data.mongodb.core.query.Update;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.Resource;
+import java.util.Objects;
+
+/**
+ * @ClassName: BrandkbsTaskServiceImpl
+ * @Description BrandkbsTaskServiceImpl
+ * @author: sjj
+ * @date: 2022-07-12 17:38
+ */
+@Service
+public class BrandkbsTaskServiceImpl implements IBrandkbsTaskService {
+
+    @Resource(name = "brandkbsTaskDao")
+    private IBrandkbsTaskDao brandkbsTaskDao;
+
+    /**
+     * 根据任务名称获取任务
+     *
+     * @param taskName 任务名称
+     * @param version  版本号
+     * @return 任务对象
+     */
+    @Override
+    public BrandkbsTask findTaskByTaskNameAndVersion(String taskName, int version) {
+        return brandkbsTaskDao.findTaskByTaskNameAndVersion(taskName, version);
+    }
+
+    /**
+     * 获取任务
+     *
+     * @param task 任务对象
+     * @param now  当前时间
+     * @return 是否获取成功
+     */
+    @Override
+    public boolean getTaskJob(BrandkbsTask task, Long now) {
+        //如果当前时间小于可执行时间，否则无法获取任务
+        if (Objects.isNull(task) || now < task.getExecutableTime()) {
+            return false;
+        }
+        //通过乐观锁的方式来更新数据表，如果结果大于0说明取到任务
+        return brandkbsTaskDao.updateTaskVersion(task) > 0;
+    }
+
+    /**
+     * 根据主键更新定时任务表
+     *
+     * @param task     任务对象
+     * @param nextTime 下次可执行时间
+     */
+    @Override
+    public void resetTask(BrandkbsTask task, Long nextTime) {
+        Update set = Update.update("version", Constant.INIT_VERSION).set("executableTime", nextTime);
+        brandkbsTaskDao.updateOneByIdWithField(task.getId(), set);
+    }
+}

src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java

@@ -22,7 +22,7 @@ import com.zhiwei.brandkbs2.pojo.vo.CustomTagVo;
 import com.zhiwei.brandkbs2.pojo.vo.PageVO;
 import com.zhiwei.brandkbs2.pojo.vo.ProjectVO;
 import com.zhiwei.brandkbs2.service.*;
-import com.zhiwei.brandkbs2.util.AggreeUtil;
+import com.zhiwei.brandkbs2.util.TextUtil;
 import com.zhiwei.brandkbs2.util.Tools;
 import com.zhiwei.middleware.mark.pojo.enums.TagField;
 import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform;
@@ -30,6 +30,8 @@ import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.time.DateUtils;
 import org.apache.commons.lang3.tuple.Pair;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
@@ -70,6 +72,7 @@ import static com.zhiwei.brandkbs2.config.Constant.DAY_PATTERN;
  */
 @Service("markDataServiceImpl")
 public class MarkDataServiceImpl implements IMarkDataService {
+    private static final Logger log = LogManager.getLogger(MarkDataServiceImpl.class);
 
     @Resource(name = "projectServiceImpl")
     private IProjectService projectService;
@@ -89,8 +92,6 @@ public class MarkDataServiceImpl implements IMarkDataService {
     @Autowired
     private StringRedisTemplate stringRedisTemplate;
 
-    private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat(Constant.DAY_PATTERN);
-
     @Override
     public PageVO<MarkFlowEntity> getYuqingMarkList(MarkSearchDTO markSearchDTO) {
         try {
@@ -125,7 +126,7 @@ public class MarkDataServiceImpl implements IMarkDataService {
         JSONObject result = new JSONObject();
         Date endDate = new Date();
         // 搜索时间
-        result.put("times", criteriaTime(endDate));
+        result.put("times", Arrays.asList("今天", "24小时", "三天", "七天", "近30天"));
         // 平台
         result.put("platformList", commonService.getQbjcPlatform("id", "name"));
         // 情感标签
@@ -319,12 +320,15 @@ public class MarkDataServiceImpl implements IMarkDataService {
             if (StringUtils.isNotEmpty(resultStr)) {
                 return JSON.parseArray(resultStr, JSONObject.class);
             }
-            List<String> text = getMarkTitleAndContent(startTime, endTime, projectId,linkedGroupId);
-            // 分析热评词
-//            AnsjSeg ansjSeg = new AnsjSeg();
-//            List<Map.Entry<String, Integer>> words = (List<Map.Entry<String, Integer>>) ansjSeg.getFenCi(text, 30).get("word");
-//            List<JSONObject> hotwords = Tools.getHotwords(words);
-//            stringRedisTemplate.opsForValue().set(redisKey, JSON.toJSONString(hotwords), 1, TimeUnit.DAYS);
+            long s = System.currentTimeMillis();
+            List<String> texts = getMarkTitleAndContent(startTime, endTime, projectId, linkedGroupId);
+            log.info("es查询size:{},耗时:{}", texts.size(), System.currentTimeMillis() - s);
+            long s1 = System.currentTimeMillis();
+            // 分析高频词
+            List<JSONObject> highWords = TextUtil.getHighWordsJson(texts, 30);
+            log.info("分析高频词耗时:{}", (System.currentTimeMillis() - s1));
+            stringRedisTemplate.opsForValue().set(redisKey, JSON.toJSONString(highWords), 1, TimeUnit.DAYS);
+            return highWords;
         } catch (IOException e) {
             ExceptionCast.cast(CommonCodeEnum.FAIL.message("es查询异常"), e);
         }
@@ -345,6 +349,12 @@ public class MarkDataServiceImpl implements IMarkDataService {
         if (markSearchDTO.getPageSize() <= 0 || markSearchDTO.getPageSize() > 10000) {
             markSearchDTO.setPageSize(20);
         }
+        // 排序默认
+        if (null == markSearchDTO.getSorter() || markSearchDTO.getSorter().isEmpty()) {
+            JSONObject sorter = new JSONObject();
+            sorter.put("time", "descend");
+            markSearchDTO.setSorter(sorter);
+        }
     }
 
     private Pair<SearchHits, Map<String, Long>> searchMarkHitsAndCount(MarkSearchDTO dto) throws IOException {
@@ -353,7 +363,11 @@ public class MarkDataServiceImpl implements IMarkDataService {
         // PostFilter 后置过滤器
         BoolQueryBuilder postFilter = QueryBuilders.boolQuery();
         // linkedGroupId projectId
-        postFilter.must(QueryBuilders.termQuery("brandkbs_cache_maps.linkedGroupId.keyword", dto.getLinkedGroupId())).must(QueryBuilders.termQuery("brandkbs_cache_maps.projectId.keyword", UserThreadLocal.getProjectId()));
+        String linkedGroupId = dto.getLinkedGroupId();
+        if (null == linkedGroupId) {
+            linkedGroupId = projectService.getProjectVOById(UserThreadLocal.getProjectId()).getBrandLinkedGroupId();
+        }
+        postFilter.must(QueryBuilders.termQuery("brandkbs_cache_maps.linkedGroupId.keyword", linkedGroupId)).must(QueryBuilders.termQuery("brandkbs_cache_maps.projectId.keyword", UserThreadLocal.getProjectId()));
         // time
         postFilter.must(QueryBuilders.rangeQuery(dto.getTimeType()).lte(dto.getStartTime()).lt(dto.getEndTime()));
         // platform
@@ -383,7 +397,7 @@ public class MarkDataServiceImpl implements IMarkDataService {
         }
         // sort
         FieldSortBuilder sort = null;
-        for (Map.Entry<String, Object> entry : JSONObject.parseObject(dto.getSorter()).entrySet()) {
+        for (Map.Entry<String, Object> entry : dto.getSorter().entrySet()) {
             if (entry.getValue().toString().contains("desc")) {
                 sort = SortBuilders.fieldSort(entry.getKey()).order(SortOrder.DESC);
             } else {
@@ -410,49 +424,49 @@ public class MarkDataServiceImpl implements IMarkDataService {
         List<JSONObject> jsonList = Arrays.stream(searchHits.getHits()).map(hit -> new JSONObject(hit.getSourceAsMap())).collect(Collectors.toList());
         Map<String, String> idTitle = jsonList.stream().collect(Collectors.toMap(json -> json.getString(uniqueField),
                 json -> Tools.getBaseFromEsMap(json).getTitleNullOptionalContent()));
-        List<List<String>> kResult = AggreeUtil.getKResult(idTitle);
-        return AggreeUtil.sortByTimeAsc(kResult, jsonList, uniqueField);
-    }
-
-    private List<JSONObject> criteriaTime(Date endDate) {
-        List<JSONObject> times = new ArrayList<>();
-        for (String criteriaTime : Arrays.asList("今天", "24小时", "三天", "七天", "近30天")) {
-            JSONObject time = new JSONObject();
-            switch (criteriaTime) {
-                case "今天":
-                    time.put("name", criteriaTime);
-                    time.put("startTime", Tools.truncDate(endDate, DAY_PATTERN).getTime());
-                    time.put("endTime", endDate.getTime());
-                    times.add(time);
-                    break;
-                case "24小时":
-                    time.put("name", criteriaTime);
-                    time.put("startTime", DateUtils.addHours(endDate, -24).getTime());
-                    time.put("endTime", endDate.getTime());
-                    times.add(time);
-                    break;
-                case "三天":
-                    time.put("name", criteriaTime);
-                    time.put("startTime", DateUtils.addDays(Tools.truncDate(endDate, DAY_PATTERN), -2).getTime());
-                    time.put("endTime", endDate.getTime());
-                    times.add(time);
-                    break;
-                case "七天":
-                    time.put("name", criteriaTime);
-                    time.put("startTime", DateUtils.addDays(Tools.truncDate(endDate, DAY_PATTERN), -6).getTime());
-                    time.put("endTime", endDate.getTime());
-                    times.add(time);
-                    break;
-                case "近30天":
-                    time.put("name", criteriaTime);
-                    time.put("startTime", DateUtils.addDays(Tools.truncDate(endDate, DAY_PATTERN), -29).getTime());
-                    time.put("endTime", endDate.getTime());
-                    times.add(time);
-                    break;
-            }
-        }
-        return times;
-    }
+        List<List<String>> kResult = TextUtil.getKResult(idTitle);
+        return TextUtil.sortByTimeAsc(kResult, jsonList, uniqueField);
+    }
+
+//    private List<JSONObject> criteriaTime(Date endDate) {
+//        List<JSONObject> times = new ArrayList<>();
+//        for (String criteriaTime : Arrays.asList("今天", "24小时", "三天", "七天", "近30天")) {
+//            JSONObject time = new JSONObject();
+//            switch (criteriaTime) {
+//                case "今天":
+//                    time.put("name", criteriaTime);
+//                    time.put("startTime", Tools.truncDate(endDate, DAY_PATTERN).getTime());
+//                    time.put("endTime", endDate.getTime());
+//                    times.add(time);
+//                    break;
+//                case "24小时":
+//                    time.put("name", criteriaTime);
+//                    time.put("startTime", DateUtils.addHours(endDate, -24).getTime());
+//                    time.put("endTime", endDate.getTime());
+//                    times.add(time);
+//                    break;
+//                case "三天":
+//                    time.put("name", criteriaTime);
+//                    time.put("startTime", DateUtils.addDays(Tools.truncDate(endDate, DAY_PATTERN), -2).getTime());
+//                    time.put("endTime", endDate.getTime());
+//                    times.add(time);
+//                    break;
+//                case "七天":
+//                    time.put("name", criteriaTime);
+//                    time.put("startTime", DateUtils.addDays(Tools.truncDate(endDate, DAY_PATTERN), -6).getTime());
+//                    time.put("endTime", endDate.getTime());
+//                    times.add(time);
+//                    break;
+//                case "近30天":
+//                    time.put("name", criteriaTime);
+//                    time.put("startTime", DateUtils.addDays(Tools.truncDate(endDate, DAY_PATTERN), -29).getTime());
+//                    time.put("endTime", endDate.getTime());
+//                    times.add(time);
+//                    break;
+//            }
+//        }
+//        return times;
+//    }
 
     private void emotionAnalyzeAdd(long startTime, long endTime, String projectId, String linkedGroupId, JSONObject result) throws IOException {
 
@@ -757,16 +771,18 @@ public class MarkDataServiceImpl implements IMarkDataService {
 
     private List<String> getMarkTitleAndContent(Long startTime, Long endTime, String projectId, String linkedGroupId) throws IOException {
         EsClientDao.SearchHelper searchHelper = EsClientDao.createSearchHelper();
-        // sort
-        searchHelper.setSort(SortBuilders.fieldSort("id").order(SortOrder.DESC));
+//        // sort
+        searchHelper.setSort(SortBuilders.fieldSort("time").order(SortOrder.DESC));
         // fetchSource
-        searchHelper.setFetchSource(new String[]{"ind_title", "ind_full_text"});
+        searchHelper.setFetchSource(new String[]{"ind_title", "ind_full_text", "c5", "foreign", "mark_cache_maps"});
         // postFilter
         BoolQueryBuilder postFilter = projectLinkedGroupQuery(projectId, linkedGroupId);
         postFilter.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
         searchHelper.setPostFilter(postFilter);
+        // TODO 临时添加
+        searchHelper.setSize(10000);
         List<String> textList = new ArrayList<>();
-        List<SearchResponse> searchResponses = esClientDao.searchScrollResponse(searchHelper);
+        List<SearchResponse> searchResponses = Collections.singletonList(esClientDao.searchResponse(searchHelper));
         for (SearchResponse searchResponse : searchResponses) {
             for (SearchHit searchHit : searchResponse.getHits().getHits()) {
                 Map<String, Object> source = searchHit.getSourceAsMap();

src/main/java/com/zhiwei/brandkbs2/service/impl/ReportServiceImpl.java

 package com.zhiwei.brandkbs2.service.impl;
 
+import com.alibaba.fastjson.JSONObject;
 import com.google.common.collect.Maps;
 import com.zhiwei.brandkbs2.auth.UserThreadLocal;
 import com.zhiwei.brandkbs2.dao.IReportDao;
@@ -76,6 +77,14 @@ public class ReportServiceImpl implements IReportService {
         reportSettingsDao.insertOneWithoutId(reportSettings);
     }
 
+    @Override
+    public JSONObject getReportsAggCount() {
+        JSONObject result = new JSONObject();
+//        result.put("types", getReportTypeAggCount(pid, uid));
+//        result.put("times", getTimeAggCount(pid, uid));
+        return result;
+    }
+
     private List<ReportSettings> getReportSettingsList() {
         String projectId = UserThreadLocal.getProjectId();
         List<ReportSettings> reports = reportSettingsDao.findList(Query.query(Criteria.where("projectId").is(projectId)));

src/main/java/com/zhiwei/brandkbs2/service/impl/TaskServiceImpl.java

 package com.zhiwei.brandkbs2.service.impl;
 
+import com.zhiwei.brandkbs2.config.Constant;
+import com.zhiwei.brandkbs2.dao.IBrandkbsTaskDao;
 import com.zhiwei.brandkbs2.dao.IChannelDao;
 import com.zhiwei.brandkbs2.dao.IEventDataDao;
 import com.zhiwei.brandkbs2.es.EsClientDao;
+import com.zhiwei.brandkbs2.pojo.BrandkbsTask;
 import com.zhiwei.brandkbs2.pojo.Channel;
 import com.zhiwei.brandkbs2.pojo.ChannelIndex;
 import com.zhiwei.brandkbs2.service.ITaskService;
@@ -11,6 +14,7 @@ import org.apache.logging.log4j.Logger;
 import org.springframework.stereotype.Service;
 
 import javax.annotation.Resource;
+import java.util.Date;
 import java.util.LinkedHashSet;
 import java.util.Map;
 
@@ -33,8 +37,11 @@ public class TaskServiceImpl implements ITaskService {
     @Resource(name = "eventDataDao")
     IEventDataDao eventDataDao;
 
+    @Resource(name = "brandkbsTaskDao")
+    IBrandkbsTaskDao brandkbsTaskDao;
+
     @Override
-    public void channelRecordFromEs(int day) {
+    public void messageFlowCount(int day) {
         Map<ChannelIndex, ChannelIndex.ChannelRecord> channelIndexChannelRecordMap = esClientDao.searchChannelRecordRecentDay(day);
         log.info("渠道信息记录-搜索到近{}天的记录条数{}条", day, channelIndexChannelRecordMap.size());
         int insertCount = 0;
@@ -54,4 +61,10 @@ public class TaskServiceImpl implements ITaskService {
         }
         log.info("渠道信息记录-录入完毕,新增渠道{}条,更新渠道{}条", insertCount, channelIndexChannelRecordMap.size() - insertCount);
     }
+
+    @Override
+    public void reportSend() {
+        Date now = new Date();
+        BrandkbsTask task = brandkbsTaskDao.findTaskByTaskNameAndVersion(Constant.TASK_REPORT_JOB, Constant.INIT_VERSION);
+    }
 }

src/main/java/com/zhiwei/brandkbs2/common/ControlCenter.java → src/main/java/com/zhiwei/brandkbs2/task/ControlCenter.java

-package com.zhiwei.brandkbs2.common;
+package com.zhiwei.brandkbs2.task;
 
 import com.zhiwei.brandkbs2.service.ITaskService;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.springframework.context.annotation.Profile;
 import org.springframework.scheduling.annotation.Async;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
@@ -16,6 +17,7 @@ import javax.annotation.Resource;
  * @date: 2022-06-08 17:39
  */
 @Component
+@Profile({"dev", "pro"})
 public class ControlCenter {
     public static final Logger log = LogManager.getLogger(ControlCenter.class);
 
@@ -27,7 +29,7 @@ public class ControlCenter {
     public void messageFlowCount() {
         log.info("定时按天录入各小时渠道进量-启动");
         try {
-            taskService.channelRecordFromEs(1);
+            taskService.messageFlowCount(1);
         } catch (Exception e) {
             log.error("定时按天录入各小时渠道进量-出错", e);
         } finally {
@@ -35,4 +37,17 @@ public class ControlCenter {
         }
     }
 
+    @Async("scheduledExecutor")
+    @Scheduled(cron = "0 0 * * * ? ")
+    public void repostSend() {
+        log.info("每小时扫描简报信息-启动");
+        try {
+            taskService.reportSend();
+        } catch (Exception e) {
+            log.info("每小时扫描简报信息-出错", e);
+        } finally {
+            log.info("每小时扫描简报信息-结束");
+        }
+    }
+
 }

src/main/java/com/zhiwei/brandkbs2/util/AggreeUtil.java

-package com.zhiwei.brandkbs2.util;
-
-import com.alibaba.fastjson.JSONObject;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Maps;
-import com.zhiwei.brandkbs2.common.GenericAttribute;
-import com.zhiwei.nlp.AggreeBootStarter;
-import com.zhiwei.nlp.common.AggreeConfig;
-import com.zhiwei.nlp.vo.KResult;
-
-import java.util.Comparator;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-/**
- * @ClassName: AggreeUtil
- * @Description 聚合工具类
- * @author: sjj
- * @date: 2022-06-29 17:07
- */
-public class AggreeUtil {
-
-    private static final Double LIMIT = 0.1;
-    private static final String DEFAULT_SORT_FIELD = GenericAttribute.ES_TIME;
-
-    static {
-        // 开启符号过滤以及关闭英文字母和数字过滤
-        AggreeConfig.getSetter().setFilterSymbol(true).setFilterEng(false).setSimplifiedChinese(true);
-    }
-
-    public static <T> List<List<T>> getKResult(Map<T, String> listMap) {
-        return AggreeBootStarter.getKResult(listMap, LIMIT).stream().map(KResult::getDataPoints).collect(Collectors.toList());
-    }
-
-    public static <T> List<List<JSONObject>> sortByTimeAsc(List<List<T>> kResultList, List<JSONObject> sourceList, String tField) {
-        ImmutableMap<T, JSONObject> idMap = Maps.uniqueIndex(sourceList, json -> (T) json.get(tField));
-        // 还原数据
-        return kResultList.stream().map(gList ->
-                gList.stream().map(idMap::get).sorted(Comparator.comparingLong(o -> o.getLong(DEFAULT_SORT_FIELD))).collect(Collectors.toList())
-        ).collect(Collectors.toList());
-    }
-
-}

src/main/java/com/zhiwei/brandkbs2/util/TextUtil.java

+package com.zhiwei.brandkbs2.util;
+
+import com.alibaba.fastjson.JSONObject;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+import com.hankcs.hanlp.HanLP;
+import com.hankcs.hanlp.dictionary.CustomDictionary;
+import com.hankcs.hanlp.dictionary.stopword.CoreStopWordDictionary;
+import com.hankcs.hanlp.seg.common.Term;
+import com.zhiwei.brandkbs2.common.GenericAttribute;
+import com.zhiwei.nlp.AggreeBootStarter;
+import com.zhiwei.nlp.common.AggreeConfig;
+import com.zhiwei.nlp.vo.KResult;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.core.io.Resource;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.PostConstruct;
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * @ClassName: AggreeUtil
+ * @Description 聚合工具类
+ * @author: sjj
+ * @date: 2022-06-29 17:07
+ */
+@Component
+public class TextUtil {
+    private static final Logger log = LogManager.getLogger(TextUtil.class);
+
+    private static final Double LIMIT = 0.1;
+    private static final String DEFAULT_SORT_FIELD = GenericAttribute.ES_TIME;
+    @Value("classpath:wordDic/customDictionary.txt")
+    private Resource customDic;
+    @Value("classpath:wordDic/stopWordDictionary.txt")
+    private Resource stopDic;
+
+    @PostConstruct
+    public void init() {
+        String errorWord = null;
+        try {
+            // 开启符号过滤以及关闭英文字母和数字过滤
+            AggreeConfig.getSetter().setFilterSymbol(true).setFilterEng(false).setSimplifiedChinese(true);
+            // 关闭词性显示
+            HanLP.Config.ShowTermNature = false;
+            // 自定义词典和停用词
+            List<String> customDics = Tools.readListFile(customDic.getInputStream());
+            List<String> stopDics = Tools.readListFile(stopDic.getInputStream());
+            for (String dic : customDics) {
+                errorWord = dic;
+                CustomDictionary.add(dic);
+            }
+            for (String dic : stopDics) {
+                errorWord = dic;
+                CoreStopWordDictionary.add(dic);
+            }
+            log.info("自定义词典加载:{}条,停用词加载:{}条", customDics.size(), stopDics.size());
+        } catch (Exception e) {
+            log.info("TextUtil-init 异常,errorWord:{}", errorWord, e);
+        }
+    }
+
+    public static <T> List<List<T>> getKResult(Map<T, String> listMap) {
+        return AggreeBootStarter.getKResult(listMap, LIMIT).stream().map(KResult::getDataPoints).collect(Collectors.toList());
+    }
+
+    public static <T> List<List<JSONObject>> sortByTimeAsc(List<List<T>> kResultList, List<JSONObject> sourceList, String tField) {
+        ImmutableMap<T, JSONObject> idMap = Maps.uniqueIndex(sourceList, json -> (T) json.get(tField));
+        // 还原数据
+        return kResultList.stream().map(gList ->
+                gList.stream().map(idMap::get).sorted(Comparator.comparingLong(o -> o.getLong(DEFAULT_SORT_FIELD))).collect(Collectors.toList())
+        ).collect(Collectors.toList());
+    }
+
+    public static Map<String, Integer> getHighWords(List<String> texts) {
+        return getHighWords(texts, null);
+    }
+
+    public static Map<String, Integer> getHighWords(List<String> texts, Integer maxSize) {
+        Map<String, Integer> result = new HashMap<>();
+        for (String text : texts) {
+            for (Term term : CoreStopWordDictionary.apply(HanLP.segment(text))) {
+                result.compute(term.toString(), (word, rate) -> {
+                    if (null == rate) {
+                        rate = 0;
+                    }
+                    return ++rate;
+                });
+            }
+        }
+        return Tools.sortMap(result, maxSize);
+    }
+
+    public static List<JSONObject> getHighWordsJson(List<String> texts, Integer maxSize) {
+        Map<String, Integer> highWords = getHighWords(texts, maxSize);
+        List<JSONObject> result = new ArrayList<>(highWords.size());
+        for (Map.Entry<String, Integer> entry : highWords.entrySet()) {
+            JSONObject jsonObject = new JSONObject();
+            jsonObject.put("name", entry.getKey());
+            jsonObject.put("num", entry.getValue());
+            result.add(jsonObject);
+        }
+        return result;
+    }
+
+}

src/main/java/com/zhiwei/brandkbs2/util/Tools.java

@@ -3,7 +3,7 @@ package com.zhiwei.brandkbs2.util;
 import com.alibaba.excel.EasyExcel;
 import com.alibaba.excel.read.listener.PageReadListener;
 import com.alibaba.fastjson.JSONObject;
-import com.zhiwei.base.category.ClassB;
+import com.zhiwei.base.category.ClassCodec;
 import com.zhiwei.base.entity.subclass.CompleteText;
 import com.zhiwei.base.entity.subclass.IncompleteText;
 import com.zhiwei.base.entity.subclass.QAText;
@@ -36,6 +36,7 @@ import java.nio.channels.WritableByteChannel;
 import java.util.*;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import static com.zhiwei.brandkbs2.common.GenericAttribute.ES_MARK_CACHE_MAPS;
 import static java.util.Objects.nonNull;
@@ -312,9 +313,9 @@ public class Tools {
         res.setRealSource(String.valueOf(map.get(GenericAttribute.ES_REAL_SOURCE)));
         res.setPlatform(getPlatform(map));
         res.setEmotion(getEmotion(map));
-        int c2 = Integer.parseInt(String.valueOf(map.get(GenericAttribute.ES_C2)));
+        int c5 = Integer.parseInt(String.valueOf(map.get(GenericAttribute.ES_C5)));
         // 单独设置属性title,content 部分url
-        switch (ClassB.TypeB.fromEncode(c2)) {
+        switch (ClassCodec.decodeClassD(c5).typeB()) {
             case COMPLETE:
                 CompleteText completeText = CompleteText.restoreFromEs(map);
                 res.setTitle(completeText.getTitle());
@@ -368,12 +369,12 @@ public class Tools {
         return null;
     }
 
-    public static JSONObject change2TagInfoByMtag(String mtag){
+    public static JSONObject change2TagInfoByMtag(String mtag) {
         JSONObject res = new JSONObject();
-        if(StringUtils.isEmpty(mtag)){
+        if (StringUtils.isEmpty(mtag)) {
             return res;
         }
-        for(Map.Entry<String, List<Tag>> entry :GlobalPojo.TAGS.entrySet()){
+        for (Map.Entry<String, List<Tag>> entry : GlobalPojo.TAGS.entrySet()) {
             for (Tag tag : entry.getValue()) {
                 if (mtag.contains(tag.getUniqueId())) {
                     res.put(tag.getGroupName(), tag.getName());
@@ -386,12 +387,13 @@ public class Tools {
 
     /**
      * 过滤 html
+     *
      * @param str
      * @return
      */
     public static String filterStrHtmlTag(String str) {
-        str =  str.replaceAll("<","&lt;");
-        return str.replaceAll(">","&gt;");
+        str = str.replaceAll("<", "&lt;");
+        return str.replaceAll(">", "&gt;");
     }
 
     /**
@@ -473,17 +475,45 @@ public class Tools {
         return resultStr.substring(0, resultStr.length() - 1);
     }
 
-//    public static String getEmotionByMtag(String mtag, String group) {
-//        if (StringUtils.isEmpty(mtag)) {
-//            return null;
-//        }
-//        List<Tag> tags = GlobalPojo.TAGS.get(group);
-//        for (Tag tag : tags) {
-//            if ("情感倾向".equals(tag.getGroupName()) && mtag.contains(tag.getUniqueId())) {
-//                return tag.getName();
-//            }
-//        }
-//        return null;
-//    }
+    public static List<String> readListFile(InputStream inputStream) throws Exception {
+        BufferedReader br = null;
+        String line;
+        List<String> result = new ArrayList<>();
+        try {
+            //根据文件路径创建缓冲输入流
+            br = new BufferedReader(new InputStreamReader(inputStream));
+            //循环读取文件的每一行，对需要修改的行进行修改，放入缓冲对象中
+            while (StringUtils.isNotEmpty(line = StringUtils.trim(br.readLine()))) {
+                // 去除多余空格
+                result.add(line);
+            }
+        } catch (Exception e) {
+            throw new Exception(e);
+        } finally {
+            if (br != null) {// 关闭流
+                try {
+                    br.close();
+                } catch (IOException ignored) {
+                }
+            }
+        }
+        return result;
+    }
+
+    public static Map<String, Integer> sortMap(Map<String, Integer> map, Integer maxSize) {
+        Map<String, Integer> result = new LinkedHashMap<>();
+        List<Map.Entry<String, Integer>> collect;
+        Stream<Map.Entry<String, Integer>> sorted = map.entrySet().stream().sorted((o1, o2) -> o2.getValue() - o1.getValue());
+        if (null != maxSize) {
+            // 按rate排序
+            collect = sorted.limit(maxSize).collect(Collectors.toList());
+        } else {
+            collect = sorted.collect(Collectors.toList());
+        }
+        for (Map.Entry<String, Integer> entry : collect) {
+            result.put(entry.getKey(), entry.getValue());
+        }
+        return result;
+    }
 
 }
\ No newline at end of file

src/main/resources/wordDic/customDictionary.txt

+淘宝浏览器
+淘宝电影
+淘宝彩票
+淘宝HD
+手机淘宝
+千牛
+飞猪
+钉钉
+阿里邮箱
+阿里星球
+阿里旺旺
+阿里通
+阿里拍卖
+阿里卖家
+阿里旅行
+阿里钉钉
+优酷土豆
+合一集团
+淘点点
+阿里钱盾
+蚂蚁金服
+阿里云
+阿里安全
+菜鸟驿站
+高德地图
+阿里文学
+阿里妈妈
+ET大脑
+菜鸟物流
+阿里公益
+95公益周
+阿里公关
+阿里O2O
+烈火如歌
+自由之战2
+旅行青蛙
+武动乾坤
+卧虎藏龙贰
+刀剑兵器谱
+偃月之日
+血染征袍
+星际要塞
+新三国志手机版
+阿里游戏公布
+阿里游戏发行
+阿里游戏上线
+阿里游戏合作
+盒马鲜生
+支付宝圈子
+社交
+支付宝红包
+小程序收藏
+共享停车
+手机退税
+延时到账
+蚂蚁森林
+淘宝卖家
+海外淘
+淘宝村
+农村淘
+虾米音乐
+天天动听
+送药上门
+码上放心
+财富号
+鹰眼监控
+POLARDB
+高德叫车
+阿里巴巴文学
+ET工业大脑
+ET医疗大脑
+ET城市大脑
+精灵书屋
+XIN公益大会|引光行动
+阿里高层
+阿里游戏
+阿里公益
+阿里智慧城市
+阿里新零售
+阿里投资
+唐家三少
+梭子蟹
+
+
+
+

src/main/resources/wordDic/stopWordDictionary.txt

+【
+日
+月
+年
+昨天
+明天
+今天
+微博
+转发
+现在
+嘿嘿
+还要
+要
+嗳
+赶早不赶晚
+这些
+日益
+倒不如
+逢
+立地
+不只
+一般
+豁然
+将近
+毫不
+嗬
+大张旗鼓
+嗯
+不可开交
+"
+#
+ 
+!
+&
+全力
+就算
+'
+$
+%
+...
+*
++
+(
+她们
+)
+.
+/
+各位
+,
+-
+︿
+3
+2
+1
+不止
+基本
+0
+不拘
+7
+这里
+6
+5
+颇
+4
+;
+:
+如此
+9
+8
+极度
+?
+>
+首先
+=
+<
+也罢
+@
+A
+见
+当庭
+隔夜
+更
+不少
+不胜
+\
+_
+替
+^
+到目前为止
+大大
+除开
+腾
+暗中
+而外
+开始
+`
+三番两次
+宁可
+这么
+权时
+结果
+大多
+除此以外
+单单
+如下
+几度
+何处
+喂
+如上
+矣
+喀
+喏
+~
+吧哒
+|
+放量
+即便
+当年
+不对
+那
+顷刻
+本人
+是
+岂非
+己
+看
+趁热
+哪边
+立马
+乘势
+啥
+何况
+这个
+啦
+人民
+率尔
+那种
+仍然
+不能
+根据
+并肩
+相对而言
+也好
+啐
+什么样
+累年
+啊
+扑通
+即使
+开外
+大概
+依照
+·
+乃至
+与否
+总而言之
+高低
+切切
+多次
+比如说
+不亦乐乎
+如期
+简言之
+何妨
+不管怎样
+顺
+顷
+将才
+呆呆地
+略为
+更为
+大约
+其次
+倍加
+满
+不定
+除了
+都
+之后
+着
+难道
+不可
+至
+风雨无阻
+陡然
+为了
+及至
+对于
+虽说
+唉
+彻夜
+嘎嘎
+臭
+不同
+大体上
+自
+！
+＃
+来得及
+哦
+而
+％
+＄
+的
+＆
+哪
+）
+老
+（
+＋
+哩
+＊
+，
+者
+人人
+比方
+０
+１
+简而言之
+这么些
+２
+３
+４
+５
+６
+不至于
+７
+这部
+谁知
+８
+几时
+９
+屡
+：
+；
+＜
+应当
+＞
+哼
+？
+人家
+如今
+＠
+哇
+哈
+哉
+尽量
+总的说来
+继之
+单纯
+方才
+哎
+这么点儿
+极
+乘胜
+八成
+光是
+倘或
+哗
+被
+忽然
+［
+从头
+出去
+哟
+］
+咦
+切不可
+尽
+哪年
+竟然
+是否
+而况
+加以
+从此以后
+省得
+就
+咳
+我们
+不力
+各个
+咱
+充其极
+｜
+｝
+次第
+～
+岂止
+｛
+某
+尽管如此
+偶而
+看上去
+截然
+甚而
+和
+如常
+任何
+极端
+接着
+嘎登
+咋
+皆可
+具体地说
+凝神
+这就是说
+将
+千万
+好在
+从早到晚
+各自
+咚
+取道
+纯粹
+这种
+只限
+上去
+恐怕
+呢
+莫非
+虽然
+碰巧
+呸
+紧接着
+即若
+本
+等等
+按照
+呵
+不单
+具体说来
+一旦
+望
+朝
+纵
+不要
+呀
+怎样
+呃
+轰然
+有
+每当
+接连不断
+呜
+呐
+不比
+呕
+纯
+呗
+各种
+理应
+连袂
+吧
+绝
+什么
+那里
+后来
+给
+日渐
+暗自
+以免
+经
+不然
+来
+饱
+别人
+吱
+看来
+沙沙
+同
+趁势
+切莫
+从重
+尽心尽力
+切勿
+果真
+各
+￥
+要不是
+白
+并排
+自己
+保管
+岂
+差一点
+默然
+此中
+能
+吗
+向
+吓
+藉以
+不惟
+的确
+此后
+让
+待到
+末##末
+哪些
+不然的话
+其他
+毫无
+连声
+趁早
+归
+几经
+当
+论
+九
+也
+挨门挨户
+挨次
+乘
+保险
+从小
+莫若
+乒
+彼
+乎
+刚好
+么
+如若
+之
+默默地
+是的
+嗡嗡
+请勿
+乃
+为
+得起
+借此
+该
+汝
+从此
+然而
+亲眼
+略微
+刚才
+一定
+反倒是
+按时
+临
+个
+倘若
+差不多
+从无到有
+—
+起来
+’
+‘
+反之则
+弗
+”
+“
+何止
+惯常
+姑且
+与其
+…
+哪个
+反而
+常言道
+大抵
+不再
+且
+到了儿
+三
+上
+再者
+不
+并且
+与
+一
+趁着
+七
+两者
+等到
+不经意
+必
+如何
+来着
+不由得
+怎么样
+尽管
+知道
+任
+旁人
+不管
+由
+个人
+哪里
+似的
+以
+甭
+甫
+倒不如说
+用
+均
+其余
+长此下去
+们
+莫
+匆匆
+多少
+当着
+就是说
+他
+既然
+虽则
+纵使
+呼哧
+沿
+快
+仅
+联袂
+没
+或许
+仍
+来看
+俺们
+从
+倘然
+只是
+往
+大凡
+而言
+当真
+待
+因此
+很
+不如
+据此
+更进一步
+那么样
+纵然
+得
+不仅...而且
+极为
+尽然
+略
+长期以来
+互
+五
+不妨
+不止一次
+地
+——
+较比
+必须
+或是
+向着
+从古到今
+在
+尽如人意
+了
+毕竟
+二
+川流不息
+确实
+于
+可以
+你
+并没有
+当场
+要不
+那儿
+纵令
+恰巧
+无宁
+四
+来讲
+局外
+近年来
+因
+并
+但
+起首
+,
+赶快
+方
+需要
+即令
+大略
+将要
+活
+不特
+然则
+极了
+何
+但是
+固
+不独
+何苦
+一则
+猛然
+屡屡
+传
+到底
+在下
+设使
+经过
+至于
+老老实实
+猛然间
+截至
+譬如
+很多
+一切
+别的
+要么
+趁机
+。
+、
+越是
+常
+　
+按期
+何尝
+》
+《
+〉
+〈
+动不动
+不外
+因为
+使得
+会
+既
+如果
+按说
+不大
+带
+自从
+以便
+宁肯
+当下
+不光
+它们
+之类
+老大
+尽可能
+尔后
+成年累月
+如上所述
+每个
+彼此
+从宽
+俺
+就此
+粗
+达旦
+当口儿
+归根结底
+看起来
+或多或少
+当中
+据我所知
+据实
+不免
+遵照
+固然
+缕缕
+换言之
+策略地
+居然
+连日来
+若
+起见
+比照
+嘎
+不成
+不仅仅是
+长话短说
+因而
+设若
+不论
+嘘
+嘛
+沿着
+恍然
+慢说
+亲身
+哼唷
+故
+便
+以至
+以致
+本着
+论说
+除外
+之所以
+简直
+前后
+大家
+嘻
+果然
+共总
+嘿
+敢
+时候
+不怎么
+如次
+依
+鄙人
+亲手
+大
+顿时
+顺着
+叮当
+敞开儿
+等
+大面儿上
+年复一年
+冲
+打开天窗说亮话
+跟
+上来
+拿
+假若
+不曾
+着呢
+快要
+此刻
+而且
+背靠背
+假使
+陈年
+多多益善
+另方面
+冒
+他人
+到处
+大体
+下来
+云云
+全然
+何须
+为着
+每逢
+内
+多
+很少
+尚且
+只要
+不仅仅
+出
+  
+顷刻间
+常常
+趁
+日臻
+恰似
+得天独厚
+另外
+敢情
+率然
+并无
+届时
+凭
+每每
+几
+她
+成为
+他们
+尔等
+尽快
+不消
+如其
+把
+反之亦然
+当即
+奇
+据悉
+奈
+前者
+第
+必定
+ [
+处处
+ ]
+断然
+绝非
+总的来看
+岂但
+分期
+古来
+我们
+啪达
+顷刻之间
+每次
+别说
+传闻
+从优
+总的来说
+非徒
+常言说得好
+非得
+由于
+难说
+可是
+从今以后
+比如
+所
+继而
+不可抗拒
+才
+如
+精光
+凭借
+略加
+起
+平素
+绝对
+赶
+于是
+打
+一样
+长线
+每时每刻
+不择手段
+理该
+共
+拦腰
+喔唷
+其
+仍旧
+屡次
+以及
+当然
+到头来
+抑或
+宁愿
+一方面
+举凡
+只有
+八
+咱们
+六
+从新
+兮
+这样
+不得已
+管
+十分
+自个儿
+呼啦
+我
+必将
+串行
+而论
+或
+牢牢
+成心
+光
+哈哈
+与此同时
+其一
+于是乎
+此
+看样子
+换句话说
+全身心
+除非
+有人
+以至于
+按理
+也许
+打从
+照着
+况且
+独
+除却
+不了
+不得
+反手
+成年
+哎呀
+关于
+恰恰相反
+这儿
+累次
+其中
+动辄
+立刻
+倒是
+毫无例外
+从古至今
+可见
+诚然
+莫不
+怎么办
+亲自
+经常
+决不
+自各儿
+这么样
+不必
+不得了
+除去
+由此可见
+像
+有些
+挨个
+不仅
+进来
+大事
+全年
+绝顶
+社会主义
+总之
+当头
+若是
+竟
+不外乎
+要不然
+如此等等
+分期分批
+那么
+毋宁
+立
+其二
+不会
+..
+背地里
+据
+此间
+哪儿
+不怕
+不问
+每
+为什么
+没有
+公然
+那会儿
+迫于
+来不及
+不起
+千万千万
+可能
+正如
+比
+还有
+借
+倘
+究竟
+及其
+不限
+偏偏
+据称
+故此
+谁
+伙同
+敢于
+弹指之间
+那些
+窃
+朝着
+叮咚
+临到
+即将
+哎哟
+而已
+尽心竭力
+到头
+亲口
+已经
+不但
+出来
+随着
+不得不
+非常
+另一个
+非但
+如前所述
+殆
+--
+诸位
+那时
+即是说
+按
+谨
+何时
+此外
+然后
+勃然
+从来
+近几年来
+近来
+莫如
+奋勇
+比起
+仅仅
+故而
+穷年累月
+历
+乌乎
+怪不得
+去
+借以
+主要
+间或
+方能
+白白
+除
+反过来
+全都
+并没
+过
+除此之外
+马上
+迄
+恰恰
+传说
+还
+这
+连
+近
+从速
+上下
+哪样
+这边
+从未
+不能不
+从不
+及
+那个
+边
+又
+迟早
+不知不觉
+挨家挨户
+多多少少
+几番
+有关
+您
+连同
+较
+互相
+怎么
+但愿
+可
+你们
+凑巧
+连日
+叫
+路经
+阿
+起先
+另
+二话没说
+之一
+这时
+即或
+连连
+其后
+各式
+当儿
+独自
+它
+宁
+哪天
+就是
+乘机
+常言说
+不下
+定
+照
+昂然
+毫无保留地
+趁便
+屡次三番
+甚至
+那末
+充其量
+该当
+另一方面
+既...又
+瑟瑟
+的话
+呜呼
+或者
+立时
+反过来说
+有的
+挨着
+再说
+够瞧的
+过于
+零
+就地
+然
+极其
+何乐而不为
+进去
+单
+随
+起头
+无论
+怎
+据说
+综上所述
+抽冷子
+才能
+怕
+千
+离
+梆
+极大
+恰逢
+半
+大举
+漫说
+接下来
+忽地
+而是
+即
+难得
+不但...而且
+格外
+怪
+倘使
+还是
+从而
+对
+本身
+乘隙
+既是
+理当
+反倒
+焉
+可好
+不满
+交口
+基本上
+认为
+这会儿
+充分
+并非
+不迭
+老是
+倍感
+鉴于
+要是
+反之
+哪怕
+除此而外
+虽
+一下
+自身
+任凭
+几乎
+顶多
+靠
+而又
+不时
+一个
+否则
+自家
+三天两头
+砰
+啊呀
+难怪
+所以
+发生
+哗啦
+多年来
+罢了
+大致
+从轻
+那边
+那么些
+不巧
+完全
+起初
+某个
+加之
+大不了
+归根到底
+偶尔
+应该
+二话不说
+日见
+不是
+大都
+愤然
+而后
+多年前
+例如
+蛮
+切
+一些
+多多
+日复一日
+较之
+即刻
+那样
+齐
+其它
+则
+不料
+刚
+初
+决非
+乘虚
+恰如
+能够
+从严
+故意
+别
+啊哟
+从中
+不已
+加上
+具体来说
+较为
+分头
+直到
+到
+>>
+隔日
+多亏
+假如
+甚么
+作为
+暗地里
+挨门逐户
+恰好
+其实
+何必
+万一
+不过
+某些
+啊哈
+基于
+不日
+尽早
+刚巧
+概
+一来
+同时
+三番五次
+为何
+更加
+绝不
+除此
+不常
+进而
+另行
+急匆匆
+通过
+话说
+若非
+极力
+存心
+http
+https
+cn
+com
+呵呵
+【
+】
+凤行
\ No newline at end of file

src/test/java/com/zhiwei/brandkbs2/Test.java

 package com.zhiwei.brandkbs2;
 
+import com.hankcs.hanlp.HanLP;
+import com.hankcs.hanlp.dictionary.CustomDictionary;
+import com.hankcs.hanlp.dictionary.stopword.CoreStopWordDictionary;
+import com.hankcs.hanlp.seg.common.Term;
+
 import java.io.UnsupportedEncodingException;
-import java.net.URLEncoder;
+import java.util.List;
 
 /**
  * @ClassName: Test
@@ -11,15 +16,29 @@ import java.net.URLEncoder;
  */
 public class Test {
 
+    static{
+        CustomDictionary.add("猪肉供应");
+        CoreStopWordDictionary.add("保障");
+    }
+
     public static void main(String[] args) throws UnsupportedEncodingException {
+        List<Term> results= HanLP.segment("国家发改委：猪肉供应有保障 猪肉价格不具备大幅上涨基础");
+        for(Term term: CoreStopWordDictionary.apply(results)){
+//            if(CoreStopWordDictionary.shouldInclude(term)){
+//                System.out.println(term);
+//            }else{
+                System.err.println(term);
+//            }
+
+        }
 
 //        String token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJwYXlsb2FkIjoie1widXNlcklkXCI6XCIyMFwiLFwia2V5XCI6XCIyMFwiLFwibmlja05hbWVcIjpcIuayiOWQm-adsFwiLFwic2VydmljZVwiOlwiXCJ9IiwiZXhwIjoxNjUzMDExNjcwLCJpYXQiOjE2NTI0MDY4MTB9.jcVXxeZkayc6-Aiq8cyYc1uyq4ugji6FdWQXCCp4M2o";
 //        String token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJwYXlsb2FkIjoiXCLlk4Hop4FcIiIsImV4cCI6NDc2MjgyMjEzMiwiaWF0IjoxNjUyNDIyMDcyfQ.DXQ8yKgfsCMjhT0xniZeWCMv4syqIoDvztU4QWsd-Fg";
 //        String t = JwtUtil.unsign(token, String.class);
 //        System.out.println(JSONObject.toJSONString(t));
 
-          String test = URLEncoder.encode("测试", "utf-8");
-        System.out.println(test);
+//          String test = URLEncoder.encode("测试", "utf-8");
+//        System.out.println(test);
 //        String projectId = "1";
 //        List<UserRole> list = new ArrayList<>();
 //        UserDTO userDto = new UserDTO();

src/test/java/com/zhiwei/brandkbs2/TestRunWith.java

@@ -3,8 +3,9 @@ package com.zhiwei.brandkbs2;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.brandkbs2.dao.IChannelLabelDao;
 import com.zhiwei.brandkbs2.dao.IEventDataDao;
-import com.zhiwei.brandkbs2.service.ITaskService;
 import com.zhiwei.brandkbs2.pojo.ChannelIndex;
+import com.zhiwei.brandkbs2.service.ITaskService;
+import com.zhiwei.brandkbs2.util.TextUtil;
 import lombok.extern.slf4j.Slf4j;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -12,6 +13,9 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
 
+import java.util.ArrayList;
+import java.util.List;
+
 /**
  * @ClassName: TestRunWith
  * @Description TODO
@@ -32,9 +36,12 @@ public class TestRunWith {
     @Autowired
     IChannelLabelDao channelLabelDao;
 
+    @Autowired
+    TextUtil textUtil;
+
     @Test
     public void test1(){
-        iTaskService.channelRecordFromEs(2);
+        iTaskService.messageFlowCount(1);
     }
 
     @Test
@@ -52,4 +59,13 @@ public class TestRunWith {
         channelLabelDao.getChannelLabelType();
     }
 
+    @Test
+    public void test4(){
+        List<String> list = new ArrayList<>();
+        list.add("分析高频词耗时");
+        list.add("分析高频词");
+        List<JSONObject> highWordsJson = TextUtil.getHighWordsJson(list,1);
+        System.out.println(JSONObject.toJSONString(highWordsJson));
+    }
+
 }