Merge branch 'feature' into 'dev'

Feature

See merge request !295

pom.xml

@@ -24,7 +24,7 @@
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
         <java.version>1.8</java.version>
         <springboot.version>2.2.5.RELEASE</springboot.version>
-        <qbjc.bean.version>1.1.6.4-SNAPSHOT</qbjc.bean.version>
+        <qbjc.bean.version>1.1.7.1-SNAPSHOT</qbjc.bean.version>
         <springfox.version>2.7.0</springfox.version>
         <fastjson.version>1.2.47</fastjson.version>
         <easyexcel.version>3.0.5</easyexcel.version>

src/main/java/com/zhiwei/brandkbs2/common/CommonConfig.java

 package com.zhiwei.brandkbs2.common;
 
+import com.zhiwei.middleware.cleaner.filter.UnifiedFilterClient;
 import com.zhiwei.middleware.event.core.EventClient;
 import com.zhiwei.middleware.event.core.EventClientFactory;
 import com.zhiwei.middleware.event.core.EventMonitorClient;
@@ -36,6 +37,9 @@ public class CommonConfig {
     @Value("${event.provider.group}")
     private String eventProviderGroup;
 
+    @Value("${filter.provider.group}")
+    private String filterGroup;
+
     @Bean
     public RestTemplate restTemplate() {
         final OkHttpClient client = new OkHttpClient().newBuilder()
@@ -50,7 +54,7 @@ public class CommonConfig {
      *
      * @return 标注中间件客户端
      */
-    @Bean
+    @Bean(name = "markerClient")
     public MarkerClient getMarkerClient() {
         return MarkerClient.getService(clientRegistry, providerGroup, appName);
     }
@@ -61,25 +65,39 @@ public class CommonConfig {
      * @return 事件中间件标签客户端
      */
     @Bean
-    public EventTagClient getEventTagClient(){
+    public EventTagClient getEventTagClient() {
         return EventClientFactory.markEventTagClient(eventClientRegistry, eventProviderGroup, appName);
     }
 
     /**
      * 获取事件中间事件客户端
+     *
      * @return 事件中间事件客户端
      */
     @Bean
-    public EventClient getEventClient(){
+    public EventClient getEventClient() {
         return EventClientFactory.eventClient(eventClientRegistry, eventProviderGroup, appName);
     }
 
     /**
      * 获取事件中间件监测客户端
+     *
      * @return 事件事件件监测客户端
      */
     @Bean
-    public EventMonitorClient getEventMonitorClient(){
+    public EventMonitorClient getEventMonitorClient() {
         return EventClientFactory.eventMonitorClient(eventClientRegistry, eventProviderGroup, appName);
     }
+
+
+    /**
+     * 获取去重中间件客户端
+     *
+     * @return 去重中间件客户端
+     */
+    @Bean(name = "filterClient")
+    public UnifiedFilterClient getUnifiedFilterClient() {
+        return UnifiedFilterClient.getClient(appName, clientRegistry, filterGroup);
+    }
+
 }

src/main/java/com/zhiwei/brandkbs2/common/GlobalPojo.java

 package com.zhiwei.brandkbs2.common;
 
 import com.alibaba.fastjson.JSONObject;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import com.zhiwei.brandkbs2.pojo.ChannelTag;
-import com.zhiwei.brandkbs2.pojo.Event;
 import com.zhiwei.brandkbs2.pojo.Project;
-import com.zhiwei.brandkbs2.service.EventService;
 import com.zhiwei.brandkbs2.service.SystemInfoService;
-import com.zhiwei.middleware.mark.vo.MarkerTag;
 import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform;
 import com.zhiwei.qbjc.bean.pojo.common.Tag;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.scheduling.annotation.Async;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
@@ -22,8 +17,6 @@ import javax.annotation.Resource;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
 import java.util.stream.Collectors;
 
 /**
@@ -47,7 +40,7 @@ public class GlobalPojo {
     /**
      * 监测系统平台
      **/
-    public static Map<String, List<Tag>> TAGS;
+    public static Map<String,List<Tag>> TAGS;
 
     /**
      * 品见情感倾向标签
@@ -95,7 +88,7 @@ public class GlobalPojo {
     private void updatePojo(String logMsg) {
         try {
             PLATFORMS = systemInfoService.getPlatforms();
-            TAGS = systemInfoService.getTags().stream().collect(Collectors.groupingBy(Tag::getGroupName));
+            TAGS = systemInfoService.getTags().stream().filter(tag -> null != tag.getProjectId()).collect(Collectors.groupingBy(Tag::getProjectId));
             LINKED_GROUP_ID_TAGS = systemInfoService.findEmotionTagByLinkedGroupId();
             CHANNEL_TAGS = systemInfoService.getChannelTags().stream().collect(Collectors.toMap(ChannelTag::getChannel, ChannelTag::getTag));
             MEDIA_TYPE = systemInfoService.getMediaTypes();
@@ -142,4 +135,13 @@ public class GlobalPojo {
         return projectMap.get(platform + source);
     }
 
+    public static String getYuQingProjectNameById(String linkedGroupId) {
+        for (JSONObject yuQingProject : YU_QING_PROJECTS) {
+            if (linkedGroupId.equals(yuQingProject.getString("id"))) {
+                return yuQingProject.getString("groupName");
+            }
+        }
+        return null;
+    }
+
 }

src/main/java/com/zhiwei/brandkbs2/config/Constant.java

@@ -28,6 +28,7 @@ public class Constant {
     public static final FastDateFormat DAY_FORMAT = FastDateFormat.getInstance(DAY_PATTERN);
     public static final FastDateFormat SPEC_DAY_FORMAT = FastDateFormat.getInstance("yyyy/MM/dd");
     public static final FastDateFormat ONLY_MONTH_FORMAT = FastDateFormat.getInstance("M月");
+    public static final FastDateFormat DF_yyyyMMdd = FastDateFormat.getInstance("yyyyMMdd");
 
     /**
      * 自定义fid分隔符号

src/main/java/com/zhiwei/brandkbs2/config/EsProperties.java

@@ -32,5 +32,19 @@ public class EsProperties {
 	private String username;
 
 	private String password;
+
+	/**
+	 * clusterNameHotES 集群名-hotes
+	 */
+	private String clusterNameHotES;
+
+	/**
+	 * 集群节点s
+	 */
+	private String clusterNodesHotES;
+
+	private String usernameHotES;
+
+	private String passwordHotES;
 	
 }

src/main/java/com/zhiwei/brandkbs2/controller/CommonController.java

@@ -6,7 +6,6 @@ import com.zhiwei.brandkbs2.enmus.RoleEnum;
 import com.zhiwei.brandkbs2.model.ResponseResult;
 import com.zhiwei.brandkbs2.service.CommonService;
 import com.zhiwei.brandkbs2.service.ProjectService;
-import com.zhiwei.middleware.mark.pojo.enums.TagField;
 import com.zhiwei.middleware.mark.vo.MarkerTag;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiImplicitParam;
@@ -17,9 +16,7 @@ import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
 
 import javax.annotation.Resource;
-import java.util.ArrayList;
 import java.util.List;
-import java.util.stream.Collectors;
 
 /**
  * @ClassName: CommonController
@@ -40,16 +37,11 @@ public class CommonController extends BaseController {
     ProjectService projectService;
 
     @ApiOperation("获取情感倾向标签信息")
-    @ApiImplicitParam(name = "contendId", value = "品牌id", required = true, paramType = "path", dataType = "string")
-    @GetMapping("/get/tag/emotion/{contendId}")
-    public ResponseResult getTagsWithEmotion(@PathVariable(value = "contendId") String contendId) {
-        List<String> res = new ArrayList<>();
-        String linkedGroupId = projectService.getProjectByContendId(UserThreadLocal.getProjectId(), contendId).getBrandLinkedGroupId();
-        List<MarkerTag> tags = commonService.getQbjcTags(linkedGroupId, TagField.GROUP_NAME.is("情感倾向"));
-        if (null != tags) {
-            res = tags.stream().map(MarkerTag::getName).collect(Collectors.toList());
-        }
-        return ResponseResult.success(res);
+    @ApiImplicitParam(name = "linkedGroupId", value = "关联项目id", required = true, paramType = "path", dataType = "string")
+    @GetMapping("/get/tag/emotion/{linkedGroupId}")
+    public ResponseResult getTagsWithEmotion(@PathVariable(value = "linkedGroupId") String linkedGroupId) {
+        List<MarkerTag> tags = commonService.getEmotionTagsWithSort(UserThreadLocal.getProjectId(), linkedGroupId);
+        return ResponseResult.success(tags);
     }
 
     @ApiOperation("获取平台类型")

src/main/java/com/zhiwei/brandkbs2/controller/admin/BehaviorController.java

@@ -77,6 +77,7 @@ public class BehaviorController extends BaseController {
 
     @ApiOperation("记录用户操作记录")
     @GetMapping("/logRecord/save")
+    @Auth(role = RoleEnum.CUSTOMER)
     public ResponseResult saveUserLogRecord(@RequestParam(value = "description") String description) {
         behaviorService.saveUserLogRecord(description);
         return ResponseResult.success();

src/main/java/com/zhiwei/brandkbs2/controller/admin/ProjectController.java

@@ -9,7 +9,6 @@ import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
 import com.zhiwei.brandkbs2.easyexcel.dto.UploadKeywordDTO;
 import com.zhiwei.brandkbs2.enmus.RoleEnum;
 import com.zhiwei.brandkbs2.model.ResponseResult;
-import com.zhiwei.brandkbs2.pojo.Behavior;
 import com.zhiwei.brandkbs2.pojo.vo.ProjectVO;
 import com.zhiwei.brandkbs2.service.BehaviorService;
 import com.zhiwei.brandkbs2.service.CommonService;
@@ -62,8 +61,6 @@ public class ProjectController extends BaseController {
     @Resource(name = "eventServiceImpl")
     private EventService eventService;
 
-    private static final Behavior.Operation OPERATION = new Behavior.Operation("项目管理", true);
-
     @Value("${brandkbs.img.url}")
     private String brandkbsImgPath;
 
@@ -79,9 +76,7 @@ public class ProjectController extends BaseController {
     @ApiOperation("添加项目配置")
     @PostMapping("/add")
     public ResponseResult addProject(@RequestBody ProjectVO projectVO) {
-        ProjectService.addProject(projectVO);
-        behaviorService.pushBehaviorOld(OPERATION, "添加项目配置:" + projectVO.getId(), request);
-        return ResponseResult.success();
+        return ProjectService.addProject(projectVO);
     }
 
     @ApiOperation("查询所有项目")
@@ -96,9 +91,7 @@ public class ProjectController extends BaseController {
     @ApiOperation("修改项目")
     @PutMapping("/update")
     public ResponseResult updateProject(@RequestBody ProjectVO projectVO) {
-        ProjectService.updateProject(projectVO);
-        behaviorService.pushBehaviorOld(OPERATION, "修改项目:" + projectVO.getId(), request);
-        return ResponseResult.success();
+        return ProjectService.updateProject(projectVO);
     }
 
     @ApiOperation("删除项目")
@@ -106,7 +99,6 @@ public class ProjectController extends BaseController {
     @DeleteMapping("/delete/{pid}")
     public ResponseResult deleteProject(@PathVariable("pid") String pid) {
         ProjectService.deleteProject(pid, UserThreadLocal.getProjectId());
-        behaviorService.pushBehaviorOld(OPERATION, "删除项目:" + pid, request);
         return ResponseResult.success();
     }
 
@@ -116,7 +108,6 @@ public class ProjectController extends BaseController {
     public ResponseResult switchProjectShow(@PathVariable String pid) {
         boolean result = ProjectService.switchProjectShow(pid);
         if (result) {
-            behaviorService.pushBehaviorOld(OPERATION, "切换项目展示状态:" + pid, request);
             return ResponseResult.success();
         } else {
             return ResponseResult.failure("启动状态下的项目，不允许调整展示状态");
@@ -128,7 +119,6 @@ public class ProjectController extends BaseController {
     @PutMapping("/switch/projectStart/{pid}")
     public ResponseResult switchProjectStart(@PathVariable String pid) {
         ProjectService.switchProjectStart(pid);
-        behaviorService.pushBehaviorOld(OPERATION, "切换项目状态:" + pid, request);
         return ResponseResult.success();
     }
 
@@ -142,9 +132,7 @@ public class ProjectController extends BaseController {
     @ApiOperation("图片上传")
     @PostMapping(value = "/upload/img", headers = "content-type=multipart/form-data")
     public ResponseResult uploadImg(@RequestParam("file") MultipartFile file) {
-        ResponseResult responseResult = Tools.uploadFile(file, brandkbsImgPath, brandkbsImageUrl);
-        behaviorService.pushBehaviorOld(OPERATION, "图片上传", request);
-        return responseResult;
+        return Tools.uploadFile(file, brandkbsImgPath, brandkbsImageUrl);
     }
 
     @ApiOperation("关键词excel解析")

src/main/java/com/zhiwei/brandkbs2/controller/app/AppArticleController.java

@@ -3,6 +3,7 @@ package com.zhiwei.brandkbs2.controller.app;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.brandkbs2.aop.LogRecord;
 import com.zhiwei.brandkbs2.auth.Auth;
+import com.zhiwei.brandkbs2.auth.UserThreadLocal;
 import com.zhiwei.brandkbs2.controller.BaseController;
 import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
 import com.zhiwei.brandkbs2.enmus.RoleEnum;
@@ -13,7 +14,10 @@ import com.zhiwei.brandkbs2.pojo.dto.ReportDTO;
 import com.zhiwei.brandkbs2.pojo.dto.ReportSearchDTO;
 import com.zhiwei.brandkbs2.service.CommonService;
 import com.zhiwei.brandkbs2.service.MarkDataService;
+import com.zhiwei.brandkbs2.service.ProjectService;
 import com.zhiwei.brandkbs2.service.ReportService;
+import com.zhiwei.brandkbs2.util.Tools;
+import com.zhiwei.middleware.mark.vo.MarkerTag;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiImplicitParam;
 import io.swagger.annotations.ApiImplicitParams;
@@ -23,6 +27,7 @@ import org.springframework.web.bind.annotation.*;
 
 import javax.annotation.Resource;
 import java.util.List;
+import java.util.stream.Collectors;
 
 /**
  * @author sjj
@@ -45,6 +50,67 @@ public class AppArticleController extends BaseController {
     @Resource(name = "commonServiceImpl")
     CommonService commonService;
 
+    @Resource(name = "projectServiceImpl")
+    ProjectService projectService;
+
+    @ApiOperation("原始数据列表")
+    @PostMapping("/origin/list")
+    public ResponseResult getOriginList(@RequestBody MarkSearchDTO markSearchDTO) {
+        return ResponseResult.success(markDataService.getOriginList(markSearchDTO));
+    }
+
+    @ApiOperation("原始数据列表-生成聚合")
+    @PostMapping("/origin/aggree")
+    public ResponseResult generateOriginAggreeList(@RequestBody JSONObject info) {
+        Long startTime = info.getLong("startTime");
+        Long endTime = info.getLong("endTime");
+        return ResponseResult.success(markDataService.generateOriginAggreeList(startTime, endTime));
+    }
+
+    @ApiOperation("原始数据列表-获取聚合结果")
+    @PostMapping("/origin/aggree/list")
+    public ResponseResult getOriginAggreeList(@RequestBody MarkSearchDTO markSearchDTO) {
+        return ResponseResult.success(markDataService.getOriginAggreeList(markSearchDTO));
+    }
+
+    @ApiOperation("原始数据导出")
+    @PostMapping(value = "/origin/list/export")
+    public ResponseResult exportOriginList(@RequestBody MarkSearchDTO markSearchDTO) {
+        Pair<String, List<ExportAppYuqingDTO>> stringListPair = markDataService.downloadYuqingMarkList(markSearchDTO);
+        EasyExcelUtil.download(stringListPair.getLeft() + "_原始数据列表数据", "sheet1", ExportAppYuqingDTO.class, stringListPair.getRight(), response);
+        return ResponseResult.success();
+    }
+
+    @ApiOperation("单条标注")
+    @PostMapping("/origin/tag")
+    public ResponseResult addOriginTag(@RequestBody JSONObject json) {
+        String zip = json.getString("zip");
+        String linkedGroupId = json.getString("linkedGroupId");
+        StringBuilder mtag = new StringBuilder(json.getString("mtag"));
+        // 带上项目配置的必要标注标签
+        if (checkMTagIllegal(mtag)){
+            return ResponseResult.failure("标注标签出现同组标签，请勿连点标注");
+        }
+        markDataService.addMtag(zip, linkedGroupId, mtag.toString());
+        return ResponseResult.success();
+    }
+
+    @ApiOperation("批量标注")
+    @PostMapping("/origin/tag/batch")
+    public ResponseResult addOriginTagBatch(@RequestBody JSONObject json) {
+        List<String> ids = (List<String>) json.get("ids");
+        String linkedGroupId = json.getString("linkedGroupId");
+        StringBuilder mtag = new StringBuilder(json.getString("mtag"));
+        if (Tools.isEmpty(ids) || null == linkedGroupId) {
+            return ResponseResult.failure("传参异常");
+        }
+        // 带上项目配置的必要标注标签
+        if (checkMTagIllegal(mtag)){
+            return ResponseResult.failure("标注标签出现同组标签，请勿连点标注");
+        }
+        return markDataService.addMtagBatch(ids, linkedGroupId, mtag.toString());
+    }
+
     @ApiOperation("舆情列表")
     @PostMapping("/mark/list")
     @LogRecord(description = "舆情库-舆情列表")
@@ -60,10 +126,17 @@ public class AppArticleController extends BaseController {
         return ResponseResult.success(markDataService.generateYuqingMarkAggreeList(startTime, endTime));
     }
 
+    @ApiOperation("聚合进度查询")
+    @GetMapping("/aggree/schedule/{aggreeId}")
+    public ResponseResult getAggreeProgress(@PathVariable String aggreeId) {
+        return ResponseResult.success(markDataService.getAggreeProgress(aggreeId));
+    }
+
     @ApiOperation("舆情列表-聚合进度查询")
     @GetMapping("/mark/aggree/schedule/{aggreeId}")
+    @Deprecated
     public ResponseResult getYuqingMarkAggreeProgress(@PathVariable String aggreeId) {
-        return ResponseResult.success(markDataService.getYuqingMarkAggreeProgress(aggreeId));
+        return ResponseResult.success(markDataService.getAggreeProgress(aggreeId));
     }
 
     @ApiOperation("舆情列表-获取聚合结果")
@@ -79,7 +152,7 @@ public class AppArticleController extends BaseController {
     }
 
     @ApiOperation("舆情导出")
-    @PostMapping(value = "mark/list/export")
+    @PostMapping(value = "/mark/list/export")
     public ResponseResult exportYuqingMarkList(@RequestBody MarkSearchDTO markSearchDTO) {
         Pair<String, List<ExportAppYuqingDTO>> stringListPair = markDataService.downloadYuqingMarkList(markSearchDTO);
         EasyExcelUtil.download(stringListPair.getLeft() + "_舆情列表数据", "sheet1", ExportAppYuqingDTO.class, stringListPair.getRight(), response);
@@ -124,7 +197,7 @@ public class AppArticleController extends BaseController {
     @GetMapping("/platform/proportion")
     public ResponseResult getMarkPlatformProportion(@RequestParam(value = "startTime", required = false) Long startTime,
                                                     @RequestParam(value = "endTime", required = false) Long endTime) {
-        return ResponseResult.success(markDataService.getMarkPlatformProportion(startTime, endTime, true));
+        return ResponseResult.success(markDataService.getMarkPlatformProportion(UserThreadLocal.getProjectId(), startTime, endTime, true));
     }
 
     @ApiOperation("舆情分析-渠道参与")
@@ -202,4 +275,14 @@ public class AppArticleController extends BaseController {
         return reportService.getReportSchedule(idList);
     }
 
+    private boolean checkMTagIllegal(StringBuilder mtag) {
+        List<MarkerTag> hitTags = projectService.getProjectById(UserThreadLocal.getProjectId()).getHitTags();
+        if (!Tools.isEmpty(hitTags)) {
+            for (String mustTag : hitTags.stream().map(MarkerTag::getUniqueId).collect(Collectors.toList())) {
+                mtag.append(mustTag);
+            }
+        }
+        return Tools.isIllegalTag(mtag.toString());
+    }
+
 }

src/main/java/com/zhiwei/brandkbs2/controller/app/AppHotController.java

@@ -127,12 +127,11 @@ public class AppHotController extends BaseController {
     public ResponseResult latestNews(@RequestParam(value = "size", defaultValue = "5") Integer size,
                                      @RequestParam(value = "include", defaultValue = "true") boolean include) throws IOException {
         String projectId = UserThreadLocal.getProjectId();
-        String linkedGroupId = projectService.getProjectVOById(projectId).getBrandLinkedGroupId();
         Date endDate = new Date();
         Date startDate = DateUtils.addHours(endDate, -24);
 
         List<Map.Entry<String, Integer>> markTopTitleList =
-                markDataService.getMarkTopTitle(startDate.getTime(), endDate.getTime(), null, projectId, linkedGroupId, Constant.PRIMARY_CONTEND_ID, size, include);
+                markDataService.getMarkTopTitle(startDate.getTime(), endDate.getTime(), null, projectId, Constant.PRIMARY_CONTEND_ID, size, include);
 
         List<JSONObject> resultList = markTopTitleList.stream().map(map -> {
             JSONObject json = new JSONObject();
@@ -142,7 +141,7 @@ public class AppHotController extends BaseController {
         }).collect(Collectors.toList());
         CompletableFuture.allOf(resultList.stream().map(json -> CompletableFuture.supplyAsync(() -> {
             try {
-                BaseMap firstArticle = markDataService.getFirstArticle(startDate.getTime(), endDate.getTime(), json.getString("title"), projectId, linkedGroupId, Constant.PRIMARY_CONTEND_ID, include);
+                BaseMap firstArticle = markDataService.getFirstArticle(startDate.getTime(), endDate.getTime(), json.getString("title"), projectId, Constant.PRIMARY_CONTEND_ID, include);
                 json.put("content", firstArticle.getContent());
                 json.put("url", firstArticle.getUrl());
                 json.put("realSource", firstArticle.getRealSource());

src/main/java/com/zhiwei/brandkbs2/controller/app/AppWarnController.java

@@ -3,9 +3,11 @@ package com.zhiwei.brandkbs2.controller.app;
 
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.brandkbs2.auth.Auth;
 import com.zhiwei.brandkbs2.auth.UserThreadLocal;
 import com.zhiwei.brandkbs2.common.GenericAttribute;
 import com.zhiwei.brandkbs2.controller.BaseController;
+import com.zhiwei.brandkbs2.enmus.RoleEnum;
 import com.zhiwei.brandkbs2.model.ResponseResult;
 import com.zhiwei.brandkbs2.service.ProjectService;
 import com.zhiwei.brandkbs2.service.ProjectWarnService;
@@ -39,6 +41,7 @@ import java.util.Map;
 @RestController
 @Api(tags = "情报预警接口", description = "情报预警")
 @RequestMapping("/app/warn")
+@Auth(role = RoleEnum.CUSTOMER)
 public class AppWarnController extends BaseController {
     private static final Logger log = LogManager.getLogger(AppWarnController.class);
 

src/main/java/com/zhiwei/brandkbs2/controller/mobile/MobileEventController.java

@@ -11,7 +11,6 @@ import com.zhiwei.brandkbs2.service.CommonService;
 import com.zhiwei.brandkbs2.service.EventService;
 import com.zhiwei.brandkbs2.service.ProjectService;
 import com.zhiwei.middleware.event.pojo.dto.BrandkbsEventSearchDTO;
-import com.zhiwei.middleware.mark.pojo.enums.TagField;
 import com.zhiwei.middleware.mark.vo.MarkerTag;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiImplicitParam;
@@ -67,7 +66,7 @@ public class MobileEventController extends BaseController {
         if (0 != emotion) {
             String emotionName = EmotionEnum.state2Name(emotion);
             String linkedGroupId = projectService.getProjectByContendId(UserThreadLocal.getProjectId(), Constant.PRIMARY_CONTEND_ID).getBrandLinkedGroupId();
-            List<MarkerTag> tags = commonService.getQbjcTags(linkedGroupId, TagField.GROUP_NAME.is("情感倾向"));
+            List<MarkerTag> tags = commonService.getEmotionTagsWithSort(UserThreadLocal.getProjectId(), linkedGroupId);
             tags.forEach(tag -> {
                 if (tag.getName().equals(emotionName)) {
                     dto.setEmotions(Collections.singletonList(tag.getUniqueId()));

src/main/java/com/zhiwei/brandkbs2/controller/mobile/MobileSearchController.java

@@ -10,7 +10,6 @@ import com.zhiwei.brandkbs2.model.ResponseResult;
 import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
 import com.zhiwei.brandkbs2.service.*;
 import com.zhiwei.middleware.event.pojo.dto.BrandkbsEventSearchDTO;
-import com.zhiwei.middleware.mark.pojo.enums.TagField;
 import com.zhiwei.middleware.mark.vo.MarkerTag;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiImplicitParam;
@@ -94,7 +93,7 @@ public class MobileSearchController {
         if (0 != emotion) {
             String emotionName = EmotionEnum.state2Name(emotion);
             String linkedGroupId = projectService.getProjectByContendId(UserThreadLocal.getProjectId(), Constant.PRIMARY_CONTEND_ID).getBrandLinkedGroupId();
-            List<MarkerTag> tags = commonService.getQbjcTags(linkedGroupId, TagField.GROUP_NAME.is("情感倾向"));
+            List<MarkerTag> tags = commonService.getEmotionTagsWithSort(UserThreadLocal.getProjectId(),linkedGroupId);
             tags.forEach(tag -> {
                 if (tag.getName().equals(emotionName)) {
                     dto.setEmotions(Collections.singletonList(tag.getUniqueId()));

src/main/java/com/zhiwei/brandkbs2/easyexcel/EasyExcelUtil.java

@@ -14,6 +14,7 @@ import org.apache.logging.log4j.Logger;
 import org.springframework.web.multipart.MultipartFile;
 
 import javax.servlet.http.HttpServletResponse;
+import java.lang.reflect.Field;
 import java.net.URLEncoder;
 import java.util.ArrayList;
 import java.util.List;
@@ -153,6 +154,7 @@ public class EasyExcelUtil {
      */
     public static <T> void download(String fileName, String sheetName, Class<T> clazz, List<T> datas, HttpServletResponse response) {
         try {
+            formatExcelExports(clazz, datas);
             response.setContentType("application/vnd.ms-excel");
             response.setCharacterEncoding("utf-8");
             // 这里URLEncoder.encode可以防止中文乱码
@@ -187,4 +189,27 @@ public class EasyExcelUtil {
             log.error("file:{}，download error：", fileName, e);
         }
     }
+
+    private static <T> void formatExcelExports(Class<T> clazz, List<T> datas){
+        List<Field> stringFields = new ArrayList<>();
+        // 记录需要设置的部分
+        for (Field field : clazz.getDeclaredFields()) {
+            // String格式截取前32767位
+            if (field.getGenericType().toString().equals(String.class.toString())) {
+                field.setAccessible(true);
+                stringFields.add(field);
+            }
+        }
+        datas.forEach(data -> {
+            for (Field field : stringFields) {
+                try {
+                    // 截取为excel单元格允许的最大长度
+                    field.set(data, StringUtils.substring((String) (field.get(data)), 0, 32767));
+                } catch (IllegalAccessException e) {
+                    log.error(e);
+                }
+            }
+        });
+    }
+
 }

src/main/java/com/zhiwei/brandkbs2/es/EsQueryTools.java

@@ -94,16 +94,15 @@ public class EsQueryTools {
         return tagBoolQuery;
     }
 
-    public static BoolQueryBuilder assembleCacheMapsQuery(String projectId, String linkedGroupId, String contendId) {
-        return assembleCacheMapsQuery(projectId, linkedGroupId, contendId, null);
+    public static BoolQueryBuilder assembleCacheMapsQuery(String projectId, String contendId) {
+        return assembleCacheMapsQuery(projectId, contendId, null);
     }
 
-    public static BoolQueryBuilder assembleCacheMapsQuery(String projectId, String linkedGroupId, String contendId, List<String> mediaTypes) {
-        return assembleCacheMapsQuery(projectId, linkedGroupId, contendId, mediaTypes,null);
+    public static BoolQueryBuilder assembleCacheMapsQuery(String projectId, String contendId, List<String> mediaTypes) {
+        return assembleCacheMapsQuery(projectId, contendId, mediaTypes,null);
     }
 
-    public static BoolQueryBuilder assembleCacheMapsQuery(String projectId, String linkedGroupId, String contendId, List<String> mediaTypes,
-                                                          Integer channelEmotion) {
+    public static BoolQueryBuilder assembleCacheMapsQuery(String projectId,String contendId, List<String> mediaTypes, Integer channelEmotion) {
         BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
         String key = concat(projectId, contendId);
         BoolQueryBuilder nestedBoolBuilder = QueryBuilders.boolQuery();
@@ -146,25 +145,23 @@ public class EsQueryTools {
         return query;
     }
 
+    public static BoolQueryBuilder assembleHotEsKeyQuery(String linkedGroupId, List<String> platformIds) {
+        BoolQueryBuilder query = QueryBuilders.boolQuery();
+        query.must(QueryBuilders.termQuery("monitor_rule_cache_maps.project.keyword", linkedGroupId));
+        if (!Tools.isEmpty(platformIds)) {
+            BoolQueryBuilder platformQuery = QueryBuilders.boolQuery();
+            for (String platformId : platformIds) {
+                platformQuery.should(QueryBuilders.termQuery("monitor_rule_cache_maps.platform.keyword", platformId));
+            }
+            query.must(platformQuery);
+        }
+        return query;
+    }
+
     public static NestedQueryBuilder cacheMapsNestedQuery(QueryBuilder query) {
         return new NestedQueryBuilder(ES_BRANDKBS_CACHE_MAPS, query, ScoreMode.None);
     }
 
-//    /**
-//     * 媒体类型请求匹配
-//     *
-//     * @param mediaTypes
-//     * @return
-//     */
-//    public static BoolQueryBuilder assembleMediaTypeQuery(List<String> mediaTypes) {
-//        BoolQueryBuilder tagQuery = QueryBuilders.boolQuery();
-//        mediaTypes.forEach(e -> {
-//            // TODO 1
-//            tagQuery.should(QueryBuilders.termQuery("brandkbs_cache_maps.channel_type.keyword", e));
-//        });
-//        return tagQuery;
-//    }
-
     public static BoolQueryBuilder assembleFiledKeywordQuery(String field, String keyword) {
         BoolQueryBuilder channelBoolQueryBuilder = QueryBuilders.boolQuery();
         String[] keys = keyword.trim().split("\\|");
@@ -181,18 +178,6 @@ public class EsQueryTools {
         return assembleFiledKeywordQuery("source", sourceKeyword);
     }
 
-//    public static BoolQueryBuilder assembleChannelSourceQuery(String sourceKeyword) {
-//        BoolQueryBuilder channelBoolQueryBuilder = QueryBuilders.boolQuery();
-//        String[] keys = sourceKeyword.trim().split("\\|");
-//        for (String key : keys) {
-//            String channelRegex = getAllRegex(key);
-//            BoolQueryBuilder keyQueryBuilder = QueryBuilders.boolQuery();
-//            keyQueryBuilder.must(QueryBuilders.regexpQuery("source.keyword", ".*" + channelRegex + ".*"));
-//            channelBoolQueryBuilder.should(keyQueryBuilder);
-//        }
-//        return channelBoolQueryBuilder;
-//    }
-
     public static void platformMustNot(BoolQueryBuilder boolQueryBuilder, String... platformNames) {
         if (null == platformNames) {
             return;
@@ -278,7 +263,7 @@ public class EsQueryTools {
      * @Title: matchPhraseQueryFields
      * @Description: 字段不拆封, 多字段 同关键词
      */
-    public static BoolQueryBuilder matchPhraseQueryFields(BoolQueryBuilder boolQueryBuilder, String esQuery,
+    private static BoolQueryBuilder matchPhraseQueryFields(BoolQueryBuilder boolQueryBuilder, String esQuery,
                                                           String word, String... fields) {
         if (ES_QUERY_AND.equals(esQuery)) {
             for (String field : fields) {

src/main/java/com/zhiwei/brandkbs2/es/EsRestClient.java

@@ -84,6 +84,52 @@ public class EsRestClient {
         return client;
     }
 
+    @Bean(name = "hotEsClient")
+    public RestHighLevelClient hotEsClient() {
+        List<HttpHost> httpHostList = new ArrayList<>();
+        try {
+            Assert.hasText(esProperties.getClusterNodesHotES(), "Cluster nodes source must not be null or empty!");
+
+            String[] nodes = StringUtils.delimitedListToStringArray(esProperties.getClusterNodesHotES(), COMMA);
+
+            Arrays.stream(nodes).forEach(node -> {
+
+                String[] segments = StringUtils.delimitedListToStringArray(node, COLON);
+
+                Assert.isTrue(segments.length == 2,
+                        () -> String.format("Invalid cluster node %s in %s! Must be in the format host:port!", node,
+                                esProperties.getClusterNodesHotES()));
+
+                String host = segments[0].trim();
+                String port = segments[1].trim();
+
+                Assert.hasText(host, () -> String.format("No host name given cluster node %s!", node));
+                Assert.hasText(port, () -> String.format("No port given in cluster node %s!", node));
+                httpHostList.add(new HttpHost(host, Integer.parseInt(port)));
+            });
+        } catch (Exception e) {
+            log.error("es client初始化异常", e);
+        }
+        HttpHost[] httpHosts = httpHostList.toArray(new HttpHost[httpHostList.size()]);
+        // 判断，如果未配置用户名，则进行无用户名密码连接，配置了用户名，则进行用户名密码连接
+        RestHighLevelClient client;
+        if (StringUtils.isEmpty(esProperties.getUsername())) {
+            client = new RestHighLevelClient(RestClient.builder(httpHosts));
+        } else {
+            final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
+            credentialsProvider.setCredentials(AuthScope.ANY,
+                    // es账号密码
+                    new UsernamePasswordCredentials(esProperties.getUsernameHotES(), esProperties.getPasswordHotES()));
+            client = new RestHighLevelClient(
+                    RestClient.builder(httpHosts).setHttpClientConfigCallback((httpClientBuilder) -> {
+                        // 这里可以设置一些参数，比如cookie存储、代理等等
+                        httpClientBuilder.disableAuthCaching();
+                        return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
+                    }));
+        }
+        return client;
+    }
+
     @Bean(name = "localEsClient")
     public RestHighLevelClient localEsClient() {
         List<HttpHost> httpHostList = new ArrayList<>();

src/main/java/com/zhiwei/brandkbs2/es/HotEsDao.java

+package com.zhiwei.brandkbs2.es;
+
+import com.zhiwei.brandkbs2.config.Constant;
+import org.apache.commons.lang3.time.DateUtils;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.Resource;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+/**
+ * @ClassName: HotEsClientDao
+ * @Description HotEsClientDao
+ * @author: sjj
+ * @date: 2023年4月3日09:31:58
+ */
+@Component("hotEsDao")
+public class HotEsDao extends EsClientDao {
+
+    @Resource(name = "hotEsClient")
+    RestHighLevelClient hotEsClient;
+
+    @Override
+    public String[] getIndexes() {
+        return getHotEsIndexes().toArray(new String[0]);
+    }
+
+    @Override
+    protected RestHighLevelClient getEsClient() {
+        return hotEsClient;
+    }
+
+    public List<String> getHotEsIndexes() {
+        List<String> res = new ArrayList<>();
+        // 近3天数据库
+        Date date = new Date();
+        for (int i = 0; i < 3; i++) {
+            String index = "hot_" + Constant.DF_yyyyMMdd.format(DateUtils.addDays(date, -i));
+            res.add(index);
+        }
+        return res;
+    }
+
+}

src/main/java/com/zhiwei/brandkbs2/function/ArticleFunction.java

+package com.zhiwei.brandkbs2.function;
+
+import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.brandkbs2.es.EsClientDao;
+import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
+import com.zhiwei.brandkbs2.service.MarkFlowService;
+import org.apache.commons.lang3.tuple.Pair;
+import org.elasticsearch.search.SearchHits;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * @ClassName: ArticleFunction
+ * @Description ArticleFunction
+ * @author: sjj
+ * @date: 2023-04-03 15:01
+ */
+public class ArticleFunction {
+
+    /**
+     * 创建搜索条件
+     */
+    @FunctionalInterface
+    public interface CreateSearchHelper {
+        EsClientDao.SearchHelper create(MarkSearchDTO dto);
+    }
+
+    /**
+     * 返回搜索结果
+     */
+    @FunctionalInterface
+    public interface SearchHitsAndCount {
+        Pair<SearchHits[], Map<String, Long>> search(MarkSearchDTO dto, boolean scroll) throws IOException;
+    }
+
+    /**
+     * 创建MarkFlowInfo实体
+     */
+    @FunctionalInterface
+    public interface CreateMarkFlowInfo {
+        JSONObject create(MarkFlowService.MarkInfoSource markInfoSource);
+    }
+
+}

src/main/java/com/zhiwei/brandkbs2/pojo/AggreeResult.java

@@ -4,6 +4,7 @@ import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.brandkbs2.common.GenericAttribute;
 import com.zhiwei.brandkbs2.common.GlobalPojo;
 import com.zhiwei.brandkbs2.util.Tools;
+import com.zhiwei.qbjc.bean.tools.BeanTools;
 import lombok.Getter;
 import lombok.Setter;
 import org.apache.commons.lang3.StringUtils;
@@ -12,6 +13,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.stream.Collectors;
 
 /**
  * @ClassName: AggreeResult
@@ -74,6 +76,11 @@ public class AggreeResult extends AbstractBaseMongo {
     private Integer aggreeSize;
 
     /**
+     * 对应es数据的id集合
+     */
+    private List<String> esIds;
+
+    /**
      * 父聚合id
      */
     private String fatherId;
@@ -83,10 +90,11 @@ public class AggreeResult extends AbstractBaseMongo {
      */
     private boolean isFather;
 
-    public static AggreeResult createFather(JSONObject json, String taskId,String projectId, Integer aggreeSize) {
-        AggreeResult aggreeResult = getInstance(json, taskId,projectId);
+    public static AggreeResult createFather(JSONObject json, String taskId, String projectId, List<JSONObject> list) {
+        AggreeResult aggreeResult = getInstance(json, taskId, projectId);
         aggreeResult.setFather(true);
-        aggreeResult.setAggreeSize(aggreeSize);
+        aggreeResult.setAggreeSize(list.size());
+        aggreeResult.setEsIds(list.stream().map(esJson -> esJson.getString("id")).collect(Collectors.toList()));
         aggreeResult.setCTime(System.currentTimeMillis());
         return aggreeResult;
     }
@@ -105,7 +113,12 @@ public class AggreeResult extends AbstractBaseMongo {
         aggreeResult.setTaskId(taskId);
         BaseMap baseMap = Tools.getBaseFromEsMap(json);
         aggreeResult.setTime(baseMap.getTime());
-        aggreeResult.setPlatformId(json.getString(GenericAttribute.ES_PLATFORM_ID));
+        // platformId分两种方式获取
+        String platformId = json.getString(GenericAttribute.ES_PLATFORM_ID);
+        if (null == platformId) {
+            platformId = BeanTools.filterPlatform(GlobalPojo.PLATFORMS, json.getInteger("c5"), json.getInteger("foreign")).getId();
+        }
+        aggreeResult.setPlatformId(platformId);
         String mediaType = GlobalPojo.getMediaType(projectId, baseMap.getPlatform(), baseMap.getSource());
         if (StringUtils.isNotEmpty(mediaType)) {
             aggreeResult.setMediaTypes(Arrays.asList(mediaType.split(",")));

src/main/java/com/zhiwei/brandkbs2/pojo/MarkFlowEntity.java

@@ -6,14 +6,15 @@ import com.zhiwei.base.entity.subclass.mark.CompleteTextMark;
 import com.zhiwei.base.entity.subclass.mark.IncompleteTextMark;
 import com.zhiwei.base.entity.subclass.mark.QATextMark;
 import com.zhiwei.base.entity.subclass.mark.VideoMark;
+import com.zhiwei.base.filter.FilterInfo;
 import com.zhiwei.brandkbs2.common.GenericAttribute;
+import com.zhiwei.brandkbs2.common.GlobalPojo;
 import com.zhiwei.brandkbs2.util.FlowEntityUtil;
 import com.zhiwei.brandkbs2.util.Tools;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
 import org.apache.commons.lang3.ObjectUtils;
-import org.elasticsearch.search.SearchHit;
 
 import java.io.Serializable;
 import java.util.List;
@@ -46,6 +47,11 @@ public class MarkFlowEntity implements Serializable {
      */
     private Object data;
 
+    /**
+     * 消息流相关信息
+     */
+    private FilterInfo filterInfo;
+
     public static MarkFlowEntity getFoldInstance(List<MarkFlowEntity> entities) {
         // 首条数据为主
         MarkFlowEntity entity = Tools.copyByJson(entities.get(0), MarkFlowEntity.class);
@@ -61,23 +67,27 @@ public class MarkFlowEntity implements Serializable {
         return entity;
     }
 
-    public static MarkFlowEntity getAggreeInstance(List<MarkFlowEntity> entities, Integer size) {
-        // 首条数据为主
+    public static MarkFlowEntity getAggreeInstance(List<MarkFlowEntity> entities, AggreeResult aggreeResult, boolean sonDetails) {
+        Integer size = aggreeResult.getAggreeSize();
+        // 首条数据为主,copy丢失FilterInfo
         MarkFlowEntity entity = Tools.copyByJson(entities.get(0), MarkFlowEntity.class);
         if (null != size && size > 1) {
             entity.getType().put("fold", true);
             entity.getType().put("size", size);
         }
+        if (sonDetails) {
+            entity.getType().put("ids", aggreeResult.getEsIds());
+        }
         return entity;
     }
 
     /**
      * Constructor
      *
-     * @param hit
+     * @param tJson
      */
-    public MarkFlowEntity(SearchHit hit) {
-        this(new JSONObject(hit.getSourceAsMap()));
+    public MarkFlowEntity(JSONObject tJson) {
+        this(tJson, null);
     }
 
     /**
@@ -85,7 +95,7 @@ public class MarkFlowEntity implements Serializable {
      *
      * @param tJson
      */
-    public MarkFlowEntity(JSONObject tJson) {
+    public MarkFlowEntity(JSONObject tJson, String linkedGroupId) {
         if (Objects.isNull(tJson.getString(GenericAttribute.ES_CNAME))) {
             tJson.put(GenericAttribute.ES_CNAME, "采集来源缺失");
         }
@@ -98,17 +108,25 @@ public class MarkFlowEntity implements Serializable {
                 if (!Tools.isEmpty(tJson.get("ind_title"))) {
                     tJson.put("ind_title", Tools.filterStrHtmlTag(ObjectUtils.defaultIfNull(tJson.getString("ind_title"), "[\"\"]")));
                 }
-                CompleteTextMark completeText = CompleteTextMark.restoreFromEs(tJson);
-                this.type = FlowEntityUtil.getCompleteTextType(completeText);
-                this.data = completeText;
+                CompleteTextMark completeTextMark = CompleteTextMark.restoreFromEs(tJson);
+                if (null != linkedGroupId) {
+                    completeTextMark.setMgroup(GlobalPojo.getYuQingProjectNameById(linkedGroupId));
+                    this.filterInfo = completeTextMark.filterInfo();
+                }
+                this.type = FlowEntityUtil.getCompleteTextType(completeTextMark);
+                this.data = completeTextMark;
                 break;
             case INCOMPLETE:
                 if (!Tools.isEmpty(tJson.get("ind_full_text"))) {
                     tJson.put("ind_full_text", Tools.filterStrHtmlTag(ObjectUtils.defaultIfNull(tJson.getString("ind_full_text"), "[\"\",\"\"]")));
                 }
-                IncompleteTextMark incompleteText = IncompleteTextMark.restoreFromEs(tJson);
-                this.type = FlowEntityUtil.getIncompleteTextType(incompleteText);
-                this.data = incompleteText;
+                IncompleteTextMark incompleteTextMark = IncompleteTextMark.restoreFromEs(tJson);
+                if (null != linkedGroupId) {
+                    incompleteTextMark.setMgroup(GlobalPojo.getYuQingProjectNameById(linkedGroupId));
+                    this.filterInfo = incompleteTextMark.filterInfo();
+                }
+                this.type = FlowEntityUtil.getIncompleteTextType(incompleteTextMark);
+                this.data = incompleteTextMark;
                 break;
             case QA:
                 if (!Tools.isEmpty(tJson.get("ind_full_text"))) {
@@ -120,9 +138,13 @@ public class MarkFlowEntity implements Serializable {
                 if (!Tools.isEmpty(tJson.get("ind_content"))) {
                     tJson.put("ind_content", Tools.filterStrHtmlTag(ObjectUtils.defaultIfNull(tJson.getString("ind_content"), "[\"\"]")));
                 }
-                QATextMark qaText = QATextMark.restoreFromEs(tJson);
-                this.type = FlowEntityUtil.getQaTextType(qaText);
-                this.data = qaText;
+                QATextMark qaTextMark = QATextMark.restoreFromEs(tJson);
+                if (null != linkedGroupId) {
+                    qaTextMark.setMgroup(GlobalPojo.getYuQingProjectNameById(linkedGroupId));
+                    this.filterInfo = qaTextMark.filterInfo();
+                }
+                this.type = FlowEntityUtil.getQaTextType(qaTextMark);
+                this.data = qaTextMark;
                 break;
             case VIDEO:
                 if (!Tools.isEmpty(tJson.get("ind_full_text"))) {
@@ -131,13 +153,24 @@ public class MarkFlowEntity implements Serializable {
                 if (!Tools.isEmpty(tJson.get("ind_title"))) {
                     tJson.put("ind_title", Tools.filterStrHtmlTag(ObjectUtils.defaultIfNull(tJson.getString("ind_title"), "[\"\"]")));
                 }
-                VideoMark video = VideoMark.restoreFromEs(tJson);
-                this.type = FlowEntityUtil.getVideoType(video);
-                this.data = video;
+                VideoMark videoMark = VideoMark.restoreFromEs(tJson);
+                if (null != linkedGroupId) {
+                    videoMark.setMgroup(GlobalPojo.getYuQingProjectNameById(linkedGroupId));
+                    this.filterInfo = videoMark.filterInfo();
+                }
+                this.type = FlowEntityUtil.getVideoType(videoMark);
+                this.data = videoMark;
                 break;
             default:
                 break;
         }
     }
 
+    public FilterInfo getFilterInfo(String linkedGroupId) {
+        if (null == filterInfo.getSelectType() || null == filterInfo.getFilterMap()) {
+            this.filterInfo = new MarkFlowEntity((JSONObject) data, linkedGroupId).filterInfo;
+        }
+        return filterInfo;
+    }
+
 }

src/main/java/com/zhiwei/brandkbs2/pojo/dto/ExportSearchWholeDTO.java

@@ -5,7 +5,6 @@ import com.alibaba.excel.annotation.write.style.ColumnWidth;
 import com.alibaba.fastjson.JSONObject;
 import lombok.Data;
 import lombok.ToString;
-import org.apache.commons.lang3.StringUtils;
 
 import java.util.Date;
 
@@ -44,10 +43,6 @@ public class ExportSearchWholeDTO {
         dto.setTime(new Date((Long) jsonObject.get("time")));
         dto.setPlatform(jsonObject.getString("platform"));
         dto.setChannel(jsonObject.getString("channel"));
-        // 截取为excel单元格允许的最大长度
-        dto.setTitle(StringUtils.substring(jsonObject.getString("title"), 0, 32767));
-        dto.setContent(StringUtils.substring(jsonObject.getString("content"), 0, 32767));
-        dto.setUrl(StringUtils.substring(jsonObject.getString("url"), 0, 32767));
         return dto;
     }
 }

src/main/java/com/zhiwei/brandkbs2/service/CommonService.java

 package com.zhiwei.brandkbs2.service;
 
 import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.base.entity.subclass.mark.MarkInfo;
 import com.zhiwei.middleware.mark.vo.MarkerTag;
 import com.zhiwei.middleware.mark.vo.TagSearch;
 
@@ -48,6 +49,16 @@ public interface CommonService {
      */
     List<JSONObject> getQbjcPlatform(String... includeFields);
 
+    /**
+     * 添加标注
+     */
+    void addMtag(MarkInfo markInfo);
+
+    /**
+     * 批量添加标注
+     */
+    void addMtagBatch(List<MarkInfo> markInfos);
+
     Long[] getTimeRangeMonth();
 
     Long[] getTimeRangeWeek();

src/main/java/com/zhiwei/brandkbs2/service/EsSearchService.java

+package com.zhiwei.brandkbs2.service;
+
+import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
+import org.apache.commons.lang3.tuple.Pair;
+import org.elasticsearch.search.SearchHits;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * @ClassName: EsSearchService
+ * @Description es数据搜索服务类
+ * @author: sjj
+ * @date: 2023-03-31 17:13
+ */
+public interface EsSearchService {
+
+    /**
+     * 搜索标注数据
+     *
+     * @param dto    搜素实体类
+     * @param scroll 滚动查询
+     * @return 搜索结果
+     */
+    Pair<SearchHits[], Map<String, Long>> searchMarkHitsAndCount(MarkSearchDTO dto, boolean scroll) throws IOException;
+
+    /**
+     * 搜索竞品标注数据
+     *
+     * @param dto    搜素实体类
+     * @param scroll 滚动查询
+     * @return 搜索结果
+     */
+    Pair<SearchHits[], Map<String, Long>> searchContendMarkHitsAndCount(MarkSearchDTO dto, boolean scroll) throws IOException;
+
+    /**
+     * 搜索原始数据
+     *
+     * @param dto    搜素实体类
+     * @param scroll 滚动查询
+     * @return 搜索结果
+     */
+    Pair<SearchHits[], Map<String, Long>> searchOriginHitsAndCount(MarkSearchDTO dto, boolean scroll) throws IOException;
+
+}

src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java

 package com.zhiwei.brandkbs2.service;
 
 import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.brandkbs2.model.ResponseResult;
 import com.zhiwei.brandkbs2.pojo.BaseMap;
 import com.zhiwei.brandkbs2.pojo.Event;
 import com.zhiwei.brandkbs2.pojo.MarkFlowEntity;
@@ -25,6 +26,51 @@ import java.util.Map;
 public interface MarkDataService {
 
     /**
+     * 获取原始数据
+     *
+     * @param markSearchDTO 标注数据搜索传输类
+     * @return 返回结果
+     */
+    PageVO<MarkFlowEntity> getOriginList(MarkSearchDTO markSearchDTO);
+
+    /**
+     * 生成聚合列表并返回id
+     *
+     * @return 聚合id
+     */
+    String generateOriginAggreeList(Long startTime, Long endTime);
+
+    /**
+     * 获取聚合结果列表
+     *
+     * @param markSearchDTO 搜索实体
+     * @return 标注消息流列表
+     */
+    PageVO<MarkFlowEntity> getOriginAggreeList(MarkSearchDTO markSearchDTO);
+
+    /**
+     * @param markSearchDTO 标注数据搜索传输类
+     * @return 返回结果
+     */
+    Pair<String, List<ExportAppYuqingDTO>> downloadOriginList(MarkSearchDTO markSearchDTO);
+
+    /**
+     * 添加标注
+     *
+     * @param zip  压缩数据源
+     * @param mtag 标注信息
+     */
+    void addMtag(String zip, String linkedGroupId, String mtag);
+
+    /**
+     * 批量添加标注
+     *
+     * @param ids  id列表
+     * @param mtag 标注信息
+     */
+    ResponseResult addMtagBatch(List<String> ids, String linkedGroupId, String mtag);
+
+    /**
      * @param markSearchDTO 标注数据搜索传输类
      * @return 返回结果
      */
@@ -49,7 +95,7 @@ public interface MarkDataService {
      * @param id 聚合id
      * @return 进度值
      */
-    String getYuqingMarkAggreeProgress(String id);
+    String getAggreeProgress(String id);
 
     /**
      * 获取聚合结果列表
@@ -114,7 +160,7 @@ public interface MarkDataService {
      * @param endTime   结束时间
      * @return 获取舆情标注数据舆情平台分布
      */
-    List<JSONObject> getMarkPlatformProportion(Long startTime, Long endTime, boolean cache);
+    List<JSONObject> getMarkPlatformProportion(String projectId, Long startTime, Long endTime, boolean cache);
 
     /**
      * 获取舆情标注数据舆情平台分布
@@ -124,7 +170,7 @@ public interface MarkDataService {
      * @param emotion 情感倾向筛选
      * @return 获取舆情标注数据舆情平台分布
      */
-    List<JSONObject> getMarkPlatformProportion(Long startTime, Long endTime,String emotion,boolean cache);
+    List<JSONObject> getMarkPlatformProportion(String projectId, Long startTime, Long endTime, String emotion, boolean cache);
 
     /**
      * 获取舆情标注数据舆情高频词分布
@@ -142,12 +188,10 @@ public interface MarkDataService {
      * @param endTime       结束时间
      * @param emotion       情感倾向名称
      * @param projectId     项目id
-     * @param linkedGroupId 关联组id
      * @param size          分页大小
      * @return 最热标题
      */
-    List<Map.Entry<String, Integer>> getMarkTopTitle(Long startTime, Long endTime, String emotion, String projectId, String linkedGroupId,
-                                                     String contendId, int size) throws IOException;
+    List<Map.Entry<String, Integer>> getMarkTopTitle(Long startTime, Long endTime, String emotion, String projectId, String contendId, int size) throws IOException;
 
     /**
      * 获取时间段某情感数据最多的标题
@@ -156,13 +200,11 @@ public interface MarkDataService {
      * @param endTime       结束时间
      * @param emotion       情感倾向名称
      * @param projectId     项目id
-     * @param linkedGroupId 关联组id
      * @param size          分页大小
      * @param include       是否包含主品牌
      * @return 最热标题
      */
-    List<Map.Entry<String, Integer>> getMarkTopTitle(Long startTime, Long endTime, String emotion, String projectId, String linkedGroupId,
-                                                     String contendId, int size, boolean include) throws IOException;
+    List<Map.Entry<String, Integer>> getMarkTopTitle(Long startTime, Long endTime, String emotion, String projectId, String contendId, int size, boolean include) throws IOException;
 
     /**
      * 根据聚合标题和时间获取首发稿件
@@ -171,10 +213,9 @@ public interface MarkDataService {
      * @param endTime       结束时间
      * @param aggTitle      聚合标题
      * @param projectId     项目id
-     * @param linkedGroupId 关联组id
      * @return 首发稿件
      */
-    BaseMap getFirstArticle(Long startTime, Long endTime, String aggTitle, String projectId, String linkedGroupId, String contendId) throws IOException;
+    BaseMap getFirstArticle(Long startTime, Long endTime, String aggTitle, String projectId, String contendId) throws IOException;
 
     /**
      * 根据聚合标题和时间获取首发稿件
@@ -183,11 +224,10 @@ public interface MarkDataService {
      * @param endTime       结束时间
      * @param aggTitle      聚合标题
      * @param projectId     项目id
-     * @param linkedGroupId 关联组id
      * @param include       是否包含主品牌
      * @return 首发稿件
      */
-    BaseMap getFirstArticle(Long startTime, Long endTime, String aggTitle, String projectId, String linkedGroupId, String contendId, boolean include) throws IOException;
+    BaseMap getFirstArticle(Long startTime, Long endTime, String aggTitle, String projectId, String contendId, boolean include) throws IOException;
 
     /**
      * 搜索标注数据通过事件
@@ -207,7 +247,7 @@ public interface MarkDataService {
      * @param fetchSource
      * @return
      */
-    List<JSONObject> searchMarkDataByTime(String projectId, String linkedGroupId, String contendId, Long startTime, Long endTime, String... fetchSource);
+    List<JSONObject> searchMarkDataByTime(String projectId,String contendId, Long startTime, Long endTime, String... fetchSource);
 
     /**
      * 竞品库-获取搜索条件
@@ -238,12 +278,11 @@ public interface MarkDataService {
     /**
      * 竞品库-获取竞品对比传播分析数据
      *
-     * @param markSearchDTO  标注数据搜索传输类
      * @param hotArticleSize 热门稿件数量
      * @param cache          是否启用缓存
      * @return JSONObject
      */
-    JSONObject getContendSpreadAnalyze(Long startTime, Long endTime, String projectId, String contendId, int hotArticleSize, boolean useCache) throws IOException;
+    JSONObject getContendSpreadAnalyze(Long startTime, Long endTime, String projectId, String contendId, int hotArticleSize, boolean cache) throws IOException;
 
     /**
      * 竞品库-舆情导出
@@ -320,9 +359,10 @@ public interface MarkDataService {
      * @param size      大小
      * @return 发布最多稿件的渠道
      */
-    List<Map<String, Object>> getEsTopSource(Long startTime, Long endTime, String projectId, String linkedGroupId, String contendId, String emotion, int size) throws IOException;
+    List<Map<String, Object>> getEsTopSource(Long startTime, Long endTime, String projectId, String contendId, String emotion, int size) throws IOException;
+
+    JSONObject getLastMarkData(String projectId, String contendId, String platform, String realSource, String source);
 
-    JSONObject getLastMarkData(String projectId, String linkedGroupId, String contendId, String platform, String realSource, String source);
+    BoolQueryBuilder projectContendIdQuery(String projectId, String contendId);
 
-    BoolQueryBuilder projectLinkedGroupContendIdQuery(String projectId, String linkedGroupId, String contendId);
 }

src/main/java/com/zhiwei/brandkbs2/service/MarkFlowService.java

 package com.zhiwei.brandkbs2.service;
 
 import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.base.entity.subclass.mark.MarkInfo;
 import com.zhiwei.brandkbs2.pojo.MarkFlowEntity;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+import java.util.List;
 
 /**
  * @ClassName: MarkFlowService
@@ -14,13 +19,36 @@ public interface MarkFlowService {
     /**
      * 创建标注消息流实体
      *
-     * @param json
-     * @param projectId
+     * @param markInfoSource
+     * @return
+     */
+    JSONObject createMarkFlowInfo(MarkInfoSource markInfoSource);
+
+    /**
+     * 创建标注消息流实体
+     *
+     * @param markInfoSource
+     * @return
+     */
+    JSONObject createMarkFlowInfo4Origin(MarkInfoSource markInfoSource);
+
+    /**
+     * 匹配标注标签值
+     * @param mes
      * @param linkedGroupId
-     * @param contendId
+     */
+    List<MarkFlowEntity> markTagInfo(List<MarkFlowEntity> mes, String linkedGroupId);
+
+    /**
+     * 解析JSON数据
+     *
+     * @param json
+     * @param mgroup
+     * @param mperson
+     * @param mtag
      * @return
      */
-    JSONObject createMarkFlowInfo(JSONObject json, String projectId,String linkedGroupId, String contendId);
+    MarkInfo markInfoFromJson(JSONObject json, String mgroup, String mperson, String mtag);
 
     /**
      * 快照页信息组装
@@ -37,4 +65,24 @@ public interface MarkFlowService {
      */
     MarkFlowEntity getShotPageFromCache(String id);
 
+    @Data
+    @AllArgsConstructor
+    class MarkInfoSource {
+        private JSONObject json;
+        private String projectId;
+        private String contendId;
+        private String linkedGroupId;
+
+        public MarkInfoSource(JSONObject json, String projectId, String contendId) {
+            this.json = json;
+            this.projectId = projectId;
+            this.contendId = contendId;
+        }
+
+        public MarkInfoSource(JSONObject json, String linkedGroupId) {
+            this.json = json;
+            this.linkedGroupId = linkedGroupId;
+        }
+    }
+
 }

src/main/java/com/zhiwei/brandkbs2/service/ProjectService.java

@@ -22,7 +22,7 @@ public interface ProjectService {
      *
      * @param projectVO 项目配置展示对象
      */
-    void addProject(ProjectVO projectVO);
+    ResponseResult addProject(ProjectVO projectVO);
 
     /**
      * 查询所有项目列表
@@ -56,7 +56,7 @@ public interface ProjectService {
      *
      * @param projectVO 项目展示对象
      */
-    void updateProject(ProjectVO projectVO);
+    ResponseResult updateProject(ProjectVO projectVO);
 
     /**
      * 根据项目ID删除项目

src/main/java/com/zhiwei/brandkbs2/service/impl/ChannelServiceImpl.java

@@ -949,7 +949,6 @@ public class ChannelServiceImpl implements ChannelService {
 		long s0 = System.currentTimeMillis();
 		String projectId = UserThreadLocal.getProjectId();
 		String contendId = "0";
-		String linkedGroupId = projectService.getProjectByContendId(projectId, contendId).getBrandLinkedGroupId();
 		Query query = new Query(Criteria.where("projectId").is(projectId).and("contendId").is(contendId));
 		channelDao.addKeywordFuzz(query, keyword, "source");
 		if (!(null == platforms || platforms.contains("全部"))) {
@@ -978,19 +977,12 @@ public class ChannelServiceImpl implements ChannelService {
 					JSONObject result = new JSONObject();
 					result.put("id", channel.getId());
 					result.put("channel", channel);
-					result.put("lastArticle", markDataService.getLastMarkData(projectId, linkedGroupId, contendId,
-							channel.getPlatform(), channel.getRealSource(), channel.getSource()));
+					result.put("lastArticle", markDataService.getLastMarkData(projectId, contendId, channel.getPlatform(), channel.getRealSource(), channel.getSource()));
 					return result;
 				}, esSearchExecutor)).collect(Collectors.toList());
 		CompletableFuture.allOf(futureList.toArray(new CompletableFuture[0])).join();
 		Map<String, JSONObject> idMap = futureList.stream().map(CompletableFuture::join)
 				.collect(Collectors.toMap(json -> json.getString("id"), a -> a));
-//        List<JSONObject> resultList = channelList.stream().map(channel -> {
-//            JSONObject result = new JSONObject();
-//            result.put("channelInfo", channel);
-//            result.put("lastArticle", markDataService.getLastMarkUrl(projectId, linkedGroupId, contendId, channel.getPlatform(), channel.getRealSource(), channel.getSource()));
-//            return result;
-//        }).collect(Collectors.toList());
 		long s3 = System.currentTimeMillis();
 		log.info("渠道发文查询耗时:" + (s3 - s2));
 		return PageVO.createPageVo(total, page, pageSize,
@@ -1515,8 +1507,7 @@ public class ChannelServiceImpl implements ChannelService {
 	}
 
 	private Long markCountByEmotion(Channel channel, String emotion) throws IOException {
-		BoolQueryBuilder postFilter = markDataService.projectLinkedGroupContendIdQuery(channel.getProjectId(),
-				channel.getLinkedGroupId(), channel.getContendId());
+		BoolQueryBuilder postFilter = markDataService.projectContendIdQuery(channel.getProjectId(),channel.getContendId());
 		postFilter.must(QueryBuilders.termQuery("brandkbs_mark_cache_maps.name.keyword", emotion));
 		return esClientDao.count(postFilter);
 	}

src/main/java/com/zhiwei/brandkbs2/service/impl/CommonServiceImpl.java

 package com.zhiwei.brandkbs2.service.impl;
 
 import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.base.entity.subclass.mark.MarkInfo;
 import com.zhiwei.brandkbs2.common.GlobalPojo;
 import com.zhiwei.brandkbs2.config.Constant;
 import com.zhiwei.brandkbs2.enmus.EmotionEnum;
@@ -112,6 +113,16 @@ public class CommonServiceImpl implements CommonService {
     }
 
     @Override
+    public void addMtag(MarkInfo markInfo) {
+        markClient.upsert(markInfo);
+    }
+
+    @Override
+    public void addMtagBatch(List<MarkInfo> markInfos) {
+        markClient.eventCollectionUpsert(markInfos);
+    }
+
+    @Override
     public Long[] getTimeRangeMonth() {
         long endTime = Tools.truncDate(new Date(), Constant.DAY_PATTERN).getTime();
         long startTime = DateUtils.addMonths(new Date(endTime), -1).getTime();

src/main/java/com/zhiwei/brandkbs2/service/impl/CustomEventServiceImpl.java

@@ -90,19 +90,17 @@ public class CustomEventServiceImpl implements CustomEventService {
         List<JSONObject> resultList = null;
         try {
             String projectId = UserThreadLocal.getProjectId();
-            String linkedGroupId = projectService.getProjectVOById(projectId).getBrandLinkedGroupId();
             String redisKey = RedisUtil.getCustomEventListKey(projectId, startTime, endTime);
             String resultStr = redisUtil.get(redisKey);
             if (StringUtils.isNotEmpty(resultStr)) {
                 return JSON.parseArray(resultStr, JSONObject.class);
             }
-            List<Map.Entry<String, Integer>> resList = markDataService.getMarkTopTitle(startTime, endTime, EmotionEnum.ALL.getName(), projectId,
-                    linkedGroupId, "0", 20);
+            List<Map.Entry<String, Integer>> resList = markDataService.getMarkTopTitle(startTime, endTime, EmotionEnum.ALL.getName(), projectId, "0", 20);
             resultList = resList.stream().map(map -> {
                 JSONObject result = new JSONObject();
                 String aggTitle = map.getKey();
                 try {
-                    BaseMap firstArticle = markDataService.getFirstArticle(startTime, endTime, aggTitle, projectId, linkedGroupId, Constant.PRIMARY_CONTEND_ID);
+                    BaseMap firstArticle = markDataService.getFirstArticle(startTime, endTime, aggTitle, projectId, Constant.PRIMARY_CONTEND_ID);
                     result.put("title", firstArticle.getTitle());
                     result.put("time", firstArticle.getTime());
                     result.put("url", firstArticle.getUrl());
@@ -297,9 +295,8 @@ public class CustomEventServiceImpl implements CustomEventService {
         customEventDao.updateOneByIdWithField(eventId, Update.update("status", false));
         ApplicationProjectListener.getThreadPool().execute(() -> {
             try {
-                String linkedGroupId = projectService.getProjectByContendId(customEvent.getProjectId(), customEvent.getContendId()).getBrandLinkedGroupId();
                 //查询符合该事件时间段的所有稿件信息
-                List<JSONObject> articles = markDataService.searchMarkDataByTime(customEvent.getProjectId(), linkedGroupId, customEvent.getContendId(),
+                List<JSONObject> articles = markDataService.searchMarkDataByTime(customEvent.getProjectId(), customEvent.getContendId(),
                         customEvent.getStartTime(), customEvent.getEndTime());
                 log.info("自定义事件:{} 查询到符合该事件时间段的所有稿件数{}个", eventId, articles.size());
                 List<String> keywords = customEvent.getKeywords();

src/main/java/com/zhiwei/brandkbs2/service/impl/IndexServiceImpl.java

@@ -261,7 +261,6 @@ public class IndexServiceImpl implements IndexService {
             endTime = timeRangeMonth[1];
             flag = false;
         }
-        String linkedGroupId = projectService.getProjectById(projectId).getBrandLinkedGroupId();
         String redisKey = RedisUtil.getIndexSpread(projectId, startTime, endTime);
         String resultStr;
         // 返回缓存
@@ -285,12 +284,12 @@ public class IndexServiceImpl implements IndexService {
             resJson.put("type", type);
             resJson.put("spreadingTend", lineList);
             JSONObject summary = new JSONObject();
-            List<Map.Entry<String, Integer>> resList = markDataService.getMarkTopTitle(startTime, endTime, EmotionEnum.ALL.getName(), projectId, linkedGroupId, contendId, 1);
+            List<Map.Entry<String, Integer>> resList = markDataService.getMarkTopTitle(startTime, endTime, EmotionEnum.ALL.getName(), projectId, contendId, 1);
             summary.put("yqSpread", this.getYqSpread(startTime, endTime, projectId, contendId, lineList, type, flag));
-            summary.put("pcTopArticle", this.getPcTopArticle(startTime, endTime, projectId, linkedGroupId, contendId, resList));
-            summary.put("topArticle", this.getMobileTopArticle(startTime, endTime, projectId, linkedGroupId, contendId, resList));
+            summary.put("pcTopArticle", this.getPcTopArticle(startTime, endTime, projectId, contendId, resList));
+            summary.put("topArticle", this.getMobileTopArticle(startTime, endTime, projectId, contendId, resList));
             summary.put("topEvent", this.getTopEvent(startTime, endTime, projectId, contendId));
-            summary.put("topSource", this.getTopSource(startTime, endTime, projectId, linkedGroupId, contendId));
+            summary.put("topSource", this.getTopSource(startTime, endTime, projectId, contendId));
             resJson.put("summary", summary);
             redisUtil.setExpire(redisKey, JSON.toJSONString(resJson));
         } catch (IOException e) {
@@ -400,12 +399,12 @@ public class IndexServiceImpl implements IndexService {
      * @param resList   首发稿件列表
      * @return 最热文章
      */
-    private JSONObject getPcTopArticle(long startTime, long endTime, String projectId, String linkedGroupId, String contendId, List<Map.Entry<String, Integer>> resList) throws IOException {
+    private JSONObject getPcTopArticle(long startTime, long endTime, String projectId, String contendId, List<Map.Entry<String, Integer>> resList) throws IOException {
         JSONObject result = new JSONObject();
         if (CollectionUtils.isNotEmpty(resList)) {
             Map.Entry<String, Integer> resMap = resList.get(0);
             String aggTitle = resMap.getKey();
-            BaseMap firstArticle = markDataService.getFirstArticle(startTime, endTime, aggTitle, projectId, linkedGroupId, contendId);
+            BaseMap firstArticle = markDataService.getFirstArticle(startTime, endTime, aggTitle, projectId, contendId);
             result.put("title", firstArticle.getTitleNullOptionalContent());
             result.put("url", firstArticle.getUrl());
             result.put("time", firstArticle.getTime());
@@ -425,13 +424,13 @@ public class IndexServiceImpl implements IndexService {
      * @param contendId 竞品ID
      * @return 最热文章
      */
-    private String getMobileTopArticle(long startTime, long endTime, String projectId, String linkedGroupId, String contendId, List<Map.Entry<String, Integer>> resList) throws IOException {
+    private String getMobileTopArticle(long startTime, long endTime, String projectId, String contendId, List<Map.Entry<String, Integer>> resList) throws IOException {
         if (CollectionUtils.isEmpty(resList)) {
             return "暂无最热文章。";
         }
         Map.Entry<String, Integer> resMap = resList.get(0);
         String aggTitle = resMap.getKey();
-        BaseMap firstArticle = markDataService.getFirstArticle(startTime, endTime, aggTitle, projectId, linkedGroupId, contendId);
+        BaseMap firstArticle = markDataService.getFirstArticle(startTime, endTime, aggTitle, projectId, contendId);
         SimpleDateFormat format = new SimpleDateFormat("M月d日");
         return "《" + firstArticle.getTitleNullOptionalContent() + "》一文于" + format.format(firstArticle.getTime()) + "在" + firstArticle.getSource()
                 + "（" + firstArticle.getPlatform() + "）上首发，相似文章数" + resMap.getValue() + "篇。";
@@ -471,11 +470,10 @@ public class IndexServiceImpl implements IndexService {
      * @param contendId 竞品ID
      * @return 高频渠道
      */
-    private JSONObject getTopSource(long startTime, long endTime, String projectId, String linkedGroupId, String contendId) throws IOException {
+    private JSONObject getTopSource(long startTime, long endTime, String projectId, String contendId) throws IOException {
         JSONObject result = new JSONObject();
-        // TODO 等待线上es数据格式调整
-        List<Map<String, Object>> positiveList = markDataService.getEsTopSource(startTime, endTime, projectId, linkedGroupId, contendId, EmotionEnum.POSITIVE.getName(), 3);
-        List<Map<String, Object>> negativeList = markDataService.getEsTopSource(startTime, endTime, projectId, linkedGroupId, contendId, EmotionEnum.NEGATIVE.getName(), 1);
+        List<Map<String, Object>> positiveList = markDataService.getEsTopSource(startTime, endTime, projectId, contendId, EmotionEnum.POSITIVE.getName(), 3);
+        List<Map<String, Object>> negativeList = markDataService.getEsTopSource(startTime, endTime, projectId, contendId, EmotionEnum.NEGATIVE.getName(), 1);
         result.put("positiveList", positiveList);
         result.put("negativeList", negativeList);
         return result;

src/main/java/com/zhiwei/brandkbs2/service/impl/ProjectServiceImpl.java

@@ -7,7 +7,6 @@ import com.zhiwei.brandkbs2.common.GlobalPojo;
 import com.zhiwei.brandkbs2.config.Constant;
 import com.zhiwei.brandkbs2.dao.EventMiddlewareDao;
 import com.zhiwei.brandkbs2.dao.ProjectDao;
-import com.zhiwei.brandkbs2.dao.QbjcPojoDao;
 import com.zhiwei.brandkbs2.dao.UserDao;
 import com.zhiwei.brandkbs2.enmus.response.ProjectCodeEnum;
 import com.zhiwei.brandkbs2.exception.ExceptionCast;
@@ -62,9 +61,6 @@ public class ProjectServiceImpl implements ProjectService {
     @Resource(name = "projectDao")
     private ProjectDao projectDao;
 
-    @Resource(name = "qbjcPojoDao")
-    private QbjcPojoDao qbjcPojoDao;
-
     @Resource(name = "mongoUtil")
     private com.zhiwei.brandkbs2.util.MongoUtil mongoUtil;
 
@@ -84,7 +80,10 @@ public class ProjectServiceImpl implements ProjectService {
     private String brandkbsImageUrl;
 
     @Override
-    public void addProject(ProjectVO projectVO) {
+    public ResponseResult addProject(ProjectVO projectVO) {
+        if (!Tools.isEmpty(projectVO.getHitKeywords()) && projectVO.getModuleShowList().contains(2)) {
+            return ResponseResult.failure("关键字和原始数据配置异常");
+        }
         if (Objects.isNull(projectVO)) {
             //抛出非法参数异常
             ExceptionCast.cast(CommonCodeEnum.INVALID_PARAM);
@@ -92,12 +91,8 @@ public class ProjectServiceImpl implements ProjectService {
         Project existsProject = projectDao.findOne("projectName", projectVO.getProjectName());
         if (Objects.nonNull(existsProject)) {
             //抛出项目名称已存在异常
-            ExceptionCast.cast(ProjectCodeEnum.PROJECT_EXISTSNAME_ERROR);
+            return ResponseResult.failure(ProjectCodeEnum.PROJECT_EXISTSNAME_ERROR.message());
         }
-        //给超管添加该项目权限，并且初始化预警推送 TODO
-//            for (Integer SUPER_ADMINUid : userMapper.getAllSUPER_ADMINUids()) {
-//                UserService.insertAdminRoleAndWarn(SUPER_ADMINUid, project.getId());
-//            }
         Project project = projectVO.creatProject(new Date());
         projectDao.insertOneWithoutId(project);
 
@@ -119,6 +114,7 @@ public class ProjectServiceImpl implements ProjectService {
                         project.getId(), contend.getBrandName(), contend.getId(), submitter, submitterId);
             }
         }
+        return ResponseResult.success();
     }
 
     @Override
@@ -180,7 +176,10 @@ public class ProjectServiceImpl implements ProjectService {
     }
 
     @Override
-    public void updateProject(ProjectVO projectVO) {
+    public ResponseResult updateProject(ProjectVO projectVO) {
+        if (!Tools.isEmpty(projectVO.getHitKeywords()) && projectVO.getModuleShowList().contains(2)) {
+            return ResponseResult.failure("关键字和原始数据配置异常");
+        }
         Project existsProject = projectDao.findOneById(projectVO.getId());
         if (Objects.isNull(existsProject)) {
             //抛出非法参数异常
@@ -213,6 +212,7 @@ public class ProjectServiceImpl implements ProjectService {
                         project.getId(), contend.getBrandName(), contend.getId(), submitter, submitterId);
             }
         }
+        return ResponseResult.success();
     }
 
     @Override
@@ -436,27 +436,6 @@ public class ProjectServiceImpl implements ProjectService {
                 .collect(Collectors.groupingBy(MarkerTag::getGroupName));
     }
 
-//    public JSONObject getUserInfoAndProjectConfig(User user) {
-//        long start = System.currentTimeMillis();
-//        JSONObject result = new JSONObject();
-//        JSONObject userJson = new JSONObject();
-//        userJson.put("id", user.getId());
-//        userJson.put("nick", user.getNickname());
-//        userJson.put("username", user.getUsername());
-//        userJson.put("projectId", UserThreadLocal.getProjectId());
-//        userJson.put("roleId", UserThreadLocal.getRoleId());
-//        userJson.put("expiredTime",userProject.getExpiredTime());
-//        //userJson.put("exportAmount",userProject.getExportAmount());
-//        Project project = projectMapper.selectByPrimaryKey(userProject.getProjectId());
-//        userJson.put("projectName", project.getProjectName());
-//        userJson.put("endTime", project.getLastTime());
-//        userJson.put("isBackUp", project.getIsBackup());
-//        result.put("user", userJson);
-//        result.put("config", this.getConfig(userProject.getProjectId()));
-//        log.info("获取项目配置信息时间：{}", System.currentTimeMillis() - start);
-//        return result;
-//    }
-
     protected List<JSONObject> getProjectListByUser(User user, boolean retainExpired) {
         List<Project> lists;
         Map<String, Date> roleExpired = new HashMap<>();
@@ -476,6 +455,7 @@ public class ProjectServiceImpl implements ProjectService {
             json.put("name", project.getProjectName());
             json.put("group", project.getBrandLinkedGroup());
             json.put("avatarUrl", project.getAvatarUrl());
+            json.put("permission", getPermissionList(project));
             if (retainExpired) {
                 Date expiredTime = roleExpired.get(project.getId());
                 if (null == expiredTime || expiredTime.after(new Date())) {
@@ -502,5 +482,18 @@ public class ProjectServiceImpl implements ProjectService {
         return project;
     }
 
+    /**
+     * 获取权限列表
+     *
+     * @param project
+     * @return
+     */
+    private List<Map<String, Object>> getPermissionList(Project project) {
+        List<Map<String, Object>> permissionList = new ArrayList<>();
+        Map<String, Object> originPermission = new HashMap<>();
+        originPermission.put("origin", project.getModuleShowList().contains(2));
+        permissionList.add(originPermission);
+        return permissionList;
+    }
 
 }

src/main/java/com/zhiwei/brandkbs2/service/impl/ProjectWarnServiceImpl.java

@@ -649,7 +649,7 @@ public class ProjectWarnServiceImpl implements ProjectWarnService {
             project = projectService.getProjectById(projectId);
             log.info("获取渠道参与预警开始,project:{}", project.getProjectName());
             EsClientDao.SearchHelper helper = EsClientDao.createSearchHelper();
-            BoolQueryBuilder postFilter = EsQueryTools.assembleCacheMapsQuery(project.getId(), project.getBrandLinkedGroupId(), "0");
+            BoolQueryBuilder postFilter = EsQueryTools.assembleCacheMapsQuery(project.getId(), "0");
             // 时间(标注时间)
             postFilter.must(QueryBuilders.rangeQuery("mtime").gte(start).lt(end));
             // 自定义渠道

src/main/java/com/zhiwei/brandkbs2/util/Tools.java

@@ -19,6 +19,7 @@ import com.zhiwei.brandkbs2.pojo.BaseMap;
 import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform;
 import com.zhiwei.qbjc.bean.pojo.common.Tag;
 import com.zhiwei.qbjc.bean.tools.BeanTools;
+import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.MapUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -41,9 +42,12 @@ import java.nio.channels.ReadableByteChannel;
 import java.nio.channels.WritableByteChannel;
 import java.text.ParseException;
 import java.util.*;
+import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
 
 import static com.zhiwei.brandkbs2.common.GenericAttribute.ES_MARK_CACHE_MAPS;
 import static com.zhiwei.brandkbs2.config.Constant.*;
@@ -959,6 +963,16 @@ public class Tools {
         return new JSONObject();
     }
 
+    public static JSONObject getHotEsHitMap(Map<String, Object> esMap, String hitKey) {
+        List<Map<String, Object>> cacheMaps = (List<Map<String, Object>>) esMap.get("monitor_rule_cache_maps");
+        for (Map<String, Object> cacheMap : cacheMaps) {
+            if (hitKey.equals(cacheMap.get("project"))) {
+                return new JSONObject(cacheMap);
+            }
+        }
+        return new JSONObject();
+    }
+
     public static JSONObject getBrandkbsHitMapWithProjectId(Map<String, Object> esMap, String projectId) {
         List<Map<String, Object>> cacheMaps = (List<Map<String, Object>>) esMap.get(GenericAttribute.ES_BRANDKBS_CACHE_MAPS);
         for (Map<String, Object> cacheMap : cacheMaps) {
@@ -1025,4 +1039,124 @@ public class Tools {
     public static boolean isNullOrUndefined(Object o) {
         return (isEmpty(o) || Objects.equals("undefined", o) || Objects.equals("null", o));
     }
+
+    /**
+     * 正则表达式匹配结果
+     *
+     * @param line
+     * @param pattern
+     * @return List<String>
+     */
+    public static List<String> patternMatchFind(String line, String pattern) {
+        List<String> res = new ArrayList<>();
+        // 创建 Pattern 对象
+        Pattern r = Pattern.compile(pattern);
+        // 现在创建 matcher 对象
+        Matcher m = r.matcher(line);
+        while (m.find()) {
+            res.add(m.group());
+        }
+        return res;
+    }
+
+    /**
+     * gzip压缩字符串
+     *
+     * @param primStr
+     * @return
+     */
+    public static String gzip(String primStr) {
+        if (primStr == null || primStr.length() == 0) {
+            return primStr;
+        }
+
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+
+        GZIPOutputStream gzip = null;
+        try {
+            gzip = new GZIPOutputStream(out);
+            gzip.write(primStr.getBytes());
+        } catch (IOException e) {
+            e.printStackTrace();
+        } finally {
+            if (gzip != null) {
+                try {
+                    gzip.close();
+                } catch (IOException e) {
+                    e.printStackTrace();
+                }
+            }
+        }
+        return new String(new Base64().encode(out.toByteArray()));
+    }
+
+    /**
+     * 使用gzip进行解压缩
+     *
+     * @param compressedStr
+     * @return
+     */
+    public static String gunzip(String compressedStr) {
+        if (compressedStr == null) {
+            return null;
+        }
+
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
+        ByteArrayInputStream in = null;
+        GZIPInputStream ginzip = null;
+        byte[] compressed = null;
+        String decompressed = null;
+        try {
+            compressed = new Base64().decode(compressedStr);
+            in = new ByteArrayInputStream(compressed);
+            ginzip = new GZIPInputStream(in);
+
+            byte[] buffer = new byte[102400];
+            int offset = -1;
+            while ((offset = ginzip.read(buffer)) != -1) {
+                out.write(buffer, 0, offset);
+            }
+            decompressed = out.toString();
+        } catch (IOException e) {
+            e.printStackTrace();
+        } finally {
+            if (ginzip != null) {
+                try {
+                    ginzip.close();
+                } catch (IOException e) {
+                }
+            }
+            if (in != null) {
+                try {
+                    in.close();
+                } catch (IOException e) {
+                }
+            }
+            if (out != null) {
+                try {
+                    out.close();
+                } catch (IOException e) {
+                }
+            }
+        }
+        return decompressed;
+    }
+
+    /**
+     * 判断标签是否合法
+     * @param mtag
+     * @return
+     */
+    public static boolean isIllegalTag(String mtag) {
+        // 非空
+        if (isEmpty(mtag)) {
+            return false;
+        }
+        String pattern = ",[\\d]+\\=";
+        List<String> mtags = Tools.isEmpty(mtag) ? new ArrayList<>() : Tools.patternMatchFind(mtag, pattern);
+        // 无重复标签组
+        long count = mtags.stream().distinct().count();
+        return count != mtags.size();
+    }
+
 }
\ No newline at end of file

src/main/resources/application-dev.properties

@@ -38,6 +38,10 @@ es.httpClusterNodes=192.168.0.130:9200:qbjc-back:yuqing.zhiweidata.com,192.168.0
 #es.password=3vh65l$i6qQA
 es.username=joker
 es.password=jokerdevops
+es.clusterNodesHotES=192.168.0.51:9400
+es.clusterNameHotES=hot-new-es
+es.usernameHotES=elastic
+es.passwordHotES=qWxZRW42OHkuOhmF5AXX
 es.index.test = false
 
 #channel-index
@@ -53,6 +57,8 @@ mark.provider.group=zhiwei-mark
 #\u4E8B\u4EF6\u4E2D\u95F4\u4EF6
 event.registry.address=zookeeper://192.168.0.203:2182?backup=192.168.0.104:2182,192.168.0.105:2182
 event.provider.group=zhiwei-event
+#\u53BB\u91CD\u4E2D\u95F4\u4EF6
+filter.provider.group=zhiwei-bloom-filter
 #\u7528\u6237\u4E2D\u5FC3
 auth.center.client.consumer.group=zhiwei-auth
 auth.center.client.registry.address=zookeeper://192.168.0.203:2182?backup=192.168.0.104:2182,192.168.0.105:2182

src/main/resources/application-local.properties

@@ -40,6 +40,10 @@ es.httpClusterNodes=202.107.192.94:1443:brandkbs2:3vh65l$i6qQA
 #es.password=3vh65l$i6qQA
 es.username=joker
 es.password=jokerdevops
+es.clusterNodesHotES=202.107.192.94:29400
+es.clusterNameHotES=hot-new-es
+es.usernameHotES=elastic
+es.passwordHotES=qWxZRW42OHkuOhmF5AXX
 es.index.test = false
 
 #channel-index
@@ -55,6 +59,8 @@ mark.provider.group=zhiwei-mark-test_liuyu
 #\u4E8B\u4EF6\u4E2D\u95F4\u4EF6
 event.registry.address=zookeeper://192.168.0.11:2181?backup=192.168.0.30:2181,192.168.0.35:2181
 event.provider.group=zhiwei-event-ygd
+#\u53BB\u91CD\u4E2D\u95F4\u4EF6
+filter.provider.group=local-filter
 #\u7528\u6237\u4E2D\u5FC3
 auth.center.client.consumer.group=zhiwei-auth-dev-liuyu
 auth.center.client.registry.address=zookeeper://192.168.0.11:2181?backup=192.168.0.30:2181,192.168.0.35:2181

src/main/resources/application-prod.properties

@@ -38,6 +38,10 @@ es.httpClusterNodes=192.168.0.130:9200:qbjc-back:yuqing.zhiweidata.com,192.168.0
 #es.password=3vh65l$i6qQA
 es.username=joker
 es.password=jokerdevops
+es.clusterNodesHotES=192.168.0.51:9400
+es.clusterNameHotES=hot-new-es
+es.usernameHotES=elastic
+es.passwordHotES=qWxZRW42OHkuOhmF5AXX
 es.index.test = false
 
 #channel-index
@@ -53,6 +57,8 @@ mark.provider.group=zhiwei-mark
 #\u4E8B\u4EF6\u4E2D\u95F4\u4EF6
 event.registry.address=zookeeper://192.168.0.203:2182?backup=192.168.0.104:2182,192.168.0.105:2182
 event.provider.group=zhiwei-event
+#\u53BB\u91CD\u4E2D\u95F4\u4EF6
+filter.provider.group=zhiwei-bloom-filter
 #\u7528\u6237\u4E2D\u5FC3
 auth.center.client.consumer.group=zhiwei-auth
 auth.center.client.registry.address=zookeeper://192.168.0.203:2182?backup=192.168.0.104:2182,192.168.0.105:2182

src/test/java/com/zhiwei/brandkbs2/BrandkbsIndexCreateTest.java

 package com.zhiwei.brandkbs2;
 
+import com.zhiwei.brandkbs2.es.ChannelEsDao;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.client.RequestOptions;
@@ -30,9 +31,48 @@ public class BrandkbsIndexCreateTest {
     public void test1() throws IOException {
         String channelIndex = "brandkbs2_channel_record_2023";
 //          String brandkbsIndex = "brandkbs2_2022";
-          putChannelIndex(channelIndex);
+//          putChannelIndex(channelIndex);
 //        putBrandkbsIndex("brandkbs2_2018","brandkbs2_2019","brandkbs2_2020","brandkbs2_2021");
 //          putBrandkbsIndex("brandkbs2_2017");
+        putChannelCopyIndex(ChannelEsDao.CHANNEL_COPY_ES_INDEX_NAME);
+    }
+
+    public void putChannelCopyIndex(String... indexes) throws IOException {
+        for (String index : indexes) {
+            CreateIndexRequest createIndexRequest = new CreateIndexRequest(index);
+            // 设置分片和副本
+            createIndexRequest.settings(Settings.builder().put("number_of_shards", "7").put("number_of_replicas", "1"));
+            Map<String, Object> properties = new HashMap<>();
+            builder(properties, "text", "id", true);
+            builder(properties, "text", "project_id", true);
+            builder(properties, "text", "contend_id", true);
+            builder(properties, "keyword", "platform", false);
+            builder(properties, "keyword", "real_source", false);
+            builder(properties, "keyword", "source", false);
+            builder(properties, "text", "fid", true);
+            builder(properties, "integer", "emotion", false);
+            builder(properties, "float", "emotion_index", false);
+
+            builder(properties, "long", "c_time", false);
+            builder(properties, "long", "last_time", false);
+            builder(properties, "text", "avatar_url", false);
+            builder(properties, "long", "article_count", false);
+            builder(properties, "long", "event_count", false);
+            builder(properties, "text", "experience_level", true);
+            builder(properties, "boolean", "show", false);
+            builder(properties, "boolean", "is_collect", false);
+            builder(properties, "long", "collect_time", false);
+            builder(properties, "float", "influence", false);
+            Map<String, Object> mapping = new HashMap<>();
+            mapping.put("properties", properties);
+            createIndexRequest.mapping(mapping);
+            //设置超时时间 分钟
+            createIndexRequest.setTimeout(TimeValue.timeValueMinutes(1));
+            //设置主节点超时时间
+            createIndexRequest.setMasterTimeout(TimeValue.timeValueMinutes(1));
+            esClient.indices().create(createIndexRequest, RequestOptions.DEFAULT);
+            log.info("索引:{}已创建", index);
+        }
     }
 
     public void putChannelIndex(String... indexes) throws IOException {

src/test/java/com/zhiwei/brandkbs2/CopyChannelFromMongo2Es.java

@@ -65,7 +65,7 @@ public class CopyChannelFromMongo2Es {
         while (mongoQueryUtil.hasNextPage()) {
             List<Channel> list = channelDao.findList(mongoQueryUtil.getNextPageQuery());
             List<CompletableFuture<Map<String, Object>>> futureList = list.stream().map(channel -> CompletableFuture.supplyAsync(() -> {
-                JSONObject markData = markDataService.getLastMarkData(channel.getProjectId(), "-1", channel.getContendId(), channel.getPlatform(), channel.getRealSource(), channel.getSource());
+                JSONObject markData = markDataService.getLastMarkData(channel.getProjectId(), channel.getContendId(), channel.getPlatform(), channel.getRealSource(), channel.getSource());
                 if (null != markData) {
                     channel.setLastArticle(markData.toJSONString());
                 }

src/test/java/com/zhiwei/brandkbs2/MarkDataServiceTest.java

 package com.zhiwei.brandkbs2;
 
 import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.brandkbs2.config.Constant;
 import com.zhiwei.brandkbs2.enmus.EmotionEnum;
 import com.zhiwei.brandkbs2.pojo.BaseMap;
 import com.zhiwei.brandkbs2.pojo.Event;
@@ -43,7 +44,7 @@ public class MarkDataServiceTest {
 
     @Test
     public void getYuqingMarkAggreeProgressTest(){
-        String yuqingMarkAggreeProgress = markDataService.getYuqingMarkAggreeProgress("2e555f743f454ff29fc35bda2668c72c");
+        String yuqingMarkAggreeProgress = markDataService.getAggreeProgress("2e555f743f454ff29fc35bda2668c72c");
         System.out.println(yuqingMarkAggreeProgress);
     }
 
@@ -90,7 +91,7 @@ public class MarkDataServiceTest {
 
     @Test
     public void getMarkPlatformProportion(){
-        List<JSONObject> markPlatformProportion = markDataService.getMarkPlatformProportion(1658073600000L, 1661937298000L, false);
+        List<JSONObject> markPlatformProportion = markDataService.getMarkPlatformProportion(Constant.DEFAULT_PROJECT_ID,1658073600000L, 1661937298000L, false);
         for (JSONObject jsonObject : markPlatformProportion) {
             System.out.println(jsonObject);
         }
@@ -107,7 +108,7 @@ public class MarkDataServiceTest {
     @Test
     public void getMarkTopTitle() throws IOException {
         List<Map.Entry<String, Integer>> markTopTitle =
-                markDataService.getMarkTopTitle(1658073600000L, 1661937298000L, null, "62beadd1bbf8eb20f96d2f2e", "136", "62e1f113b746930599b616a0", 10);
+                markDataService.getMarkTopTitle(1658073600000L, 1661937298000L, "62beadd1bbf8eb20f96d2f2e", "136", "62e1f113b746930599b616a0", 10);
         for (Map.Entry<String, Integer> stringIntegerEntry : markTopTitle) {
             System.out.println(stringIntegerEntry);
         }
@@ -116,7 +117,7 @@ public class MarkDataServiceTest {
     @Test
     public void getFirstArticleTest() throws IOException {
         BaseMap article = markDataService.getFirstArticle(1658073600000L, 1661727700000L, "飞鹤杯小小朗读者活动北上广津等八位公益推广大使来啦",
-                "62beadd1bbf8eb20f96d2f2e", "136", "62e1f113b746930599b616a0");
+                "62beadd1bbf8eb20f96d2f2e", "136");
         System.out.println(article.getContent());
     }
 
@@ -136,7 +137,7 @@ public class MarkDataServiceTest {
     @Test
     public void searchMarkDataByTimeTest(){
         List<JSONObject> jsonObjects =
-                markDataService.searchMarkDataByTime("62beadd1bbf8eb20f96d2f2d", "23", "62e1f113b746930599b6167e", 1658073600000L, 1661727700000L);
+                markDataService.searchMarkDataByTime("62beadd1bbf8eb20f96d2f2d","62e1f113b746930599b6167e", 1658073600000L, 1661727700000L);
         for (JSONObject jsonObject : jsonObjects) {
             System.out.println(jsonObject);
         }
@@ -206,12 +207,12 @@ public class MarkDataServiceTest {
 
     @Test
     public void getEsTopSourceTest() throws IOException {
-        List<Map<String, Object>> esTopSource = markDataService.getEsTopSource(1663516800000L, 1666108800000L, "62beadd1bbf8eb20f96d2f1a", "25", "0",
+        List<Map<String, Object>> esTopSource = markDataService.getEsTopSource(1663516800000L, 1666108800000L, "62beadd1bbf8eb20f96d2f1a", "0",
                 EmotionEnum.POSITIVE.getName(), 3);
         for (Map<String, Object> stringObjectMap : esTopSource) {
             System.out.println(stringObjectMap.entrySet());
         }
-        List<Map<String, Object>> esTopSource2 = markDataService.getEsTopSource(1663516800000L, 1666108800000L, "62beadd1bbf8eb20f96d2f1a", "25", "0",
+        List<Map<String, Object>> esTopSource2 = markDataService.getEsTopSource(1663516800000L, 1666108800000L, "62beadd1bbf8eb20f96d2f1a", "0",
                 EmotionEnum.NEGATIVE.getName(), 3);
         for (Map<String, Object> stringObjectMap : esTopSource2) {
             System.out.println(stringObjectMap.entrySet());

src/test/java/com/zhiwei/brandkbs2/Test.java

 package com.zhiwei.brandkbs2;
 
-import com.alibaba.fastjson.JSONObject;
+import com.alibaba.fastjson.JSON;
 import com.hankcs.hanlp.dictionary.CustomDictionary;
 import com.hankcs.hanlp.dictionary.stopword.CoreStopWordDictionary;
-import com.zhiwei.brandkbs2.util.Tools;
+import com.zhiwei.brandkbs2.pojo.dto.ExportAppYuqingDTO;
+import org.apache.commons.lang3.StringUtils;
 
-import java.io.UnsupportedEncodingException;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Objects;
 
 /**
  * @ClassName: Test
@@ -15,44 +20,76 @@ import java.io.UnsupportedEncodingException;
  */
 public class Test {
 
+    public static final String PROJECT_WARN_NEW_CASE = "BRANDKBS:NEW_CASE:";
+    private static final String SEPARATOR = ":";
+
     static{
         CustomDictionary.add("猪肉供应");
         CoreStopWordDictionary.add("保障");
     }
 
-    public static void main(String[] args) throws UnsupportedEncodingException {
-
-
-        System.out.println(JSONObject.toJSONString(Tools.parseToDays(1658512800000L,1659031200000L)));
-
-//        List<Term> results= HanLP.segment("国家发改委：猪肉供应有保障 猪肉价格不具备大幅上涨基础");
-//        for(Term term: CoreStopWordDictionary.apply(results)){
-////            if(CoreStopWordDictionary.shouldInclude(term)){
-////                System.out.println(term);
-////            }else{
-//                System.err.println(term);
-////            }
-//
-//        }
-
-//        String token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJwYXlsb2FkIjoie1widXNlcklkXCI6XCIyMFwiLFwia2V5XCI6XCIyMFwiLFwibmlja05hbWVcIjpcIuayiOWQm-adsFwiLFwic2VydmljZVwiOlwiXCJ9IiwiZXhwIjoxNjUzMDExNjcwLCJpYXQiOjE2NTI0MDY4MTB9.jcVXxeZkayc6-Aiq8cyYc1uyq4ugji6FdWQXCCp4M2o";
-//        String token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJwYXlsb2FkIjoiXCLlk4Hop4FcIiIsImV4cCI6NDc2MjgyMjEzMiwiaWF0IjoxNjUyNDIyMDcyfQ.DXQ8yKgfsCMjhT0xniZeWCMv4syqIoDvztU4QWsd-Fg";
-//        String t = JwtUtil.unsign(token, String.class);
-//        System.out.println(JSONObject.toJSONString(t));
-
-//          String test = URLEncoder.encode("测试", "utf-8");
-//        System.out.println(test);
-//        String projectId = "1";
-//        List<UserRole> list = new ArrayList<>();
-//        UserDTO userDto = new UserDTO();
-//        userDto.setProjectId(projectId);
-//        userDto.setRoleId(1);
-//        list.add(UserRole.createFromUserDto(userDto));
-//
-//        list.stream().filter(userRole -> userRole.getProjectId().equals(projectId)).findAny().ifPresent(role -> {
-//                role.setRoleId(2);
-//        });
-//        System.out.println(JSONObject.toJSONString(list));
+    public static String newCrisisCaseListKey() {
+       return generateRedisKey(PROJECT_WARN_NEW_CASE, "LIST");
+    }
+
+    private static String generateRedisKey(String... keys) {
+        Objects.requireNonNull(keys);
+        boolean contains = keys[0].endsWith(":");
+        StringBuilder sb = new StringBuilder(keys[0]);
+        for (int i = 1; i < keys.length; i++) {
+            if (1 == i && contains) {
+                sb.append(keys[i]);
+            } else {
+                sb.append(SEPARATOR).append(keys[i]);
+            }
+        }
+        return sb.toString();
+    }
+
+    public static void main(String[] args) {
+        List<ExportAppYuqingDTO> list = new ArrayList<>();
+        for (int i = 0; i < 2; i++) {
+            ExportAppYuqingDTO dto = new ExportAppYuqingDTO();
+            dto.setTitle("真的是离大谱啊真的是离大谱啊真的是离大谱啊真的是离大谱啊真的是离大谱啊真的是离大谱啊真的是离大谱啊真的是离大谱啊真的是离大谱啊");
+            dto.setSource("JJ" + i);
+            dto.setTime(new Date(System.currentTimeMillis()));
+            list.add(dto);
+        }
+//        beforeTest(ExportAppYuqingDTO.class);
+        test(ExportAppYuqingDTO.class, list);
+        for (ExportAppYuqingDTO dto : list) {
+            System.out.println(JSON.toJSONString(dto));
+        }
+    }
+
+    private static <T> void beforeTest(Class<T> clazz){
+        for (Field field : clazz.getDeclaredFields()) {
+            // String格式截取前32767位
+            if (field.getGenericType().toString().equals(String.class.toString())) {
+                field.setAccessible(true);
+            }
+        }
+    }
+
+    private static <T> void test(Class<T> clazz, List<T> datas) {
+        List<Field> stringFields = new ArrayList<>();
+        // 记录需要设置的部分
+        for (Field field : clazz.getDeclaredFields()) {
+            // String格式截取前32767位
+            if (field.getGenericType().toString().equals(String.class.toString())) {
+                field.setAccessible(true);
+                stringFields.add(field);
+            }
+        }
+        datas.forEach(data -> {
+            for (Field field : stringFields) {
+                try {
+                    field.set(data, StringUtils.substring((String) (field.get(data)), 0, 10));
+                } catch (IllegalAccessException e) {
+                    e.printStackTrace();
+                }
+            }
+        });
     }
 
 }