舆情库聚合接口完成

src/main/java/com/zhiwei/brandkbs2/common/GenericAttribute.java

@@ -29,6 +29,13 @@ public class GenericAttribute {
      */
     public static final String ES_FOLLOWERS_NUM = "followers_num";
     /**
+     * es media_type
+     */
+    public static final String ES_MEDIA_TYPE = "media_type";
+    // ss platform_id
+    // TODO 待修改为 platform_id
+    public static final String ES_PLATFORM_ID= "platformId";
+    /**
      * es rootSource
      **/
     public static final String ES_ROOT_SOURCE = "root_source";

src/main/java/com/zhiwei/brandkbs2/common/RedisKeyPrefix.java

@@ -25,6 +25,7 @@ public class RedisKeyPrefix {
     public static final String MARK_HIGH_WORD = "BRANDKBS:MARK:HIGH_WORD:";
 
     public static final String REDIS_SYSTEM_COPY = "BRANDKBS:SYSTEM:COPY:";
+    public static final String AGGREE_RESULT_CACHE = "BRANDKBS:AGGREE_CACHE:";
 
     /**
      * 项目简报报相关缓存KEY

src/main/java/com/zhiwei/brandkbs2/controller/admin/EventController.java

@@ -47,7 +47,7 @@ import java.util.Set;
 public class EventController extends BaseController {
 
     @Resource(name = "eventServiceImpl")
-    private EventService EventService;
+    private EventService eventService;
 
     @Resource(name = "behaviorServiceImpl")
     private BehaviorService behaviorService;
@@ -64,13 +64,13 @@ public class EventController extends BaseController {
     @ApiImplicitParam(name = "linkedGroupId", value = "关联项目组id", paramType = "query", dataType = "string")
     @GetMapping("/yq/tag")
     public ResponseResult searchCriteria(@RequestParam(value = "linkedGroupId") String linkedGroupId) {
-        return ResponseResult.success(EventService.findEventTagListAll(linkedGroupId));
+        return ResponseResult.success(eventService.findEventTagListAll(linkedGroupId));
     }
 
     @ApiOperation("分页查询舆情事件列表")
     @PostMapping("/yq/list")
     public ResponseResult findYqEventList(@RequestBody YqEventSearchVO yqEventSearchVO) {
-        PageVO<JSONObject> yqEventList = EventService.findYqEventList(yqEventSearchVO);
+        PageVO<JSONObject> yqEventList = eventService.findYqEventList(yqEventSearchVO);
         return ResponseResult.success(yqEventList);
     }
 
@@ -87,27 +87,27 @@ public class EventController extends BaseController {
                                         @RequestParam(value = "linkedGroupId") String linkedGroupId,
                                         @RequestParam(value = "keyword", defaultValue = "") String keyword,
                                         @RequestParam(value = "sorter", defaultValue = "{\"startTime\":\"descend\"}") String sorter) {
-        return ResponseResult.success(EventService.findEventList(page, size, linkedGroupId, keyword, sorter));
+        return ResponseResult.success(eventService.findEventList(page, size, linkedGroupId, keyword, sorter));
     }
 
     @ApiOperation("获取单个事件信息")
     @ApiImplicitParam(name = "eventId", value = "事件ID", required = true, paramType = "path", dataType = "string")
     @GetMapping("/get/{eventId}")
     public ResponseResult getEvent(@PathVariable("eventId") String eventId) {
-        return ResponseResult.success(EventService.getEventJSONByEventId(eventId));
+        return ResponseResult.success(eventService.getEventJSONByEventId(eventId));
     }
 
     @ApiOperation("获取事件数据列表")
     @ApiImplicitParams({@ApiImplicitParam(name = "page", value = "页码", required = false, defaultValue = "1", paramType = "query", dataType = "int"), @ApiImplicitParam(name = "size", value = "每页记录数", required = false, defaultValue = "10", paramType = "query", dataType = "int"), @ApiImplicitParam(name = "eventId", value = "事件ID", required = true, paramType = "query", dataType = "string")})
     @GetMapping("/data/get")
     public ResponseResult getEventDatas(@RequestParam(value = "page", defaultValue = "1") int page, @RequestParam(value = "size", defaultValue = "10") int size, @RequestParam(value = "eventId") String eventId) {
-        return ResponseResult.success(EventService.getEventDatas(page, size, eventId));
+        return ResponseResult.success(eventService.getEventDatas(page, size, eventId));
     }
 
     @ApiOperation("修改事件信息")
     @PutMapping("/update")
     public ResponseResult updateEvent(@RequestBody EventVO eventVO) {
-        EventService.updateEvent(eventVO);
+        eventService.updateEvent(eventVO);
         behaviorService.pushBehavior(OPERATION, "修改事件信息:" + eventVO.getId(), request);
         return ResponseResult.success();
     }
@@ -115,7 +115,7 @@ public class EventController extends BaseController {
     @ApiOperation("修改事件首发信息")
     @PutMapping("/updateFirst")
     public ResponseResult updateEventFirst(@RequestBody EventDataDTO eventDataDTO) {
-        EventService.updateEventFirst(eventDataDTO);
+        eventService.updateEventFirst(eventDataDTO);
         behaviorService.pushBehavior(OPERATION, "修改事件首发信息:" + eventDataDTO.getEventId(), request);
         return ResponseResult.success();
     }
@@ -124,7 +124,7 @@ public class EventController extends BaseController {
     @ApiImplicitParam(name = "eventIds", value = "事件ID集合", paramType = "body", dataType = "list")
     @PostMapping("/analyze")
     public ResponseResult analysisEvents(@RequestBody JSONObject info) {
-        EventService.analysisEvents(info.getJSONArray("eventIds").toJavaList(String.class));
+        eventService.analysisEvents(info.getJSONArray("eventIds").toJavaList(String.class));
         behaviorService.pushBehavior(OPERATION, "批量更新事件", request);
         return ResponseResult.success();
     }
@@ -133,7 +133,7 @@ public class EventController extends BaseController {
     @ApiImplicitParam(name = "eventId", value = "事件ID", required = true, paramType = "path", dataType = "string")
     @PutMapping("/end/{eventId}")
     public ResponseResult endEvent(@PathVariable("eventId") String eventId) {
-        EventService.endEvent(eventId);
+        eventService.endEvent(eventId);
         behaviorService.pushBehavior(OPERATION, "结束事件:" + eventId, request);
         return ResponseResult.success();
     }
@@ -142,7 +142,7 @@ public class EventController extends BaseController {
     @ApiImplicitParam(name = "eventId", value = "事件ID", required = true, paramType = "path", dataType = "string")
     @DeleteMapping("/delete/{eventId}")
     public ResponseResult deleteEvent(@PathVariable("eventId") String eventId) {
-        String title = EventService.deleteEvent(eventId);
+        String title = eventService.deleteEvent(eventId);
         behaviorService.pushBehavior(OPERATION, "删除事件:" + title, request);
         return ResponseResult.success();
     }
@@ -153,7 +153,7 @@ public class EventController extends BaseController {
     public ResponseResult deleteEventData(@RequestBody JSONObject info) {
         String id = info.getString("id");
         String eventId = info.getString("eventId");
-        String title = EventService.deleteEventData(id, eventId);
+        String title = eventService.deleteEventData(id, eventId);
         behaviorService.pushBehavior(OPERATION, "删除事件单条数据,eventId:" + eventId + ",数据id:" + id + ",事件标题:" + title, request);
         return ResponseResult.success();
     }
@@ -172,14 +172,14 @@ public class EventController extends BaseController {
     @ApiOperation("事件上传进度获取")
     @GetMapping("data/upload/progress/{ticket}")
     public ResponseResult getEventDataUploadProgress(@PathVariable String ticket) {
-        return ResponseResult.success(EventService.getEventDataUploadProgress(ticket));
+        return ResponseResult.success(eventService.getEventDataUploadProgress(ticket));
     }
 
     @ApiOperation("事件数据上传进度获取")
     @ApiImplicitParams(@ApiImplicitParam(name = "eventIds", value = "事件id列表", paramType = "body", dataType = "list"))
     @PostMapping("analyze/progress")
     public ResponseResult getEventAnalyzeProgress(@RequestBody JSONObject info) {
-        return ResponseResult.success(EventService.getEventAnalyzeProgress(info.getJSONArray("eventIds").toJavaList(String.class)));
+        return ResponseResult.success(eventService.getEventAnalyzeProgress(info.getJSONArray("eventIds").toJavaList(String.class)));
     }
 
     @ApiOperation("事件上传模板下载")
@@ -202,7 +202,7 @@ public class EventController extends BaseController {
     @ApiImplicitParam(name = "linkedGroupId", value = "关联项目组id", required = true, paramType = "query", dataType = "string")
     @GetMapping("/download")
     public ResponseResult downloadEvents(@RequestParam("linkedGroupId") String linkedGroupId) {
-        Pair<String, List<ExportEventDTO>> result = EventService.downloadEvents(linkedGroupId);
+        Pair<String, List<ExportEventDTO>> result = eventService.downloadEvents(linkedGroupId);
         String fileName = result.getLeft();
         EasyExcelUtil.download(fileName + "_事件列表数据", fileName, ExportEventDTO.class, result.getRight(), response);
         return ResponseResult.success();
@@ -212,7 +212,7 @@ public class EventController extends BaseController {
     @ApiImplicitParam(name = "eventId", value = "事件ID", required = true, paramType = "query", dataType = "string")
     @GetMapping("/data/download")
     public ResponseResult downloadEventDatas(@RequestParam("eventId") String eventId) {
-        Pair<String, List<ExportEventDataDTO>> result = EventService.downloadEventDatas(eventId);
+        Pair<String, List<ExportEventDataDTO>> result = eventService.downloadEventDatas(eventId);
         String title = result.getLeft();
         EasyExcelUtil.download(title, title, ExportEventDataDTO.class, result.getRight(), response);
         return ResponseResult.success();
@@ -224,7 +224,7 @@ public class EventController extends BaseController {
     public ResponseResult addEventsByYq(@RequestBody JSONObject info) {
         String linkedGroupId = info.getString("linkedGroupId");
         List<YqEventDTO> yqEventList = info.getJSONArray("list").toJavaList(YqEventDTO.class);
-        EventService.addYqEvents(linkedGroupId, yqEventList);
+        eventService.addYqEvents(linkedGroupId, yqEventList);
         behaviorService.pushBehavior(OPERATION, "批量导入舆情事件数据", request);
         return ResponseResult.success();
     }
@@ -232,7 +232,7 @@ public class EventController extends BaseController {
     @ApiOperation("按搜索条件全部导入舆情事件数据")
     @PutMapping("/upload/yqAll")
     public ResponseResult addEventAllByYq(@RequestBody YqEventSearchVO yqEventSearchVO) {
-        EventService.addYqEventAll(yqEventSearchVO);
+        eventService.addYqEventAll(yqEventSearchVO);
         behaviorService.pushBehavior(OPERATION, "全部导入舆情事件数据", request);
         return ResponseResult.success();
     }
@@ -242,7 +242,7 @@ public class EventController extends BaseController {
     @PostMapping(value = "/upload/file", headers = "content-type=multipart/form-data")
     @Auth(role = RoleEnum.SUPER_ADMIN)
     public ResponseResult addEventsByFile(@RequestParam(value = "linkedGroupId") String linkedGroupId, @RequestParam("fileUrl") String fileUrl) {
-        EventService.addFileEvents(linkedGroupId, fileUrl);
+        eventService.addFileEvents(linkedGroupId, fileUrl);
         behaviorService.pushBehavior(OPERATION, "文件上传事件", request);
         return ResponseResult.success();
     }
@@ -252,7 +252,7 @@ public class EventController extends BaseController {
             "linkedGroupId", value = "关联项目id", paramType = "form", dataType = "string")})
     @PostMapping(value = "/data/upload", headers = "content-type=multipart/form-data")
     public ResponseResult uploadEventDatas(@RequestParam("linkedGroupId") String linkedGroupId, @RequestParam("file") MultipartFile file) {
-        EventService.uploadEventDatas(linkedGroupId, file);
+        eventService.uploadEventDatas(linkedGroupId, file);
         behaviorService.pushBehavior(OPERATION, "事件数据上传", request);
         return ResponseResult.success();
     }
@@ -261,13 +261,13 @@ public class EventController extends BaseController {
     @ApiImplicitParams({@ApiImplicitParam(name = "page", value = "页码", required = false, defaultValue = "1", paramType = "query", dataType = "int"), @ApiImplicitParam(name = "size", value = "每页记录数", required = false, defaultValue = "10", paramType = "query", dataType = "int"), @ApiImplicitParam(name = "linkedGroupId", value = "关联性项目组id", required = true, paramType = "query", dataType = "string"), @ApiImplicitParam(name = "tagGroupName", value = "标签组名", required = true, paramType = "query", dataType = "string")})
     @GetMapping("/tag/list")
     public ResponseResult findEventTagList(@RequestParam(value = "page", defaultValue = "1") int page, @RequestParam(value = "size", defaultValue = "10") int size, @RequestParam(value = "linkedGroupId") String linkedGroupId, @RequestParam(value = "tagGroupName") String tagGroupName) {
-        return ResponseResult.success(EventService.findEventTagList(page, size, linkedGroupId, tagGroupName));
+        return ResponseResult.success(eventService.findEventTagList(page, size, linkedGroupId, tagGroupName));
     }
 
     @ApiOperation("查询所有事件标签组名")
     @GetMapping("/tag/groupName")
     public ResponseResult findEventTagGroupName(@RequestParam(value = "linkedGroupId") String linkedGroupId) {
-        return ResponseResult.success(EventService.findEventTagGroupName(linkedGroupId));
+        return ResponseResult.success(eventService.findEventTagGroupName(linkedGroupId));
     }
 
 }

src/main/java/com/zhiwei/brandkbs2/controller/app/AppArticleController.java

@@ -2,8 +2,10 @@ package com.zhiwei.brandkbs2.controller.app;
 
 import com.zhiwei.brandkbs2.auth.Auth;
 import com.zhiwei.brandkbs2.controller.BaseController;
+import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
 import com.zhiwei.brandkbs2.enmus.RoleEnum;
 import com.zhiwei.brandkbs2.model.ResponseResult;
+import com.zhiwei.brandkbs2.pojo.dto.ExportAppYuqingDTO;
 import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
 import com.zhiwei.brandkbs2.pojo.dto.ReportDTO;
 import com.zhiwei.brandkbs2.pojo.dto.ReportSearchDTO;
@@ -14,9 +16,11 @@ import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiImplicitParam;
 import io.swagger.annotations.ApiImplicitParams;
 import io.swagger.annotations.ApiOperation;
+import org.apache.commons.lang3.tuple.Pair;
 import org.springframework.web.bind.annotation.*;
 
 import javax.annotation.Resource;
+import java.util.List;
 
 /**
  * @author sjj
@@ -46,17 +50,38 @@ public class AppArticleController extends BaseController {
     }
 
     @ApiOperation("舆情列表-生成聚合")
-    @PostMapping("/mark/agree")
+    @PostMapping("/mark/aggree")
     public ResponseResult generateYuqingMarkAggreeList(@RequestBody MarkSearchDTO markSearchDTO) {
         return ResponseResult.success(markDataService.generateYuqingMarkAggreeList(markSearchDTO));
     }
 
+    @ApiOperation("舆情列表-聚合进度查询")
+    @GetMapping("/mark/aggree/progress/{aggreeId}")
+    public ResponseResult getYuqingMarkAggreeProgress(@PathVariable String aggreeId) {
+        return ResponseResult.success(markDataService.getYuqingMarkAggreeProgress(aggreeId));
+    }
+
+    @ApiOperation("舆情列表-获取聚合结果")
+    @PostMapping("/mark/aggree/list")
+    public ResponseResult getYuqingMarkAggreeList(@RequestBody MarkSearchDTO markSearchDTO) {
+        return ResponseResult.success(markDataService.getYuqingMarkAggreeList(markSearchDTO));
+    }
+
     @ApiOperation("舆情列表-搜索条件")
     @GetMapping("/mark/list/criteria")
     public ResponseResult getYuqingMark(@RequestParam(required = false) String linkedGroupId) {
         return ResponseResult.success(markDataService.getYuqingMarkCriteria(linkedGroupId));
     }
 
+    @ApiOperation("舆情导出")
+    @PostMapping(value = "mark/list/export")
+    public ResponseResult exportYuqingMarkList(@RequestBody MarkSearchDTO markSearchDTO) {
+        Pair<String, List<ExportAppYuqingDTO>> stringListPair = markDataService.downloadYuqingMarkList(markSearchDTO);
+        EasyExcelUtil.download(stringListPair.getLeft() + "_舆情列表数据", "sheet1", ExportAppYuqingDTO.class, stringListPair.getRight(), response);
+        return ResponseResult.success();
+
+    }
+
     @ApiOperation("舆情分析-舆情概览")
     @ApiImplicitParams({@ApiImplicitParam(name = "startTime", value = "开始时间", paramType = "query", dataType = "long"), @ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long")})
     @GetMapping("/analyze/summary")

src/main/java/com/zhiwei/brandkbs2/controller/app/GlobalController.java

@@ -19,7 +19,7 @@ import javax.annotation.Resource;
  */
 @RestController
 @RequestMapping("/app/global")
-@Api(tags = "全局通用接口")
+@Api(tags = "全局通用接口", description = "提供全局通用相关")
 @Auth(role = RoleEnum.CUSTOMER)
 public class GlobalController extends BaseController {
 

src/main/java/com/zhiwei/brandkbs2/dao/AggreeResultDao.java

+package com.zhiwei.brandkbs2.dao;
+
+import com.zhiwei.brandkbs2.pojo.AggreeResult;
+
+/**
+ * @ClassName: AggreeResultDao
+ * @Description AggreeResultDao
+ * @author: sjj
+ * @date: 2022-07-18 09:47
+ */
+public interface AggreeResultDao extends BaseMongoDao<AggreeResult>{
+}

src/main/java/com/zhiwei/brandkbs2/dao/impl/AggreeResultDaoImpl.java

+package com.zhiwei.brandkbs2.dao.impl;
+
+import com.zhiwei.brandkbs2.dao.AggreeResultDao;
+import com.zhiwei.brandkbs2.pojo.AggreeResult;
+import org.springframework.stereotype.Service;
+
+/**
+ * @ClassName: AggreeResultDaoImpl
+ * @Description AggreeResultDaoImpl
+ * @author: sjj
+ * @date: 2022-07-18 09:50
+ */
+@Service("aggreeResultDaoImpl")
+public class AggreeResultDaoImpl extends BaseMongoDaoImpl<AggreeResult> implements AggreeResultDao {
+    private static final String COLLECTION_PREFIX = "brandkbs_aggree_result";
+    
+    public AggreeResultDaoImpl() {
+        super(COLLECTION_PREFIX);
+    }
+}

src/main/java/com/zhiwei/brandkbs2/dao/impl/UserProjectOldDaoImpl.java

@@ -6,8 +6,6 @@ import lombok.Setter;
 import org.springframework.data.mongodb.core.mapping.Document;
 import org.springframework.stereotype.Component;
 
-import java.util.Date;
-
 /**
  * @ClassName: UserProjectOldImpl
  * @Description 用户项目关联旧表
@@ -28,7 +26,7 @@ public class UserProjectOldDaoImpl extends BaseMongoDaoImpl<UserProjectOldDaoImp
     public static class UserProjectOld extends AbstractBaseMongo{
         String userId;
         Integer roleId;
-        Date expiredTime;
+        Long expiredTime;
         Integer exportAmount;
         String projectName;
     }

src/main/java/com/zhiwei/brandkbs2/easyexcel/listener/EventFileListener.java

@@ -65,6 +65,6 @@ public class EventFileListener extends AnalysisEventListener<UploadEventDTO> {
      * 保存更新事件信息逻辑
      */
     private void addFileEvent() {
-        datas.forEach(eventDTO -> EventService.addFileEvent(projectId, linkedGroupId, eventDTO));
+        datas.forEach(eventDTO -> eventService.addFileEvent(projectId, linkedGroupId, eventDTO));
     }
 }

src/main/java/com/zhiwei/brandkbs2/pojo/AggreeResult.java

+package com.zhiwei.brandkbs2.pojo;
+
+import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.brandkbs2.common.GenericAttribute;
+import com.zhiwei.brandkbs2.util.Tools;
+import lombok.Getter;
+import lombok.Setter;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * @ClassName: AggreeResult
+ * @Description 聚合实体
+ * @author: sjj
+ * @date: 2022-07-18 09:41
+ */
+@Setter
+@Getter
+public class AggreeResult extends AbstractBaseMongo {
+
+    /**
+     * 任务id
+     */
+    private String taskId;
+
+    /**
+     * 创建时间
+     */
+    private Long cTime;
+
+    /**
+     * 原数据
+     */
+    private JSONObject data;
+
+    /**
+     * 聚合标题
+     */
+    private String aggreeTitle;
+
+    /**
+     * 发文时间
+     */
+    private Long time;
+
+    /**
+     * 平台id
+     */
+    private String platformId;
+
+    /**
+     * 重要发声方
+     */
+    private List<String> mediaTypes;
+
+    /**
+     * 情感倾向(舆情调性)
+     */
+    private List<String> tags;
+
+    /**
+     * 聚合量
+     */
+    private Integer aggreeSize;
+
+    /**
+     * 父聚合id
+     */
+    private String fatherId;
+
+    /**
+     * 是否是父聚合
+     */
+    private boolean isFather;
+
+    public static AggreeResult createFather(JSONObject json, String taskId, Integer aggreeSize) {
+        AggreeResult aggreeResult = getInstance(json, taskId);
+        aggreeResult.setFather(true);
+        aggreeResult.setAggreeSize(aggreeSize);
+        aggreeResult.setCTime(System.currentTimeMillis());
+        return aggreeResult;
+    }
+
+    public static AggreeResult createSon(JSONObject json, String taskId, String fatherId, String fatherTitle) {
+        AggreeResult aggreeResult = getInstance(json, taskId);
+        aggreeResult.setFatherId(fatherId);
+        // 使用父聚合标题
+        aggreeResult.setAggreeTitle(fatherTitle);
+        aggreeResult.setCTime(System.currentTimeMillis());
+        return aggreeResult;
+    }
+
+    private static AggreeResult getInstance(JSONObject json, String taskId) {
+        AggreeResult aggreeResult = new AggreeResult();
+        aggreeResult.setTaskId(taskId);
+        aggreeResult.setData(json);
+        BaseMap baseMap = Tools.getBaseFromEsMap(json);
+        aggreeResult.setTime(baseMap.getTime());
+        aggreeResult.setPlatformId(json.getString(GenericAttribute.ES_PLATFORM_ID));
+        String mediaType = json.getString(GenericAttribute.ES_MEDIA_TYPE);
+        if (StringUtils.isNotEmpty(mediaType)) {
+            aggreeResult.setMediaTypes(Arrays.asList(mediaType.split(",")));
+        }
+        aggreeResult.setTags(parseMatg(json.getString(GenericAttribute.ES_MTAG)));
+        aggreeResult.setAggreeTitle(baseMap.getTitleNullOptionalContent());
+        return aggreeResult;
+    }
+
+    private static List<String> parseMatg(String mtag) {
+        if (StringUtils.isEmpty(mtag)) {
+            return null;
+        }
+        List<String> resList = new ArrayList<>();
+        String[] uniques = mtag.substring(1).split(",");
+        for (String unique : uniques) {
+            resList.add("," + unique);
+        }
+        return resList;
+    }
+
+}

src/main/java/com/zhiwei/brandkbs2/pojo/MarkFlowEntity.java

@@ -15,6 +15,7 @@ import lombok.Setter;
 import org.apache.commons.lang3.ObjectUtils;
 import org.elasticsearch.search.SearchHit;
 
+import java.io.Serializable;
 import java.util.List;
 import java.util.Objects;
 
@@ -28,7 +29,7 @@ import java.util.Objects;
 @Getter
 @Setter
 @NoArgsConstructor
-public class MarkFlowEntity {
+public class MarkFlowEntity implements Serializable {
 
     /**
      * 消息类型
@@ -47,16 +48,29 @@ public class MarkFlowEntity {
 
     public static MarkFlowEntity getFoldInstance(List<MarkFlowEntity> entities) {
         // 首条数据为主
-        MarkFlowEntity entity = entities.get(0);
+        MarkFlowEntity entity = Tools.copyByJson(entities.get(0), MarkFlowEntity.class);
         if (entities.size() > 1) {
             entity.getType().put("fold", true);
             entity.getType().put("size", entities.size());
             // info内容
+            if (null == entity.getInfo()) {
+                entity.setInfo(new JSONObject());
+            }
             entity.getInfo().put("foldInfo", entities);
         }
         return entity;
     }
 
+    public static MarkFlowEntity getAggreeInstance(List<MarkFlowEntity> entities,int size) {
+        // 首条数据为主
+        MarkFlowEntity entity = Tools.copyByJson(entities.get(0), MarkFlowEntity.class);
+        if (size > 1) {
+            entity.getType().put("fold", true);
+            entity.getType().put("size",size);
+        }
+        return entity;
+    }
+
     /**
      * Constructor
      *

src/main/java/com/zhiwei/brandkbs2/pojo/UserRole.java

@@ -3,6 +3,7 @@ package com.zhiwei.brandkbs2.pojo;
 import com.zhiwei.brandkbs2.enmus.RoleEnum;
 import com.zhiwei.brandkbs2.pojo.dto.UserDTO;
 import lombok.Getter;
+import lombok.NoArgsConstructor;
 import lombok.Setter;
 import org.apache.commons.lang3.time.DateUtils;
 
@@ -17,6 +18,7 @@ import java.util.Date;
  */
 @Setter
 @Getter
+@NoArgsConstructor
 public class UserRole {
 
     // 默认绑定阿里项目
@@ -51,10 +53,10 @@ public class UserRole {
      */
     private Integer exportAmount;
 
-    public UserRole(String projectId, int roleId, Date expiredTime, Integer exportAmount) {
+    public UserRole(String projectId, int roleId, Long expiredTime, Integer exportAmount) {
         this.projectId = projectId;
         this.roleId = roleId;
-        this.expiredTime = expiredTime.getTime();
+        this.expiredTime = expiredTime;
         this.exportAmount = exportAmount;
     }
 
@@ -63,7 +65,7 @@ public class UserRole {
         if (null != userDTO.getExportAmount() && userDTO.getExportAmount() < MAX_EXPORT_LIMIT) {
             exportAmount = userDTO.getExportAmount();
         }
-        Date expiredTime = null;
+        Long expiredTime = null;
         //当创建用户为客户时，添加过期时间
         if (userDTO.getRoleId() == RoleEnum.CUSTOMER.getState()) {
             Date date = new Date(userDTO.getExpiredTime());
@@ -72,13 +74,13 @@ public class UserRole {
             calendar.set(Calendar.HOUR_OF_DAY, 23);
             calendar.set(Calendar.MINUTE, 59);
             calendar.set(Calendar.SECOND, 59);
-            expiredTime = calendar.getTime();
+            expiredTime = calendar.getTime().getTime();
         }
         return new UserRole(userDTO.getProjectId(), userDTO.getRoleId(), expiredTime, exportAmount);
     }
 
     public static UserRole defaultUserRole(Date now) {
-        return new UserRole(DEFAULT_PROJECT_ID, DEFAULT_ROLE_ID, DateUtils.addDays(now, 30), DEFAULT_EXPORT_LIMIT);
+        return new UserRole(DEFAULT_PROJECT_ID, DEFAULT_ROLE_ID, DateUtils.addDays(now, 30).getTime(), DEFAULT_EXPORT_LIMIT);
     }
 
 }

src/main/java/com/zhiwei/brandkbs2/pojo/dto/ExportAppYuqingDTO.java

+package com.zhiwei.brandkbs2.pojo.dto;
+
+import com.alibaba.excel.annotation.ExcelProperty;
+import com.alibaba.excel.annotation.write.style.ColumnWidth;
+import com.zhiwei.brandkbs2.pojo.BaseMap;
+import com.zhiwei.brandkbs2.util.Tools;
+import lombok.Data;
+import lombok.ToString;
+
+import java.util.Date;
+import java.util.Map;
+
+/**
+ * 舆情导出
+ */
+@Data
+@ToString
+public class ExportAppYuqingDTO {
+
+    @ColumnWidth(20)
+    @ExcelProperty("时间")
+    private Date time;
+
+    @ColumnWidth(20)
+    @ExcelProperty("平台")
+    private String platform;
+
+    @ColumnWidth(20)
+    @ExcelProperty("渠道")
+    private String source;
+
+    @ColumnWidth(50)
+    @ExcelProperty("标题")
+    private String title;
+
+    @ColumnWidth(50)
+    @ExcelProperty("文本")
+    private String content;
+
+    @ColumnWidth(50)
+    @ExcelProperty("地址")
+    private String url;
+
+    @ColumnWidth(25)
+    @ExcelProperty("情感调性")
+    private String emotion;
+
+    public static ExportAppYuqingDTO createFromEsMap(Map<String, Object> map) {
+        ExportAppYuqingDTO dto = new ExportAppYuqingDTO();
+        BaseMap baseFromEsMap = Tools.getBaseFromEsMap(map);
+        dto.setTime(new Date(baseFromEsMap.getTime()));
+        dto.setPlatform(baseFromEsMap.getPlatform());
+        dto.setSource(baseFromEsMap.getSource());
+        dto.setTitle(baseFromEsMap.getTitle());
+        dto.setContent(baseFromEsMap.getContent());
+        dto.setUrl(baseFromEsMap.getUrl());
+        dto.setEmotion(baseFromEsMap.getEmotion());
+        return dto;
+    }
+
+}

src/main/java/com/zhiwei/brandkbs2/pojo/dto/MarkSearchDTO.java

@@ -12,6 +12,12 @@ import java.util.List;
 @ToString
 @ApiModel("标注数据搜索传输类")
 public class MarkSearchDTO {
+
+    /**
+     * 聚合id
+     */
+    @ApiModelProperty("聚合id")
+    private String aggreeId;
     /**
      * 页码
      */
@@ -25,6 +31,8 @@ public class MarkSearchDTO {
     private int pageSize;
 
     // postFilter
+    @ApiModelProperty(value = "项目id")
+    private String projectId;
     /**
      * 关联项目组id
      */

src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java

@@ -2,8 +2,10 @@ package com.zhiwei.brandkbs2.service;
 
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.brandkbs2.pojo.MarkFlowEntity;
+import com.zhiwei.brandkbs2.pojo.dto.ExportAppYuqingDTO;
 import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
 import com.zhiwei.brandkbs2.pojo.vo.PageVO;
+import org.apache.commons.lang3.tuple.Pair;
 
 import java.util.List;
 
@@ -22,6 +24,12 @@ public interface MarkDataService {
     PageVO<MarkFlowEntity> getYuqingMarkList(MarkSearchDTO markSearchDTO);
 
     /**
+     * @param markSearchDTO 标注数据搜索传输类
+     * @return 返回结果
+     */
+    Pair<String,List<ExportAppYuqingDTO>> downloadYuqingMarkList(MarkSearchDTO markSearchDTO);
+
+    /**
      * 生成聚合列表并返回id
      *
      * @return 聚合id
@@ -29,6 +37,21 @@ public interface MarkDataService {
     String generateYuqingMarkAggreeList(MarkSearchDTO markSearchDTO);
 
     /**
+     * 获取聚合进度结果
+     * @param id 聚合id
+     * @return 进度值
+     */
+    String getYuqingMarkAggreeProgress(String id);
+
+    /**
+     * 获取聚合结果列表
+     *
+     * @param markSearchDTO 搜索实体
+     * @return 标注消息流列表
+     */
+    PageVO<MarkFlowEntity> getYuqingMarkAggreeList(MarkSearchDTO markSearchDTO);
+
+    /**
      * 舆情标注数据搜索条件
      */
     JSONObject getYuqingMarkCriteria(String linkedGroup);

src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java

@@ -8,6 +8,7 @@ import com.zhiwei.brandkbs2.auth.UserThreadLocal;
 import com.zhiwei.brandkbs2.common.GenericAttribute;
 import com.zhiwei.brandkbs2.common.GlobalPojo;
 import com.zhiwei.brandkbs2.common.RedisKeyPrefix;
+import com.zhiwei.brandkbs2.dao.AggreeResultDao;
 import com.zhiwei.brandkbs2.enmus.EmotionEnum;
 import com.zhiwei.brandkbs2.enmus.ImportantChannelEnum;
 import com.zhiwei.brandkbs2.es.EsClientDao;
@@ -15,13 +16,14 @@ import com.zhiwei.brandkbs2.es.EsQueryTools;
 import com.zhiwei.brandkbs2.exception.ExceptionCast;
 import com.zhiwei.brandkbs2.listener.ApplicationProjectListener;
 import com.zhiwei.brandkbs2.model.CommonCodeEnum;
-import com.zhiwei.brandkbs2.pojo.BaseMap;
-import com.zhiwei.brandkbs2.pojo.MarkFlowEntity;
+import com.zhiwei.brandkbs2.pojo.*;
+import com.zhiwei.brandkbs2.pojo.dto.ExportAppYuqingDTO;
 import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
 import com.zhiwei.brandkbs2.pojo.vo.CustomTagVo;
 import com.zhiwei.brandkbs2.pojo.vo.PageVO;
 import com.zhiwei.brandkbs2.pojo.vo.ProjectVO;
 import com.zhiwei.brandkbs2.service.*;
+import com.zhiwei.brandkbs2.util.MongoUtil;
 import com.zhiwei.brandkbs2.util.RedisUtil;
 import com.zhiwei.brandkbs2.util.TextUtil;
 import com.zhiwei.brandkbs2.util.Tools;
@@ -52,6 +54,8 @@ import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilde
 import org.elasticsearch.search.sort.FieldSortBuilder;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.data.mongodb.core.query.Query;
 import org.springframework.stereotype.Service;
 
 import javax.annotation.Resource;
@@ -88,19 +92,25 @@ public class MarkDataServiceImpl implements MarkDataService {
     @Resource(name = "markFlowServiceImpl")
     MarkFlowService markFlowService;
 
+    @Resource(name = "aggreeResultDaoImpl")
+    AggreeResultDao aggreeResultDao;
+
     @Resource(name = "redisUtil")
     RedisUtil redisUtil;
 
+    @Resource(name = "mongoUtil")
+    MongoUtil mongoUtil;
+
     @Override
     public PageVO<MarkFlowEntity> getYuqingMarkList(MarkSearchDTO markSearchDTO) {
         try {
             defaultMarkSearch(markSearchDTO);
             // 搜索es数据
-            Pair<SearchHits, Map<String, Long>> hitsAndCounts = searchMarkHitsAndCount(markSearchDTO);
+            Pair<SearchHits[], Map<String, Long>> hitsAndCounts = searchMarkHitsAndCount(markSearchDTO, false);
             // 总量
-            long total = hitsAndCounts.getLeft().getTotalHits().value > 10000 ? 10000 : hitsAndCounts.getLeft().getTotalHits().value;
+            long total = hitsAndCounts.getLeft()[0].getTotalHits().value > 10000 ? 10000 : hitsAndCounts.getLeft()[0].getTotalHits().value;
             // 消息列表
-            List<MarkFlowEntity> flowEntityList = getMarkFlowEntity(markSearchDTO, hitsAndCounts.getLeft());
+            List<MarkFlowEntity> flowEntityList = getMarkFlowEntity(markSearchDTO, hitsAndCounts.getLeft()[0]);
             // 返回分页结果并设置平台count
             return PageVO.createPageVo(total, markSearchDTO.getPage(), markSearchDTO.getPageSize(), flowEntityList)
                     // 各平台计量
@@ -112,25 +122,159 @@ public class MarkDataServiceImpl implements MarkDataService {
     }
 
     @Override
+    public Pair<String, List<ExportAppYuqingDTO>> downloadYuqingMarkList(MarkSearchDTO markSearchDTO) {
+        try {
+            ProjectVO project = projectService.getProjectVOById(UserThreadLocal.getProjectId());
+            List<ExportAppYuqingDTO> returnList = new ArrayList<>();
+            Pair<SearchHits[], Map<String, Long>> hitsAndCounts = searchMarkHitsAndCount(markSearchDTO, false);
+            for (SearchHits searchHits : hitsAndCounts.getLeft()) {
+                List<ExportAppYuqingDTO> collect = Arrays.stream(searchHits.getHits()).map(SearchHit::getSourceAsMap).map(ExportAppYuqingDTO::createFromEsMap).collect(Collectors.toList());
+                returnList.addAll(collect);
+            }
+
+            return Pair.of(project.getBrandName() + "_" + markSearchDTO.getStartTime() + "_" + markSearchDTO.getEndTime(), returnList);
+        } catch (IOException e) {
+            ExceptionCast.cast(CommonCodeEnum.FAIL.message("es检索异常"));
+        }
+        return null;
+    }
+
+    @Override
     public String generateYuqingMarkAggreeList(MarkSearchDTO markSearchDTO) {
         String uuid = Tools.getUUID();
+        String projectId = UserThreadLocal.getProjectId();
+        defaultMarkSearch4Aggree(markSearchDTO);
         ApplicationProjectListener.getThreadPool().execute(() -> {
+            String redisKey = RedisUtil.getAggreeCacheKey(uuid, projectId);
             try {
-                defaultMarkSearch(markSearchDTO);
+                redisUtil.setExpire(redisKey, "0");
                 // 搜索es数据
-                Pair<SearchHits, Map<String, Long>> hitsAndCounts = searchMarkHitsAndCount(markSearchDTO);
+                Pair<SearchHits[], Map<String, Long>> hitsAndCounts = searchMarkHitsAndCount(markSearchDTO, true);
+                log.info("taskId:{},聚合前数据:{}条", uuid, hitsAndCounts.getLeft()[0].getTotalHits().value);
                 // 聚合分组后数据
                 List<List<JSONObject>> groupList = aggregation(hitsAndCounts.getLeft(), "id");
-
-
+                log.info("taskId:{},聚合完成,聚合组:{}个", uuid, groupList.size());
+                // 入mongo缓存
+                cacheAggreeResult(groupList, uuid);
+                redisUtil.setExpire(redisKey, "100");
             } catch (Exception e) {
                 log.error("generateYuqingMarkAggreeList-", e);
+                redisUtil.remove(redisKey);
             }
         });
         return uuid;
     }
 
     @Override
+    public String getYuqingMarkAggreeProgress(String id) {
+        String redisKey = RedisUtil.getAggreeCacheKey(id, UserThreadLocal.getProjectId());
+        return redisUtil.get(redisKey);
+    }
+
+    @Override
+    public PageVO<MarkFlowEntity> getYuqingMarkAggreeList(MarkSearchDTO dto) {
+        if (null == dto.getAggreeId()) {
+            ExceptionCast.cast(CommonCodeEnum.INVALID_PARAM.message("聚合id不得为空"));
+        }
+        defaultMarkSearch4Aggree(dto);
+        Query query = assembleAggreeQuery(dto);
+        // 量查询
+        long count = aggreeResultDao.count(query);
+        // 开启分页
+        mongoUtil.start(dto.getPage(), dto.getPageSize(), query);
+        // 排序
+        aggreeResultDao.addSort(query, "{\"aggreeSize\":\"descend\"}");
+        // 返回结果
+        List<MarkFlowEntity> resList = new ArrayList<>();
+        List<AggreeResult> fatherList = aggreeResultDao.findList(query);
+
+//        List<String> fatherIds = fatherList.stream().map(AggreeResult::getFatherId).collect(Collectors.toList());
+//        Map<String, List<AggreeResult>> fatherMap = new HashMap<>();
+//        for (AggreeResult aggreeResult : aggreeResultDao.findList(Query.query(Criteria.where("fatherId").in(fatherIds)))) {
+//            fatherMap.compute(aggreeResult.getFatherId(), (k, v) -> {
+//                if (null == v) {
+//                    v = new ArrayList<>();
+//                }
+//                v.add(aggreeResult);
+//                return v;
+//            });
+//        }
+
+//        Map<String, List<AggreeResult>> fatherMap = aggreeResultDao.findList(Query.query(Criteria.where("fatherId").in(fatherIds))).stream().collect(Collectors.groupingBy(AggreeResult::getFatherId));
+        for (AggreeResult aggreeResult : fatherList) {
+            List<MarkFlowEntity> markFlowEntities = new ArrayList<>();
+            MarkFlowEntity instance = new MarkFlowEntity(aggreeResult.getData());
+            instance.setInfo(markFlowService.createMarkFlowInfo(aggreeResult.getData(), dto.getProjectId(), dto.getLinkedGroupId()));
+            markFlowEntities.add(instance);
+            // 添加父标题集下的子标题集
+//            markFlowEntities.addAll(fatherMap.get(aggreeResult.getFatherId()).stream().map(sonResult -> new MarkFlowEntity(sonResult.getData())).collect(Collectors.toList()));
+            resList.add(MarkFlowEntity.getAggreeInstance(markFlowEntities, aggreeResult.getAggreeSize()));
+        }
+        return PageVO.createPageVo(count, dto.getPage(), dto.getPageSize(), resList);
+    }
+
+    private Query assembleAggreeQuery(MarkSearchDTO dto) {
+        Query query = new Query();
+        query.addCriteria(Criteria.where("taskId").is(dto.getAggreeId()));
+        // 平台
+        if (CollectionUtils.isNotEmpty(dto.getPlatforms())) {
+            Criteria orCriteria = new Criteria();
+            List<Criteria> platformCriteria = new ArrayList<>();
+            for (String platformId : dto.getPlatforms()) {
+                platformCriteria.add(Criteria.where("platformId").is(platformId));
+            }
+            query.addCriteria(orCriteria.orOperator(platformCriteria.toArray(new Criteria[0])));
+        }
+        // 媒体类型
+        if (CollectionUtils.isNotEmpty(dto.getMediaTypes())) {
+            Criteria orCriteria = new Criteria();
+            List<Criteria> mediaCriteria = new ArrayList<>();
+            for (String mediaType : dto.getMediaTypes()) {
+                mediaCriteria.add(Criteria.where("mediaTypes").is(mediaType));
+            }
+            query.addCriteria(orCriteria.orOperator(mediaCriteria.toArray(new Criteria[0])));
+        }
+        // 情感调性
+        if (CollectionUtils.isNotEmpty(dto.getTags())) {
+            Criteria orCriteria = new Criteria();
+            List<Criteria> tagCriteria = new ArrayList<>();
+            for (String tag : dto.getTags()) {
+                tagCriteria.add(Criteria.where("tags").is(tag));
+            }
+            query.addCriteria((orCriteria.orOperator(tagCriteria.toArray(new Criteria[0]))));
+        }
+        // 关键词模糊查询
+        aggreeResultDao.addKeywordFuzz(query, dto.getKeyword(), "aggreeTitle");
+        return query;
+    }
+
+    private void cacheAggreeResult(List<List<JSONObject>> groupList, String taskId) {
+        int count = 0;
+        int percent;
+        for (List<JSONObject> list : groupList) {
+            String fatherId = null;
+            String fatherTitle = null;
+            for (int i = 0; i < list.size(); i++) {
+                JSONObject json = list.get(i);
+                AggreeResult aggreeResult;
+                // 父聚合处理
+                if (i == 0) {
+                    aggreeResult = AggreeResult.createFather(json, taskId, list.size());
+                    aggreeResultDao.insertOneWithoutId(aggreeResult);
+                    fatherId = aggreeResult.getId();
+                    fatherTitle = aggreeResult.getAggreeTitle();
+                } else {
+                    aggreeResult = AggreeResult.createSon(json, taskId, fatherId, fatherTitle);
+                    aggreeResultDao.insertOneWithoutId(aggreeResult);
+                }
+            }
+            if ((percent = (++count / list.size())) % (10 / list.size()) == 0) {
+                log.info("taskId:{},fatherId:{} 入库完毕,进度:{}%", taskId, fatherId, percent);
+            }
+        }
+    }
+
+    @Override
     public JSONObject getYuqingMarkCriteria(String linkedGroupId) {
         if (null == linkedGroupId) {
             linkedGroupId = projectService.getProjectVOById(UserThreadLocal.getProjectId()).getBrandLinkedGroupId();
@@ -183,8 +327,7 @@ public class MarkDataServiceImpl implements MarkDataService {
             // 获取所有平台稿件倾向稿件数量信息
             List<JSONObject> platformsCount = getPlatformMarkEmotionCount(startTime, endTime, projectId, linkedGroupId);
             // 正面舆情峰值平台
-            platformsCount.stream().max(Comparator.comparing(o -> o.getInteger(EmotionEnum.POSITIVE.getName()))).ifPresent(e -> result.put("posTopPlatform",
-                    e.getString("platform_id")));
+            platformsCount.stream().max(Comparator.comparing(o -> o.getInteger(EmotionEnum.POSITIVE.getName()))).ifPresent(e -> result.put("posTopPlatform", e.getString("platform_id")));
             // 负面舆情峰值平台
             platformsCount.stream().max(Comparator.comparing(o -> o.getInteger(EmotionEnum.NEGATIVE.getName()))).ifPresent(e -> result.put("negTopPlatform", e.getString("platform_id")));
             // 获取时间段正面情感数据最多的标题
@@ -278,11 +421,7 @@ public class MarkDataServiceImpl implements MarkDataService {
         for (JSONObject result : channelEmotions) {
             int channelEmotion = result.getIntValue("channelEmotion");
             long channelEmotionCount = result.getLongValue("count");
-            List<String> mediaTypeList = Arrays.asList(
-                    ImportantChannelEnum.YANGJI.getState(),
-                    ImportantChannelEnum.KEJI.getState(),
-                    ImportantChannelEnum.CAIJING.getState(),
-                    ImportantChannelEnum.QITA.getState());
+            List<String> mediaTypeList = Arrays.asList(ImportantChannelEnum.YANGJI.getState(), ImportantChannelEnum.KEJI.getState(), ImportantChannelEnum.CAIJING.getState(), ImportantChannelEnum.QITA.getState());
             List<JSONObject> mediaTypeResultList = new ArrayList<>(mediaTypeList.size());
             for (String mediaType : mediaTypeList) {
                 BoolQueryBuilder boolQueryBuilder = projectLinkedGroupQuery(projectId, linkedGroupId);
@@ -291,9 +430,7 @@ public class MarkDataServiceImpl implements MarkDataService {
                         .must(QueryBuilders.termQuery("channel_emotion", channelEmotion));
                 // TODO 其他类别标签
                 if (ImportantChannelEnum.QITA.getState().equals(mediaType)) {
-                    boolQueryBuilder.mustNot(QueryBuilders.matchQuery("tag", ImportantChannelEnum.YANGJI.getState()))
-                            .mustNot(QueryBuilders.matchQuery("tag", ImportantChannelEnum.KEJI.getState()))
-                            .mustNot(QueryBuilders.matchQuery("tag", ImportantChannelEnum.CAIJING.getState()));
+                    boolQueryBuilder.mustNot(QueryBuilders.matchQuery("tag", ImportantChannelEnum.YANGJI.getState())).mustNot(QueryBuilders.matchQuery("tag", ImportantChannelEnum.KEJI.getState())).mustNot(QueryBuilders.matchQuery("tag", ImportantChannelEnum.CAIJING.getState()));
                 } else {
                     boolQueryBuilder.must(QueryBuilders.matchQuery("tag", mediaType));
                 }
@@ -349,6 +486,16 @@ public class MarkDataServiceImpl implements MarkDataService {
         return null;
     }
 
+    private void defaultMarkSearch4Aggree(MarkSearchDTO markSearchDTO) {
+        // 时间范围默认一星期
+        if (Objects.isNull(markSearchDTO.getStartTime()) || Objects.isNull(markSearchDTO.getEndTime())) {
+            Date now = new Date();
+            markSearchDTO.setEndTime(now.getTime());
+            markSearchDTO.setStartTime(DateUtils.addDays(now, -7).getTime());
+        }
+        defaultMarkSearch(markSearchDTO);
+    }
+
     private void defaultMarkSearch(MarkSearchDTO markSearchDTO) {
         // 时间范围默认一个月
         if (Objects.isNull(markSearchDTO.getStartTime()) || Objects.isNull(markSearchDTO.getEndTime())) {
@@ -369,19 +516,22 @@ public class MarkDataServiceImpl implements MarkDataService {
             sorter.put("time", "descend");
             markSearchDTO.setSorter(sorter);
         }
+        String projectId = UserThreadLocal.getProjectId();
+        markSearchDTO.setProjectId(projectId);
+        String linkedGroupId = markSearchDTO.getLinkedGroupId();
+        if (null == linkedGroupId) {
+            linkedGroupId = projectService.getProjectVOById(projectId).getBrandLinkedGroupId();
+            markSearchDTO.setLinkedGroupId(linkedGroupId);
+        }
     }
 
-    private Pair<SearchHits, Map<String, Long>> searchMarkHitsAndCount(MarkSearchDTO dto) throws IOException {
-        // 获取索引
-        String[] indexes = esClientDao.getIndexes();
-        // PostFilter 后置过滤器
-        BoolQueryBuilder postFilter = QueryBuilders.boolQuery();
+    private EsClientDao.SearchHelper createSearchHelperByMarkSearchDTO(MarkSearchDTO dto) {
+        EsClientDao.SearchHelper helper = EsClientDao.createSearchHelper();
         // linkedGroupId projectId
+        String projectId = dto.getProjectId();
         String linkedGroupId = dto.getLinkedGroupId();
-        if (null == linkedGroupId) {
-            linkedGroupId = projectService.getProjectVOById(UserThreadLocal.getProjectId()).getBrandLinkedGroupId();
-        }
-        postFilter.must(QueryBuilders.termQuery("brandkbs_cache_maps.linkedGroupId.keyword", linkedGroupId)).must(QueryBuilders.termQuery("brandkbs_cache_maps.projectId.keyword", UserThreadLocal.getProjectId()));
+        // PostFilter 后置过滤器
+        BoolQueryBuilder postFilter = projectLinkedGroupQuery(projectId, linkedGroupId);
         // time
         postFilter.must(QueryBuilders.rangeQuery(dto.getTimeType()).lte(dto.getStartTime()).lt(dto.getEndTime()));
         // platform
@@ -397,6 +547,7 @@ public class MarkDataServiceImpl implements MarkDataService {
         if (CollectionUtils.isNotEmpty(dto.getCustomTags())) {
             postFilter.must(EsQueryTools.assembleTagQuery(dto.getCustomTags()));
         }
+        helper.setPostFilter(postFilter);
         // Query 查询条件
         BoolQueryBuilder query = QueryBuilders.boolQuery();
         // keyword
@@ -409,6 +560,7 @@ public class MarkDataServiceImpl implements MarkDataService {
         if (StringUtils.isNotEmpty(dto.getSourceKeyword())) {
             query.must(QueryBuilders.matchQuery(GenericAttribute.ES_SOURCE, dto.getSourceKeyword()));
         }
+        helper.setQuery(query);
         // sort
         FieldSortBuilder sort = null;
         for (Map.Entry<String, Object> entry : dto.getSorter().entrySet()) {
@@ -422,28 +574,46 @@ public class MarkDataServiceImpl implements MarkDataService {
                 sort = SortBuilders.fieldSort(entry.getKey()).order(SortOrder.ASC);
             }
         }
+        helper.setSort(sort);
         // from size
-        int from = (dto.getPage() - 1) * dto.getPageSize();
+        helper.setFrom((dto.getPage() - 1) * dto.getPageSize());
+        helper.setSize(dto.getPageSize());
         // HighlightBuilder ???
-        SearchHits searchHits = esClientDao.searchHits(indexes, postFilter, query, null, sort, from, dto.getPageSize(), null);
+        return helper;
+    }
+
+    private Pair<SearchHits[], Map<String, Long>> searchMarkHitsAndCount(MarkSearchDTO dto, boolean aggree) throws IOException {
+        EsClientDao.SearchHelper helper = createSearchHelperByMarkSearchDTO(dto);
+        if (aggree) {
+            helper.setSize(0);
+            // TODO 本地调试不滚动查询
+//           List<SearchResponse> searchResponses = esClientDao.searchScrollResponse(helper);
+            List<SearchResponse> searchResponses = Collections.singletonList(esClientDao.searchResponse(helper));
+            return Pair.of(searchResponses.stream().map(SearchResponse::getHits).toArray(SearchHits[]::new), null);
+        }
+        SearchHits searchHits = esClientDao.searchHits(helper);
         //平台计量统计 仅第一页且无平台限制
         Map<String, Long> counts = new HashMap<>();
         if (1 == dto.getPage() && CollectionUtils.isEmpty(dto.getPlatforms())) {
             for (MessagePlatform platform : GlobalPojo.PLATFORMS) {
-                BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery().must(query).must(EsQueryTools.assemblePlatformQuery(Collections.singletonList(platform)));
-                Long count = esClientDao.count(indexes, postFilter, queryBuilder);
+                BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery().must(helper.getQuery()).must(EsQueryTools.assemblePlatformQuery(Collections.singletonList(platform)));
+                Long count = esClientDao.count(esClientDao.getIndexes(), helper.getPostFilter(), queryBuilder);
                 counts.put(platform.getName(), count);
             }
         }
-        return Pair.of(searchHits, counts);
+        return Pair.of(new SearchHits[]{searchHits}, counts);
     }
 
-    private List<List<JSONObject>> aggregation(SearchHits searchHits, String uniqueField) {
-        List<JSONObject> jsonList = Arrays.stream(searchHits.getHits()).map(hit -> new JSONObject(hit.getSourceAsMap())).collect(Collectors.toList());
-        Map<String, String> idTitle = jsonList.stream().collect(Collectors.toMap(json -> json.getString(uniqueField),
-                json -> Tools.getBaseFromEsMap(json).getTitleNullOptionalContent()));
+    private List<List<JSONObject>> aggregation(SearchHits[] searchHitsList, String uniqueField) {
+        List<SearchHit> hitList = new ArrayList<>();
+        for (SearchHits searchHits : searchHitsList) {
+            hitList.addAll(Arrays.stream(searchHits.getHits()).collect(Collectors.toList()));
+        }
+        Map<String, String> idTitle = hitList.stream().collect(Collectors.toMap(SearchHit::getId, hit -> Tools.getBaseFromEsMap(hit.getSourceAsMap()).getTitleNullOptionalContent()));
         List<List<String>> kResult = TextUtil.getKResult(idTitle);
-        return TextUtil.restoreAndSort(kResult, jsonList, uniqueField);
+//        return TextUtil.restoreThenSort(kResult, hitList.stream().map(hit -> new JSONObject(hit.getSourceAsMap())).collect(Collectors.toList()),
+//                uniqueField);
+        return TextUtil.restoreThenSort(kResult, hitList);
     }
 
 //    private List<JSONObject> criteriaTime(Date endDate) {
@@ -536,8 +706,7 @@ public class MarkDataServiceImpl implements MarkDataService {
         query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
 
         List<JSONObject> resultList = new ArrayList<>(emotionList.size());
-        Map<String, Aggregation> aggMap =
-                esClientDao.searchResponse(indexes, null, query, aggregationBuilder, null, null, 0, 0, null).getAggregations().asMap();
+        Map<String, Aggregation> aggMap = esClientDao.searchResponse(indexes, null, query, aggregationBuilder, null, null, 0, 0, null).getAggregations().asMap();
         ParsedStringTerms teamAgg = (ParsedStringTerms) aggMap.get("count");
         List<? extends Terms.Bucket> buckets = teamAgg.getBuckets();
         for (String emotion : emotionList) {
@@ -635,14 +804,11 @@ public class MarkDataServiceImpl implements MarkDataService {
         TermsAggregationBuilder emotionAggregationBuilder = AggregationBuilders.terms("count").field("mark_cache_maps.name.keyword");
         // query
         BoolQueryBuilder query = QueryBuilders.boolQuery();
-        query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime)).
-                must(QueryBuilders.termQuery("brandkbs_cache_maps.projectId.keyword", projectId)).
-                must(QueryBuilders.termQuery("brandkbs_cache_maps.linkedGroupId.keyword", linkedGroupId));
+        query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime)).must(QueryBuilders.termQuery("brandkbs_cache_maps.projectId.keyword", projectId)).must(QueryBuilders.termQuery("brandkbs_cache_maps.linkedGroupId.keyword", linkedGroupId));
         // 情感倾向限制
         query.must(QueryBuilders.termQuery("mark_cache_maps.group_name.keyword", "情感倾向"));
         // response
-        SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, query,
-                daysAggregationBuilder.subAggregation(emotionAggregationBuilder), null, null, 0, 0, null);
+        SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, query, daysAggregationBuilder.subAggregation(emotionAggregationBuilder), null, null, 0, 0, null);
         Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
         ParsedDateHistogram dayAggTeam = (ParsedDateHistogram) aggMap.get("dayAgg");
         List<? extends Histogram.Bucket> buckets = dayAggTeam.getBuckets();
@@ -680,8 +846,7 @@ public class MarkDataServiceImpl implements MarkDataService {
         query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
         // 情感倾向限制
         query.must(QueryBuilders.termQuery("mark_cache_maps.group_name.keyword", "情感倾向"));
-        SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, query,
-                platformAggregationBuilder.subAggregation(emotionAggregationBuilder), null, null, 0, 0, null);
+        SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, query, platformAggregationBuilder.subAggregation(emotionAggregationBuilder), null, null, 0, 0, null);
         Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
         ParsedStringTerms teamAgg = (ParsedStringTerms) aggMap.get("platform_count");
         List<? extends Terms.Bucket> buckets = teamAgg.getBuckets();
@@ -723,8 +888,7 @@ public class MarkDataServiceImpl implements MarkDataService {
         List<JSONObject> platformList = commonService.getQbjcPlatform("id", "name");
         List<JSONObject> platformsCount = getPlatformsCount(startTime, endTime, null, null, projectId, linkedGroupId, platformList);
         long articlesCount = platformsCount.stream().mapToLong(platform -> platform.getLongValue("count")).sum();
-        List<JSONObject> resultList = platformsCount.stream().peek(platform -> platform.put("proportion",
-                0 == articlesCount ? 0 : platform.getLongValue("count") * 1.0 / articlesCount)).collect(Collectors.toList());
+        List<JSONObject> resultList = platformsCount.stream().peek(platform -> platform.put("proportion", 0 == articlesCount ? 0 : platform.getLongValue("count") * 1.0 / articlesCount)).collect(Collectors.toList());
         redisUtil.setExpire(redisKey, JSON.toJSONString(resultList), 1, TimeUnit.HOURS);
         return resultList;
     }
@@ -777,8 +941,7 @@ public class MarkDataServiceImpl implements MarkDataService {
      *
      * @return 所有稿件倾向稿件数量信息
      */
-    private List<JSONObject> getChannelEmotionsCount(Long startTime, Long endTime, String keyword, String searchField, String projectId, String linkedGroupId,
-                                                     List<String> emotionList) {
+    private List<JSONObject> getChannelEmotionsCount(Long startTime, Long endTime, String keyword, String searchField, String projectId, String linkedGroupId, List<String> emotionList) {
         return Collections.emptyList();
     }
 

src/main/java/com/zhiwei/brandkbs2/service/impl/ReportServiceImpl.java

@@ -88,7 +88,7 @@ public class ReportServiceImpl implements ReportService {
 
     @Override
     public List<Report> getCustomReportByStatus(String projectId, boolean status) {
-        Criteria criteria = Criteria.where("projectId").is(projectId).and("status").is(status).is("type").is(ReportTypeEnum.CUSTOM.getState());
+        Criteria criteria = Criteria.where("projectId").is(projectId).and("status").is(status).and("type").is(ReportTypeEnum.CUSTOM.getState());
         return reportDao.findList(Query.query(criteria));
     }
 
@@ -158,7 +158,7 @@ public class ReportServiceImpl implements ReportService {
         }
         JSONObject all = new JSONObject();
         all.put("name", "不限");
-        all.put("count",collect.values().stream().mapToInt(v -> v).sum());
+        all.put("count", collect.values().stream().mapToInt(v -> v).sum());
         result.add(0, all);
         return result;
     }
@@ -220,7 +220,18 @@ public class ReportServiceImpl implements ReportService {
 
     @Override
     public PageVO<JSONObject> findReportList(ReportSearchDTO reportSearch) {
-        Query query = new Query();
+        Criteria criteria = new Criteria();
+        if (null != reportSearch.getStartTime()) {
+            criteria.and("startTime").gte(reportSearch.getStartTime());
+        }
+        if (null != reportSearch.getEndTime()) {
+            criteria.and("endTime").lt(reportSearch.getEndTime());
+        }
+        if (StringUtils.isNotEmpty(reportSearch.getType())) {
+            criteria.and("type").is(reportSearch.getType());
+        }
+        Query query = new Query(criteria);
+        // 添加关键字查询并排序
         long count = reportDao.count(query, reportSearch.getKeyword(), new String[]{"title"});
         mongoUtil.start(reportSearch.getPage(), reportSearch.getPageSize(), query);
         // 添加排序

src/main/java/com/zhiwei/brandkbs2/util/RedisUtil.java

@@ -23,6 +23,10 @@ public class RedisUtil {
         return RedisKeyPrefix.REDIS_SYSTEM_COPY + "SHOT_PAGE:" + projectId + "-" + id;
     }
 
+    public static String getAggreeCacheKey(String id, String projectId) {
+        return RedisKeyPrefix.AGGREE_RESULT_CACHE + projectId + ":" + id;
+    }
+
     public void setExpire(String key, String value, long timeout, TimeUnit unit) {
         stringRedisTemplate.opsForValue().set(key, value, timeout, unit);
     }
@@ -35,4 +39,8 @@ public class RedisUtil {
         return stringRedisTemplate.opsForValue().get(key);
     }
 
+    public void remove(String key) {
+        setExpire(key, "-1", 1, TimeUnit.SECONDS);
+    }
+
 }

src/main/java/com/zhiwei/brandkbs2/util/TextUtil.java

@@ -13,6 +13,7 @@ import com.zhiwei.nlp.common.AggreeConfig;
 import com.zhiwei.nlp.vo.KResult;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.search.SearchHit;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.core.io.Resource;
 import org.springframework.stereotype.Component;
@@ -67,7 +68,7 @@ public class TextUtil {
         return AggreeBootStarter.getKResult(listMap, LIMIT).stream().map(KResult::getDataPoints).collect(Collectors.toList());
     }
 
-    public static <T> List<List<JSONObject>> restoreAndSort(List<List<T>> kResultList, List<JSONObject> sourceList, String tField) {
+    public static <T> List<List<JSONObject>> restoreThenSort(List<List<T>> kResultList, List<JSONObject> sourceList, String tField) {
         ImmutableMap<T, JSONObject> idMap = Maps.uniqueIndex(sourceList, json -> (T) json.get(tField));
         // 还原数据
         return kResultList.stream().map(gList ->
@@ -75,6 +76,14 @@ public class TextUtil {
         ).collect(Collectors.toList());
     }
 
+    public static List<List<JSONObject>> restoreThenSort(List<List<String>> kResultList, List<SearchHit> sourceList) {
+        ImmutableMap<String, SearchHit> idMap = Maps.uniqueIndex(sourceList, SearchHit::getId);
+        // 还原数据
+        return kResultList.stream().map(gList ->
+                gList.stream().map(title -> new JSONObject(idMap.get(title).getSourceAsMap())).sorted(Comparator.comparingLong(o -> o.getLong(DEFAULT_SORT_FIELD))).collect(Collectors.toList())
+        ).collect(Collectors.toList());
+    }
+
     public static Map<String, Integer> getHighWords(List<String> texts) {
         return getHighWords(texts, null);
     }

src/main/java/com/zhiwei/brandkbs2/util/Tools.java

@@ -2,6 +2,7 @@ package com.zhiwei.brandkbs2.util;
 
 import com.alibaba.excel.EasyExcel;
 import com.alibaba.excel.read.listener.PageReadListener;
+import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.base.category.ClassCodec;
 import com.zhiwei.base.entity.subclass.CompleteText;
@@ -554,4 +555,8 @@ public class Tools {
         return calendar.getTime().getTime();
     }
 
+    public static <T> T copyByJson(T obj, Class<T> clazz) {
+        return JSON.parseObject(JSON.toJSONString(obj), clazz);
+    }
+
 }
\ No newline at end of file