添加定时清理聚合任务

src/main/java/com/zhiwei/brandkbs2/config/Constant.java

@@ -10,6 +10,8 @@ import org.apache.commons.lang3.time.FastDateFormat;
  */
 public class Constant {
 
+    public static final Long ONE_DAY = 24 * 60 * 60 * 1000L;
+
     public static final String HOUR_PATTERN = "yyyy-MM-dd HH";
     public static final String DAY_PATTERN = "yyyy-MM-dd";
 

src/main/java/com/zhiwei/brandkbs2/controller/app/AppArticleController.java

@@ -79,7 +79,6 @@ public class AppArticleController extends BaseController {
         Pair<String, List<ExportAppYuqingDTO>> stringListPair = markDataService.downloadYuqingMarkList(markSearchDTO);
         EasyExcelUtil.download(stringListPair.getLeft() + "_舆情列表数据", "sheet1", ExportAppYuqingDTO.class, stringListPair.getRight(), response);
         return ResponseResult.success();
-
     }
 
     @ApiOperation("舆情分析-舆情概览")

src/main/java/com/zhiwei/brandkbs2/dao/AggreeResultDao.java

@@ -9,4 +9,6 @@ import com.zhiwei.brandkbs2.pojo.AggreeResult;
  * @date: 2022-07-18 09:47
  */
 public interface AggreeResultDao extends BaseMongoDao<AggreeResult>{
+
+    long deleteExpire(long expireTime);
 }

src/main/java/com/zhiwei/brandkbs2/dao/impl/AggreeResultDaoImpl.java

 package com.zhiwei.brandkbs2.dao.impl;
 
+import com.mongodb.client.result.DeleteResult;
 import com.zhiwei.brandkbs2.dao.AggreeResultDao;
 import com.zhiwei.brandkbs2.pojo.AggreeResult;
+import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.data.mongodb.core.query.Query;
 import org.springframework.stereotype.Service;
 
 /**
@@ -13,8 +16,15 @@ import org.springframework.stereotype.Service;
 @Service("aggreeResultDaoImpl")
 public class AggreeResultDaoImpl extends BaseMongoDaoImpl<AggreeResult> implements AggreeResultDao {
     private static final String COLLECTION_PREFIX = "brandkbs_aggree_result";
-    
+
     public AggreeResultDaoImpl() {
         super(COLLECTION_PREFIX);
     }
+
+    @Override
+    public long deleteExpire(long expireTime) {
+        Query query = Query.query(Criteria.where("cTime").lt(expireTime));
+        DeleteResult remove = mongoTemplate.remove(query, COLLECTION_PREFIX);
+        return remove.getDeletedCount();
+    }
 }

src/main/java/com/zhiwei/brandkbs2/easyexcel/listener/EventFileListener.java

@@ -25,12 +25,12 @@ public class EventFileListener extends AnalysisEventListener<UploadEventDTO> {
      * 储存解析数据集合
      */
     private final List<UploadEventDTO> datas = new ArrayList<>(BATCH_COUNT);
-    private final EventService EventService;
+    private final EventService eventService;
     private final String projectId;
     private final String linkedGroupId;
 
-    public EventFileListener(EventService EventService, String projectId, String linkedGroupId) {
-        this.EventService = EventService;
+    public EventFileListener(EventService eventService, String projectId, String linkedGroupId) {
+        this.eventService = eventService;
         this.projectId = projectId;
         this.linkedGroupId = linkedGroupId;
     }

src/main/java/com/zhiwei/brandkbs2/pojo/BaseMap.java

 package com.zhiwei.brandkbs2.pojo;
 
+import com.zhiwei.base.category.ClassD;
 import lombok.Getter;
 import lombok.Setter;
 
@@ -14,6 +15,11 @@ import lombok.Setter;
 public class BaseMap {
 
     /**
+     * typeB
+     */
+    private ClassD.TypeB typeB;
+
+    /**
      * url
      */
     private String url;

src/main/java/com/zhiwei/brandkbs2/pojo/Channel.java

@@ -34,11 +34,6 @@ public class Channel extends ChannelIndex {
     private String imgUrl;
 
     /**
-     * 渠道标签
-     */
-    private String tag;
-
-    /**
      * 发布稿件id列表
      */
     private LinkedHashSet<String> articleIds;
@@ -99,10 +94,11 @@ public class Channel extends ChannelIndex {
         channel.setPlatform(channelIndex.getPlatform());
         channel.setRealSource(channelIndex.getRealSource());
         channel.setSource(channelIndex.getSource());
-        channel.setLastTime(channelRecord.getLastTime());
-        channel.setArticleIds(channelRecord.getArticleIds());
         // fid 用来便捷搜索
         channel.setFid(channelIndex.getFid());
+
+        channel.setLastTime(channelRecord.getLastTime());
+        channel.setArticleIds(channelRecord.getArticleIds());
         // TODO 调性随机分配
         double random = Math.random();
         if (random < 0.3) {

src/main/java/com/zhiwei/brandkbs2/pojo/dto/MarkSearchDTO.java

@@ -110,6 +110,6 @@ public class MarkSearchDTO {
      * 是否去重
      */
     @ApiModelProperty(value = "是否去重")
-    private boolean duplicate;
+    private boolean fold;
 
 }

src/main/java/com/zhiwei/brandkbs2/service/TaskService.java

@@ -18,4 +18,9 @@ public interface TaskService{
      */
     void generateReportAndSend();
 
+    /**
+     * 清理过期聚合任务
+     */
+    void cleanAggreeTask();
+
 }

src/main/java/com/zhiwei/brandkbs2/service/impl/ChannelServiceImpl.java

 package com.zhiwei.brandkbs2.service.impl;
 
 import com.alibaba.fastjson.JSONObject;
-import com.zhiwei.brandkbs2.dao.ChannelDao;
-import com.zhiwei.brandkbs2.dao.ChannelLabelDao;
-import com.zhiwei.brandkbs2.dao.EventDao;
-import com.zhiwei.brandkbs2.dao.EventDataDao;
+import com.zhiwei.brandkbs2.dao.*;
 import com.zhiwei.brandkbs2.easyexcel.dto.ExportAdminChannelArticleDTO;
 import com.zhiwei.brandkbs2.easyexcel.dto.ExportAdminChannelEventDTO;
 import com.zhiwei.brandkbs2.easyexcel.dto.ExportChannelDTO;
@@ -55,12 +52,14 @@ public class ChannelServiceImpl implements ChannelService {
     @Resource(name = "esClientDao")
     EsClientDao esClientDao;
 
+    @Resource(name = "channelTagDao")
+    ChannelTagDao channelTagDao;
+
     @Resource(name = "mongoUtil")
     MongoUtil mongoUtil;
 
     @Override
-    public PageVO<JSONObject> findChannelList(int page, int size, String linkedGroupId, String emotion, String platform, Boolean show,
-                                              String keyword, String sorter) {
+    public PageVO<JSONObject> findChannelList(int page, int size, String linkedGroupId, String emotion, String platform, Boolean show, String keyword, String sorter) {
         Query query = channelListQuery(linkedGroupId, show, emotion, platform, keyword, sorter);
         long total = channelDao.count(query);
         // 开启分页
@@ -80,7 +79,7 @@ public class ChannelServiceImpl implements ChannelService {
             result.put("lastTime", channel.getLastTime());
             result.put("show", channel.isShow());
             result.put("imgUrl", channel.getImgUrl());
-            result.put("tag", channel.getTag());
+            result.put("tag", channelTagDao.getTagByChannelName(channel.getSource()));
             return result;
         }).collect(Collectors.toList());
         MongoUtil.PageHelper<JSONObject> pageHelper = mongoUtil.pageHelper(total, resList);
@@ -100,7 +99,7 @@ public class ChannelServiceImpl implements ChannelService {
                 JSONObject json = new JSONObject();
                 BaseMap baseMap = Tools.getBaseFromEsMap(sourceJson);
                 json.put("id", sourceJson.get("id"));
-                json.put("title",baseMap.getTitleNullOptionalContent());
+                json.put("title", baseMap.getTitleNullOptionalContent());
                 json.put("url", baseMap.getUrl());
                 json.put("time", new Date(baseMap.getTime()));
                 json.put("emotion", baseMap.getEmotion());
@@ -170,7 +169,7 @@ public class ChannelServiceImpl implements ChannelService {
         jsonObject.put("realSource", channel.getRealSource());
         jsonObject.put("source", channel.getSource());
         jsonObject.put("imgUrl", channel.getImgUrl());
-        jsonObject.put("tag", channel.getTag());
+        jsonObject.put("tag", channelTagDao.getTagByChannelName(channel.getSource()));
         return jsonObject;
     }
 

src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java

@@ -977,11 +977,22 @@ public class MarkDataServiceImpl implements MarkDataService {
     private List<MarkFlowEntity> getMarkFlowEntity(MarkSearchDTO markSearchDTO, SearchHits searchHits) {
         String projectId = UserThreadLocal.getProjectId();
         String linkedGroupId = markSearchDTO.getLinkedGroupId();
-        return Arrays.stream(searchHits.getHits()).map(entity -> {
-            MarkFlowEntity instance = new MarkFlowEntity(entity);
-            instance.setInfo(markFlowService.createMarkFlowInfo(new JSONObject(entity.getSourceAsMap()), projectId, linkedGroupId));
-            return instance;
-        }).collect(Collectors.toList());
+        // 重复消息折叠
+        if (markSearchDTO.isFold()) {
+            Map<String, List<Map<String, Object>>> collect = Arrays.stream(searchHits.getHits()).map(SearchHit::getSourceAsMap).collect(Collectors.groupingBy(map -> {
+                BaseMap baseMap = Tools.getBaseFromEsMap(map);
+                String title = baseMap.getTitleNullOptionalContent();
+                return Tools.filterSpecialCharacter(title) + baseMap.getTypeB().encode();
+            }));
+            return collect.values().stream().map(list -> MarkFlowEntity.getFoldInstance(list.stream().map(map -> getMarkFlowEntity(map, projectId, linkedGroupId)).collect(Collectors.toList()))).collect(Collectors.toList());
+        }
+        return Arrays.stream(searchHits.getHits()).map(hit -> getMarkFlowEntity(hit.getSourceAsMap(), projectId, linkedGroupId)).collect(Collectors.toList());
+    }
+
+    private MarkFlowEntity getMarkFlowEntity(Map<String, Object> map, String projectId, String linkedGroupId) {
+        MarkFlowEntity instance = new MarkFlowEntity(new JSONObject(map));
+        instance.setInfo(markFlowService.createMarkFlowInfo(new JSONObject(map), projectId, linkedGroupId));
+        return instance;
     }
 
 }
\ No newline at end of file

src/main/java/com/zhiwei/brandkbs2/service/impl/TaskServiceImpl.java

 package com.zhiwei.brandkbs2.service.impl;
 
 import com.zhiwei.brandkbs2.config.Constant;
-import com.zhiwei.brandkbs2.dao.BrandkbsTaskDao;
-import com.zhiwei.brandkbs2.dao.ChannelDao;
-import com.zhiwei.brandkbs2.dao.EventDataDao;
-import com.zhiwei.brandkbs2.dao.ReportSettingsDao;
+import com.zhiwei.brandkbs2.dao.*;
 import com.zhiwei.brandkbs2.enmus.ReportTypeEnum;
 import com.zhiwei.brandkbs2.es.EsClientDao;
 import com.zhiwei.brandkbs2.listener.ApplicationProjectListener;
@@ -52,6 +49,9 @@ public class TaskServiceImpl implements TaskService {
     @Resource(name = "reportSettingsDao")
     ReportSettingsDao reportSettingsDao;
 
+    @Resource(name = "aggreeResultDaoImpl")
+    AggreeResultDao aggreeResultDao;
+
     @Resource(name = "brandkbsTaskServiceImpl")
     BrandkbsTaskService brandkbsTaskService;
 
@@ -123,6 +123,12 @@ public class TaskServiceImpl implements TaskService {
         }
     }
 
+    @Override
+    public void cleanAggreeTask() {
+        long deleteCount = aggreeResultDao.deleteExpire(System.currentTimeMillis() - Constant.ONE_DAY);
+        log.info("本次清理过期聚合:{}条", deleteCount);
+    }
+
     private boolean reportSendByProject(Project project) {
         boolean flag = false;
         // 扫描setting信息

src/main/java/com/zhiwei/brandkbs2/task/ControlCenter.java

@@ -50,4 +50,17 @@ public class ControlCenter {
         }
     }
 
+    @Async("scheduledExecutor")
+    @Scheduled(cron = "0 0 * * * ? ")
+    public void cleanAggreeTask(){
+        log.info("每小时清理过期聚合任务-启动");
+        try {
+            taskService.cleanAggreeTask();
+        } catch (Exception e) {
+            log.info("每小时清理过期聚合任务-出错", e);
+        } finally {
+            log.info("每小时清理过期聚合任务-结束");
+        }
+    }
+
 }

src/main/java/com/zhiwei/brandkbs2/util/Tools.java

@@ -4,6 +4,7 @@ import com.alibaba.excel.EasyExcel;
 import com.alibaba.excel.read.listener.PageReadListener;
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.base.category.ClassB;
 import com.zhiwei.base.category.ClassCodec;
 import com.zhiwei.base.entity.subclass.CompleteText;
 import com.zhiwei.base.entity.subclass.IncompleteText;
@@ -313,6 +314,7 @@ public class Tools {
     public static BaseMap getBaseFromEsMap(Map<String, Object> map) {
         // 设置source,forward,time
         BaseMap res = Tools.convertMap(map, BaseMap.class);
+        res.setTypeB(ClassB.TypeB.fromEncode((int) map.get(GenericAttribute.ES_C2)));
         // 统一设置属性:realSource,platform,emotion
         res.setRealSource(String.valueOf(map.get(GenericAttribute.ES_REAL_SOURCE)));
         res.setPlatform(getPlatform(map));