Skip to content

B
brandkbs2
Commit 57631a16 by shenjunjie
Options

添加定时清理聚合任务

parent b68d9b1f
Showing with 83 additions and 31 deletions
src/main/java/com/zhiwei/brandkbs2/config/Constant.java
......@@ -10,6 +10,8 @@ import org.apache.commons.lang3.time.FastDateFormat;
*/
public class Constant {
public static final Long ONE_DAY = 24 * 60 * 60 * 1000L;
public static final String HOUR_PATTERN = "yyyy-MM-dd HH";
public static final String DAY_PATTERN = "yyyy-MM-dd";
......
src/main/java/com/zhiwei/brandkbs2/controller/app/AppArticleController.java
......@@ -79,7 +79,6 @@ public class AppArticleController extends BaseController {
Pair<String, List<ExportAppYuqingDTO>> stringListPair = markDataService.downloadYuqingMarkList(markSearchDTO);
EasyExcelUtil.download(stringListPair.getLeft() + "_舆情列表数据", "sheet1", ExportAppYuqingDTO.class, stringListPair.getRight(), response);
return ResponseResult.success();
}
@ApiOperation("舆情分析-舆情概览")
......
src/main/java/com/zhiwei/brandkbs2/dao/AggreeResultDao.java
......@@ -9,4 +9,6 @@ import com.zhiwei.brandkbs2.pojo.AggreeResult;
* @date: 2022-07-18 09:47
*/
public interface AggreeResultDao extends BaseMongoDao<AggreeResult>{
long deleteExpire(long expireTime);
}
src/main/java/com/zhiwei/brandkbs2/dao/impl/AggreeResultDaoImpl.java
package com.zhiwei.brandkbs2.dao.impl;
import com.mongodb.client.result.DeleteResult;
import com.zhiwei.brandkbs2.dao.AggreeResultDao;
import com.zhiwei.brandkbs2.pojo.AggreeResult;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
/**
......@@ -13,8 +16,15 @@ import org.springframework.stereotype.Service;
@Service("aggreeResultDaoImpl")
public class AggreeResultDaoImpl extends BaseMongoDaoImpl<AggreeResult> implements AggreeResultDao {
private static final String COLLECTION_PREFIX = "brandkbs_aggree_result";
public AggreeResultDaoImpl() {
super(COLLECTION_PREFIX);
}
@Override
public long deleteExpire(long expireTime) {
Query query = Query.query(Criteria.where("cTime").lt(expireTime));
DeleteResult remove = mongoTemplate.remove(query, COLLECTION_PREFIX);
return remove.getDeletedCount();
}
}
src/main/java/com/zhiwei/brandkbs2/easyexcel/listener/EventFileListener.java
......@@ -25,12 +25,12 @@ public class EventFileListener extends AnalysisEventListener<UploadEventDTO> {
* 储存解析数据集合
*/
private final List<UploadEventDTO> datas = new ArrayList<>(BATCH_COUNT);
private final EventService EventService;
private final EventService eventService;
private final String projectId;
private final String linkedGroupId;
public EventFileListener(EventService EventService, String projectId, String linkedGroupId) {
this.EventService = EventService;
public EventFileListener(EventService eventService, String projectId, String linkedGroupId) {
this.eventService = eventService;
this.projectId = projectId;
this.linkedGroupId = linkedGroupId;
}
......
src/main/java/com/zhiwei/brandkbs2/pojo/BaseMap.java
package com.zhiwei.brandkbs2.pojo;
import com.zhiwei.base.category.ClassD;
import lombok.Getter;
import lombok.Setter;
......@@ -14,6 +15,11 @@ import lombok.Setter;
public class BaseMap {
/**
* typeB
*/
private ClassD.TypeB typeB;
/**
* url
*/
private String url;
......
src/main/java/com/zhiwei/brandkbs2/pojo/Channel.java
......@@ -34,11 +34,6 @@ public class Channel extends ChannelIndex {
private String imgUrl;
/**
* 渠道标签
*/
private String tag;
/**
* 发布稿件id列表
*/
private LinkedHashSet<String> articleIds;
......@@ -99,10 +94,11 @@ public class Channel extends ChannelIndex {
channel.setPlatform(channelIndex.getPlatform());
channel.setRealSource(channelIndex.getRealSource());
channel.setSource(channelIndex.getSource());
channel.setLastTime(channelRecord.getLastTime());
channel.setArticleIds(channelRecord.getArticleIds());
// fid 用来便捷搜索
channel.setFid(channelIndex.getFid());
channel.setLastTime(channelRecord.getLastTime());
channel.setArticleIds(channelRecord.getArticleIds());
// TODO 调性随机分配
double random = Math.random();
if (random < 0.3) {
......
src/main/java/com/zhiwei/brandkbs2/pojo/dto/MarkSearchDTO.java
......@@ -110,6 +110,6 @@ public class MarkSearchDTO {
* 是否去重
*/
@ApiModelProperty(value = "是否去重")
private boolean duplicate;
private boolean fold;
}
src/main/java/com/zhiwei/brandkbs2/service/TaskService.java
......@@ -18,4 +18,9 @@ public interface TaskService{
*/
void generateReportAndSend();
/**
* 清理过期聚合任务
*/
void cleanAggreeTask();
}
src/main/java/com/zhiwei/brandkbs2/service/impl/ChannelServiceImpl.java
package com.zhiwei.brandkbs2.service.impl;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.dao.ChannelDao;
import com.zhiwei.brandkbs2.dao.ChannelLabelDao;
import com.zhiwei.brandkbs2.dao.EventDao;
import com.zhiwei.brandkbs2.dao.EventDataDao;
import com.zhiwei.brandkbs2.dao.*;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportAdminChannelArticleDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportAdminChannelEventDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportChannelDTO;
......@@ -55,12 +52,14 @@ public class ChannelServiceImpl implements ChannelService {
@Resource(name = "esClientDao")
EsClientDao esClientDao;
@Resource(name = "channelTagDao")
ChannelTagDao channelTagDao;
@Resource(name = "mongoUtil")
MongoUtil mongoUtil;
@Override
public PageVO<JSONObject> findChannelList(int page, int size, String linkedGroupId, String emotion, String platform, Boolean show,
String keyword, String sorter) {
public PageVO<JSONObject> findChannelList(int page, int size, String linkedGroupId, String emotion, String platform, Boolean show, String keyword, String sorter) {
Query query = channelListQuery(linkedGroupId, show, emotion, platform, keyword, sorter);
long total = channelDao.count(query);
// 开启分页
......@@ -80,7 +79,7 @@ public class ChannelServiceImpl implements ChannelService {
result.put("lastTime", channel.getLastTime());
result.put("show", channel.isShow());
result.put("imgUrl", channel.getImgUrl());
result.put("tag", channel.getTag());
result.put("tag", channelTagDao.getTagByChannelName(channel.getSource()));
return result;
}).collect(Collectors.toList());
MongoUtil.PageHelper<JSONObject> pageHelper = mongoUtil.pageHelper(total, resList);
......@@ -100,7 +99,7 @@ public class ChannelServiceImpl implements ChannelService {
JSONObject json = new JSONObject();
BaseMap baseMap = Tools.getBaseFromEsMap(sourceJson);
json.put("id", sourceJson.get("id"));
json.put("title",baseMap.getTitleNullOptionalContent());
json.put("title", baseMap.getTitleNullOptionalContent());
json.put("url", baseMap.getUrl());
json.put("time", new Date(baseMap.getTime()));
json.put("emotion", baseMap.getEmotion());
......@@ -170,7 +169,7 @@ public class ChannelServiceImpl implements ChannelService {
jsonObject.put("realSource", channel.getRealSource());
jsonObject.put("source", channel.getSource());
jsonObject.put("imgUrl", channel.getImgUrl());
jsonObject.put("tag", channel.getTag());
jsonObject.put("tag", channelTagDao.getTagByChannelName(channel.getSource()));
return jsonObject;
}
......
src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java
......@@ -977,11 +977,22 @@ public class MarkDataServiceImpl implements MarkDataService {
private List<MarkFlowEntity> getMarkFlowEntity(MarkSearchDTO markSearchDTO, SearchHits searchHits) {
String projectId = UserThreadLocal.getProjectId();
String linkedGroupId = markSearchDTO.getLinkedGroupId();
return Arrays.stream(searchHits.getHits()).map(entity -> {
MarkFlowEntity instance = new MarkFlowEntity(entity);
instance.setInfo(markFlowService.createMarkFlowInfo(new JSONObject(entity.getSourceAsMap()), projectId, linkedGroupId));
return instance;
}).collect(Collectors.toList());
// 重复消息折叠
if (markSearchDTO.isFold()) {
Map<String, List<Map<String, Object>>> collect = Arrays.stream(searchHits.getHits()).map(SearchHit::getSourceAsMap).collect(Collectors.groupingBy(map -> {
BaseMap baseMap = Tools.getBaseFromEsMap(map);
String title = baseMap.getTitleNullOptionalContent();
return Tools.filterSpecialCharacter(title) + baseMap.getTypeB().encode();
}));
return collect.values().stream().map(list -> MarkFlowEntity.getFoldInstance(list.stream().map(map -> getMarkFlowEntity(map, projectId, linkedGroupId)).collect(Collectors.toList()))).collect(Collectors.toList());
}
return Arrays.stream(searchHits.getHits()).map(hit -> getMarkFlowEntity(hit.getSourceAsMap(), projectId, linkedGroupId)).collect(Collectors.toList());
}
private MarkFlowEntity getMarkFlowEntity(Map<String, Object> map, String projectId, String linkedGroupId) {
MarkFlowEntity instance = new MarkFlowEntity(new JSONObject(map));
instance.setInfo(markFlowService.createMarkFlowInfo(new JSONObject(map), projectId, linkedGroupId));
return instance;
}
}
\ No newline at end of file
src/main/java/com/zhiwei/brandkbs2/service/impl/TaskServiceImpl.java
package com.zhiwei.brandkbs2.service.impl;
import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.dao.BrandkbsTaskDao;
import com.zhiwei.brandkbs2.dao.ChannelDao;
import com.zhiwei.brandkbs2.dao.EventDataDao;
import com.zhiwei.brandkbs2.dao.ReportSettingsDao;
import com.zhiwei.brandkbs2.dao.*;
import com.zhiwei.brandkbs2.enmus.ReportTypeEnum;
import com.zhiwei.brandkbs2.es.EsClientDao;
import com.zhiwei.brandkbs2.listener.ApplicationProjectListener;
......@@ -52,6 +49,9 @@ public class TaskServiceImpl implements TaskService {
@Resource(name = "reportSettingsDao")
ReportSettingsDao reportSettingsDao;
@Resource(name = "aggreeResultDaoImpl")
AggreeResultDao aggreeResultDao;
@Resource(name = "brandkbsTaskServiceImpl")
BrandkbsTaskService brandkbsTaskService;
......@@ -123,6 +123,12 @@ public class TaskServiceImpl implements TaskService {
}
}
@Override
public void cleanAggreeTask() {
long deleteCount = aggreeResultDao.deleteExpire(System.currentTimeMillis() - Constant.ONE_DAY);
log.info("本次清理过期聚合:{}条", deleteCount);
}
private boolean reportSendByProject(Project project) {
boolean flag = false;
// 扫描setting信息
......
src/main/java/com/zhiwei/brandkbs2/task/ControlCenter.java
......@@ -50,4 +50,17 @@ public class ControlCenter {
}
}
@Async("scheduledExecutor")
@Scheduled(cron = "0 0 * * * ? ")
public void cleanAggreeTask(){
log.info("每小时清理过期聚合任务-启动");
try {
taskService.cleanAggreeTask();
} catch (Exception e) {
log.info("每小时清理过期聚合任务-出错", e);
} finally {
log.info("每小时清理过期聚合任务-结束");
}
}
}
src/main/java/com/zhiwei/brandkbs2/util/Tools.java
......@@ -4,6 +4,7 @@ import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.read.listener.PageReadListener;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.base.category.ClassB;
import com.zhiwei.base.category.ClassCodec;
import com.zhiwei.base.entity.subclass.CompleteText;
import com.zhiwei.base.entity.subclass.IncompleteText;
......@@ -313,6 +314,7 @@ public class Tools {
public static BaseMap getBaseFromEsMap(Map<String, Object> map) {
// 设置source,forward,time
BaseMap res = Tools.convertMap(map, BaseMap.class);
res.setTypeB(ClassB.TypeB.fromEncode((int) map.get(GenericAttribute.ES_C2)));
// 统一设置属性:realSource,platform,emotion
res.setRealSource(String.valueOf(map.get(GenericAttribute.ES_REAL_SOURCE)));
res.setPlatform(getPlatform(map));
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment