Skip to content

B
brandkbs2
Commit ea21057a by shenjunjie
Options

事件模块提交

parent 64c458a8
Showing with 510 additions and 207 deletions
src/main/java/com/zhiwei/brandkbs2/common/GlobalPojo.java
package com.zhiwei.brandkbs2.common;
import com.zhiwei.brandkbs2.pojo.ChannelTag;
import com.zhiwei.brandkbs2.service.SystemInfoService;
import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform;
import com.zhiwei.qbjc.bean.pojo.common.Tag;
......@@ -38,6 +39,14 @@ public class GlobalPojo {
**/
public static Map<String, List<Tag>> TAGS;
public static Map<String, String> CHANNEL_TAGS;
/**
* 渠道类型（划分重要渠道）
*/
public static Map<String, Map<String, String>> MEDIA_TYPE;
@PostConstruct
public void start() {
try {
......@@ -60,12 +69,14 @@ public class GlobalPojo {
private void updatePojo(String logMsg) {
PLATFORMS = systemInfoService.getPlatforms();
TAGS = systemInfoService.getTags().stream().collect(Collectors.groupingBy(Tag::getGroupName));
log.info("{}-获取PLATFORMS-size:{},TAGS-size:{}", logMsg, PLATFORMS.size(), TAGS.size());
CHANNEL_TAGS = systemInfoService.getChannelTags().stream().collect(Collectors.toMap(ChannelTag::getChannel, ChannelTag::getTag));
MEDIA_TYPE = systemInfoService.getMediaTypes();
log.info("{}-获取PLATFORMS-size:{},TAGS-size:{},CHANNEL_TAGS:{}", logMsg, PLATFORMS.size(), TAGS.size(), CHANNEL_TAGS.size());
}
public static String getPlatformIdByName(String platformName){
public static String getPlatformIdByName(String platformName) {
for (MessagePlatform platform : PLATFORMS) {
if(platform.getName().equals(platformName)){
if (platform.getName().equals(platformName)) {
return platform.getId();
}
}
......@@ -81,4 +92,8 @@ public class GlobalPojo {
return null;
}
public static String getMediaType(String projectId, String platform, String source) {
return MEDIA_TYPE.get(projectId).get(platform + source);
}
}
src/main/java/com/zhiwei/brandkbs2/config/Constant.java
......@@ -22,6 +22,7 @@ public class Constant {
public static final FastDateFormat HOUR_FORMAT = FastDateFormat.getInstance(HOUR_PATTERN);
public static final FastDateFormat DAY_FORMAT = FastDateFormat.getInstance(DAY_PATTERN);
public static final FastDateFormat SPEC_DAY_FORMAT = FastDateFormat.getInstance("yyyy/MM/dd");
public static final FastDateFormat MONTH_FORMAT = FastDateFormat.getInstance(MONTH_PATTERN);
/**
......
src/main/java/com/zhiwei/brandkbs2/controller/admin/EventController.java
......@@ -222,9 +222,9 @@ public class EventController extends BaseController {
@ApiImplicitParams({@ApiImplicitParam(name = "list", paramType = "body", dataType = "list"), @ApiImplicitParam(name = "linkedGroupId", value = "关联项目id", paramType = "body", dataType = "string")})
@PutMapping("/upload/yq")
public ResponseResult addEventsByYq(@RequestBody JSONObject info) {
String linkedGroupId = info.getString("linkedGroupId");
String contendId = null == info.getString("contendId") ? "0" : info.getString("contendId");
List<YqEventDTO> yqEventList = info.getJSONArray("list").toJavaList(YqEventDTO.class);
eventService.addYqEvents(linkedGroupId, yqEventList);
eventService.addYqEvents(contendId, yqEventList);
behaviorService.pushBehavior(OPERATION, "批量导入舆情事件数据", request);
return ResponseResult.success();
}
......@@ -238,21 +238,22 @@ public class EventController extends BaseController {
}
@ApiOperation("文件上传事件")
@ApiImplicitParams({@ApiImplicitParam(name = "linkedGroupId", value = "关联项目组id", paramType = "form", dataType = "string"), @ApiImplicitParam(name = "fileUrl", value = "文件路径", required = true, paramType = "form", dataType = "string")})
@ApiImplicitParams({@ApiImplicitParam(name = "contendId", value = "竞品id", paramType = "form", dataType = "string"), @ApiImplicitParam(name = "fileUrl",
value = "文件路径", required = true, paramType = "form", dataType = "string")})
@PostMapping(value = "/upload/file", headers = "content-type=multipart/form-data")
@Auth(role = RoleEnum.SUPER_ADMIN)
public ResponseResult addEventsByFile(@RequestParam(value = "linkedGroupId") String linkedGroupId, @RequestParam("fileUrl") String fileUrl) {
eventService.addFileEvents(linkedGroupId, fileUrl);
public ResponseResult addEventsByFile(@RequestParam(value = "contendId",defaultValue = "0") String contendId, @RequestParam("fileUrl") String fileUrl) {
eventService.addFileEvents(contendId, fileUrl);
behaviorService.pushBehavior(OPERATION, "文件上传事件", request);
return ResponseResult.success();
}
@ApiOperation("事件数据上传")
@ApiImplicitParams({@ApiImplicitParam(name = "file", value = "上传表格", paramType = "form", dataType = "multipartFile"), @ApiImplicitParam(name =
"linkedGroupId", value = "关联项目id", paramType = "form", dataType = "string")})
"contendId", value = "竞品id", paramType = "form", dataType = "string")})
@PostMapping(value = "/data/upload", headers = "content-type=multipart/form-data")
public ResponseResult uploadEventDatas(@RequestParam("linkedGroupId") String linkedGroupId, @RequestParam("file") MultipartFile file) {
eventService.uploadEventDatas(linkedGroupId, file);
public ResponseResult uploadEventDatas(@RequestParam(value = "contendId",defaultValue = "0") String contendId, @RequestParam("file") MultipartFile file) {
eventService.uploadEventDatas(contendId, file);
behaviorService.pushBehavior(OPERATION, "事件数据上传", request);
return ResponseResult.success();
}
......
src/main/java/com/zhiwei/brandkbs2/dao/EventDao.java
......@@ -29,10 +29,10 @@ public interface EventDao extends BaseMongoDao<Event> {
*
* @param yqEventId 舆情事件id
* @param projectId 项目id
* @param linkedGroupId 关联项目id
* @param contendId 竞品id
* @return 事件
*/
Event getEventByUniqueIds(String yqEventId, String projectId, String linkedGroupId);
Event getEventByUniqueIds(String yqEventId, String projectId, String contendId);
/**
......@@ -41,7 +41,7 @@ public interface EventDao extends BaseMongoDao<Event> {
* @param channelIndex 渠道标识
* @return 参与事件数
*/
List<String> getEventCount(ChannelIndex channelIndex);
List<String> getEvents(ChannelIndex channelIndex);
/**
......
src/main/java/com/zhiwei/brandkbs2/dao/EventTopArticlesAnalysisDao.java
......@@ -9,4 +9,6 @@ import com.zhiwei.brandkbs2.pojo.EventTopArticlesAnalysis;
*/
public interface EventTopArticlesAnalysisDao extends BaseMongoDao<EventTopArticlesAnalysis>{
void deleteByEventId(String eventId);
}
src/main/java/com/zhiwei/brandkbs2/dao/impl/EventDaoImpl.java
......@@ -34,25 +34,28 @@ public class EventDaoImpl extends BaseMongoDaoImpl<Event> implements EventDao {
}
@Override
public boolean existEventByUniqueIds(String yqEventId, String projectId, String linkedGroupId) {
return null != getEventByUniqueIds(yqEventId, projectId, linkedGroupId);
public boolean existEventByUniqueIds(String yqEventId, String projectId, String contendId) {
return null != getEventByUniqueIds(yqEventId, projectId, contendId);
}
@Override
public Event getEventByUniqueIds(String yqEventId, String projectId, String linkedGroupId) {
public Event getEventByUniqueIds(String yqEventId, String projectId, String contendId) {
Query query = new Query();
query.addCriteria(Criteria.where("yqEventId").is(yqEventId).and("projectId").is(projectId).and("linkedGroupId").is(linkedGroupId));
query.addCriteria(Criteria.where("yqEventId").is(yqEventId).and("projectId").is(projectId).and("contendId").is(contendId));
return mongoTemplate.findOne(query, Event.class);
}
@Override
public List<String> getEventCount(ChannelIndex channelIndex) {
public List<String> getEvents(ChannelIndex channelIndex) {
// 添加渠道唯一标识
Criteria criteria = addChannelIndex(channelIndex);
// 分组
Aggregation agg = Aggregation.newAggregation(Aggregation.match(criteria), Aggregation.group("eventId").count().as("eventCount"));
AggregationResults<JSONObject> aggregate = mongoTemplate.aggregate(agg, getAggreeCollection(), JSONObject.class);
List<JSONObject> mappedResults = aggregate.getMappedResults();
List<JSONObject> mappedResults = new ArrayList<>();
for (String aggreeCollection : getAggreeCollections(3)) {
// 分组
Aggregation agg = Aggregation.newAggregation(Aggregation.match(criteria), Aggregation.group("eventId").count().as("eventCount"));
AggregationResults<JSONObject> aggregate = mongoTemplate.aggregate(agg, aggreeCollection, JSONObject.class);
mappedResults.addAll(aggregate.getMappedResults());
}
return mappedResults.stream().map(json -> json.getString("_id")).collect(Collectors.toList());
}
......@@ -68,6 +71,32 @@ public class EventDaoImpl extends BaseMongoDaoImpl<Event> implements EventDao {
@Override
public long getEventCount(ChannelIndex channelIndex, List<String> eventEmotions, String emotion) {
long res = 0;
Criteria criteria = Criteria.where("channelFid").is(channelIndex.getFid());
if (null != eventEmotions) {
criteria.and("eventMapper.emotion").in(eventEmotions);
}
if (null != emotion) {
criteria.and("emotion").is(emotion);
}
for (String aggreeCollection : getAggreeCollections(3)) {
Aggregation agg = Aggregation.newAggregation(Aggregation.match(criteria), Aggregation.group("eventId").count().as("eventCount"));
AggregationResults<JSONObject> aggregate = mongoTemplate.aggregate(agg, aggreeCollection, JSONObject.class);
res += aggregate.getMappedResults().size();
}
return res;
}
/**
* mongo版本不支持toString/toObjectId 表达式
*
* @param channelIndex
* @param eventEmotions
* @param emotion
* @return
*/
@Deprecated
private long getEventCountOrigin(ChannelIndex channelIndex, List<String> eventEmotions, String emotion) {
String primaryCollection = getAggreeCollection();
Criteria criteria = Criteria.where("channelFid").is(channelIndex.getFid());
if (null != emotion) {
......@@ -75,10 +104,7 @@ public class EventDaoImpl extends BaseMongoDaoImpl<Event> implements EventDao {
}
String aliasName = "events";
Criteria lookUpCriteria = Criteria.where(aliasName + ".emotion").in(eventEmotions);
List<AggregationOperation> operations = Arrays.asList(Aggregation.match(criteria),
Aggregation.lookup(COLLECTION_NAME, "eventId", "_id", aliasName),
Aggregation.match(lookUpCriteria),
Aggregation.project("events._id"));
List<AggregationOperation> operations = Arrays.asList(Aggregation.match(criteria), Aggregation.lookup(COLLECTION_NAME, "eventId", "_id", aliasName), Aggregation.match(lookUpCriteria), Aggregation.project("events._id"));
Aggregation aggregation = Aggregation.newAggregation(operations);
AggregationResults<JSONObject> aggregate = mongoTemplate.aggregate(aggregation, primaryCollection, JSONObject.class);
List<JSONObject> mappedResults = aggregate.getMappedResults();
......@@ -87,6 +113,7 @@ public class EventDaoImpl extends BaseMongoDaoImpl<Event> implements EventDao {
/**
* 以事件为主表联合查询 应用于getEventCount,已被替代
*
* @param channelIndex
* @param eventEmotions
* @param emotion
......@@ -116,9 +143,7 @@ public class EventDaoImpl extends BaseMongoDaoImpl<Event> implements EventDao {
Criteria lookUpCriteria = Criteria.where(aliasName.concat(".channelFid")).is(channelIndex.getFid());
// 分组
Aggregation agg = Aggregation.newAggregation(Aggregation.match(criteria), Aggregation.project("startTime", "_id", "emotion", "influence", "title", "endTime", "eventTag"),
// 想通过截取的方式来分组,但是会有8小时误差问题未解决,故舍弃
// .andExpression("add(new java.util.Date(8),startTime)").substring(0, nrOfChars).as("patternDate"),
Aggregation.lookup(getAggreeCollection(), "_id", "eventId", aliasName), Aggregation.match(lookUpCriteria));
Aggregation.lookup(getAggreeCollection(), "_id", "eventMapper.id", aliasName), Aggregation.match(lookUpCriteria));
AggregationResults<JSONObject> aggregate = mongoTemplate.aggregate(agg, COLLECTION_NAME, JSONObject.class);
List<JSONObject> mappedResults = aggregate.getMappedResults();
for (JSONObject mappedResult : mappedResults) {
......@@ -142,10 +167,18 @@ public class EventDaoImpl extends BaseMongoDaoImpl<Event> implements EventDao {
return res;
}
private String getAggreeCollection() {
private List<String> getAggreeCollections(int years) {
List<String> res = new ArrayList<>();
Calendar date = Calendar.getInstance();
int year = date.get(Calendar.YEAR);
return COLLECTION_PREFIX + "_" + year;
for (int i = 0; i < years; i++) {
res.add(COLLECTION_PREFIX + (year - i));
}
return res;
}
private String getAggreeCollection() {
return getAggreeCollections(1).get(0);
}
}
src/main/java/com/zhiwei/brandkbs2/dao/impl/EventTopArticlesAnalysisDaoImpl.java
......@@ -3,6 +3,8 @@ package com.zhiwei.brandkbs2.dao.impl;
import com.zhiwei.brandkbs2.dao.EventTopArticlesAnalysisDao;
import com.zhiwei.brandkbs2.pojo.EventTopArticlesAnalysis;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Component;
/**
......@@ -17,4 +19,9 @@ public class EventTopArticlesAnalysisDaoImpl extends BaseMongoDaoImpl<EventTopAr
public EventTopArticlesAnalysisDaoImpl() {
super(COLLECTION_NAME);
}
@Override
public void deleteByEventId(String eventId) {
mongoTemplate.remove(Query.query(Criteria.where("eventId").is(eventId)), COLLECTION_NAME);
}
}
src/main/java/com/zhiwei/brandkbs2/easyexcel/dto/UploadEventDataDTO.java
package com.zhiwei.brandkbs2.easyexcel.dto;
import com.alibaba.excel.annotation.ExcelProperty;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.pojo.Event;
import com.zhiwei.brandkbs2.pojo.EventData;
import com.zhiwei.brandkbs2.util.Tools;
import lombok.Data;
import lombok.ToString;
import org.apache.logging.log4j.util.Strings;
import org.bson.types.ObjectId;
import java.util.Date;
import java.util.Optional;
/**
* @ClassName: UploadEventDataDTO
......@@ -61,36 +53,5 @@ public class UploadEventDataDTO {
@ExcelProperty("原创/转发(微博平台)")
private boolean isForward;
/**
* 表格解析后的数据转换为事件数据传输格式
*
* @return 事件数据传输格式
*/
public EventData createEventData(Event event) {
EventData eventData = new EventData();
eventData.setPlatform(this.getPlatform());
eventData.setRealSource(this.getRealSource());
eventData.setSource(this.getSource());
eventData.setUrl(this.getUrl());
eventData.setTime(this.getTime().getTime());
//如果标题不为空，取前64个字符作为标题，如果为空，取内容的前32位作为标题
eventData.setTitle(Strings.isNotBlank(title) ? (title.length() > 64 ? title.substring(0, 64) : title) : content.length() > 32 ? content.substring(0, 32) : content);
eventData.setAggTitle(Tools.filterSpecialCharacter(title));
eventData.setContent(content);
eventData.setTime(this.getTime().getTime());
JSONObject tagInfo = new JSONObject();
Optional.ofNullable(this.getEmotion()).ifPresent(emotion -> tagInfo.put(Constant.EMOTION_LABEL_KEY, emotion));
Optional.ofNullable(this.getBrand()).ifPresent(brand -> tagInfo.put(Constant.BRAND_LABEL_KEY, brand));
eventData.setTagInfo(tagInfo);
eventData.setEmotion(this.getEmotion());
eventData.setForward(this.isForward);
eventData.setEventId(new ObjectId(event.getId()));
eventData.setProjectId(event.getProjectId());
eventData.setLinkedGroupId(event.getLinkedGroupId());
eventData.setCTime(System.currentTimeMillis());
// TODO Type字段类型是否需要？？？
return eventData;
}
}
src/main/java/com/zhiwei/brandkbs2/easyexcel/listener/EventDataListener.java
......@@ -5,10 +5,8 @@ import com.alibaba.excel.event.AnalysisEventListener;
import com.zhiwei.brandkbs2.dao.EventDao;
import com.zhiwei.brandkbs2.dao.EventDataDao;
import com.zhiwei.brandkbs2.easyexcel.dto.UploadEventDataDTO;
import com.zhiwei.brandkbs2.listener.ApplicationProjectListener;
import com.zhiwei.brandkbs2.pojo.Event;
import com.zhiwei.brandkbs2.pojo.EventData;
import lombok.extern.slf4j.Slf4j;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.data.redis.core.StringRedisTemplate;
......@@ -27,15 +25,15 @@ public class EventDataListener extends AnalysisEventListener<UploadEventDataDTO>
private final StringRedisTemplate stringRedisTemplate;
private final String linkedGroupId;
private final String contendId;
private final String redisKey;
public EventDataListener(EventDao eventDao, EventDataDao eventDataDao, StringRedisTemplate stringRedisTemplate, String linkedGroupId,
public EventDataListener(EventDao eventDao, EventDataDao eventDataDao, StringRedisTemplate stringRedisTemplate, String contendId,
String redisKey) {
this.eventDao = eventDao;
this.eventDataDao = eventDataDao;
this.stringRedisTemplate = stringRedisTemplate;
this.linkedGroupId = linkedGroupId;
this.contendId = contendId;
this.redisKey = redisKey;
}
......@@ -43,8 +41,8 @@ public class EventDataListener extends AnalysisEventListener<UploadEventDataDTO>
public void invoke(UploadEventDataDTO eventDataDTO, AnalysisContext analysisContext) {
Event event = eventDao.findOneById(eventDataDTO.getEventId());
// 不允许上传其他关联组数据
if (event.getLinkedGroupId().equals(linkedGroupId)) {
EventData eventData = eventDataDTO.createEventData(event);
if (event.getContendId().equals(contendId)) {
EventData eventData = EventData.createFromUploadEventDataDTO(eventDataDTO, event);
eventDataDao.insertOneWithoutId(eventData);
}
int progress = analysisContext.readRowHolder().getRowIndex() * 100 / (analysisContext.readSheetHolder().getApproximateTotalRowNumber() - 1);
......
src/main/java/com/zhiwei/brandkbs2/easyexcel/listener/EventFileListener.java
......@@ -4,7 +4,6 @@ import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener;
import com.zhiwei.brandkbs2.easyexcel.dto.UploadEventDTO;
import com.zhiwei.brandkbs2.service.EventService;
import lombok.extern.slf4j.Slf4j;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
......@@ -29,12 +28,12 @@ public class EventFileListener extends AnalysisEventListener<UploadEventDTO> {
private final List<UploadEventDTO> datas = new ArrayList<>(BATCH_COUNT);
private final EventService eventService;
private final String projectId;
private final String linkedGroupId;
private final String contendId;
public EventFileListener(EventService eventService, String projectId, String linkedGroupId) {
public EventFileListener(EventService eventService, String projectId, String contendId) {
this.eventService = eventService;
this.projectId = projectId;
this.linkedGroupId = linkedGroupId;
this.contendId = contendId;
}
/**
......@@ -67,6 +66,6 @@ public class EventFileListener extends AnalysisEventListener<UploadEventDTO> {
* 保存更新事件信息逻辑
*/
private void addFileEvent() {
datas.forEach(eventDTO -> eventService.addFileEvent(projectId, linkedGroupId, eventDTO));
datas.forEach(eventDTO -> eventService.addFileEvent(projectId, contendId, eventDTO));
}
}
src/main/java/com/zhiwei/brandkbs2/enmus/EmotionEnum.java
......@@ -112,7 +112,16 @@ public enum EmotionEnum {
return emotion;
}
public static int parseFromName(String name) {
public static EmotionEnum parseFromName(String name) {
for (EmotionEnum value : EmotionEnum.values()) {
if (value.getName().equals(name)) {
return value;
}
}
return UNDEFINED;
}
public static int parseFromName2State(String name) {
for (EmotionEnum value : EmotionEnum.values()) {
if (value.getName().equals(name)) {
return value.getState();
......
src/main/java/com/zhiwei/brandkbs2/pojo/BaseMap.java
......@@ -2,6 +2,7 @@ package com.zhiwei.brandkbs2.pojo;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.base.category.ClassD;
import com.zhiwei.brandkbs2.util.Tools;
import lombok.Getter;
import lombok.Setter;
......@@ -69,12 +70,13 @@ public class BaseMap {
private JSONObject sourceJson;
/**
* 当标题为空时用文本截取前20个字代替
*
* @return
*/
public String getTitleNullOptionalContent() {
if (null != title) {
return title;
} else {
return incompleteContent;
}
return Tools.getTitleWithContent(title, incompleteContent, content);
}
}
src/main/java/com/zhiwei/brandkbs2/pojo/ChannelIndex.java
......@@ -231,7 +231,7 @@ public class ChannelIndex extends AbstractBaseMongo {
// String url = baseMap.getUrl();
// String title = baseMap.getTitle();
String emotionStr = baseMap.getEmotion();
return new Article(time, mtime, id, EmotionEnum.parseFromName(emotionStr));
return new Article(time, mtime, id, EmotionEnum.parseFromName2State(emotionStr));
}
public Map<String, Object> toEsMap() {
......
src/main/java/com/zhiwei/brandkbs2/pojo/Event.java
......@@ -66,10 +66,10 @@ public class Event extends AbstractBaseMongo {
* 项目ID
*/
private String projectId;
/**
* 关联项目组ID
*/
private String linkedGroupId;
// /**
// * 关联项目组ID
// */
// private String linkedGroupId;
/**
* 品牌ID
*/
......@@ -103,13 +103,12 @@ public class Event extends AbstractBaseMongo {
*/
private String eventTopArticlesAnalysisDetail;
public static Event createFromYqEventDTO(YqEventDTO yqEventDTO, String collectionName, String projectId, String linkedGroupId) {
public static Event createFromYqEventDTO(YqEventDTO yqEventDTO, String collectionName, String projectId, String contendId) {
Event event = new Event();
event.setTitle(yqEventDTO.getName());
event.setStartTime(yqEventDTO.getStartTime());
boolean isEnd = "结束".equals(yqEventDTO.getSpreadStatus());
event.setEndStatus(isEnd);
event.setEndTime(isEnd ? yqEventDTO.getEndTime() : null);
event.setEndTime(yqEventDTO.getEndTime());
event.setEndStatus(Objects.nonNull(yqEventDTO.getEndTime()));
event.setKeyword(yqEventDTO.getKeyword());
event.setEmotion(yqEventDTO.getTagInfo().getString(EventTagEnum.EVENT_ATTRIBUTE.getName()));
event.setEventTag(yqEventDTO.getTagInfo());
......@@ -117,13 +116,12 @@ public class Event extends AbstractBaseMongo {
// 关联id
event.setCollectionName(collectionName);
event.setProjectId(projectId);
event.setLinkedGroupId(linkedGroupId);
event.setContendId(contendId);
return event;
}
public static Event createFromUploadEventDTO(UploadEventDTO eventDTO, String collectionName, String projectId, String linkedGroupId) {
public static Event createFromUploadEventDTO(UploadEventDTO eventDTO, String collectionName, String projectId, String contendId) {
Event event = new Event();
event.setCollectionName(collectionName);
event.setTitle(eventDTO.getTitle());
event.setStartTime(eventDTO.getStartTime().getTime());
event.setEndTime(eventDTO.getEndTime().getTime());
......@@ -134,9 +132,18 @@ public class Event extends AbstractBaseMongo {
// 关联id
event.setCollectionName(collectionName);
event.setProjectId(projectId);
event.setLinkedGroupId(linkedGroupId);
event.setContendId(contendId);
return event;
}
public void setStaticState(long totalDisseminationVolume, long totalChannelVolume, long negativeArticleVolume, JSONObject articleEmotionProportions,
JSONObject articlePlatformProportions, String eventTopArticlesAnalysisDetail) {
this.totalDisseminationVolume = totalDisseminationVolume;
this.totalChannelVolume = totalChannelVolume;
this.negativeArticleVolume = negativeArticleVolume;
this.articleEmotionProportions = articleEmotionProportions;
this.articlePlatformProportions = articlePlatformProportions;
this.eventTopArticlesAnalysisDetail = eventTopArticlesAnalysisDetail;
}
}
src/main/java/com/zhiwei/brandkbs2/pojo/EventData.java
......@@ -2,13 +2,18 @@ package com.zhiwei.brandkbs2.pojo;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.common.GlobalPojo;
import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.easyexcel.dto.UploadEventDataDTO;
import com.zhiwei.brandkbs2.pojo.dto.EventDataDTO;
import com.zhiwei.brandkbs2.util.Tools;
import lombok.Data;
import lombok.Getter;
import lombok.Setter;
import org.bson.types.ObjectId;
import java.util.Map;
import java.util.Optional;
/**
* @author sjj
......@@ -27,7 +32,12 @@ public class EventData extends AbstractBaseMongo {
/**
* 事件ID
*/
private ObjectId eventId;
private String eventId;
/**
* 跟事件关联关系
*/
private EventMapper eventMapper;
/**
* 平台
*/
......@@ -95,32 +105,76 @@ public class EventData extends AbstractBaseMongo {
/**
* 渠道标签（重要渠道）
*/
private String channelTag;
private String mediaType;
public static EventData createFromEsMap(Map<String, Object> map, Event event) {
EventData data = new EventData();
JSONObject jsonMap = new JSONObject(map);
BaseMap baseMap = Tools.getBaseFromEsMap(map);
data.setEventId(new ObjectId(event.getId()));
data.setEventId(event.getId());
data.setProjectId(event.getProjectId());
data.setContendId(event.getContendId());
data.setLinkedGroupId(event.getLinkedGroupId());
data.setPlatform(baseMap.getPlatform());
data.setRealSource(baseMap.getRealSource());
data.setSource(baseMap.getSource());
data.setChannelFid(Tools.concat(data.projectId, data.contendId, data.platform, data.realSource, data.source));
data.setUrl(baseMap.getUrl());
data.setTitle(baseMap.getTitle());
data.setAggTitle(Tools.filterSpecialCharacter(baseMap.getTitle()));
data.setTitle(baseMap.getTitleNullOptionalContent());
data.setAggTitle(Tools.filterSpecialCharacter(data.getTitle()));
data.setContent(baseMap.getContent());
data.setEmotion(baseMap.getEmotion());
data.setTime(baseMap.getTime());
data.setForward(baseMap.isForward());
data.setCTime(jsonMap.getLong(GenericAttribute.ES_CTIME));
data.setCTime(System.currentTimeMillis());
data.setMediaType(GlobalPojo.getMediaType(data.getProjectId(), data.getPlatform(), data.getSource()));
// TagInfo
data.setTagInfo(Tools.change2TagInfoByMtag(jsonMap.getString(GenericAttribute.ES_MTAG)));
data.setEmotion(baseMap.getEmotion());
data.setArticleId(jsonMap.getString("id"));
// setMapper
EventMapper eventMapper = new EventMapper();
eventMapper.setId(new ObjectId(event.getId()));
eventMapper.setEmotion(event.getEmotion());
eventMapper.setStartTime(event.getStartTime());
eventMapper.setEndTime(event.getEndTime());
data.setEventMapper(eventMapper);
return data;
}
/**
* 表格解析后的数据转换为事件数据传输格式
*
* @return 事件数据传输格式
*/
public static EventData createFromUploadEventDataDTO(UploadEventDataDTO uploadEventDataDTO, Event event) {
EventData data = new EventData();
data.setEventId(event.getId());
data.setProjectId(event.getProjectId());
data.setContendId(event.getContendId());
data.setPlatform(uploadEventDataDTO.getPlatform());
data.setRealSource(uploadEventDataDTO.getRealSource());
data.setSource(uploadEventDataDTO.getSource());
data.setChannelFid(Tools.concat(data.projectId, data.contendId, data.platform, data.realSource, data.source));
data.setUrl(uploadEventDataDTO.getUrl());
data.setTitle(Tools.getTitleWithContent(uploadEventDataDTO.getTitle(), uploadEventDataDTO.getContent()));
data.setAggTitle(Tools.filterSpecialCharacter(data.getTitle()));
data.setContent(uploadEventDataDTO.getContent());
data.setTime(uploadEventDataDTO.getTime().getTime());
data.setForward(uploadEventDataDTO.isForward());
data.setCTime(System.currentTimeMillis());
data.setMediaType(GlobalPojo.getMediaType(data.getProjectId(), data.getPlatform(), data.getSource()));
// TagInfo
JSONObject tagInfo = new JSONObject();
Optional.ofNullable(uploadEventDataDTO.getEmotion()).ifPresent(emotion -> tagInfo.put(Constant.EMOTION_LABEL_KEY, emotion));
Optional.ofNullable(uploadEventDataDTO.getBrand()).ifPresent(brand -> tagInfo.put(Constant.BRAND_LABEL_KEY, brand));
data.setTagInfo(tagInfo);
data.setEmotion(uploadEventDataDTO.getEmotion());
// setMapper
EventMapper eventMapper = new EventMapper();
eventMapper.setId(new ObjectId(event.getId()));
eventMapper.setEmotion(event.getEmotion());
eventMapper.setStartTime(event.getStartTime());
eventMapper.setEndTime(event.getEndTime());
data.setEventMapper(eventMapper);
return data;
}
......@@ -128,4 +182,13 @@ public class EventData extends AbstractBaseMongo {
return Tools.convertMap(eventDataDTO, EventData.class);
}
@Data
public static class EventMapper {
ObjectId id;
String emotion;
Long startTime;
Long endTime;
}
}
src/main/java/com/zhiwei/brandkbs2/pojo/EventTopArticlesAnalysis.java
......@@ -12,20 +12,20 @@ import org.springframework.data.mongodb.core.mapping.Document;
@Getter
@Setter
@Document(collection = "brandkbs_event_top_articles_analysis")
public class EventTopArticlesAnalysis extends AbstractBaseMongo{
public class EventTopArticlesAnalysis extends AbstractBaseMongo {
/**
* 聚合标题
*/
private String aggTitle;
/**
* 聚合标题
* 聚合情感
*/
private String emotion;
/**
* 聚合数
*/
private String count;
private Integer count;
/**
* 发布时间
*/
......@@ -42,4 +42,28 @@ public class EventTopArticlesAnalysis extends AbstractBaseMongo{
* 关联事件id
*/
private String eventId;
public static EventTopArticlesAnalysis createWithTimeType(String aggTitle, EventData templateData, String timePoint, Integer count, String eventId) {
EventTopArticlesAnalysis eventTopArticlesAnalysis = new EventTopArticlesAnalysis();
eventTopArticlesAnalysis.setAggTitle(aggTitle);
eventTopArticlesAnalysis.setEmotion(templateData.getEmotion());
eventTopArticlesAnalysis.setCount(count);
eventTopArticlesAnalysis.setTime(templateData.getTime());
eventTopArticlesAnalysis.setTimePoint(timePoint);
eventTopArticlesAnalysis.setType("按时间");
eventTopArticlesAnalysis.setEventId(eventId);
return eventTopArticlesAnalysis;
}
public static EventTopArticlesAnalysis createWithAmountType(String aggTitle, EventData templateData, Integer count, String eventId) {
EventTopArticlesAnalysis eventTopArticlesAnalysis = new EventTopArticlesAnalysis();
eventTopArticlesAnalysis.setAggTitle(aggTitle);
eventTopArticlesAnalysis.setEmotion(templateData.getEmotion());
eventTopArticlesAnalysis.setCount(count);
eventTopArticlesAnalysis.setTime(templateData.getTime());
eventTopArticlesAnalysis.setType("按数量");
eventTopArticlesAnalysis.setEventId(eventId);
return eventTopArticlesAnalysis;
}
}
src/main/java/com/zhiwei/brandkbs2/pojo/vo/EventTopArticlesAnalysisVO.java
......@@ -23,7 +23,7 @@ public class EventTopArticlesAnalysisVO {
/**
* 聚合数
*/
private String count;
private Integer count;
/**
* 发布时间
*/
......
src/main/java/com/zhiwei/brandkbs2/pojo/vo/YqEventSearchVO.java
......@@ -24,6 +24,9 @@ public class YqEventSearchVO {
@ApiModelProperty("关联项目组id")
private String linkedGroupId;
@ApiModelProperty("关联品牌id")
private String contendId = "0";
@ApiModelProperty("起始时间")
private Long startTime;
......
src/main/java/com/zhiwei/brandkbs2/service/EventService.java
......@@ -5,16 +5,17 @@ import com.zhiwei.brandkbs2.easyexcel.dto.ExportEventDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportEventDataDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.UploadEventDTO;
import com.zhiwei.brandkbs2.pojo.Event;
import com.zhiwei.brandkbs2.pojo.EventData;
import com.zhiwei.brandkbs2.pojo.EventDisseminationTrend;
import com.zhiwei.brandkbs2.pojo.dto.EventDataDTO;
import com.zhiwei.brandkbs2.pojo.dto.YqEventDTO;
import com.zhiwei.brandkbs2.pojo.vo.*;
import com.zhiwei.brandkbs2.pojo.vo.EventListInfoVO;
import com.zhiwei.brandkbs2.pojo.vo.EventVO;
import com.zhiwei.brandkbs2.pojo.vo.PageVO;
import com.zhiwei.brandkbs2.pojo.vo.YqEventSearchVO;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.web.multipart.MultipartFile;
import java.util.List;
import java.util.Map;
/**
* @ClassName: EventService
......@@ -35,10 +36,10 @@ public interface EventService {
/**
* 批量添加舆情事件数据
*
* @param linkedGroupId 关联项目ID
* @param contendId 竞品id
* @param yqEventList 舆情事件列表
*/
void addYqEvents(String linkedGroupId, List<YqEventDTO> yqEventList);
void addYqEvents(String contendId, List<YqEventDTO> yqEventList);
/**
* 批量添加舆情事件数据
......@@ -70,11 +71,11 @@ public interface EventService {
/**
* 稿件数据上传
*
* @param linkedGroupId 关联项目id
* @param contendId 竞品id
* @param file 文件
* @return 文件信息
*/
JSONObject uploadEventDatas(String linkedGroupId, MultipartFile file);
JSONObject uploadEventDatas(String contendId, MultipartFile file);
/**
* 根据票据获取文件上传进度
......@@ -166,19 +167,19 @@ public interface EventService {
/**
* 添加文件上传事件信息
*
* @param linkedGroupId 关联项目组id
* @param contendId 竞品id
* @param fileUrl 文件路径
*/
void addFileEvents(String linkedGroupId, String fileUrl);
void addFileEvents(String contendId, String fileUrl);
/**
* 添加文件上传事件信息
*
* @param projectId 项目id
* @param linkedGroupId 关联项目组id
* @param contendId 竞品id
* @param eventDTO 事件传输对象
*/
void addFileEvent(String projectId, String linkedGroupId, UploadEventDTO eventDTO);
void addFileEvent(String projectId, String contendId, UploadEventDTO eventDTO);
/**
* 批量更新事件
......@@ -257,7 +258,7 @@ public interface EventService {
* @param sorter
* @return
*/
PageVO<EventData> getEventChannelVoices(String eventId, String type, int page, int pageSize, String sorter);
List<JSONObject> getEventChannelVoices(String eventId, String type, int page, int pageSize, String sorter);
/**
* 事件详情-热门文章分析
......
src/main/java/com/zhiwei/brandkbs2/service/SystemInfoService.java
package com.zhiwei.brandkbs2.service;
import com.zhiwei.brandkbs2.pojo.ChannelTag;
import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform;
import com.zhiwei.qbjc.bean.pojo.common.Tag;
import java.util.List;
import java.util.Map;
/**
* @ClassName: SystemInfoService
......@@ -27,4 +29,13 @@ public interface SystemInfoService {
*/
List<Tag> getTags();
/**
* 获取ChannelTags
*
* @return List<ChannelTag>
*/
List<ChannelTag> getChannelTags();
Map<String, Map<String, String>> getMediaTypes();
}
src/main/java/com/zhiwei/brandkbs2/service/impl/ChannelServiceImpl.java
......@@ -165,7 +165,7 @@ public class ChannelServiceImpl implements ChannelService {
@Override
public PageVO<JSONObject> findEventList(int page, int size, String channelId) {
Channel channel = channelDao.findOneById(channelId);
List<String> eventIds = eventDao.getEventCount(channel.getChannelIndex());
List<String> eventIds = eventDao.getEvents(channel.getChannelIndex());
Query query = Query.query(Criteria.where("_id").in(eventIds));
long total = eventDao.count(query);
mongoUtil.start(page, size, query);
......@@ -262,7 +262,7 @@ public class ChannelServiceImpl implements ChannelService {
public List<ExportAdminChannelEventDTO> findDownloadChannelEventList(String channelId) {
List<ExportAdminChannelEventDTO> resList = new ArrayList<>();
Channel channel = channelDao.findOneById(channelId);
List<String> eventIds = eventDao.getEventCount(channel.getChannelIndex());
List<String> eventIds = eventDao.getEvents(channel.getChannelIndex());
if (CollectionUtils.isEmpty(eventIds)) {
return resList;
}
......@@ -460,9 +460,8 @@ public class ChannelServiceImpl implements ChannelService {
@Override
public List<JSONObject> getCollectList(String contendId) {
String projectId = UserThreadLocal.getProjectId();
String linkedGroupId = projectService.getProjectByContendId(projectId, contendId).getBrandLinkedGroupId();
Query query = new Query();
query.addCriteria(Criteria.where("projectId").is(projectId).and("linkedGroupId").is(linkedGroupId).and("isCollect").is(true));
query.addCriteria(Criteria.where("projectId").is(projectId).and("contendId").is(contendId).and("isCollect").is(true));
channelDao.addSort(query, "{\"collectTime\":\"descend\"}");
return channelDao.findList(query).stream().map(channel -> {
JSONObject json = new JSONObject();
......@@ -808,7 +807,7 @@ public class ChannelServiceImpl implements ChannelService {
}
channel.setEmotionIndex(index);
channel.setEmotion(emotion);
channel.setEventCount(eventDao.getEventCount(new ChannelIndex(channel)).size());
channel.setEventCount(eventDao.getEvents(new ChannelIndex(channel)).size());
}
/**
......
src/main/java/com/zhiwei/brandkbs2/service/impl/CommonServiceImpl.java
......@@ -59,7 +59,7 @@ public class CommonServiceImpl implements CommonService {
@Override
public List<MarkerTag> getEmotionTagsWithSort(String projectId, String linkedGroupId, TagSearch... tagSearches) {
List<MarkerTag> emotionList = getQbjcTags(linkedGroupId, TagField.GROUP_NAME.is("情感倾向"));
return emotionList.stream().filter(tag -> 0 != EmotionEnum.parseFromName(tag.getName())).sorted(Comparator.comparingInt(tag -> EmotionEnum.parseFromName(tag.getName()))).collect(Collectors.toList());
return emotionList.stream().filter(tag -> 0 != EmotionEnum.parseFromName2State(tag.getName())).sorted(Comparator.comparingInt(tag -> EmotionEnum.parseFromName2State(tag.getName()))).collect(Collectors.toList());
}
@Override
......
src/main/java/com/zhiwei/brandkbs2/service/impl/EventDataServiceImpl.java
package com.zhiwei.brandkbs2.service.impl;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.dao.EventDao;
import com.zhiwei.brandkbs2.dao.EventDataDao;
import com.zhiwei.brandkbs2.dao.EventTopArticlesAnalysisDao;
import com.zhiwei.brandkbs2.enmus.EmotionEnum;
import com.zhiwei.brandkbs2.pojo.Event;
import com.zhiwei.brandkbs2.pojo.EventData;
import com.zhiwei.brandkbs2.pojo.EventTopArticlesAnalysis;
import com.zhiwei.brandkbs2.service.EventDataService;
import com.zhiwei.brandkbs2.service.MarkDataService;
import com.zhiwei.brandkbs2.util.Tools;
......@@ -15,9 +20,9 @@ import org.joda.time.PeriodType;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.text.MessageFormat;
import java.util.*;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
/**
......@@ -42,6 +47,11 @@ public class EventDataServiceImpl implements EventDataService {
@Resource(name = "eventServiceImpl")
private EventServiceImpl eventServiceImpl;
@Resource(name = "eventTopArticlesAnalysisDao")
private EventTopArticlesAnalysisDao eventTopArticlesAnalysisDao;
private static final String TOP_DETAIL_TEMPLATE = "Top媒体传播方向：全局Top8的文章中，{0}报道方向占比最高，为{1}%，{2}报道方向占比{3}%。Top报道方向中，《{4}》相似文章数最多，总文章数为{5}篇。";
@Override
public void analysisEvent(Event event) {
long startTime = event.getStartTime();
......@@ -72,13 +82,15 @@ public class EventDataServiceImpl implements EventDataService {
}
private void updateEventData(Event event, List<EventData> eventDataList) {
String keyword = event.getKeyword();
// 删除历史数据
eventDataDao.deleteByEventId(event.getId(), event.getCollectionName());
deleteEventRelated(event);
// 设置快照属性值
setEventStaticState(event, eventDataList);
List<EventData> insertList = new ArrayList<>();
String keyword = event.getKeyword();
for (EventData data : eventDataList) {
// 关键词筛选
if (Tools.containsKeyword(Arrays.asList(keyword.split(keyword.contains("，") ? "，" : "\\|")), (data.getTitle() + data.getContent()).toLowerCase())) {
if (Tools.isContains(Arrays.asList(keyword.split(keyword.contains("，") ? "，" : "\\|")), (data.getTitle() + data.getContent()))) {
insertList.add(data);
}
}
......@@ -87,4 +99,118 @@ public class EventDataServiceImpl implements EventDataService {
});
log.info("analysisEvent-eventId:{},更新事件数据完毕,实际更新:{}条", event.getId(), insertList.size());
}
private void deleteEventRelated(Event event) {
// 删除历史数据
eventDataDao.deleteByEventId(event.getId(), event.getCollectionName());
eventTopArticlesAnalysisDao.deleteByEventId(event.getId());
}
private void setEventStaticState(Event event, List<EventData> eventDataList) {
long totalDisseminationVolume = eventDataList.size();
Set<String> totalChannelVolumes = new HashSet<>();
Map<String, AtomicLong> platformIdCounts = new HashMap<>();
long positiveArticleVolume = 0;
long negativeArticleVolume = 0;
long neutralArticleVolume = 0;
long totalArticleVolume = 0;
for (EventData eventData : eventDataList) {
switch (EmotionEnum.parseFromName(eventData.getEmotion())) {
case POSITIVE:
positiveArticleVolume++;
totalArticleVolume++;
break;
case NEGATIVE:
negativeArticleVolume++;
totalArticleVolume++;
break;
case NEUTRAL:
neutralArticleVolume++;
totalArticleVolume++;
break;
}
totalChannelVolumes.add(eventData.getPlatform() + eventData.getRealSource() + eventData.getSource());
platformIdCounts.putIfAbsent(eventData.getPlatform(), new AtomicLong());
platformIdCounts.get(eventData.getPlatform()).incrementAndGet();
}
// 文章情感占比
JSONObject articleEmotionProportions = new JSONObject();
articleEmotionProportions.put("positive", getProportion(positiveArticleVolume, totalArticleVolume));
articleEmotionProportions.put("negative", getProportion(negativeArticleVolume, totalArticleVolume));
articleEmotionProportions.put("neutral", getProportion(neutralArticleVolume, totalArticleVolume));
// 文章平台占比
JSONObject articlePlatformProportions = new JSONObject();
platformIdCounts.forEach((platform, count) -> articlePlatformProportions.put(platform, getProportion(count.get(), totalDisseminationVolume)));
// 热门文章分析描述
String eventTopArticlesAnalysisDetail = MessageFormat.format(TOP_DETAIL_TEMPLATE, topArticlesAnalysis(eventDataList, event));
event.setStaticState(totalDisseminationVolume, totalChannelVolumes.size(), negativeArticleVolume, articleEmotionProportions,
articlePlatformProportions, eventTopArticlesAnalysisDetail);
eventDao.updateOne(event);
}
private List<String> topArticlesAnalysis(List<EventData> eventDataList, Event event) {
// 标题前8数据
List<Map.Entry<String, List<EventData>>> top8Titles =
eventDataList.stream().collect(Collectors.groupingBy(EventData::getAggTitle)).entrySet().stream()
.sorted((e1, e2) -> Integer.compare(e2.getValue().size(), e1.getValue().size())).limit(8).collect(Collectors.toList());
List<EventTopArticlesAnalysis> insertList = new ArrayList<>();
// 按时间（天级）录入
top8Titles.forEach(entry -> {
Map<String, List<EventData>> dayMap = entry.getValue().stream().collect(Collectors.groupingBy(data -> Constant.SPEC_DAY_FORMAT.format(data.getTime())));
for (Map.Entry<String, List<EventData>> dayEntry : dayMap.entrySet()) {
EventData eventData = dayEntry.getValue().stream().min(Comparator.comparingLong(EventData::getTime)).get();
EventTopArticlesAnalysis eventTopArticlesAnalysis = EventTopArticlesAnalysis.createWithTimeType(entry.getKey(), eventData, dayEntry.getKey(),
dayEntry.getValue().size(), event.getId());
insertList.add(eventTopArticlesAnalysis);
}
});
// 按数量录入
top8Titles.forEach(entry -> {
EventData eventData = entry.getValue().stream().min(Comparator.comparingLong(EventData::getTime)).get();
EventTopArticlesAnalysis eventTopArticlesAnalysis = EventTopArticlesAnalysis.createWithAmountType(entry.getKey(), eventData,
entry.getValue().size(), event.getId());
insertList.add(eventTopArticlesAnalysis);
});
eventTopArticlesAnalysisDao.insertMany(insertList);
return fillDetail(top8Titles);
}
private List<String> fillDetail(List<Map.Entry<String, List<EventData>>> top8Titles) {
List<String> details = new ArrayList<>();
Map<String, AtomicLong> emotionMap = new HashMap<>();
for (Map.Entry<String, List<EventData>> entry : top8Titles) {
for (EventData eventData : entry.getValue()) {
switch (EmotionEnum.parseFromName(eventData.getEmotion())) {
case POSITIVE:
emotionMap.putIfAbsent("正面", new AtomicLong());
emotionMap.get("正面").incrementAndGet();
break;
case NEGATIVE:
emotionMap.putIfAbsent("负面", new AtomicLong());
emotionMap.get("负面").incrementAndGet();
break;
case NEUTRAL:
emotionMap.putIfAbsent("中性", new AtomicLong());
emotionMap.get("中性").incrementAndGet();
break;
}
}
}
long sum = emotionMap.values().stream().mapToLong(AtomicLong::get).sum();
emotionMap.entrySet().stream().sorted((x, y) -> Long.compare(y.getValue().get(), x.getValue().get())).limit(2).forEach(entry -> {
details.add(entry.getKey());
details.add(String.valueOf((entry.getValue().get()) / sum));
});
details.add(top8Titles.get(0).getKey());
details.add(String.valueOf(top8Titles.get(0).getValue().size()));
return details;
}
private Double getProportion(long part, long total) {
if (0 == part || 0 == total) {
return 0.00;
}
return (double) part / total;
}
}
src/main/java/com/zhiwei/brandkbs2/service/impl/EventServiceImpl.java
......@@ -43,7 +43,6 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.domain.Sort;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;
......@@ -110,6 +109,11 @@ public class EventServiceImpl implements EventService {
@Resource
private RedisUtil redisUtil;
/**
* 重要渠道集合
*/
private static final List<String> IMPORTANT_CHANNEL_LIST = new ArrayList<>(Arrays.asList("央级", "科技", "财经", "其他"));
// private static final Pattern PATTERN = Pattern.compile("[\ud83c\udc00-\ud83c\udfff]|[\ud83d\udc00-\ud83d\udfff]|[\u2600-\u27ff]");
@Override
......@@ -123,9 +127,9 @@ public class EventServiceImpl implements EventService {
}
@Override
public void addYqEvents(String linkedGroupId, List<YqEventDTO> yqEventList) {
public void addYqEvents(String contendId, List<YqEventDTO> yqEventList) {
String projectId = UserThreadLocal.getProjectId();
String redisKey = RedisKeyPrefix.yuqingProgressKey(linkedGroupId);
String redisKey = RedisKeyPrefix.yuqingProgressKey(contendId);
if (CollectionUtils.isEmpty(yqEventList)) {
stringRedisTemplate.opsForValue().set(redisKey, String.valueOf(100), 1, TimeUnit.MINUTES);
}
......@@ -139,8 +143,8 @@ public class EventServiceImpl implements EventService {
//事件太多塞满队列了
ApplicationProjectListener.getThreadPool().execute(() -> {
yqEventList.forEach(yqEventDTO -> {
Event existEvent = eventDao.getEventByUniqueIds(yqEventDTO.getYqEventId(), projectId, linkedGroupId);
Event event = Event.createFromYqEventDTO(yqEventDTO, collectionName, projectId, linkedGroupId);
Event existEvent = eventDao.getEventByUniqueIds(yqEventDTO.getYqEventId(), projectId, contendId);
Event event = Event.createFromYqEventDTO(yqEventDTO, collectionName, projectId, contendId);
if (null != existEvent) {
event.setId(existEvent.getId());
event.setCTime(existEvent.getCTime());
......@@ -179,7 +183,7 @@ public class EventServiceImpl implements EventService {
}
log.info("开始导入" + resList.size() + "个事件...");
List<YqEventDTO> list = resList.stream().map(YqEventDTO::createFromUrlJson).collect(Collectors.toList());
addYqEvents(yqEventSearchVO.getLinkedGroupId(), list);
addYqEvents(yqEventSearchVO.getContendId(), list);
}
@Override
......@@ -239,14 +243,14 @@ public class EventServiceImpl implements EventService {
}
@Override
public JSONObject uploadEventDatas(String linkedGroupId, MultipartFile file) {
public JSONObject uploadEventDatas(String contendId, MultipartFile file) {
String uuid = Tools.getUUID();
String redisKey = RedisKeyPrefix.eventDataProgressKey(uuid);
stringRedisTemplate.opsForValue().set(redisKey, "0");
ApplicationProjectListener.getThreadPool().execute(() -> {
ReadExcelDTO<UploadEventDataDTO> readExcel = new ReadExcelDTO<>();
readExcel.setClazz(UploadEventDataDTO.class);
readExcel.setAnalysisEventListener(new EventDataListener(eventDao, eventDataDao, stringRedisTemplate, linkedGroupId, redisKey));
readExcel.setAnalysisEventListener(new EventDataListener(eventDao, eventDataDao, stringRedisTemplate, contendId, redisKey));
EasyExcelUtil.read(file, readExcel);
stringRedisTemplate.delete(redisKey);
});
......@@ -432,31 +436,30 @@ public class EventServiceImpl implements EventService {
}
@Override
public void addFileEvents(String linkedGroupId, String fileUrl) {
public void addFileEvents(String contendId, String fileUrl) {
//获取事件信息
ApplicationProjectListener.getThreadPool().execute(() -> {
ReadExcelDTO<UploadEventDTO> readExcelDTO = new ReadExcelDTO<>();
readExcelDTO.setClazz(UploadEventDTO.class);
EventFileListener eventFileListener = new EventFileListener(this, UserThreadLocal.getProjectId(), linkedGroupId);
EventFileListener eventFileListener = new EventFileListener(this, UserThreadLocal.getProjectId(), contendId);
readExcelDTO.setAnalysisEventListener(eventFileListener);
EasyExcelUtil.read(brandkbsFileUrl + fileUrl, readExcelDTO);
});
}
@Override
public void addFileEvent(String projectId, String linkedGroupId, UploadEventDTO eventDTO) {
public void addFileEvent(String projectId, String contendId, UploadEventDTO eventDTO) {
if (StringUtils.isEmpty(eventDTO.getTitle()) || StringUtils.isEmpty(eventDTO.getKeyword())) {
return;
}
Event event = Event.createFromUploadEventDTO(eventDTO, eventDataDao.generateCollectionName(), projectId, linkedGroupId);
Event event = Event.createFromUploadEventDTO(eventDTO, eventDataDao.generateCollectionName(), projectId, contendId);
event.setCTime(new Date().getTime());
eventDao.insertOne(event);
}
@Override
public void analysisEvents(List<String> eventIds) {
// ApplicationProjectListener.getThreadPool().execute(() -> eventIds.forEach(eventId -> {
eventIds.forEach(eventId -> {
ApplicationProjectListener.getThreadPool().execute(() -> eventIds.forEach(eventId -> {
String redisKey = null;
try {
Event event = getEventById(eventId);
......@@ -469,8 +472,7 @@ public class EventServiceImpl implements EventService {
log.error("事件id:{}更新失败", eventId, e);
redisUtil.set(redisKey, "-1");
}
});
// }));
}));
}
@Override
......@@ -549,7 +551,7 @@ public class EventServiceImpl implements EventService {
// vo.setFirstEventData(eventDataDao.findFirstData(event.getId(), event.getCollectionName()));
// return vo;
// }).collect(Collectors.toList());
// 未保证排序
// 为保证排序
Map<String, EventListInfoVO> sortMap = eventList.stream().collect(Collectors.toConcurrentMap(Event::getId, EventListInfoVO::new));
CompletableFuture.allOf(eventList.stream().map(event -> eventDataDao.findFirstDataAsync(event.getId(), event.getCollectionName()).thenApply((r) -> {
// 设置首发稿件
......@@ -576,31 +578,31 @@ public class EventServiceImpl implements EventService {
}
@Override
public PageVO<EventData> getEventChannelVoices(String eventId, String type, int page, int pageSize, String sorter) {
public List<JSONObject> getEventChannelVoices(String eventId, String type, int page, int pageSize, String sorter) {
Event event = getEventById(eventId);
Query query = Query.query(Criteria.where("eventId").is(eventId));
if (Objects.equals("重要渠道", type)) {
query.addCriteria(Criteria.where("sourceTag").ne(null));
query.addCriteria(Criteria.where("mediaType").in(IMPORTANT_CHANNEL_LIST));
}
// 排序
JSONObject sortJson = JSONObject.parseObject(sorter);
sortJson.entrySet().forEach(sortEntry -> {
String sort = String.valueOf(sortEntry.getValue()).replace("end", "");
String sortField = sortEntry.getKey();
if (Sort.Direction.ASC.name().equalsIgnoreCase(sort)) {
query.with(Sort.by(Sort.Direction.ASC, sortField));
} else {
query.with(Sort.by(Sort.Direction.DESC, sortField));
}
});
mongoUtil.start(page, pageSize, query);
// 总数
long total = eventDataDao.count(query, event.getCollectionName());
int start = pageSize * (page - 1);
query.limit(pageSize).skip(start);
// 排序
eventDataDao.addSort(query, sorter);
// 数据
List<EventData> eventDataList = eventDataDao.findList(query, event.getCollectionName());
PageVO<EventData> pageVo = PageVO.createPageVo(total, page, pageSize, eventDataList);
return pageVo;
// 分组排序
Map<String, List<EventData>> groupMap = new HashMap<>();
for (EventData eventData : eventDataList) {
groupMap.putIfAbsent(eventData.getMediaType(), new ArrayList<>());
groupMap.get(eventData.getMediaType()).add(eventData);
}
return groupMap.entrySet().stream().map(entry -> {
JSONObject json = new JSONObject();
json.put("type", entry.getKey());
json.put("list", entry.getValue());
return json;
}).collect(Collectors.toList());
}
@Override
......@@ -763,7 +765,7 @@ public class EventServiceImpl implements EventService {
List<JSONObject> resList = new ArrayList<>(array.size());
for (int i = 0; i < array.size(); i++) {
JSONObject jsonObject = array.getJSONObject(i);
jsonObject.put("exists", eventDao.existEventByUniqueIds(jsonObject.getString("id"), UserThreadLocal.getProjectId(), yqEventSearchVO.getLinkedGroupId()));
jsonObject.put("exists", eventDao.existEventByUniqueIds(jsonObject.getString("id"), UserThreadLocal.getProjectId(), yqEventSearchVO.getContendId()));
resList.add(jsonObject);
}
return Pair.of(result.getLongValue("total"), resList);
......
src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java
......@@ -79,7 +79,8 @@ import java.util.stream.Collectors;
public class MarkDataServiceImpl implements MarkDataService {
private static final Logger log = LogManager.getLogger(MarkDataServiceImpl.class);
private static final String[] EVENT_FETCH_SOURCE = new String[]{"ind_full_text", "c5", "real_source", "source", "mtime", "time", "url", "mtag", "foreign", "brandkbs_mark_cache_maps"};
private static final String[] EVENT_FETCH_SOURCE =
new String[]{"id", "ind_full_text", "c5", "real_source", "source", "mtime", "time", "url", "mtag", "foreign", "brandkbs_mark_cache_maps"};
@Resource(name = "projectServiceImpl")
private ProjectService projectService;
......@@ -829,7 +830,8 @@ public class MarkDataServiceImpl implements MarkDataService {
@Override
public List<JSONObject> searchMarkDataByEvent(Event event) {
return searchMarkDataByTime(event.getProjectId(), event.getLinkedGroupId(), event.getContendId(), event.getStartTime(), event.getEndTime(), EVENT_FETCH_SOURCE);
String linkedGroupId = projectService.getProjectByContendId(event.getProjectId(), event.getContendId()).getBrandLinkedGroupId();
return searchMarkDataByTime(event.getProjectId(), linkedGroupId, event.getContendId(), event.getStartTime(), event.getEndTime(), EVENT_FETCH_SOURCE);
}
@Override
......
src/main/java/com/zhiwei/brandkbs2/service/impl/SystemInfoServiceImpl.java
package com.zhiwei.brandkbs2.service.impl;
import com.zhiwei.brandkbs2.dao.ChannelTagDao;
import com.zhiwei.brandkbs2.dao.MediaTypeDao;
import com.zhiwei.brandkbs2.dao.QbjcPojoDao;
import com.zhiwei.brandkbs2.pojo.ChannelTag;
import com.zhiwei.brandkbs2.pojo.MediaType;
import com.zhiwei.brandkbs2.service.SystemInfoService;
import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform;
import com.zhiwei.qbjc.bean.pojo.common.Tag;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* @ClassName: SystemInfoServiceImpl
......@@ -21,6 +27,12 @@ public class SystemInfoServiceImpl implements SystemInfoService {
@Resource(name = "qbjcPojoDao")
private QbjcPojoDao qbjcPojoDao;
@Resource
private ChannelTagDao channelTagDao;
@Resource
private MediaTypeDao mediaTypeDao;
@Override
public List<MessagePlatform> getPlatforms() {
return qbjcPojoDao.findMessagePlatformAll();
......@@ -30,4 +42,20 @@ public class SystemInfoServiceImpl implements SystemInfoService {
public List<Tag> getTags() {
return qbjcPojoDao.findTagAll();
}
@Override
public List<ChannelTag> getChannelTags() {
return channelTagDao.findList(null);
}
@Override
public Map<String, Map<String, String>> getMediaTypes() {
Map<String, Map<String, String>> res = new HashMap<>();
List<MediaType> list = mediaTypeDao.findList(null);
for (MediaType mediaType : list) {
res.putIfAbsent(mediaType.getProjectId(), new HashMap<>());
res.get(mediaType.getProjectId()).put(mediaType.getPlatform() + mediaType.getChannel(), mediaType.getType());
}
return res;
}
}
src/main/java/com/zhiwei/brandkbs2/util/Tools.java
......@@ -210,25 +210,26 @@ public class Tools {
}
}
/**
* 是否包含关键字
*
* @param keywordsList 关键词数据
* @param content 字符串标题+内容
* @return 是否包含关键字
*/
public static boolean containsKeyword(List<String> keywordsList, String content) {
//按空格分割必须全部命中
for (String keywords : keywordsList) {
for (String keyword : keywords.split(" ")) {
if (!content.contains(keyword.toLowerCase())) {
//一个不命中，直接返回false
return false;
}
}
}
return true;
}
// /**
// * 是否包含关键字
// *
// * @param keywordsList 关键词数据
// * @param content 字符串标题+内容
// * @return 是否包含关键字
// */
// public static boolean containsKeyword(List<String> keywordsList, String content) {
// keywordsList = keywordsList.stream().map(keywords->keywords.split(keywords.contains("，") ? "，" : "\\|")).collect(Collectors.toList());
// //按空格分割必须全部命中
// for (String keywords : keywordsList) {
// for (String keyword : keywords.split(" ")) {
// if (!content.contains(keyword.toLowerCase())) {
// //一个不命中，直接返回false
// return false;
// }
// }
// }
// return true;
// }
public static <T> T convertMap(Object source, Class<T> destinationClass) throws MappingException {
return DOZER_BEAN_MAPPER.map(source, destinationClass);
......@@ -761,20 +762,14 @@ public class Tools {
public static boolean isContains(List<String> keywords, String content) {
boolean contains = true;
content = content.toLowerCase();
for (String value : keywords) {
//按空格分割必须全部命中
String[] keys = value.split(" ");
for (String key : keys) {
if (!content.contains(key.toLowerCase())) {
//一个不命中，直接结束当前循环
contains = false;
break;
// 按空格分割必须全部命中
for (String keyword : keywords) {
for (String word : keyword.split(" ")) {
if (!content.contains(word.toLowerCase())) {
//一个不命中，直接返回false
return false;
}
}
if (contains) {
//已经满足条件，视为有效数据，结束循环
break;
}
}
return contains;
}
......@@ -803,4 +798,17 @@ public class Tools {
return res;
}
public static String getTitleWithContent(String title, String... contents) {
if (null != title) {
return title;
} else {
for (String content : contents) {
if (null != content) {
return content.substring(0, Math.min(20, content.length()));
}
}
}
return null;
}
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment