Skip to content

B
brandkbs2
Commit 0e7e2f1a by shenjunjie
Options

2022/7/29 12:05

parent 404cb010
Showing with 189 additions and 39 deletions
src/main/java/com/zhiwei/brandkbs2/es/EsClientDao.java
......@@ -2,6 +2,7 @@ package com.zhiwei.brandkbs2.es;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.enmus.EmotionEnum;
import com.zhiwei.brandkbs2.pojo.ChannelIndex;
import com.zhiwei.brandkbs2.pojo.Event;
import com.zhiwei.brandkbs2.util.Tools;
......@@ -179,7 +180,8 @@ public class EsClientDao {
v = new ChannelIndex.Record();
}
try {
return v.mergeRecord(new ChannelIndex.Record((long) result.get(GenericAttribute.ES_TIME), String.valueOf(result.get("id"))));
return v.mergeRecord(new ChannelIndex.Record((long) result.get(GenericAttribute.ES_TIME), (long) result.get(GenericAttribute.ES_MTIME),
String.valueOf(result.get("id")), EmotionEnum.parseFromName(Tools.getEmotion(result))));
} catch (Exception e) {
log.error("searchRecord-error-id:{}", result.get("id"), e);
return null;
......@@ -198,7 +200,7 @@ public class EsClientDao {
return getIndexList().toArray(new String[0]);
}
protected RestHighLevelClient getEsClient(){
protected RestHighLevelClient getEsClient() {
return esClient;
}
......
src/main/java/com/zhiwei/brandkbs2/pojo/ChannelIndex.java
......@@ -92,9 +92,7 @@ public class ChannelIndex extends AbstractBaseMongo {
ChannelIndex channelIndex = new ChannelIndex(projectId, linkedGroupId, messagePlatform.getName(), realSource, source);
// 默认主品牌
channelIndex.setContendId(String.valueOf(0));
Optional.ofNullable(cacheMap.get("contend_id")).ifPresent(e -> {
channelIndex.setContendId(String.valueOf(e));
});
Optional.ofNullable(cacheMap.get("contend_id")).ifPresent(e -> channelIndex.setContendId(String.valueOf(e)));
res.add(channelIndex);
}
return res;
......@@ -109,14 +107,12 @@ public class ChannelIndex extends AbstractBaseMongo {
public static Map<ChannelIndex, Record> mergeRecord(List<Map<ChannelIndex, Record>> channelIndexMaps) {
Map<ChannelIndex, Record> mergeRecord = new HashMap<>();
for (Map<ChannelIndex, Record> channelIndexMap : channelIndexMaps) {
channelIndexMap.forEach((channelIndex, record) -> {
mergeRecord.compute(channelIndex, (k, v) -> {
if (null == v) {
return record;
}
return v.mergeRecord(record);
});
});
channelIndexMap.forEach((channelIndex, record) -> mergeRecord.compute(channelIndex, (k, v) -> {
if (null == v) {
return record;
}
return v.mergeRecord(record);
}));
}
return mergeRecord;
}
......@@ -132,9 +128,9 @@ public class ChannelIndex extends AbstractBaseMongo {
public Record() {
}
public Record(Long lastTime, String articleId) {
public Record(Long lastTime, Long mtime, String articleId, int emotion) {
this.lastTime = lastTime;
this.articles.add(new Article(lastTime, articleId));
this.articles.add(new Article(lastTime, mtime, articleId, emotion));
}
public void setLastTime(Long lastTime) {
......@@ -162,18 +158,24 @@ public class ChannelIndex extends AbstractBaseMongo {
@AllArgsConstructor
public static class Article {
Long time;
Long mtime;
String id;
int emotion;
public static Article fromEsMap(Map<String, Object> esMap) {
return new Article(Long.valueOf(esMap.get("time") + ""), String.valueOf(esMap.get("id")));
return new Article(Long.parseLong(esMap.get("time") + ""), Long.parseLong(esMap.get("mtime") + ""), String.valueOf(esMap.get("id")), Integer.parseInt(
esMap.get("emotion") + ""));
}
public Map<String, Object> toEsMap() {
Map<String, Object> map = new HashMap<>();
map.put("time", time);
map.put("mtime", mtime);
map.put("id", id);
map.put("emotion", emotion);
return map;
}
}
@Override
......
src/main/java/com/zhiwei/brandkbs2/service/impl/ChannelServiceImpl.java
package com.zhiwei.brandkbs2.service.impl;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.common.GlobalPojo;
import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.dao.*;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportAdminChannelArticleDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportAdminChannelEventDTO;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportChannelDTO;
import com.zhiwei.brandkbs2.enmus.ChannelEmotion;
import com.zhiwei.brandkbs2.enmus.EmotionEnum;
import com.zhiwei.brandkbs2.enmus.EventTagEnum;
import com.zhiwei.brandkbs2.enmus.ExperienceEnum;
import com.zhiwei.brandkbs2.es.ChannelEsDao;
......@@ -45,6 +48,8 @@ import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.io.IOException;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.util.*;
import java.util.stream.Collectors;
......@@ -380,39 +385,130 @@ public class ChannelServiceImpl implements ChannelService {
@Override
public JSONObject getSpreadingTend(String channelId, String type, Set<String> contends, Long startTime, Long endTime) {
try {
JSONObject res = new JSONObject();
// 默认搜索一周
if (null == startTime || null == endTime) {
Long[] timeRangeWeek = commonService.getTimeRangeWeek();
startTime = timeRangeWeek[0];
endTime = timeRangeWeek[1];
}
Channel channel = channelDao.findOneById(channelId);
EsClientDao.SearchHelper searchHelper = createSearchHelperByChannelCriteria(UserThreadLocal.getProjectId(), null, null, null, startTime, endTime);
// 添加contends集合 查询
EsQueryTools.assembleContendsQuery(searchHelper.getQuery(), contends);
// 添加渠道限制
searchHelper.getQuery().must(QueryBuilders.termQuery("channel_fid", channel.getFid()));
searchHelper.getQuery().must(QueryBuilders.termQuery("channel_fid.keyword", channel.getFid()));
// 分页查询所有结果
List<SearchResponse> searchResponses = channelEsDao.searchScrollResponse(searchHelper);
// 根据竞品分类
Map<String, List<ChannelIndex.Article>> contendMap = new HashMap<>();
for (SearchResponse searchResponse : searchResponses) {
for (SearchHit hit : searchResponse.getHits().getHits()) {
ChannelRecord channelRecord = new ChannelRecord(hit.getSourceAsMap());
// 过滤掉不符合时间条件的数据
List<ChannelIndex.Article> articles = channelRecord.getRecord().getArticles().stream().filter(article -> Tools.hitTimeRange(startTime, endTime, article.getTime())).collect(Collectors.toList());
contendMap.compute(channelRecord.getContendId(), (k, v) -> {
if (null == v) {
return articles;
// 根据品牌分类
Map<String, List<ChannelIndex.Article>> contendMap = convert2ContendMap(searchResponses, startTime, endTime);
long articleTotal = 0;
List<JSONObject> spreadDatas = new ArrayList<>();
for (Map.Entry<String, List<ChannelIndex.Article>> entry : contendMap.entrySet()) {
JSONObject spreadData = new JSONObject();
// 按日分组并根据id去重保留最近标注时间
Map<Long, List<ChannelIndex.Article>> dateListMap = partitionWithDuplicate(startTime, endTime, entry.getValue());
List<JSONObject> spread = dateListMap.entrySet().stream().map(e -> {
JSONObject spreadJson = new JSONObject();
spreadJson.put("time", e.getKey());
spreadJson.put("sum", e.getValue().size());
return spreadJson;
}).collect(Collectors.toList());
spreadData.put("spreadingTend", spread);
// 各稿件数统计
long positiveCount = 0;
long negativeCount = 0;
long total = 0;
for (List<ChannelIndex.Article> articles : dateListMap.values()) {
for (ChannelIndex.Article article : articles) {
if (article.getEmotion() == EmotionEnum.POSITIVE.getState()) {
positiveCount++;
} else if (article.getEmotion() == EmotionEnum.NEGATIVE.getState()) {
negativeCount++;
}
v.addAll(articles);
return v;
});
total++;
}
}
spreadData.put("positiveCount", positiveCount);
spreadData.put("negativeCount", negativeCount);
spreadData.put("articleTotal", total);
spreadData.put("positivePercent", new BigDecimal((double) positiveCount * 100 / total).setScale(1, RoundingMode.UP));
spreadData.put("negativePercent", new BigDecimal((double) negativeCount * 100 / total).setScale(1, RoundingMode.UP));
articleTotal += dateListMap.values().stream().mapToInt(List::size).sum();
spreadDatas.add(spreadData);
}
// 根据竞品分类细分
// TODO
res.put("startTime", startTime);
res.put("endTime", endTime);
res.put("articleTotal", articleTotal);
res.put("eventTotal", 0);
res.put("spreadDatas", spreadDatas);
return res;
} catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("es查询异常"));
}
return null;
}
// private List<JSONObject> getSpreadData(){
//
// }
private Map<String, List<ChannelIndex.Article>> convert2ContendMap(List<SearchResponse> searchResponses, Long startTime, Long endTime) {
Map<String, List<ChannelIndex.Article>> contendMap = new HashMap<>();
for (SearchResponse searchResponse : searchResponses) {
for (SearchHit hit : searchResponse.getHits().getHits()) {
ChannelRecord channelRecord = new ChannelRecord(hit.getSourceAsMap());
// 过滤掉不符合时间条件的数据
List<ChannelIndex.Article> articles = channelRecord.getRecord().getArticles().stream().filter(article -> Tools.hitTimeRange(startTime, endTime, article.getTime())).collect(Collectors.toList());
contendMap.compute(channelRecord.getContendId(), (k, v) -> {
if (null == v) {
return articles;
}
v.addAll(articles);
return v;
});
}
}
return contendMap;
}
/**
* 按日分组并根据id去重保留最近标注时间
*
* @return
*/
private Map<Long, List<ChannelIndex.Article>> partitionWithDuplicate(Long startTime, Long endTime, List<ChannelIndex.Article> articles) {
Map<Long, List<ChannelIndex.Article>> res = new HashMap<>();
// 自动补全时间段
for (Long timeKey : Tools.parseToDays(startTime, endTime)) {
res.put(timeKey, Lists.newArrayList());
}
Map<String, ChannelIndex.Article> setMap = new HashMap<>();
// 去重并保留最近标注时间
for (ChannelIndex.Article article : articles) {
setMap.compute(article.getId(), (k, v) -> {
// 旧值为null或标注时间更新
if (null == v || article.getMtime() > v.getMtime()) {
return article;
}
return v;
});
}
for (ChannelIndex.Article article : setMap.values()) {
// 按日分组
long key = Tools.truncDate(new Date(article.getTime()), Constant.DAY_PATTERN).getTime();
res.compute(key, (k, v) -> {
if (null == v) {
v = new ArrayList<>();
}
v.add(article);
return v;
});
}
return res;
}
private EsClientDao.SearchHelper createSearchHelperByChannelCriteria(String projectId, String contendId, String platform, String keyword, Long startTime, Long endTime) {
EsClientDao.SearchHelper helper = EsClientDao.createSearchHelper();
......@@ -425,9 +521,9 @@ public class ChannelServiceImpl implements ChannelService {
// query
BoolQueryBuilder query = QueryBuilders.boolQuery();
// project和contendId
query.must(QueryBuilders.termQuery("project_id", projectId));
query.must(QueryBuilders.termQuery("project_id.keyword", projectId));
if (null != contendId) {
query.must(QueryBuilders.termQuery("contend_id", contendId));
query.must(QueryBuilders.termQuery("contend_id.keyword", contendId));
}
// platformId
String platformId = GlobalPojo.getPlatformIdByName(platform);
......
src/main/java/com/zhiwei/brandkbs2/util/Tools.java
......@@ -24,6 +24,8 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.commons.lang3.time.FastDateFormat;
import org.dozer.DozerBeanMapper;
import org.joda.time.Period;
import org.joda.time.PeriodType;
import org.springframework.data.mapping.MappingException;
import org.springframework.web.multipart.MultipartFile;
......@@ -576,6 +578,7 @@ public class Tools {
/**
* 自定义HttpStatus和response内容，返回response
*
* @param response
* @param status
* @param returnData
......@@ -630,4 +633,28 @@ public class Tools {
return null == endTime || endTime > targetTime;
}
/**
* 解析时间转换成按日的集合
*
* @param startTime 开始时间
* @param endTime 结束时间
* @return 按日分割的map集合
*/
public static List<Long> parseToDays(Long startTime, Long endTime) {
Date start = new Date(startTime);
Date end = new Date(endTime);
start = Tools.truncDate(start, Constant.DAY_PATTERN);
end = Tools.truncDate(end, Constant.DAY_PATTERN);
Period periodDays = new Period(start.getTime(), end.getTime(), PeriodType.days());
int days = periodDays.getDays();
if (days < 0) {
return Collections.emptyList();
}
List<Long> resList = new ArrayList<>(days);
for (int i = 0; i <= days; i++) {
resList.add(DateUtils.addDays(start, i).getTime());
}
return resList;
}
}
\ No newline at end of file
src/main/resources/application-dev.properties
......@@ -56,4 +56,15 @@ qbjc.interface.upload.token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJwYXlsb2FkIjo
qbjc.project.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/project/resource
qbjc.event.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/aplan/event
qbjc.event.tag.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/tag/event?project={1}
qbjc.platform.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/platform/resource
\ No newline at end of file
qbjc.platform.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/platform/resource
#\u5371\u673A\u5E93\u5916\u90E8\u63A5\u53E3
crisis.search.url=https://crisis.zhiweidata.com/app/brandkbs/crisisSearch?page={1}&size={2}&keyword={3}
crisis.searchTags.url=https://crisis.zhiweidata.com/app/brandkbs/searchCrisisByTags?page={1}&size={2}&brand={3}&category={4}
crisis.top3.url=https://crisis.zhiweidata.com/app/brandkbs/event/online-top3
crisis.searchCriteria.url=https://crisis.zhiweidata.com/app/brandkbs/condition
crisis.list.url=https://crisis.zhiweidata.com/app/brandkbs/crisisList?page={1}&size={2}&startTime={3}&endTime={4}&category={5}
crisis.share.url=https://crisis.zhiweidata.com/app/brandkbs/share/{1}
#\u70ED\u70B9\u5E93\u5916\u90E8\u63A5\u53E3
trends.longTimeInListSearchByInner.url=https://trends.zhiweidata.com/hotSearchTrend/inner/longTimeInListSearchByInner?sortType={1}&type={2}
trends.findHotSearchESDataInTimeByInner.url=https://hotsearch-manage.zhiweidata.com/hotsearch/hotSearch/findHotSearchESDataInTimeByInner?limit={1}&page={2}&type={3}&word={4}
trends.longTimeInListSearch.url=https://trends.zhiweidata.com/hotSearchTrend/search/longTimeInListSearch?type={1}&sortType=realTime
\ No newline at end of file
src/main/resources/application-prod.properties
......@@ -57,4 +57,15 @@ qbjc.interface.upload.token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJwYXlsb2FkIjo
qbjc.project.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/project/resource
qbjc.event.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/aplan/event
qbjc.event.tag.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/tag/event?project={1}
qbjc.platform.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/platform/resource
\ No newline at end of file
qbjc.platform.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/platform/resource
#\u5371\u673A\u5E93\u5916\u90E8\u63A5\u53E3
crisis.search.url=https://crisis.zhiweidata.com/app/brandkbs/crisisSearch?page={1}&size={2}&keyword={3}
crisis.searchTags.url=https://crisis.zhiweidata.com/app/brandkbs/searchCrisisByTags?page={1}&size={2}&brand={3}&category={4}
crisis.top3.url=https://crisis.zhiweidata.com/app/brandkbs/event/online-top3
crisis.searchCriteria.url=https://crisis.zhiweidata.com/app/brandkbs/condition
crisis.list.url=https://crisis.zhiweidata.com/app/brandkbs/crisisList?page={1}&size={2}&startTime={3}&endTime={4}&category={5}
crisis.share.url=https://crisis.zhiweidata.com/app/brandkbs/share/{1}
#\u70ED\u70B9\u5E93\u5916\u90E8\u63A5\u53E3
trends.longTimeInListSearchByInner.url=https://trends.zhiweidata.com/hotSearchTrend/inner/longTimeInListSearchByInner?sortType={1}&type={2}
trends.findHotSearchESDataInTimeByInner.url=https://hotsearch-manage.zhiweidata.com/hotsearch/hotSearch/findHotSearchESDataInTimeByInner?limit={1}&page={2}&type={3}&word={4}
trends.longTimeInListSearch.url=https://trends.zhiweidata.com/hotSearchTrend/search/longTimeInListSearch?type={1}&sortType=realTime
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment