Skip to content

B
brandkbs2
Commit a8a8ef17 by shenjunjie
Options

Merge branch 'feature' into 'release' 

Feature

See merge request !465
parents 2da615fd 44eb4486
Showing with 445 additions and 46 deletions
src/main/java/com/zhiwei/brandkbs2/common/GlobalPojo.java
......@@ -5,6 +5,7 @@ import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.pojo.ChannelTag;
import com.zhiwei.brandkbs2.pojo.HighlightWord;
import com.zhiwei.brandkbs2.pojo.Project;
import com.zhiwei.brandkbs2.pojo.SensitiveChannel;
import com.zhiwei.brandkbs2.service.ChannelService;
import com.zhiwei.brandkbs2.service.SystemInfoService;
import com.zhiwei.brandkbs2.util.Tools;
......@@ -92,6 +93,16 @@ public class GlobalPojo {
public static final String ELSE_PLATFORM_NAME = "其他自媒体";
/**
* 舆情项目重要渠道
*/
public static Map<String, Map<String, SensitiveChannel>> PROJECT_SENSITIVE_CHANNEL = new HashMap<>();
/**
* 舆情通用重要渠道
*/
public static Map<String, SensitiveChannel> COMMON_SENSITIVE_CHANNEL = new HashMap<>();
@PostConstruct
public void start() {
try {
......@@ -126,9 +137,11 @@ public class GlobalPojo {
PROJECT_MAP = systemInfoService.getProjects();
YU_QING_PROJECTS = systemInfoService.getYuQingProjects();
PROJECT_EMOTION_CHANNEL_DATA = channelService.getProjectEmotionChannelListData();
PROJECT_SENSITIVE_CHANNEL = systemInfoService.getProjectSensitiveChannel();
COMMON_SENSITIVE_CHANNEL = systemInfoService.getCommonSensitiveChannel();
updateHighlightGraphs();
log.info("{}-获取PLATFORMS-size:{},TAGS-size:{},LINKED_GROUP_ID_TAGS:{},CHANNEL_TAGS:{},MEDIA_TYPE:{},PROJECT_MAP:{},YUQING-PROJECTS-size:{},PROJECT_EMOTION_CHANNEL_DATA-size:{}", logMsg, PLATFORMS.size(), TAGS.size(),
LINKED_GROUP_ID_TAGS.size(), CHANNEL_TAGS.size(), MEDIA_TYPE.size(), PROJECT_MAP.size(), YU_QING_PROJECTS.size(), PROJECT_EMOTION_CHANNEL_DATA.size());
log.info("{}-获取PLATFORMS-size:{},TAGS-size:{},LINKED_GROUP_ID_TAGS:{},CHANNEL_TAGS:{},MEDIA_TYPE:{},PROJECT_MAP:{},YUQING-PROJECTS-size:{},PROJECT_EMOTION_CHANNEL_DATA-size:{},PROJECT_SENSITIVE_CHANNEL-size:{}, COMMON_SENSITIVE_CHANNEL-size:{}", logMsg, PLATFORMS.size(), TAGS.size(),
LINKED_GROUP_ID_TAGS.size(), CHANNEL_TAGS.size(), MEDIA_TYPE.size(), PROJECT_MAP.size(), YU_QING_PROJECTS.size(), PROJECT_EMOTION_CHANNEL_DATA.size(), PROJECT_SENSITIVE_CHANNEL.size(), COMMON_SENSITIVE_CHANNEL.size());
} catch (Exception e) {
log.info("{}-获取缓存值异常", logMsg, e);
}
......
src/main/java/com/zhiwei/brandkbs2/controller/app/AppArticleController.java
......@@ -526,6 +526,45 @@ public class AppArticleController extends BaseController {
return ResponseResult.success(markDataService.getYuqingAnalyzeDetail(markSearchDTO));
}
@ApiOperation("新-舆情分析-高频标题-基础信息")
@GetMapping("/analyze/frequent-title/info")
public ResponseResult getAggTitleBaseInfo(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "aggTitle") String aggTitle,
@RequestParam(value = "planId", required = false) String planId) {
return ResponseResult.success(markDataService.getAggTitleBaseInfo(aggTitle, startTime, endTime, planId));
}
@ApiOperation("新-舆情分析-高频标题-发布节点")
@GetMapping("/analyze/frequent-title/article-point")
public ResponseResult getArticlePoints(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "aggTitle") String aggTitle,
@RequestParam(value = "planId", required = false) String planId) {
return ResponseResult.success(markDataService.getAggTitleArticlePoints(aggTitle, startTime, endTime, planId));
}
@ApiOperation("新-舆情分析-高频标题-平台分布")
@GetMapping("/analyze/frequent-title/platform-percent")
public ResponseResult getAggTitlePlatformPercentage(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "aggTitle") String aggTitle,
@RequestParam(value = "planId", required = false) String planId) {
return ResponseResult.success(markDataService.getAggTitlePlatformPercentage(aggTitle, startTime, endTime, planId));
}
@ApiOperation("新-舆情分析-高频标题-发文列表")
@GetMapping("/analyze/frequent-title/articles")
public ResponseResult getArticleList(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "aggTitle") String aggTitle,
@RequestParam(value = "platform") String platform,
@RequestParam(value = "planId", required = false) String planId,
@RequestParam(value = "page", defaultValue = "1") Integer page,
@RequestParam(value = "pageSize", defaultValue = "10") Integer pageSize) {
return ResponseResult.success(markDataService.getArticleList(aggTitle, startTime, endTime, planId, platform, page, pageSize));
}
private boolean checkMTagIllegal(StringBuilder mtag) {
List<MarkerTag> hitTags = projectService.getProjectById(UserThreadLocal.getProjectId()).getHitTags();
if (!Tools.isEmpty(hitTags)) {
......
src/main/java/com/zhiwei/brandkbs2/dao/QbjcPojoDao.java
......@@ -51,4 +51,16 @@ public interface QbjcPojoDao {
* @return
*/
SensitiveChannel findSensitiveChannelBySource(String source);
/**
* qbjc项目重要渠道
* @return
*/
List<SensitiveChannel> findProjectSensitiveChannel();
/**
* qbjc通用重要渠道
* @return
*/
List<SensitiveChannel> findCommonSensitiveChannel();
}
src/main/java/com/zhiwei/brandkbs2/dao/impl/QbjcPojoDaoImpl.java
......@@ -55,5 +55,17 @@ public class QbjcPojoDaoImpl implements QbjcPojoDao {
query.addCriteria(Criteria.where("name").is(source));
return mongoTemplate.findOne(query, SensitiveChannel.class, "qbjc_sensitive_channel_system");
}
@Override
public List<SensitiveChannel> findProjectSensitiveChannel() {
Query query = new Query();
return mongoTemplate.find(query, SensitiveChannel.class, "qbjc_sensitive_channel");
}
@Override
public List<SensitiveChannel> findCommonSensitiveChannel() {
Query query = new Query();
return mongoTemplate.find(query, SensitiveChannel.class, "qbjc_sensitive_channel_system");
}
}
src/main/java/com/zhiwei/brandkbs2/service/ChannelService.java
......@@ -308,4 +308,6 @@ public interface ChannelService {
* @throws IOException
*/
Map<String, JSONObject> getProjectEmotionChannelListData() throws IOException;
JSONObject matchYuQingSensitiveChannel(String linkedGroupId, String source);
}
src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java
......@@ -634,4 +634,46 @@ public interface MarkDataService {
* @return
*/
List<JSONObject> getLastNews(Long startTime, Long endTime, String planId, int size, boolean include);
/**
* 舆情分析-高频标题详情页-基础信息
* @param aggTitle
* @param startTime
* @param endTime
* @param planId
* @return
*/
JSONObject getAggTitleBaseInfo(String aggTitle, Long startTime, Long endTime, String planId);
/**
* 舆情分析-高频标题详情页-发布节点
* @param aggTitle
* @param startTime
* @param endTime
* @param planId
* @return
*/
List<JSONObject> getAggTitleArticlePoints(String aggTitle, Long startTime, Long endTime, String planId);
/**
* 舆情分析-高频标题详情页-平台分布
* @param aggTitle
* @param startTime
* @param endTime
* @param planId
* @return
*/
List<JSONObject> getAggTitlePlatformPercentage(String aggTitle, Long startTime, Long endTime, String planId);
/**
* 舆情分析-高频标题详情页-发文列表
* @param aggTitle
* @param startTime
* @param endTime
* @param planId
* @param page
* @param pageSize
* @return
*/
PageVO<JSONObject> getArticleList(String aggTitle, Long startTime, Long endTime, String planId, String platform, int page, int pageSize);
}
src/main/java/com/zhiwei/brandkbs2/service/SystemInfoService.java
......@@ -3,6 +3,7 @@ package com.zhiwei.brandkbs2.service;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.pojo.ChannelTag;
import com.zhiwei.brandkbs2.pojo.Project;
import com.zhiwei.brandkbs2.pojo.SensitiveChannel;
import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform;
import com.zhiwei.qbjc.bean.pojo.common.Tag;
......@@ -58,4 +59,15 @@ public interface SystemInfoService {
List<JSONObject> getYuQingProjects();
/**
* 从舆情系统获取项目重要渠道
* @return
*/
Map<String, Map<String, SensitiveChannel>> getProjectSensitiveChannel();
/**
* 从舆情系统获取通用重要渠道
* @return
*/
Map<String, SensitiveChannel> getCommonSensitiveChannel();
}
src/main/java/com/zhiwei/brandkbs2/service/impl/ChannelServiceImpl.java
......@@ -731,15 +731,18 @@ public class ChannelServiceImpl implements ChannelService {
* @param source
* @return
*/
private JSONObject matchYuQingSensitiveChannel(String linkedGroupId, String source){
@Override
public JSONObject matchYuQingSensitiveChannel(String linkedGroupId, String source){
JSONObject sensitiveChannelInfo = new JSONObject();
SensitiveChannel sensitiveChannel;
// 先匹配舆情项目重要渠道，未匹配上时转而匹配舆情通用重要渠道
SensitiveChannel projectSensitiveChannel = qbjcPojoDao.findSensitiveChannelBySource(linkedGroupId, source);
if (Objects.nonNull(projectSensitiveChannel)) {
sensitiveChannel = projectSensitiveChannel;
Map<String, SensitiveChannel> sensitiveChannelMap = GlobalPojo.PROJECT_SENSITIVE_CHANNEL.get(linkedGroupId);
// SensitiveChannel projectSensitiveChannel = GlobalPojo.PROJECT_SENSITIVE_CHANNEL.get(linkedGroupId).get(source);
if (Objects.nonNull(sensitiveChannelMap) && Objects.nonNull(sensitiveChannelMap.get(source))) {
sensitiveChannel = sensitiveChannelMap.get(source);
}else {
sensitiveChannel = qbjcPojoDao.findSensitiveChannelBySource(source);
// sensitiveChannel = qbjcPojoDao.findSensitiveChannelBySource(source);
sensitiveChannel = GlobalPojo.COMMON_SENSITIVE_CHANNEL.get(source);
}
if (Objects.nonNull(sensitiveChannel)){
sensitiveChannelInfo.put("mainBodyType", sensitiveChannel.getMainBodyType());
......
src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java
......@@ -38,6 +38,7 @@ import com.zhiwei.brandkbs2.util.MongoUtil;
import com.zhiwei.brandkbs2.util.RedisUtil;
import com.zhiwei.brandkbs2.util.TextUtil;
import com.zhiwei.brandkbs2.util.Tools;
import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.ListUtils;
import org.apache.commons.lang3.StringUtils;
......@@ -45,6 +46,7 @@ import org.apache.commons.lang3.time.DateUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ElasticsearchStatusException;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
......@@ -180,6 +182,9 @@ public class MarkDataServiceImpl implements MarkDataService {
@Resource(name = "systemInfoServiceImpl")
SystemInfoService systemInfoService;
@Resource(name = "channelServiceImpl")
ChannelService channelService;
@Value("${brandkbs.file.url}")
private String brandkbsFilePath;
......@@ -871,8 +876,14 @@ public class MarkDataServiceImpl implements MarkDataService {
if (null != emotion && !Objects.equals(emotion, EmotionEnum.ALL.getName())) {
query.must(QueryBuilders.termQuery("brandkbs_mark_cache_maps.name.keyword", emotion));
}
SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, query,
aggregationBuilder.subAggregation(sourceAggregationBuilder), null, null, 0, 0, null);
SearchResponse searchResponse;
// 单独处理buckets超过上限异常，出错时按时间分段查询
try {
searchResponse = esClientDao.searchResponse(indexes, null, query,
aggregationBuilder.subAggregation(sourceAggregationBuilder), null, null, 0, 0, null);
}catch (ElasticsearchStatusException e){
return getMarkTopTitleDivided(query, indexes, aggregationBuilder, startTime, endTime, size);
}
List<JSONObject> res = new ArrayList<>();
Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
ParsedStringTerms teamAgg = (ParsedStringTerms) aggMap.get("titles");
......@@ -883,7 +894,6 @@ public class MarkDataServiceImpl implements MarkDataService {
if ("分享一篇文章".equals(title)) {
continue;
}
// result.merge(title, num, Integer::sum);
jsonObject.put("title", title);
jsonObject.put("num", bucket.getDocCount());
Map<String, Aggregation> aggregationMap = bucket.getAggregations().asMap();
......@@ -899,6 +909,59 @@ public class MarkDataServiceImpl implements MarkDataService {
return res;
}
/**
* getMarkTopTitle按时间分段查询
* @param query
* @param indexes
* @param aggregationBuilder
* @param startTime
* @param endTime
* @param size
* @return
* @throws IOException
*/
private List<JSONObject> getMarkTopTitleDivided(BoolQueryBuilder query, String[] indexes, TermsAggregationBuilder aggregationBuilder, Long startTime, Long endTime, int size) throws IOException {
List<Long[]> timeRange = Tools.cutTimeRange(startTime, endTime, Constant.ONE_DAY);
Map<String, JSONObject> map = new HashMap<>();
for (Long[] time : timeRange) {
TermsAggregationBuilder sourceAggregationBuilder = AggregationBuilders.terms(Tools.concat(time[0], time[1])).field("source").size(10000);
query.must(QueryBuilders.rangeQuery("time").gte(time[0]).lt(time[1]));
SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, query,
aggregationBuilder.subAggregation(sourceAggregationBuilder), null, null, 0, 0, null);
Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
ParsedStringTerms teamAgg = (ParsedStringTerms) aggMap.get("titles");
for (Terms.Bucket bucket : teamAgg.getBuckets()) {
JSONObject jsonObject = new JSONObject();
String title = bucket.getKeyAsString();
// 过滤 “分享一篇文章” 的标题
if ("分享一篇文章".equals(title)) {
continue;
}
jsonObject.put("title", title);
jsonObject.put("num", bucket.getDocCount());
Map<String, Aggregation> aggregationMap = bucket.getAggregations().asMap();
ParsedStringTerms sourceAgg = (ParsedStringTerms) aggregationMap.get(Tools.concat(time[0], time[1]));
jsonObject.put("sourceCount", sourceAgg.getBuckets().size());
List<String> sources = new ArrayList<>();
for (Terms.Bucket sourceBucket : sourceAgg.getBuckets()) {
sources.add(sourceBucket.getKeyAsString());
}
jsonObject.put("sources", sources);
// 合并map
map.merge(title, jsonObject, (v1, v2) -> {
v1.put("num", v1.getLongValue("num") + v2.getLongValue("num"));
v1.put("sourceCount", v1.getIntValue("sourceCount") + v2.getIntValue("sourceCount"));
v1.put("sources", v1.getJSONArray("sources").addAll(v2.getJSONArray("sources")));
return v1;
});
}
}
return map.values().stream()
.filter(Objects::nonNull)
.sorted(Comparator.comparingLong((JSONObject json) -> json.getLongValue("num")).reversed())
.limit(size + 1).collect(Collectors.toList());
}
@Override
public BaseMap getFirstArticle(Long startTime, Long endTime, String aggTitle, String projectId, String contendId) throws IOException {
return getFirstArticle(startTime, endTime, aggTitle, projectId, contendId, true);
......@@ -910,6 +973,14 @@ public class MarkDataServiceImpl implements MarkDataService {
}
private BaseMap getFirstArticle(Long startTime, Long endTime, String aggTitle, String projectId, String contendId, String planId, boolean include) throws IOException{
return getArticleSort(startTime, endTime, aggTitle, projectId, contendId, planId, include, "{\"time\" : \"asc\"}");
}
private BaseMap getLastArticle(Long startTime, Long endTime, String aggTitle, String projectId, String contendId, String planId, boolean include) throws IOException {
return getArticleSort(startTime, endTime, aggTitle, projectId, contendId, planId, include, "{\"time\" : \"desc\"}");
}
private BaseMap getArticleSort(Long startTime, Long endTime, String aggTitle, String projectId, String contendId, String planId, boolean include, String sorter) throws IOException {
// 索引
String[] indexes = esClientDao.getIndexes();
// postFilter
......@@ -927,6 +998,14 @@ public class MarkDataServiceImpl implements MarkDataService {
postFilter.must(QueryBuilders.termQuery("agg_title.keyword", aggTitle));
//sort
FieldSortBuilder sort = new FieldSortBuilder("time").order(SortOrder.ASC);
JSONObject sortJson = JSONObject.parseObject(sorter);
for (Map.Entry<String, Object> entry : sortJson.entrySet()) {
if (entry.getValue().toString().contains("desc")) {
sort = SortBuilders.fieldSort(entry.getKey()).order(SortOrder.DESC);
} else {
sort = SortBuilders.fieldSort(entry.getKey()).order(SortOrder.ASC);
}
}
//hits
SearchHits hits = esClientDao.searchHits(indexes, postFilter, null, null, sort, 0, 1, null);
if (0 == hits.getTotalHits().value) {
......@@ -2407,7 +2486,7 @@ public class MarkDataServiceImpl implements MarkDataService {
String projectId = UserThreadLocal.getProjectId();
Long total = getYuqingAnalyzeCount(startTime, endTime, planId, importantPlatforms);
// 平台聚合
SearchResponse searchResponse = platformAggSearchResponse(startTime, endTime, planId, importantPlatforms);
SearchResponse searchResponse = platformAggSearchResponse(startTime, endTime, planId, importantPlatforms, null);
Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
ParsedStringTerms teamAgg = (ParsedStringTerms) aggMap.get("platform_count");
List<? extends Terms.Bucket> buckets = teamAgg.getBuckets();
......@@ -2455,33 +2534,7 @@ public class MarkDataServiceImpl implements MarkDataService {
@Override
public List<JSONObject> getPlatformPercentage(Long startTime, Long endTime, String planId) {
try {
List<String> platforms = commonService.getQbjcPlatformNames();
List<JSONObject> list = new ArrayList<>();
// query
BoolQueryBuilder query = yuqingAnalyzeQuery(startTime, endTime, planId, null, platforms);
// total
Long total = esClientDao.count(query);
// 平台聚合
SearchResponse searchResponse = platformAggSearchResponse(startTime, endTime, planId, platforms);
Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
ParsedStringTerms teamAgg = (ParsedStringTerms) aggMap.get("platform_count");
List<? extends Terms.Bucket> buckets = teamAgg.getBuckets();
buckets.forEach(bucket -> {
JSONObject platformResult = new JSONObject();
platformResult.put("platform", GlobalPojo.getPlatformNameById(bucket.getKeyAsString()));
platformResult.put("percent", 0 == total ? 0d : bucket.getDocCount() / (double) total);
list.add(platformResult);
});
// 取占比最高的前9
List<JSONObject> res = list.stream()
.sorted(Comparator.comparingDouble((JSONObject jsonObject) -> jsonObject.getDoubleValue("percent"))
.reversed()).limit(9).collect(Collectors.toList());
// 第10个算做其他平台
JSONObject other = new JSONObject();
other.put("platform", "其他");
other.put("percent", 1 - res.stream().mapToDouble(count -> count.getDoubleValue("percent")).sum());
res.add(other);
return res;
return getPlatformPercentage(null, startTime, endTime, planId);
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "新舆情分析getPlatformPercentage异常-", e);
}
......@@ -2489,6 +2542,45 @@ public class MarkDataServiceImpl implements MarkDataService {
}
/**
* 获取舆情分析平台分布
* @param aggTitle
* @param startTime
* @param endTime
* @param planId
* @return
* @throws IOException
*/
private List<JSONObject> getPlatformPercentage(String aggTitle, Long startTime, Long endTime, String planId) throws IOException {
List<String> platforms = commonService.getQbjcPlatformNames();
List<JSONObject> list = new ArrayList<>();
// query
BoolQueryBuilder query = yuqingAnalyzeQuery(startTime, endTime, null, planId, null, null, aggTitle);
// total
Long total = esClientDao.count(query);
// 平台聚合
SearchResponse searchResponse = platformAggSearchResponse(startTime, endTime, planId, platforms, aggTitle);
Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
ParsedStringTerms teamAgg = (ParsedStringTerms) aggMap.get("platform_count");
List<? extends Terms.Bucket> buckets = teamAgg.getBuckets();
buckets.forEach(bucket -> {
JSONObject platformResult = new JSONObject();
platformResult.put("platform", GlobalPojo.getPlatformNameById(bucket.getKeyAsString()));
platformResult.put("percent", 0 == total ? 0d : bucket.getDocCount() / (double) total);
list.add(platformResult);
});
// 取占比最高的前9
List<JSONObject> res = list.stream()
.sorted(Comparator.comparingDouble((JSONObject jsonObject) -> jsonObject.getDoubleValue("percent"))
.reversed()).limit(9).collect(Collectors.toList());
// 第10个算做其他平台
JSONObject other = new JSONObject();
other.put("platform", "其他");
other.put("percent", 1 - res.stream().mapToDouble(count -> count.getDoubleValue("percent")).sum());
res.add(other);
return res;
}
/**
* 平台聚合查询
* @param startTime
* @param endTime
......@@ -2497,13 +2589,18 @@ public class MarkDataServiceImpl implements MarkDataService {
* @return
* @throws IOException
*/
private SearchResponse platformAggSearchResponse(Long startTime, Long endTime, String planId, List<String> platforms) throws IOException {
private SearchResponse platformAggSearchResponse(Long startTime, Long endTime, String planId, List<String> platforms, String aggTitle) throws IOException {
// 索引
String[] indexes = esClientDao.getIndexes();
// 聚合请求
TermsAggregationBuilder platformAggregationBuilder = AggregationBuilders.terms("platform_count").field("platform_id").order(BucketOrder.count(false));
// query
BoolQueryBuilder query = yuqingAnalyzeQuery(startTime, endTime, planId, null, platforms);
BoolQueryBuilder query;
if (Objects.isNull(aggTitle)){
query = yuqingAnalyzeQuery(startTime, endTime, planId, null, platforms);
}else {
query = yuqingAnalyzeQuery(startTime, endTime, null, planId, null, null, aggTitle);
}
return esClientDao.searchResponse(indexes, null, query, platformAggregationBuilder, null, null, 0, 0, null);
}
......@@ -2742,8 +2839,8 @@ public class MarkDataServiceImpl implements MarkDataService {
@Override
public List<JSONObject> getLastNews(Long startTime, Long endTime, String planId, int size, boolean include) {
String projectId = UserThreadLocal.getProjectId();
try {
String projectId = UserThreadLocal.getProjectId();
List<JSONObject> markTopTitleList = getMarkTopTitle(startTime, endTime, null, projectId, Constant.PRIMARY_CONTEND_ID, planId, size);
CompletableFuture.allOf(markTopTitleList.stream().map(json -> CompletableFuture.supplyAsync(() -> {
try {
......@@ -2759,11 +2856,150 @@ public class MarkDataServiceImpl implements MarkDataService {
}, executor)).toArray(CompletableFuture[]::new)).join();
return markTopTitleList.stream().filter(Objects::nonNull).limit(size).collect(Collectors.toList());
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "新舆情分析getLastNews异常-", e);
log.error("新舆情分析getLastNews异常-projectId:{}", projectId,e);
}
return Collections.emptyList();
}
@Override
public JSONObject getAggTitleBaseInfo(String aggTitle, Long startTime, Long endTime, String planId) {
JSONObject res = new JSONObject();
try {
String projectId = UserThreadLocal.getProjectId();
String linkedGroupId = projectService.getProjectById(projectId).getBrandLinkedGroupId();
CompletableFuture<JSONObject> firstArticleFuture = CompletableFuture.supplyAsync(() -> {
try {
// 最早发布
BaseMap firstArticle = getFirstArticle(startTime, endTime, aggTitle, projectId, Constant.PRIMARY_CONTEND_ID, planId, true);
// 匹配重要渠道
JSONObject firstSensitiveChannel = channelService.matchYuQingSensitiveChannel(linkedGroupId, firstArticle.getSource());
JSONObject firstArticleJson = JSONObject.parseObject(JSONObject.toJSONString(firstArticle));
firstArticleJson.put("politicsLevel", Objects.nonNull(firstSensitiveChannel) ? firstSensitiveChannel.getString("politicsLevel") : null);
return firstArticleJson;
} catch (IOException ignore) {
}
return null;
}, executor);
CompletableFuture<JSONObject> lastArticleFuture = CompletableFuture.supplyAsync(() -> {
try {
// 最新发布
BaseMap lastArticle = getLastArticle(startTime, endTime, aggTitle, projectId, Constant.PRIMARY_CONTEND_ID, planId, true);
// 匹配重要渠道
JSONObject lastSensitiveChannel = channelService.matchYuQingSensitiveChannel(linkedGroupId, lastArticle.getSource());
JSONObject lastArticleJson = JSONObject.parseObject(JSONObject.toJSONString(lastArticle));
lastArticleJson.put("politicsLevel", Objects.nonNull(lastSensitiveChannel) ? lastSensitiveChannel.getString("politicsLevel") : null);
return lastArticleJson;
} catch (IOException ignore) {
}
return null;
}, executor);
CompletableFuture.allOf(firstArticleFuture, lastArticleFuture).join();
JSONObject firstArticle = firstArticleFuture.join();
JSONObject lastArticle = lastArticleFuture.join();
res.put("title", firstArticle.getString("title"));
res.put("emotion", firstArticle.getString("emotion"));
res.put("content", firstArticle.getString("content"));
res.put("firstArticle", firstArticle);
res.put("lastArticle", lastArticle);
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "高频标题详情页getAggTitleBaseInfo异常-", e);
}
return res;
}
@Override
public List<JSONObject> getAggTitleArticlePoints(String aggTitle, Long startTime, Long endTime, String planId) {
List<JSONObject> jsonObject = new ArrayList<>();
try {
String[] indexes = esClientDao.getIndexes();
// query
BoolQueryBuilder query = yuqingAnalyzeQuery(startTime, endTime, null, planId, null, null, aggTitle);
DateHistogramAggregationBuilder daysAggregationBuilder =
AggregationBuilders.dateHistogram("timeAgg").field("time").calendarInterval(DateHistogramInterval.DAY);
TermsAggregationBuilder sourceAggregationBuilder = AggregationBuilders.terms("sourceAgg").field("source").size(10000);
// response
SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, query,
daysAggregationBuilder.subAggregation(sourceAggregationBuilder), null, null, 0, 0, null);
Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
ParsedDateHistogram teamAgg = (ParsedDateHistogram) aggMap.get("timeAgg");
List<? extends Histogram.Bucket> buckets = teamAgg.getBuckets();
// 重要渠道
String linkedGroupId = projectService.getProjectById(UserThreadLocal.getProjectId()).getBrandLinkedGroupId();
Map<String, SensitiveChannel> sensitiveChannelMap = GlobalPojo.PROJECT_SENSITIVE_CHANNEL.get(linkedGroupId);
// 项目重要渠道未匹配 转而获取通用重要渠道
if (Objects.isNull(sensitiveChannelMap)){
sensitiveChannelMap = GlobalPojo.COMMON_SENSITIVE_CHANNEL;
}
Map<String, SensitiveChannel> finalSensitiveChannelMap = sensitiveChannelMap;
buckets.forEach(bucket -> {
JSONObject point = new JSONObject();
point.put("time", Long.parseLong(bucket.getKeyAsString()));
// 发文渠道
Map<String, Aggregation> sourceMap = bucket.getAggregations().asMap();
ParsedStringTerms countTeam = (ParsedStringTerms) sourceMap.get("sourceAgg");
List<? extends Terms.Bucket> sourceBuckets = countTeam.getBuckets();
List<JSONObject> sources = new ArrayList<>();
sourceBuckets.forEach(sourceBucket -> {
JSONObject source = new JSONObject();
source.put("name", sourceBucket.getKeyAsString());
// 重要渠道判断
source.put("isImportant", finalSensitiveChannelMap.containsKey(sourceBucket.getKeyAsString()));
sources.add(source);
});
point.put("sources", sources);
point.put("count", sources.size());
jsonObject.add(point);
});
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "高频标题详情页articlePoints异常-", e);
}
return jsonObject;
}
@Override
public List<JSONObject> getAggTitlePlatformPercentage(String aggTitle, Long startTime, Long endTime, String planId) {
try {
return getPlatformPercentage(aggTitle, startTime, endTime, planId);
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "新舆情分析getAggTitlePlatformPercentage异常-", e);
}
return Collections.emptyList();
}
@Override
public PageVO<JSONObject> getArticleList(String aggTitle, Long startTime, Long endTime, String planId, String platform, int page, int pageSize) {
try {
String[] indexes = esClientDao.getIndexes();
// query
BoolQueryBuilder query = yuqingAnalyzeQuery(startTime, endTime, null, planId, null, Collections.singletonList(platform), aggTitle);
if (Objects.equals("其他", platform)){
List<String> platforms = getPlatformPercentage(aggTitle, startTime, endTime, planId).stream().map(json -> json.getString("platform")).collect(Collectors.toList());
List<String> allPlatforms = GlobalPojo.PLATFORMS.stream().map(MessagePlatform::getName).collect(Collectors.toList());
allPlatforms.removeAll(platforms);
query = yuqingAnalyzeQuery(startTime, endTime, null, planId, null, allPlatforms, aggTitle);
}
// sort
FieldSortBuilder sort = SortBuilders.fieldSort("time").order(SortOrder.DESC);
// response
SearchHits searchHits = esClientDao.searchHits(indexes, query, null, null, sort, (page - 1) * pageSize, pageSize, null);
long total = searchHits.getTotalHits().value;
List<JSONObject> list = Arrays.stream(searchHits.getHits()).map(hit -> {
BaseMap baseMap = Tools.getBaseFromEsMap(hit.getSourceAsMap());
JSONObject jsonObject = new JSONObject();
jsonObject.put("source", baseMap.getSource());
jsonObject.put("platform", baseMap.getPlatform());
jsonObject.put("title", baseMap.getTitle());
jsonObject.put("time", baseMap.getTime());
jsonObject.put("url", baseMap.getUrl());
return jsonObject;
}).collect(Collectors.toList());
return PageVO.createPageVo(total, page, pageSize, list);
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "新舆情分析getArticleList异常-", e);
}
return PageVO.createPageVo(0, page, pageSize, Collections.emptyList());
}
/**
* 微博、微信、抖音、小红书平台趋势图
* @param startTime
......@@ -2868,15 +3104,15 @@ public class MarkDataServiceImpl implements MarkDataService {
}
private Long getYuqingAnalyzeCount(Long startTime, Long endTime, String projectId, String planId, String emotion, List<String> platforms) throws IOException {
BoolQueryBuilder query = yuqingAnalyzeQuery(startTime, endTime, projectId, planId, emotion, platforms);
BoolQueryBuilder query = yuqingAnalyzeQuery(startTime, endTime, projectId, planId, emotion, platforms, null);
return esClientDao.count(query);
}
private BoolQueryBuilder yuqingAnalyzeQuery(Long startTime, Long endTime, String planId, String emotion, List<String> platforms){
return yuqingAnalyzeQuery(startTime, endTime, null, planId, emotion, platforms);
return yuqingAnalyzeQuery(startTime, endTime, null, planId, emotion, platforms, null);
}
private BoolQueryBuilder yuqingAnalyzeQuery(Long startTime, Long endTime, String projectId, String planId, String emotion, List<String> platforms){
private BoolQueryBuilder yuqingAnalyzeQuery(Long startTime, Long endTime, String projectId, String planId, String emotion, List<String> platforms, String aggTitle){
if (Objects.isNull(projectId)) {
projectId = UserThreadLocal.getProjectId();
}
......@@ -2891,7 +3127,7 @@ public class MarkDataServiceImpl implements MarkDataService {
}
// platform
if (CollectionUtils.isNotEmpty(platforms) && Objects.nonNull(platforms.get(0))){
List<String> platformIds = new ArrayList<>(6);
List<String> platformIds = new ArrayList<>();
for (String platform : platforms) {
String platformId = GlobalPojo.getPlatformIdByName(platform);
platformIds.add(platformId);
......@@ -2906,6 +3142,10 @@ public class MarkDataServiceImpl implements MarkDataService {
if (Objects.nonNull(endTime)) {
query.must(QueryBuilders.rangeQuery("time").lt(endTime));
}
// aggTitle
if (Objects.nonNull(aggTitle)){
query.must(QueryBuilders.termQuery("agg_title.keyword", aggTitle));
}
return query;
}
......
src/main/java/com/zhiwei/brandkbs2/service/impl/SystemInfoServiceImpl.java
......@@ -10,6 +10,7 @@ import com.zhiwei.brandkbs2.dao.QbjcPojoDao;
import com.zhiwei.brandkbs2.pojo.ChannelTag;
import com.zhiwei.brandkbs2.pojo.MediaType;
import com.zhiwei.brandkbs2.pojo.Project;
import com.zhiwei.brandkbs2.pojo.SensitiveChannel;
import com.zhiwei.brandkbs2.service.SystemInfoService;
import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform;
import com.zhiwei.qbjc.bean.pojo.common.Tag;
......@@ -113,4 +114,22 @@ public class SystemInfoServiceImpl implements SystemInfoService {
return res;
}).collect(Collectors.toList());
}
@Override
public Map<String, Map<String, SensitiveChannel>> getProjectSensitiveChannel() {
Map<String, Map<String, SensitiveChannel>> res = new HashMap<>();
List<SensitiveChannel> projectSensitiveChannel = qbjcPojoDao.findProjectSensitiveChannel();
Map<String, List<SensitiveChannel>> projectMap = projectSensitiveChannel.stream().collect(Collectors.groupingBy(SensitiveChannel::getProjectId));
for (Map.Entry<String, List<SensitiveChannel>> entry : projectMap.entrySet()) {
Map<String, SensitiveChannel> sourceMap = entry.getValue().stream().collect(Collectors.toMap(SensitiveChannel::getName, o-> o, (k1, k2) -> k1));
res.put(entry.getKey(), sourceMap);
}
return res;
}
@Override
public Map<String, SensitiveChannel> getCommonSensitiveChannel() {
List<SensitiveChannel> commonSensitiveChannel = qbjcPojoDao.findCommonSensitiveChannel();
return commonSensitiveChannel.stream().collect(Collectors.toMap(SensitiveChannel::getName, o-> o, (k1, k2) -> k1));
}
}
src/main/java/com/zhiwei/brandkbs2/service/impl/UserServiceImpl.java
......@@ -235,6 +235,7 @@ public class UserServiceImpl implements UserService {
OptionalInt.of(userDTO.getExportAmount()).ifPresent(userRole::setExportAmount);
userDao.updateOneByIdWithField(userDTO.getId(), new Update().set("roles", roles));
});
Optional.of(userDTO.getNickname()).ifPresent(nickName -> userDao.updateOneByIdWithField(userDTO.getId(), new Update().set("nickname", nickName)));
}
@Override
......
src/main/java/com/zhiwei/brandkbs2/util/Tools.java
......@@ -319,6 +319,10 @@ public class Tools {
public static List<Long[]> cutTimeRange(long startTime, long endTime, long range) {
List<Long[]> res = new ArrayList<>();
if (endTime - startTime <= range){
res.add(new Long[]{startTime, endTime});
return res;
}
while (startTime < endTime) {
long rangeEnd = Math.min(startTime + range, endTime);
res.add(new Long[]{startTime, rangeEnd});
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment