Skip to content

B
brandkbs2
Commit fd799dfb by 陈健智
Options

原发追溯

parent e9c73d49
Showing with 471 additions and 15 deletions
src/main/java/com/zhiwei/brandkbs2/common/GenericAttribute.java
......@@ -22,6 +22,11 @@ public class GenericAttribute {
// public static final String ES_INDEX_TEST = "brandkbs2_2022";
public static final String ES_CHANNEL_INDEX_TEST = "brandkbs2_channel_record_test";
public static final String ES_CHANNEL_INDEX_PREFIX = "brandkbs2_channel_record_";
public static final String ES_COMPLETE_TEXT_INDEX_PRE = "complete_text_";
public static final String ES_INCOMPLETE_TEXT_INDEX_PRE = "incomplete_text_";
public static final String ES_QA_TEXT_INDEX_PRE = "qa_text_";
public static final String ES_VIDEO_INDEX_PRE = "video_";
/**
* es ind_title
**/
......
src/main/java/com/zhiwei/brandkbs2/config/Constant.java
......@@ -34,6 +34,8 @@ public class Constant {
public static final FastDateFormat ONLY_MONTH_FORMAT = FastDateFormat.getInstance("M月");
public static final FastDateFormat DF_yyyyMMdd = FastDateFormat.getInstance("yyyyMMdd");
public static final FastDateFormat DF_yyyyMM = FastDateFormat.getInstance("yyyyMM");
/**
* 自定义fid分隔符号
*/
......
src/main/java/com/zhiwei/brandkbs2/controller/app/AppArticleController.java
......@@ -553,6 +553,36 @@ public class AppArticleController extends BaseController {
return ResponseResult.success(markDataService.getDailyReportDetail(id));
}
@ApiOperation("原发溯源-首发信息")
@PostMapping("/search-whole/origin/first-article")
public ResponseResult getWholeSearchFirstArticle(@RequestBody MarkSearchDTO dto) {
return ResponseResult.success(markDataService.getWholeSearchFirstArticle(dto));
}
@ApiOperation("原发溯源-媒体参与")
@PostMapping("/search-whole/origin/media-info")
public ResponseResult getWholeSearchMediaParticipation(@RequestBody MarkSearchDTO dto) {
return ResponseResult.success(markDataService.getWholeSearchMediaParticipation(dto));
}
@ApiOperation("原发溯源-发布节点")
@PostMapping("/search-whole/origin/article-point")
public ResponseResult getWholeSearchArticlePoint(@RequestBody MarkSearchDTO dto) {
return ResponseResult.success(markDataService.getWholeSearchArticlePoint(dto));
}
@ApiOperation("原发溯源-平台分布")
@PostMapping("/search-whole/origin/platform-percent")
public ResponseResult getWholeSearchPlatformPercentage(@RequestBody MarkSearchDTO dto) {
return ResponseResult.success(markDataService.getWholeSearchPlatformPercentage(dto));
}
@ApiOperation("原发溯源-发文列表")
@PostMapping("/search-whole/origin/articles")
public ResponseResult getWholeSearchArticleList(@RequestBody MarkSearchDTO dto) {
return ResponseResult.success(markDataService.getWholeSearchArticleList(dto));
}
private boolean checkMTagIllegal(StringBuilder mtag) {
List<MarkerTag> hitTags = projectService.getProjectById(UserThreadLocal.getProjectId()).getHitTags();
if (!Tools.isEmpty(hitTags)) {
......
src/main/java/com/zhiwei/brandkbs2/es/EsClientDao.java
......@@ -7,6 +7,7 @@ import com.zhiwei.brandkbs2.pojo.ChannelIndex;
import com.zhiwei.brandkbs2.util.Tools;
import lombok.Getter;
import lombok.Setter;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.commons.lang3.time.FastDateFormat;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.logging.log4j.LogManager;
......@@ -29,6 +30,8 @@ import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.sort.FieldSortBuilder;
import org.joda.time.Period;
import org.joda.time.PeriodType;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.retry.support.RetryTemplate;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
......@@ -195,6 +198,10 @@ public class EsClientDao {
return getIndexList().toArray(new String[0]);
}
public String[] getAllIndexes(Long startTime, Long endTime) {
return getAllIndexList(startTime, endTime).toArray(new String[0]);
}
protected RestHighLevelClient getEsClient() {
return esClient;
}
......@@ -214,6 +221,25 @@ public class EsClientDao {
return res;
}
public List<String> getAllIndexList(Long start, Long end) {
List<String> list = new ArrayList<>();
Date endTime = new Date(end);
Date startTime = new Date(start);
String pattern = "yyyyMM";
startTime = Tools.truncDate(startTime, pattern);
endTime = Tools.truncDate(endTime, pattern);
Period period = new Period(startTime.getTime(), endTime.getTime(), PeriodType.months());
int mouths = period.getMonths();
for (int i = 0; i <= mouths; i++) {
String suffix = Constant.DF_yyyyMM.format(DateUtils.addMonths(startTime, i));
list.add(GenericAttribute.ES_COMPLETE_TEXT_INDEX_PRE + suffix);
list.add(GenericAttribute.ES_INCOMPLETE_TEXT_INDEX_PRE + suffix);
list.add(GenericAttribute.ES_QA_TEXT_INDEX_PRE + suffix);
list.add(GenericAttribute.ES_VIDEO_INDEX_PRE + suffix);
}
return list;
}
// private SearchSourceBuilder addSort(SearchSourceBuilder builder, String sorter) {
// if (StringUtils.isNotEmpty(sorter)) {
// for (Map.Entry<String, Object> entry : JSONObject.parseObject(sorter).entrySet()) {
......
src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java
......@@ -715,4 +715,39 @@ public interface MarkDataService {
* @return
*/
DailyReport getDailyReportDetail(String id);
/**
* 获取原发溯源-首发信息
* @param dto 全网搜数据搜索传输类
* @return
*/
MarkFlowEntity getWholeSearchFirstArticle(MarkSearchDTO dto);
/**
* 获取原发溯源-平台分布
* @param dto 全网搜数据搜索传输类
* @return
*/
List<JSONObject> getWholeSearchMediaParticipation(MarkSearchDTO dto);
/**
* 获取原发溯源-发布节点
* @param dto 全网搜数据搜索传输类
* @return
*/
List<JSONObject> getWholeSearchArticlePoint(MarkSearchDTO dto);
/**
* 获取原发溯源-平台分布
* @param dto 全网搜数据搜索传输类
* @return
*/
List<JSONObject> getWholeSearchPlatformPercentage(MarkSearchDTO dto);
/**
* 获取原发溯源-发文列表
* @param dto 全网搜数据搜索传输类
* @return
*/
PageVO<JSONObject> getWholeSearchArticleList(MarkSearchDTO dto);
}
src/main/java/com/zhiwei/brandkbs2/service/MarkFlowService.java
......@@ -3,6 +3,7 @@ package com.zhiwei.brandkbs2.service;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.base.entity.subclass.mark.MarkInfo;
import com.zhiwei.brandkbs2.pojo.MarkFlowEntity;
import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
import com.zhiwei.brandkbs2.pojo.dto.TagFilterDTO;
import lombok.AllArgsConstructor;
import lombok.Data;
......@@ -74,6 +75,14 @@ public interface MarkFlowService {
*/
MarkFlowEntity getShotPageFromCache(String id);
/**
* 获取大库数据实体
* @param json
* @param dto
* @return
*/
MarkFlowEntity createAllIndexFlowInfo(JSONObject json, MarkSearchDTO dto);
@Data
@AllArgsConstructor
class MarkInfoSource {
......
src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java
......@@ -36,6 +36,7 @@ import com.zhiwei.brandkbs2.util.RedisUtil;
import com.zhiwei.brandkbs2.util.TextUtil;
import com.zhiwei.brandkbs2.util.Tools;
import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform;
import com.zhiwei.qbjc.bean.tools.BeanTools;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.ListUtils;
import org.apache.commons.lang3.StringUtils;
......@@ -2683,12 +2684,12 @@ public class MarkDataServiceImpl implements MarkDataService {
*/
private List<JSONObject> getPlatformPercentage(MarkSearchDTO dto) throws IOException {
if (CollectionUtils.isEmpty(dto.getPlatforms())) {
List<String> platforms = commonService.getQbjcPlatform().stream().map(jsonObject -> jsonObject.getString("id")).collect(Collectors.toList());
List<String> platforms = GlobalPojo.PLATFORMS.stream().map(MessagePlatform::getId).collect(Collectors.toList());
dto.setPlatforms(platforms);
}
List<JSONObject> list = new ArrayList<>();
// total
dto.setProjectId(UserThreadLocal.getProjectId());
defaultMarkSearch(dto);
Long total = getYuqingAnalyzeCount(dto);
// 平台聚合
SearchResponse searchResponse = platformAggSearchResponse(dto);
......@@ -2743,11 +2744,15 @@ public class MarkDataServiceImpl implements MarkDataService {
Pair<String, List<LineVO>> spreadTendency = getEmotionSpreadTendency(dto, null);
// 负面趋势图
Pair<String, List<LineVO>> negativeSpreadTendency = getEmotionSpreadTendency(dto, EmotionEnum.NEGATIVE.getName());
// 正面趋势图
Pair<String, List<LineVO>> positiveSpreadTendency = getEmotionSpreadTendency(dto, EmotionEnum.POSITIVE.getName());
Map<String, BaseMap> baseMaps = new HashMap<>(2);
BaseMap totalBaseMaps = null;
BaseMap negativeBaseMaps = null;
if (Objects.nonNull(negativeSpreadTendency.getLeft()) || Objects.nonNull(spreadTendency.getLeft())) {
CompletableFuture.allOf(Stream.of(spreadTendency.getLeft(), negativeSpreadTendency.getLeft()).filter(Objects::nonNull).map(aggTitle -> CompletableFuture.runAsync(() -> {
BaseMap positiveBaseMaps = null;
if (Objects.nonNull(negativeSpreadTendency.getLeft()) || Objects.nonNull(spreadTendency.getLeft()) || Objects.nonNull(positiveSpreadTendency.getLeft())) {
CompletableFuture.allOf(Stream.of(spreadTendency.getLeft(), negativeSpreadTendency.getLeft(), positiveSpreadTendency.getLeft())
.filter(Objects::nonNull).map(aggTitle -> CompletableFuture.runAsync(() -> {
try {
baseMaps.put(aggTitle, getAnalyzeFirstArticle(dto, aggTitle));
} catch (IOException ignored) {
......@@ -2759,7 +2764,10 @@ public class MarkDataServiceImpl implements MarkDataService {
totalBaseMaps = getWeiboFirstArticle(endTime, spreadTendency.getRight(), planId, projectId, null);
}
if (Objects.isNull(negativeSpreadTendency.getLeft()) && CollectionUtils.isNotEmpty(negativeSpreadTendency.getRight())){
negativeBaseMaps = getWeiboFirstArticle(endTime, spreadTendency.getRight(), planId, projectId, EmotionEnum.NEGATIVE.getName());
negativeBaseMaps = getWeiboFirstArticle(endTime, negativeSpreadTendency.getRight(), planId, projectId, EmotionEnum.NEGATIVE.getName());
}
if (Objects.isNull(positiveSpreadTendency.getLeft()) && CollectionUtils.isNotEmpty(positiveSpreadTendency.getRight())){
positiveBaseMaps = getWeiboFirstArticle(endTime, positiveSpreadTendency.getRight(), planId, projectId, EmotionEnum.POSITIVE.getName());
}
// 最高点
BaseMap baseMap = Objects.isNull(baseMaps.get(spreadTendency.getLeft())) ? totalBaseMaps : baseMaps.get(spreadTendency.getLeft());
......@@ -2775,11 +2783,20 @@ public class MarkDataServiceImpl implements MarkDataService {
negativeHighestJson.put("title", Tools.filterUselessTitle(negativeBaseMap.getTitle()) ? StringUtils.substring(negativeBaseMap.getContent(), 0, 15) : negativeBaseMap.getTitle());
negativeHighestJson.put("url", negativeBaseMap.getUrl());
}
// 正面最高点
BaseMap positiveBaseMap = Objects.isNull(baseMaps.get(positiveSpreadTendency.getLeft())) ? positiveBaseMaps : baseMaps.get(positiveSpreadTendency.getLeft());
JSONObject positiveHighestJson = new JSONObject();
if (Objects.nonNull(positiveBaseMap)) {
positiveHighestJson.put("title", Tools.filterUselessTitle(positiveBaseMap.getTitle()) ? StringUtils.substring(positiveBaseMap.getContent(), 0, 15) : positiveBaseMap.getTitle());
positiveHighestJson.put("url", positiveBaseMap.getUrl());
}
res.putAll(platformSpreadTendency);
res.put("总量", spreadTendency.getRight());
res.put("负面", negativeSpreadTendency.getRight());
res.put("正面", positiveSpreadTendency.getRight());
res.put("highest", highestJson);
res.put("negativeHighest", negativeHighestJson);
res.put("positiveHighest", positiveHighestJson);
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "新舆情分析getSpreadTendency异常-", e);
}
......@@ -3122,9 +3139,9 @@ public class MarkDataServiceImpl implements MarkDataService {
public List<JSONObject> getAggTitleArticlePoints(MarkSearchDTO dto) {
List<JSONObject> jsonObject = new ArrayList<>();
try {
defaultMarkSearch(dto);
String[] indexes = esClientDao.getIndexes();
// query
dto.setProjectId(UserThreadLocal.getProjectId());
BoolQueryBuilder query = yuqingAnalyzeQuery(dto);
DateHistogramAggregationBuilder daysAggregationBuilder =
AggregationBuilders.dateHistogram("timeAgg").field("time").calendarInterval(DateHistogramInterval.DAY).offset("16h");
......@@ -3194,18 +3211,19 @@ public class MarkDataServiceImpl implements MarkDataService {
@Override
public PageVO<JSONObject> getArticleList(MarkSearchDTO dto) {
int page = Objects.isNull(dto.getPage()) ? 1 : dto.getPage();
int pageSize = Objects.isNull(dto.getPageSize()) ? 10 : dto.getPageSize();
try {
defaultMarkSearch(dto);
String[] indexes = esClientDao.getIndexes();
dto.setProjectId(UserThreadLocal.getProjectId());
int page = dto.getPage();
int pageSize = dto.getPageSize();
// query
if (Objects.equals("其他", dto.getPlatforms().get(0))){
List<String> platforms = getPlatformPercentage(dto).stream().map(json -> json.getString("platform")).collect(Collectors.toList());
List<String> allPlatforms = GlobalPojo.PLATFORMS.stream().map(MessagePlatform::getName).collect(Collectors.toList());
allPlatforms.removeAll(platforms);
dto.setPlatforms(allPlatforms.stream().map(GlobalPojo::getPlatformIdByName).collect(Collectors.toList()));
}else {
}
if (!Objects.equals("全部", dto.getPlatforms().get(0))){
dto.setPlatforms(Collections.singletonList(GlobalPojo.getPlatformIdByName(dto.getPlatforms().get(0))));
}
BoolQueryBuilder query = yuqingAnalyzeQuery(dto);
......@@ -3219,6 +3237,7 @@ public class MarkDataServiceImpl implements MarkDataService {
JSONObject jsonObject = new JSONObject();
jsonObject.put("source", baseMap.getSource());
jsonObject.put("platform", baseMap.getPlatform());
jsonObject.put("realSource", baseMap.getRealSource());
jsonObject.put("title", baseMap.getTitle());
jsonObject.put("time", baseMap.getTime());
jsonObject.put("url", baseMap.getUrl());
......@@ -3228,7 +3247,7 @@ public class MarkDataServiceImpl implements MarkDataService {
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "新舆情分析getArticleList异常-", e);
}
return PageVO.createPageVo(0, page, pageSize, Collections.emptyList());
return null;
}
@Override
......@@ -3278,6 +3297,272 @@ public class MarkDataServiceImpl implements MarkDataService {
return dailyReportDao.findOneById(id);
}
@Override
public MarkFlowEntity getWholeSearchFirstArticle(MarkSearchDTO dto) {
try {
BoolQueryBuilder query = getAllIndexSearchOriginQuery(dto);
String[] indexes = esClientDao.getAllIndexes(dto.getStartTime(), dto.getEndTime());
// sort
FieldSortBuilder sort = new FieldSortBuilder("time").order(SortOrder.ASC);
// hits
SearchHits hits = esClientDao.searchHits(indexes, query, null, null, sort, 0, 1, null);
if (0 == hits.getTotalHits().value) {
return null;
}
Map<String, Object> hit = hits.getAt(0).getSourceAsMap();
JSONObject tJson = new JSONObject(hit);
return markFlowService.createAllIndexFlowInfo(tJson, dto);
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "全网搜原发溯源getWholeSearchFirstArticle异常-", e);
}
return null;
}
@Override
public List<JSONObject> getWholeSearchMediaParticipation(MarkSearchDTO dto) {
List<JSONObject> res = new ArrayList<>();
try {
BoolQueryBuilder query = getAllIndexSearchOriginQuery(dto);
String[] indexes = esClientDao.getAllIndexes(dto.getStartTime(), dto.getEndTime());
// sort
FieldSortBuilder sort = new FieldSortBuilder("time").order(SortOrder.ASC);
// ipAgg
TermsAggregationBuilder provinceAggregationBuilder = AggregationBuilders.terms("ipAgg").field("ip_location").size(6);
// response
SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, query, provinceAggregationBuilder, null, sort, 0, 1000, null);
// hits
SearchHits hits = searchResponse.getHits();
JSONObject firstJson = new JSONObject(hits.getAt(0).getSourceAsMap());
Map<String, Integer> platformCount = new HashMap<>();
Map<String, Integer> fieldCount = new HashMap<>();
int centralLevelCount = 0;
int provincialLevelCount = 0;
int prefectureLevelCount = 0;
// 重要渠道
Map<String, SensitiveChannel> sensitiveChannelMap = GlobalPojo.COMMON_SENSITIVE_CHANNEL;
for (SearchHit hit : hits.getHits()) {
JSONObject hitJson = new JSONObject(hit.getSourceAsMap());
String platform = BeanTools.filterPlatform(GlobalPojo.PLATFORMS, hitJson.getIntValue(GenericAttribute.ES_C5), hitJson.getIntValue(GenericAttribute.ES_FOREIGN)).getName();
// 平台统计
platformCount.compute(platform, (k, v) -> Objects.isNull(v) ? 1 : ++v);
String source = hitJson.getString(GenericAttribute.ES_SOURCE);
SensitiveChannel sensitiveChannel = sensitiveChannelMap.get(source);
if (Objects.isNull(sensitiveChannel)){
continue;
}
// 重要渠道领域统计
if (StringUtils.isNotBlank(sensitiveChannel.getField())) {
fieldCount.compute(sensitiveChannel.getField(), (k, v) -> Objects.isNull(v) ? 1 : ++v);
}
if (StringUtils.isNotBlank(sensitiveChannel.getPoliticsLevel())){
if (Objects.equals("央级", sensitiveChannel.getPoliticsLevel())){
centralLevelCount ++;
}
if (Objects.equals("省级", sensitiveChannel.getPoliticsLevel())){
provincialLevelCount ++;
}
if (Objects.equals("市级", sensitiveChannel.getPoliticsLevel())){
prefectureLevelCount ++;
}
}
}
// 首发平台
JSONObject firstPlatform = new JSONObject();
firstPlatform.put("name", "首发平台");
firstPlatform.put("value", BeanTools.filterPlatform(GlobalPojo.PLATFORMS, firstJson.getIntValue(GenericAttribute.ES_C5), firstJson.getIntValue(GenericAttribute.ES_FOREIGN)).getName());
res.add(firstPlatform);
// 爆发平台
Set<String> maxPlatforms = Tools.sortMap(platformCount, 1).keySet();
JSONObject maxPlatform = new JSONObject();
maxPlatform.put("name", "爆发平台");
maxPlatform.put("value", maxPlatforms.toArray()[0]);
res.add(maxPlatform);
// 传播量
JSONObject count = new JSONObject();
count.put("name", "传播量");
count.put("value", hits.getHits().length);
res.add(count);
// 媒体领域
JSONObject mediaField = new JSONObject();
mediaField.put("name", "媒体领域");
mediaField.put("value", Tools.sortMap(fieldCount, 6).keySet());
res.add(mediaField);
// 央级
JSONObject centralLevel = new JSONObject();
centralLevel.put("name", "央级");
centralLevel.put("value", centralLevelCount);
res.add(centralLevel);
// 省级
JSONObject provincialLevel = new JSONObject();
provincialLevel.put("name", "省级");
provincialLevel.put("value", provincialLevelCount);
res.add(provincialLevel);
// 地级
JSONObject prefectureLevel = new JSONObject();
prefectureLevel.put("name", "地级");
prefectureLevel.put("value", prefectureLevelCount);
res.add(prefectureLevel);
// 媒体地域
Map<String, Aggregation> provinceMap = searchResponse.getAggregations().asMap();
ParsedStringTerms provinceCountTeam = (ParsedStringTerms) provinceMap.get("ipAgg");
List<? extends Terms.Bucket> provinceBuckets = provinceCountTeam.getBuckets();
JSONObject mediaProvince = new JSONObject();
mediaProvince.put("name", "媒体地域");
mediaProvince.put("value", provinceBuckets.stream().map(MultiBucketsAggregation.Bucket::getKeyAsString).collect(Collectors.toList()));
res.add(mediaProvince);
return res;
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "全网搜原发溯源getWholeSearchMediaParticipation异常-", e);
}
return res;
}
@Override
public List<JSONObject> getWholeSearchArticlePoint(MarkSearchDTO dto) {
List<JSONObject> jsonObjects = new ArrayList<>();
try {
BoolQueryBuilder query = getAllIndexSearchOriginQuery(dto);
String[] indexes = esClientDao.getAllIndexes(dto.getStartTime(), dto.getEndTime());
// aggregation
DateHistogramAggregationBuilder daysAggregationBuilder =
AggregationBuilders.dateHistogram("timeAgg").field("time").calendarInterval(DateHistogramInterval.DAY).offset("16h");
TermsAggregationBuilder sourceAggregationBuilder = AggregationBuilders.terms("sourceAgg").field("source").size(10000);
// response
SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, query,
daysAggregationBuilder.subAggregation(sourceAggregationBuilder), null, null, 0, 0, null);
Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
ParsedDateHistogram teamAgg = (ParsedDateHistogram) aggMap.get("timeAgg");
List<? extends Histogram.Bucket> buckets = teamAgg.getBuckets();
// 重要渠道
Map<String, SensitiveChannel> sensitiveChannelMap = GlobalPojo.COMMON_SENSITIVE_CHANNEL;
buckets.forEach(bucket -> {
JSONObject point = new JSONObject();
point.put("time", Long.parseLong(bucket.getKeyAsString()));
// 发文渠道
Map<String, Aggregation> sourceMap = bucket.getAggregations().asMap();
ParsedStringTerms countTeam = (ParsedStringTerms) sourceMap.get("sourceAgg");
List<? extends Terms.Bucket> sourceBuckets = countTeam.getBuckets();
List<JSONObject> sources = new ArrayList<>();
sourceBuckets.forEach(sourceBucket -> {
JSONObject source = new JSONObject();
source.put("name", sourceBucket.getKeyAsString());
// 重要渠道判断
source.put("isImportant", sensitiveChannelMap.containsKey(sourceBucket.getKeyAsString()));
sources.add(source);
});
point.put("sources", sources);
point.put("count", sources.size());
jsonObjects.add(point);
});
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "全网搜原发溯源发布节点getWholeSearchArticlePoint异常-", e);
}
return jsonObjects;
}
@Override
public List<JSONObject> getWholeSearchPlatformPercentage(MarkSearchDTO dto) {
try {
BoolQueryBuilder query = getAllIndexSearchOriginQuery(dto);
String[] indexes = esClientDao.getAllIndexes(dto.getStartTime(), dto.getEndTime());
Long total = esClientDao.count(indexes, query, null);
// 聚合条件
Script script = new Script("doc['c5'].value +'_' +doc['foreign'].value");
TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("agg").script(script).size(1000);
// response
SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, query, aggregationBuilder, null, null, 0, 0, null);
// buckets
Map<String, Aggregation> aggregationMap = searchResponse.getAggregations().asMap();
ParsedStringTerms terms = (ParsedStringTerms) aggregationMap.get("agg");
List<? extends Terms.Bucket> buckets = terms.getBuckets();
Map<String, Long> platformCount = new HashMap<>();
for (Terms.Bucket bucket : buckets) {
String[] split = bucket.getKeyAsString().split("_");
String platform = BeanTools.filterPlatform(GlobalPojo.PLATFORMS, Integer.valueOf(split[0]), Integer.valueOf(split[1])).getName();
if (StringUtils.isBlank(platform)){
continue;
}
platformCount.compute(platform, (k, v) -> Objects.isNull(v) ? bucket.getDocCount() : v + bucket.getDocCount());
}
List<JSONObject> list = new ArrayList<>();
for (Map.Entry<String, Long> entry : platformCount.entrySet()) {
JSONObject platformResult = new JSONObject();
platformResult.put("platform", entry.getKey());
platformResult.put("percent", 0 == total ? 0d : entry.getValue() / (double) total);
list.add(platformResult);
}
// 取占比最高的前9
List<JSONObject> res = list.stream()
.sorted(Comparator.comparingDouble((JSONObject jsonObject) -> jsonObject.getDoubleValue("percent"))
.reversed()).limit(9).collect(Collectors.toList());
// 第10个算做其他平台
JSONObject other = new JSONObject();
other.put("platform", "其他");
other.put("percent", 1 - res.stream().mapToDouble(count -> count.getDoubleValue("percent")).sum());
res.add(other);
return res;
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "全网搜原发溯源平台分布getWholeSearchPlatformPercentage异常-", e);
}
return null;
}
@Override
public PageVO<JSONObject> getWholeSearchArticleList(MarkSearchDTO dto) {
try {
defaultMarkSearch(dto);
// keyword
BoolQueryBuilder keywordQuery = QueryBuilders.boolQuery();
keywordQuery.must(EsQueryTools.assembleNormalKeywordQuery(dto.getKeyword(), new String[]{GenericAttribute.ES_IND_FULL_TEXT}));
BoolQueryBuilder query = QueryBuilders.boolQuery().must(keywordQuery);
// platform query
if (Objects.equals("其他", dto.getPlatforms().get(0))){
List<String> platforms = getWholeSearchPlatformPercentage(dto).stream().map(json -> json.getString("platform")).collect(Collectors.toList());
query.must(EsQueryTools.assemblePlatformQuery(GlobalPojo.PLATFORMS.stream()
.filter(platform -> !platforms.contains(platform.getName())).collect(Collectors.toList())));
}
if (!Objects.equals("全部", dto.getPlatforms().get(0))){
query.must(EsQueryTools.assemblePlatformQuery(GlobalPojo.PLATFORMS.stream()
.filter(platform -> Objects.equals(dto.getPlatforms().get(0), platform.getName())).collect(Collectors.toList())));
}
int page = dto.getPage();
int pageSize = dto.getPageSize();
String[] indexes = esClientDao.getAllIndexes(dto.getStartTime(), dto.getEndTime());
FieldSortBuilder sort = new FieldSortBuilder("time").order(SortOrder.ASC);
SearchHits hits = esClientDao.searchHits(indexes, query, null, null, sort, (page - 1) * pageSize, pageSize, null);
List<JSONObject> list = new ArrayList<>();
for (SearchHit hit : hits) {
JSONObject hitJson = new JSONObject(hit.getSourceAsMap());
JSONObject data = JSONObject.parseObject(JSONObject.toJSONString(new MarkFlowEntity(hitJson).getData()));
JSONObject jsonObject = new JSONObject();
jsonObject.put("platform", BeanTools.filterPlatform(GlobalPojo.PLATFORMS, hitJson.getIntValue(GenericAttribute.ES_C5), hitJson.getIntValue(GenericAttribute.ES_FOREIGN)).getName());
jsonObject.put("source", hitJson.getString(GenericAttribute.ES_SOURCE));
jsonObject.put("realSource", hitJson.getString(GenericAttribute.ES_REAL_SOURCE));
jsonObject.put("time", hitJson.getLongValue("time"));
String title = StringUtils.isNotBlank(Tools.getTitleWithContent(data.getString("title"), data.getString("content")))
? Tools.getTitleWithContent(data.getString("title"), data.getString("content"))
: Tools.getTitleWithContent(data.getString("questionTitle"), data.getString("questionContent"));
jsonObject.put("title", title);
list.add(jsonObject);
}
return PageVO.createPageVo(hits.getHits().length, page, pageSize, list);
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "全网搜原发溯源文章列表getWholeSearchArticleList异常-", e);
}
return null;
}
private BoolQueryBuilder getAllIndexSearchOriginQuery(MarkSearchDTO dto) {
dto.setStartTime(dto.getEndTime() - Constant.ONE_MONTH);
// keyword
BoolQueryBuilder keywordQuery = QueryBuilders.boolQuery();
keywordQuery.must(QueryBuilders.matchPhraseQuery(GenericAttribute.ES_IND_FULL_TEXT, dto.getKeyword()));
// platform
BoolQueryBuilder platformQuery = EsQueryTools.assemblePlatformQuery(GlobalPojo.PLATFORMS);
return QueryBuilders.boolQuery().must(keywordQuery).must(platformQuery)
.must(QueryBuilders.rangeQuery("time").gte(dto.getStartTime()).lte(dto.getEndTime()));
}
private JSONObject getDailyReportSummary(JSONObject yuqingAmount, JSONObject yuqingEmotion, JSONObject platformPercentage){
JSONObject res = new JSONObject();
long total = yuqingAmount.getLongValue("total");
......@@ -3469,8 +3754,14 @@ public class MarkDataServiceImpl implements MarkDataService {
if (Objects.equals("视频", dto.getSearchType())){
postFilter.must(QueryBuilders.existsQuery(GenericAttribute.ES_VIDEO_URLS));
}
// time
postFilter.must(QueryBuilders.rangeQuery("time").gte(dto.getStartTime()).lt(dto.getEndTime()));
// startTime
if (Objects.nonNull(dto.getStartTime())) {
postFilter.must(QueryBuilders.rangeQuery("time").gte(dto.getStartTime()));
}
// endTime
if (Objects.nonNull(dto.getEndTime())) {
postFilter.must(QueryBuilders.rangeQuery("time").lt(dto.getEndTime()));
}
// platform
if (CollectionUtils.isNotEmpty(dto.getPlatforms())) {
postFilter.must(EsQueryTools.assemblePlatformQuery(Tools.getPlatformByIds(dto.getPlatforms())));
......
src/main/java/com/zhiwei/brandkbs2/service/impl/MarkFlowServiceImpl.java
......@@ -10,14 +10,14 @@ import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.common.GlobalPojo;
import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.dao.HighlightWordDao;
import com.zhiwei.brandkbs2.enmus.ChannelEmotion;
import com.zhiwei.brandkbs2.exception.ExceptionCast;
import com.zhiwei.brandkbs2.model.CommonCodeEnum;
import com.zhiwei.brandkbs2.pojo.MarkFlowEntity;
import com.zhiwei.brandkbs2.pojo.SensitiveChannel;
import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
import com.zhiwei.brandkbs2.pojo.dto.TagFilterDTO;
import com.zhiwei.brandkbs2.service.MarkFlowService;
import com.zhiwei.brandkbs2.service.TagFilterService;
import com.zhiwei.brandkbs2.util.RedisUtil;
import com.zhiwei.brandkbs2.util.Tools;
import com.zhiwei.middleware.automaticmark.vo.Keyword;
......@@ -27,6 +27,7 @@ import com.zhiwei.middleware.mark.vo.QueryResult;
import com.zhiwei.qbjc.bean.pojo.common.Tag;
import com.zhiwei.qbjc.bean.tools.BeanTools;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.stereotype.Service;
......@@ -197,6 +198,63 @@ public class MarkFlowServiceImpl implements MarkFlowService {
return JSON.parseObject(data, MarkFlowEntity.class);
}
@Override
public MarkFlowEntity createAllIndexFlowInfo(JSONObject json, MarkSearchDTO dto) {
MarkFlowEntity markFlowEntity = new MarkFlowEntity(json);
JSONObject info = new JSONObject();
info.put("sourceDetails", getSourceDetails(json));
info.put("startTime", dto.getStartTime());
info.put("endTime", dto.getEndTime());
info.put("highlightWordMap", getHighlightWordMap(UserThreadLocal.getProjectId(), json));
markFlowEntity.setInfo(info);
return markFlowEntity;
}
/**
* 底层大库数据SourceDetails提取
* @param tJson
* @return
*/
private JSONObject getSourceDetails(JSONObject tJson){
JSONObject sourceDetails = new JSONObject();
String source = tJson.getString(GenericAttribute.ES_SOURCE);
String rootSource = tJson.getString(GenericAttribute.ES_ROOT_SOURCE);
// 是否原创
if (tJson.containsKey(GenericAttribute.ES_ROOT_SOURCE) && (rootSource.contains(source) || source.contains(rootSource))) {
sourceDetails.put("rootPublish", "原创");
}
// C4,realSource提取展示
sourceDetails.put("clientFrom", Tools.isEmpty(tJson.getString(GenericAttribute.ES_REAL_SOURCE)) ? ClassD.TypeD.fromEncode(tJson.getIntValue(GenericAttribute.ES_C4)).title() : tJson.getString(GenericAttribute.ES_REAL_SOURCE));
// platform
sourceDetails.put("platform", BeanTools.filterPlatform(GlobalPojo.PLATFORMS, tJson.getIntValue(GenericAttribute.ES_C5), tJson.getIntValue(GenericAttribute.ES_FOREIGN)).getName());
// source
sourceDetails.put("source", source);
// 粉丝量提取
long followersNum = tJson.getLongValue("followers_num");
if (followersNum > 0) {
sourceDetails.put("followersNum", followersNum);
}
// 渠道标签
SensitiveChannel sensitiveChannel = GlobalPojo.COMMON_SENSITIVE_CHANNEL.get(source);
if (Objects.nonNull(sensitiveChannel)) {
List<String> channelTagStr = new ArrayList<>(4);
if (StringUtils.isNotBlank(sensitiveChannel.getPoliticsLevel())){
channelTagStr.add(sensitiveChannel.getPoliticsLevel());
}
if (StringUtils.isNotBlank(sensitiveChannel.getMainBodyType())){
channelTagStr.add(sensitiveChannel.getMainBodyType());
}
if (StringUtils.isNotBlank(sensitiveChannel.getField())){
channelTagStr.add(sensitiveChannel.getField());
}
if (StringUtils.isNotBlank(sensitiveChannel.getRegion())){
channelTagStr.add(sensitiveChannel.getRegion());
}
sourceDetails.put("channelTag", StringUtils.join(channelTagStr, "|"));
}
return sourceDetails;
}
/**
* 获取自定义标签
* @param markInfoSource
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment