Skip to content

B
brandkbs2
Commit 06ebd048 by shenjunjie
Options

Merge branch 'release' into 'master' 

Release

See merge request !581
parents fe234347 e4ed37fd
Showing with 1363 additions and 15 deletions
src/main/java/com/zhiwei/brandkbs2/controller/app/AppDownloadController.java
......@@ -67,6 +67,9 @@ public class AppDownloadController extends BaseController {
@Resource(name = "markDataServiceImpl")
MarkDataService markDataService;
@Resource(name = "searchWholeServiceImpl")
SearchWholeService searchWholeService;
@Resource(name = "toolsetServiceImpl")
private ToolsetService toolsetService;
......@@ -431,7 +434,7 @@ public class AppDownloadController extends BaseController {
return ResponseResult.failure("实时采集余额不足");
}
}
List<ExportSearchWholeDTO> exportList = markDataService.exportSearchWhole(dto);
List<ExportSearchWholeDTO> exportList = searchWholeService.exportSearchWhole(dto);
// excel写入至指定路径
String projectName = projectService.getProjectById(UserThreadLocal.getProjectId()).getProjectName();
String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), "全网搜舆情列表数据");
......
src/main/java/com/zhiwei/brandkbs2/controller/app/AppSearchController.java
......@@ -91,6 +91,9 @@ public class AppSearchController extends BaseController {
@Resource(name = "extraServiceImpl")
ExtraService extraService;
@Resource(name = "searchWholeServiceImpl")
SearchWholeService searchWholeService;
@Resource(name = "redisUtil")
RedisUtil redisUtil;
......@@ -153,7 +156,7 @@ public class AppSearchController extends BaseController {
@ApiOperation("搜索-全网搜条件")
@GetMapping("/getWholeSearchCriteria")
public ResponseResult getWholeSearchCriteria(@RequestParam(value = "externalDataSource", defaultValue = "false") boolean externalDataSource) {
JSONObject wholeSearchCriteria = markDataService.getWholeNetworkSearchCriteria(externalDataSource);
JSONObject wholeSearchCriteria = searchWholeService.getWholeNetworkSearchCriteria(externalDataSource);
return ResponseResult.success(wholeSearchCriteria);
}
......@@ -181,7 +184,7 @@ public class AppSearchController extends BaseController {
return ResponseResult.failure("实时采集余额不足");
}
}
Pair<JSONObject, Integer> pair = markDataService.searchWholeNetworkWithBalance(dto);
Pair<JSONObject, Integer> pair = searchWholeService.searchWholeNetworkWithBalance(dto);
if(dto.isSentiment()){
List<JSONObject> list = pair.getLeft().getJSONArray("list").toJavaList(JSONObject.class);
commonService.sentimentAnalysis(list);
......@@ -219,7 +222,7 @@ public class AppSearchController extends BaseController {
return ResponseResult.failure("实时采集余额不足");
}
}
List<ExportSearchWholeDTO> exportList = markDataService.exportSearchWhole(dto);
List<ExportSearchWholeDTO> exportList = searchWholeService.exportSearchWhole(dto);
EasyExcelUtil.download("全网搜舆情列表数据", "sheet1", ExportSearchWholeDTO.class, exportList, response);
if (dto.isExternalDataSource()) {
extraService.decreaseRecord(dto.getKeyword(), WholeSearchRecord.UsedType.output, exportList.size());
......@@ -380,6 +383,79 @@ public class AppSearchController extends BaseController {
return ResponseResult.success(markDataService.getAIReferenceQuestionCache(true));
}
@ApiOperation("新-舆情分析-舆情总量")
@PostMapping("/analyze/amount")
@LogRecord(description = "舆情库-舆情分析")
public ResponseResult getYuqingAmount(@RequestBody SearchFilterDTO searchFilterDTO) {
if (searchFilterDTO.isExternalDataSource()){
return ResponseResult.failure("多元接口不支持数据分析");
}
return ResponseResult.success(searchWholeService.getSearchWholeYuqingAmount(searchFilterDTO));
}
@ApiOperation("新-舆情分析-重点平台")
@PostMapping("/analyze/important-platform")
public ResponseResult getImportantPlatformPercentage(@RequestBody SearchFilterDTO searchFilterDTO) {
if (searchFilterDTO.isExternalDataSource()){
return ResponseResult.failure("多元接口不支持数据分析");
}
return ResponseResult.success(searchWholeService.getSearchWholeImportantPlatformPercentage(searchFilterDTO));
}
@ApiOperation("新-舆情分析-平台占比")
@PostMapping("/analyze/platform-percent")
public ResponseResult getPlatformPercentage(@RequestBody SearchFilterDTO searchFilterDTO) {
if (searchFilterDTO.isExternalDataSource()){
return ResponseResult.failure("多元接口不支持数据分析");
}
return ResponseResult.success(searchWholeService.getSearchWholeAnalyzePlatformPercentage(searchFilterDTO));
}
@ApiOperation("新-舆情分析-舆情走势图")
@PostMapping("/analyze/tendency")
public ResponseResult getSpreadTendency(@RequestBody SearchFilterDTO searchFilterDTO) {
if (searchFilterDTO.isExternalDataSource()){
return ResponseResult.failure("多元接口不支持数据分析");
}
return ResponseResult.success(searchWholeService.getSearchWholeSpreadTendency(searchFilterDTO));
}
@ApiOperation("新-舆情分析-活跃渠道")
@PostMapping("/analyze/active-channel")
public ResponseResult getActiveChannels(@RequestBody SearchFilterDTO searchFilterDTO) {
if (searchFilterDTO.isExternalDataSource()){
return ResponseResult.failure("多元接口不支持数据分析");
}
return ResponseResult.success(searchWholeService.getSearchWholeActiveChannels(searchFilterDTO));
}
@ApiOperation("新-舆情分析-ip分布")
@PostMapping("/analyze/ip-located")
public ResponseResult getArticleIpLocated(@RequestBody SearchFilterDTO searchFilterDTO) {
if (searchFilterDTO.isExternalDataSource()){
return ResponseResult.failure("多元接口不支持数据分析");
}
return ResponseResult.success(searchWholeService.getSearchWholeArticleIpLocated(searchFilterDTO));
}
@ApiOperation("新-舆情分析-词云")
@PostMapping("/analyze/high-word")
public ResponseResult getHighWord(@RequestBody SearchFilterDTO searchFilterDTO) {
if (searchFilterDTO.isExternalDataSource()){
return ResponseResult.failure("多元接口不支持数据分析");
}
return ResponseResult.success(searchWholeService.getSearchWholeAnalyzeHighWord(searchFilterDTO));
}
@ApiOperation("新-舆情分析-活跃渠道、ip分布、词云详情页面,社媒平台发文")
@PostMapping("/analyze/data")
public ResponseResult getYuqingAnalyzeDetail(@RequestBody SearchFilterDTO searchFilterDTO) {
if (searchFilterDTO.isExternalDataSource()){
return ResponseResult.failure("多元接口不支持数据分析");
}
return ResponseResult.success(searchWholeService.getSearchWholeAnalyzeDetail(searchFilterDTO));
}
@ApiOperation("搜索-搜索关键词历史记录")
@GetMapping("/keyword/cache")
public ResponseResult getSearchKeywordCache(@ApiParam(name = "searchType",
......
src/main/java/com/zhiwei/brandkbs2/enmus/SensitiveChannelsEnum.java 0 → 100644
package com.zhiwei.brandkbs2.enmus;
import com.zhiwei.brandkbs2.util.Tools;
import java.util.*;
/**
* @ClassName: SensitiveChannels
* @Description: 重要渠道
* @author: YaoGuoDong
* @create: 2022-06-10 14:50
**/
public enum SensitiveChannelsEnum {
YANG_MEI("央媒", "央媒", Arrays.asList("中国新闻网","中国网","人民日报", "新华社", "求是", "解放军报", "光明日报", "经济日报", "中国日报", "中央人民广播电台", "央视网", "国际在线", "科技日报", "中国纪检监察报", "工人日报", "中国青年报", "农民日报", "法制日报", "中新社")),
ZHU_LIU_CAI_CAI_JING("主流财经", "主流财经", Arrays.asList("证券时报", "国际金融报", "中国经济周刊", "经济参考报", "中国证券报", "上海证券报", "财经国家周刊", "中新经纬", "经济日报", "证券日报", "中国企业家", "中国经济信息社", "中国财经报", "中国商报", "中国企业报", "国际商报", "中国经济导报", "中国经济时报", "金融时报", "中国经营报", "财新周刊", "《财经》杂志", "证券市场红周刊", "界面新闻", "财联社", "蓝鲸财经", "摩尔金融", "第一财经日报", "第一财经周刊", "21世纪经济报道", "21世纪商业评论", "大众证券报", "深圳商报", "新财富", "香港商报", "成都商报", "每日经济新闻", "金融投资报", "长江商报", "支点财经", "经济观察报", "华商报", "东亚经贸新闻", "时代财经", "资本邦", "投资界", "投中网", "创业邦", "财经天下周刊", "FT中文网", "财富", "福布斯", "彭博商业周刊", "哈佛商业评论", "金融界", "金投网", "中研网", "FX168财经网", "中金在线 ", "中商情报网", "财界网", "东方财富网", "南方财富网", "同花顺财经", "云掌财经", "华尔街见闻", "智通财经", "深蓝财经", "乐居财经", "格隆汇", "棱镜", "凤凰WEEKLY财经 ", "野马财经", "燃财经", "晚点LatePost ", "LateNews by 小晚（微信公众号）", "秦朔朋友圈", "吴晓波频道", "兽楼处", "叶檀财经", "牛弹琴", "占豪", "新财迷", "智谷趋势", "智东西", "三言财经", "华商韬略", "正商参阅", "水木然专栏", "春暖花开", "饭统戴老板", "石述思", "刘军洛财经", "郎咸平", "花荣", "水皮", "光远看经济", "正和岛", "功夫财经", "李大霄", "肖锋", "宋清辉", "何加盐", "财经十一人", "但斌", "启阳路4号", "棱镜", "经鉴新闻", "蓝鲸浑水")),
ZHU_LIU_KE_JI("主流科技", "主流科技", Arrays.asList("科技日报", "计算机世界", "中国计算机报", "赛迪网", "中国电子报", "通信世界", "通信产业报", "虎嗅", "36氪", "36kr", "钛媒体", "动点科技", "雷锋网", "驱动之家", "快科技", "砍柴网", "驱动中国", "科技资讯网", "TechCrunch", "Business Insider", "新智元", "量子位", "机器之心", "TechWeb", "IT168", "IT之家", "天极网", "比特网", "中关村在线", "PCHOME", "电脑之家", "CSDN", "51CTO", "硅谷动力", "至顶网", "IT时代周刊", "C114通信网", "飞象网", "DoNews", "cnBeta", "亿欧网", "鞭牛士", "PingWest品玩", "镁客网", "极客公园", "猎云网", "泡泡网", "爱范儿", "速途网", "太平洋电脑网", "i黑马", "手机中国", "科技美学", "量子位", "少数派", "酷玩实验室", "笔吧评测室", "安兔兔", "差评", "科技每日推送", "全是黑科技", "机器之心", "小白测评", "AppSo ", "爱范儿", "大米评测", "最黑科技", "DeepTech深科技", "好机友", "果粉之家", "刘旷", "金错刀", "雷科技", "新智元", "爱否科技", "InfoQ", "小道消息", "人人都是产品经理", "花果科技", "智东西", "阑夕", "雷帝触网", "丁道师", "黑马公社", "周兴斌", "科技狐", "躺倒鸭", "春公子", "郝闻郝看", "全天候科技", "闫跃龙", "小声比比", "乌鸦校尉", "江湖老刘", "科技兽", "郭静的互联网圈", "电手", "超先声", "猿大侠", "IT三剑客", "果壳", "哎咆科技")),
BEI_JING("区域", "北京", Arrays.asList("北京时间", "北京广播电视台", "北京青年报", "北京青年网", "北青网", "北京青年周刊", "北京日报", "京报网", "北京商报", "北京现代商报", "北京商报网", "首都建设报", "北京社区报", "北京头条", "北京晚报", "北晚新视觉", "北晚在线", "北京晨报", "新京报", "千龙网")),
SHANG_HAI("区域", "上海", Arrays.asList("东方卫视", "解放日报", "上观新闻", "看看新闻", "SMG", "上海日报", "上海日报SHINE", "文汇报", "新民邻声", "新民晚报", "新民网", "新民周刊", "文汇网", "新闻晨报", "周到上海", "界面新闻", "财联社", "蓝鲸财经", "科创板日报", "澎湃新闻", "上海法治报", "上海证券报", "第一财经", "第一财经日报", "第一财经周刊", "一财网", "劳动报", "东方网", "观察者网")),
GUANG_ZHOU("区域", "广州", Arrays.asList("南方日报", "南方网", "南方+", "南方Plus", "南方都市报", "奥一网", "南方周末", "南都周刊", "南方人物周刊", "信息时报", "时代周报", "时代财经", "新周刊", "消费者报道", "大洋网", "21世纪经济报道", "21世纪商业评论", "广州日报", "新花城", "南风窗", "羊城晚报", "金羊网", "羊城派", "新快报", "广东卫视", "广东南方卫视", "触电新闻", "广东广播电视台", "触电新闻", "广东经视", "珠江频道")),
SHEN_ZHEN("区域", "深圳", Arrays.asList("深圳商报", "读创", "深圳特区报", "读特", "深圳卫视", "晶报", "壹深圳", "深圳热线", "深圳新闻网", "深圳晚报", "Shenzhen Daily")),
ZHE_JIANG("区域", "浙江", Arrays.asList("钱江晚报", "浙江日报", "浙江新闻", "天目新闻", "浙江法制报", "都市快报", "每日商报", "青年时报", "杭州日报", "浙江在线", "科技金融时报", "浙江之声", "浙江卫视", "中国蓝TV", "中国蓝新闻", "新蓝网", "杭州网", "1818黄金眼", "小强热线", "钱江视频", "浙样红TV", "FM93浙江交通之声", "FM93交通之声", "杭州日报", "杭州网", "网经社", "宁波日报", "甬派", "宁波晚报", "中国宁波网")),
CHUAN_YU("区域", "川渝", Arrays.asList("成都日报", "成都商报", "红星新闻", "每日经济新闻", "微成都", "成都晚报", "封面新闻", "华西都市报", "四川在线", "四川新闻网", "川报观察", "四川日报", "重庆日报", "上游新闻", "重庆晚报", "重庆晨报", "都市热报", "华龙网", "四川卫视", "四川观察", "就成都", "四川广播电视台", "重庆电视台"));
private final String group;
private final String name;
private final List<String> channels;
private static final List<String> GROUPS = Arrays.asList("全部", "央媒", "主流财经", "主流科技", "区域");
private static final List<String> FILTER_GROUPS = Arrays.asList("全部", "央媒", "主流财经", "主流科技");
public String getGroup() {
return group;
}
public String getName() {
return name;
}
public List<String> getChannels() {
return channels;
}
SensitiveChannelsEnum(String group, String name, List<String> channels) {
this.group = group;
this.name = name;
this.channels = channels;
}
public static List<String> getSensitiveChannelsByGroupAndName(String group, String name) {
for (SensitiveChannelsEnum channels : values()) {
// 条件一：group无子级、name传参为空、枚举存在group
boolean conditionOne = FILTER_GROUPS.contains(group) && Tools.isEmpty(name) && channels.getGroup().equals(group);
// 条件二：group有子级、枚举存在name、枚举存在group
boolean conditionTwo = channels.getGroup().equals(group) && channels.getName().equals(name);
if (conditionOne || conditionTwo) {
return channels.getChannels();
}
}
return new ArrayList<>();
}
public static List<String> getSensitiveChannelsNameByGroup(String group) {
List<String> list = new ArrayList<>();
for (SensitiveChannelsEnum channels : values()) {
if (channels.getGroup().equals(group)) {
list.add(channels.getName());
}
}
return list;
}
/**
* 获取重要渠道传参
* @return
*/
public static List<Map<String, Object>> getExtraParam() {
List<Map<String, Object>> list = new ArrayList<>();
for (int i = 0; i < SensitiveChannelsEnum.GROUPS.size(); i++) {
Map<String, Object> map = new HashMap<>();
map.put("name", SensitiveChannelsEnum.GROUPS.get(i));
if (FILTER_GROUPS.contains(GROUPS.get(i))) {
map.put("value", new ArrayList<>());
} else {
map.put("value", SensitiveChannelsEnum.getSensitiveChannelsNameByGroup(SensitiveChannelsEnum.GROUPS.get(i)));
}
list.add(map);
}
return list;
}
}
src/main/java/com/zhiwei/brandkbs2/es/EsQueryTools.java
......@@ -80,6 +80,58 @@ public class EsQueryTools {
}
/**
* keyword类型字段匹配 | 作分割
*
* @Title: assembleSeniorTermQuery
* @Description: keyword类型字段匹配
* @param @param source
* @param @param fieldlist
* @param @return 设定文件
* @return BoolQueryBuilder 返回类型
*/
public static BoolQueryBuilder assembleSeniorTermOrQuery(String source, String[] fieldlist) {
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
if (!Tools.isEmpty(source)) {
String[] sourcewords = source.split("\\|");
for (int i = 0; i < sourcewords.length; i++) {
String sourceword = sourcewords[i];
boolQuery.should(termQueryField(QueryBuilders.boolQuery(), "should", sourceword, fieldlist));
}
}
return boolQuery;
}
/**
* termQuery 多字段，单一匹配值
*
* @Title: termQueryField
* @Description: termQuery 多字段，单一匹配值
* @param @param boolQueryBuilder
* @param @param esQuery
* @param @param word
* @param @param fields
* @param @return 设定文件
* @return BoolQueryBuilder 返回类型
*/
public static <T> BoolQueryBuilder termQueryField(BoolQueryBuilder boolQueryBuilder, String esQuery, T word,
String[] fields) {
if (ES_QUERY_AND.equals(esQuery)) {
for (String field : fields) {
boolQueryBuilder.must(QueryBuilders.matchPhraseQuery(field, word));
}
} else if (ES_QUERY_OR.equals(esQuery)) {
for (String field : fields) {
boolQueryBuilder.should(QueryBuilders.matchPhraseQuery(field, word));
}
} else if (ES_QUERY_NOT.equals(esQuery)) {
for (String field : fields) {
boolQueryBuilder.mustNot(QueryBuilders.matchPhraseQuery(field, word));
}
}
return boolQueryBuilder;
}
/**
* 标签请求匹配
*
* @param tagIds
......
src/main/java/com/zhiwei/brandkbs2/pojo/MarkSearchAnalyzeTask.java
......@@ -2,6 +2,7 @@ package com.zhiwei.brandkbs2.pojo;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
import com.zhiwei.brandkbs2.pojo.dto.SearchFilterDTO;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.Setter;
......@@ -23,6 +24,10 @@ public class MarkSearchAnalyzeTask extends AbstractBaseMongo {
*/
private MarkSearchDTO markSearchDTO;
/**
* 全网搜搜索条件
*/
private SearchFilterDTO searchFilterDTO;
/**
* 数据分析结果-词云
*/
List<JSONObject> highWord;
......
src/main/java/com/zhiwei/brandkbs2/pojo/dto/SearchFilterDTO.java
......@@ -136,6 +136,12 @@ public class SearchFilterDTO {
@ApiModelProperty(value = "导出数据量")
private int outputCount;
/**
* ip地址
*/
@ApiModelProperty(value = "ip地址")
private String ip;
// /**
// * 这部分字段无需前端传参
// */
......@@ -162,7 +168,7 @@ public class SearchFilterDTO {
public Map<String, Object> toMap() {
Map<String, Object> map = new HashMap<>();
map.put("timeType", "time");
map.put("pt", getPlatforms());
map.put("pt", getDefaultPlatforms());
map.put("dataType", searchType);
map.put("matchType", matchType);
map.put("search", keyword);
......@@ -190,7 +196,7 @@ public class SearchFilterDTO {
return map;
}
public List<String> getPlatforms() {
public List<String> getDefaultPlatforms() {
if (Objects.equals("视频", this.searchType)){
return CollectionUtils.isNotEmpty(this.videoPlatforms) ? this.videoPlatforms : new ArrayList<>(GenericAttribute.VIDEO_PLATFORM);
}
......
src/main/java/com/zhiwei/brandkbs2/service/SearchWholeService.java 0 → 100644
package com.zhiwei.brandkbs2.service;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.pojo.dto.ExportSearchWholeDTO;
import com.zhiwei.brandkbs2.pojo.dto.SearchFilterDTO;
import com.zhiwei.brandkbs2.pojo.vo.PageVO;
import org.apache.commons.lang3.tuple.Pair;
import java.io.IOException;
import java.util.List;
/**
* @ClassName: SearchWholeService
* @Description 全网搜服务
* @author: cjz
* @date: 2024-08-29 9:43
*/
public interface SearchWholeService {
/**
* 搜索-全网搜
*
* @param dto 全网搜搜索传输类
* @return JSONObject
*/
Pair<JSONObject, Integer> searchWholeNetworkWithBalance(SearchFilterDTO dto);
/**
* 搜索-全网搜条件
* @return JSONObject
*/
JSONObject getWholeNetworkSearchCriteria(boolean externalDataSource);
/**
* 搜索-全网搜-舆情导出
* @param dto 全网搜搜索传输类
* @return List<ExportSearchWholeDTO>
*/
List<ExportSearchWholeDTO> exportSearchWhole(SearchFilterDTO dto);
/**
* 全网搜舆情分析-舆情总量
* @param dto 全网搜数据传输类
* @return
* @throws IOException
*/
JSONObject getSearchWholeYuqingAmount(SearchFilterDTO dto);
/**
* 全网搜舆情分析-重点平台
* @param dto 全网搜数据传输类
* @return
*/
JSONObject getSearchWholeImportantPlatformPercentage(SearchFilterDTO dto);
/**
* 全网搜舆情分析-平台分布
* @param dto 全网搜数据传输类
* @return
*/
List<JSONObject> getSearchWholeAnalyzePlatformPercentage(SearchFilterDTO dto);
/**
* 全网搜舆情分析-舆情走势图
* @param dto 全网搜数据传输类
* @return
*/
JSONObject getSearchWholeSpreadTendency(SearchFilterDTO dto);
/**
* 全网搜舆情分析-活跃渠道
* @param dto 全网搜数据传输类
* @return
*/
List<JSONObject> getSearchWholeActiveChannels(SearchFilterDTO dto);
/**
* 全网搜舆情分析-ip分布
* @param dto 全网搜数据传输类
* @return
*/
List<JSONObject> getSearchWholeArticleIpLocated(SearchFilterDTO dto);
/**
* 全网搜舆情分析-词云
* @param dto 全网搜数据传输类
* @return
*/
List<JSONObject> getSearchWholeAnalyzeHighWord(SearchFilterDTO dto);
/**
* 全网搜舆情分析-活跃渠道、ip分布、词云详情页面、社媒平台发文
* @param dto 标注数据搜索传输类
* @return
*/
PageVO<JSONObject> getSearchWholeAnalyzeDetail(SearchFilterDTO dto);
}
src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java
......@@ -15,10 +15,7 @@ import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.dao.*;
import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportYuqingInteractionUpdateDTO;
import com.zhiwei.brandkbs2.enmus.ChannelEmotion;
import com.zhiwei.brandkbs2.enmus.EmotionEnum;
import com.zhiwei.brandkbs2.enmus.ImportantChannelEnum;
import com.zhiwei.brandkbs2.enmus.InteractionEnum;
import com.zhiwei.brandkbs2.enmus.*;
import com.zhiwei.brandkbs2.es.EsClientDao;
import com.zhiwei.brandkbs2.es.EsQueryTools;
import com.zhiwei.brandkbs2.exception.ExceptionCast;
......@@ -1389,6 +1386,7 @@ public class MarkDataServiceImpl implements MarkDataService {
}
@Override
@Deprecated
public Pair<JSONObject, Integer> searchWholeNetworkWithBalance(SearchFilterDTO dto) {
if (dto.isExternalDataSource()) { // 查商业数据库
try {
......@@ -1442,6 +1440,7 @@ public class MarkDataServiceImpl implements MarkDataService {
}
@Override
@Deprecated
public List<ExportSearchWholeDTO> exportSearchWhole(SearchFilterDTO dto) {
Integer exportAmount = userService.queryUserInfo(UserThreadLocal.getUserId(), UserThreadLocal.getProjectId()).getExportAmount();
exportAmount = Objects.isNull(exportAmount) ? 10000 : exportAmount;
......@@ -1973,6 +1972,9 @@ public class MarkDataServiceImpl implements MarkDataService {
if (nonManualProjectPlanDao.count(new Query(Criteria.where("projectId").is(projectId))) >= 5) {
return ResponseResult.failure("超过方案数量上限5");
}
if (!dto.getLasting() && Objects.isNull(dto.getEndTime())){
dto.setEndTime(System.currentTimeMillis() + Constant.ONE_YEAR);
}
NonManualProjectPlan plan = new NonManualProjectPlan(dto);
Project project = projectService.getProjectById(projectId);
if (project.getExternalDataSource()) {
......@@ -2025,6 +2027,9 @@ public class MarkDataServiceImpl implements MarkDataService {
@Override
public ResponseResult updateNonManualProjectPlan(NonManualProjectPlanDTO dto) {
if (!dto.getLasting() && Objects.isNull(dto.getEndTime())){
dto.setEndTime(System.currentTimeMillis() + Constant.ONE_YEAR);
}
NonManualProjectPlan existPlan = nonManualProjectPlanDao.findOneById(dto.getId());
NonManualProjectPlan plan = new NonManualProjectPlan(dto);
plan.setCTime(existPlan.getCTime());
......@@ -2625,7 +2630,10 @@ public class MarkDataServiceImpl implements MarkDataService {
// 项目日均
String projectYuqingCountAvgKey = RedisUtil.getYuqingAnalyzeProjectAvgCountKey(projectId, Constant.PRIMARY_CONTEND_ID, planId, null, false);
if (Objects.isNull(redisUtil.get(projectYuqingCountAvgKey))) {
countYuqingAmountAvg(dto.getStartTime(), dto.getEndTime(), projectId, planId);
Project project = projectService.getProjectById(projectId);
long endTime = System.currentTimeMillis();
long startTime = project.getCTime() > endTime - Constant.ONE_YEAR ? project.getCTime() : endTime - Constant.ONE_YEAR;
countYuqingAmountAvg(startTime, endTime, projectId, planId);
}
double projectYuqingAvgCount = Objects.isNull(redisUtil.get(projectYuqingCountAvgKey)) ? 0d : Double.parseDouble(redisUtil.get(projectYuqingCountAvgKey));
jsonObject.put("projectAvg", projectYuqingAvgCount);
......@@ -2674,10 +2682,13 @@ public class MarkDataServiceImpl implements MarkDataService {
String neutralKey = RedisUtil.getYuqingAnalyzeEmotionDistributionAvgKey(projectId, Constant.PRIMARY_CONTEND_ID, EmotionEnum.NEUTRAL.getName(), planId, customPlanId, isCustom);
String negativeKey = RedisUtil.getYuqingAnalyzeEmotionDistributionAvgKey(projectId, Constant.PRIMARY_CONTEND_ID, EmotionEnum.NEGATIVE.getName(), planId, customPlanId, isCustom);
if (Objects.isNull(redisUtil.get(positiveKey)) || Objects.isNull(redisUtil.get(neutralKey)) || Objects.isNull(redisUtil.get(negativeKey))){
Project project = projectService.getProjectById(projectId);
long endTime = System.currentTimeMillis();
long startTime = project.getCTime() > endTime - Constant.ONE_YEAR ? project.getCTime() : endTime - Constant.ONE_YEAR;
if (isCustom){
countCustomEmotionDistributionAvg(dto.getStartTime(), dto.getEndTime(), projectId, customPlanId);
countCustomEmotionDistributionAvg(startTime, endTime, projectId, customPlanId);
}else {
countEmotionDistributionAvg(dto.getStartTime(), dto.getEndTime(), projectId, planId);
countEmotionDistributionAvg(startTime, endTime, projectId, planId);
}
}
double positiveAvg = Objects.isNull(redisUtil.get(positiveKey)) ? 0d : Double.parseDouble(redisUtil.get(positiveKey));
......@@ -2734,7 +2745,10 @@ public class MarkDataServiceImpl implements MarkDataService {
// 项目均值
String key = RedisUtil.getYuqingAnalyzePlatformAvgCountKey(projectId, Constant.PRIMARY_CONTEND_ID, dto.getPlanId());
if (Objects.isNull(redisUtil.get(key))){
countImportantPlatformPercentageAvg(dto.getStartTime(), dto.getEndTime(), projectId, dto.getPlanId());
Project project = projectService.getProjectById(projectId);
long endTime = System.currentTimeMillis();
long startTime = project.getCTime() > endTime - Constant.ONE_YEAR ? project.getCTime() : endTime - Constant.ONE_YEAR;
countImportantPlatformPercentageAvg(startTime, endTime, projectId, dto.getPlanId());
}
double projectAvg = Objects.isNull(redisUtil.get(key)) ? 0d : Double.parseDouble(redisUtil.get(key));
overview.put("compare", compare(avg, projectAvg));
......@@ -3105,7 +3119,7 @@ public class MarkDataServiceImpl implements MarkDataService {
if (Objects.nonNull(existTask)){
return existTask.getHighWord();
}
MarkSearchAnalyzeTask task = new MarkSearchAnalyzeTask(dto, null, false, System.currentTimeMillis());
MarkSearchAnalyzeTask task = new MarkSearchAnalyzeTask(dto, null,null, false, System.currentTimeMillis());
markSearchAnalyzeTaskDao.insertOne(task);
List<JSONObject> highWord = generateHighWord(dto);
Update update = new Update();
......
src/main/java/com/zhiwei/brandkbs2/service/impl/SearchWholeServiceImpl.java 0 → 100644
package com.zhiwei.brandkbs2.service.impl;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.base.category.ClassB;
import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.common.GlobalPojo;
import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.dao.MarkSearchAnalyzeTaskDao;
import com.zhiwei.brandkbs2.enmus.SensitiveChannelsEnum;
import com.zhiwei.brandkbs2.es.EsClientDao;
import com.zhiwei.brandkbs2.es.EsQueryTools;
import com.zhiwei.brandkbs2.exception.ExceptionCast;
import com.zhiwei.brandkbs2.model.CommonCodeEnum;
import com.zhiwei.brandkbs2.pojo.BaseMap;
import com.zhiwei.brandkbs2.pojo.MarkSearchAnalyzeTask;
import com.zhiwei.brandkbs2.pojo.Project;
import com.zhiwei.brandkbs2.pojo.dto.ExportSearchWholeDTO;
import com.zhiwei.brandkbs2.pojo.dto.SearchFilterDTO;
import com.zhiwei.brandkbs2.pojo.vo.LineVO;
import com.zhiwei.brandkbs2.pojo.vo.PageVO;
import com.zhiwei.brandkbs2.service.*;
import com.zhiwei.brandkbs2.util.RedisUtil;
import com.zhiwei.brandkbs2.util.TextUtil;
import com.zhiwei.brandkbs2.util.Tools;
import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform;
import com.zhiwei.qbjc.bean.tools.BeanTools;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.script.Script;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.BucketOrder;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.bucket.histogram.ParsedDateHistogram;
import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.sort.FieldSortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.joda.time.Period;
import org.joda.time.PeriodType;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.mongodb.core.query.Update;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
import javax.annotation.Resource;
import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
* @ClassName: SearchWholeService
* @Description 全网搜服务
* @author: cjz
* @date: 2024-08-29 9:43
*/
@Service("searchWholeServiceImpl")
public class SearchWholeServiceImpl implements SearchWholeService {
private static final Logger log = LogManager.getLogger(SearchWholeServiceImpl.class);
@Resource(name = "esClientDao")
private EsClientDao esClientDao;
@Resource(name = "markSearchAnalyzeTaskDao")
MarkSearchAnalyzeTaskDao markSearchAnalyzeTaskDao;
@Resource(name = "redisUtil")
RedisUtil redisUtil;
@Resource(name = "textUtil")
TextUtil textUtil;
@Resource(name = "restTemplate")
private RestTemplate restTemplate;
@Resource(name = "markDataServiceImpl")
MarkDataService markDataService;
@Resource(name = "projectServiceImpl")
private ProjectService projectService;
@Resource(name = "userServiceImpl")
private UserService userService;
@Resource(name = "commonServiceImpl")
private CommonService commonService;
@Resource(name = "systemInfoServiceImpl")
SystemInfoService systemInfoService;
@Value("${istarshine.addIStarShineKSData.url}")
private String addIStarShineKSDataUrl;
@Value("${istarshine.getIStarShineKSInfoData.url}")
private String getIStarShineKSInfoDataUrl;
@Value("${whole.search.url}")
private String wholeSearchUrl;
@Override
public Pair<JSONObject, Integer> searchWholeNetworkWithBalance(SearchFilterDTO dto) {
if (dto.isExternalDataSource()) { // 查商业数据库
try {
JSONObject data = getSearchWholeOpinionJson(dto);
JSONObject json = packageExternalDataSourceResult(data);
// 仅当第一页时统计平台进量（声量）
if (1 == dto.getPage()) {
json.put("info", getExternalDataSourcePlatformCount(dto));
}
return Pair.of(json, json.getInteger("pageLimit"));
} catch (Exception e) {
log.error("error searchWholeNetwork - ", e);
return Pair.of(null, null);
}
} else { // 查舆情库
ResponseEntity<JSONObject> responseEntity = restTemplate.postForEntity(wholeSearchUrl, dto.toMap(), JSONObject.class);
return Pair.of(searchWholeOpinion(responseEntity), null);
}
}
@Override
public JSONObject getWholeNetworkSearchCriteria(boolean externalDataSource) {
JSONObject result = new JSONObject();
// 搜索时间
result.put("times", Arrays.asList("今天", "24小时", "三天", "七天", "近30天"));
result.put("ninetyDays", DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), -89).getTime());
// 外部数据源（实时采集）条件
if (externalDataSource) {
result.put("platformList", Arrays.asList("App", "博客", "平媒", "新闻", "微博", "微信", "论坛", "视频", "短视频"));
} else {
result.put("platformList", commonService.getQbjcPlatform("id", "name")
.stream().filter(s -> !s.get("name").equals("脉脉")).collect(Collectors.toList()));
result.putAll(systemInfoService.getExtraParam());
}
return result;
}
@Override
public List<ExportSearchWholeDTO> exportSearchWhole(SearchFilterDTO dto) {
Integer exportAmount = userService.queryUserInfo(UserThreadLocal.getUserId(), UserThreadLocal.getProjectId()).getExportAmount();
exportAmount = Objects.isNull(exportAmount) ? 10000 : exportAmount;
JSONArray jsonArray = new JSONArray();
dto.setPageSize(50);
dto.setPage(1);
while (true) {
if (dto.getPage() * dto.getPageSize() > exportAmount) {
break;
}
// 获取当页数据
JSONObject jsonObject = searchWholeNetworkWithBalance(dto).getLeft();
JSONArray array = jsonObject.getJSONArray("list");
if (Objects.isNull(array) || 0 == array.size()) {
break;
}
jsonArray.addAll(array);
dto.setPage(dto.getPage() + 1);
}
return jsonArray.stream().map(json -> ExportSearchWholeDTO.creatExportSearchWholeDTO((JSONObject) json)).collect(Collectors.toList());
}
/**
* 舆情库数据结果组装成标准分页返回格式
*
* @param responseEntity
* @return
*/
private JSONObject searchWholeOpinion(ResponseEntity<JSONObject> responseEntity) {
JSONObject res = new JSONObject();
JSONObject bodyData = Objects.requireNonNull(responseEntity.getBody()).getJSONObject("data");
JSONObject data = bodyData.getJSONObject("data");
JSONArray listArray = data.getJSONArray("list");
List<JSONObject> resultList = new ArrayList<>();
for (Object o : listArray) {
JSONObject jsonObject = (JSONObject) o;
JSONObject result = searchWholeDataProcess(jsonObject);
resultList.add(result);
}
// 仅第一页返回平台进量（声量）统计
if (1 == data.getInteger("page")) {
JSONObject count = bodyData.getJSONObject("count");
List<JSONObject> platformCounts = count.getJSONArray("count").toJavaList(JSONObject.class);
// 去除外媒和脉脉两个平台
List<JSONObject> list = platformCounts.stream().filter(json -> !Objects.equals(json.getString("name"), "脉脉") &&
!Objects.equals(json.getString("name"), "外媒")).collect(Collectors.toList());
// 重新计算总数
Long total = list.stream().map(json -> json.getLongValue("count")).reduce(Long::sum).orElse(0L);
JSONObject counts = new JSONObject();
counts.put("count", list);
counts.put("total", total);
res.put("info", counts);
}
res.put("list", resultList);
res.put("page", data.get("page"));
res.put("pageLimit", data.get("pageLimit"));
res.put("total", data.get("total"));
res.put("totalPage", data.get("totalPage"));
res.put("isBackUp", false);
return res;
}
/**
* 获取实时采集（外部数据源）平台进量（声量）
*
* @param dto
* @return
*/
private JSONObject getExternalDataSourcePlatformCount(SearchFilterDTO dto) {
JSONObject res = new JSONObject();
List<JSONObject> list = new ArrayList<>();
List<String> platforms = Arrays.asList("App", "博客", "平媒", "新闻", "微博", "微信", "论坛", "视频", "短视频");
for (String platform : platforms) {
// 重置平台条件
dto.setPlatforms(Collections.singletonList(platform));
JSONObject data = getSearchWholeOpinionJson(dto);
JSONObject task = data.getJSONObject("task");
JSONObject jsonObject = new JSONObject();
jsonObject.put("count", task.getLongValue("searchCount"));
jsonObject.put("name", platform);
list.add(jsonObject);
}
Long total = list.stream().map(jsonObject -> jsonObject.getLongValue("count")).reduce(Long::sum).orElse(0L);
res.put("count", list);
res.put("total", total);
return res;
}
/**
* 将外部数据源JSONObject组装成标准的返回结果
*
* @param data
* @return
*/
private JSONObject packageExternalDataSourceResult(JSONObject data) {
JSONObject jsonObject = new JSONObject();
List<JSONObject> resultList = new ArrayList<>();
JSONObject task = data.getJSONObject("task");
Integer receiveCount = task.getInteger("receiveCount");
Integer searchCount = task.getInteger("searchCount");
int totalPage;
if (0 == receiveCount || 0 == searchCount) {
totalPage = 0;
} else {
totalPage = searchCount % receiveCount == 0 ? searchCount / receiveCount : (searchCount / receiveCount + 1);
}
List<JSONObject> listArray = data.getJSONArray("data").toJavaList(JSONObject.class);
for (JSONObject json : listArray) {
JSONObject result = new JSONObject();
String platform = getPlatformExternal(json.getJSONObject("gather"));
result.put("platform", platform);
result.put("title", json.get("title"));
result.put("content", json.get("content"));
// 渠道获取分平台
result.put("source", getSourceExternal(json, getExternalType(json, platform)));
result.put("url", getUrl(json));
result.put("time", json.getString("ctime").length() == 10 ? json.getLong("ctime") * 1000L : json.getLong("ctime"));
resultList.add(result);
}
jsonObject.put("list", resultList);
//jsonObject.put("page",data1.get("page"));
jsonObject.put("pageLimit", receiveCount);
jsonObject.put("total", searchCount);
jsonObject.put("totalPage", totalPage);
jsonObject.put("balance", projectService.getProjectById(UserThreadLocal.getProjectId()).getWholeSearchBalance() - receiveCount);
jsonObject.put("isBackUp", true);
return jsonObject;
}
private ClassB.TypeB getExternalType(JSONObject tJson, String platform) {
int wtype = tJson.getIntValue("wtype");
if (platform.equals("微博") && 2 == wtype) {
return ClassB.TypeB.INCOMPLETE;
} else if (platform.equals("微博") && 1 == wtype) {
return ClassB.TypeB.INCOMPLETE;
} else if (platform.equals("视频") || platform.equals("短视频")) {
return ClassB.TypeB.VIDEO;
}
return ClassB.TypeB.COMPLETE;
}
private String getSourceExternal(JSONObject tJson, ClassB.TypeB typeB) {
String source = null;
switch (typeB) {
case COMPLETE:
case INCOMPLETE:
case VIDEO:
if (!StringUtils.isEmpty(tJson.getString("user"))) {
source = tJson.getJSONObject("user").getString("name");
} else {
source = tJson.getString("repostSource");
}
break;
default:
break;
}
return source;
}
/**
* 获取全网搜接口信息url
*
* @param json 数据
* @return url链接
*/
private static String getUrl(JSONObject json) {
String url = json.getString("url");
// 如果url为空,则查询QA类型url字段
if (null == url) {
url = json.getString("answer_url");
}
if (null == url) {
url = json.getString("question_url");
}
return url;
}
private String getPlatformExternal(JSONObject gather) {
String siteDomain = gather.getString("site_domain");
String infoFlag = JSON.toJSONString(gather.get("info_flag"));
if (Objects.isNull(siteDomain) && gather.getString("site_name").contains("今日头条")) {
return "今日头条";
}
if (Objects.nonNull(siteDomain) && siteDomain.equals("toutiao.com")) {
return "今日头条";
}
if (infoFlag.contains("06")) {
return "微信";
}
if (infoFlag.contains("04") || infoFlag.contains("08")) {
return "微博";
}
if (infoFlag.contains("02")) {
return "贴吧论坛";
}
if (infoFlag.contains("0105")) {
return "平媒";
}
if (infoFlag.contains("0109")) {
return "自媒体";
}
if (infoFlag.contains("07")) {
return "视频";
}
if (infoFlag.contains("11")) {
return "短视频";
}
// '01'：新闻, '0105'：平媒,0109'：APP, '02'：论坛, '03'：博客, '04'：微博,'0401':新浪微博, '06'：微信, '07'： 视频,
// '08'：长微博,' '11'： 小视频, '12'：境外资讯,'1201': 境外,'1202'：外文, '13': 境外社交媒体,'1301': Twitter',1302': Facebook, '99':搜索
return "网媒";
}
/**
* 获取外部数据源搜索结果json
*
* @param dto
* @return
*/
private JSONObject getSearchWholeOpinionJson(SearchFilterDTO dto) {
// 组装请求参数
JSONObject taskEntity = packageExternalDataSourceParams(dto);
ResponseEntity<JSONObject> responseEntity = restTemplate.postForEntity(addIStarShineKSDataUrl, taskEntity, JSONObject.class);
JSONObject result = Objects.requireNonNull(responseEntity.getBody()).getJSONObject("data");
return searchWholeBackUp(result.get("id").toString());
}
/**
* 获取外部数据源搜索结果JSONObject
*
* @param taskId
* @return
*/
private JSONObject searchWholeBackUp(String taskId) {
JSONObject jsonObject = new JSONObject();
for (int i = 0; i < 30; i++) {
ResponseEntity<JSONObject> responseEntity = restTemplate.getForEntity(getIStarShineKSInfoDataUrl, JSONObject.class, taskId);
JSONObject data = Objects.requireNonNull(responseEntity.getBody()).getJSONObject("data");
JSONObject task = data.getJSONObject("task");
if (2 == task.getInteger("crawlerFinish")) {
throw new IllegalArgumentException(task.getString("searchResultMsg"));
}
if (1 == task.getInteger("crawlerFinish")) {
return data;
}
Tools.sleep(500L);
}
return jsonObject;
}
/**
* 外部数据源组装请求参数
*
* @param dto
* @return
*/
private JSONObject packageExternalDataSourceParams(SearchFilterDTO dto) {
JSONObject params = externalTransform(dto);
JSONObject taskEntity = new JSONObject();
taskEntity.put("appId", "6183571e0d710000f6003a12"); // 应用id, 由张志伟提供给使用者
taskEntity.put("taskType", null); // 可为null, 任务类型
taskEntity.put("taskDescription", null); // 可为null, 任务描述
taskEntity.put("taskName", UserThreadLocal.getProjectId() + "_" + UserThreadLocal.getUserId());
taskEntity.put("userName", "品见"); // 提交任务的人
taskEntity.put("searchParams", params); // 任务详情参数
return taskEntity;
}
/**
* 外部数据源字段转换
*
* @param dto
*/
private JSONObject externalTransform(SearchFilterDTO dto) {
JSONObject params = new JSONObject();
params.put("crawlerWord", dto.getKeyword());
params.put("startTime", dto.getStartTime());
params.put("endTime", dto.getEndTime());
String platforms;
// 平台为空时，传全部
if (Tools.isEmpty(dto.getPlatforms())) {
platforms = "App,博客,平媒,新闻,微博,微信,论坛,视频,短视频";
} else {
platforms = StringUtils.join(dto.getPlatforms(), ",");
}
params.put("platforms", platforms);
params.put("sortOf", "ctime+" + (dto.getSorter().toJSONString().contains("asc") ? "asc" : "desc"));
params.put("page", dto.getPage());
params.put("pageSize", dto.getPageSize());
params.put("crawlerTime", dto.getCrawlerTime());
params.put("filterWord", dto.getFilterWords());
// matchFields
if (dto.getMatchType().equals("标题")) {
params.put("matchFields", Collections.singletonList("title"));
}else if (dto.getMatchType().equals("全文")){
params.put("matchFields", Collections.singletonList("content"));
}
// sourceKeyword
if (StringUtils.isNotBlank(dto.getSourceKeyword())){
params.put("userList", Arrays.asList(dto.getSourceKeyword().trim().split("\\|")));
}
// // 历史部分
// if (Tools.isEmpty(dto.getPlatforms())) {
// dto.setPlatformsStr("App,博客,平媒,新闻,微博,微信,论坛,视频,短视频");
// }
return params;
}
@Override
public JSONObject getSearchWholeYuqingAmount(SearchFilterDTO dto) {
JSONObject jsonObject = new JSONObject();
try {
String projectId = UserThreadLocal.getProjectId();
String[] indexes = esClientDao.getAllIndexes(dto.getStartTime(), dto.getEndTime());
BoolQueryBuilder postFilter = searchWholeAnalyzeQuery(dto);
// 舆情总量
long total = esClientDao.count(indexes, null, postFilter);
jsonObject.put("total", total);
// 项目日均
String projectYuqingCountAvgKey = RedisUtil.getYuqingAnalyzeProjectAvgCountKey(projectId, Constant.PRIMARY_CONTEND_ID, null, null, false);
if (Objects.isNull(redisUtil.get(projectYuqingCountAvgKey))) {
Project project = GlobalPojo.PROJECT_MAP.get(projectId);
long endTime = System.currentTimeMillis();
long startTime = project.getCTime() > endTime - Constant.ONE_YEAR ? project.getCTime() : endTime - Constant.ONE_YEAR;
markDataService.countYuqingAmountAvg(startTime, endTime, projectId, null);
}
double projectYuqingAvgCount = Objects.isNull(redisUtil.get(projectYuqingCountAvgKey)) ? 0d : Double.parseDouble(redisUtil.get(projectYuqingCountAvgKey));
jsonObject.put("projectAvg", projectYuqingAvgCount);
// 本次日均
int days = new Period(dto.getStartTime(), dto.getEndTime(), PeriodType.days()).getDays();
days = 0 == days ? 1 : days;
double yuqingAvgCount = total == 0 ? 0d : total / (double) days;
jsonObject.put("compare", compare(yuqingAvgCount, projectYuqingAvgCount));
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "全网搜舆情分析getSearchWholeYuqingAmount异常-", e);
}
return jsonObject;
}
@Override
public JSONObject getSearchWholeImportantPlatformPercentage(SearchFilterDTO dto) {
JSONObject res = new JSONObject();
try {
List<JSONObject> list = new ArrayList<>();
if (CollectionUtils.isEmpty(dto.getPlatforms())) {
dto.setPlatforms(Stream.of("微博", "微信", "抖音", "小红书", "今日头条", "网媒").map(GlobalPojo::getPlatformIdByName).collect(Collectors.toList()));
}
// indexes
String[] indexes = esClientDao.getAllIndexes(dto.getStartTime(), dto.getEndTime());
// query
BoolQueryBuilder postFilter = searchWholeAnalyzeQuery(dto);
long total = esClientDao.count(indexes, postFilter, null);
// 平台聚合
Script script = new Script("doc['c5'].value +'_' +doc['foreign'].value");
TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("agg").script(script).size(1000);
SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, postFilter, aggregationBuilder, null, null, 0, 0, null);
// buckets
Map<String, Aggregation> aggregationMap = searchResponse.getAggregations().asMap();
ParsedStringTerms terms = (ParsedStringTerms) aggregationMap.get("agg");
List<? extends Terms.Bucket> buckets = terms.getBuckets();
Map<String, Long> platformCount = new HashMap<>();
for (Terms.Bucket bucket : buckets) {
String[] split = bucket.getKeyAsString().split("_");
String platform = BeanTools.filterPlatform(GlobalPojo.PLATFORMS, Integer.valueOf(split[0]), Integer.valueOf(split[1])).getName();
if (StringUtils.isBlank(platform)){
continue;
}
platformCount.compute(platform, (k, v) -> Objects.isNull(v) ? bucket.getDocCount() : v + bucket.getDocCount());
}
for (Map.Entry<String, Long> entry : platformCount.entrySet()) {
JSONObject platformResult = new JSONObject();
platformResult.put("platform", entry.getKey());
platformResult.put("count", entry.getValue());
list.add(platformResult);
}
res.put("platformCount", list.stream().sorted(Comparator.comparingLong((JSONObject json) -> json.getLongValue("count")).reversed()));
JSONObject overview = new JSONObject();
// 总量
overview.put("total", total);
// 日均
int days = new Period(dto.getStartTime(), dto.getEndTime(), PeriodType.days()).getDays();
days = 0 == days ? 1 : days;
double avg = total == 0 ? 0d : total / (double) days;
// 项目均值
String projectId = UserThreadLocal.getProjectId();
String key = RedisUtil.getYuqingAnalyzePlatformAvgCountKey(projectId, Constant.PRIMARY_CONTEND_ID, null);
if (Objects.isNull(redisUtil.get(key))){
Project project = GlobalPojo.PROJECT_MAP.get(projectId);
long endTime = System.currentTimeMillis();
long startTime = project.getCTime() > endTime - Constant.ONE_YEAR ? project.getCTime() : endTime - Constant.ONE_YEAR;
markDataService.countImportantPlatformPercentageAvg(startTime, endTime, projectId, null);
}
double projectAvg = Objects.isNull(redisUtil.get(key)) ? 0d : Double.parseDouble(redisUtil.get(key));
overview.put("compare", compare(avg, projectAvg));
overview.put("avg", projectAvg);
res.put("overview", overview);
return res;
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "全网搜舆情分析getSearchWholeImportantPlatformPercentage异常-", e);
}
return res;
}
@Override
public List<JSONObject> getSearchWholeAnalyzePlatformPercentage(SearchFilterDTO dto) {
try {
if (CollectionUtils.isEmpty(dto.getPlatforms())) {
List<String> platforms = GlobalPojo.PLATFORMS.stream().map(MessagePlatform::getId).collect(Collectors.toList());
dto.setPlatforms(platforms);
}
List<JSONObject> list = new ArrayList<>();
// indexes
String[] indexes = esClientDao.getAllIndexes(dto.getStartTime(), dto.getEndTime());
// query
BoolQueryBuilder postFilter = searchWholeAnalyzeQuery(dto);
// total
long total = esClientDao.count(indexes, postFilter, null);
// 平台聚合
Script script = new Script("doc['c5'].value +'_' +doc['foreign'].value");
TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("agg").script(script).size(1000).order(BucketOrder.count(false));
SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, postFilter, aggregationBuilder, null, null, 0, 0, null);
// buckets
Map<String, Aggregation> aggregationMap = searchResponse.getAggregations().asMap();
ParsedStringTerms terms = (ParsedStringTerms) aggregationMap.get("agg");
List<? extends Terms.Bucket> buckets = terms.getBuckets();
Map<String, Integer> platformCount = new HashMap<>();
for (Terms.Bucket bucket : buckets) {
String[] split = bucket.getKeyAsString().split("_");
String platform = BeanTools.filterPlatform(GlobalPojo.PLATFORMS, Integer.valueOf(split[0]), Integer.valueOf(split[1])).getName();
if (StringUtils.isBlank(platform)){
continue;
}
platformCount.compute(platform, (k, v) -> Objects.isNull(v) ? (int) bucket.getDocCount() : v + (int) bucket.getDocCount());
}
// 取前9
Map<String, Integer> sortMap = Tools.sortMap(platformCount, 9);
for (Map.Entry<String, Integer> entry : sortMap.entrySet()) {
JSONObject platformResult = new JSONObject();
platformResult.put("platform", entry.getKey());
platformResult.put("percent", 0 == total ? 0d : entry.getValue() / (double) total);
list.add(platformResult);
}
// 第10个算做其他平台
JSONObject other = new JSONObject();
other.put("platform", "其他");
other.put("percent", 1 - list.stream().mapToDouble(count -> count.getDoubleValue("percent")).sum());
list.add(other);
return list;
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "全网搜舆情分析geAnalyzePlatformPercentage异常-", e);
}
return Collections.emptyList();
}
@Override
public JSONObject getSearchWholeSpreadTendency(SearchFilterDTO dto) {
JSONObject res = new JSONObject();
try {
List<LineVO> totalSpreadTendency = getTotalSpreadTendency(dto);
Map<String, List<LineVO>> platformSpreadTendency = getPlatformSpreadTendency(dto);
BaseMap firstArticle = getFirstArticle(dto, totalSpreadTendency);
JSONObject highestJson = new JSONObject();
if (Objects.nonNull(firstArticle)) {
String title = Objects.isNull(firstArticle.getTitle()) || Tools.filterUselessTitle(firstArticle.getTitle())
? firstArticle.getContent() : firstArticle.getTitle();
highestJson.put("title", StringUtils.substring(title, 0, 15));
highestJson.put("url", firstArticle.getUrl());
}
res.putAll(platformSpreadTendency);
res.put("总量", totalSpreadTendency);
res.put("highest", highestJson);
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "全网搜舆情分析getSearchWholeSpreadTendency异常-", e);
}
return res;
}
@Override
public List<JSONObject> getSearchWholeActiveChannels(SearchFilterDTO dto) {
List<JSONObject> list = new ArrayList<>();
try {
// 索引
String[] indexes = esClientDao.getAllIndexes(dto.getStartTime(), dto.getEndTime());
// query
BoolQueryBuilder query = searchWholeAnalyzeQuery(dto);
// 渠道聚合，取前20
TermsAggregationBuilder sourceAggregationBuilder = AggregationBuilders.terms("source_count").field("source").order(BucketOrder.count(false)).size(20 + Tools.FILTER_SOURCE.size());
SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, query,
sourceAggregationBuilder, null, null, 0, 0, null);
Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
ParsedStringTerms sourceCountTeam = (ParsedStringTerms) aggMap.get("source_count");
List<? extends Terms.Bucket> buckets = sourceCountTeam.getBuckets();
for (Terms.Bucket bucket : buckets) {
JSONObject jsonObject = new JSONObject();
String source = bucket.getKeyAsString();
// 过滤渠道名：”匿名“
if (Tools.filterUselessSource(source)){
continue;
}
jsonObject.put("source", source);
// 发文次数
jsonObject.put("count", bucket.getDocCount());
list.add(jsonObject);
}
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "全网搜舆情分析getActiveChannels异常-", e);
}
return list.stream().limit(20).collect(Collectors.toList());
}
@Override
public List<JSONObject> getSearchWholeArticleIpLocated(SearchFilterDTO dto) {
List<JSONObject> list = new ArrayList<>();
try {
// 索引
String[] indexes = esClientDao.getAllIndexes(dto.getStartTime(), dto.getEndTime());
// query
BoolQueryBuilder query = searchWholeAnalyzeQuery(dto);
// 聚合请求
TermsAggregationBuilder ipAggregationBuilder = AggregationBuilders.terms("ip_count").field("ip_location").order(BucketOrder.count(false)).size(10);
SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, query,
ipAggregationBuilder, null, null, 0, 0, null);
Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
ParsedStringTerms sourceCountTeam = (ParsedStringTerms) aggMap.get("ip_count");
List<? extends Terms.Bucket> buckets = sourceCountTeam.getBuckets();
buckets.forEach(bucket -> {
JSONObject jsonObject = new JSONObject();
jsonObject.put("ip", bucket.getKeyAsString());
jsonObject.put("count", bucket.getDocCount());
list.add(jsonObject);
});
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "全网搜舆情分析getArticleIpLocated异常-", e);
}
return list;
}
@Override
public List<JSONObject> getSearchWholeAnalyzeHighWord(SearchFilterDTO dto) {
MarkSearchAnalyzeTask existTask = markSearchAnalyzeTaskDao.findOne("searchFilterDTO", dto);
if (Objects.nonNull(existTask)){
return existTask.getHighWord();
}
MarkSearchAnalyzeTask task = new MarkSearchAnalyzeTask(null, dto,null, false, System.currentTimeMillis());
markSearchAnalyzeTaskDao.insertOne(task);
List<JSONObject> highWord = generateHighWord(dto);
Update update = new Update();
update.set("finish", true);
update.set("highWord", highWord);
markSearchAnalyzeTaskDao.updateOneByIdWithField(task.getId(), update);
return highWord;
}
@Override
public PageVO<JSONObject> getSearchWholeAnalyzeDetail(SearchFilterDTO dto) {
try {
// 索引
String[] indexes = esClientDao.getAllIndexes(dto.getStartTime(), dto.getEndTime());
// query
BoolQueryBuilder query = searchWholeAnalyzeQuery(dto);
long total = esClientDao.count(indexes, query, null);
// sort
FieldSortBuilder sort = null;
for (Map.Entry<String, Object> entry : dto.getSorter().entrySet()) {
String key = entry.getKey();
if (null == entry.getValue() || entry.getValue().toString().contains("desc")) {
sort = SortBuilders.fieldSort(key).order(SortOrder.DESC);
} else {
sort = SortBuilders.fieldSort(key).order(SortOrder.ASC);
}
}
int from = (dto.getPage() - 1) * dto.getPageSize();
int size = dto.getPageSize();
SearchHits searchHits = esClientDao.searchHits(indexes, query, null, null, sort, from, size, null);
List<JSONObject> resList = Arrays.stream(searchHits.getHits()).map(hit -> searchWholeDataProcess(new JSONObject(hit.getSourceAsMap()))).collect(Collectors.toList());
JSONObject jsonObject = new JSONObject();
// 高亮关键词
jsonObject.put("highlightWord", GlobalPojo.HIGHLIGHT_MAP.get(UserThreadLocal.getProjectId()));
return PageVO.createPageVo(total, dto.getPage(), dto.getPageSize(), resList).setInfo(jsonObject);
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "全网搜舆情分析getSearchWholeAnalyzeDetail异常-", e);
}
return null;
}
private JSONObject searchWholeDataProcess(JSONObject jsonObject){
JSONObject result = new JSONObject();
String platform = BeanTools.filterPlatform(GlobalPojo.PLATFORMS, jsonObject.getIntValue(GenericAttribute.ES_C5), jsonObject.getIntValue(GenericAttribute.ES_FOREIGN)).getName();
platform = Objects.isNull(platform) ? jsonObject.getString("real_source") : platform;
result.put("platform", platform);
JSONArray contentArray = jsonObject.getJSONArray("ind_full_text");
result.put("title", contentArray.get(0));
result.put("content", contentArray.get(1));
if (3 == contentArray.size()) {
result.put("content", result.getString("content") + contentArray.get(2));
}
result.put("channel", jsonObject.getString("source"));
String url = jsonObject.getString("url");
// 如果url为空,则查询question_url字段
if (null == url) {
url = jsonObject.getString("question_url");
}
result.put("url", url);
result.put("time", jsonObject.getLongValue("time"));
result.put("avatarUrl", jsonObject.getString("avatar_url"));
result.put("source", jsonObject.getString("source"));
result.put("followersNum", jsonObject.getIntValue("followers_num"));
result.put("realSource", jsonObject.getString("real_source"));
result.put("ipLocation", jsonObject.getString("ip_location"));
result.put("picUrls", jsonObject.getString("pic_urls"));
result.put("videoUrls", jsonObject.getJSONArray("video_urls"));
// 视频时长处理
if (StringUtils.isNotBlank(jsonObject.getString("length"))){
result.put("length", Tools.formatVideoLength(jsonObject.getString("length")));
}
return result;
}
private List<JSONObject> generateHighWord(SearchFilterDTO dto){
List<JSONObject> res = new ArrayList<>();
try {
EsClientDao.SearchHelper searchHelper = EsClientDao.createSearchHelper();
// sort
searchHelper.setSort(SortBuilders.fieldSort(GenericAttribute.ES_TIME).order(SortOrder.DESC));
// fetchSource
searchHelper.setFetchSource(new String[]{GenericAttribute.ES_IND_TITLE, GenericAttribute.ES_IND_FULL_TEXT});
// postFilter
BoolQueryBuilder postFilter = searchWholeAnalyzeQuery(dto);
searchHelper.setPostFilter(postFilter);
searchHelper.setSize(1000);
searchHelper.setIndexes(esClientDao.getAllIndexes(dto.getStartTime(), dto.getEndTime()));
List<String> textList = new ArrayList<>();
for (int i = 1; i <= 10; i++) {
searchHelper.setFrom((i - 1) * 1000);
SearchResponse searchResponse = esClientDao.searchResponse(searchHelper);
for (SearchHit searchHit : searchResponse.getHits().getHits()) {
Map<String, Object> source = searchHit.getSourceAsMap();
String title = (String) source.getOrDefault(GenericAttribute.ES_IND_TITLE, "");
String content = (String) source.getOrDefault(GenericAttribute.ES_IND_FULL_TEXT, "");
textList.add(title + content);
}
}
res = textUtil.getHighWordsJsonDifferentFieldName(textList, 20);
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "全网搜舆情分析getHighWord异常-", e);
}
return res;
}
/**
* 时间段内最早发文
* @param dto
* @param totalSpreadTendency
* @return
* @throws IOException
*/
private BaseMap getFirstArticle(SearchFilterDTO dto, List<LineVO> totalSpreadTendency) throws IOException {
// 最高点的时间为开始时间
LineVO startLine = totalSpreadTendency.stream().max(Comparator.comparingLong(LineVO::getCount)).get();
Long startTime = startLine.getDate();
int startIndex = totalSpreadTendency.indexOf(startLine);
// 最高点后一个点的时间为结束时间
Long endTime = dto.getEndTime();
endTime = startIndex == totalSpreadTendency.size() - 1 ? endTime : totalSpreadTendency.get(startIndex + 1).getDate();
dto.setStartTime(startTime);
dto.setEndTime(endTime);
// indexes
String[] indexes = esClientDao.getAllIndexes(dto.getStartTime(), dto.getEndTime());
BoolQueryBuilder query = searchWholeAnalyzeQuery(dto);
// sort
FieldSortBuilder sort = new FieldSortBuilder("time").order(SortOrder.ASC);
SearchHits searchHits = esClientDao.searchHits(indexes, query, null, null, sort, 0, 1, null);
if (0 == searchHits.getTotalHits().value) {
return null;
}
return Tools.getBaseFromEsMap(searchHits.getAt(0).getSourceAsMap());
}
/**
* 舆情趋势图
* @param dto 标注数据搜索传输类
* @return
* @throws IOException
*/
private List<LineVO> getTotalSpreadTendency(SearchFilterDTO dto) throws IOException {
List<LineVO> res = new ArrayList<>();
String[] indexes = esClientDao.getAllIndexes(dto.getStartTime(), dto.getEndTime());
DateHistogramAggregationBuilder daysAggregationBuilder;
// 天级以小时为颗粒度，其他以天作为颗粒度
if (dto.getEndTime() - dto.getStartTime() <= Constant.ONE_DAY){
daysAggregationBuilder = AggregationBuilders.dateHistogram("timeAgg").field("time").calendarInterval(DateHistogramInterval.HOUR);
}else {
daysAggregationBuilder = AggregationBuilders.dateHistogram("timeAgg").field("time").calendarInterval(DateHistogramInterval.DAY);
}
// query
BoolQueryBuilder query = searchWholeAnalyzeQuery(dto);
// response
SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, query,
daysAggregationBuilder, null, null, 0, 0, null);
Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
ParsedDateHistogram teamAgg = (ParsedDateHistogram) aggMap.get("timeAgg");
List<? extends Histogram.Bucket> buckets = teamAgg.getBuckets();
// 走势图
buckets.forEach(bucket -> {
long time = Long.parseLong(bucket.getKeyAsString());
long count = bucket.getDocCount();
res.add(new LineVO(count, time));
});
return res;
}
/**
* 微博、微信、抖音、小红书平台趋势图
* @param dto
* @return
* @throws IOException
*/
private Map<String, List<LineVO>> getPlatformSpreadTendency(SearchFilterDTO dto) throws IOException{
Map<String, List<LineVO>> res = new HashMap<>(4);
if (CollectionUtils.isEmpty(dto.getPlatforms())){
dto.setPlatforms(Stream.of("微博", "微信", "抖音", "小红书").map(GlobalPojo::getPlatformIdByName).collect(Collectors.toList()));
}
String[] indexes = esClientDao.getAllIndexes(dto.getStartTime(), dto.getEndTime());
// 平台聚合
Script script = new Script("doc['c5'].value +'_' +doc['foreign'].value");
TermsAggregationBuilder platformAggregationBuilder = AggregationBuilders.terms("platformAgg").script(script).order(BucketOrder.count(false));
DateHistogramAggregationBuilder daysAggregationBuilder;
// 天级以小时为颗粒度，其他以天作为颗粒度
if (dto.getEndTime() - dto.getStartTime() <= Constant.ONE_DAY){
daysAggregationBuilder = AggregationBuilders.dateHistogram("timeAgg").field("time").calendarInterval(DateHistogramInterval.HOUR);
}else {
daysAggregationBuilder = AggregationBuilders.dateHistogram("timeAgg").field("time").calendarInterval(DateHistogramInterval.DAY);
}
// query
BoolQueryBuilder query = searchWholeAnalyzeQuery(dto);
// response
SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, query, platformAggregationBuilder.subAggregation(daysAggregationBuilder), null, null, 0, 0, null);
Map<String, Aggregation> map = searchResponse.getAggregations().asMap();
ParsedStringTerms countTeam = (ParsedStringTerms) map.get("platformAgg");
List<? extends Terms.Bucket> platformBuckets = countTeam.getBuckets();
Map<String, Map<String, Long>> platformLines = new HashMap<>();
platformBuckets.forEach(bucket -> {
String[] split = bucket.getKeyAsString().split("_");
String platform = BeanTools.filterPlatform(GlobalPojo.PLATFORMS, Integer.valueOf(split[0]), Integer.valueOf(split[1])).getName();
Map<String, Aggregation> aggMap = bucket.getAggregations().asMap();
ParsedDateHistogram teamAgg = (ParsedDateHistogram) aggMap.get("timeAgg");
List<? extends Histogram.Bucket> buckets = teamAgg.getBuckets();
// 合并平台相同时间节点稿件数
for (Histogram.Bucket timeBucket : buckets) {
platformLines.compute(platform, (k1, v1) -> {
if (Objects.isNull(v1)){
Map<String, Long> lineMap = new HashMap<>();
lineMap.put(timeBucket.getKeyAsString(), timeBucket.getDocCount());
platformLines.put(platform, lineMap);
return lineMap;
}else {
String timeStr = timeBucket.getKeyAsString();
long count = timeBucket.getDocCount();
Map<String, Long> lineMap = platformLines.get(platform);
lineMap.compute(timeStr, (k2, v2) -> Objects.isNull(v2) ? count : count + v2);
return lineMap;
}
});
}
});
platformLines.forEach((k1, v1) -> {
List<LineVO> line = new ArrayList<>();
v1.forEach((k2, v2) -> line.add(new LineVO(v2, Long.valueOf(k2))));
res.put(k1, line);
});
return res;
}
private String compare(double avg, double projectAvg){
// 本次查询日均数据 > 项目日均1.2倍为【本次偏高】，项目日均0.8倍<本次查询日均数据<项目日均1.2倍为【数据量正常】，本次查询日均数据 < 项目日均0.8倍为【本次偏低】
String compare = "normal";
double highSide = 1.2 * projectAvg;
double lowSize = 0.8 * projectAvg;
if (avg > highSide){
compare = "higher";
}
if (lowSize > avg){
compare = "lower";
}
return compare;
}
private BoolQueryBuilder searchWholeAnalyzeQuery(SearchFilterDTO dto){
BoolQueryBuilder postFilter = QueryBuilders.boolQuery();
// searchType 视频查询
if (Objects.nonNull(dto.getSearchType()) && Objects.equals("视频", dto.getSearchType())){
postFilter.must(QueryBuilders.existsQuery(GenericAttribute.ES_VIDEO_URLS));
}
// startTime endTime
if (Objects.nonNull(dto.getStartTime()) && Objects.nonNull(dto.getEndTime())) {
postFilter.must(QueryBuilders.rangeQuery("time").gte(dto.getStartTime()).lt(dto.getEndTime()));
}
// Query 查询条件
String[] fieldSearch = "标题".equals(dto.getMatchType()) ? new String[]{GenericAttribute.ES_IND_TITLE} : new String[]{GenericAttribute.ES_IND_FULL_TEXT};
// keyword
if (StringUtils.isNotEmpty(dto.getKeyword())) {
// Query
postFilter.must(EsQueryTools.assembleNormalKeywordQuery(dto.getKeyword(), fieldSearch));
}
// fans
if (StringUtils.isNotEmpty(dto.getFans())) {
if ("百万".equals(dto.getFans())) {
postFilter.must(QueryBuilders.rangeQuery("followers_num").gte(1000000).lt(10000000));
}
if ("千万".equals(dto.getFans())) {
postFilter.must(QueryBuilders.rangeQuery("followers_num").gte(10000000).lt(100000000));
}
}
// sensitiveChannels
if (!Tools.isEmpty(dto.getSensitiveChannels()) && !"全部".equals(dto.getSensitiveChannels().get("father"))) {
// 重要渠道提取
List<String> channels = SensitiveChannelsEnum.getSensitiveChannelsByGroupAndName(dto.getSensitiveChannels().get("father"), dto.getSensitiveChannels().get("son"));
String source = String.join("|", channels);
postFilter.must(EsQueryTools.assembleSeniorTermOrQuery(source, new String[]{GenericAttribute.ES_SOURCE}));
}
// platform
if (CollectionUtils.isNotEmpty(dto.getPlatforms())) {
postFilter.must(EsQueryTools.assemblePlatformQuery(Tools.getPlatformByIds(dto.getPlatforms())));
}
// videoPlatforms
if (Objects.nonNull(dto.getSearchType()) && Objects.equals("视频", dto.getSearchType()) && CollectionUtils.isNotEmpty(dto.getVideoPlatforms())){
postFilter.must(EsQueryTools.assembleVideoPlatformQuery(dto.getVideoPlatforms()));
}
// source
if (StringUtils.isNotEmpty(dto.getSourceKeyword())) {
postFilter.must(EsQueryTools.assembleSeniorTermOrQuery(dto.getSourceKeyword(), new String[]{GenericAttribute.ES_SOURCE}));
}
// FilterWords
if (StringUtils.isNotEmpty(dto.getFilterType()) && StringUtils.isNotBlank(dto.getFilterWords())) {
String[] fieldFilter = "标题".equals(dto.getFilterType()) ? new String[]{GenericAttribute.ES_IND_TITLE}
: new String[]{GenericAttribute.ES_IND_FULL_TEXT};
postFilter.mustNot(EsQueryTools.assembleNormalKeywordQuery(dto.getFilterWords(), fieldFilter));
}
// ip
if (StringUtils.isNotBlank(dto.getIp())){
postFilter.must(QueryBuilders.termQuery("ip_location", dto.getIp()));
}
return postFilter;
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment