Skip to content

B
brandkbs2
Commit 5761750e by 陈健智
Options

查舆情、全网搜声量显示

parent 051521e4
Showing with 222 additions and 43 deletions
src/main/java/com/zhiwei/brandkbs2/controller/app/AppSearchController.java
......@@ -11,9 +11,11 @@ import com.zhiwei.brandkbs2.controller.BaseController;
import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
import com.zhiwei.brandkbs2.enmus.RoleEnum;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.MarkFlowEntity;
import com.zhiwei.brandkbs2.pojo.WholeSearchRecord;
import com.zhiwei.brandkbs2.pojo.dto.*;
import com.zhiwei.brandkbs2.pojo.vo.ChannelListVO;
import com.zhiwei.brandkbs2.pojo.vo.PageVO;
import com.zhiwei.brandkbs2.service.*;
import com.zhiwei.brandkbs2.util.Tools;
import com.zhiwei.middleware.event.pojo.dto.BrandkbsEventSearchDTO;
......@@ -199,14 +201,28 @@ public class AppSearchController extends BaseController {
@LogRecord(values = "keyword", description = "查舆情", arguments = true, entity = true)
@PostMapping("/mark/list")
public ResponseResult getYuqingMarkList(@RequestBody MarkSearchDTO markSearchDTO) {
return ResponseResult.success(markDataService.getYuqingMarkList(markSearchDTO));
PageVO<MarkFlowEntity> yuqingMarkList = markDataService.getYuqingMarkList(markSearchDTO);
// 仅第一页增加平台进量（声量）统计
if (1 == markSearchDTO.getPage()) {
JSONObject info = yuqingMarkList.getInfo();
info.put("counts", markDataService.countMarkPlatform(markSearchDTO));
yuqingMarkList.setInfo(info);
}
return ResponseResult.success(yuqingMarkList);
}
@ApiOperation("原始数据列表")
@LogRecord(values = "keyword", description = "查原始数据", arguments = true, entity = true)
@PostMapping("/origin/list")
public ResponseResult getOriginList(@RequestBody MarkSearchDTO markSearchDTO) {
return ResponseResult.success(markDataService.getOriginList(markSearchDTO));
PageVO<MarkFlowEntity> originList = markDataService.getOriginList(markSearchDTO);
// 仅第一页增加平台进量（声量）统计
if (1 == markSearchDTO.getPage()) {
JSONObject info = originList.getInfo();
info.put("counts", markDataService.countOriginPlatform(markSearchDTO));
originList.setInfo(info);
}
return ResponseResult.success(originList);
}
@ApiOperation("搜索-舆情列表-搜索条件")
......
src/main/java/com/zhiwei/brandkbs2/service/EsSearchService.java
package com.zhiwei.brandkbs2.service;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
import org.apache.commons.lang3.tuple.Pair;
import org.elasticsearch.search.SearchHits;
......@@ -18,7 +19,7 @@ public interface EsSearchService {
/**
* 搜索标注数据
*
* @param dto 搜素实体类
* @param dto 搜索实体类
* @param scroll 滚动查询
* @return 搜索结果
*/
......@@ -27,7 +28,7 @@ public interface EsSearchService {
/**
* 搜索竞品标注数据
*
* @param dto 搜素实体类
* @param dto 搜索实体类
* @param scroll 滚动查询
* @return 搜索结果
*/
......@@ -36,10 +37,25 @@ public interface EsSearchService {
/**
* 搜索原始数据
*
* @param dto 搜素实体类
* @param dto 搜索实体类
* @param scroll 滚动查询
* @return 搜索结果
*/
Pair<SearchHits[], Map<String, Long>> searchOriginHitsAndCount(MarkSearchDTO dto, boolean scroll) throws IOException;
/**
* 查舆情舆情库-平台进量（声量）统计
* @param dto 搜索实体类
* @return
* @throws IOException
*/
JSONObject countMarkPlatform(MarkSearchDTO dto) throws IOException;
/**
* 查舆情原始数据-平台进量（声量）统计
* @param dto 搜索实体类
* @return
* @throws IOException
*/
JSONObject countOriginPlatform(MarkSearchDTO dto) throws IOException;
}
src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java
......@@ -378,4 +378,18 @@ public interface MarkDataService {
* @return
*/
Long interactionUpdatePrediction(MarkSearchDTO dto);
/**
* 查舆情舆情库-平台进量（声量）统计
* @param dto 标注数据搜索传输类
* @return
*/
JSONObject countMarkPlatform(MarkSearchDTO dto);
/**
* 查舆情原始数据-平台进量（声量）统计
* @param dto 标注数据搜索传输类
* @return
*/
JSONObject countOriginPlatform(MarkSearchDTO dto);
}
src/main/java/com/zhiwei/brandkbs2/service/impl/EsSearchServiceImpl.java
package com.zhiwei.brandkbs2.service.impl;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.common.GlobalPojo;
import com.zhiwei.brandkbs2.es.EsClientDao;
import com.zhiwei.brandkbs2.es.EsQueryTools;
import com.zhiwei.brandkbs2.function.ArticleFunction;
import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
import com.zhiwei.brandkbs2.service.EsSearchService;
import com.zhiwei.brandkbs2.util.Tools;
import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
......@@ -21,9 +24,8 @@ import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.stream.Collectors;
/**
* @ClassName: EsSearchServiceImpl
......@@ -55,6 +57,18 @@ public class EsSearchServiceImpl implements EsSearchService {
return searchHitsAndCount(dto, this::createOriginSearchHelper, hotEsDao, scroll);
}
@Override
public JSONObject countMarkPlatform(MarkSearchDTO dto) throws IOException {
EsClientDao.SearchHelper helper = createSearchHelper(dto);
return platformCount(helper.getPostFilter(), esClientDao);
}
@Override
public JSONObject countOriginPlatform(MarkSearchDTO dto) throws IOException {
EsClientDao.SearchHelper helper = createOriginSearchHelper(dto);
return platformCount(helper.getPostFilter(), hotEsDao);
}
private Pair<SearchHits[], Map<String, Long>> searchHitsAndCount(MarkSearchDTO dto, ArticleFunction.CreateSearchHelper createSearchHelper,
EsClientDao clientDao, boolean scroll) throws IOException {
EsClientDao.SearchHelper helper = createSearchHelper.create(dto);
......@@ -63,8 +77,8 @@ public class EsSearchServiceImpl implements EsSearchService {
return Pair.of(clientDao.searchScrollResponse(helper).stream().map(SearchResponse::getHits).toArray(SearchHits[]::new), null);
}
SearchHits searchHits = clientDao.searchHits(helper);
Map<String, Long> platformCount = platformCount(dto.getPage(), dto.getPlatforms(), helper.getPostFilter(), clientDao);
return Pair.of(new SearchHits[]{searchHits}, platformCount);
// Map<String, Long> platformCount = platformCount(dto.getPage(), dto.getPlatforms(), helper.getPostFilter(), clientDao);
return Pair.of(new SearchHits[]{searchHits}, new HashMap<>());
}
private EsClientDao.SearchHelper createSearchHelper(MarkSearchDTO dto) {
......@@ -250,18 +264,23 @@ public class EsSearchServiceImpl implements EsSearchService {
*
* @return
*/
private Map<String, Long> platformCount(int page, List<String> platforms, BoolQueryBuilder postFilter, EsClientDao esClientDao) throws IOException {
Map<String, Long> counts = new HashMap<>();
// TODO 平台进量暂不统计
private JSONObject platformCount(BoolQueryBuilder postFilter, EsClientDao esClientDao) throws IOException {
List<JSONObject> list = new ArrayList<>();
//平台计量统计 仅第一页且无平台限制
// if (1 == page && CollectionUtils.isEmpty(platforms)) {
// for (MessagePlatform platform : GlobalPojo.PLATFORMS) {
// BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery().must(postFilter).must(EsQueryTools.assemblePlatformQuery(Collections.singletonList(platform)));
// Long count = esClientDao.count(esClientDao.getIndexes(), queryBuilder, null);
// counts.put(platform.getName(), count);
// }
// }
return counts;
for (MessagePlatform platform : GlobalPojo.PLATFORMS.stream().filter(messagePlatform -> !"外媒".equals(messagePlatform.getName())).collect(Collectors.toList())) {
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery().must(postFilter).must(EsQueryTools.assemblePlatformQuery(Collections.singletonList(platform)));
Long count = esClientDao.count(esClientDao.getIndexes(), queryBuilder, null);
JSONObject jsonObject = new JSONObject();
jsonObject.put("id", platform.getId());
jsonObject.put("name", platform.getName());
jsonObject.put("count", count);
list.add(jsonObject);
}
Long total = list.stream().map(jsonObject -> jsonObject.getLongValue("count")).reduce(Long::sum).orElse(0L);
JSONObject res = new JSONObject();
res.put("count", list);
res.put("total", total);
return res;
}
}
src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java
......@@ -184,10 +184,12 @@ public class MarkDataServiceImpl implements MarkDataService {
long total = hitsAndCounts.getLeft()[0].getTotalHits().value;
// 消息列表
List<MarkFlowEntity> flowEntityList = getMarkFlowEntity4Origin(markSearchDTO, hitsAndCounts.getLeft()[0]);
JSONObject jsonObject = new JSONObject();
jsonObject.put("highlightWord", highlightWordDao.findList(Query.query(Criteria.where("projectId").is(UserThreadLocal.getProjectId()))).stream().map(HighlightWord::getKeyword).collect(Collectors.toList()));
// 返回分页结果并设置平台count
return PageVO.createPageVo(total, markSearchDTO.getPage(), markSearchDTO.getPageSize(), flowEntityList)
// 各平台计量
.setInfo(new JSONObject(ImmutableMap.of("highlightWord", highlightWordDao.findList(Query.query(Criteria.where("projectId").is(UserThreadLocal.getProjectId()))).stream().map(HighlightWord::getKeyword).collect(Collectors.toList()))));
.setInfo(jsonObject);
} catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.FAIL, "es查询异常");
}
......@@ -266,10 +268,12 @@ public class MarkDataServiceImpl implements MarkDataService {
long total = hitsAndCounts.getLeft()[0].getTotalHits().value;
// 消息列表
List<MarkFlowEntity> flowEntityList = getMarkFlowEntity(markSearchDTO, hitsAndCounts.getLeft()[0]);
JSONObject jsonObject = new JSONObject();
// 高亮关键词
jsonObject.put("highlightWord", GlobalPojo.HIGHLIGHT_MAP.get(UserThreadLocal.getProjectId()));
// 返回分页结果并返回高亮关键词
return PageVO.createPageVo(total, markSearchDTO.getPage(), markSearchDTO.getPageSize(), flowEntityList)
// 高亮关键词
.setInfo(new JSONObject(ImmutableMap.of("highlightWord", GlobalPojo.HIGHLIGHT_MAP.get(UserThreadLocal.getProjectId()))));
.setInfo(jsonObject);
} catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.FAIL, "es查询异常");
}
......@@ -1150,17 +1154,12 @@ public class MarkDataServiceImpl implements MarkDataService {
public Pair<JSONObject, Integer> searchWholeNetworkWithBalance(SearchFilterDTO dto) {
if (dto.isExternalDataSource()) { // 查商业数据库
try {
JSONObject params = externalTransform(dto);
JSONObject taskEntity = new JSONObject();
taskEntity.put("appId", "6183571e0d710000f6003a12"); // 应用id, 由张志伟提供给使用者
taskEntity.put("taskType", null); // 可为null, 任务类型
taskEntity.put("taskDescription", null); // 可为null, 任务描述
taskEntity.put("taskName", UserThreadLocal.getProjectId() + "_" + UserThreadLocal.getUserId());
taskEntity.put("userName", "品见"); // 提交任务的人
taskEntity.put("searchParams", params); // 任务详情参数
ResponseEntity<JSONObject> responseEntity = restTemplate.postForEntity(addIStarShineKSDataUrl, taskEntity, JSONObject.class);
JSONObject result = Objects.requireNonNull(responseEntity.getBody()).getJSONObject("data");
JSONObject json = searchWholeBackUp(result.get("id").toString());
JSONObject data = getSearchWholeOpinionJson(dto);
JSONObject json = packageExternalDataSourceResult(data);
// 仅当第一页时统计平台进量（声量）
if (1 == dto.getPage()){
json.put("info", getExternalDataSourcePlatformCount(dto));
}
return Pair.of(json, json.getInteger("pageLimit"));
} catch (Exception e) {
log.error("error searchWholeNetwork - ", e);
......@@ -1172,6 +1171,36 @@ public class MarkDataServiceImpl implements MarkDataService {
}
}
/**
* 获取外部数据源搜索结果json
* @param dto
* @return
*/
private JSONObject getSearchWholeOpinionJson(SearchFilterDTO dto){
// 组装请求参数
JSONObject taskEntity = packageExternalDataSourceParams(dto);
ResponseEntity<JSONObject> responseEntity = restTemplate.postForEntity(addIStarShineKSDataUrl, taskEntity, JSONObject.class);
JSONObject result = Objects.requireNonNull(responseEntity.getBody()).getJSONObject("data");
return searchWholeBackUp(result.get("id").toString());
}
/**
* 外部数据源组装请求参数
* @param dto
* @return
*/
private JSONObject packageExternalDataSourceParams(SearchFilterDTO dto){
JSONObject params = externalTransform(dto);
JSONObject taskEntity = new JSONObject();
taskEntity.put("appId", "6183571e0d710000f6003a12"); // 应用id, 由张志伟提供给使用者
taskEntity.put("taskType", null); // 可为null, 任务类型
taskEntity.put("taskDescription", null); // 可为null, 任务描述
taskEntity.put("taskName", UserThreadLocal.getProjectId() + "_" + UserThreadLocal.getUserId());
taskEntity.put("userName", "品见"); // 提交任务的人
taskEntity.put("searchParams", params); // 任务详情参数
return taskEntity;
}
@Override
public List<ExportSearchWholeDTO> exportSearchWhole(SearchFilterDTO dto) {
Integer exportAmount = userService.queryUserInfo(UserThreadLocal.getUserId(), UserThreadLocal.getProjectId()).getExportAmount();
......@@ -1195,6 +1224,11 @@ public class MarkDataServiceImpl implements MarkDataService {
return jsonArray.stream().map(json -> ExportSearchWholeDTO.creatExportSearchWholeDTO((JSONObject) json)).collect(Collectors.toList());
}
/**
* 舆情库数据结果组装成标准分页返回格式
* @param responseEntity
* @return
*/
private JSONObject searchWholeOpinion(ResponseEntity<JSONObject> responseEntity) {
JSONObject jsonObject = new JSONObject();
JSONObject bodyData = Objects.requireNonNull(responseEntity.getBody()).getJSONObject("data");
......@@ -1227,6 +1261,20 @@ public class MarkDataServiceImpl implements MarkDataService {
result.put("ipLocation", list.get("ip_location"));
resultList.add(result);
}
// 仅第一页返回平台进量（声量）统计
if (1 == data.getInteger("page")) {
JSONObject count = bodyData.getJSONObject("count");
List<JSONObject> platformCounts = count.getJSONArray("count").toJavaList(JSONObject.class);
// 去除外媒和脉脉两个平台
List<JSONObject> list = platformCounts.stream().filter(json -> !Objects.equals(json.getString("name"), "脉脉") &&
!Objects.equals(json.getString("name"), "外媒")).collect(Collectors.toList());
// 重新计算总数
Long total = list.stream().map(json -> json.getLongValue("count")).reduce(Long::sum).orElse(0L);
JSONObject counts = new JSONObject();
counts.put("count", list);
counts.put("total", total);
jsonObject.put("info", counts);
}
jsonObject.put("list", resultList);
jsonObject.put("page", data.get("page"));
jsonObject.put("pageLimit", data.get("pageLimit"));
......@@ -1236,10 +1284,13 @@ public class MarkDataServiceImpl implements MarkDataService {
return jsonObject;
}
/**
* 获取外部数据源搜索结果JSONObject
* @param taskId
* @return
*/
private JSONObject searchWholeBackUp(String taskId) {
JSONObject jsonObject = new JSONObject();
List<JSONObject> resultList = new ArrayList<>();
for (int i = 0; i < 30; i++) {
ResponseEntity<JSONObject> responseEntity = restTemplate.getForEntity(getIStarShineKSInfoDataUrl, JSONObject.class, taskId);
JSONObject data = Objects.requireNonNull(responseEntity.getBody()).getJSONObject("data");
......@@ -1248,8 +1299,22 @@ public class MarkDataServiceImpl implements MarkDataService {
throw new IllegalArgumentException(task.getString("searchResultMsg"));
}
if (1 == task.getInteger("crawlerFinish")) {
return data;
}
Tools.sleep(500L);
}
return jsonObject;
}
//pageSize
/**
* 将外部数据源JSONObject组装成标准的返回结果
* @param data
* @return
*/
private JSONObject packageExternalDataSourceResult(JSONObject data){
JSONObject jsonObject = new JSONObject();
List<JSONObject> resultList = new ArrayList<>();
JSONObject task = data.getJSONObject("task");
Integer receiveCount = task.getInteger("receiveCount");
Integer searchCount = task.getInteger("searchCount");
int totalPage;
......@@ -1280,9 +1345,30 @@ public class MarkDataServiceImpl implements MarkDataService {
jsonObject.put("isBackUp", true);
return jsonObject;
}
Tools.sleep(500L);
/**
* 获取实时采集（外部数据源）平台进量（声量）
* @param dto
* @return
*/
private JSONObject getExternalDataSourcePlatformCount(SearchFilterDTO dto){
JSONObject res = new JSONObject();
List<JSONObject> list = new ArrayList<>();
List<String> platforms = Arrays.asList("App", "博客", "平媒", "新闻", "微博", "微信", "论坛", "视频", "短视频");
for (String platform : platforms) {
// 重置平台条件
dto.setPlatforms(Collections.singletonList(platform));
JSONObject data = getSearchWholeOpinionJson(dto);
JSONObject task = data.getJSONObject("task");
JSONObject jsonObject = new JSONObject();
jsonObject.put("count", task.getLongValue("searchCount"));
jsonObject.put("name", platform);
list.add(jsonObject);
}
return jsonObject;
Long total = list.stream().map(jsonObject -> jsonObject.getLongValue("count")).reduce(Long::sum).orElse(0L);
res.put("count", list);
res.put("total", total);
return res;
}
private ClassB.TypeB getExternalType(JSONObject tJson, String platform) {
......@@ -1580,6 +1666,34 @@ public class MarkDataServiceImpl implements MarkDataService {
return null;
}
@Override
public JSONObject countMarkPlatform(MarkSearchDTO dto) {
defaultMarkSearch(dto);
try {
return esSearchService.countMarkPlatform(dto);
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "es查询异常", e);
}
return null;
}
@Override
public JSONObject countOriginPlatform(MarkSearchDTO dto) {
// 时间范围默认当天
if (Objects.isNull(dto.getStartTime()) || Objects.isNull(dto.getEndTime())) {
Date now = new Date();
dto.setEndTime(now.getTime());
dto.setStartTime(DateUtils.addDays(now, -1).getTime());
}
defaultMarkSearch(dto);
try {
return esSearchService.countOriginPlatform(dto);
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "es查询异常", e);
}
return null;
}
private static JSONObject getTitleAndUrl(Map<String, Object> map) {
JSONObject json = new JSONObject();
BaseMap baseMap = Tools.getBaseFromEsMap(map);
......
src/main/resources/application-dev.properties
......@@ -77,7 +77,7 @@ qbjc.platform.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/platfo
qbjc.userCenter.url=https://login.zhiweidata.com/plogin/center
qbjc.userCenter.token=AoJ0ooy3HV1EElWnvQw9YTS9b5Y+fmtkbM6DdpPgDO6D/OhNqH4qrJKarzMr
qbjc.channel.application.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/channel-application
qbjc.channel.influence=https://yuqing.test.zhiweidata.com/qbjcbackPhoenix/interface/middleware/channel-influence?platform={1}&channel={2}
qbjc.channel.influence=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/channel-influence?platform={1}&channel={2}
#\u5371\u673A\u5E93\u5916\u90E8\u63A5\u53E3
crisis.search.url=https://crisis.zhiweidata.com/app/brandkbs/crisisSearch?page={1}&size={2}&keyword={3}
crisis.searchTags.url=https://crisis.zhiweidata.com/app/brandkbs/searchCrisisByTags?page={1}&size={2}&brand={3}&category={4}
......
src/main/resources/application-local.properties
......@@ -80,7 +80,7 @@ qbjc.platform.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/platfo
qbjc.userCenter.url=http://login.zhiweidata.top/plogin/center
qbjc.userCenter.token=AoJ0ooy3HV1EElWnvQw9YTS9b5Y+fmtkbM6DdpPgDO6D/OhNqH4qrJKarzMr
qbjc.channel.application.url=http://192.168.0.79:11000/qbjcbackPhoenix/interface/middleware/channel-application
qbjc.channel.influence=https://yuqing.test.zhiweidata.com/qbjcbackPhoenix/interface/middleware/channel-influence?platform={1}&channel={2}
qbjc.channel.influence=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/channel-influence?platform={1}&channel={2}
#\u5371\u673A\u5E93\u5916\u90E8\u63A5\u53E3
crisis.search.url=https://crisis.zhiweidata.com/app/brandkbs/crisisSearch?page={1}&size={2}&keyword={3}
crisis.searchTags.url=https://crisis.zhiweidata.com/app/brandkbs/searchCrisisByTags?page={1}&size={2}&brand={3}&category={4}
......
src/main/resources/application-prod.properties
......@@ -77,7 +77,7 @@ qbjc.platform.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/platfo
qbjc.userCenter.url=https://login.zhiweidata.com/plogin/center
qbjc.userCenter.token=AoJ0ooy3HV1EElWnvQw9YTS9b5Y+fmtkbM6DdpPgDO6D/OhNqH4qrJKarzMr
qbjc.channel.application.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/channel-application
qbjc.channel.influence=https://yuqing.test.zhiweidata.com/qbjcbackPhoenix/interface/middleware/channel-influence?platform={1}&channel={2}
qbjc.channel.influence=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/channel-influence?platform={1}&channel={2}
#\u5371\u673A\u5E93\u5916\u90E8\u63A5\u53E3
crisis.search.url=https://crisis.zhiweidata.com/app/brandkbs/crisisSearch?page={1}&size={2}&keyword={3}
crisis.searchTags.url=https://crisis.zhiweidata.com/app/brandkbs/searchCrisisByTags?page={1}&size={2}&brand={3}&category={4}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment