Skip to content

B
brandkbs2
Commit 3a21a0ba by shenjunjie
Options

Merge branch 'feature' into 'dev' 

Feature

See merge request !573
parents 674a5a9e 1190a500
Showing with 121 additions and 29 deletions
src/main/java/com/zhiwei/brandkbs2/controller/app/AppDownloadController.java
......@@ -396,9 +396,9 @@ public class AppDownloadController extends BaseController {
@PostMapping(value = "/contend/mark")
@DownloadTask(taskName = "竞品库竞品舆情下载", description = "竞品库竞品舆情")
public ResponseResult exportContendMarkList(@RequestBody MarkSearchDTO markSearchDTO) {
if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
return ResponseResult.failure("不支持特殊符号字段查询");
}
// if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
// return ResponseResult.failure("不支持特殊符号字段查询");
// }
Pair<String, List<ExportAppYuqingDTO>> stringListPair = markDataService.downloadContendMarkList(markSearchDTO);
// excel写入至指定路径
String projectName = projectService.getProjectById(UserThreadLocal.getProjectId()).getProjectName();
......@@ -412,9 +412,9 @@ public class AppDownloadController extends BaseController {
@LogRecord(description = "全网搜-舆情导出", values = {"startTime", "endTime", "fans", "filterType", "filterWords", "search", "keyword", "platforms", "sensitiveChannels", "sourceKeyword"}, entity = true, arguments = true)
@DownloadTask(taskName = "全网搜舆情下载", description = "全网搜舆情")
public ResponseResult exportSearchWhole(@RequestBody SearchFilterDTO dto) {
if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
return ResponseResult.failure("不支持特殊符号字段查询");
}
// if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
// return ResponseResult.failure("不支持特殊符号字段查询");
// }
// 针对商业数据库做限制
if (dto.isExternalDataSource()) {
long time = DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), -89).getTime();
......
src/main/java/com/zhiwei/brandkbs2/controller/app/AppSearchController.java
......@@ -161,9 +161,9 @@ public class AppSearchController extends BaseController {
@LogRecord(values = {"fans", "sensitiveChannels:father,son", "keyword", "search"}, description = "全网搜", arguments = true, entity = true)
@PostMapping("/searchWhole")
public ResponseResult searchWholeNetwork(@RequestBody SearchFilterDTO dto) {
if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
return ResponseResult.failure("不支持特殊符号字段查询");
}
// if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
// return ResponseResult.failure("不支持特殊符号字段查询");
// }
cacheSearchKeyword(dto.getKeyword(), "whole");
// 针对商业数据库做限制
if (dto.isExternalDataSource()) {
......@@ -200,9 +200,9 @@ public class AppSearchController extends BaseController {
@PostMapping("/exportSearchWhole")
@LogRecord(description = "全网搜-舆情导出", values = {"startTime", "endTime", "fans", "filterType", "filterWords", "search", "keyword", "platforms", "sensitiveChannels", "sourceKeyword"}, entity = true, arguments = true)
public ResponseResult exportSearchWhole(@RequestBody SearchFilterDTO dto) {
if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
return ResponseResult.failure("不支持特殊符号字段查询");
}
// if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
// return ResponseResult.failure("不支持特殊符号字段查询");
// }
// 针对商业数据库做限制
if (dto.isExternalDataSource()) {
long time = DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), -89).getTime();
......@@ -231,9 +231,9 @@ public class AppSearchController extends BaseController {
@LogRecord(values = {"searchType", "keyword"}, description = "查舆情", arguments = true, entity = true)
@PostMapping("/mark/list")
public ResponseResult getYuqingMarkList(@RequestBody MarkSearchDTO markSearchDTO) {
if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
return ResponseResult.failure("不支持特殊符号字段查询");
}
// if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
// return ResponseResult.failure("不支持特殊符号字段查询");
// }
cacheSearchKeyword(markSearchDTO.getKeyword(), "yuqing");
PageVO<MarkFlowEntity> yuqingMarkList = markDataService.getYuqingMarkList(markSearchDTO);
// 仅第一页增加平台进量（声量）统计
......@@ -339,9 +339,9 @@ public class AppSearchController extends BaseController {
@LogRecord(values = "keyword", description = "查竞品",arguments = true, entity = true)
@PostMapping("/contend/list")
public ResponseResult getContendSearchList(@RequestBody MarkSearchDTO markSearchDTO) {
if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
return ResponseResult.failure("不支持特殊符号字段查询");
}
// if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
// return ResponseResult.failure("不支持特殊符号字段查询");
// }
cacheSearchKeyword(markSearchDTO.getKeyword(), "contend");
return ResponseResult.success(markDataService.getContendSearchList(markSearchDTO));
}
......@@ -354,8 +354,17 @@ public class AppSearchController extends BaseController {
@ApiOperation("搜索-AI搜索")
@GetMapping("/ai/answer")
public ResponseResult getAISearchResult(@RequestParam(value = "question") String question) {
return ResponseResult.success(markDataService.getAISearchResult(question));
public ResponseResult getAISearchResult(@RequestParam(value = "question") String question,
@RequestParam(value = "keyword", required = false) String keyword,
@RequestParam(value = "startTime", required = false) Long startTime,
@RequestParam(value = "endTime", required = false) Long endTime) {
return ResponseResult.success(markDataService.getAISearchResult(question, keyword, startTime, endTime));
}
@ApiOperation("搜索-AI搜索-联网搜索")
@GetMapping("/ai/answer/online")
public ResponseResult getAIOnlineSearchResult(@RequestParam(value = "question") String question) {
return ResponseResult.success(markDataService.getAIOnlineSearchResult(question));
}
@ApiOperation("搜索-AI推荐提问")
......
src/main/java/com/zhiwei/brandkbs2/es/EsClientDao.java
......@@ -2,9 +2,13 @@ package com.zhiwei.brandkbs2.es;
import com.alibaba.fastjson.JSONObject;
import com.hankcs.hanlp.HanLP;
import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.common.GlobalPojo;
import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.pojo.ChannelIndex;
import com.zhiwei.brandkbs2.pojo.Contend;
import com.zhiwei.brandkbs2.pojo.Project;
import com.zhiwei.brandkbs2.pojo.ai.FieldMapping;
import com.zhiwei.brandkbs2.util.TextUtil;
import com.zhiwei.brandkbs2.util.Tools;
......@@ -335,6 +339,63 @@ public class EsClientDao {
return Pair.of(new Long[]{startTime, endTime}, res);
}
public List<JSONObject> findSearch(String question, String keyword, Long startTime, Long endTime) throws IOException {
List<JSONObject> list = new ArrayList<>();
String projectId = UserThreadLocal.getProjectId();
BoolQueryBuilder query = QueryBuilders.boolQuery();
// 默认一周
if (Objects.isNull(startTime) || Objects.isNull(endTime)){
endTime = System.currentTimeMillis();
startTime = System.currentTimeMillis() - Constant.ONE_WEEK;
}
// time
query.must(QueryBuilders.rangeQuery(GenericAttribute.ES_TIME).gte(startTime).lt(endTime));
// contend
Project project = GlobalPojo.PROJECT_MAP.get(projectId);
if (CollectionUtils.isNotEmpty(project.getContendList())) {
List<Contend> contendList = new ArrayList<>();
for (Contend contend : project.getContendList()) {
if (question.contains(contend.getBrandName())) {
contendList.add(contend);
}
}
if (CollectionUtils.isNotEmpty(contendList)) {
BoolQueryBuilder contendQuery = QueryBuilders.boolQuery();
for (Contend contend : contendList) {
contendQuery.should(EsQueryTools.assembleCacheMapsQuery(projectId, contend.getId()));
}
contendQuery.minimumShouldMatch(1);
query.must(contendQuery);
}else {
query.must(EsQueryTools.assembleCacheMapsQuery(projectId, Constant.PRIMARY_CONTEND_ID));
}
}
// keyword
query.must(EsQueryTools.assembleNormalKeywordQuery(keyword, new String[]{GenericAttribute.ES_IND_FULL_TEXT}));
String[] fetchSource = {"id", GenericAttribute.ES_TIME, GenericAttribute.ES_IND_TITLE};
// hit
SearchHit[] hits = searchHits(getIndexes(), query, null, fetchSource, null, 0, 10000, null).getHits();
Pair<Map<String, JSONObject>, Map<String, String>> searchProcess = findSearchResultProcess(hits);
Map<String, JSONObject> idBaseMap = searchProcess.getLeft();
Map<String, String> idTitle = searchProcess.getRight();
if (idTitle.isEmpty()){
return list;
}
// 按标题聚合，取聚合结果集前9，并取结果集中最新的文章的id
List<String> idList = TextUtil.getKResult(idTitle).stream()
.sorted(Comparator.comparing(List<String>::size, Comparator.reverseOrder()))
.limit(9)
.map(ids -> ids.stream().map(idBaseMap::get).max(Comparator.comparingLong(json -> json.getLongValue(GenericAttribute.ES_TIME))).orElse(null))
.filter(Objects::nonNull)
.map(json -> json.getString("id"))
.collect(Collectors.toList());
// 反查原数据
for (String id : idList) {
list.add(getTopTitleLatest(id));
}
return list;
}
public List<JSONObject> findSearch(List<FieldMapping> fieldMappings) throws IOException {
List<JSONObject> list = new ArrayList<>();
BoolQueryBuilder query = getBoolQueryBuilder(fieldMappings);
......@@ -364,7 +425,7 @@ public class EsClientDao {
.collect(Collectors.toList());
// 反查原数据
for (String id : idList) {
list.add(getTopTitleLatest(getBoolQueryBuilder(fieldMappings), id));
list.add(getTopTitleLatest(id));
}
return list;
}
......@@ -389,11 +450,10 @@ public class EsClientDao {
return searchHits(getIndexes(), query, null, fetchSource, null, 0, 10000, null).getHits();
}
private JSONObject getTopTitleLatest(BoolQueryBuilder query, String id) throws IOException {
private JSONObject getTopTitleLatest(String id) throws IOException {
BoolQueryBuilder query = QueryBuilders.boolQuery();
query.must(QueryBuilders.termQuery("id", id));
FieldSortBuilder sort = new FieldSortBuilder(GenericAttribute.ES_TIME).order(SortOrder.DESC);
SearchHits searchHits = searchHits(getIndexes(), query, null, null, sort, 0, 1, null);
return new JSONObject(searchHits.getAt(0).getSourceAsMap());
return searchById(id);
}
private BoolQueryBuilder getBoolQueryBuilder(List<FieldMapping> fieldMappings) {
......
src/main/java/com/zhiwei/brandkbs2/pojo/UserLogRecord.java
......@@ -62,4 +62,16 @@ public class UserLogRecord extends AbstractBaseMongo{
record.setCost(cost);
return record;
}
public static UserLogRecord defaultUserLogRecord(String description){
UserLogRecord record = new UserLogRecord();
record.setProjectId(UserThreadLocal.getProjectId());
record.setUserId(UserThreadLocal.getUserId());
record.setNickname(UserThreadLocal.getNickname());
record.setDescription(description);
record.setRoleId(UserThreadLocal.getRoleId());
record.setCTime(System.currentTimeMillis());
record.setUpdateTime(System.currentTimeMillis());
return record;
}
}
src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java
......@@ -853,5 +853,12 @@ public interface MarkDataService {
* @param question
* @return
*/
JSONObject getAISearchResult(String question);
JSONObject getAISearchResult(String question, String keyword, Long startTime, Long endTime);
/**
* AI搜索-联网搜索
* @param question
* @return
*/
JSONObject getAIOnlineSearchResult(String question);
}
src/main/java/com/zhiwei/brandkbs2/service/impl/ProjectWarnServiceImpl.java
......@@ -89,7 +89,7 @@ public class ProjectWarnServiceImpl implements ProjectWarnService {
static {
TYPE_SEARCH.put("微博热搜", "weibo");
TYPE_SEARCH.put("微博话题", "weibo-topic");
// TYPE_SEARCH.put("微博话题", "weibo-topic");
TYPE_SEARCH.put("微博预热", "weibo-rise");
TYPE_SEARCH.put("头条热搜", "toutiao");
TYPE_SEARCH.put("抖音热搜", "douyin");
......@@ -385,7 +385,8 @@ public class ProjectWarnServiceImpl implements ProjectWarnService {
// key2
String key2 = "";
List<String> key2Element = new ArrayList<>();
config.getListType().forEach(type -> {
// 2024/8/19 微博话题榜采集下线，防止微博话题配置历史数据影响
config.getListType().stream().filter(type -> !Objects.equals("微博话题", type)).collect(Collectors.toList()).forEach(type -> {
if (config.getFirstTop()) {
key2Element.add(type + "榜-首次上榜");
}
......
src/main/java/com/zhiwei/brandkbs2/util/Tools.java
......@@ -554,10 +554,13 @@ public class Tools {
String separator = "-";
StringBuilder sb = new StringBuilder();
for (Object obj : objects) {
if (Objects.isNull(obj)){
continue;
}
sb.append(obj).append(separator);
}
String resultStr = sb.toString();
return resultStr.substring(0, resultStr.length() - 1);
return StringUtils.isBlank(resultStr) ? resultStr : resultStr.substring(0, resultStr.length() - 1);
}
public static String[] split(String concatStr) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment