Skip to content

B
brandkbs2
Commit 7e4e6f47 by 陈健智
Options

搜索优化、新增视频中心

parent d99ec5a1
Showing with 257 additions and 37 deletions
src/main/java/com/zhiwei/brandkbs2/common/GenericAttribute.java
package com.zhiwei.brandkbs2.common;
import java.util.Arrays;
import java.util.List;
/**
* @ClassName: GenericAttribute
* @Description 一般属性值
......@@ -131,6 +134,10 @@ public class GenericAttribute {
* 地域
*/
public static final String ES_FIELD = "field";
/**
* video_urls
*/
public static final String ES_VIDEO_URLS = "video_urls";
// public static final String ES_CACHE_MAP_PROJECT = "brandkbs_cache_maps.project_id.keyword";
// public static final String ES_CACHE_MAP_LINKED_GROUP_ID = "brandkbs_cache_maps.linked_group_id.keyword";
......@@ -166,5 +173,6 @@ public class GenericAttribute {
// return value;
// }
// }
/**视频real_source*/
public static final List<String> VIDEO_PLATFORM = Arrays.asList("抖音", "快手", "小红书", "bilibili", "新浪微博视频", "今日头条", "微信视频号", "西瓜视频", "其他");
}
src/main/java/com/zhiwei/brandkbs2/controller/app/AppArticleController.java
......@@ -542,7 +542,7 @@ public class AppArticleController extends BaseController {
@GetMapping("/daily-report/list")
@LogRecord(description = "舆情简报-每日一报")
public ResponseResult getDailyReportList(@RequestParam(value = "page", defaultValue = "1") Integer page,
@RequestParam(value = "pageSize", defaultValue = "10")Integer pageSize) {
@RequestParam(value = "pageSize", defaultValue = "10") Integer pageSize) {
return ResponseResult.success(markDataService.getDailyReportList(page, pageSize));
}
......
src/main/java/com/zhiwei/brandkbs2/controller/app/AppDownloadController.java
......@@ -18,6 +18,7 @@ import com.zhiwei.brandkbs2.pojo.vo.ProjectVO;
import com.zhiwei.brandkbs2.service.*;
import com.zhiwei.brandkbs2.util.Tools;
import io.swagger.annotations.*;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.logging.log4j.LogManager;
......@@ -395,6 +396,9 @@ public class AppDownloadController extends BaseController {
@PostMapping(value = "/contend/mark")
@DownloadTask(taskName = "竞品库竞品舆情下载", description = "竞品库竞品舆情")
public ResponseResult exportContendMarkList(@RequestBody MarkSearchDTO markSearchDTO) {
if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
return ResponseResult.failure("不支持特殊符号字段查询");
}
Pair<String, List<ExportAppYuqingDTO>> stringListPair = markDataService.downloadContendMarkList(markSearchDTO);
// excel写入至指定路径
String projectName = projectService.getProjectById(UserThreadLocal.getProjectId()).getProjectName();
......@@ -408,6 +412,9 @@ public class AppDownloadController extends BaseController {
@LogRecord(description = "全网搜-舆情导出", values = {"startTime", "endTime", "fans", "filterType", "filterWords", "search", "keyword", "platforms", "sensitiveChannels", "sourceKeyword"}, entity = true, arguments = true)
@DownloadTask(taskName = "全网搜舆情下载", description = "全网搜舆情")
public ResponseResult exportSearchWhole(@RequestBody SearchFilterDTO dto) {
if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
return ResponseResult.failure("不支持特殊符号字段查询");
}
// 针对商业数据库做限制
if (dto.isExternalDataSource()) {
long time = DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), -89).getTime();
......@@ -430,7 +437,7 @@ public class AppDownloadController extends BaseController {
String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), "全网搜舆情列表数据");
EasyExcelUtil.write(filePath, "sheet1", ExportSearchWholeDTO.class, exportList);
if (dto.isExternalDataSource()) {
extraService.decreaseRecord(dto.getSearch(), WholeSearchRecord.UsedType.output, exportList.size());
extraService.decreaseRecord(dto.getKeyword(), WholeSearchRecord.UsedType.output, exportList.size());
}
return ResponseResult.success(filePath);
}
......
src/main/java/com/zhiwei/brandkbs2/controller/app/AppSearchController.java
......@@ -21,6 +21,7 @@ import com.zhiwei.brandkbs2.util.Tools;
import com.zhiwei.middleware.event.pojo.dto.BrandkbsEventSearchDTO;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.joda.time.Period;
......@@ -147,6 +148,9 @@ public class AppSearchController extends BaseController {
@LogRecord(values = {"fans", "sensitiveChannels:father,son", "keyword", "search"}, description = "全网搜", arguments = true, entity = true)
@PostMapping("/searchWhole")
public ResponseResult searchWholeNetwork(@RequestBody SearchFilterDTO dto) {
if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
return ResponseResult.failure("不支持特殊符号字段查询");
}
// 针对商业数据库做限制
if (dto.isExternalDataSource()) {
long time = DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), -89).getTime();
......@@ -168,14 +172,23 @@ public class AppSearchController extends BaseController {
List<JSONObject> list = pair.getLeft().getJSONArray("list").toJavaList(JSONObject.class);
commonService.sentimentAnalysis(list);
}
extraService.decreaseRecord(dto.getSearch(), WholeSearchRecord.UsedType.query, pair.getRight());
extraService.decreaseRecord(dto.getKeyword(), WholeSearchRecord.UsedType.query, pair.getRight());
return ResponseResult.success(pair.getLeft());
}
@ApiOperation("搜索-获取全网搜余额")
@PostMapping("/searchWhole/balance")
public ResponseResult getWholeSearchBalance() {
return ResponseResult.success(extraService.getWholeSearchBalance());
}
@ApiOperation("搜索-全网搜-舆情导出")
@PostMapping("/exportSearchWhole")
@LogRecord(description = "全网搜-舆情导出", values = {"startTime", "endTime", "fans", "filterType", "filterWords", "search", "keyword", "platforms", "sensitiveChannels", "sourceKeyword"}, entity = true, arguments = true)
public ResponseResult exportSearchWhole(@RequestBody SearchFilterDTO dto) {
if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
return ResponseResult.failure("不支持特殊符号字段查询");
}
// 针对商业数据库做限制
if (dto.isExternalDataSource()) {
long time = DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), -89).getTime();
......@@ -195,20 +208,27 @@ public class AppSearchController extends BaseController {
List<ExportSearchWholeDTO> exportList = markDataService.exportSearchWhole(dto);
EasyExcelUtil.download("全网搜舆情列表数据", "sheet1", ExportSearchWholeDTO.class, exportList, response);
if (dto.isExternalDataSource()) {
extraService.decreaseRecord(dto.getSearch(), WholeSearchRecord.UsedType.output, exportList.size());
extraService.decreaseRecord(dto.getKeyword(), WholeSearchRecord.UsedType.output, exportList.size());
}
return ResponseResult.success();
}
@ApiOperation("舆情列表")
@LogRecord(values = {"keyword", "sourceKeyword", "hostKeyword"}, description = "查舆情", arguments = true, entity = true)
@LogRecord(values = {"searchType", "keyword"}, description = "查舆情", arguments = true, entity = true)
@PostMapping("/mark/list")
public ResponseResult getYuqingMarkList(@RequestBody MarkSearchDTO markSearchDTO) {
if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
return ResponseResult.failure("不支持特殊符号字段查询");
}
PageVO<MarkFlowEntity> yuqingMarkList = markDataService.getYuqingMarkList(markSearchDTO);
// 仅第一页增加平台进量（声量）统计
if (1 == markSearchDTO.getPage()) {
JSONObject info = yuqingMarkList.getInfo();
info.put("counts", markDataService.countMarkPlatform(markSearchDTO));
if (Objects.equals("视频", markSearchDTO.getSearchType())) {
info.put("counts", markDataService.countMarkVideoPlatform(markSearchDTO));
}else {
info.put("counts", markDataService.countMarkPlatform(markSearchDTO));
}
yuqingMarkList.setInfo(info);
}
return ResponseResult.success(yuqingMarkList);
......@@ -220,7 +240,7 @@ public class AppSearchController extends BaseController {
public ResponseResult getOriginList(@RequestBody MarkSearchDTO markSearchDTO) {
PageVO<MarkFlowEntity> originList = markDataService.getOriginList(markSearchDTO);
// 仅第一页增加平台进量（声量）统计
if (1 == markSearchDTO.getPage()) {
if (1 == markSearchDTO.getPage() && !Objects.equals("视频", markSearchDTO.getSearchType())) {
JSONObject info = originList.getInfo();
info.put("counts", markDataService.countOriginPlatform(markSearchDTO));
originList.setInfo(info);
......@@ -301,6 +321,9 @@ public class AppSearchController extends BaseController {
@LogRecord(values = "keyword", description = "查竞品",arguments = true, entity = true)
@PostMapping("/contend/list")
public ResponseResult getContendSearchList(@RequestBody MarkSearchDTO markSearchDTO) {
if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
return ResponseResult.failure("不支持特殊符号字段查询");
}
return ResponseResult.success(markDataService.getContendSearchList(markSearchDTO));
}
......
src/main/java/com/zhiwei/brandkbs2/es/EsQueryTools.java
......@@ -291,6 +291,27 @@ public class EsQueryTools {
}
/**
* 视频平台查询
* @param platforms
* @return
*/
public static BoolQueryBuilder assembleVideoPlatformQuery(List<String> platforms){
BoolQueryBuilder realSourceQuery = QueryBuilders.boolQuery();
// realSource
// 其他 特殊处理
if (platforms.contains("其他")) {
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
for (int i = 0; i < GenericAttribute.VIDEO_PLATFORM.size() - 1; i++) {
queryBuilder.mustNot(QueryBuilders.termQuery(GenericAttribute.ES_REAL_SOURCE, GenericAttribute.VIDEO_PLATFORM.get(i)));
}
realSourceQuery.should(queryBuilder);
platforms.remove("其他");
}
platforms.forEach(pt -> realSourceQuery.should(QueryBuilders.termQuery(GenericAttribute.ES_REAL_SOURCE, pt)));
return realSourceQuery;
}
/**
* 针对微博平台转发 查询语句
*
* @param isForward
......
src/main/java/com/zhiwei/brandkbs2/pojo/MarkFlowEntity.java
......@@ -15,6 +15,7 @@ import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import java.io.Serializable;
import java.util.List;
......@@ -163,6 +164,10 @@ public class MarkFlowEntity implements Serializable {
videoMark.setMgroup(GlobalPojo.getYuQingProjectNameById(linkedGroupId));
this.filterInfo = videoMark.filterInfo();
}
// 视频时长处理
if (StringUtils.isNotBlank(videoMark.getLength())){
videoMark.setLength(Tools.formatVideoLength(videoMark.getLength()));
}
this.type = FlowEntityUtil.getVideoType(videoMark);
this.data = videoMark;
break;
......
src/main/java/com/zhiwei/brandkbs2/pojo/dto/MarkSearchDTO.java
......@@ -64,12 +64,24 @@ public class MarkSearchDTO {
private Long endTime;
/**
* 数据类型 全量数据 | 视频
*/
@ApiModelProperty("搜索类型:全量数据|视频")
private String searchType;
/**
* 平台
*/
@ApiModelProperty(value = "平台")
private List<String> platforms;
/**
* 视频平台
*/
@ApiModelProperty(value = "视频平台")
private List<String> videoPlatforms;
/**
* 媒体类型
*/
@ApiModelProperty(value = "重要发声方")
......@@ -119,7 +131,7 @@ public class MarkSearchDTO {
// query部分
/**
* 检索字段
* 关键字检索字段
*/
@ApiModelProperty(value = "检索字段")
private String searchField;
......@@ -155,10 +167,16 @@ public class MarkSearchDTO {
private boolean fold;
/**
*
* 屏蔽词
*/
@ApiModelProperty(value = "屏蔽词")
private String shieldWord;
private String filterWords;
/**
* 屏蔽词检索字段 全文|标题
*/
@ApiModelProperty(value = "屏蔽词检索字段：全文|标题")
private String filterType;
/**
* 方案id,仅非人工项目
......
src/main/java/com/zhiwei/brandkbs2/pojo/dto/SearchFilterDTO.java
package com.zhiwei.brandkbs2.pojo.dto;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.common.GlobalPojo;
import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import org.apache.commons.collections4.CollectionUtils;
import java.util.*;
import java.util.stream.Collectors;
@Data
@ApiModel("全网搜搜索传输类")
......@@ -42,10 +47,16 @@ public class SearchFilterDTO {
private Long crawlerTime;
/**
* 数据类型 全量数据 | 视频
*/
@ApiModelProperty("搜索类型:全量数据 | 视频")
private String searchType;
/**
* 关键字
*/
@ApiModelProperty(value = "关键字")
private String search;
private String keyword;
/**
* 平台
......@@ -54,6 +65,12 @@ public class SearchFilterDTO {
private List<String> platforms;
/**
* 视频平台
*/
@ApiModelProperty(value = "视频平台")
private List<String> videoPlatforms;
/**
* 排序字段
*/
@ApiModelProperty(value = "排序字段")
......@@ -146,8 +163,9 @@ public class SearchFilterDTO {
Map<String, Object> map = new HashMap<>();
map.put("timeType", "time");
map.put("pt", getPlatforms());
map.put("dataType", searchType);
map.put("matchType", matchType);
map.put("search", search);
map.put("search", keyword);
map.put("startTime", startTime);
map.put("endTime", endTime);
map.put("source", sourceKeyword);
......@@ -173,7 +191,10 @@ public class SearchFilterDTO {
}
public List<String> getPlatforms() {
return Objects.nonNull(platforms) ? this.platforms : Collections.emptyList();
if (Objects.equals("视频", this.searchType)){
return CollectionUtils.isNotEmpty(this.videoPlatforms) ? this.videoPlatforms : new ArrayList<>(GenericAttribute.VIDEO_PLATFORM);
}
return CollectionUtils.isNotEmpty(this.platforms) ? this.platforms : GlobalPojo.PLATFORMS.stream().map(MessagePlatform::getId).collect(Collectors.toList());
}
public String getMatchType() {
......
src/main/java/com/zhiwei/brandkbs2/service/EsSearchService.java
......@@ -75,4 +75,12 @@ public interface EsSearchService {
* @throws IOException
*/
JSONObject countNonManualPlatform(MarkSearchDTO dto) throws IOException;
/**
* 查舆情视频中心进量（声量）统计
* @param dto
* @return
* @throws IOException
*/
JSONObject countMarkVideoPlatform(MarkSearchDTO dto) throws IOException;
}
src/main/java/com/zhiwei/brandkbs2/service/ExtraService.java
......@@ -56,6 +56,12 @@ public interface ExtraService {
JSONObject overview();
/**
* 获取全网搜余额
* @return
*/
Integer getWholeSearchBalance();
/**
* @param keyword 关键词
* @param usedType 使用类别
* @param usedCount 使用量
......
src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java
......@@ -393,6 +393,13 @@ public interface MarkDataService {
JSONObject countOriginPlatform(MarkSearchDTO dto);
/**
* 查舆情视频中心-平台进量（声量）统计
* @param dto 标注数据搜索传输类
* @return
*/
JSONObject countMarkVideoPlatform(MarkSearchDTO dto);
/**
* 新建非人工项目方案
* @param dto
* @return
......
src/main/java/com/zhiwei/brandkbs2/service/impl/EsSearchServiceImpl.java
......@@ -77,6 +77,11 @@ public class EsSearchServiceImpl implements EsSearchService {
return platformCount(dto, this::createNonManualSearchHelper, esClientDao);
}
@Override
public JSONObject countMarkVideoPlatform(MarkSearchDTO dto) throws IOException {
return videoPlatformCount(dto, this::createSearchHelper, esClientDao);
}
private Pair<SearchHits[], Map<String, Long>> searchHitsAndCount(MarkSearchDTO dto, ArticleFunction.CreateSearchHelper createSearchHelper,
EsClientDao clientDao, boolean scroll) throws IOException {
EsClientDao.SearchHelper helper = createSearchHelper.create(dto);
......@@ -99,10 +104,18 @@ public class EsSearchServiceImpl implements EsSearchService {
BoolQueryBuilder postFilter = EsQueryTools.assembleCacheMapsQuery(projectId, contendId, dto.getPoliticsLevel(), dto.getField(), dto.getRegion(), dto.getMainBodyType(), dto.getChannelEmotion());
// time
postFilter.must(QueryBuilders.rangeQuery(dto.getTimeType()).gte(dto.getStartTime()).lt(dto.getEndTime()));
// searchType
if (Objects.equals("视频", dto.getSearchType())){
postFilter.must(QueryBuilders.existsQuery(GenericAttribute.ES_VIDEO_URLS));
}
// platform
if (CollectionUtils.isNotEmpty(dto.getPlatforms())) {
postFilter.must(EsQueryTools.assemblePlatformQuery(Tools.getPlatformByIds(dto.getPlatforms())));
}
// videoPlatforms
if (CollectionUtils.isNotEmpty(dto.getVideoPlatforms())){
postFilter.must(EsQueryTools.assembleVideoPlatformQuery(dto.getVideoPlatforms()));
}
// tags
if (CollectionUtils.isNotEmpty(dto.getTags())) {
postFilter.must(EsQueryTools.assembleTagQuery(dto.getTags()));
......@@ -119,9 +132,10 @@ public class EsSearchServiceImpl implements EsSearchService {
// Query
postFilter.must(EsQueryTools.assembleNormalKeywordQuery(dto.getKeyword(), fieldSearch));
}
// shieldWord
if (StringUtils.isNotEmpty(dto.getShieldWord())) {
postFilter.mustNot(EsQueryTools.assembleNormalKeywordQuery(dto.getShieldWord(), fieldSearch));
// filterWords
String[] filterFieldSearch = "标题".equals(dto.getFilterType()) ? new String[]{GenericAttribute.ES_IND_TITLE} : new String[]{GenericAttribute.ES_IND_FULL_TEXT};
if (StringUtils.isNotEmpty(dto.getFilterWords())) {
postFilter.mustNot(EsQueryTools.assembleNormalKeywordQuery(dto.getFilterWords(), filterFieldSearch));
}
// sourceKeyword
if (StringUtils.isNotEmpty(dto.getSourceKeyword())) {
......@@ -383,4 +397,31 @@ public class EsSearchServiceImpl implements EsSearchService {
res.put("total", total);
return res;
}
/**
* 视频realSource进量统计
* @param dto
* @param createSearchHelper
* @param esClientDao
* @return
* @throws IOException
*/
private JSONObject videoPlatformCount(MarkSearchDTO dto, ArticleFunction.CreateSearchHelper createSearchHelper, EsClientDao esClientDao) throws IOException {
List<JSONObject> list = new ArrayList<>();
for (String realSource : GenericAttribute.VIDEO_PLATFORM) {
List<String> oneRealSource = new ArrayList<>(1);
oneRealSource.add(realSource);
dto.setVideoPlatforms(oneRealSource);
EsClientDao.SearchHelper helper = createSearchHelper.create(dto);
JSONObject jsonObject = new JSONObject();
jsonObject.put("count", esClientDao.count(helper.getPostFilter()));
jsonObject.put("name", realSource);
list.add(jsonObject);
}
Long total = list.stream().map(jsonObject -> jsonObject.getLongValue("count")).reduce(Long::sum).orElse(0L);
JSONObject res = new JSONObject();
res.put("count", list);
res.put("total", total);
return res;
}
}
src/main/java/com/zhiwei/brandkbs2/service/impl/ExtraServiceImpl.java
......@@ -109,6 +109,11 @@ public class ExtraServiceImpl implements ExtraService {
}
@Override
public Integer getWholeSearchBalance() {
return projectDao.findOneById(UserThreadLocal.getProjectId()).getWholeSearchBalance();
}
@Override
public void decreaseRecord(String keyword, WholeSearchRecord.UsedType usedType, Integer usedCount) {
if (null == usedCount) {
return;
......
src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java
......@@ -1438,6 +1438,12 @@ public class MarkDataServiceImpl implements MarkDataService {
result.put("followersNum", list.get("followers_num"));
result.put("realSource", list.get("real_source"));
result.put("ipLocation", list.get("ip_location"));
result.put("picUrls", list.get("pic_urls"));
result.put("videoUrls", list.getJSONArray("video_urls"));
// 视频时长处理
if (StringUtils.isNotBlank(list.getString("length"))){
result.put("length", Tools.formatVideoLength(list.getString("length")));
}
resultList.add(result);
}
// 仅第一页返回平台进量（声量）统计
......@@ -1643,7 +1649,7 @@ public class MarkDataServiceImpl implements MarkDataService {
*/
private JSONObject externalTransform(SearchFilterDTO dto) {
JSONObject params = new JSONObject();
params.put("crawlerWord", dto.getSearch());
params.put("crawlerWord", dto.getKeyword());
params.put("startTime", dto.getStartTime());
params.put("endTime", dto.getEndTime());
String platforms;
......@@ -1662,6 +1668,12 @@ public class MarkDataServiceImpl implements MarkDataService {
// matchFields
if (dto.getMatchType().equals("标题")) {
params.put("matchFields", Collections.singletonList("title"));
}else if (dto.getMatchType().equals("全文")){
params.put("matchFields", Collections.singletonList("content"));
}
// sourceKeyword
if (StringUtils.isNotBlank(dto.getSourceKeyword())){
params.put("userList", Arrays.asList(dto.getSourceKeyword().trim().split("\\|")));
}
// // 历史部分
// if (Tools.isEmpty(dto.getPlatforms())) {
......@@ -1883,6 +1895,17 @@ public class MarkDataServiceImpl implements MarkDataService {
}
@Override
public JSONObject countMarkVideoPlatform(MarkSearchDTO dto) {
defaultMarkSearch(dto);
try {
return esSearchService.countMarkVideoPlatform(dto);
} catch (Exception e) {
ExceptionCast.cast(CommonCodeEnum.FAIL, "es查询异常", e);
}
return null;
}
@Override
public ResponseResult addNonManualProjectPlan(NonManualProjectPlanDTO dto) {
String projectId = UserThreadLocal.getProjectId();
if (nonManualProjectPlanDao.count(new Query(Criteria.where("projectId").is(projectId))) >= 5) {
......@@ -2052,7 +2075,7 @@ public class MarkDataServiceImpl implements MarkDataService {
private SearchFilterDTO createSearchFilterDTO(String keyword, String filterWord){
SearchFilterDTO dto = new SearchFilterDTO();
dto.setSearch(keyword);
dto.setKeyword(keyword);
dto.setFilterWords(filterWord);
// 预估一天
dto.setEndTime(System.currentTimeMillis());
......@@ -2415,15 +2438,15 @@ public class MarkDataServiceImpl implements MarkDataService {
List<JSONObject> res = new ArrayList<>();
String projectId = UserThreadLocal.getProjectId();
List<NonManualProjectPlan> planList = nonManualProjectPlanDao.findList(new Query(Criteria.where("projectId").is(projectId)));
long startTime = Constant.DF_yyyyMMdd.parse(Constant.DF_yyyyMMdd.format(new Date())).getTime();
long endTime = System.currentTimeMillis();
// long startTime = Constant.DF_yyyyMMdd.parse(Constant.DF_yyyyMMdd.format(new Date())).getTime();
// long endTime = System.currentTimeMillis();
for (NonManualProjectPlan plan : planList) {
JSONObject jsonObject = new JSONObject();
jsonObject.put("id", plan.getId());
jsonObject.put("name", plan.getName());
BoolQueryBuilder query = EsQueryTools.assembleCacheMapsPlanQuery(projectId, plan.getId());
query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
jsonObject.put("count", esClientDao.count(query));
// BoolQueryBuilder query = EsQueryTools.assembleCacheMapsPlanQuery(projectId, plan.getId());
// query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
// jsonObject.put("count", esClientDao.count(query));
res.add(jsonObject);
}
return res;
......@@ -2950,10 +2973,11 @@ public class MarkDataServiceImpl implements MarkDataService {
// postFilter
BoolQueryBuilder postFilter = yuqingAnalyzeQuery(dto);
searchHelper.setPostFilter(postFilter);
searchHelper.setSize(10000);
searchHelper.setSize(1000);
List<String> textList = new ArrayList<>();
List<SearchResponse> searchResponses = Collections.singletonList(esClientDao.searchResponse(searchHelper));
for (SearchResponse searchResponse : searchResponses) {
for (int i = 1; i <= 10; i++) {
searchHelper.setFrom((i - 1) * 1000);
SearchResponse searchResponse = esClientDao.searchResponse(searchHelper);
for (SearchHit searchHit : searchResponse.getHits().getHits()) {
Map<String, Object> source = searchHit.getSourceAsMap();
BaseMap baseMap = Tools.getBaseFromEsMap(source);
......@@ -3260,11 +3284,6 @@ public class MarkDataServiceImpl implements MarkDataService {
*/
private Map<String, List<LineVO>> getPlatformSpreadTendency(MarkSearchDTO dto) throws IOException{
Map<String, List<LineVO>> res = new HashMap<>(4);
// 平台
// List<String> platforms = new ArrayList<>();
// for (String platformNames : Arrays.asList("微博", "微信", "抖音", "小红书")) {
// platforms.add(GlobalPojo.getPlatformIdByName(platformNames));
// }
if (CollectionUtils.isEmpty(dto.getPlatforms())){
dto.setPlatforms(Stream.of("微博", "微信", "抖音", "小红书").map(GlobalPojo::getPlatformIdByName).collect(Collectors.toList()));
}
......@@ -3409,12 +3428,20 @@ public class MarkDataServiceImpl implements MarkDataService {
if (Objects.nonNull(dto.getPlanId())){
postFilter = EsQueryTools.assembleCacheMapsPlanQuery(projectId, dto.getPlanId(), dto.getPoliticsLevel());
}
// searchType
if (Objects.equals("视频", dto.getSearchType())){
postFilter.must(QueryBuilders.existsQuery(GenericAttribute.ES_VIDEO_URLS));
}
// time
postFilter.must(QueryBuilders.rangeQuery("time").gte(dto.getStartTime()).lt(dto.getEndTime()));
// platform
if (CollectionUtils.isNotEmpty(dto.getPlatforms())) {
postFilter.must(EsQueryTools.assemblePlatformQuery(Tools.getPlatformByIds(dto.getPlatforms())));
}
// videoPlatforms
if (CollectionUtils.isNotEmpty(dto.getVideoPlatforms())){
postFilter.must(EsQueryTools.assembleVideoPlatformQuery(dto.getVideoPlatforms()));
}
// tags
if (CollectionUtils.isNotEmpty(dto.getTags())) {
postFilter.must(EsQueryTools.assembleTagQuery(dto.getTags()));
......@@ -3430,14 +3457,15 @@ public class MarkDataServiceImpl implements MarkDataService {
// Query
postFilter.must(EsQueryTools.assembleNormalKeywordQuery(dto.getKeyword(), fieldSearch));
}
// filterWords
String[] filterFieldSearch = "标题".equals(dto.getFilterType()) ? new String[]{GenericAttribute.ES_IND_TITLE} : new String[]{GenericAttribute.ES_IND_FULL_TEXT};
if (StringUtils.isNotEmpty(dto.getFilterWords())) {
postFilter.mustNot(EsQueryTools.assembleNormalKeywordQuery(dto.getFilterWords(), filterFieldSearch));
}
// emotion
if (StringUtils.isNotEmpty(emotion) && !EmotionEnum.ALL.getName().equals(emotion)) {
postFilter.must(QueryBuilders.termQuery("brandkbs_mark_cache_maps.name.keyword", emotion));
}
// shieldWord
if (StringUtils.isNotEmpty(dto.getShieldWord())) {
postFilter.mustNot(EsQueryTools.assembleNormalKeywordQuery(dto.getShieldWord(), fieldSearch));
}
// sourceKeyword
if (StringUtils.isNotEmpty(dto.getSourceKeyword())) {
postFilter.must(EsQueryTools.assembleSourceKeywordQuery(dto.getSourceKeyword()));
......
src/main/java/com/zhiwei/brandkbs2/util/Tools.java
......@@ -37,6 +37,8 @@ import org.springframework.web.multipart.MultipartFile;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.*;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.net.URLEncoder;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
......@@ -1356,6 +1358,10 @@ public class Tools {
return Arrays.stream(str.split("[\\u007c]+")).filter(StringUtils::isNotBlank).collect(Collectors.joining("|"));
}
public static boolean checkUniteString(String str){
return str.split("[\\u4e28]+").length > 1 || str.split("[\\uff5c]+").length > 1 || str.split("[\\u007c]+").length > 1;
}
/**
* 过滤 “分享一篇文章”，”暂无标题“，”Notitle“ 的无用标题
* @param title
......@@ -1373,4 +1379,19 @@ public class Tools {
public static Boolean filterUselessSource(String source){
return FILTER_SOURCE.contains(source);
}
/**
* 格式化视频时长秒
* @param videoLength
* @return
*/
public static String formatVideoLength(String videoLength){
if (videoLength.contains(":")){
return videoLength;
}
double temp = Double.parseDouble(videoLength);
long mm = BigDecimal.valueOf(temp).divide(BigDecimal.valueOf(60), 0, RoundingMode.HALF_UP).longValue();
long ss = BigDecimal.valueOf(temp).divideAndRemainder(BigDecimal.valueOf(60))[1].setScale(0, RoundingMode.HALF_UP).longValue();
return ((mm < 10 ? ("0" + mm) : mm) + ":" + (ss < 10 ? ("0" + ss) : ss));
}
}
\ No newline at end of file
src/main/resources/application-dev.properties
......@@ -102,7 +102,7 @@ ef.checkCaptcha.url=https://ef.zhiweidata.com/captchaApi/checkeCaptcha.do.do?id=
#\u5168\u7F51\u641C\u5916\u90E8\u63A5\u53E3
istarshine.addIStarShineKSData.url=https://istarshine-service.zhiweidata.com/istarshine/ks/addIStarShineKSData
istarshine.getIStarShineKSInfoData.url=https://istarshine-service.zhiweidata.com/istarshine/ks/getIStarShineKSInfoData?taskId={1}&appId=6183571e0d710000f6003a12
whole.search.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/search/yuqing
whole.search.url=https://yuqing.test.zhiweidata.com/qbjcbackPhoenix/interface/middleware/search/yuqing
whole.searchCriteria.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/search/pt
whole.extraParam.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/search/extra/param
brief.report.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/brief-report-web
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment