Skip to content

B
brandkbs2
Commit 615eabd6 by shenjunjie
Options

Merge branch 'feature' into 'dev' 

Feature

See merge request !509
parents 6744d566 7e4e6f47
Showing with 348 additions and 57 deletions
src/main/java/com/zhiwei/brandkbs2/common/GenericAttribute.java
package com.zhiwei.brandkbs2.common;
import java.util.Arrays;
import java.util.List;
/**
* @ClassName: GenericAttribute
* @Description 一般属性值
......@@ -131,6 +134,10 @@ public class GenericAttribute {
* 地域
*/
public static final String ES_FIELD = "field";
/**
* video_urls
*/
public static final String ES_VIDEO_URLS = "video_urls";
// public static final String ES_CACHE_MAP_PROJECT = "brandkbs_cache_maps.project_id.keyword";
// public static final String ES_CACHE_MAP_LINKED_GROUP_ID = "brandkbs_cache_maps.linked_group_id.keyword";
......@@ -166,5 +173,6 @@ public class GenericAttribute {
// return value;
// }
// }
/**视频real_source*/
public static final List<String> VIDEO_PLATFORM = Arrays.asList("抖音", "快手", "小红书", "bilibili", "新浪微博视频", "今日头条", "微信视频号", "西瓜视频", "其他");
}
src/main/java/com/zhiwei/brandkbs2/controller/app/AppArticleController.java
......@@ -542,7 +542,7 @@ public class AppArticleController extends BaseController {
@GetMapping("/daily-report/list")
@LogRecord(description = "舆情简报-每日一报")
public ResponseResult getDailyReportList(@RequestParam(value = "page", defaultValue = "1") Integer page,
@RequestParam(value = "pageSize", defaultValue = "10")Integer pageSize) {
@RequestParam(value = "pageSize", defaultValue = "10") Integer pageSize) {
return ResponseResult.success(markDataService.getDailyReportList(page, pageSize));
}
......
src/main/java/com/zhiwei/brandkbs2/controller/app/AppDownloadController.java
......@@ -18,6 +18,7 @@ import com.zhiwei.brandkbs2.pojo.vo.ProjectVO;
import com.zhiwei.brandkbs2.service.*;
import com.zhiwei.brandkbs2.util.Tools;
import io.swagger.annotations.*;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.logging.log4j.LogManager;
......@@ -395,6 +396,9 @@ public class AppDownloadController extends BaseController {
@PostMapping(value = "/contend/mark")
@DownloadTask(taskName = "竞品库竞品舆情下载", description = "竞品库竞品舆情")
public ResponseResult exportContendMarkList(@RequestBody MarkSearchDTO markSearchDTO) {
if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
return ResponseResult.failure("不支持特殊符号字段查询");
}
Pair<String, List<ExportAppYuqingDTO>> stringListPair = markDataService.downloadContendMarkList(markSearchDTO);
// excel写入至指定路径
String projectName = projectService.getProjectById(UserThreadLocal.getProjectId()).getProjectName();
......@@ -408,6 +412,9 @@ public class AppDownloadController extends BaseController {
@LogRecord(description = "全网搜-舆情导出", values = {"startTime", "endTime", "fans", "filterType", "filterWords", "search", "keyword", "platforms", "sensitiveChannels", "sourceKeyword"}, entity = true, arguments = true)
@DownloadTask(taskName = "全网搜舆情下载", description = "全网搜舆情")
public ResponseResult exportSearchWhole(@RequestBody SearchFilterDTO dto) {
if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
return ResponseResult.failure("不支持特殊符号字段查询");
}
// 针对商业数据库做限制
if (dto.isExternalDataSource()) {
long time = DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), -89).getTime();
......@@ -430,7 +437,7 @@ public class AppDownloadController extends BaseController {
String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), "全网搜舆情列表数据");
EasyExcelUtil.write(filePath, "sheet1", ExportSearchWholeDTO.class, exportList);
if (dto.isExternalDataSource()) {
extraService.decreaseRecord(dto.getSearch(), WholeSearchRecord.UsedType.output, exportList.size());
extraService.decreaseRecord(dto.getKeyword(), WholeSearchRecord.UsedType.output, exportList.size());
}
return ResponseResult.success(filePath);
}
......
src/main/java/com/zhiwei/brandkbs2/controller/app/AppSearchController.java
......@@ -21,6 +21,7 @@ import com.zhiwei.brandkbs2.util.Tools;
import com.zhiwei.middleware.event.pojo.dto.BrandkbsEventSearchDTO;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.joda.time.Period;
......@@ -147,6 +148,9 @@ public class AppSearchController extends BaseController {
@LogRecord(values = {"fans", "sensitiveChannels:father,son", "keyword", "search"}, description = "全网搜", arguments = true, entity = true)
@PostMapping("/searchWhole")
public ResponseResult searchWholeNetwork(@RequestBody SearchFilterDTO dto) {
if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
return ResponseResult.failure("不支持特殊符号字段查询");
}
// 针对商业数据库做限制
if (dto.isExternalDataSource()) {
long time = DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), -89).getTime();
......@@ -168,14 +172,23 @@ public class AppSearchController extends BaseController {
List<JSONObject> list = pair.getLeft().getJSONArray("list").toJavaList(JSONObject.class);
commonService.sentimentAnalysis(list);
}
extraService.decreaseRecord(dto.getSearch(), WholeSearchRecord.UsedType.query, pair.getRight());
extraService.decreaseRecord(dto.getKeyword(), WholeSearchRecord.UsedType.query, pair.getRight());
return ResponseResult.success(pair.getLeft());
}
@ApiOperation("搜索-获取全网搜余额")
@PostMapping("/searchWhole/balance")
public ResponseResult getWholeSearchBalance() {
return ResponseResult.success(extraService.getWholeSearchBalance());
}
@ApiOperation("搜索-全网搜-舆情导出")
@PostMapping("/exportSearchWhole")
@LogRecord(description = "全网搜-舆情导出", values = {"startTime", "endTime", "fans", "filterType", "filterWords", "search", "keyword", "platforms", "sensitiveChannels", "sourceKeyword"}, entity = true, arguments = true)
public ResponseResult exportSearchWhole(@RequestBody SearchFilterDTO dto) {
if (StringUtils.isNotEmpty(dto.getKeyword()) && Tools.checkUniteString(dto.getKeyword())){
return ResponseResult.failure("不支持特殊符号字段查询");
}
// 针对商业数据库做限制
if (dto.isExternalDataSource()) {
long time = DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), -89).getTime();
......@@ -195,20 +208,27 @@ public class AppSearchController extends BaseController {
List<ExportSearchWholeDTO> exportList = markDataService.exportSearchWhole(dto);
EasyExcelUtil.download("全网搜舆情列表数据", "sheet1", ExportSearchWholeDTO.class, exportList, response);
if (dto.isExternalDataSource()) {
extraService.decreaseRecord(dto.getSearch(), WholeSearchRecord.UsedType.output, exportList.size());
extraService.decreaseRecord(dto.getKeyword(), WholeSearchRecord.UsedType.output, exportList.size());
}
return ResponseResult.success();
}
@ApiOperation("舆情列表")
@LogRecord(values = "keyword", description = "查舆情", arguments = true, entity = true)
@LogRecord(values = {"searchType", "keyword"}, description = "查舆情", arguments = true, entity = true)
@PostMapping("/mark/list")
public ResponseResult getYuqingMarkList(@RequestBody MarkSearchDTO markSearchDTO) {
if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
return ResponseResult.failure("不支持特殊符号字段查询");
}
PageVO<MarkFlowEntity> yuqingMarkList = markDataService.getYuqingMarkList(markSearchDTO);
// 仅第一页增加平台进量（声量）统计
if (1 == markSearchDTO.getPage()) {
JSONObject info = yuqingMarkList.getInfo();
if (Objects.equals("视频", markSearchDTO.getSearchType())) {
info.put("counts", markDataService.countMarkVideoPlatform(markSearchDTO));
}else {
info.put("counts", markDataService.countMarkPlatform(markSearchDTO));
}
yuqingMarkList.setInfo(info);
}
return ResponseResult.success(yuqingMarkList);
......@@ -220,7 +240,7 @@ public class AppSearchController extends BaseController {
public ResponseResult getOriginList(@RequestBody MarkSearchDTO markSearchDTO) {
PageVO<MarkFlowEntity> originList = markDataService.getOriginList(markSearchDTO);
// 仅第一页增加平台进量（声量）统计
if (1 == markSearchDTO.getPage()) {
if (1 == markSearchDTO.getPage() && !Objects.equals("视频", markSearchDTO.getSearchType())) {
JSONObject info = originList.getInfo();
info.put("counts", markDataService.countOriginPlatform(markSearchDTO));
originList.setInfo(info);
......@@ -301,6 +321,9 @@ public class AppSearchController extends BaseController {
@LogRecord(values = "keyword", description = "查竞品",arguments = true, entity = true)
@PostMapping("/contend/list")
public ResponseResult getContendSearchList(@RequestBody MarkSearchDTO markSearchDTO) {
if (StringUtils.isNotEmpty(markSearchDTO.getKeyword()) && Tools.checkUniteString(markSearchDTO.getKeyword())){
return ResponseResult.failure("不支持特殊符号字段查询");
}
return ResponseResult.success(markDataService.getContendSearchList(markSearchDTO));
}
......
src/main/java/com/zhiwei/brandkbs2/controller/app/AppToolsetController.java
......@@ -77,8 +77,7 @@ public class AppToolsetController {
@LogRecord(description = "工具库-本品事件影响力更新")
public ResponseResult primaryEventUpdate(@RequestParam MultipartFile file,
@RequestParam Boolean extraCompute){
toolsetService.primaryEventUpdate(file, extraCompute);
return ResponseResult.success();
return toolsetService.primaryEventUpdate(file, extraCompute);
}
@ApiOperation("工具库-事件影响力更新-竞品更新")
......
src/main/java/com/zhiwei/brandkbs2/easyexcel/listener/BytedanceEventListener.java
......@@ -4,8 +4,10 @@ import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener;
import com.zhiwei.brandkbs2.easyexcel.dto.UploadBytedanceEventDTO;
import com.zhiwei.brandkbs2.pojo.BytedanceCustomEventUpdateTaskData;
import org.apache.commons.collections4.CollectionUtils;
import java.util.List;
import java.util.Map;
import java.util.Objects;
/**
......@@ -16,20 +18,39 @@ import java.util.Objects;
*/
public class BytedanceEventListener extends AnalysisEventListener<UploadBytedanceEventDTO> {
private String taskId;
private List<BytedanceCustomEventUpdateTaskData> data;
private List<Boolean> correctHead;
public BytedanceEventListener(String taskId, List<BytedanceCustomEventUpdateTaskData> data){
this.taskId = taskId;
public BytedanceEventListener(List<BytedanceCustomEventUpdateTaskData> data, List<Boolean> correctHead){
this.data = data;
this.correctHead = correctHead;
}
/**
* 表头格式校验
* @param headMap
* @param context
*/
@Override
public void invokeHeadMap(Map<Integer, String> headMap, AnalysisContext context) {
if (!"序号".equals(headMap.get(0)) || !"平台类型".equals(headMap.get(1)) || !"发布平台".equals(headMap.get(2))
|| !"渠道".equals(headMap.get(3)) || !"时间".equals(headMap.get(4)) || !"标题".equals(headMap.get(5))
|| !"文本".equals(headMap.get(6)) || !"地址".equals(headMap.get(7)) || !"命中词".equals(headMap.get(8))
|| !"情感倾向".equals(headMap.get(9)) || !"事件标签".equals(headMap.get(10)) || !"在看".equals(headMap.get(11))
|| !"阅读".equals(headMap.get(12)) || !"点赞".equals(headMap.get(13)) || !"评论".equals(headMap.get(14))
|| !"转发".equals(headMap.get(15))) {
correctHead.add(false);
}
}
@Override
public void invoke(UploadBytedanceEventDTO dto, AnalysisContext context) {
if (CollectionUtils.isNotEmpty(correctHead) && !correctHead.get(0)){
return;
}
BytedanceCustomEventUpdateTaskData taskData = new BytedanceCustomEventUpdateTaskData();
taskData.setTaskId(taskId);
taskData.setPlatformType(dto.getPlatformType());
taskData.setPlatform(dto.getPlatform());
taskData.setChannel(dto.getChannel());
......
src/main/java/com/zhiwei/brandkbs2/es/EsQueryTools.java
......@@ -291,6 +291,27 @@ public class EsQueryTools {
}
/**
* 视频平台查询
* @param platforms
* @return
*/
public static BoolQueryBuilder assembleVideoPlatformQuery(List<String> platforms){
BoolQueryBuilder realSourceQuery = QueryBuilders.boolQuery();
// realSource
// 其他 特殊处理
if (platforms.contains("其他")) {
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
for (int i = 0; i < GenericAttribute.VIDEO_PLATFORM.size() - 1; i++) {
queryBuilder.mustNot(QueryBuilders.termQuery(GenericAttribute.ES_REAL_SOURCE, GenericAttribute.VIDEO_PLATFORM.get(i)));
}
realSourceQuery.should(queryBuilder);
platforms.remove("其他");
}
platforms.forEach(pt -> realSourceQuery.should(QueryBuilders.termQuery(GenericAttribute.ES_REAL_SOURCE, pt)));
return realSourceQuery;
}
/**
* 针对微博平台转发 查询语句
*
* @param isForward
......
src/main/java/com/zhiwei/brandkbs2/pojo/BytedanceCustomEventUpdateTaskData.java
......@@ -14,8 +14,6 @@ import lombok.Setter;
@Setter
@NoArgsConstructor
public class BytedanceCustomEventUpdateTaskData extends AbstractBaseMongo {
private String taskId;
private String platformType;
private String platform;
......
src/main/java/com/zhiwei/brandkbs2/pojo/MarkFlowEntity.java
......@@ -15,6 +15,7 @@ import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import java.io.Serializable;
import java.util.List;
......@@ -163,6 +164,10 @@ public class MarkFlowEntity implements Serializable {
videoMark.setMgroup(GlobalPojo.getYuQingProjectNameById(linkedGroupId));
this.filterInfo = videoMark.filterInfo();
}
// 视频时长处理
if (StringUtils.isNotBlank(videoMark.getLength())){
videoMark.setLength(Tools.formatVideoLength(videoMark.getLength()));
}
this.type = FlowEntityUtil.getVideoType(videoMark);
this.data = videoMark;
break;
......
src/main/java/com/zhiwei/brandkbs2/pojo/dto/MarkSearchDTO.java
......@@ -64,12 +64,24 @@ public class MarkSearchDTO {
private Long endTime;
/**
* 数据类型 全量数据 | 视频
*/
@ApiModelProperty("搜索类型:全量数据|视频")
private String searchType;
/**
* 平台
*/
@ApiModelProperty(value = "平台")
private List<String> platforms;
/**
* 视频平台
*/
@ApiModelProperty(value = "视频平台")
private List<String> videoPlatforms;
/**
* 媒体类型
*/
@ApiModelProperty(value = "重要发声方")
......@@ -119,7 +131,7 @@ public class MarkSearchDTO {
// query部分
/**
* 检索字段
* 关键字检索字段
*/
@ApiModelProperty(value = "检索字段")
private String searchField;
......@@ -155,10 +167,16 @@ public class MarkSearchDTO {
private boolean fold;
/**
*
* 屏蔽词
*/
@ApiModelProperty(value = "屏蔽词")
private String shieldWord;
private String filterWords;
/**
* 屏蔽词检索字段 全文|标题
*/
@ApiModelProperty(value = "屏蔽词检索字段：全文|标题")
private String filterType;
/**
* 方案id,仅非人工项目
......
src/main/java/com/zhiwei/brandkbs2/pojo/dto/SearchFilterDTO.java
package com.zhiwei.brandkbs2.pojo.dto;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.common.GlobalPojo;
import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import org.apache.commons.collections4.CollectionUtils;
import java.util.*;
import java.util.stream.Collectors;
@Data
@ApiModel("全网搜搜索传输类")
......@@ -42,10 +47,16 @@ public class SearchFilterDTO {
private Long crawlerTime;
/**
* 数据类型 全量数据 | 视频
*/
@ApiModelProperty("搜索类型:全量数据 | 视频")
private String searchType;
/**
* 关键字
*/
@ApiModelProperty(value = "关键字")
private String search;
private String keyword;
/**
* 平台
......@@ -54,6 +65,12 @@ public class SearchFilterDTO {
private List<String> platforms;
/**
* 视频平台
*/
@ApiModelProperty(value = "视频平台")
private List<String> videoPlatforms;
/**
* 排序字段
*/
@ApiModelProperty(value = "排序字段")
......@@ -146,8 +163,9 @@ public class SearchFilterDTO {
Map<String, Object> map = new HashMap<>();
map.put("timeType", "time");
map.put("pt", getPlatforms());
map.put("dataType", searchType);
map.put("matchType", matchType);
map.put("search", search);
map.put("search", keyword);
map.put("startTime", startTime);
map.put("endTime", endTime);
map.put("source", sourceKeyword);
......@@ -173,7 +191,10 @@ public class SearchFilterDTO {
}
public List<String> getPlatforms() {
return Objects.nonNull(platforms) ? this.platforms : Collections.emptyList();
if (Objects.equals("视频", this.searchType)){
return CollectionUtils.isNotEmpty(this.videoPlatforms) ? this.videoPlatforms : new ArrayList<>(GenericAttribute.VIDEO_PLATFORM);
}
return CollectionUtils.isNotEmpty(this.platforms) ? this.platforms : GlobalPojo.PLATFORMS.stream().map(MessagePlatform::getId).collect(Collectors.toList());
}
public String getMatchType() {
......
src/main/java/com/zhiwei/brandkbs2/service/EsSearchService.java
......@@ -75,4 +75,12 @@ public interface EsSearchService {
* @throws IOException
*/
JSONObject countNonManualPlatform(MarkSearchDTO dto) throws IOException;
/**
* 查舆情视频中心进量（声量）统计
* @param dto
* @return
* @throws IOException
*/
JSONObject countMarkVideoPlatform(MarkSearchDTO dto) throws IOException;
}
src/main/java/com/zhiwei/brandkbs2/service/ExtraService.java
......@@ -56,6 +56,12 @@ public interface ExtraService {
JSONObject overview();
/**
* 获取全网搜余额
* @return
*/
Integer getWholeSearchBalance();
/**
* @param keyword 关键词
* @param usedType 使用类别
* @param usedCount 使用量
......
src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java
......@@ -393,6 +393,13 @@ public interface MarkDataService {
JSONObject countOriginPlatform(MarkSearchDTO dto);
/**
* 查舆情视频中心-平台进量（声量）统计
* @param dto 标注数据搜索传输类
* @return
*/
JSONObject countMarkVideoPlatform(MarkSearchDTO dto);
/**
* 新建非人工项目方案
* @param dto
* @return
......
src/main/java/com/zhiwei/brandkbs2/service/ToolsetService.java
......@@ -76,7 +76,7 @@ public interface ToolsetService {
* @param file excel文件
* @param extraCompute 是否计算实时传播力与传播影响力
*/
void primaryEventUpdate(MultipartFile file, Boolean extraCompute);
ResponseResult primaryEventUpdate(MultipartFile file, Boolean extraCompute);
/**
* 竞品事件影响力计算
......
src/main/java/com/zhiwei/brandkbs2/service/impl/EsSearchServiceImpl.java
......@@ -77,6 +77,11 @@ public class EsSearchServiceImpl implements EsSearchService {
return platformCount(dto, this::createNonManualSearchHelper, esClientDao);
}
@Override
public JSONObject countMarkVideoPlatform(MarkSearchDTO dto) throws IOException {
return videoPlatformCount(dto, this::createSearchHelper, esClientDao);
}
private Pair<SearchHits[], Map<String, Long>> searchHitsAndCount(MarkSearchDTO dto, ArticleFunction.CreateSearchHelper createSearchHelper,
EsClientDao clientDao, boolean scroll) throws IOException {
EsClientDao.SearchHelper helper = createSearchHelper.create(dto);
......@@ -99,10 +104,18 @@ public class EsSearchServiceImpl implements EsSearchService {
BoolQueryBuilder postFilter = EsQueryTools.assembleCacheMapsQuery(projectId, contendId, dto.getPoliticsLevel(), dto.getField(), dto.getRegion(), dto.getMainBodyType(), dto.getChannelEmotion());
// time
postFilter.must(QueryBuilders.rangeQuery(dto.getTimeType()).gte(dto.getStartTime()).lt(dto.getEndTime()));
// searchType
if (Objects.equals("视频", dto.getSearchType())){
postFilter.must(QueryBuilders.existsQuery(GenericAttribute.ES_VIDEO_URLS));
}
// platform
if (CollectionUtils.isNotEmpty(dto.getPlatforms())) {
postFilter.must(EsQueryTools.assemblePlatformQuery(Tools.getPlatformByIds(dto.getPlatforms())));
}
// videoPlatforms
if (CollectionUtils.isNotEmpty(dto.getVideoPlatforms())){
postFilter.must(EsQueryTools.assembleVideoPlatformQuery(dto.getVideoPlatforms()));
}
// tags
if (CollectionUtils.isNotEmpty(dto.getTags())) {
postFilter.must(EsQueryTools.assembleTagQuery(dto.getTags()));
......@@ -119,9 +132,10 @@ public class EsSearchServiceImpl implements EsSearchService {
// Query
postFilter.must(EsQueryTools.assembleNormalKeywordQuery(dto.getKeyword(), fieldSearch));
}
// shieldWord
if (StringUtils.isNotEmpty(dto.getShieldWord())) {
postFilter.mustNot(EsQueryTools.assembleNormalKeywordQuery(dto.getShieldWord(), fieldSearch));
// filterWords
String[] filterFieldSearch = "标题".equals(dto.getFilterType()) ? new String[]{GenericAttribute.ES_IND_TITLE} : new String[]{GenericAttribute.ES_IND_FULL_TEXT};
if (StringUtils.isNotEmpty(dto.getFilterWords())) {
postFilter.mustNot(EsQueryTools.assembleNormalKeywordQuery(dto.getFilterWords(), filterFieldSearch));
}
// sourceKeyword
if (StringUtils.isNotEmpty(dto.getSourceKeyword())) {
......@@ -383,4 +397,31 @@ public class EsSearchServiceImpl implements EsSearchService {
res.put("total", total);
return res;
}
/**
* 视频realSource进量统计
* @param dto
* @param createSearchHelper
* @param esClientDao
* @return
* @throws IOException
*/
private JSONObject videoPlatformCount(MarkSearchDTO dto, ArticleFunction.CreateSearchHelper createSearchHelper, EsClientDao esClientDao) throws IOException {
List<JSONObject> list = new ArrayList<>();
for (String realSource : GenericAttribute.VIDEO_PLATFORM) {
List<String> oneRealSource = new ArrayList<>(1);
oneRealSource.add(realSource);
dto.setVideoPlatforms(oneRealSource);
EsClientDao.SearchHelper helper = createSearchHelper.create(dto);
JSONObject jsonObject = new JSONObject();
jsonObject.put("count", esClientDao.count(helper.getPostFilter()));
jsonObject.put("name", realSource);
list.add(jsonObject);
}
Long total = list.stream().map(jsonObject -> jsonObject.getLongValue("count")).reduce(Long::sum).orElse(0L);
JSONObject res = new JSONObject();
res.put("count", list);
res.put("total", total);
return res;
}
}
src/main/java/com/zhiwei/brandkbs2/service/impl/ExtraServiceImpl.java
......@@ -109,6 +109,11 @@ public class ExtraServiceImpl implements ExtraService {
}
@Override
public Integer getWholeSearchBalance() {
return projectDao.findOneById(UserThreadLocal.getProjectId()).getWholeSearchBalance();
}
@Override
public void decreaseRecord(String keyword, WholeSearchRecord.UsedType usedType, Integer usedCount) {
if (null == usedCount) {
return;
......
src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java
......@@ -1438,6 +1438,12 @@ public class MarkDataServiceImpl implements MarkDataService {
result.put("followersNum", list.get("followers_num"));
result.put("realSource", list.get("real_source"));
result.put("ipLocation", list.get("ip_location"));
result.put("picUrls", list.get("pic_urls"));
result.put("videoUrls", list.getJSONArray("video_urls"));
// 视频时长处理
if (StringUtils.isNotBlank(list.getString("length"))){
result.put("length", Tools.formatVideoLength(list.getString("length")));
}
resultList.add(result);
}
// 仅第一页返回平台进量（声量）统计
......@@ -1643,7 +1649,7 @@ public class MarkDataServiceImpl implements MarkDataService {
*/
private JSONObject externalTransform(SearchFilterDTO dto) {
JSONObject params = new JSONObject();
params.put("crawlerWord", dto.getSearch());
params.put("crawlerWord", dto.getKeyword());
params.put("startTime", dto.getStartTime());
params.put("endTime", dto.getEndTime());
String platforms;
......@@ -1662,6 +1668,12 @@ public class MarkDataServiceImpl implements MarkDataService {
// matchFields
if (dto.getMatchType().equals("标题")) {
params.put("matchFields", Collections.singletonList("title"));
}else if (dto.getMatchType().equals("全文")){
params.put("matchFields", Collections.singletonList("content"));
}
// sourceKeyword
if (StringUtils.isNotBlank(dto.getSourceKeyword())){
params.put("userList", Arrays.asList(dto.getSourceKeyword().trim().split("\\|")));
}
// // 历史部分
// if (Tools.isEmpty(dto.getPlatforms())) {
......@@ -1883,28 +1895,50 @@ public class MarkDataServiceImpl implements MarkDataService {
}
@Override
public JSONObject countMarkVideoPlatform(MarkSearchDTO dto) {
defaultMarkSearch(dto);
try {
return esSearchService.countMarkVideoPlatform(dto);
} catch (Exception e) {
ExceptionCast.cast(CommonCodeEnum.FAIL, "es查询异常", e);
}
return null;
}
@Override
public ResponseResult addNonManualProjectPlan(NonManualProjectPlanDTO dto) {
if (nonManualProjectPlanDao.count(new Query(Criteria.where("projectId").is(UserThreadLocal.getProjectId()))) >= 5) {
String projectId = UserThreadLocal.getProjectId();
if (nonManualProjectPlanDao.count(new Query(Criteria.where("projectId").is(projectId))) >= 5) {
return ResponseResult.failure("超过方案数量上限5");
}
NonManualProjectPlan plan = new NonManualProjectPlan(dto);
Project project = projectService.getProjectById(projectId);
if (project.getExternalDataSource()) {
// 同时添加快搜订阅任务
String iStarShineSubjectTaskId = addIStarShineSubjectTask(dto);
if (Objects.isNull(iStarShineSubjectTaskId)){
if (Objects.isNull(iStarShineSubjectTaskId)) {
return ResponseResult.failure("添加方案失败");
}
NonManualProjectPlan plan = new NonManualProjectPlan(dto);
plan.setIStarShineSubjectTaskId(iStarShineSubjectTaskId);
}
nonManualProjectPlanDao.insertOne(plan);
return ResponseResult.success();
}
@Override
public ResponseResult deleteNonManualProjectPlan(String id) {
Project project = projectService.getProjectById(UserThreadLocal.getProjectId());
// 商业数据源同时删除快搜订阅任务
if (project.getExternalDataSource()){
ResponseResult responseResult = deleteIStarShineSubjectTask(id);
if (responseResult.isStatus()){
nonManualProjectPlanDao.deleteOneById(id);
}
return responseResult;
}else {
nonManualProjectPlanDao.deleteOneById(id);
return ResponseResult.success();
}
}
@Override
......@@ -1929,12 +1963,16 @@ public class MarkDataServiceImpl implements MarkDataService {
NonManualProjectPlan plan = new NonManualProjectPlan(dto);
plan.setCTime(existPlan.getCTime());
plan.setIStarShineSubjectTaskId(existPlan.getIStarShineSubjectTaskId());
Project project = projectService.getProjectById(UserThreadLocal.getProjectId());
// 商业数据源同时修改快搜订阅任务
if (project.getExternalDataSource()){
ResponseResult responseResult = updateIStarShineSubjectTask(dto);
if (!responseResult.isStatus()){
return responseResult;
}
}
nonManualProjectPlanDao.updateOne(plan);
return responseResult;
return ResponseResult.success();
}
/**
......@@ -2001,7 +2039,7 @@ public class MarkDataServiceImpl implements MarkDataService {
HttpEntity<String> request = new HttpEntity<>(param.toJSONString(), httpHeaders);
JSONObject body = restTemplate.postForEntity(updateIStarShineSubjectTaskUrl, request, JSONObject.class).getBody();
if (Objects.isNull(body) || body.isEmpty() || Boolean.FALSE.equals(body.getBooleanValue("status"))){
log.info(JSONObject.toJSONString(plan) + "-修改快搜订阅任务失败-" + body);
log.error(JSONObject.toJSONString(plan) + "-修改快搜订阅任务失败-" + body);
return ResponseResult.failure("修改失败");
}
return ResponseResult.success();
......@@ -2009,14 +2047,35 @@ public class MarkDataServiceImpl implements MarkDataService {
@Override
public Integer getPlanDataPrediction(String keyword, String filterWord) {
try {
Project project = projectService.getProjectById(UserThreadLocal.getProjectId());
if (project.getExternalDataSource()) {
JSONObject data = getSearchWholeOpinionJson(createSearchFilterDTO(keyword, filterWord));
JSONObject task = data.getJSONObject("task");
return task.getInteger("searchCount");
} else {
long endTime = System.currentTimeMillis();
long startTime = endTime - Constant.ONE_DAY;
BoolQueryBuilder postFilter = QueryBuilders.boolQuery();
// time
postFilter.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
// keyword
postFilter.must(EsQueryTools.assembleNormalKeywordQuery(keyword, new String[]{GenericAttribute.ES_IND_FULL_TEXT}));
// filterWord
if (StringUtils.isNotBlank(filterWord)) {
postFilter.mustNot(EsQueryTools.assembleNormalKeywordQuery(filterWord, new String[]{GenericAttribute.ES_IND_FULL_TEXT}));
}
return hotEsDao.count(postFilter).intValue();
}
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "方案预估失败", e);
}
return null;
}
private SearchFilterDTO createSearchFilterDTO(String keyword, String filterWord){
SearchFilterDTO dto = new SearchFilterDTO();
dto.setSearch(keyword);
dto.setKeyword(keyword);
dto.setFilterWords(filterWord);
// 预估一天
dto.setEndTime(System.currentTimeMillis());
......@@ -2379,15 +2438,15 @@ public class MarkDataServiceImpl implements MarkDataService {
List<JSONObject> res = new ArrayList<>();
String projectId = UserThreadLocal.getProjectId();
List<NonManualProjectPlan> planList = nonManualProjectPlanDao.findList(new Query(Criteria.where("projectId").is(projectId)));
long startTime = Constant.DF_yyyyMMdd.parse(Constant.DF_yyyyMMdd.format(new Date())).getTime();
long endTime = System.currentTimeMillis();
// long startTime = Constant.DF_yyyyMMdd.parse(Constant.DF_yyyyMMdd.format(new Date())).getTime();
// long endTime = System.currentTimeMillis();
for (NonManualProjectPlan plan : planList) {
JSONObject jsonObject = new JSONObject();
jsonObject.put("id", plan.getId());
jsonObject.put("name", plan.getName());
BoolQueryBuilder query = EsQueryTools.assembleCacheMapsPlanQuery(projectId, plan.getId());
query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
jsonObject.put("count", esClientDao.count(query));
// BoolQueryBuilder query = EsQueryTools.assembleCacheMapsPlanQuery(projectId, plan.getId());
// query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
// jsonObject.put("count", esClientDao.count(query));
res.add(jsonObject);
}
return res;
......@@ -2914,10 +2973,11 @@ public class MarkDataServiceImpl implements MarkDataService {
// postFilter
BoolQueryBuilder postFilter = yuqingAnalyzeQuery(dto);
searchHelper.setPostFilter(postFilter);
searchHelper.setSize(10000);
searchHelper.setSize(1000);
List<String> textList = new ArrayList<>();
List<SearchResponse> searchResponses = Collections.singletonList(esClientDao.searchResponse(searchHelper));
for (SearchResponse searchResponse : searchResponses) {
for (int i = 1; i <= 10; i++) {
searchHelper.setFrom((i - 1) * 1000);
SearchResponse searchResponse = esClientDao.searchResponse(searchHelper);
for (SearchHit searchHit : searchResponse.getHits().getHits()) {
Map<String, Object> source = searchHit.getSourceAsMap();
BaseMap baseMap = Tools.getBaseFromEsMap(source);
......@@ -3224,11 +3284,6 @@ public class MarkDataServiceImpl implements MarkDataService {
*/
private Map<String, List<LineVO>> getPlatformSpreadTendency(MarkSearchDTO dto) throws IOException{
Map<String, List<LineVO>> res = new HashMap<>(4);
// 平台
// List<String> platforms = new ArrayList<>();
// for (String platformNames : Arrays.asList("微博", "微信", "抖音", "小红书")) {
// platforms.add(GlobalPojo.getPlatformIdByName(platformNames));
// }
if (CollectionUtils.isEmpty(dto.getPlatforms())){
dto.setPlatforms(Stream.of("微博", "微信", "抖音", "小红书").map(GlobalPojo::getPlatformIdByName).collect(Collectors.toList()));
}
......@@ -3373,12 +3428,20 @@ public class MarkDataServiceImpl implements MarkDataService {
if (Objects.nonNull(dto.getPlanId())){
postFilter = EsQueryTools.assembleCacheMapsPlanQuery(projectId, dto.getPlanId(), dto.getPoliticsLevel());
}
// searchType
if (Objects.equals("视频", dto.getSearchType())){
postFilter.must(QueryBuilders.existsQuery(GenericAttribute.ES_VIDEO_URLS));
}
// time
postFilter.must(QueryBuilders.rangeQuery("time").gte(dto.getStartTime()).lt(dto.getEndTime()));
// platform
if (CollectionUtils.isNotEmpty(dto.getPlatforms())) {
postFilter.must(EsQueryTools.assemblePlatformQuery(Tools.getPlatformByIds(dto.getPlatforms())));
}
// videoPlatforms
if (CollectionUtils.isNotEmpty(dto.getVideoPlatforms())){
postFilter.must(EsQueryTools.assembleVideoPlatformQuery(dto.getVideoPlatforms()));
}
// tags
if (CollectionUtils.isNotEmpty(dto.getTags())) {
postFilter.must(EsQueryTools.assembleTagQuery(dto.getTags()));
......@@ -3394,14 +3457,15 @@ public class MarkDataServiceImpl implements MarkDataService {
// Query
postFilter.must(EsQueryTools.assembleNormalKeywordQuery(dto.getKeyword(), fieldSearch));
}
// filterWords
String[] filterFieldSearch = "标题".equals(dto.getFilterType()) ? new String[]{GenericAttribute.ES_IND_TITLE} : new String[]{GenericAttribute.ES_IND_FULL_TEXT};
if (StringUtils.isNotEmpty(dto.getFilterWords())) {
postFilter.mustNot(EsQueryTools.assembleNormalKeywordQuery(dto.getFilterWords(), filterFieldSearch));
}
// emotion
if (StringUtils.isNotEmpty(emotion) && !EmotionEnum.ALL.getName().equals(emotion)) {
postFilter.must(QueryBuilders.termQuery("brandkbs_mark_cache_maps.name.keyword", emotion));
}
// shieldWord
if (StringUtils.isNotEmpty(dto.getShieldWord())) {
postFilter.mustNot(EsQueryTools.assembleNormalKeywordQuery(dto.getShieldWord(), fieldSearch));
}
// sourceKeyword
if (StringUtils.isNotEmpty(dto.getSourceKeyword())) {
postFilter.must(EsQueryTools.assembleSourceKeywordQuery(dto.getSourceKeyword()));
......
src/main/java/com/zhiwei/brandkbs2/service/impl/ProjectServiceImpl.java
......@@ -252,6 +252,13 @@ public class ProjectServiceImpl implements ProjectService {
// 删除非人工项目维护记录和方案
if (!project.isManual()){
Query query = new Query(Criteria.where("projectId").is(pid));
List<NonManualProjectPlan> planList = nonManualProjectPlanDao.findList(query);
// 删除非人工项目数据订阅任务
if (project.getExternalDataSource()){
for (NonManualProjectPlan plan : planList) {
markDataService.deleteIStarShineSubjectTask(plan.getId());
}
}
nonManualProjectPlanDao.deleteOneByQuery(query);
nonManualProjectDataUpdateRecordDao.deleteOneByQuery(query);
}
......@@ -531,6 +538,7 @@ public class ProjectServiceImpl implements ProjectService {
project.setShow(existsProject.isShow());
project.setDataBalance(existsProject.getDataBalance());
project.setStart(existsProject.isStart());
project.setExternalDataSource(existsProject.getExternalDataSource());
projectDao.updateOne(project);
return ResponseResult.success();
}
......
src/main/java/com/zhiwei/brandkbs2/service/impl/ToolsetServiceImpl.java
......@@ -368,19 +368,23 @@ public class ToolsetServiceImpl implements ToolsetService {
}
@Override
public void primaryEventUpdate(MultipartFile file, Boolean extraCompute) {
BytedanceCustomEventUpdateTask task = BytedanceCustomEventUpdateTask.createTaskPrimary(extraCompute);
bytedanceCustomEventUpdateTaskDao.insertOne(task);
public ResponseResult primaryEventUpdate(MultipartFile file, Boolean extraCompute) {
List<BytedanceCustomEventUpdateTaskData> data = new ArrayList<>();
List<Boolean> correctHead = new ArrayList<>(1);
// excel信息提取
ReadExcelDTO<UploadBytedanceEventDTO> readExcel = new ReadExcelDTO<>();
readExcel.setClazz(UploadBytedanceEventDTO.class);
readExcel.setAnalysisEventListener(new BytedanceEventListener(task.getId(), data));
readExcel.setAnalysisEventListener(new BytedanceEventListener(data, correctHead));
EasyExcelUtil.read(file, readExcel);
if (CollectionUtils.isNotEmpty(correctHead) && !correctHead.get(0)){
return ResponseResult.failure("表格模板错误,计算失败");
}
BytedanceCustomEventUpdateTask task = BytedanceCustomEventUpdateTask.createTaskPrimary(extraCompute);
bytedanceCustomEventUpdateTaskDao.insertOne(task);
// 更新处理进度
resetProcessSchedule(task.getId(), data.size());
ApplicationProjectListener.getThreadPool().execute(() -> processEventUpdate(data, task, extraCompute));
return ResponseResult.success();
}
@Override
......@@ -399,7 +403,7 @@ public class ToolsetServiceImpl implements ToolsetService {
// excel信息提取
ReadExcelDTO<UploadBytedanceEventDTO> readExcel = new ReadExcelDTO<>();
readExcel.setClazz(UploadBytedanceEventDTO.class);
readExcel.setAnalysisEventListener(new BytedanceEventListener(taskId, data));
readExcel.setAnalysisEventListener(new BytedanceEventListener(data, null));
EasyExcelUtil.read(file, readExcel);
// 将旧数据与新数据合并
List<BytedanceCustomEventUpdateTaskData> taskData = bytedanceCustomEventUpdateTaskDataDao.findList(new Query(Criteria.where("taskId").is(taskId)));
......@@ -688,6 +692,7 @@ public class ToolsetServiceImpl implements ToolsetService {
private void updateStatus(String taskId, BytedanceCustomEventUpdateTask.TaskStatus taskStatus){
BytedanceCustomEventUpdateTask task = bytedanceCustomEventUpdateTaskDao.findOneById(taskId);
task.setRead(true);
updateStatus(task, taskStatus, null);
}
......@@ -724,6 +729,10 @@ public class ToolsetServiceImpl implements ToolsetService {
? data.getPlatformType() : "网媒", data.getChannel()));
// 1.分别将等级为S,A,B,C的渠道各自加和
if (Objects.nonNull(channelInfluence)){
// 2024/5/21修改：无论表中是否匹配，当类型为D，取值=1
if (Objects.equals("D", channelInfluence.getLevel())){
channelInfluence.setInfluence(1d);
}
levelSumH.compute(channelInfluence.getLevel(), (k, v) -> Objects.isNull(v)
? channelInfluence.getInfluence()
: BigDecimal.valueOf(v).add(BigDecimal.valueOf(channelInfluence.getInfluence())).doubleValue());
......
src/main/java/com/zhiwei/brandkbs2/util/Tools.java
......@@ -37,6 +37,8 @@ import org.springframework.web.multipart.MultipartFile;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.*;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.net.URLEncoder;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
......@@ -1356,6 +1358,10 @@ public class Tools {
return Arrays.stream(str.split("[\\u007c]+")).filter(StringUtils::isNotBlank).collect(Collectors.joining("|"));
}
public static boolean checkUniteString(String str){
return str.split("[\\u4e28]+").length > 1 || str.split("[\\uff5c]+").length > 1 || str.split("[\\u007c]+").length > 1;
}
/**
* 过滤 “分享一篇文章”，”暂无标题“，”Notitle“ 的无用标题
* @param title
......@@ -1373,4 +1379,19 @@ public class Tools {
public static Boolean filterUselessSource(String source){
return FILTER_SOURCE.contains(source);
}
/**
* 格式化视频时长秒
* @param videoLength
* @return
*/
public static String formatVideoLength(String videoLength){
if (videoLength.contains(":")){
return videoLength;
}
double temp = Double.parseDouble(videoLength);
long mm = BigDecimal.valueOf(temp).divide(BigDecimal.valueOf(60), 0, RoundingMode.HALF_UP).longValue();
long ss = BigDecimal.valueOf(temp).divideAndRemainder(BigDecimal.valueOf(60))[1].setScale(0, RoundingMode.HALF_UP).longValue();
return ((mm < 10 ? ("0" + mm) : mm) + ":" + (ss < 10 ? ("0" + ss) : ss));
}
}
\ No newline at end of file
src/main/resources/application-dev.properties
......@@ -102,7 +102,7 @@ ef.checkCaptcha.url=https://ef.zhiweidata.com/captchaApi/checkeCaptcha.do.do?id=
#\u5168\u7F51\u641C\u5916\u90E8\u63A5\u53E3
istarshine.addIStarShineKSData.url=https://istarshine-service.zhiweidata.com/istarshine/ks/addIStarShineKSData
istarshine.getIStarShineKSInfoData.url=https://istarshine-service.zhiweidata.com/istarshine/ks/getIStarShineKSInfoData?taskId={1}&appId=6183571e0d710000f6003a12
whole.search.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/search/yuqing
whole.search.url=https://yuqing.test.zhiweidata.com/qbjcbackPhoenix/interface/middleware/search/yuqing
whole.searchCriteria.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/search/pt
whole.extraParam.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/search/extra/param
brief.report.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/brief-report-web
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment