Skip to content

B
brandkbs2
Commit 1cf8e1ca by shenjunjie
Options

原始数据库上线

parent eae36092
Showing with 1708 additions and 820 deletions
pom.xml
......@@ -24,7 +24,7 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<java.version>1.8</java.version>
<springboot.version>2.2.5.RELEASE</springboot.version>
<qbjc.bean.version>1.1.6.4-SNAPSHOT</qbjc.bean.version>
<qbjc.bean.version>1.1.7.1-SNAPSHOT</qbjc.bean.version>
<springfox.version>2.7.0</springfox.version>
<fastjson.version>1.2.47</fastjson.version>
<easyexcel.version>3.0.5</easyexcel.version>
......
src/main/java/com/zhiwei/brandkbs2/common/CommonConfig.java
package com.zhiwei.brandkbs2.common;
import com.zhiwei.middleware.cleaner.filter.UnifiedFilterClient;
import com.zhiwei.middleware.event.core.EventClient;
import com.zhiwei.middleware.event.core.EventClientFactory;
import com.zhiwei.middleware.event.core.EventMonitorClient;
......@@ -36,6 +37,9 @@ public class CommonConfig {
@Value("${event.provider.group}")
private String eventProviderGroup;
@Value("${filter.provider.group}")
private String filterGroup;
@Bean
public RestTemplate restTemplate() {
final OkHttpClient client = new OkHttpClient().newBuilder()
......@@ -50,7 +54,7 @@ public class CommonConfig {
*
* @return 标注中间件客户端
*/
@Bean
@Bean(name = "markerClient")
public MarkerClient getMarkerClient() {
return MarkerClient.getService(clientRegistry, providerGroup, appName);
}
......@@ -61,25 +65,39 @@ public class CommonConfig {
* @return 事件中间件标签客户端
*/
@Bean
public EventTagClient getEventTagClient(){
public EventTagClient getEventTagClient() {
return EventClientFactory.markEventTagClient(eventClientRegistry, eventProviderGroup, appName);
}
/**
* 获取事件中间事件客户端
*
* @return 事件中间事件客户端
*/
@Bean
public EventClient getEventClient(){
public EventClient getEventClient() {
return EventClientFactory.eventClient(eventClientRegistry, eventProviderGroup, appName);
}
/**
* 获取事件中间件监测客户端
*
* @return 事件事件件监测客户端
*/
@Bean
public EventMonitorClient getEventMonitorClient(){
public EventMonitorClient getEventMonitorClient() {
return EventClientFactory.eventMonitorClient(eventClientRegistry, eventProviderGroup, appName);
}
/**
* 获取去重中间件客户端
*
* @return 去重中间件客户端
*/
@Bean(name = "filterClient")
public UnifiedFilterClient getUnifiedFilterClient() {
return UnifiedFilterClient.getClient(appName, clientRegistry, filterGroup);
}
}
src/main/java/com/zhiwei/brandkbs2/common/GlobalPojo.java
package com.zhiwei.brandkbs2.common;
import com.alibaba.fastjson.JSONObject;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.zhiwei.brandkbs2.pojo.ChannelTag;
import com.zhiwei.brandkbs2.pojo.Event;
import com.zhiwei.brandkbs2.pojo.Project;
import com.zhiwei.brandkbs2.service.EventService;
import com.zhiwei.brandkbs2.service.SystemInfoService;
import com.zhiwei.middleware.mark.vo.MarkerTag;
import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform;
import com.zhiwei.qbjc.bean.pojo.common.Tag;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
......@@ -22,8 +17,6 @@ import javax.annotation.Resource;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.stream.Collectors;
/**
......@@ -47,7 +40,7 @@ public class GlobalPojo {
/**
* 监测系统平台
**/
public static Map<String, List<Tag>> TAGS;
public static Map<String,List<Tag>> TAGS;
/**
* 品见情感倾向标签
......@@ -95,7 +88,7 @@ public class GlobalPojo {
private void updatePojo(String logMsg) {
try {
PLATFORMS = systemInfoService.getPlatforms();
TAGS = systemInfoService.getTags().stream().collect(Collectors.groupingBy(Tag::getGroupName));
TAGS = systemInfoService.getTags().stream().filter(tag -> null != tag.getProjectId()).collect(Collectors.groupingBy(Tag::getProjectId));
LINKED_GROUP_ID_TAGS = systemInfoService.findEmotionTagByLinkedGroupId();
CHANNEL_TAGS = systemInfoService.getChannelTags().stream().collect(Collectors.toMap(ChannelTag::getChannel, ChannelTag::getTag));
MEDIA_TYPE = systemInfoService.getMediaTypes();
......@@ -142,4 +135,13 @@ public class GlobalPojo {
return projectMap.get(platform + source);
}
public static String getYuQingProjectNameById(String linkedGroupId) {
for (JSONObject yuQingProject : YU_QING_PROJECTS) {
if (linkedGroupId.equals(yuQingProject.getString("id"))) {
return yuQingProject.getString("groupName");
}
}
return null;
}
}
src/main/java/com/zhiwei/brandkbs2/config/Constant.java
......@@ -28,6 +28,7 @@ public class Constant {
public static final FastDateFormat DAY_FORMAT = FastDateFormat.getInstance(DAY_PATTERN);
public static final FastDateFormat SPEC_DAY_FORMAT = FastDateFormat.getInstance("yyyy/MM/dd");
public static final FastDateFormat ONLY_MONTH_FORMAT = FastDateFormat.getInstance("M月");
public static final FastDateFormat DF_yyyyMMdd = FastDateFormat.getInstance("yyyyMMdd");
/**
* 自定义fid分隔符号
......
src/main/java/com/zhiwei/brandkbs2/config/EsProperties.java
......@@ -32,5 +32,19 @@ public class EsProperties {
private String username;
private String password;
/**
* clusterNameHotES 集群名-hotes
*/
private String clusterNameHotES;
/**
* 集群节点s
*/
private String clusterNodesHotES;
private String usernameHotES;
private String passwordHotES;
}
src/main/java/com/zhiwei/brandkbs2/controller/CommonController.java
......@@ -6,7 +6,6 @@ import com.zhiwei.brandkbs2.enmus.RoleEnum;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.service.CommonService;
import com.zhiwei.brandkbs2.service.ProjectService;
import com.zhiwei.middleware.mark.pojo.enums.TagField;
import com.zhiwei.middleware.mark.vo.MarkerTag;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
......@@ -17,9 +16,7 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
/**
* @ClassName: CommonController
......@@ -40,16 +37,11 @@ public class CommonController extends BaseController {
ProjectService projectService;
@ApiOperation("获取情感倾向标签信息")
@ApiImplicitParam(name = "contendId", value = "品牌id", required = true, paramType = "path", dataType = "string")
@GetMapping("/get/tag/emotion/{contendId}")
public ResponseResult getTagsWithEmotion(@PathVariable(value = "contendId") String contendId) {
List<String> res = new ArrayList<>();
String linkedGroupId = projectService.getProjectByContendId(UserThreadLocal.getProjectId(), contendId).getBrandLinkedGroupId();
List<MarkerTag> tags = commonService.getQbjcTags(linkedGroupId, TagField.GROUP_NAME.is("情感倾向"));
if (null != tags) {
res = tags.stream().map(MarkerTag::getName).collect(Collectors.toList());
}
return ResponseResult.success(res);
@ApiImplicitParam(name = "linkedGroupId", value = "关联项目id", required = true, paramType = "path", dataType = "string")
@GetMapping("/get/tag/emotion/{linkedGroupId}")
public ResponseResult getTagsWithEmotion(@PathVariable(value = "linkedGroupId") String linkedGroupId) {
List<MarkerTag> tags = commonService.getEmotionTagsWithSort(UserThreadLocal.getProjectId(), linkedGroupId);
return ResponseResult.success(tags);
}
@ApiOperation("获取平台类型")
......
src/main/java/com/zhiwei/brandkbs2/controller/admin/ProjectController.java
......@@ -9,7 +9,6 @@ import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
import com.zhiwei.brandkbs2.easyexcel.dto.UploadKeywordDTO;
import com.zhiwei.brandkbs2.enmus.RoleEnum;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.Behavior;
import com.zhiwei.brandkbs2.pojo.vo.ProjectVO;
import com.zhiwei.brandkbs2.service.BehaviorService;
import com.zhiwei.brandkbs2.service.CommonService;
......@@ -62,8 +61,6 @@ public class ProjectController extends BaseController {
@Resource(name = "eventServiceImpl")
private EventService eventService;
private static final Behavior.Operation OPERATION = new Behavior.Operation("项目管理", true);
@Value("${brandkbs.img.url}")
private String brandkbsImgPath;
......@@ -79,9 +76,7 @@ public class ProjectController extends BaseController {
@ApiOperation("添加项目配置")
@PostMapping("/add")
public ResponseResult addProject(@RequestBody ProjectVO projectVO) {
ProjectService.addProject(projectVO);
behaviorService.pushBehaviorOld(OPERATION, "添加项目配置:" + projectVO.getId(), request);
return ResponseResult.success();
return ProjectService.addProject(projectVO);
}
@ApiOperation("查询所有项目")
......@@ -96,9 +91,7 @@ public class ProjectController extends BaseController {
@ApiOperation("修改项目")
@PutMapping("/update")
public ResponseResult updateProject(@RequestBody ProjectVO projectVO) {
ProjectService.updateProject(projectVO);
behaviorService.pushBehaviorOld(OPERATION, "修改项目:" + projectVO.getId(), request);
return ResponseResult.success();
return ProjectService.updateProject(projectVO);
}
@ApiOperation("删除项目")
......@@ -106,7 +99,6 @@ public class ProjectController extends BaseController {
@DeleteMapping("/delete/{pid}")
public ResponseResult deleteProject(@PathVariable("pid") String pid) {
ProjectService.deleteProject(pid, UserThreadLocal.getProjectId());
behaviorService.pushBehaviorOld(OPERATION, "删除项目:" + pid, request);
return ResponseResult.success();
}
......@@ -116,7 +108,6 @@ public class ProjectController extends BaseController {
public ResponseResult switchProjectShow(@PathVariable String pid) {
boolean result = ProjectService.switchProjectShow(pid);
if (result) {
behaviorService.pushBehaviorOld(OPERATION, "切换项目展示状态:" + pid, request);
return ResponseResult.success();
} else {
return ResponseResult.failure("启动状态下的项目，不允许调整展示状态");
......@@ -128,7 +119,6 @@ public class ProjectController extends BaseController {
@PutMapping("/switch/projectStart/{pid}")
public ResponseResult switchProjectStart(@PathVariable String pid) {
ProjectService.switchProjectStart(pid);
behaviorService.pushBehaviorOld(OPERATION, "切换项目状态:" + pid, request);
return ResponseResult.success();
}
......@@ -142,9 +132,7 @@ public class ProjectController extends BaseController {
@ApiOperation("图片上传")
@PostMapping(value = "/upload/img", headers = "content-type=multipart/form-data")
public ResponseResult uploadImg(@RequestParam("file") MultipartFile file) {
ResponseResult responseResult = Tools.uploadFile(file, brandkbsImgPath, brandkbsImageUrl);
behaviorService.pushBehaviorOld(OPERATION, "图片上传", request);
return responseResult;
return Tools.uploadFile(file, brandkbsImgPath, brandkbsImageUrl);
}
@ApiOperation("关键词excel解析")
......
src/main/java/com/zhiwei/brandkbs2/controller/app/AppArticleController.java
......@@ -3,6 +3,7 @@ package com.zhiwei.brandkbs2.controller.app;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.aop.LogRecord;
import com.zhiwei.brandkbs2.auth.Auth;
import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.controller.BaseController;
import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
import com.zhiwei.brandkbs2.enmus.RoleEnum;
......@@ -13,7 +14,10 @@ import com.zhiwei.brandkbs2.pojo.dto.ReportDTO;
import com.zhiwei.brandkbs2.pojo.dto.ReportSearchDTO;
import com.zhiwei.brandkbs2.service.CommonService;
import com.zhiwei.brandkbs2.service.MarkDataService;
import com.zhiwei.brandkbs2.service.ProjectService;
import com.zhiwei.brandkbs2.service.ReportService;
import com.zhiwei.brandkbs2.util.Tools;
import com.zhiwei.middleware.mark.vo.MarkerTag;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiImplicitParams;
......@@ -23,6 +27,7 @@ import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource;
import java.util.List;
import java.util.stream.Collectors;
/**
* @author sjj
......@@ -45,6 +50,67 @@ public class AppArticleController extends BaseController {
@Resource(name = "commonServiceImpl")
CommonService commonService;
@Resource(name = "projectServiceImpl")
ProjectService projectService;
@ApiOperation("原始数据列表")
@PostMapping("/origin/list")
public ResponseResult getOriginList(@RequestBody MarkSearchDTO markSearchDTO) {
return ResponseResult.success(markDataService.getOriginList(markSearchDTO));
}
@ApiOperation("原始数据列表-生成聚合")
@PostMapping("/origin/aggree")
public ResponseResult generateOriginAggreeList(@RequestBody JSONObject info) {
Long startTime = info.getLong("startTime");
Long endTime = info.getLong("endTime");
return ResponseResult.success(markDataService.generateOriginAggreeList(startTime, endTime));
}
@ApiOperation("原始数据列表-获取聚合结果")
@PostMapping("/origin/aggree/list")
public ResponseResult getOriginAggreeList(@RequestBody MarkSearchDTO markSearchDTO) {
return ResponseResult.success(markDataService.getOriginAggreeList(markSearchDTO));
}
@ApiOperation("原始数据导出")
@PostMapping(value = "/origin/list/export")
public ResponseResult exportOriginList(@RequestBody MarkSearchDTO markSearchDTO) {
Pair<String, List<ExportAppYuqingDTO>> stringListPair = markDataService.downloadYuqingMarkList(markSearchDTO);
EasyExcelUtil.download(stringListPair.getLeft() + "_原始数据列表数据", "sheet1", ExportAppYuqingDTO.class, stringListPair.getRight(), response);
return ResponseResult.success();
}
@ApiOperation("单条标注")
@PostMapping("/origin/tag")
public ResponseResult addOriginTag(@RequestBody JSONObject json) {
String zip = json.getString("zip");
String linkedGroupId = json.getString("linkedGroupId");
StringBuilder mtag = new StringBuilder(json.getString("mtag"));
// 带上项目配置的必要标注标签
if (checkMTagIllegal(mtag)){
return ResponseResult.failure("标注标签出现同组标签，请勿连点标注");
}
markDataService.addMtag(zip, linkedGroupId, mtag.toString());
return ResponseResult.success();
}
@ApiOperation("批量标注")
@PostMapping("/origin/tag/batch")
public ResponseResult addOriginTagBatch(@RequestBody JSONObject json) {
List<String> ids = (List<String>) json.get("ids");
String linkedGroupId = json.getString("linkedGroupId");
StringBuilder mtag = new StringBuilder(json.getString("mtag"));
if (Tools.isEmpty(ids) || null == linkedGroupId) {
return ResponseResult.failure("传参异常");
}
// 带上项目配置的必要标注标签
if (checkMTagIllegal(mtag)){
return ResponseResult.failure("标注标签出现同组标签，请勿连点标注");
}
return markDataService.addMtagBatch(ids, linkedGroupId, mtag.toString());
}
@ApiOperation("舆情列表")
@PostMapping("/mark/list")
@LogRecord(description = "舆情库-舆情列表")
......@@ -60,10 +126,17 @@ public class AppArticleController extends BaseController {
return ResponseResult.success(markDataService.generateYuqingMarkAggreeList(startTime, endTime));
}
@ApiOperation("聚合进度查询")
@GetMapping("/aggree/schedule/{aggreeId}")
public ResponseResult getAggreeProgress(@PathVariable String aggreeId) {
return ResponseResult.success(markDataService.getAggreeProgress(aggreeId));
}
@ApiOperation("舆情列表-聚合进度查询")
@GetMapping("/mark/aggree/schedule/{aggreeId}")
@Deprecated
public ResponseResult getYuqingMarkAggreeProgress(@PathVariable String aggreeId) {
return ResponseResult.success(markDataService.getYuqingMarkAggreeProgress(aggreeId));
return ResponseResult.success(markDataService.getAggreeProgress(aggreeId));
}
@ApiOperation("舆情列表-获取聚合结果")
......@@ -79,7 +152,7 @@ public class AppArticleController extends BaseController {
}
@ApiOperation("舆情导出")
@PostMapping(value = "mark/list/export")
@PostMapping(value = "/mark/list/export")
public ResponseResult exportYuqingMarkList(@RequestBody MarkSearchDTO markSearchDTO) {
Pair<String, List<ExportAppYuqingDTO>> stringListPair = markDataService.downloadYuqingMarkList(markSearchDTO);
EasyExcelUtil.download(stringListPair.getLeft() + "_舆情列表数据", "sheet1", ExportAppYuqingDTO.class, stringListPair.getRight(), response);
......@@ -124,7 +197,7 @@ public class AppArticleController extends BaseController {
@GetMapping("/platform/proportion")
public ResponseResult getMarkPlatformProportion(@RequestParam(value = "startTime", required = false) Long startTime,
@RequestParam(value = "endTime", required = false) Long endTime) {
return ResponseResult.success(markDataService.getMarkPlatformProportion(startTime, endTime, true));
return ResponseResult.success(markDataService.getMarkPlatformProportion(UserThreadLocal.getProjectId(), startTime, endTime, true));
}
@ApiOperation("舆情分析-渠道参与")
......@@ -202,4 +275,14 @@ public class AppArticleController extends BaseController {
return reportService.getReportSchedule(idList);
}
private boolean checkMTagIllegal(StringBuilder mtag) {
List<MarkerTag> hitTags = projectService.getProjectById(UserThreadLocal.getProjectId()).getHitTags();
if (!Tools.isEmpty(hitTags)) {
for (String mustTag : hitTags.stream().map(MarkerTag::getUniqueId).collect(Collectors.toList())) {
mtag.append(mustTag);
}
}
return Tools.isIllegalTag(mtag.toString());
}
}
src/main/java/com/zhiwei/brandkbs2/controller/app/AppHotController.java
......@@ -127,12 +127,11 @@ public class AppHotController extends BaseController {
public ResponseResult latestNews(@RequestParam(value = "size", defaultValue = "5") Integer size,
@RequestParam(value = "include", defaultValue = "true") boolean include) throws IOException {
String projectId = UserThreadLocal.getProjectId();
String linkedGroupId = projectService.getProjectVOById(projectId).getBrandLinkedGroupId();
Date endDate = new Date();
Date startDate = DateUtils.addHours(endDate, -24);
List<Map.Entry<String, Integer>> markTopTitleList =
markDataService.getMarkTopTitle(startDate.getTime(), endDate.getTime(), null, projectId, linkedGroupId, Constant.PRIMARY_CONTEND_ID, size, include);
markDataService.getMarkTopTitle(startDate.getTime(), endDate.getTime(), null, projectId, Constant.PRIMARY_CONTEND_ID, size, include);
List<JSONObject> resultList = markTopTitleList.stream().map(map -> {
JSONObject json = new JSONObject();
......@@ -142,7 +141,7 @@ public class AppHotController extends BaseController {
}).collect(Collectors.toList());
CompletableFuture.allOf(resultList.stream().map(json -> CompletableFuture.supplyAsync(() -> {
try {
BaseMap firstArticle = markDataService.getFirstArticle(startDate.getTime(), endDate.getTime(), json.getString("title"), projectId, linkedGroupId, Constant.PRIMARY_CONTEND_ID, include);
BaseMap firstArticle = markDataService.getFirstArticle(startDate.getTime(), endDate.getTime(), json.getString("title"), projectId, Constant.PRIMARY_CONTEND_ID, include);
json.put("content", firstArticle.getContent());
json.put("url", firstArticle.getUrl());
json.put("realSource", firstArticle.getRealSource());
......
src/main/java/com/zhiwei/brandkbs2/controller/mobile/MobileEventController.java
......@@ -11,7 +11,6 @@ import com.zhiwei.brandkbs2.service.CommonService;
import com.zhiwei.brandkbs2.service.EventService;
import com.zhiwei.brandkbs2.service.ProjectService;
import com.zhiwei.middleware.event.pojo.dto.BrandkbsEventSearchDTO;
import com.zhiwei.middleware.mark.pojo.enums.TagField;
import com.zhiwei.middleware.mark.vo.MarkerTag;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
......@@ -67,7 +66,7 @@ public class MobileEventController extends BaseController {
if (0 != emotion) {
String emotionName = EmotionEnum.state2Name(emotion);
String linkedGroupId = projectService.getProjectByContendId(UserThreadLocal.getProjectId(), Constant.PRIMARY_CONTEND_ID).getBrandLinkedGroupId();
List<MarkerTag> tags = commonService.getQbjcTags(linkedGroupId, TagField.GROUP_NAME.is("情感倾向"));
List<MarkerTag> tags = commonService.getEmotionTagsWithSort(UserThreadLocal.getProjectId(), linkedGroupId);
tags.forEach(tag -> {
if (tag.getName().equals(emotionName)) {
dto.setEmotions(Collections.singletonList(tag.getUniqueId()));
......
src/main/java/com/zhiwei/brandkbs2/controller/mobile/MobileSearchController.java
......@@ -10,7 +10,6 @@ import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
import com.zhiwei.brandkbs2.service.*;
import com.zhiwei.middleware.event.pojo.dto.BrandkbsEventSearchDTO;
import com.zhiwei.middleware.mark.pojo.enums.TagField;
import com.zhiwei.middleware.mark.vo.MarkerTag;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
......@@ -94,7 +93,7 @@ public class MobileSearchController {
if (0 != emotion) {
String emotionName = EmotionEnum.state2Name(emotion);
String linkedGroupId = projectService.getProjectByContendId(UserThreadLocal.getProjectId(), Constant.PRIMARY_CONTEND_ID).getBrandLinkedGroupId();
List<MarkerTag> tags = commonService.getQbjcTags(linkedGroupId, TagField.GROUP_NAME.is("情感倾向"));
List<MarkerTag> tags = commonService.getEmotionTagsWithSort(UserThreadLocal.getProjectId(),linkedGroupId);
tags.forEach(tag -> {
if (tag.getName().equals(emotionName)) {
dto.setEmotions(Collections.singletonList(tag.getUniqueId()));
......
src/main/java/com/zhiwei/brandkbs2/es/EsQueryTools.java
......@@ -94,16 +94,15 @@ public class EsQueryTools {
return tagBoolQuery;
}
public static BoolQueryBuilder assembleCacheMapsQuery(String projectId, String linkedGroupId, String contendId) {
return assembleCacheMapsQuery(projectId, linkedGroupId, contendId, null);
public static BoolQueryBuilder assembleCacheMapsQuery(String projectId, String contendId) {
return assembleCacheMapsQuery(projectId, contendId, null);
}
public static BoolQueryBuilder assembleCacheMapsQuery(String projectId, String linkedGroupId, String contendId, List<String> mediaTypes) {
return assembleCacheMapsQuery(projectId, linkedGroupId, contendId, mediaTypes,null);
public static BoolQueryBuilder assembleCacheMapsQuery(String projectId, String contendId, List<String> mediaTypes) {
return assembleCacheMapsQuery(projectId, contendId, mediaTypes,null);
}
public static BoolQueryBuilder assembleCacheMapsQuery(String projectId, String linkedGroupId, String contendId, List<String> mediaTypes,
Integer channelEmotion) {
public static BoolQueryBuilder assembleCacheMapsQuery(String projectId,String contendId, List<String> mediaTypes, Integer channelEmotion) {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
String key = concat(projectId, contendId);
BoolQueryBuilder nestedBoolBuilder = QueryBuilders.boolQuery();
......@@ -146,25 +145,23 @@ public class EsQueryTools {
return query;
}
public static BoolQueryBuilder assembleHotEsKeyQuery(String linkedGroupId, List<String> platformIds) {
BoolQueryBuilder query = QueryBuilders.boolQuery();
query.must(QueryBuilders.termQuery("monitor_rule_cache_maps.project.keyword", linkedGroupId));
if (!Tools.isEmpty(platformIds)) {
BoolQueryBuilder platformQuery = QueryBuilders.boolQuery();
for (String platformId : platformIds) {
platformQuery.should(QueryBuilders.termQuery("monitor_rule_cache_maps.platform.keyword", platformId));
}
query.must(platformQuery);
}
return query;
}
public static NestedQueryBuilder cacheMapsNestedQuery(QueryBuilder query) {
return new NestedQueryBuilder(ES_BRANDKBS_CACHE_MAPS, query, ScoreMode.None);
}
// /**
// * 媒体类型请求匹配
// *
// * @param mediaTypes
// * @return
// */
// public static BoolQueryBuilder assembleMediaTypeQuery(List<String> mediaTypes) {
// BoolQueryBuilder tagQuery = QueryBuilders.boolQuery();
// mediaTypes.forEach(e -> {
// // TODO 1
// tagQuery.should(QueryBuilders.termQuery("brandkbs_cache_maps.channel_type.keyword", e));
// });
// return tagQuery;
// }
public static BoolQueryBuilder assembleFiledKeywordQuery(String field, String keyword) {
BoolQueryBuilder channelBoolQueryBuilder = QueryBuilders.boolQuery();
String[] keys = keyword.trim().split("\\|");
......@@ -181,18 +178,6 @@ public class EsQueryTools {
return assembleFiledKeywordQuery("source", sourceKeyword);
}
// public static BoolQueryBuilder assembleChannelSourceQuery(String sourceKeyword) {
// BoolQueryBuilder channelBoolQueryBuilder = QueryBuilders.boolQuery();
// String[] keys = sourceKeyword.trim().split("\\|");
// for (String key : keys) {
// String channelRegex = getAllRegex(key);
// BoolQueryBuilder keyQueryBuilder = QueryBuilders.boolQuery();
// keyQueryBuilder.must(QueryBuilders.regexpQuery("source.keyword", ".*" + channelRegex + ".*"));
// channelBoolQueryBuilder.should(keyQueryBuilder);
// }
// return channelBoolQueryBuilder;
// }
public static void platformMustNot(BoolQueryBuilder boolQueryBuilder, String... platformNames) {
if (null == platformNames) {
return;
......@@ -278,7 +263,7 @@ public class EsQueryTools {
* @Title: matchPhraseQueryFields
* @Description: 字段不拆封, 多字段 同关键词
*/
public static BoolQueryBuilder matchPhraseQueryFields(BoolQueryBuilder boolQueryBuilder, String esQuery,
private static BoolQueryBuilder matchPhraseQueryFields(BoolQueryBuilder boolQueryBuilder, String esQuery,
String word, String... fields) {
if (ES_QUERY_AND.equals(esQuery)) {
for (String field : fields) {
......
src/main/java/com/zhiwei/brandkbs2/es/EsRestClient.java
......@@ -84,6 +84,52 @@ public class EsRestClient {
return client;
}
@Bean(name = "hotEsClient")
public RestHighLevelClient hotEsClient() {
List<HttpHost> httpHostList = new ArrayList<>();
try {
Assert.hasText(esProperties.getClusterNodesHotES(), "Cluster nodes source must not be null or empty!");
String[] nodes = StringUtils.delimitedListToStringArray(esProperties.getClusterNodesHotES(), COMMA);
Arrays.stream(nodes).forEach(node -> {
String[] segments = StringUtils.delimitedListToStringArray(node, COLON);
Assert.isTrue(segments.length == 2,
() -> String.format("Invalid cluster node %s in %s! Must be in the format host:port!", node,
esProperties.getClusterNodesHotES()));
String host = segments[0].trim();
String port = segments[1].trim();
Assert.hasText(host, () -> String.format("No host name given cluster node %s!", node));
Assert.hasText(port, () -> String.format("No port given in cluster node %s!", node));
httpHostList.add(new HttpHost(host, Integer.parseInt(port)));
});
} catch (Exception e) {
log.error("es client初始化异常", e);
}
HttpHost[] httpHosts = httpHostList.toArray(new HttpHost[httpHostList.size()]);
// 判断，如果未配置用户名，则进行无用户名密码连接，配置了用户名，则进行用户名密码连接
RestHighLevelClient client;
if (StringUtils.isEmpty(esProperties.getUsername())) {
client = new RestHighLevelClient(RestClient.builder(httpHosts));
} else {
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY,
// es账号密码
new UsernamePasswordCredentials(esProperties.getUsernameHotES(), esProperties.getPasswordHotES()));
client = new RestHighLevelClient(
RestClient.builder(httpHosts).setHttpClientConfigCallback((httpClientBuilder) -> {
// 这里可以设置一些参数，比如cookie存储、代理等等
httpClientBuilder.disableAuthCaching();
return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
}));
}
return client;
}
@Bean(name = "localEsClient")
public RestHighLevelClient localEsClient() {
List<HttpHost> httpHostList = new ArrayList<>();
......
src/main/java/com/zhiwei/brandkbs2/es/HotEsDao.java 0 → 100644
package com.zhiwei.brandkbs2.es;
import com.zhiwei.brandkbs2.config.Constant;
import org.apache.commons.lang3.time.DateUtils;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
/**
* @ClassName: HotEsClientDao
* @Description HotEsClientDao
* @author: sjj
* @date: 2023年4月3日09:31:58
*/
@Component("hotEsDao")
public class HotEsDao extends EsClientDao {
@Resource(name = "hotEsClient")
RestHighLevelClient hotEsClient;
@Override
public String[] getIndexes() {
return getHotEsIndexes().toArray(new String[0]);
}
@Override
protected RestHighLevelClient getEsClient() {
return hotEsClient;
}
public List<String> getHotEsIndexes() {
List<String> res = new ArrayList<>();
// 近3天数据库
Date date = new Date();
for (int i = 0; i < 3; i++) {
String index = "hot_" + Constant.DF_yyyyMMdd.format(DateUtils.addDays(date, -i));
res.add(index);
}
return res;
}
}
src/main/java/com/zhiwei/brandkbs2/function/ArticleFunction.java 0 → 100644
package com.zhiwei.brandkbs2.function;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.es.EsClientDao;
import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
import com.zhiwei.brandkbs2.service.MarkFlowService;
import org.apache.commons.lang3.tuple.Pair;
import org.elasticsearch.search.SearchHits;
import java.io.IOException;
import java.util.Map;
/**
* @ClassName: ArticleFunction
* @Description ArticleFunction
* @author: sjj
* @date: 2023-04-03 15:01
*/
public class ArticleFunction {
/**
* 创建搜索条件
*/
@FunctionalInterface
public interface CreateSearchHelper {
EsClientDao.SearchHelper create(MarkSearchDTO dto);
}
/**
* 返回搜索结果
*/
@FunctionalInterface
public interface SearchHitsAndCount {
Pair<SearchHits[], Map<String, Long>> search(MarkSearchDTO dto, boolean scroll) throws IOException;
}
/**
* 创建MarkFlowInfo实体
*/
@FunctionalInterface
public interface CreateMarkFlowInfo {
JSONObject create(MarkFlowService.MarkInfoSource markInfoSource);
}
}
src/main/java/com/zhiwei/brandkbs2/pojo/AggreeResult.java
......@@ -4,6 +4,7 @@ import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.common.GlobalPojo;
import com.zhiwei.brandkbs2.util.Tools;
import com.zhiwei.qbjc.bean.tools.BeanTools;
import lombok.Getter;
import lombok.Setter;
import org.apache.commons.lang3.StringUtils;
......@@ -12,6 +13,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
/**
* @ClassName: AggreeResult
......@@ -74,6 +76,11 @@ public class AggreeResult extends AbstractBaseMongo {
private Integer aggreeSize;
/**
* 对应es数据的id集合
*/
private List<String> esIds;
/**
* 父聚合id
*/
private String fatherId;
......@@ -83,10 +90,11 @@ public class AggreeResult extends AbstractBaseMongo {
*/
private boolean isFather;
public static AggreeResult createFather(JSONObject json, String taskId,String projectId, Integer aggreeSize) {
AggreeResult aggreeResult = getInstance(json, taskId,projectId);
public static AggreeResult createFather(JSONObject json, String taskId, String projectId, List<JSONObject> list) {
AggreeResult aggreeResult = getInstance(json, taskId, projectId);
aggreeResult.setFather(true);
aggreeResult.setAggreeSize(aggreeSize);
aggreeResult.setAggreeSize(list.size());
aggreeResult.setEsIds(list.stream().map(esJson -> esJson.getString("id")).collect(Collectors.toList()));
aggreeResult.setCTime(System.currentTimeMillis());
return aggreeResult;
}
......@@ -105,7 +113,12 @@ public class AggreeResult extends AbstractBaseMongo {
aggreeResult.setTaskId(taskId);
BaseMap baseMap = Tools.getBaseFromEsMap(json);
aggreeResult.setTime(baseMap.getTime());
aggreeResult.setPlatformId(json.getString(GenericAttribute.ES_PLATFORM_ID));
// platformId分两种方式获取
String platformId = json.getString(GenericAttribute.ES_PLATFORM_ID);
if (null == platformId) {
platformId = BeanTools.filterPlatform(GlobalPojo.PLATFORMS, json.getInteger("c5"), json.getInteger("foreign")).getId();
}
aggreeResult.setPlatformId(platformId);
String mediaType = GlobalPojo.getMediaType(projectId, baseMap.getPlatform(), baseMap.getSource());
if (StringUtils.isNotEmpty(mediaType)) {
aggreeResult.setMediaTypes(Arrays.asList(mediaType.split(",")));
......
src/main/java/com/zhiwei/brandkbs2/pojo/MarkFlowEntity.java
......@@ -6,14 +6,15 @@ import com.zhiwei.base.entity.subclass.mark.CompleteTextMark;
import com.zhiwei.base.entity.subclass.mark.IncompleteTextMark;
import com.zhiwei.base.entity.subclass.mark.QATextMark;
import com.zhiwei.base.entity.subclass.mark.VideoMark;
import com.zhiwei.base.filter.FilterInfo;
import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.common.GlobalPojo;
import com.zhiwei.brandkbs2.util.FlowEntityUtil;
import com.zhiwei.brandkbs2.util.Tools;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.apache.commons.lang3.ObjectUtils;
import org.elasticsearch.search.SearchHit;
import java.io.Serializable;
import java.util.List;
......@@ -46,6 +47,11 @@ public class MarkFlowEntity implements Serializable {
*/
private Object data;
/**
* 消息流相关信息
*/
private FilterInfo filterInfo;
public static MarkFlowEntity getFoldInstance(List<MarkFlowEntity> entities) {
// 首条数据为主
MarkFlowEntity entity = Tools.copyByJson(entities.get(0), MarkFlowEntity.class);
......@@ -61,23 +67,27 @@ public class MarkFlowEntity implements Serializable {
return entity;
}
public static MarkFlowEntity getAggreeInstance(List<MarkFlowEntity> entities, Integer size) {
// 首条数据为主
public static MarkFlowEntity getAggreeInstance(List<MarkFlowEntity> entities, AggreeResult aggreeResult, boolean sonDetails) {
Integer size = aggreeResult.getAggreeSize();
// 首条数据为主,copy丢失FilterInfo
MarkFlowEntity entity = Tools.copyByJson(entities.get(0), MarkFlowEntity.class);
if (null != size && size > 1) {
entity.getType().put("fold", true);
entity.getType().put("size", size);
}
if (sonDetails) {
entity.getType().put("ids", aggreeResult.getEsIds());
}
return entity;
}
/**
* Constructor
*
* @param hit
* @param tJson
*/
public MarkFlowEntity(SearchHit hit) {
this(new JSONObject(hit.getSourceAsMap()));
public MarkFlowEntity(JSONObject tJson) {
this(tJson, null);
}
/**
......@@ -85,7 +95,7 @@ public class MarkFlowEntity implements Serializable {
*
* @param tJson
*/
public MarkFlowEntity(JSONObject tJson) {
public MarkFlowEntity(JSONObject tJson, String linkedGroupId) {
if (Objects.isNull(tJson.getString(GenericAttribute.ES_CNAME))) {
tJson.put(GenericAttribute.ES_CNAME, "采集来源缺失");
}
......@@ -98,17 +108,25 @@ public class MarkFlowEntity implements Serializable {
if (!Tools.isEmpty(tJson.get("ind_title"))) {
tJson.put("ind_title", Tools.filterStrHtmlTag(ObjectUtils.defaultIfNull(tJson.getString("ind_title"), "[\"\"]")));
}
CompleteTextMark completeText = CompleteTextMark.restoreFromEs(tJson);
this.type = FlowEntityUtil.getCompleteTextType(completeText);
this.data = completeText;
CompleteTextMark completeTextMark = CompleteTextMark.restoreFromEs(tJson);
if (null != linkedGroupId) {
completeTextMark.setMgroup(GlobalPojo.getYuQingProjectNameById(linkedGroupId));
this.filterInfo = completeTextMark.filterInfo();
}
this.type = FlowEntityUtil.getCompleteTextType(completeTextMark);
this.data = completeTextMark;
break;
case INCOMPLETE:
if (!Tools.isEmpty(tJson.get("ind_full_text"))) {
tJson.put("ind_full_text", Tools.filterStrHtmlTag(ObjectUtils.defaultIfNull(tJson.getString("ind_full_text"), "[\"\",\"\"]")));
}
IncompleteTextMark incompleteText = IncompleteTextMark.restoreFromEs(tJson);
this.type = FlowEntityUtil.getIncompleteTextType(incompleteText);
this.data = incompleteText;
IncompleteTextMark incompleteTextMark = IncompleteTextMark.restoreFromEs(tJson);
if (null != linkedGroupId) {
incompleteTextMark.setMgroup(GlobalPojo.getYuQingProjectNameById(linkedGroupId));
this.filterInfo = incompleteTextMark.filterInfo();
}
this.type = FlowEntityUtil.getIncompleteTextType(incompleteTextMark);
this.data = incompleteTextMark;
break;
case QA:
if (!Tools.isEmpty(tJson.get("ind_full_text"))) {
......@@ -120,9 +138,13 @@ public class MarkFlowEntity implements Serializable {
if (!Tools.isEmpty(tJson.get("ind_content"))) {
tJson.put("ind_content", Tools.filterStrHtmlTag(ObjectUtils.defaultIfNull(tJson.getString("ind_content"), "[\"\"]")));
}
QATextMark qaText = QATextMark.restoreFromEs(tJson);
this.type = FlowEntityUtil.getQaTextType(qaText);
this.data = qaText;
QATextMark qaTextMark = QATextMark.restoreFromEs(tJson);
if (null != linkedGroupId) {
qaTextMark.setMgroup(GlobalPojo.getYuQingProjectNameById(linkedGroupId));
this.filterInfo = qaTextMark.filterInfo();
}
this.type = FlowEntityUtil.getQaTextType(qaTextMark);
this.data = qaTextMark;
break;
case VIDEO:
if (!Tools.isEmpty(tJson.get("ind_full_text"))) {
......@@ -131,13 +153,24 @@ public class MarkFlowEntity implements Serializable {
if (!Tools.isEmpty(tJson.get("ind_title"))) {
tJson.put("ind_title", Tools.filterStrHtmlTag(ObjectUtils.defaultIfNull(tJson.getString("ind_title"), "[\"\"]")));
}
VideoMark video = VideoMark.restoreFromEs(tJson);
this.type = FlowEntityUtil.getVideoType(video);
this.data = video;
VideoMark videoMark = VideoMark.restoreFromEs(tJson);
if (null != linkedGroupId) {
videoMark.setMgroup(GlobalPojo.getYuQingProjectNameById(linkedGroupId));
this.filterInfo = videoMark.filterInfo();
}
this.type = FlowEntityUtil.getVideoType(videoMark);
this.data = videoMark;
break;
default:
break;
}
}
public FilterInfo getFilterInfo(String linkedGroupId) {
if (null == filterInfo.getSelectType() || null == filterInfo.getFilterMap()) {
this.filterInfo = new MarkFlowEntity((JSONObject) data, linkedGroupId).filterInfo;
}
return filterInfo;
}
}
src/main/java/com/zhiwei/brandkbs2/service/CommonService.java
package com.zhiwei.brandkbs2.service;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.base.entity.subclass.mark.MarkInfo;
import com.zhiwei.middleware.mark.vo.MarkerTag;
import com.zhiwei.middleware.mark.vo.TagSearch;
......@@ -48,6 +49,16 @@ public interface CommonService {
*/
List<JSONObject> getQbjcPlatform(String... includeFields);
/**
* 添加标注
*/
void addMtag(MarkInfo markInfo);
/**
* 批量添加标注
*/
void addMtagBatch(List<MarkInfo> markInfos);
Long[] getTimeRangeMonth();
Long[] getTimeRangeWeek();
......
src/main/java/com/zhiwei/brandkbs2/service/EsSearchService.java 0 → 100644
package com.zhiwei.brandkbs2.service;
import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
import org.apache.commons.lang3.tuple.Pair;
import org.elasticsearch.search.SearchHits;
import java.io.IOException;
import java.util.Map;
/**
* @ClassName: EsSearchService
* @Description es数据搜索服务类
* @author: sjj
* @date: 2023-03-31 17:13
*/
public interface EsSearchService {
/**
* 搜索标注数据
*
* @param dto 搜素实体类
* @param scroll 滚动查询
* @return 搜索结果
*/
Pair<SearchHits[], Map<String, Long>> searchMarkHitsAndCount(MarkSearchDTO dto, boolean scroll) throws IOException;
/**
* 搜索竞品标注数据
*
* @param dto 搜素实体类
* @param scroll 滚动查询
* @return 搜索结果
*/
Pair<SearchHits[], Map<String, Long>> searchContendMarkHitsAndCount(MarkSearchDTO dto, boolean scroll) throws IOException;
/**
* 搜索原始数据
*
* @param dto 搜素实体类
* @param scroll 滚动查询
* @return 搜索结果
*/
Pair<SearchHits[], Map<String, Long>> searchOriginHitsAndCount(MarkSearchDTO dto, boolean scroll) throws IOException;
}
src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java
package com.zhiwei.brandkbs2.service;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.BaseMap;
import com.zhiwei.brandkbs2.pojo.Event;
import com.zhiwei.brandkbs2.pojo.MarkFlowEntity;
......@@ -25,6 +26,51 @@ import java.util.Map;
public interface MarkDataService {
/**
* 获取原始数据
*
* @param markSearchDTO 标注数据搜索传输类
* @return 返回结果
*/
PageVO<MarkFlowEntity> getOriginList(MarkSearchDTO markSearchDTO);
/**
* 生成聚合列表并返回id
*
* @return 聚合id
*/
String generateOriginAggreeList(Long startTime, Long endTime);
/**
* 获取聚合结果列表
*
* @param markSearchDTO 搜索实体
* @return 标注消息流列表
*/
PageVO<MarkFlowEntity> getOriginAggreeList(MarkSearchDTO markSearchDTO);
/**
* @param markSearchDTO 标注数据搜索传输类
* @return 返回结果
*/
Pair<String, List<ExportAppYuqingDTO>> downloadOriginList(MarkSearchDTO markSearchDTO);
/**
* 添加标注
*
* @param zip 压缩数据源
* @param mtag 标注信息
*/
void addMtag(String zip, String linkedGroupId, String mtag);
/**
* 批量添加标注
*
* @param ids id列表
* @param mtag 标注信息
*/
ResponseResult addMtagBatch(List<String> ids, String linkedGroupId, String mtag);
/**
* @param markSearchDTO 标注数据搜索传输类
* @return 返回结果
*/
......@@ -49,7 +95,7 @@ public interface MarkDataService {
* @param id 聚合id
* @return 进度值
*/
String getYuqingMarkAggreeProgress(String id);
String getAggreeProgress(String id);
/**
* 获取聚合结果列表
......@@ -114,7 +160,7 @@ public interface MarkDataService {
* @param endTime 结束时间
* @return 获取舆情标注数据舆情平台分布
*/
List<JSONObject> getMarkPlatformProportion(Long startTime, Long endTime, boolean cache);
List<JSONObject> getMarkPlatformProportion(String projectId, Long startTime, Long endTime, boolean cache);
/**
* 获取舆情标注数据舆情平台分布
......@@ -124,7 +170,7 @@ public interface MarkDataService {
* @param emotion 情感倾向筛选
* @return 获取舆情标注数据舆情平台分布
*/
List<JSONObject> getMarkPlatformProportion(Long startTime, Long endTime,String emotion,boolean cache);
List<JSONObject> getMarkPlatformProportion(String projectId, Long startTime, Long endTime, String emotion, boolean cache);
/**
* 获取舆情标注数据舆情高频词分布
......@@ -142,12 +188,10 @@ public interface MarkDataService {
* @param endTime 结束时间
* @param emotion 情感倾向名称
* @param projectId 项目id
* @param linkedGroupId 关联组id
* @param size 分页大小
* @return 最热标题
*/
List<Map.Entry<String, Integer>> getMarkTopTitle(Long startTime, Long endTime, String emotion, String projectId, String linkedGroupId,
String contendId, int size) throws IOException;
List<Map.Entry<String, Integer>> getMarkTopTitle(Long startTime, Long endTime, String emotion, String projectId, String contendId, int size) throws IOException;
/**
* 获取时间段某情感数据最多的标题
......@@ -156,13 +200,11 @@ public interface MarkDataService {
* @param endTime 结束时间
* @param emotion 情感倾向名称
* @param projectId 项目id
* @param linkedGroupId 关联组id
* @param size 分页大小
* @param include 是否包含主品牌
* @return 最热标题
*/
List<Map.Entry<String, Integer>> getMarkTopTitle(Long startTime, Long endTime, String emotion, String projectId, String linkedGroupId,
String contendId, int size, boolean include) throws IOException;
List<Map.Entry<String, Integer>> getMarkTopTitle(Long startTime, Long endTime, String emotion, String projectId, String contendId, int size, boolean include) throws IOException;
/**
* 根据聚合标题和时间获取首发稿件
......@@ -171,10 +213,9 @@ public interface MarkDataService {
* @param endTime 结束时间
* @param aggTitle 聚合标题
* @param projectId 项目id
* @param linkedGroupId 关联组id
* @return 首发稿件
*/
BaseMap getFirstArticle(Long startTime, Long endTime, String aggTitle, String projectId, String linkedGroupId, String contendId) throws IOException;
BaseMap getFirstArticle(Long startTime, Long endTime, String aggTitle, String projectId, String contendId) throws IOException;
/**
* 根据聚合标题和时间获取首发稿件
......@@ -183,11 +224,10 @@ public interface MarkDataService {
* @param endTime 结束时间
* @param aggTitle 聚合标题
* @param projectId 项目id
* @param linkedGroupId 关联组id
* @param include 是否包含主品牌
* @return 首发稿件
*/
BaseMap getFirstArticle(Long startTime, Long endTime, String aggTitle, String projectId, String linkedGroupId, String contendId, boolean include) throws IOException;
BaseMap getFirstArticle(Long startTime, Long endTime, String aggTitle, String projectId, String contendId, boolean include) throws IOException;
/**
* 搜索标注数据通过事件
......@@ -207,7 +247,7 @@ public interface MarkDataService {
* @param fetchSource
* @return
*/
List<JSONObject> searchMarkDataByTime(String projectId, String linkedGroupId, String contendId, Long startTime, Long endTime, String... fetchSource);
List<JSONObject> searchMarkDataByTime(String projectId,String contendId, Long startTime, Long endTime, String... fetchSource);
/**
* 竞品库-获取搜索条件
......@@ -238,12 +278,11 @@ public interface MarkDataService {
/**
* 竞品库-获取竞品对比传播分析数据
*
* @param markSearchDTO 标注数据搜索传输类
* @param hotArticleSize 热门稿件数量
* @param cache 是否启用缓存
* @return JSONObject
*/
JSONObject getContendSpreadAnalyze(Long startTime, Long endTime, String projectId, String contendId, int hotArticleSize, boolean useCache) throws IOException;
JSONObject getContendSpreadAnalyze(Long startTime, Long endTime, String projectId, String contendId, int hotArticleSize, boolean cache) throws IOException;
/**
* 竞品库-舆情导出
......@@ -320,9 +359,10 @@ public interface MarkDataService {
* @param size 大小
* @return 发布最多稿件的渠道
*/
List<Map<String, Object>> getEsTopSource(Long startTime, Long endTime, String projectId, String linkedGroupId, String contendId, String emotion, int size) throws IOException;
List<Map<String, Object>> getEsTopSource(Long startTime, Long endTime, String projectId, String contendId, String emotion, int size) throws IOException;
JSONObject getLastMarkData(String projectId, String contendId, String platform, String realSource, String source);
JSONObject getLastMarkData(String projectId, String linkedGroupId, String contendId, String platform, String realSource, String source);
BoolQueryBuilder projectContendIdQuery(String projectId, String contendId);
BoolQueryBuilder projectLinkedGroupContendIdQuery(String projectId, String linkedGroupId, String contendId);
}
src/main/java/com/zhiwei/brandkbs2/service/MarkFlowService.java
package com.zhiwei.brandkbs2.service;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.base.entity.subclass.mark.MarkInfo;
import com.zhiwei.brandkbs2.pojo.MarkFlowEntity;
import lombok.AllArgsConstructor;
import lombok.Data;
import java.util.List;
/**
* @ClassName: MarkFlowService
......@@ -14,13 +19,36 @@ public interface MarkFlowService {
/**
* 创建标注消息流实体
*
* @param json
* @param projectId
* @param markInfoSource
* @return
*/
JSONObject createMarkFlowInfo(MarkInfoSource markInfoSource);
/**
* 创建标注消息流实体
*
* @param markInfoSource
* @return
*/
JSONObject createMarkFlowInfo4Origin(MarkInfoSource markInfoSource);
/**
* 匹配标注标签值
* @param mes
* @param linkedGroupId
* @param contendId
*/
List<MarkFlowEntity> markTagInfo(List<MarkFlowEntity> mes, String linkedGroupId);
/**
* 解析JSON数据
*
* @param json
* @param mgroup
* @param mperson
* @param mtag
* @return
*/
JSONObject createMarkFlowInfo(JSONObject json, String projectId,String linkedGroupId, String contendId);
MarkInfo markInfoFromJson(JSONObject json, String mgroup, String mperson, String mtag);
/**
* 快照页信息组装
......@@ -37,4 +65,24 @@ public interface MarkFlowService {
*/
MarkFlowEntity getShotPageFromCache(String id);
@Data
@AllArgsConstructor
class MarkInfoSource {
private JSONObject json;
private String projectId;
private String contendId;
private String linkedGroupId;
public MarkInfoSource(JSONObject json, String projectId, String contendId) {
this.json = json;
this.projectId = projectId;
this.contendId = contendId;
}
public MarkInfoSource(JSONObject json, String linkedGroupId) {
this.json = json;
this.linkedGroupId = linkedGroupId;
}
}
}
src/main/java/com/zhiwei/brandkbs2/service/ProjectService.java
......@@ -22,7 +22,7 @@ public interface ProjectService {
*
* @param projectVO 项目配置展示对象
*/
void addProject(ProjectVO projectVO);
ResponseResult addProject(ProjectVO projectVO);
/**
* 查询所有项目列表
......@@ -56,7 +56,7 @@ public interface ProjectService {
*
* @param projectVO 项目展示对象
*/
void updateProject(ProjectVO projectVO);
ResponseResult updateProject(ProjectVO projectVO);
/**
* 根据项目ID删除项目
......
src/main/java/com/zhiwei/brandkbs2/service/impl/ChannelServiceImpl.java
......@@ -949,7 +949,6 @@ public class ChannelServiceImpl implements ChannelService {
long s0 = System.currentTimeMillis();
String projectId = UserThreadLocal.getProjectId();
String contendId = "0";
String linkedGroupId = projectService.getProjectByContendId(projectId, contendId).getBrandLinkedGroupId();
Query query = new Query(Criteria.where("projectId").is(projectId).and("contendId").is(contendId));
channelDao.addKeywordFuzz(query, keyword, "source");
if (!(null == platforms || platforms.contains("全部"))) {
......@@ -978,19 +977,12 @@ public class ChannelServiceImpl implements ChannelService {
JSONObject result = new JSONObject();
result.put("id", channel.getId());
result.put("channel", channel);
result.put("lastArticle", markDataService.getLastMarkData(projectId, linkedGroupId, contendId,
channel.getPlatform(), channel.getRealSource(), channel.getSource()));
result.put("lastArticle", markDataService.getLastMarkData(projectId, contendId, channel.getPlatform(), channel.getRealSource(), channel.getSource()));
return result;
}, esSearchExecutor)).collect(Collectors.toList());
CompletableFuture.allOf(futureList.toArray(new CompletableFuture[0])).join();
Map<String, JSONObject> idMap = futureList.stream().map(CompletableFuture::join)
.collect(Collectors.toMap(json -> json.getString("id"), a -> a));
// List<JSONObject> resultList = channelList.stream().map(channel -> {
// JSONObject result = new JSONObject();
// result.put("channelInfo", channel);
// result.put("lastArticle", markDataService.getLastMarkUrl(projectId, linkedGroupId, contendId, channel.getPlatform(), channel.getRealSource(), channel.getSource()));
// return result;
// }).collect(Collectors.toList());
long s3 = System.currentTimeMillis();
log.info("渠道发文查询耗时:" + (s3 - s2));
return PageVO.createPageVo(total, page, pageSize,
......@@ -1515,8 +1507,7 @@ public class ChannelServiceImpl implements ChannelService {
}
private Long markCountByEmotion(Channel channel, String emotion) throws IOException {
BoolQueryBuilder postFilter = markDataService.projectLinkedGroupContendIdQuery(channel.getProjectId(),
channel.getLinkedGroupId(), channel.getContendId());
BoolQueryBuilder postFilter = markDataService.projectContendIdQuery(channel.getProjectId(),channel.getContendId());
postFilter.must(QueryBuilders.termQuery("brandkbs_mark_cache_maps.name.keyword", emotion));
return esClientDao.count(postFilter);
}
......
src/main/java/com/zhiwei/brandkbs2/service/impl/CommonServiceImpl.java
package com.zhiwei.brandkbs2.service.impl;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.base.entity.subclass.mark.MarkInfo;
import com.zhiwei.brandkbs2.common.GlobalPojo;
import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.enmus.EmotionEnum;
......@@ -112,6 +113,16 @@ public class CommonServiceImpl implements CommonService {
}
@Override
public void addMtag(MarkInfo markInfo) {
markClient.upsert(markInfo);
}
@Override
public void addMtagBatch(List<MarkInfo> markInfos) {
markClient.eventCollectionUpsert(markInfos);
}
@Override
public Long[] getTimeRangeMonth() {
long endTime = Tools.truncDate(new Date(), Constant.DAY_PATTERN).getTime();
long startTime = DateUtils.addMonths(new Date(endTime), -1).getTime();
......
src/main/java/com/zhiwei/brandkbs2/service/impl/CustomEventServiceImpl.java
......@@ -90,19 +90,17 @@ public class CustomEventServiceImpl implements CustomEventService {
List<JSONObject> resultList = null;
try {
String projectId = UserThreadLocal.getProjectId();
String linkedGroupId = projectService.getProjectVOById(projectId).getBrandLinkedGroupId();
String redisKey = RedisUtil.getCustomEventListKey(projectId, startTime, endTime);
String resultStr = redisUtil.get(redisKey);
if (StringUtils.isNotEmpty(resultStr)) {
return JSON.parseArray(resultStr, JSONObject.class);
}
List<Map.Entry<String, Integer>> resList = markDataService.getMarkTopTitle(startTime, endTime, EmotionEnum.ALL.getName(), projectId,
linkedGroupId, "0", 20);
List<Map.Entry<String, Integer>> resList = markDataService.getMarkTopTitle(startTime, endTime, EmotionEnum.ALL.getName(), projectId, "0", 20);
resultList = resList.stream().map(map -> {
JSONObject result = new JSONObject();
String aggTitle = map.getKey();
try {
BaseMap firstArticle = markDataService.getFirstArticle(startTime, endTime, aggTitle, projectId, linkedGroupId, Constant.PRIMARY_CONTEND_ID);
BaseMap firstArticle = markDataService.getFirstArticle(startTime, endTime, aggTitle, projectId, Constant.PRIMARY_CONTEND_ID);
result.put("title", firstArticle.getTitle());
result.put("time", firstArticle.getTime());
result.put("url", firstArticle.getUrl());
......@@ -297,9 +295,8 @@ public class CustomEventServiceImpl implements CustomEventService {
customEventDao.updateOneByIdWithField(eventId, Update.update("status", false));
ApplicationProjectListener.getThreadPool().execute(() -> {
try {
String linkedGroupId = projectService.getProjectByContendId(customEvent.getProjectId(), customEvent.getContendId()).getBrandLinkedGroupId();
//查询符合该事件时间段的所有稿件信息
List<JSONObject> articles = markDataService.searchMarkDataByTime(customEvent.getProjectId(), linkedGroupId, customEvent.getContendId(),
List<JSONObject> articles = markDataService.searchMarkDataByTime(customEvent.getProjectId(), customEvent.getContendId(),
customEvent.getStartTime(), customEvent.getEndTime());
log.info("自定义事件:{} 查询到符合该事件时间段的所有稿件数{}个", eventId, articles.size());
List<String> keywords = customEvent.getKeywords();
......
src/main/java/com/zhiwei/brandkbs2/service/impl/EsSearchServiceImpl.java 0 → 100644
package com.zhiwei.brandkbs2.service.impl;
import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.common.GlobalPojo;
import com.zhiwei.brandkbs2.es.EsClientDao;
import com.zhiwei.brandkbs2.es.EsQueryTools;
import com.zhiwei.brandkbs2.function.ArticleFunction;
import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
import com.zhiwei.brandkbs2.service.EsSearchService;
import com.zhiwei.brandkbs2.util.Tools;
import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.sort.FieldSortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* @ClassName: EsSearchServiceImpl
* @Description EsSearchServiceImpl
* @author: sjj
* @date: 2023-03-31 17:17
*/
@Service
public class EsSearchServiceImpl implements EsSearchService {
@Resource(name = "esClientDao")
private EsClientDao esClientDao;
@Resource(name = "hotEsDao")
private EsClientDao hotEsDao;
@Override
public Pair<SearchHits[], Map<String, Long>> searchMarkHitsAndCount(MarkSearchDTO dto, boolean scroll) throws IOException {
return searchHitsAndCount(dto, this::createSearchHelper, esClientDao, scroll);
}
@Override
public Pair<SearchHits[], Map<String, Long>> searchContendMarkHitsAndCount(MarkSearchDTO dto, boolean scroll) throws IOException {
return searchHitsAndCount(dto, this::createContendSearchHelper, esClientDao, scroll);
}
@Override
public Pair<SearchHits[], Map<String, Long>> searchOriginHitsAndCount(MarkSearchDTO dto, boolean scroll) throws IOException {
return searchHitsAndCount(dto, this::createOriginSearchHelper, hotEsDao, scroll);
}
private Pair<SearchHits[], Map<String, Long>> searchHitsAndCount(MarkSearchDTO dto, ArticleFunction.CreateSearchHelper createSearchHelper,
EsClientDao clientDao, boolean scroll) throws IOException {
EsClientDao.SearchHelper helper = createSearchHelper.create(dto);
// 滚动查询
if (scroll) {
return Pair.of(clientDao.searchScrollResponse(helper).stream().map(SearchResponse::getHits).toArray(SearchHits[]::new), null);
}
SearchHits searchHits = clientDao.searchHits(helper);
Map<String, Long> platformCount = platformCount(dto.getPage(), dto.getPlatforms(), helper.getPostFilter(), clientDao);
return Pair.of(new SearchHits[]{searchHits}, platformCount);
}
private EsClientDao.SearchHelper createSearchHelper(MarkSearchDTO dto) {
EsClientDao.SearchHelper helper = EsClientDao.createSearchHelper();
// projectId、contendId
String projectId = dto.getProjectId();
String contendId = dto.getContendId();
// PostFilter 后置过滤器
// 新增channelEmotion
BoolQueryBuilder postFilter = EsQueryTools.assembleCacheMapsQuery(projectId, contendId, dto.getMediaTypes(), dto.getChannelEmotion());
// time
postFilter.must(QueryBuilders.rangeQuery(dto.getTimeType()).gte(dto.getStartTime()).lt(dto.getEndTime()));
// platform
if (CollectionUtils.isNotEmpty(dto.getPlatforms())) {
postFilter.must(EsQueryTools.assemblePlatformQuery(Tools.getPlatformByIds(dto.getPlatforms())));
}
// tags
if (CollectionUtils.isNotEmpty(dto.getTags())) {
postFilter.must(EsQueryTools.assembleTagQuery(dto.getTags()));
}
// customTags
if (CollectionUtils.isNotEmpty(dto.getCustomTags())) {
postFilter.must(EsQueryTools.assembleTagQuery(dto.getCustomTags()));
}
// Query 查询条件
// BoolQueryBuilder query = QueryBuilders.boolQuery();
// keyword
if (StringUtils.isNotEmpty(dto.getKeyword())) {
// Query
String[] fieldSearch = "仅标题".equals(dto.getSearchField()) ? new String[]{GenericAttribute.ES_IND_TITLE} : new String[]{GenericAttribute.ES_IND_FULL_TEXT};
postFilter.must(EsQueryTools.assembleNormalKeywordQuery(dto.getKeyword(), fieldSearch));
}
// sourceKeyword
if (StringUtils.isNotEmpty(dto.getSourceKeyword())) {
postFilter.must(EsQueryTools.assembleSourceQuery(dto.getSourceKeyword()));
}
// hostKeyword
if (StringUtils.isNotEmpty(dto.getHostKeyword())) {
postFilter.must(EsQueryTools.assembleFiledKeywordQuery("host", dto.getHostKeyword()));
}
helper.setPostFilter(postFilter);
// helper.setQuery(query);
// sort
FieldSortBuilder sort = null;
if (null != dto.getSorter()) {
for (Map.Entry<String, Object> entry : dto.getSorter().entrySet()) {
String key = entry.getKey();
if (key.equals("influence")) {
key = "channel_influence";
} else if (key.equals("followers")) {
key = "channel_followers";
}
if (null == entry.getValue() || entry.getValue().toString().contains("desc")) {
sort = SortBuilders.fieldSort(key).order(SortOrder.DESC);
} else {
sort = SortBuilders.fieldSort(key).order(SortOrder.ASC);
}
}
}
helper.setSort(sort);
// from size
if (null != dto.getPageSize()) {
helper.setFrom((dto.getPage() - 1) * dto.getPageSize());
helper.setSize(dto.getPageSize());
}
// HighlightBuilder ???
return helper;
}
private EsClientDao.SearchHelper createContendSearchHelper(MarkSearchDTO dto) {
EsClientDao.SearchHelper helper = EsClientDao.createSearchHelper();
// projectId、contendId
String projectId = dto.getProjectId();
String contendId = dto.getContendId();
// PostFilter 后置过滤器
BoolQueryBuilder postFilter = EsQueryTools.assembleCacheMapsQuery(projectId, contendId, dto.getMediaTypes());
// time
postFilter.must(QueryBuilders.rangeQuery(dto.getTimeType()).gte(dto.getStartTime()).lt(dto.getEndTime()));
// platform
if (CollectionUtils.isNotEmpty(dto.getPlatforms())) {
postFilter.must(EsQueryTools.assemblePlatformQuery(Tools.getPlatformByIds(dto.getPlatforms())));
}
// tags
if (CollectionUtils.isNotEmpty(dto.getTags())) {
postFilter.must(EsQueryTools.assembleTagQuery(dto.getTags()));
}
// customTags
if (CollectionUtils.isNotEmpty(dto.getCustomTags())) {
postFilter.must(EsQueryTools.assembleTagQuery(dto.getCustomTags()));
}
// Query 查询条件
// BoolQueryBuilder query = QueryBuilders.boolQuery();
// keyword
if (StringUtils.isNotEmpty(dto.getKeyword())) {
// Query
String[] fieldSearch = "仅标题".equals(dto.getSearchField()) ? new String[]{GenericAttribute.ES_IND_TITLE} : new String[]{GenericAttribute.ES_IND_FULL_TEXT};
postFilter.must(EsQueryTools.assembleNormalKeywordQuery(dto.getKeyword(), fieldSearch));
}
// sourceKeyword
if (StringUtils.isNotEmpty(dto.getSourceKeyword())) {
postFilter.must(EsQueryTools.assembleSourceQuery(dto.getSourceKeyword()));
}
// hostKeyword
if (StringUtils.isNotEmpty(dto.getHostKeyword())) {
postFilter.must(EsQueryTools.assembleFiledKeywordQuery("host", dto.getHostKeyword()));
}
helper.setPostFilter(postFilter);
// helper.setQuery(query);
// sort
FieldSortBuilder sort = null;
if (null != dto.getSorter()) {
for (Map.Entry<String, Object> entry : dto.getSorter().entrySet()) {
String key = entry.getKey();
if (key.equals("influence")) {
key = "channel_influence";
} else if (key.equals("followers")) {
key = "channel_followers";
}
if (null == entry.getValue() || entry.getValue().toString().contains("desc")) {
sort = SortBuilders.fieldSort(key).order(SortOrder.DESC);
} else {
sort = SortBuilders.fieldSort(key).order(SortOrder.ASC);
}
}
}
helper.setSort(sort);
// from size
helper.setFrom((dto.getPage() - 1) * dto.getPageSize());
helper.setSize(dto.getPageSize());
// HighlightBuilder ???
return helper;
}
private EsClientDao.SearchHelper createOriginSearchHelper(MarkSearchDTO dto) {
EsClientDao.SearchHelper helper = EsClientDao.createSearchHelper();
// PostFilter 后置过滤器
BoolQueryBuilder postFilter = EsQueryTools.assembleHotEsKeyQuery(dto.getLinkedGroupId(), dto.getPlatforms());
// time
postFilter.must(QueryBuilders.rangeQuery(dto.getTimeType()).gte(dto.getStartTime()).lt(dto.getEndTime()));
// keyword
if (StringUtils.isNotEmpty(dto.getKeyword())) {
// Query
String[] fieldSearch = "仅标题".equals(dto.getSearchField()) ? new String[]{GenericAttribute.ES_IND_TITLE} : new String[]{GenericAttribute.ES_IND_FULL_TEXT};
postFilter.must(EsQueryTools.assembleNormalKeywordQuery(dto.getKeyword(), fieldSearch));
}
// sourceKeyword
if (StringUtils.isNotEmpty(dto.getSourceKeyword())) {
postFilter.must(EsQueryTools.assembleSourceQuery(dto.getSourceKeyword()));
}
// hostKeyword
if (StringUtils.isNotEmpty(dto.getHostKeyword())) {
postFilter.must(EsQueryTools.assembleFiledKeywordQuery("host", dto.getHostKeyword()));
}
helper.setPostFilter(postFilter);
// sort
FieldSortBuilder sort = null;
if (null != dto.getSorter()) {
for (Map.Entry<String, Object> entry : dto.getSorter().entrySet()) {
String key = entry.getKey();
if (null == entry.getValue() || entry.getValue().toString().contains("desc")) {
sort = SortBuilders.fieldSort(key).order(SortOrder.DESC);
} else {
sort = SortBuilders.fieldSort(key).order(SortOrder.ASC);
}
}
}
helper.setSort(sort);
// from size
helper.setFrom((dto.getPage() - 1) * dto.getPageSize());
helper.setSize(dto.getPageSize());
// HighlightBuilder ???
return helper;
}
/**
* 平台进量统计
*
* @return
*/
private Map<String, Long> platformCount(int page, List<String> platforms, BoolQueryBuilder postFilter, EsClientDao esClientDao) throws IOException {
Map<String, Long> counts = new HashMap<>();
//平台计量统计 仅第一页且无平台限制
if (1 == page && CollectionUtils.isEmpty(platforms)) {
for (MessagePlatform platform : GlobalPojo.PLATFORMS) {
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery().must(postFilter).must(EsQueryTools.assemblePlatformQuery(Collections.singletonList(platform)));
Long count = esClientDao.count(esClientDao.getIndexes(), queryBuilder, null);
counts.put(platform.getName(), count);
}
}
return counts;
}
}
src/main/java/com/zhiwei/brandkbs2/service/impl/IndexServiceImpl.java
......@@ -261,7 +261,6 @@ public class IndexServiceImpl implements IndexService {
endTime = timeRangeMonth[1];
flag = false;
}
String linkedGroupId = projectService.getProjectById(projectId).getBrandLinkedGroupId();
String redisKey = RedisUtil.getIndexSpread(projectId, startTime, endTime);
String resultStr;
// 返回缓存
......@@ -285,12 +284,12 @@ public class IndexServiceImpl implements IndexService {
resJson.put("type", type);
resJson.put("spreadingTend", lineList);
JSONObject summary = new JSONObject();
List<Map.Entry<String, Integer>> resList = markDataService.getMarkTopTitle(startTime, endTime, EmotionEnum.ALL.getName(), projectId, linkedGroupId, contendId, 1);
List<Map.Entry<String, Integer>> resList = markDataService.getMarkTopTitle(startTime, endTime, EmotionEnum.ALL.getName(), projectId, contendId, 1);
summary.put("yqSpread", this.getYqSpread(startTime, endTime, projectId, contendId, lineList, type, flag));
summary.put("pcTopArticle", this.getPcTopArticle(startTime, endTime, projectId, linkedGroupId, contendId, resList));
summary.put("topArticle", this.getMobileTopArticle(startTime, endTime, projectId, linkedGroupId, contendId, resList));
summary.put("pcTopArticle", this.getPcTopArticle(startTime, endTime, projectId, contendId, resList));
summary.put("topArticle", this.getMobileTopArticle(startTime, endTime, projectId, contendId, resList));
summary.put("topEvent", this.getTopEvent(startTime, endTime, projectId, contendId));
summary.put("topSource", this.getTopSource(startTime, endTime, projectId, linkedGroupId, contendId));
summary.put("topSource", this.getTopSource(startTime, endTime, projectId, contendId));
resJson.put("summary", summary);
redisUtil.setExpire(redisKey, JSON.toJSONString(resJson));
} catch (IOException e) {
......@@ -400,12 +399,12 @@ public class IndexServiceImpl implements IndexService {
* @param resList 首发稿件列表
* @return 最热文章
*/
private JSONObject getPcTopArticle(long startTime, long endTime, String projectId, String linkedGroupId, String contendId, List<Map.Entry<String, Integer>> resList) throws IOException {
private JSONObject getPcTopArticle(long startTime, long endTime, String projectId, String contendId, List<Map.Entry<String, Integer>> resList) throws IOException {
JSONObject result = new JSONObject();
if (CollectionUtils.isNotEmpty(resList)) {
Map.Entry<String, Integer> resMap = resList.get(0);
String aggTitle = resMap.getKey();
BaseMap firstArticle = markDataService.getFirstArticle(startTime, endTime, aggTitle, projectId, linkedGroupId, contendId);
BaseMap firstArticle = markDataService.getFirstArticle(startTime, endTime, aggTitle, projectId, contendId);
result.put("title", firstArticle.getTitleNullOptionalContent());
result.put("url", firstArticle.getUrl());
result.put("time", firstArticle.getTime());
......@@ -425,13 +424,13 @@ public class IndexServiceImpl implements IndexService {
* @param contendId 竞品ID
* @return 最热文章
*/
private String getMobileTopArticle(long startTime, long endTime, String projectId, String linkedGroupId, String contendId, List<Map.Entry<String, Integer>> resList) throws IOException {
private String getMobileTopArticle(long startTime, long endTime, String projectId, String contendId, List<Map.Entry<String, Integer>> resList) throws IOException {
if (CollectionUtils.isEmpty(resList)) {
return "暂无最热文章。";
}
Map.Entry<String, Integer> resMap = resList.get(0);
String aggTitle = resMap.getKey();
BaseMap firstArticle = markDataService.getFirstArticle(startTime, endTime, aggTitle, projectId, linkedGroupId, contendId);
BaseMap firstArticle = markDataService.getFirstArticle(startTime, endTime, aggTitle, projectId, contendId);
SimpleDateFormat format = new SimpleDateFormat("M月d日");
return "《" + firstArticle.getTitleNullOptionalContent() + "》一文于" + format.format(firstArticle.getTime()) + "在" + firstArticle.getSource()
+ "（" + firstArticle.getPlatform() + "）上首发，相似文章数" + resMap.getValue() + "篇。";
......@@ -471,11 +470,10 @@ public class IndexServiceImpl implements IndexService {
* @param contendId 竞品ID
* @return 高频渠道
*/
private JSONObject getTopSource(long startTime, long endTime, String projectId, String linkedGroupId, String contendId) throws IOException {
private JSONObject getTopSource(long startTime, long endTime, String projectId, String contendId) throws IOException {
JSONObject result = new JSONObject();
// TODO 等待线上es数据格式调整
List<Map<String, Object>> positiveList = markDataService.getEsTopSource(startTime, endTime, projectId, linkedGroupId, contendId, EmotionEnum.POSITIVE.getName(), 3);
List<Map<String, Object>> negativeList = markDataService.getEsTopSource(startTime, endTime, projectId, linkedGroupId, contendId, EmotionEnum.NEGATIVE.getName(), 1);
List<Map<String, Object>> positiveList = markDataService.getEsTopSource(startTime, endTime, projectId, contendId, EmotionEnum.POSITIVE.getName(), 3);
List<Map<String, Object>> negativeList = markDataService.getEsTopSource(startTime, endTime, projectId, contendId, EmotionEnum.NEGATIVE.getName(), 1);
result.put("positiveList", positiveList);
result.put("negativeList", negativeList);
return result;
......
src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java
......@@ -5,6 +5,8 @@ import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.zhiwei.base.entity.subclass.mark.MarkInfo;
import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.common.GlobalPojo;
......@@ -18,8 +20,10 @@ import com.zhiwei.brandkbs2.enmus.ImportantChannelEnum;
import com.zhiwei.brandkbs2.es.EsClientDao;
import com.zhiwei.brandkbs2.es.EsQueryTools;
import com.zhiwei.brandkbs2.exception.ExceptionCast;
import com.zhiwei.brandkbs2.function.ArticleFunction;
import com.zhiwei.brandkbs2.listener.ApplicationProjectListener;
import com.zhiwei.brandkbs2.model.CommonCodeEnum;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.*;
import com.zhiwei.brandkbs2.pojo.dto.ExportAppYuqingDTO;
import com.zhiwei.brandkbs2.pojo.dto.ExportSearchWholeDTO;
......@@ -34,7 +38,6 @@ import com.zhiwei.brandkbs2.util.MongoUtil;
import com.zhiwei.brandkbs2.util.RedisUtil;
import com.zhiwei.brandkbs2.util.TextUtil;
import com.zhiwei.brandkbs2.util.Tools;
import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform;
import com.zhiwei.tools.istarshine.IstarShineUtil;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
......@@ -64,7 +67,6 @@ import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.joda.time.Period;
import org.joda.time.PeriodType;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
......@@ -101,10 +103,7 @@ public class MarkDataServiceImpl implements MarkDataService {
@Value("${whole.search.url}")
private String wholeSearchUrl;
@Value("${whole.searchCriteria.url}")
private String wholeSearchCriteriaUrl;
@Autowired
@Resource(name = "restTemplate")
private RestTemplate restTemplate;
@Resource(name = "projectServiceImpl")
......@@ -113,6 +112,9 @@ public class MarkDataServiceImpl implements MarkDataService {
@Resource(name = "esClientDao")
private EsClientDao esClientDao;
@Resource(name = "hotEsDao")
private EsClientDao hotEsDao;
@Resource(name = "commonServiceImpl")
private CommonService commonService;
......@@ -140,21 +142,28 @@ public class MarkDataServiceImpl implements MarkDataService {
@Resource(name = "mongoUtil")
MongoUtil mongoUtil;
@Autowired
@Resource(name = "textUtil")
TextUtil textUtil;
@Resource(name = "esSearchServiceImpl")
EsSearchService esSearchService;
@Override
public PageVO<MarkFlowEntity> getYuqingMarkList(MarkSearchDTO markSearchDTO) {
public PageVO<MarkFlowEntity> getOriginList(MarkSearchDTO markSearchDTO) {
try {
//设置默认的时间、页码、排序方式等
// 时间范围默认当天
if (Objects.isNull(markSearchDTO.getStartTime()) || Objects.isNull(markSearchDTO.getEndTime())) {
Date now = new Date();
markSearchDTO.setEndTime(now.getTime());
markSearchDTO.setStartTime(DateUtils.addDays(now, -1).getTime());
}
defaultMarkSearch(markSearchDTO);
// 搜索es数据
Pair<SearchHits[], Map<String, Long>> hitsAndCounts = searchMarkHitsAndCount(markSearchDTO, false);
Pair<SearchHits[], Map<String, Long>> hitsAndCounts = esSearchService.searchOriginHitsAndCount(markSearchDTO, false);
// 总量
// long total = hitsAndCounts.getLeft()[0].getTotalHits().value > 10000 ? 10000 : hitsAndCounts.getLeft()[0].getTotalHits().value;
long total = hitsAndCounts.getLeft()[0].getTotalHits().value;
// 消息列表
List<MarkFlowEntity> flowEntityList = getMarkFlowEntity(markSearchDTO, hitsAndCounts.getLeft()[0]);
List<MarkFlowEntity> flowEntityList = getMarkFlowEntity4Origin(markSearchDTO, hitsAndCounts.getLeft()[0]);
// 返回分页结果并设置平台count
return PageVO.createPageVo(total, markSearchDTO.getPage(), markSearchDTO.getPageSize(), flowEntityList)
// 各平台计量
......@@ -166,19 +175,81 @@ public class MarkDataServiceImpl implements MarkDataService {
}
@Override
public Pair<String, List<ExportAppYuqingDTO>> downloadYuqingMarkList(MarkSearchDTO markSearchDTO) {
public String generateOriginAggreeList(Long startTime, Long endTime) {
// 时间范围默认当天
if (Objects.isNull(startTime) || Objects.isNull(endTime)) {
Date now = new Date();
startTime = now.getTime();
endTime = DateUtils.addDays(now, -1).getTime();
}
return generateAggreeList(UserThreadLocal.getProjectId(), startTime, endTime, esSearchService::searchOriginHitsAndCount);
}
@Override
public PageVO<MarkFlowEntity> getOriginAggreeList(MarkSearchDTO markSearchDTO) {
// 时间范围默认当天
if (Objects.isNull(markSearchDTO.getStartTime()) || Objects.isNull(markSearchDTO.getEndTime())) {
Date now = new Date();
markSearchDTO.setEndTime(now.getTime());
markSearchDTO.setStartTime(DateUtils.addDays(now, -1).getTime());
}
return getAggreeList(markSearchDTO, markFlowService::createMarkFlowInfo4Origin, true);
}
@Override
public Pair<String, List<ExportAppYuqingDTO>> downloadOriginList(MarkSearchDTO markSearchDTO) {
return downloadList(markSearchDTO, esSearchService::searchOriginHitsAndCount);
}
@Override
public void addMtag(String zip, String linkedGroupId, String mtag) {
JSONObject markInfoJson = JSONObject.parseObject(Tools.gunzip(zip));
String nickname = UserThreadLocal.getNickname();
commonService.addMtag(markFlowService.markInfoFromJson(markInfoJson, GlobalPojo.getYuQingProjectNameById(linkedGroupId), nickname, mtag));
}
@Override
public ResponseResult addMtagBatch(List<String> ids, String linkedGroupId, String mtag) {
try {
ProjectVO project = projectService.getProjectVOById(UserThreadLocal.getProjectId());
List<ExportAppYuqingDTO> returnList = new ArrayList<>();
defaultMarkSearch(markSearchDTO);
markSearchDTO.setPageSize(null);
Pair<SearchHits[], Map<String, Long>> hitsAndCounts = searchMarkHitsAndCount(markSearchDTO, false);
for (SearchHits searchHits : hitsAndCounts.getLeft()) {
List<ExportAppYuqingDTO> collect = Arrays.stream(searchHits.getHits()).map(SearchHit::getSourceAsMap).map(ExportAppYuqingDTO::createFromEsMap).collect(Collectors.toList());
returnList.addAll(collect);
String nickname = UserThreadLocal.getNickname();
// 单条实时入库,其他延迟入库
JSONObject markInfoJson = hotEsDao.searchById(ids.get(0));
commonService.addMtag(markFlowService.markInfoFromJson(markInfoJson, GlobalPojo.getYuQingProjectNameById(linkedGroupId), nickname, mtag));
if (ids.size() > 1) {
ids.remove(0);
ApplicationProjectListener.getThreadPool().execute(() -> {
for (List<String> list : Lists.partition(ids, 100)) {
try {
List<MarkInfo> collect = hotEsDao.searchByIds(list).values().stream().map(json -> markFlowService.markInfoFromJson(json,
GlobalPojo.getYuQingProjectNameById(linkedGroupId), nickname, mtag)).collect(Collectors.toList());
commonService.addMtagBatch(collect);
} catch (IOException e) {
log.error("addMtagBatch-", e);
}
}
});
}
} catch (Exception e) {
return ResponseResult.failure("操作失败");
}
return ResponseResult.success();
}
return Pair.of(project.getBrandName() + "_" + markSearchDTO.getStartTime() + "_" + markSearchDTO.getEndTime(), returnList);
@Override
public PageVO<MarkFlowEntity> getYuqingMarkList(MarkSearchDTO markSearchDTO) {
try {
//设置默认的时间、页码、排序方式等
defaultMarkSearch(markSearchDTO);
// 搜索es数据
Pair<SearchHits[], Map<String, Long>> hitsAndCounts = esSearchService.searchMarkHitsAndCount(markSearchDTO, false);
// 总量
long total = hitsAndCounts.getLeft()[0].getTotalHits().value;
// 消息列表
List<MarkFlowEntity> flowEntityList = getMarkFlowEntity(markSearchDTO, hitsAndCounts.getLeft()[0]);
// 返回分页结果并设置平台count
return PageVO.createPageVo(total, markSearchDTO.getPage(), markSearchDTO.getPageSize(), flowEntityList)
// 各平台计量
.setInfo(new JSONObject(ImmutableMap.of("platformCount", hitsAndCounts.getRight())));
} catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.FAIL, "es查询异常");
}
......@@ -186,138 +257,24 @@ public class MarkDataServiceImpl implements MarkDataService {
}
@Override
public Pair<String, List<ExportAppYuqingDTO>> downloadYuqingMarkList(MarkSearchDTO markSearchDTO) {
return downloadList(markSearchDTO, esSearchService::searchMarkHitsAndCount);
}
@Override
public String generateYuqingMarkAggreeList(Long startTime, Long endTime) {
String uuid = Tools.getUUID();
String projectId = UserThreadLocal.getProjectId();
MarkSearchDTO markSearchDTO = defaultMarkSearch4Aggree(startTime, endTime);
ApplicationProjectListener.getThreadPool().execute(() -> {
String redisKey = RedisUtil.getAggreeCacheKey(uuid, projectId);
try {
redisUtil.setExpire(redisKey, "0");
// 搜索es数据
Pair<SearchHits[], Map<String, Long>> hitsAndCounts = searchMarkHitsAndCount(markSearchDTO, true);
log.info("taskId:{},聚合前数据:{}条", uuid, hitsAndCounts.getLeft()[0].getTotalHits().value);
// 聚合分组后数据
List<List<JSONObject>> groupList = aggregation(hitsAndCounts.getLeft(), "id");
log.info("taskId:{},聚合完成,聚合组:{}个", uuid, groupList.size());
// 入mongo缓存
cacheAggreeResult(groupList, uuid, projectId);
redisUtil.setExpire(redisKey, "100");
} catch (Exception e) {
log.error("generateYuqingMarkAggreeList-", e);
redisUtil.remove(redisKey);
}
});
return uuid;
return generateAggreeList(UserThreadLocal.getProjectId(), startTime, endTime, esSearchService::searchMarkHitsAndCount);
}
@Override
public String getYuqingMarkAggreeProgress(String id) {
public String getAggreeProgress(String id) {
String redisKey = RedisUtil.getAggreeCacheKey(id, UserThreadLocal.getProjectId());
return redisUtil.get(redisKey);
}
@Override
public PageVO<MarkFlowEntity> getYuqingMarkAggreeList(MarkSearchDTO dto) {
if (null == dto.getAggreeId()) {
ExceptionCast.cast(CommonCodeEnum.INVALID_PARAM);
}
defaultMarkSearch(dto);
Query query = assembleAggreeQuery(dto);
// 量查询
long count = aggreeResultDao.count(query);
// 开启分页
mongoUtil.start(dto.getPage(), dto.getPageSize(), query);
// 排序
aggreeResultDao.addSort(query, "{\"aggreeSize\":\"descend\"}");
// 返回结果
List<MarkFlowEntity> resList = new ArrayList<>();
List<AggreeResult> fatherList = aggreeResultDao.findList(query);
// List<String> fatherIds = fatherList.stream().map(AggreeResult::getFatherId).collect(Collectors.toList());
// Map<String, List<AggreeResult>> fatherMap = new HashMap<>();
// for (AggreeResult aggreeResult : aggreeResultDao.findList(Query.query(Criteria.where("fatherId").in(fatherIds)))) {
// fatherMap.compute(aggreeResult.getFatherId(), (k, v) -> {
// if (null == v) {
// v = new ArrayList<>();
// }
// v.add(aggreeResult);
// return v;
// });
// }
// Map<String, List<AggreeResult>> fatherMap = aggreeResultDao.findList(Query.query(Criteria.where("fatherId").in(fatherIds))).stream().collect(Collectors.groupingBy(AggreeResult::getFatherId));
for (AggreeResult aggreeResult : fatherList) {
List<MarkFlowEntity> markFlowEntities = new ArrayList<>();
MarkFlowEntity instance = new MarkFlowEntity(aggreeResult.getData());
instance.setInfo(markFlowService.createMarkFlowInfo(aggreeResult.getData(), dto.getProjectId(), dto.getLinkedGroupId(), dto.getContendId()));
markFlowEntities.add(instance);
// 添加父标题集下的子标题集
// markFlowEntities.addAll(fatherMap.get(aggreeResult.getFatherId()).stream().map(sonResult -> new MarkFlowEntity(sonResult.getData())).collect(Collectors.toList()));
resList.add(MarkFlowEntity.getAggreeInstance(markFlowEntities, aggreeResult.getAggreeSize()));
}
return PageVO.createPageVo(count, dto.getPage(), dto.getPageSize(), resList);
}
private Query assembleAggreeQuery(MarkSearchDTO dto) {
Query query = new Query();
query.addCriteria(Criteria.where("taskId").is(dto.getAggreeId()));
// 平台
if (CollectionUtils.isNotEmpty(dto.getPlatforms())) {
Criteria orCriteria = new Criteria();
List<Criteria> platformCriteria = new ArrayList<>();
for (String platformId : dto.getPlatforms()) {
platformCriteria.add(Criteria.where("platformId").is(platformId));
}
query.addCriteria(orCriteria.orOperator(platformCriteria.toArray(new Criteria[0])));
}
// 媒体类型
if (CollectionUtils.isNotEmpty(dto.getMediaTypes())) {
Criteria orCriteria = new Criteria();
List<Criteria> mediaCriteria = new ArrayList<>();
for (String mediaType : dto.getMediaTypes()) {
mediaCriteria.add(Criteria.where("mediaTypes").is(mediaType));
}
query.addCriteria(orCriteria.orOperator(mediaCriteria.toArray(new Criteria[0])));
}
// 情感调性
if (CollectionUtils.isNotEmpty(dto.getTags())) {
Criteria orCriteria = new Criteria();
List<Criteria> tagCriteria = new ArrayList<>();
for (String tag : dto.getTags()) {
tagCriteria.add(Criteria.where("tags").is(tag));
}
query.addCriteria((orCriteria.orOperator(tagCriteria.toArray(new Criteria[0]))));
}
// 关键词模糊查询
aggreeResultDao.addKeywordFuzz(query, dto.getKeyword(), "data.source");
return query;
}
private void cacheAggreeResult(List<List<JSONObject>> groupList, String taskId, String projectId) {
int count = 0;
int percent;
for (List<JSONObject> list : groupList) {
String fatherId = null;
String fatherTitle = null;
for (int i = 0; i < list.size(); i++) {
JSONObject json = list.get(i);
AggreeResult aggreeResult;
// 父聚合处理
if (i == 0) {
aggreeResult = AggreeResult.createFather(json, taskId, projectId, list.size());
aggreeResultDao.insertOneWithoutId(aggreeResult);
fatherId = aggreeResult.getId();
fatherTitle = aggreeResult.getAggreeTitle();
} else {
aggreeResult = AggreeResult.createSon(json, taskId, projectId, fatherId, fatherTitle);
aggreeResultDao.insertOneWithoutId(aggreeResult);
}
}
// if ((percent = (++count / list.size())) % (10 / list.size()) == 0) {
// log.info("taskId:{},fatherId:{} 入库完毕,进度:{}%", taskId, fatherId, percent);
// }
}
return getAggreeList(dto, markFlowService::createMarkFlowInfo, false);
}
@Override
......@@ -390,7 +347,6 @@ public class MarkDataServiceImpl implements MarkDataService {
startTime = timeRange[0];
endTime = timeRange[1];
String projectId = UserThreadLocal.getProjectId();
String linkedGroupId = projectService.getProjectVOById(projectId).getBrandLinkedGroupId();
String redisKey = RedisKeyPrefix.MARK_ANALYZE_SUMMARY + Tools.concat(projectId, startTime, endTime);
String resultStr;
// 返回缓存
......@@ -400,19 +356,19 @@ public class MarkDataServiceImpl implements MarkDataService {
// 返回值
JSONObject result = new JSONObject();
// 添加各情感发文数统计
emotionAnalyzeAdd(startTime, endTime, projectId, linkedGroupId, result);
emotionAnalyzeAdd(startTime, endTime, projectId, result);
// 获取标注数据传播趋势
List<JSONObject> markSpread = getMarkSpread(startTime, endTime, projectId, true);
markSpread.stream().max(Comparator.comparing(o -> o.getInteger("posNormalCount"))).ifPresent(e -> result.put("posMaxTime", e.getDate("time")));
// 获取所有平台稿件倾向稿件数量信息
List<JSONObject> platformsCount = getPlatformMarkEmotionCount(startTime, endTime, projectId, linkedGroupId);
List<JSONObject> platformsCount = getPlatformMarkEmotionCount(startTime, endTime, projectId);
// 正面舆情峰值平台
platformsCount.stream().max(Comparator.comparing(o -> o.getInteger(EmotionEnum.POSITIVE.getName()))).ifPresent(e -> result.put("posTopPlatform", GlobalPojo.getPlatformNameById(e.getString("platform_id"))));
// 负面舆情峰值平台
platformsCount.stream().max(Comparator.comparing(o -> o.getInteger(EmotionEnum.NEGATIVE.getName()))).ifPresent(e -> result.put("negTopPlatform", GlobalPojo.getPlatformNameById(e.getString("platform_id"))));
// 获取时间段正面情感数据最多的标题
String topTitle = "";
List<Map.Entry<String, Integer>> markTopTitle = getMarkTopTitle(startTime, endTime, EmotionEnum.POSITIVE.getName(), projectId, linkedGroupId, "0", 1);
List<Map.Entry<String, Integer>> markTopTitle = getMarkTopTitle(startTime, endTime, EmotionEnum.POSITIVE.getName(), projectId, "0", 1);
if (!markTopTitle.isEmpty()) {
topTitle = markTopTitle.get(0).getKey();
}
......@@ -421,7 +377,7 @@ public class MarkDataServiceImpl implements MarkDataService {
markSpread.stream().max(Comparator.comparing(o -> o.getInteger("negNormalCount"))).ifPresent(e -> result.put("negMaxTime", e.getLong("time")));
// 获取时间段负面情感数据最多的标题
topTitle = "";
List<Map.Entry<String, Integer>> negTopTitle = getMarkTopTitle(startTime, endTime, EmotionEnum.NEGATIVE.getName(), projectId, linkedGroupId, "0", 1);
List<Map.Entry<String, Integer>> negTopTitle = getMarkTopTitle(startTime, endTime, EmotionEnum.NEGATIVE.getName(), projectId, "0", 1);
if (!negTopTitle.isEmpty()) {
topTitle = negTopTitle.get(0).getKey();
}
......@@ -455,7 +411,6 @@ public class MarkDataServiceImpl implements MarkDataService {
startTime = timeRange[0];
endTime = timeRange[1];
String projectId = UserThreadLocal.getProjectId();
String linkedGroupId = projectService.getProjectVOById(projectId).getBrandLinkedGroupId();
String redisKey = RedisKeyPrefix.MARK_EMOTION_PROPORTION + Tools.concat(projectId, startTime, endTime);
// 返回缓存
String resultStr;
......@@ -464,8 +419,8 @@ public class MarkDataServiceImpl implements MarkDataService {
}
long lastStartTime = startTime - (endTime - startTime);
List<String> emotionList = Arrays.asList(EmotionEnum.POSITIVE.getName(), EmotionEnum.NEUTRAL.getName(), EmotionEnum.NEGATIVE.getName());
List<JSONObject> curEmotions = getMarkEmotionsCount(startTime, endTime, null, null, projectId, linkedGroupId, emotionList);
List<JSONObject> oldEmotions = getMarkEmotionsCount(lastStartTime, startTime, null, null, projectId, linkedGroupId, emotionList);
List<JSONObject> curEmotions = getMarkEmotionsCount(startTime, endTime, null, null, projectId, emotionList);
List<JSONObject> oldEmotions = getMarkEmotionsCount(lastStartTime, startTime, null, null, projectId, emotionList);
JSONObject result = new JSONObject();
result.put("curCount", curEmotions.stream().mapToLong(emotion -> emotion.getLongValue("count")).sum());
result.put("curPosCount", curEmotions.get(0).getLongValue("count"));
......@@ -489,7 +444,6 @@ public class MarkDataServiceImpl implements MarkDataService {
startTime = timeRange[0];
endTime = timeRange[1];
String projectId = UserThreadLocal.getProjectId();
String linkedGroupId = projectService.getProjectVOById(projectId).getBrandLinkedGroupId();
String redisKey = RedisKeyPrefix.MARK_CHANNEL_PROPORTION + Tools.concat(projectId, startTime, endTime);
String resultStr;
if (cache && StringUtils.isNotEmpty(resultStr = redisUtil.get(redisKey))) {
......@@ -498,9 +452,9 @@ public class MarkDataServiceImpl implements MarkDataService {
JSONObject result = new JSONObject();
try {
// 渠道情感倾向占比
result.put("channelEmotionPercent", getChannelEmotionPercent(projectId, linkedGroupId, startTime, endTime));
result.put("channelEmotionPercent", getChannelEmotionPercent(projectId, startTime, endTime));
// 渠道标签占比
result.put("importantChannelPercent", getImportantChannelPercent(projectId, linkedGroupId, startTime, endTime));
result.put("importantChannelPercent", getImportantChannelPercent(projectId, startTime, endTime));
} catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.FAIL, "es查询异常", e);
}
......@@ -509,18 +463,17 @@ public class MarkDataServiceImpl implements MarkDataService {
}
@Override
public List<JSONObject> getMarkPlatformProportion(Long startTime, Long endTime, boolean cache) {
return getMarkPlatformProportion(startTime, endTime, null, cache);
public List<JSONObject> getMarkPlatformProportion(String projectId, Long startTime, Long endTime, boolean cache) {
return getMarkPlatformProportion(projectId, startTime, endTime, null, cache);
}
@Override
public List<JSONObject> getMarkPlatformProportion(Long startTime, Long endTime, String emotion, boolean cache) {
public List<JSONObject> getMarkPlatformProportion(String projectId, Long startTime, Long endTime, String emotion, boolean cache) {
List<JSONObject> res = null;
try {
Long[] timeRange = Tools.formatTimeRange(startTime, endTime);
startTime = timeRange[0];
endTime = timeRange[1];
String projectId = UserThreadLocal.getProjectId();
// 舆情库默认contendId为0
res = getMarkPlatformProportion(startTime, endTime, projectId, Constant.PRIMARY_CONTEND_ID, emotion, cache);
} catch (IOException e) {
......@@ -536,14 +489,13 @@ public class MarkDataServiceImpl implements MarkDataService {
startTime = timeRange[0];
endTime = timeRange[1];
String projectId = UserThreadLocal.getProjectId();
String linkedGroupId = projectService.getProjectVOById(projectId).getBrandLinkedGroupId();
String redisKey = RedisKeyPrefix.MARK_HIGH_WORD + Tools.concat(projectId, startTime, endTime);
String resultStr = redisUtil.get(redisKey);
if (StringUtils.isNotEmpty(resultStr)) {
return JSON.parseArray(resultStr, JSONObject.class);
}
long s = System.currentTimeMillis();
List<String> texts = getMarkTitleAndContent(startTime, endTime, projectId, linkedGroupId);
List<String> texts = getMarkTitleAndContent(startTime, endTime, projectId);
log.info("es查询size:{},耗时:{}", texts.size(), System.currentTimeMillis() - s);
long s1 = System.currentTimeMillis();
// 分析高频词
......@@ -552,7 +504,7 @@ public class MarkDataServiceImpl implements MarkDataService {
redisUtil.setExpire(redisKey, JSON.toJSONString(highWords));
return highWords;
} catch (IOException e) {
log.error("getMarkHighWord-",e);
log.error("getMarkHighWord-", e);
ExceptionCast.cast(CommonCodeEnum.FAIL);
}
return null;
......@@ -609,156 +561,25 @@ public class MarkDataServiceImpl implements MarkDataService {
}
}
private EsClientDao.SearchHelper createSearchHelperByMarkSearchDTO(MarkSearchDTO dto) {
EsClientDao.SearchHelper helper = EsClientDao.createSearchHelper();
// linkedGroupId、projectId、contendId
String projectId = dto.getProjectId();
String linkedGroupId = dto.getLinkedGroupId();
String contendId = dto.getContendId();
// PostFilter 后置过滤器
// BoolQueryBuilder postFilter = projectLinkedGroupQuery(projectId, linkedGroupId);
// 新增channelEmotion
BoolQueryBuilder postFilter = EsQueryTools.assembleCacheMapsQuery(projectId, linkedGroupId, contendId, dto.getMediaTypes(), dto.getChannelEmotion());
// time
postFilter.must(QueryBuilders.rangeQuery(dto.getTimeType()).gte(dto.getStartTime()).lt(dto.getEndTime()));
// platform
if (CollectionUtils.isNotEmpty(dto.getPlatforms())) {
postFilter.must(EsQueryTools.assemblePlatformQuery(Tools.getPlatformByIds(dto.getPlatforms())));
}
// tags
if (CollectionUtils.isNotEmpty(dto.getTags())) {
postFilter.must(EsQueryTools.assembleTagQuery(dto.getTags()));
}
// customTags
if (CollectionUtils.isNotEmpty(dto.getCustomTags())) {
postFilter.must(EsQueryTools.assembleTagQuery(dto.getCustomTags()));
}
// Query 查询条件
// BoolQueryBuilder query = QueryBuilders.boolQuery();
// keyword
if (StringUtils.isNotEmpty(dto.getKeyword())) {
// Query
String[] fieldSearch = "仅标题".equals(dto.getSearchField()) ? new String[]{GenericAttribute.ES_IND_TITLE} : new String[]{GenericAttribute.ES_IND_FULL_TEXT};
postFilter.must(EsQueryTools.assembleNormalKeywordQuery(dto.getKeyword(), fieldSearch));
}
// sourceKeyword
if (StringUtils.isNotEmpty(dto.getSourceKeyword())) {
postFilter.must(EsQueryTools.assembleSourceQuery(dto.getSourceKeyword()));
}
// hostKeyword
if (StringUtils.isNotEmpty(dto.getHostKeyword())) {
postFilter.must(EsQueryTools.assembleFiledKeywordQuery("host", dto.getHostKeyword()));
}
helper.setPostFilter(postFilter);
// helper.setQuery(query);
// sort
FieldSortBuilder sort = null;
if (null != dto.getSorter()) {
for (Map.Entry<String, Object> entry : dto.getSorter().entrySet()) {
String key = entry.getKey();
if (key.equals("influence")) {
key = "channel_influence";
} else if (key.equals("followers")) {
key = "channel_followers";
}
if (null == entry.getValue() || entry.getValue().toString().contains("desc")) {
sort = SortBuilders.fieldSort(key).order(SortOrder.DESC);
} else {
sort = SortBuilders.fieldSort(key).order(SortOrder.ASC);
}
}
}
helper.setSort(sort);
// from size
if (null != dto.getPageSize()) {
helper.setFrom((dto.getPage() - 1) * dto.getPageSize());
helper.setSize(dto.getPageSize());
}
// HighlightBuilder ???
return helper;
}
private Pair<SearchHits[], Map<String, Long>> searchMarkHitsAndCount(MarkSearchDTO dto, boolean aggree) throws IOException {
EsClientDao.SearchHelper helper = createSearchHelperByMarkSearchDTO(dto);
if (aggree) {
List<SearchResponse> searchResponses = esClientDao.searchScrollResponse(helper);
// <SearchResponse> searchResponses = Collections.singletonList(esClientDao.searchResponse(helper));
return Pair.of(searchResponses.stream().map(SearchResponse::getHits).toArray(SearchHits[]::new), null);
}
SearchHits searchHits = esClientDao.searchHits(helper);
//平台计量统计 仅第一页且无平台限制
Map<String, Long> counts = new HashMap<>();
if (1 == dto.getPage() && CollectionUtils.isEmpty(dto.getPlatforms())) {
for (MessagePlatform platform : GlobalPojo.PLATFORMS) {
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery().must(helper.getPostFilter()).must(EsQueryTools.assemblePlatformQuery(Collections.singletonList(platform)));
Long count = esClientDao.count(esClientDao.getIndexes(), queryBuilder, null);
counts.put(platform.getName(), count);
}
}
return Pair.of(new SearchHits[]{searchHits}, counts);
}
private List<List<JSONObject>> aggregation(SearchHits[] searchHitsList, String uniqueField) {
private List<List<JSONObject>> aggregation(SearchHits[] searchHitsList) {
List<SearchHit> hitList = new ArrayList<>();
for (SearchHits searchHits : searchHitsList) {
hitList.addAll(Arrays.stream(searchHits.getHits()).collect(Collectors.toList()));
}
Map<String, String> idTitle = hitList.stream().collect(Collectors.toMap(SearchHit::getId, hit -> Tools.getBaseFromEsMap(hit.getSourceAsMap()).getTitleNullOptionalContent()));
List<List<String>> kResult = TextUtil.getKResult(idTitle);
// return TextUtil.restoreThenSort(kResult, hitList.stream().map(hit -> new JSONObject(hit.getSourceAsMap())).collect(Collectors.toList()),
// uniqueField);
return TextUtil.restoreThenSort(kResult, hitList);
}
// private List<JSONObject> criteriaTime(Date endDate) {
// List<JSONObject> times = new ArrayList<>();
// for (String criteriaTime : Arrays.asList("今天", "24小时", "三天", "七天", "近30天")) {
// JSONObject time = new JSONObject();
// switch (criteriaTime) {
// case "今天":
// time.put("name", criteriaTime);
// time.put("startTime", Tools.truncDate(endDate, DAY_PATTERN).getTime());
// time.put("endTime", endDate.getTime());
// times.add(time);
// break;
// case "24小时":
// time.put("name", criteriaTime);
// time.put("startTime", DateUtils.addHours(endDate, -24).getTime());
// time.put("endTime", endDate.getTime());
// times.add(time);
// break;
// case "三天":
// time.put("name", criteriaTime);
// time.put("startTime", DateUtils.addDays(Tools.truncDate(endDate, DAY_PATTERN), -2).getTime());
// time.put("endTime", endDate.getTime());
// times.add(time);
// break;
// case "七天":
// time.put("name", criteriaTime);
// time.put("startTime", DateUtils.addDays(Tools.truncDate(endDate, DAY_PATTERN), -6).getTime());
// time.put("endTime", endDate.getTime());
// times.add(time);
// break;
// case "近30天":
// time.put("name", criteriaTime);
// time.put("startTime", DateUtils.addDays(Tools.truncDate(endDate, DAY_PATTERN), -29).getTime());
// time.put("endTime", endDate.getTime());
// times.add(time);
// break;
// }
// }
// return times;
// }
private void emotionAnalyzeAdd(long startTime, long endTime, String projectId, String linkedGroupId, JSONObject result) throws IOException {
private void emotionAnalyzeAdd(long startTime, long endTime, String projectId, JSONObject result) throws IOException {
long lastStartTime = startTime - (endTime - startTime);
List<String> emotionList = Arrays.asList(EmotionEnum.POSITIVE.getName(), EmotionEnum.NEGATIVE.getName(), EmotionEnum.NEUTRAL.getName());
List<JSONObject> curArticleEmotions = getMarkEmotionsCount(startTime, endTime, null, null, projectId, linkedGroupId, emotionList);
List<JSONObject> curArticleEmotions = getMarkEmotionsCount(startTime, endTime, null, null, projectId, emotionList);
int curPosCount = curArticleEmotions.get(0).getIntValue("count");
int curNegCount = curArticleEmotions.get(1).getIntValue("count");
int curNeuCount = curArticleEmotions.get(2).getIntValue("count");
List<JSONObject> lastArticleEmotions = getMarkEmotionsCount(lastStartTime, startTime, null, null, projectId, linkedGroupId, emotionList);
List<JSONObject> lastArticleEmotions = getMarkEmotionsCount(lastStartTime, startTime, null, null, projectId, emotionList);
int lastPosCount = lastArticleEmotions.get(0).getIntValue("count");
int lastNegCount = lastArticleEmotions.get(1).getIntValue("count");
int lastNeuCount = lastArticleEmotions.get(2).getIntValue("count");
......@@ -774,22 +595,21 @@ public class MarkDataServiceImpl implements MarkDataService {
/**
* 获取所有稿件倾向稿件数量信息
*
* @param startTime 开始时间
* @param endTime 结束时间
* @param keyword 关键词
* @param searchField 检索字段
* @param projectId 项目ID
* @param linkedGroupId 关联组id
* @param emotionList 情感倾向集合
* @param startTime 开始时间
* @param endTime 结束时间
* @param keyword 关键词
* @param searchField 检索字段
* @param projectId 项目ID
* @param emotionList 情感倾向集合
* @return 所有稿件倾向稿件数量信息
*/
private List<JSONObject> getMarkEmotionsCount(Long startTime, Long endTime, String keyword, String searchField, String projectId, String linkedGroupId, List<String> emotionList) throws IOException {
private List<JSONObject> getMarkEmotionsCount(Long startTime, Long endTime, String keyword, String searchField, String projectId, List<String> emotionList) throws IOException {
// 获取索引
String[] indexes = esClientDao.getIndexes();
// 聚合查询
TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("count").field("brandkbs_mark_cache_maps.name.keyword");
// query
BoolQueryBuilder query = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, "0");
BoolQueryBuilder query = projectContendIdQuery(projectId, "0");
// keyword
if (StringUtils.isNotEmpty(keyword)) {
String[] fieldSearch = "标题".equals(searchField) ? new String[]{GenericAttribute.ES_IND_TITLE} : new String[]{GenericAttribute.ES_IND_FULL_TEXT};
......@@ -812,9 +632,66 @@ public class MarkDataServiceImpl implements MarkDataService {
return resultList;
}
public List<JSONObject> getMarkSpread(Long startTime, Long endTime, String projectId, boolean cache) throws IOException {
ProjectVO project = projectService.getProjectVOById(projectId);
String linkedGroupId = project.getBrandLinkedGroupId();
private Query assembleAggreeQuery(MarkSearchDTO dto) {
Query query = new Query();
query.addCriteria(Criteria.where("taskId").is(dto.getAggreeId()));
// 仅返回父聚合集
query.addCriteria(Criteria.where("isFather").is(true));
// 平台
if (CollectionUtils.isNotEmpty(dto.getPlatforms())) {
Criteria orCriteria = new Criteria();
List<Criteria> platformCriteria = new ArrayList<>();
for (String platformId : dto.getPlatforms()) {
platformCriteria.add(Criteria.where("platformId").is(platformId));
}
query.addCriteria(orCriteria.orOperator(platformCriteria.toArray(new Criteria[0])));
}
// 媒体类型
if (CollectionUtils.isNotEmpty(dto.getMediaTypes())) {
Criteria orCriteria = new Criteria();
List<Criteria> mediaCriteria = new ArrayList<>();
for (String mediaType : dto.getMediaTypes()) {
mediaCriteria.add(Criteria.where("mediaTypes").is(mediaType));
}
query.addCriteria(orCriteria.orOperator(mediaCriteria.toArray(new Criteria[0])));
}
// 情感调性
if (CollectionUtils.isNotEmpty(dto.getTags())) {
Criteria orCriteria = new Criteria();
List<Criteria> tagCriteria = new ArrayList<>();
for (String tag : dto.getTags()) {
tagCriteria.add(Criteria.where("tags").is(tag));
}
query.addCriteria((orCriteria.orOperator(tagCriteria.toArray(new Criteria[0]))));
}
// 关键词模糊查询
aggreeResultDao.addKeywordFuzz(query, dto.getKeyword(), "data.source");
return query;
}
private void cacheAggreeResult(List<List<JSONObject>> groupList, String taskId, String projectId) {
for (List<JSONObject> list : groupList) {
String fatherId = null;
String fatherTitle = null;
for (int i = 0; i < list.size(); i++) {
JSONObject json = list.get(i);
AggreeResult aggreeResult;
// 父聚合处理
if (i == 0) {
aggreeResult = AggreeResult.createFather(json, taskId, projectId, list);
aggreeResultDao.insertOneWithoutId(aggreeResult);
fatherId = aggreeResult.getId();
fatherTitle = aggreeResult.getAggreeTitle();
} else {
aggreeResult = AggreeResult.createSon(json, taskId, projectId, fatherId, fatherTitle);
aggreeResultDao.insertOneWithoutId(aggreeResult);
}
}
}
}
private List<JSONObject> getMarkSpread(Long startTime, Long endTime, String projectId, boolean cache) throws IOException {
Long[] timeRange = Tools.formatTimeRange(startTime, endTime);
startTime = timeRange[0];
endTime = timeRange[1];
......@@ -824,7 +701,7 @@ public class MarkDataServiceImpl implements MarkDataService {
return JSON.parseArray(resultStr, JSONObject.class);
}
List<JSONObject> lineList = new ArrayList<>();
List<JSONObject> result = getDayMarkEmotionCount(startTime, endTime, projectId, linkedGroupId);
List<JSONObject> result = getDayMarkEmotionCount(startTime, endTime, projectId);
result.forEach(day -> {
JSONObject line = new JSONObject();
line.put("normalCount", day.getIntValue("total"));
......@@ -838,33 +715,31 @@ public class MarkDataServiceImpl implements MarkDataService {
return lineList;
}
public List<Map.Entry<String, Integer>> getMarkTopTitle(Long startTime, Long endTime, String emotion, String projectId, String linkedGroupId, String contendId, int size) throws IOException {
return getMarkTopTitle(startTime, endTime, emotion, projectId, linkedGroupId, contendId, size, true);
@Override
public List<Map.Entry<String, Integer>> getMarkTopTitle(Long startTime, Long endTime, String emotion, String projectId, String contendId, int size) throws IOException {
return getMarkTopTitle(startTime, endTime, emotion, projectId, contendId, size, true);
}
/**
* 获取时间段某情感数据最多的标题
*
* @param startTime 开始时间
* @param endTime 结束时间
* @param emotion 情感倾向名称
* @param projectId 项目id
* @param linkedGroupId 关联组id
* @param size 分页大小
* @param include 是否包含主品牌
* @param startTime 开始时间
* @param endTime 结束时间
* @param emotion 情感倾向名称
* @param projectId 项目id
* @param size 分页大小
* @param include 是否包含主品牌
* @return 最热标题
*/
public List<Map.Entry<String, Integer>> getMarkTopTitle(Long startTime, Long endTime, String emotion, String projectId, String linkedGroupId, String contendId, int size, boolean include) throws IOException {
@Override
public List<Map.Entry<String, Integer>> getMarkTopTitle(Long startTime, Long endTime, String emotion, String projectId, String contendId, int size, boolean include) throws IOException {
// 索引
String[] indexes = esClientDao.getIndexes();
// TODO script使用
// Script script = new Script("params._source.ind_title");
// TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("titles").script(script).order(BucketOrder.count(false)).size(60000);
TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("titles").field("agg_title.keyword").size(size + 1);
// query
BoolQueryBuilder query;
if (include) {
query = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, contendId);
query = projectContendIdQuery(projectId, contendId);
} else {
query = EsQueryTools.assembleCacheMapsQueryExcludePrimaryId(projectId);
}
......@@ -890,18 +765,19 @@ public class MarkDataServiceImpl implements MarkDataService {
return result.entrySet().stream().sorted(Map.Entry.comparingByValue(Comparator.reverseOrder())).limit(size).collect(Collectors.toList());
}
public BaseMap getFirstArticle(Long startTime, Long endTime, String aggTitle, String projectId, String linkedGroupId, String contendId) throws IOException {
return getFirstArticle(startTime, endTime, aggTitle, projectId, linkedGroupId, contendId, true);
@Override
public BaseMap getFirstArticle(Long startTime, Long endTime, String aggTitle, String projectId, String contendId) throws IOException {
return getFirstArticle(startTime, endTime, aggTitle, projectId, contendId, true);
}
@Override
public BaseMap getFirstArticle(Long startTime, Long endTime, String aggTitle, String projectId, String linkedGroupId, String contendId, boolean include) throws IOException {
public BaseMap getFirstArticle(Long startTime, Long endTime, String aggTitle, String projectId, String contendId, boolean include) throws IOException {
// 索引
String[] indexes = esClientDao.getIndexes();
// postFilter
BoolQueryBuilder postFilter;
if (include) {
postFilter = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, contendId);
postFilter = projectContendIdQuery(projectId, contendId);
} else {
postFilter = EsQueryTools.assembleCacheMapsQueryExcludePrimaryId(projectId);
}
......@@ -913,29 +789,23 @@ public class MarkDataServiceImpl implements MarkDataService {
if (0 == hits.getTotalHits().value) {
return new BaseMap();
}
// JSONObject resultJSONObject = new JSONObject();
// BaseMap baseMap = Tools.getBaseFromEsMap(hit.getSourceAsMap());
// resultJSONObject.put("content", baseMap.getContent());
// resultJSONObject.put("url", baseMap.getUrl());
// resultJSONObject.put("realSource", baseMap.getRealSource());
return Tools.getBaseFromEsMap(hits.getAt(0).getSourceAsMap());
}
@Override
public List<JSONObject> searchMarkDataByEvent(Event event) {
String linkedGroupId = projectService.getProjectByContendId(event.getProjectId(), event.getContendId()).getBrandLinkedGroupId();
Long endTime = null;
if (event.isEndStatus()) {
endTime = event.getEndTime();
}
return searchMarkDataByTime(event.getProjectId(), linkedGroupId, event.getContendId(), event.getStartTime(), endTime, EVENT_FETCH_SOURCE);
return searchMarkDataByTime(event.getProjectId(), event.getContendId(), event.getStartTime(), endTime, EVENT_FETCH_SOURCE);
}
@Override
public List<JSONObject> searchMarkDataByTime(String projectId, String linkedGroupId, String contendId, Long startTime, Long endTime, String... fetchSource) {
public List<JSONObject> searchMarkDataByTime(String projectId, String contendId, Long startTime, Long endTime, String... fetchSource) {
try {
BoolQueryBuilder query = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, contendId);
BoolQueryBuilder query = projectContendIdQuery(projectId, contendId);
RangeQueryBuilder timeBuilder = QueryBuilders.rangeQuery("time").gte(startTime);
if (null != endTime) {
timeBuilder.lt(endTime);
......@@ -990,7 +860,7 @@ public class MarkDataServiceImpl implements MarkDataService {
try {
defaultMarkSearch(dto);
// 搜索es数据
Pair<SearchHits[], Map<String, Long>> hitsAndCounts = searchContendMarkHitsAndCount(dto, false);
Pair<SearchHits[], Map<String, Long>> hitsAndCounts = esSearchService.searchContendMarkHitsAndCount(dto, false);
// 总量
// long total = hitsAndCounts.getLeft()[0].getTotalHits().value > 10000 ? 10000 : hitsAndCounts.getLeft()[0].getTotalHits().value;
long total = hitsAndCounts.getLeft()[0].getTotalHits().value;
......@@ -1006,91 +876,6 @@ public class MarkDataServiceImpl implements MarkDataService {
return null;
}
private Pair<SearchHits[], Map<String, Long>> searchContendMarkHitsAndCount(MarkSearchDTO dto, boolean aggree) throws IOException {
EsClientDao.SearchHelper helper = createContendSearchHelper(dto);
if (aggree) {
List<SearchResponse> searchResponses = esClientDao.searchScrollResponse(helper);
// <SearchResponse> searchResponses = Collections.singletonList(esClientDao.searchResponse(helper));
return Pair.of(searchResponses.stream().map(SearchResponse::getHits).toArray(SearchHits[]::new), null);
}
SearchHits searchHits = esClientDao.searchHits(helper);
//平台计量统计 仅第一页且无平台限制
Map<String, Long> counts = new HashMap<>();
if (1 == dto.getPage() && CollectionUtils.isEmpty(dto.getPlatforms())) {
for (MessagePlatform platform : GlobalPojo.PLATFORMS) {
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery().must(helper.getPostFilter()).must(EsQueryTools.assemblePlatformQuery(Collections.singletonList(platform)));
Long count = esClientDao.count(esClientDao.getIndexes(), queryBuilder, null);
counts.put(platform.getName(), count);
}
}
return Pair.of(new SearchHits[]{searchHits}, counts);
}
private EsClientDao.SearchHelper createContendSearchHelper(MarkSearchDTO dto) {
EsClientDao.SearchHelper helper = EsClientDao.createSearchHelper();
// linkedGroupId、projectId、contendId
String projectId = dto.getProjectId();
String contendId = dto.getContendId();
String contendLinkedGroupId = projectService.getProjectByContendId(projectId, dto.getContendId()).getBrandLinkedGroupId();
// PostFilter 后置过滤器
BoolQueryBuilder postFilter = EsQueryTools.assembleCacheMapsQuery(projectId, contendLinkedGroupId, contendId, dto.getMediaTypes());
// time
postFilter.must(QueryBuilders.rangeQuery(dto.getTimeType()).gte(dto.getStartTime()).lt(dto.getEndTime()));
// platform
if (CollectionUtils.isNotEmpty(dto.getPlatforms())) {
postFilter.must(EsQueryTools.assemblePlatformQuery(Tools.getPlatformByIds(dto.getPlatforms())));
}
// tags
if (CollectionUtils.isNotEmpty(dto.getTags())) {
postFilter.must(EsQueryTools.assembleTagQuery(dto.getTags()));
}
// customTags
if (CollectionUtils.isNotEmpty(dto.getCustomTags())) {
postFilter.must(EsQueryTools.assembleTagQuery(dto.getCustomTags()));
}
// Query 查询条件
// BoolQueryBuilder query = QueryBuilders.boolQuery();
// keyword
if (StringUtils.isNotEmpty(dto.getKeyword())) {
// Query
String[] fieldSearch = "仅标题".equals(dto.getSearchField()) ? new String[]{GenericAttribute.ES_IND_TITLE} : new String[]{GenericAttribute.ES_IND_FULL_TEXT};
postFilter.must(EsQueryTools.assembleNormalKeywordQuery(dto.getKeyword(), fieldSearch));
}
// sourceKeyword
if (StringUtils.isNotEmpty(dto.getSourceKeyword())) {
postFilter.must(EsQueryTools.assembleSourceQuery(dto.getSourceKeyword()));
}
// hostKeyword
if (StringUtils.isNotEmpty(dto.getHostKeyword())) {
postFilter.must(EsQueryTools.assembleFiledKeywordQuery("host", dto.getHostKeyword()));
}
helper.setPostFilter(postFilter);
// helper.setQuery(query);
// sort
FieldSortBuilder sort = null;
if (null != dto.getSorter()) {
for (Map.Entry<String, Object> entry : dto.getSorter().entrySet()) {
String key = entry.getKey();
if (key.equals("influence")) {
key = "channel_influence";
} else if (key.equals("followers")) {
key = "channel_followers";
}
if (null == entry.getValue() || entry.getValue().toString().contains("desc")) {
sort = SortBuilders.fieldSort(key).order(SortOrder.DESC);
} else {
sort = SortBuilders.fieldSort(key).order(SortOrder.ASC);
}
}
}
helper.setSort(sort);
// from size
helper.setFrom((dto.getPage() - 1) * dto.getPageSize());
helper.setSize(dto.getPageSize());
// HighlightBuilder ???
return helper;
}
@Override
public JSONObject getContendSummary(MarkSearchDTO dto, boolean cache) throws IOException {
// 为dto设置默认值
......@@ -1120,20 +905,20 @@ public class MarkDataServiceImpl implements MarkDataService {
long primaryTotalCount = primaryPlatformsProportion.stream().mapToLong(platform -> platform.getLongValue("count")).sum();
result.put("priNorTotal", primaryTotalCount);
// 主品牌传播量最大的平台的数据
JSONObject primaryMaxPlatformProportion = primaryPlatformsProportion.stream().max(Comparator.comparing(o -> o.getLong("count"))).get();
JSONObject primaryMaxPlatformProportion =
primaryPlatformsProportion.stream().max(Comparator.comparing(o -> o.getLong("count"))).orElseGet(JSONObject::new);
result.put("priMaxPlatform", primaryMaxPlatformProportion.getString("platform"));
result.put("priMaxPlatformPro", primaryMaxPlatformProportion.getDoubleValue("proportion"));
// 竞品的名字
String brandName = projectService.getProjectByContendId(dto.getProjectId(), dto.getContendId()).getBrandName();
result.put("conName", brandName);
// 获取时间段竞品在每个平台传播量
String contendLinkedGroupId = projectService.getProjectByContendId(dto.getProjectId(), dto.getContendId()).getBrandLinkedGroupId();
List<JSONObject> contendPlatformsProportion = getMarkPlatformProportion(dto.getStartTime(), dto.getEndTime(), dto.getProjectId(), dto.getContendId(), null, true);
// 获取时间段竞品总传播量
long contendTotalCount = contendPlatformsProportion.stream().mapToLong(platform -> platform.getLongValue("count")).sum();
result.put("conNorTotal", contendTotalCount);
// 竞品传播量最大的平台的数据
JSONObject contendMaxPlatformProportion = contendPlatformsProportion.stream().max(Comparator.comparing(o -> o.getLong("count"))).get();
JSONObject contendMaxPlatformProportion = contendPlatformsProportion.stream().max(Comparator.comparing(o -> o.getLong("count"))).orElseGet(JSONObject::new);
result.put("conMaxPlatform", contendMaxPlatformProportion.getString("platform"));
result.put("conMaxPlatformPro", contendMaxPlatformProportion.getDoubleValue("proportion"));
result.put("compare", String.format("%.1f", primaryTotalCount * 1.0 / contendTotalCount));
......@@ -1154,7 +939,6 @@ public class MarkDataServiceImpl implements MarkDataService {
// 竞品的名字
AbstractProject projectByContendId = projectService.getProjectByContendId(projectId, contendId);
String brandName = projectByContendId.getBrandName();
String contendLinkedGroupId = projectByContendId.getBrandLinkedGroupId();
// 缓存
SimpleDateFormat sdf = new SimpleDateFormat(Constant.DAY_PATTERN);
String startTimeStr = sdf.format(startTime);
......@@ -1177,7 +961,7 @@ public class MarkDataServiceImpl implements MarkDataService {
JSONObject primaryLine = new JSONObject();
primaryLine.put("id", Constant.PRIMARY_CONTEND_ID);
primaryLine.put("brand", project.getBrandName());
List<LineVO> primarySpread = getArticleSpread(projectId, project.getBrandLinkedGroupId(), Constant.PRIMARY_CONTEND_ID, cutList);
List<LineVO> primarySpread = getArticleSpread(projectId, Constant.PRIMARY_CONTEND_ID, cutList);
primaryLine.put("spread", primarySpread);
resList.add(primaryLine);
result.put("days", primarySpread.size());
......@@ -1186,7 +970,7 @@ public class MarkDataServiceImpl implements MarkDataService {
JSONObject contendLine = new JSONObject();
contendLine.put("id", contendId);
contendLine.put("brand", brandName);
List<LineVO> contendSpread = getArticleSpread(projectId, contendLinkedGroupId, contendId, cutList);
List<LineVO> contendSpread = getArticleSpread(projectId, contendId, cutList);
contendLine.put("spread", contendSpread);
resList.add(contendLine);
result.put("spread", resList);
......@@ -1206,24 +990,20 @@ public class MarkDataServiceImpl implements MarkDataService {
long priMaxDayStartTime = Tools.truncDate(primaryMax.getDate(), Constant.DAY_PATTERN);
long priMaxDayEndTime = DateUtils.addDays(new Date(priMaxDayStartTime), 1).getTime();
// 获取时间段某情感数据最多的标题
List<Map.Entry<String, Integer>> priTopTitle = getMarkTopTitle(priMaxDayStartTime, priMaxDayEndTime, EmotionEnum.ALL.getName(), projectId,
project.getBrandLinkedGroupId(), Constant.PRIMARY_CONTEND_ID, 1);
List<Map.Entry<String, Integer>> priTopTitle = getMarkTopTitle(priMaxDayStartTime, priMaxDayEndTime, EmotionEnum.ALL.getName(), projectId, Constant.PRIMARY_CONTEND_ID, 1);
if (CollectionUtils.isNotEmpty(priTopTitle)) {
// 首发稿件
BaseMap priFirstArticle = getFirstArticle(priMaxDayStartTime, priMaxDayEndTime, priTopTitle.get(0).getKey(), projectId, project.getBrandLinkedGroupId(),
Constant.PRIMARY_CONTEND_ID);
BaseMap priFirstArticle = getFirstArticle(priMaxDayStartTime, priMaxDayEndTime, priTopTitle.get(0).getKey(), projectId, Constant.PRIMARY_CONTEND_ID);
result.put("priTopTitle", priFirstArticle.getTitle());
result.put("priTopTitleUrl", priFirstArticle.getUrl());
}
// 获取主品牌热门媒体方向
List<Map.Entry<String, Integer>> priHotTitles = getMarkTopTitle(startTime, endTime, EmotionEnum.ALL.getName(), projectId,
project.getBrandLinkedGroupId(), Constant.PRIMARY_CONTEND_ID, hotArticleSize);
List<Map.Entry<String, Integer>> priHotTitles = getMarkTopTitle(startTime, endTime, EmotionEnum.ALL.getName(), projectId, Constant.PRIMARY_CONTEND_ID, hotArticleSize);
if (CollectionUtils.isNotEmpty(priHotTitles)) {
List<JSONObject> priHotArticle = priHotTitles.stream().map(map -> {
JSONObject jsonObject = new JSONObject();
try {
BaseMap priFirstArticle = getFirstArticle(startTime, endTime, map.getKey(), projectId, project.getBrandLinkedGroupId(),
Constant.PRIMARY_CONTEND_ID);
BaseMap priFirstArticle = getFirstArticle(startTime, endTime, map.getKey(), projectId, Constant.PRIMARY_CONTEND_ID);
if (null != priFirstArticle) {
jsonObject.put("title", priFirstArticle.getTitle());
jsonObject.put("url", priFirstArticle.getUrl());
......@@ -1253,23 +1033,20 @@ public class MarkDataServiceImpl implements MarkDataService {
long conMaxDayStartTime = Tools.truncDate(contendMax.getDate(), Constant.DAY_PATTERN);
long conMaxDayEndTime = DateUtils.addDays(new Date(conMaxDayStartTime), 1).getTime();
List<Map.Entry<String, Integer>> conTopTitle = getMarkTopTitle(conMaxDayStartTime, conMaxDayEndTime, EmotionEnum.ALL.getName(),projectId,
contendLinkedGroupId, contendId, 1);
List<Map.Entry<String, Integer>> conTopTitle = getMarkTopTitle(conMaxDayStartTime, conMaxDayEndTime, EmotionEnum.ALL.getName(), projectId, contendId, 1);
if (CollectionUtils.isNotEmpty(conTopTitle)) {
// 首发稿件
BaseMap conFirstArticle = getFirstArticle(conMaxDayStartTime, conMaxDayEndTime, conTopTitle.get(0).getKey(), projectId, contendLinkedGroupId,
contendId);
BaseMap conFirstArticle = getFirstArticle(conMaxDayStartTime, conMaxDayEndTime, conTopTitle.get(0).getKey(), projectId, contendId);
result.put("conTopTitle", conFirstArticle.getTitle());
result.put("conTopTitleUrl", conFirstArticle.getUrl());
}
// 获取竞品热门媒体方向
List<Map.Entry<String, Integer>> conHotTitles = getMarkTopTitle(startTime, endTime, EmotionEnum.ALL.getName(), projectId,
contendLinkedGroupId, contendId, hotArticleSize);
List<Map.Entry<String, Integer>> conHotTitles = getMarkTopTitle(startTime, endTime, EmotionEnum.ALL.getName(), projectId, contendId, hotArticleSize);
if (CollectionUtils.isNotEmpty(conHotTitles)) {
List<JSONObject> conHotArticle = conHotTitles.stream().map(map -> {
JSONObject jsonObject = new JSONObject();
try {
BaseMap conFirstArticle = getFirstArticle(startTime, endTime, map.getKey(), projectId, contendLinkedGroupId, contendId);
BaseMap conFirstArticle = getFirstArticle(startTime, endTime, map.getKey(), projectId, contendId);
if (null != conFirstArticle) {
jsonObject.put("title", conFirstArticle.getTitle());
jsonObject.put("url", conFirstArticle.getUrl());
......@@ -1288,21 +1065,7 @@ public class MarkDataServiceImpl implements MarkDataService {
@Override
public Pair<String, List<ExportAppYuqingDTO>> downloadContendMarkList(MarkSearchDTO markSearchDTO) {
try {
ProjectVO project = projectService.getProjectVOById(UserThreadLocal.getProjectId());
List<ExportAppYuqingDTO> returnList = new ArrayList<>();
defaultMarkSearch(markSearchDTO);
Pair<SearchHits[], Map<String, Long>> hitsAndCounts = searchContendMarkHitsAndCount(markSearchDTO, false);
for (SearchHits searchHits : hitsAndCounts.getLeft()) {
List<ExportAppYuqingDTO> collect = Arrays.stream(searchHits.getHits()).map(SearchHit::getSourceAsMap).map(ExportAppYuqingDTO::createFromEsMap).collect(Collectors.toList());
returnList.addAll(collect);
}
return Pair.of(project.getBrandName() + "_" + markSearchDTO.getStartTime() + "_" + markSearchDTO.getEndTime(), returnList);
} catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.FAIL, "es查询异常", e);
}
return null;
return downloadList(markSearchDTO, esSearchService::searchContendMarkHitsAndCount);
}
@Override
......@@ -1327,7 +1090,7 @@ public class MarkDataServiceImpl implements MarkDataService {
}
} else {
result.put("origin", "舆情");
platformList = commonService.getQbjcPlatform("id","name").stream().filter(s -> !s.get("name").equals("脉脉")).collect(Collectors.toList());
platformList = commonService.getQbjcPlatform("id", "name").stream().filter(s -> !s.get("name").equals("脉脉")).collect(Collectors.toList());
}
result.put("platformList", platformList);
return result;
......@@ -1383,14 +1146,14 @@ public class MarkDataServiceImpl implements MarkDataService {
JSONArray jsonArray = new JSONArray();
dto.setPageSize(50);
dto.setPage(1);
while (true){
if (dto.getPage() * dto.getPageSize() > exportAmount){
while (true) {
if (dto.getPage() * dto.getPageSize() > exportAmount) {
break;
}
// 获取当页数据
JSONObject jsonObject = searchWholeNetwork(dto);
JSONArray array = jsonObject.getJSONArray("list");
if (Objects.isNull(array) || 0 == array.size()){
if (Objects.isNull(array) || 0 == array.size()) {
break;
}
jsonArray.addAll(array);
......@@ -1439,7 +1202,7 @@ public class MarkDataServiceImpl implements MarkDataService {
return jsonObject;
}
public JSONObject searchWholeBackUp(String taskId) {
private JSONObject searchWholeBackUp(String taskId) {
JSONObject jsonObject = new JSONObject();
List<JSONObject> resultList = new ArrayList<>();
......@@ -1481,8 +1244,8 @@ public class MarkDataServiceImpl implements MarkDataService {
/**
* 获取全网搜接口信息url
*
* @param json
* @return
* @param json 数据
* @return url链接
*/
private static String getUrl(JSONObject json) {
String url = json.getString("url");
......@@ -1525,7 +1288,7 @@ public class MarkDataServiceImpl implements MarkDataService {
@Override
public long getYuqingMarkCount(Long startTime, Long endTime, String emotion, String platformId, String projectId, String contendId) throws IOException {
BoolQueryBuilder filter = projectLinkedGroupContendIdQuery(projectId, projectService.getProjectById(projectId).getBrandLinkedGroupId(), contendId);
BoolQueryBuilder filter = projectContendIdQuery(projectId, contendId);
if (null != startTime) {
filter.must(QueryBuilders.rangeQuery("time").gte(startTime));
}
......@@ -1544,7 +1307,7 @@ public class MarkDataServiceImpl implements MarkDataService {
@Override
public long getYuqingMarkFirstTime(String projectId, String contendId) throws IOException {
EsClientDao.SearchHelper searchHelper = EsClientDao.createSearchHelper();
BoolQueryBuilder filter = projectLinkedGroupContendIdQuery(projectId, projectService.getProjectById(projectId).getBrandLinkedGroupId(), contendId);
BoolQueryBuilder filter = projectContendIdQuery(projectId, contendId);
searchHelper.setPostFilter(filter);
searchHelper.setSort(SortBuilders.fieldSort("time").order(SortOrder.ASC));
searchHelper.setSize(1);
......@@ -1557,14 +1320,14 @@ public class MarkDataServiceImpl implements MarkDataService {
}
@Override
public List<Map<String, Object>> getEsTopSource(Long startTime, Long endTime, String projectId, String linkedGroupId, String contendId, String emotion, int size) throws IOException {
public List<Map<String, Object>> getEsTopSource(Long startTime, Long endTime, String projectId, String contendId, String emotion, int size) throws IOException {
EsClientDao.SearchHelper searchHelper = EsClientDao.createSearchHelper();
// 聚合条件
Script script = new Script("doc['platform_id'].value +'_' +doc['real_source'].value+'_' +doc['source'].value");
TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("agg").script(script).order(BucketOrder.count(false));
searchHelper.setAggregationBuilder(aggregationBuilder);
// query
BoolQueryBuilder query = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, contendId);
BoolQueryBuilder query = projectContendIdQuery(projectId, contendId);
query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
if (StringUtils.isNotEmpty(emotion) && !EmotionEnum.ALL.getName().equals(emotion)) {
query.must(QueryBuilders.termQuery("brandkbs_mark_cache_maps.name.keyword", emotion));
......@@ -1597,11 +1360,11 @@ public class MarkDataServiceImpl implements MarkDataService {
}
@Override
public JSONObject getLastMarkData(String projectId, String linkedGroupId, String contendId, String platform, String realSource, String source) {
public JSONObject getLastMarkData(String projectId, String contendId, String platform, String realSource, String source) {
try {
EsClientDao.SearchHelper searchHelper = EsClientDao.createSearchHelper();
BoolQueryBuilder query = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, contendId);
query.must(QueryBuilders.termQuery("platform_id", GlobalPojo.getPlatformIdByName(platform)));
BoolQueryBuilder query = projectContendIdQuery(projectId, contendId);
query.must(QueryBuilders.termQuery("platform_id", Objects.requireNonNull(GlobalPojo.getPlatformIdByName(platform))));
query.must(QueryBuilders.termQuery("real_source", realSource));
query.must(QueryBuilders.termQuery("source", source));
searchHelper.setQuery(query);
......@@ -1613,11 +1376,16 @@ public class MarkDataServiceImpl implements MarkDataService {
}
return getTitleAndUrl(searchHits.getAt(0).getSourceAsMap());
} catch (Exception e) {
log.info("getLastMarkData-",e);
log.info("getLastMarkData-", e);
return null;
}
}
@Override
public BoolQueryBuilder projectContendIdQuery(String projectId, String contendId) {
return EsQueryTools.assembleCacheMapsQuery(projectId, contendId);
}
private static JSONObject getTitleAndUrl(Map<String, Object> map) {
JSONObject json = new JSONObject();
BaseMap baseMap = Tools.getBaseFromEsMap(map);
......@@ -1629,13 +1397,12 @@ public class MarkDataServiceImpl implements MarkDataService {
/**
* 获取传播图谱
*
* @param projectId 项目ID
* @param linkedGroupId 关联组Id
* @param contendId 竞品ID
* @param cutList 拆分list
* @return
* @param projectId 项目ID
* @param contendId 竞品ID
* @param cutList 拆分list
* @return List<LineVO>
*/
private List<LineVO> getArticleSpread(String projectId, String linkedGroupId, String contendId, List<Map<String, Long>> cutList) {
private List<LineVO> getArticleSpread(String projectId, String contendId, List<Map<String, Long>> cutList) {
List<LineVO> lineList = new ArrayList<>(cutList.size());
cutList.forEach(day -> {
LineVO lineVO = new LineVO();
......@@ -1643,7 +1410,7 @@ public class MarkDataServiceImpl implements MarkDataService {
Long endTime = day.get("endTime");
Long articleCount;
try {
articleCount = getMarkArticleCount(startTime, endTime, projectId, linkedGroupId, contendId);
articleCount = getMarkArticleCount(startTime, endTime, projectId, contendId);
} catch (IOException e) {
throw new RuntimeException(e);
}
......@@ -1657,21 +1424,17 @@ public class MarkDataServiceImpl implements MarkDataService {
/**
* 获取稿件数量
*
* @param startTime 开始时间
* @param endTime 结束时间
* @param projectId 项目ID
* @param linkedGroupId 关联组Id
* @param contendId 竞品ID
* @param startTime 开始时间
* @param endTime 结束时间
* @param projectId 项目ID
* @param contendId 竞品ID
* @return 每日稿件数量
*/
private Long getMarkArticleCount(Long startTime, Long endTime, String projectId, String linkedGroupId, String contendId) throws IOException {
// index
String[] indexes = esClientDao.getIndexes();
BoolQueryBuilder postFilter = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, contendId);
private Long getMarkArticleCount(Long startTime, Long endTime, String projectId, String contendId) throws IOException {
BoolQueryBuilder postFilter = projectContendIdQuery(projectId, contendId);
// time range
postFilter.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
//hits
// hits
return esClientDao.count(postFilter);
}
......@@ -1701,13 +1464,12 @@ public class MarkDataServiceImpl implements MarkDataService {
/**
* 获取每日稿件倾向稿件数量信息
*
* @param startTime 开始时间
* @param endTime 结束时间
* @param projectId 项目id
* @param linkedGroupId 关联组id
* @param startTime 开始时间
* @param endTime 结束时间
* @param projectId 项目id
* @return 所有稿件倾向稿件数量信息
*/
private List<JSONObject> getDayMarkEmotionCount(Long startTime, Long endTime, String projectId, String linkedGroupId) throws IOException {
private List<JSONObject> getDayMarkEmotionCount(Long startTime, Long endTime, String projectId) throws IOException {
List<JSONObject> resultList = new ArrayList<>();
// 索引
String[] indexes = esClientDao.getIndexes();
......@@ -1715,7 +1477,7 @@ public class MarkDataServiceImpl implements MarkDataService {
DateHistogramAggregationBuilder daysAggregationBuilder = AggregationBuilders.dateHistogram("dayAgg").field("time").calendarInterval(DateHistogramInterval.DAY);
TermsAggregationBuilder emotionAggregationBuilder = AggregationBuilders.terms("count").field("brandkbs_mark_cache_maps.name.keyword");
// query
BoolQueryBuilder query = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, "0");
BoolQueryBuilder query = projectContendIdQuery(projectId, Constant.PRIMARY_CONTEND_ID);
// time
query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
// 情感倾向限制
......@@ -1740,13 +1502,12 @@ public class MarkDataServiceImpl implements MarkDataService {
/**
* 获取所有平台稿件倾向稿件数量信息
*
* @param startTime 开始时间
* @param endTime 结束时间
* @param projectId 项目ID
* @param linkedGroupId 关联组id
* @param startTime 开始时间
* @param endTime 结束时间
* @param projectId 项目ID
* @return 所有稿件倾向稿件数量信息
*/
private List<JSONObject> getPlatformMarkEmotionCount(Long startTime, Long endTime, String projectId, String linkedGroupId) throws IOException {
private List<JSONObject> getPlatformMarkEmotionCount(Long startTime, Long endTime, String projectId) throws IOException {
List<JSONObject> ResultList = new ArrayList<>();
// 索引
String[] indexes = esClientDao.getIndexes();
......@@ -1754,7 +1515,7 @@ public class MarkDataServiceImpl implements MarkDataService {
TermsAggregationBuilder platformAggregationBuilder = AggregationBuilders.terms("platform_count").field("platform_id").order(BucketOrder.count(false));
TermsAggregationBuilder emotionAggregationBuilder = AggregationBuilders.terms("emotion_count").field("brandkbs_mark_cache_maps.name.keyword");
// postFilter
BoolQueryBuilder query = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, "0");
BoolQueryBuilder query = projectContendIdQuery(projectId, Constant.PRIMARY_CONTEND_ID);
// time
query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
// 情感倾向限制
......@@ -1824,7 +1585,7 @@ public class MarkDataServiceImpl implements MarkDataService {
// 聚合请求
TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("count").field("platform_id").order(BucketOrder.count(false)).size(100);
// query
BoolQueryBuilder query = projectLinkedGroupContendIdQuery(projectId, null, contendId);
BoolQueryBuilder query = projectContendIdQuery(projectId, contendId);
// keyword
if (StringUtils.isNotEmpty(keyword)) {
String[] fieldSearch = "标题".equals(searchField) ? new String[]{GenericAttribute.ES_IND_TITLE} : new String[]{GenericAttribute.ES_IND_FULL_TEXT};
......@@ -1856,11 +1617,11 @@ public class MarkDataServiceImpl implements MarkDataService {
*
* @return 所有渠道倾向占比
*/
private JSONObject getChannelEmotionPercent(String projectId, String linkedGroupId, Long startTime, Long endTime) throws IOException {
private JSONObject getChannelEmotionPercent(String projectId, Long startTime, Long endTime) throws IOException {
JSONObject channelEmotionResult = new JSONObject();
// 搜索符合条件的数据
EsClientDao.SearchHelper searchHelper = EsClientDao.createSearchHelper();
BoolQueryBuilder postFilter = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, "0");
BoolQueryBuilder postFilter = projectContendIdQuery(projectId, Constant.PRIMARY_CONTEND_ID);
postFilter.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
postFilter.mustNot(EsQueryTools.cacheMapsNestedQuery(QueryBuilders.termQuery("brandkbs_cache_maps.channel_emotion", 0)));
searchHelper.setPostFilter(postFilter);
......@@ -1886,31 +1647,30 @@ public class MarkDataServiceImpl implements MarkDataService {
return channelEmotionResult;
}
private JSONObject getImportantChannelPercent(String projectId, String linkedGroupId, Long startTime, Long endTime) throws IOException {
private JSONObject getImportantChannelPercent(String projectId, Long startTime, Long endTime) throws IOException {
JSONObject mediaTypePercent = new JSONObject();
Map<String, Long> mediaTypeMap = new HashMap<>();
// 目前只要该部分列表
List<ImportantChannelEnum> mediaTypeList = Arrays.asList(ImportantChannelEnum.YANGJI, ImportantChannelEnum.KEJI, ImportantChannelEnum.CAIJING, ImportantChannelEnum.QITA);
for (ImportantChannelEnum channelEnum : mediaTypeList) {
BoolQueryBuilder postFilter = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, "0");
BoolQueryBuilder postFilter = projectContendIdQuery(projectId, Constant.PRIMARY_CONTEND_ID);
postFilter.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
postFilter.must(EsQueryTools.cacheMapsNestedQuery(QueryBuilders.matchQuery("brandkbs_cache_maps.channel_type.keyword", channelEnum.getState())));
Long count = esClientDao.count(postFilter);
mediaTypeMap.put(channelEnum.name().toLowerCase() + "Percent", count);
}
long total = mediaTypeMap.values().stream().mapToLong(v -> v).sum();
mediaTypePercent.putAll(mediaTypeMap);
return mediaTypePercent;
}
private List<String> getMarkTitleAndContent(Long startTime, Long endTime, String projectId, String linkedGroupId) throws IOException {
private List<String> getMarkTitleAndContent(Long startTime, Long endTime, String projectId) throws IOException {
EsClientDao.SearchHelper searchHelper = EsClientDao.createSearchHelper();
// // sort
searchHelper.setSort(SortBuilders.fieldSort("time").order(SortOrder.DESC));
// fetchSource
searchHelper.setFetchSource(new String[]{"ind_title", "ind_full_text", "c5", "foreign", "brandkbs_mark_cache_maps"});
// postFilter
BoolQueryBuilder postFilter = projectLinkedGroupContendIdQuery(projectId, linkedGroupId, "0");
BoolQueryBuilder postFilter = projectContendIdQuery(projectId, Constant.PRIMARY_CONTEND_ID);
postFilter.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
searchHelper.setPostFilter(postFilter);
searchHelper.setSize(10000);
......@@ -1928,13 +1688,8 @@ public class MarkDataServiceImpl implements MarkDataService {
return textList;
}
public BoolQueryBuilder projectLinkedGroupContendIdQuery(String projectId, String linkedGroupId, String contendId) {
return EsQueryTools.assembleCacheMapsQuery(projectId, linkedGroupId, contendId);
}
private List<MarkFlowEntity> getMarkFlowEntity(MarkSearchDTO markSearchDTO, SearchHits searchHits) {
String projectId = markSearchDTO.getProjectId();
String linkedGroupId = markSearchDTO.getLinkedGroupId();
String contendId = markSearchDTO.getContendId();
// 重复消息折叠
if (markSearchDTO.isFold()) {
......@@ -1949,18 +1704,126 @@ public class MarkDataServiceImpl implements MarkDataService {
return sortList.stream().map(id -> {
List<Map<String, Object>> maps = collect.get(id);
if (null != maps) {
return MarkFlowEntity.getFoldInstance(maps.stream().map(map -> getMarkFlowEntity(map, projectId, linkedGroupId, contendId)).collect(Collectors.toList()));
return MarkFlowEntity.getFoldInstance(maps.stream().map(map -> getMarkFlowEntity(map, projectId, contendId)).collect(Collectors.toList()));
}
return null;
}).filter(Objects::nonNull).collect(Collectors.toList());
}
return Arrays.stream(searchHits.getHits()).map(hit -> getMarkFlowEntity(hit.getSourceAsMap(), projectId, linkedGroupId, contendId)).collect(Collectors.toList());
return Arrays.stream(searchHits.getHits()).map(hit -> getMarkFlowEntity(hit.getSourceAsMap(), projectId, contendId)).collect(Collectors.toList());
}
private MarkFlowEntity getMarkFlowEntity(Map<String, Object> map, String projectId, String linkedGroupId, String contendId) {
private MarkFlowEntity getMarkFlowEntity(Map<String, Object> map, String projectId, String contendId) {
MarkFlowEntity instance = new MarkFlowEntity(new JSONObject(map));
instance.setInfo(markFlowService.createMarkFlowInfo(new JSONObject(map), projectId, linkedGroupId, contendId));
instance.setInfo(markFlowService.createMarkFlowInfo(new MarkFlowService.MarkInfoSource(new JSONObject(map), projectId, contendId)));
return instance;
}
private List<MarkFlowEntity> getMarkFlowEntity4Origin(MarkSearchDTO markSearchDTO, SearchHits searchHits) {
String linkedGroupId = markSearchDTO.getLinkedGroupId();
// 重复消息折叠
if (markSearchDTO.isFold()) {
// 根据旧顺序排序
List<String> sortList = Arrays.stream(searchHits.getHits()).map(hit -> hit.getSourceAsMap().get("id") + "").collect(Collectors.toList());
Map<String, List<Map<String, Object>>> groupMap = Arrays.stream(searchHits.getHits()).map(SearchHit::getSourceAsMap).collect(Collectors.groupingBy(map -> {
BaseMap baseMap = Tools.getBaseFromEsMap(map);
String title = baseMap.getTitleNullOptionalContent();
return Tools.filterSpecialCharacter(title) + baseMap.getTypeB().encode();
}));
Map<String, List<Map<String, Object>>> collect = groupMap.values().stream().map(list -> list.stream().sorted(Comparator.comparingLong(o -> Long.parseLong(o.get("time") + ""))).collect(Collectors.toList())).collect(Collectors.toMap(list -> list.get(0).get("id") + "", list -> list));
return sortList.stream().map(id -> {
List<Map<String, Object>> maps = collect.get(id);
if (null != maps) {
return MarkFlowEntity.getFoldInstance(maps.stream().map(map -> getMarkFlowEntity4Origin(map, linkedGroupId)).collect(Collectors.toList()));
}
return null;
}).filter(Objects::nonNull).collect(Collectors.toList());
}
List<MarkFlowEntity> collect = Arrays.stream(searchHits.getHits()).map(hit -> getMarkFlowEntity4Origin(hit.getSourceAsMap(), linkedGroupId)).collect(Collectors.toList());
return markFlowService.markTagInfo(collect, linkedGroupId);
}
/**
* 组装单条信息
* @param map
* @param linkedGroupId
* @return
*/
private MarkFlowEntity getMarkFlowEntity4Origin(Map<String, Object> map, String linkedGroupId) {
MarkFlowEntity instance = new MarkFlowEntity(new JSONObject(map), linkedGroupId);
instance.setInfo(markFlowService.createMarkFlowInfo4Origin(new MarkFlowService.MarkInfoSource(new JSONObject(map), linkedGroupId)));
return instance;
}
private String generateAggreeList(String projectId, Long startTime, Long endTime, ArticleFunction.SearchHitsAndCount searchHitsAndCount) {
String uuid = Tools.getUUID();
MarkSearchDTO markSearchDTO = defaultMarkSearch4Aggree(startTime, endTime);
ApplicationProjectListener.getThreadPool().execute(() -> {
String redisKey = RedisUtil.getAggreeCacheKey(uuid, projectId);
try {
redisUtil.setExpire(redisKey, "0");
log.info("taskId:{},聚合任务开始", uuid);
// 搜索es数据
Pair<SearchHits[], Map<String, Long>> hitsAndCounts = searchHitsAndCount.search(markSearchDTO, true);
log.info("taskId:{},聚合前数据:{}条", uuid, hitsAndCounts.getLeft()[0].getTotalHits().value);
// 聚合分组后数据
List<List<JSONObject>> groupList = aggregation(hitsAndCounts.getLeft());
log.info("taskId:{},聚合完成,聚合组:{}个", uuid, groupList.size());
// 入mongo缓存
cacheAggreeResult(groupList, uuid, projectId);
redisUtil.setExpire(redisKey, "100");
} catch (Exception e) {
log.error("generateAggreeList-", e);
redisUtil.remove(redisKey);
}
});
return uuid;
}
private PageVO<MarkFlowEntity> getAggreeList(MarkSearchDTO dto, ArticleFunction.CreateMarkFlowInfo createMarkFlowInfo, boolean sonDetails) {
if (null == dto.getAggreeId()) {
ExceptionCast.cast(CommonCodeEnum.INVALID_PARAM);
}
defaultMarkSearch(dto);
Query query = assembleAggreeQuery(dto);
// 量查询
long count = aggreeResultDao.count(query);
// 开启分页
mongoUtil.start(dto.getPage(), dto.getPageSize(), query);
// 排序
aggreeResultDao.addSort(query, null == dto.getSorter() ? "{\"aggreeSize\":\"descend\"}" : dto.getSorter().toString());
// 返回结果
List<MarkFlowEntity> resList = new ArrayList<>();
List<AggreeResult> fatherList = aggreeResultDao.findList(query);
for (AggreeResult aggreeResult : fatherList) {
List<MarkFlowEntity> markFlowEntities = new ArrayList<>();
MarkFlowEntity instance = new MarkFlowEntity(aggreeResult.getData(), dto.getLinkedGroupId());
instance.setInfo(createMarkFlowInfo.create(new MarkFlowService.MarkInfoSource(aggreeResult.getData(), dto.getProjectId(), dto.getContendId(), dto.getLinkedGroupId())));
markFlowEntities.add(instance);
// 添加父标题集下的子标题集
resList.add(MarkFlowEntity.getAggreeInstance(markFlowEntities, aggreeResult, sonDetails));
}
if (sonDetails) {
markFlowService.markTagInfo(resList, dto.getLinkedGroupId());
}
return PageVO.createPageVo(count, dto.getPage(), dto.getPageSize(), resList);
}
private Pair<String, List<ExportAppYuqingDTO>> downloadList(MarkSearchDTO markSearchDTO,ArticleFunction.SearchHitsAndCount searchHitsAndCount) {
try {
List<ExportAppYuqingDTO> returnList = new ArrayList<>();
Project project = projectService.getProjectById(UserThreadLocal.getProjectId());
defaultMarkSearch(markSearchDTO);
markSearchDTO.setPageSize(null);
for (SearchHits searchHits : searchHitsAndCount.search(markSearchDTO, false).getLeft()) {
List<ExportAppYuqingDTO> collect = Arrays.stream(searchHits.getHits()).map(SearchHit::getSourceAsMap).map(ExportAppYuqingDTO::createFromEsMap).collect(Collectors.toList());
returnList.addAll(collect);
}
return Pair.of(project.getBrandName() + "_" + markSearchDTO.getStartTime() + "_" + markSearchDTO.getEndTime(), returnList);
} catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.FAIL, "es查询异常");
}
return null;
}
}
\ No newline at end of file
src/main/java/com/zhiwei/brandkbs2/service/impl/MarkFlowServiceImpl.java
......@@ -2,13 +2,14 @@ package com.zhiwei.brandkbs2.service.impl;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.base.category.ClassB;
import com.zhiwei.base.category.ClassD;
import com.zhiwei.base.entity.subclass.mark.*;
import com.zhiwei.base.filter.FilterInfo;
import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.common.GlobalPojo;
import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.dao.ChannelDao;
import com.zhiwei.brandkbs2.dao.ChannelTagDao;
import com.zhiwei.brandkbs2.enmus.ChannelEmotion;
import com.zhiwei.brandkbs2.exception.ExceptionCast;
import com.zhiwei.brandkbs2.model.CommonCodeEnum;
......@@ -16,11 +17,23 @@ import com.zhiwei.brandkbs2.pojo.MarkFlowEntity;
import com.zhiwei.brandkbs2.service.MarkFlowService;
import com.zhiwei.brandkbs2.util.RedisUtil;
import com.zhiwei.brandkbs2.util.Tools;
import com.zhiwei.middleware.cleaner.filter.UnifiedFilterClient;
import com.zhiwei.middleware.mark.service.MarkerClient;
import com.zhiwei.middleware.mark.vo.QueryResult;
import com.zhiwei.qbjc.bean.pojo.common.Tag;
import com.zhiwei.qbjc.bean.tools.BeanTools;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
/**
* @ClassName: MarkFlowServiceImpl
......@@ -30,23 +43,122 @@ import java.math.RoundingMode;
*/
@Service("markFlowServiceImpl")
public class MarkFlowServiceImpl implements MarkFlowService {
private static final Logger log = LogManager.getLogger(MarkFlowServiceImpl.class);
@Resource(name = "channelTagDao")
ChannelTagDao channelTagDao;
@Resource(name = "channelDao")
ChannelDao channelDao;
// @Resource(name = "channelTagDao")
// ChannelTagDao channelTagDao;
//
// @Resource(name = "channelDao")
// ChannelDao channelDao;
@Resource(name = "redisUtil")
RedisUtil redisUtil;
@Resource(name = "markerClient")
MarkerClient markerClient;
@Resource(name = "filterClient")
UnifiedFilterClient filterClient;
@Override
public JSONObject createMarkFlowInfo(JSONObject json, String projectId, String linkedGroupId, String contendId) {
public JSONObject createMarkFlowInfo(MarkInfoSource markInfoSource) {
JSONObject resultInfo = createInfoWithEmotion(markInfoSource.getJson());
resultInfo.put("sourceDetails", getSourceDetails(markInfoSource));
return resultInfo;
}
@Override
public JSONObject createMarkFlowInfo4Origin(MarkInfoSource markInfoSource) {
JSONObject json = markInfoSource.getJson();
String linkedGroupId = markInfoSource.getLinkedGroupId();
JSONObject resultInfo = createInfoWithEmotion(json);
resultInfo.put("sourceDetails", getSourceDetails(json, projectId, linkedGroupId, contendId));
resultInfo.put("sourceDetails", getSourceDetails4Origin(json, linkedGroupId));
String zip = Tools.gzip(JSON.toJSONString(json));
resultInfo.put("zip", zip);
return resultInfo;
}
/**
* 消息实体批量处理标注信息
*
* @param mes
* @param linkedGroupId
* @return void
*/
@Override
public List<MarkFlowEntity> markTagInfo(List<MarkFlowEntity> mes, String linkedGroupId) {
if (!Tools.isEmpty(mes)) {
List<FilterInfo> filterInfos = mes.stream().map(m -> m.getFilterInfo(linkedGroupId)).collect(Collectors.toList());
Map<String, QueryResult> markInfos = markerClient.matchQueryResult(filterInfos);
if (Tools.isEmpty(markInfos)) {
log.error("dubbo-markInfos为空，filterInfosSize:{};ex.:{}", filterInfos.size(), JSONObject.toJSONString(filterInfos));
} else {
List<Tag> tags = GlobalPojo.TAGS.get(linkedGroupId);
for (MarkFlowEntity me : mes) {
String[] mupdates = filterClient.getUpdateInfo(me.getFilterInfo());
boolean isHit = mupdates.length > 1 ? markInfos.containsKey(mupdates[0])
|| markInfos.containsKey(mupdates[1]) :
markInfos.containsKey(mupdates[0]);
if (isHit) {
QueryResult result = markInfos.get(mupdates[0]);
if (Objects.isNull(result)) {
result = markInfos.get(mupdates[1]);
}
me.getInfo().put("tagInfo", getTagInfo(result, tags));
}
}
}
}
return mes;
}
/**
* 将json转为markInfo
*
* @param json
* @param mtag
* @param mperson
* @param mgroup
* @return MarkInfo
*/
@Override
public MarkInfo markInfoFromJson(JSONObject json, String mgroup, String mperson, String mtag) {
int c2 = json.getIntValue(GenericAttribute.ES_C2);
switch (ClassB.TypeB.fromEncode(c2)) {
case COMPLETE:
CompleteTextMark completeTextMark = CompleteTextMark.restoreFromEs(json);
completeTextMark.setMgroup(mgroup);
completeTextMark.setMperson(mperson);
completeTextMark.setMtag(mtag);
completeTextMark.setMtime(System.currentTimeMillis());
return new MarkInfo(completeTextMark);
case INCOMPLETE:
IncompleteTextMark incompleteTextMark = IncompleteTextMark.restoreFromEs(json);
incompleteTextMark.setMgroup(mgroup);
incompleteTextMark.setMperson(mperson);
incompleteTextMark.setMtag(mtag);
incompleteTextMark.setMtime(System.currentTimeMillis());
return new MarkInfo(incompleteTextMark);
case QA:
QATextMark qaTextMark = QATextMark.restoreFromEs(json);
qaTextMark.setMgroup(mgroup);
qaTextMark.setMperson(mperson);
qaTextMark.setMtag(mtag);
qaTextMark.setMtime(System.currentTimeMillis());
return new MarkInfo(qaTextMark);
case VIDEO:
VideoMark videoMark = VideoMark.restoreFromEs(json);
videoMark.setMgroup(mgroup);
videoMark.setMperson(mperson);
videoMark.setMtag(mtag);
videoMark.setMtime(System.currentTimeMillis());
return new MarkInfo(videoMark);
default:
return null;
}
}
@Override
public String setShotPage(String markFlowEntity) {
MarkFlowEntity data = JSON.parseObject(markFlowEntity, MarkFlowEntity.class);
......@@ -64,28 +176,6 @@ public class MarkFlowServiceImpl implements MarkFlowService {
ExceptionCast.cast(CommonCodeEnum.FAIL);
}
return JSON.parseObject(data, MarkFlowEntity.class);
// JSONObject dataJson = JSON.parseObject(data);
// FilterInfo filterInfo = null;
// JSONObject filterInfoJson = dataJson.getJSONObject("filterInfo");
// if (Objects.nonNull(filterInfoJson) && filterInfoJson.containsKey("selectType")
// && filterInfoJson.containsKey("filterMap")) {
// SelectType selectType = SelectType.valueOf(filterInfoJson.getString("selectType"));
// JSONObject filterMap = filterInfoJson.getJSONObject("filterMap");
// // 2020/5/27 15:17 论坛回答部分特殊去重解析
// if (SelectType.SPEC_FORUM_CONTENT_MARK == selectType) {
// filterInfo = FilterInfo.getInstanceWithNoTime(selectType,
// requireNonNull(filterMap.getString("url"), "url 为空"),
// requireNonNull(filterMap.getString("text"), "text 为空"),
// requireNonNull(filterMap.getString("spareText"), "spareText 为空"),
// requireNonNull(filterMap.getString("source"), "source 为空"), filterMap.getString("mgroup"));
// } else {
// filterInfo = new FilterInfo(selectType, requireNonNull(filterMap.getString("unique"), "unique 为空"),
// requireNonNull(filterMap.getString("url"), "url 为空"),
// requireNonNull(filterMap.getLong("time"), "time 为空"),
// requireNonNull(filterMap.getString("text"), "text 为空"),
// requireNonNull(filterMap.getString("source"), "source 为空"), filterMap.getString("mgroup"));
// }
// }
}
private JSONObject createInfoWithEmotion(JSONObject json) {
......@@ -102,8 +192,9 @@ public class MarkFlowServiceImpl implements MarkFlowService {
return info;
}
private JSONObject getSourceDetails(JSONObject tJson, String projectId, String linkedGroupId, String contendId) {
private JSONObject getSourceDetails(MarkInfoSource markInfoSource) {
JSONObject sourceDetails = new JSONObject();
JSONObject tJson = markInfoSource.getJson();
String source = tJson.getString(GenericAttribute.ES_SOURCE);
// 是否原创
if (tJson.containsKey(GenericAttribute.ES_ROOT_SOURCE) && isOrigin(source, tJson.getString(GenericAttribute.ES_ROOT_SOURCE))) {
......@@ -139,7 +230,7 @@ public class MarkFlowServiceImpl implements MarkFlowService {
// sourceDetails.put("channelEmotion", ChannelEmotion.getNameFromState(hitMap.get("channel_emotion")));
// }
// }
JSONObject brandkbsHitJson = Tools.getBrandkbsHitMap(tJson, Tools.concat(projectId, contendId));
JSONObject brandkbsHitJson = Tools.getBrandkbsHitMap(tJson, Tools.concat(markInfoSource.getProjectId(), markInfoSource.getContendId()));
if (null != brandkbsHitJson.get("channel_emotion")) {
sourceDetails.put("channelEmotion", ChannelEmotion.getNameFromState(brandkbsHitJson.getIntValue("channel_emotion")));
}
......@@ -152,6 +243,28 @@ public class MarkFlowServiceImpl implements MarkFlowService {
return sourceDetails;
}
private JSONObject getSourceDetails4Origin(JSONObject tJson, String linkedGroupId) {
JSONObject sourceDetails = new JSONObject();
String source = tJson.getString(GenericAttribute.ES_SOURCE);
// 是否原创
if (tJson.containsKey(GenericAttribute.ES_ROOT_SOURCE) && isOrigin(source, tJson.getString(GenericAttribute.ES_ROOT_SOURCE))) {
sourceDetails.put("rootPublish", "原创");
}
// C4,realSource提取展示
sourceDetails.put("clientFrom", getClientFrom(tJson.getIntValue(GenericAttribute.ES_C4), tJson.getString(GenericAttribute.ES_REAL_SOURCE)));
// platform
JSONObject hitJson = Tools.getHotEsHitMap(tJson, linkedGroupId);
sourceDetails.put("platform", GlobalPojo.getPlatformNameById(hitJson.getString("platform")));
// source
sourceDetails.put("source", source);
// 粉丝量提取
long followersNum = tJson.getLongValue(GenericAttribute.ES_FOLLOWERS_NUM);
if (followersNum > 0) {
sourceDetails.put("followersNum", followersNum);
}
return sourceDetails;
}
/**
* 按c4、origin获取ClientFrom
*
......@@ -159,7 +272,7 @@ public class MarkFlowServiceImpl implements MarkFlowService {
* @param realSource
* @return String
*/
public String getClientFrom(int c4, String realSource) {
private String getClientFrom(int c4, String realSource) {
if (Tools.isEmpty(realSource)) {
return ClassD.TypeD.fromEncode(c4).title();
} else {
......@@ -174,10 +287,32 @@ public class MarkFlowServiceImpl implements MarkFlowService {
* @param rootSource
* @return boolean
*/
public boolean isOrigin(String source, String rootSource) {
private boolean isOrigin(String source, String rootSource) {
// 条件一 泛用
return rootSource.contains(source) || source.contains(rootSource);
}
/**
* 标签信息转换
*
* @param queryResult
* @param tags
* @return Object
*/
private Map<String, Object> getTagInfo(QueryResult queryResult, List<Tag> tags) {
Map<String, Object> res = new HashMap<>(5);
if (queryResult.isDuplicate() && !Tools.isEmpty(queryResult.getMtag()) && !Tools.isEmpty(queryResult.getMperson())) {
res.put("hasTag", queryResult.isDuplicate());
res.put("mupdate", queryResult.getMupdates()[0]);
res.put("mupdateTwo", queryResult.getMupdates().length > 1 ? queryResult.getMupdates()[1] : null);
res.put("tagType", "自动化机器人".equals(queryResult.getMperson()) ? "自动标注" : "正常标注");
res.put("mperson", queryResult.getMperson());
res.put("mtag", queryResult.getMtag());
res.putAll(BeanTools.decodeMtag(queryResult.getMtag(), tags));
} else {
res.put("hasTag", false);
}
return res;
}
}
src/main/java/com/zhiwei/brandkbs2/service/impl/ProjectServiceImpl.java
......@@ -7,7 +7,6 @@ import com.zhiwei.brandkbs2.common.GlobalPojo;
import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.dao.EventMiddlewareDao;
import com.zhiwei.brandkbs2.dao.ProjectDao;
import com.zhiwei.brandkbs2.dao.QbjcPojoDao;
import com.zhiwei.brandkbs2.dao.UserDao;
import com.zhiwei.brandkbs2.enmus.response.ProjectCodeEnum;
import com.zhiwei.brandkbs2.exception.ExceptionCast;
......@@ -62,9 +61,6 @@ public class ProjectServiceImpl implements ProjectService {
@Resource(name = "projectDao")
private ProjectDao projectDao;
@Resource(name = "qbjcPojoDao")
private QbjcPojoDao qbjcPojoDao;
@Resource(name = "mongoUtil")
private com.zhiwei.brandkbs2.util.MongoUtil mongoUtil;
......@@ -84,7 +80,10 @@ public class ProjectServiceImpl implements ProjectService {
private String brandkbsImageUrl;
@Override
public void addProject(ProjectVO projectVO) {
public ResponseResult addProject(ProjectVO projectVO) {
if (!Tools.isEmpty(projectVO.getHitKeywords()) && projectVO.getModuleShowList().contains(2)) {
return ResponseResult.failure("关键字和原始数据配置异常");
}
if (Objects.isNull(projectVO)) {
//抛出非法参数异常
ExceptionCast.cast(CommonCodeEnum.INVALID_PARAM);
......@@ -92,12 +91,8 @@ public class ProjectServiceImpl implements ProjectService {
Project existsProject = projectDao.findOne("projectName", projectVO.getProjectName());
if (Objects.nonNull(existsProject)) {
//抛出项目名称已存在异常
ExceptionCast.cast(ProjectCodeEnum.PROJECT_EXISTSNAME_ERROR);
return ResponseResult.failure(ProjectCodeEnum.PROJECT_EXISTSNAME_ERROR.message());
}
//给超管添加该项目权限，并且初始化预警推送 TODO
// for (Integer SUPER_ADMINUid : userMapper.getAllSUPER_ADMINUids()) {
// UserService.insertAdminRoleAndWarn(SUPER_ADMINUid, project.getId());
// }
Project project = projectVO.creatProject(new Date());
projectDao.insertOneWithoutId(project);
......@@ -119,6 +114,7 @@ public class ProjectServiceImpl implements ProjectService {
project.getId(), contend.getBrandName(), contend.getId(), submitter, submitterId);
}
}
return ResponseResult.success();
}
@Override
......@@ -180,7 +176,10 @@ public class ProjectServiceImpl implements ProjectService {
}
@Override
public void updateProject(ProjectVO projectVO) {
public ResponseResult updateProject(ProjectVO projectVO) {
if (!Tools.isEmpty(projectVO.getHitKeywords()) && projectVO.getModuleShowList().contains(2)) {
return ResponseResult.failure("关键字和原始数据配置异常");
}
Project existsProject = projectDao.findOneById(projectVO.getId());
if (Objects.isNull(existsProject)) {
//抛出非法参数异常
......@@ -213,6 +212,7 @@ public class ProjectServiceImpl implements ProjectService {
project.getId(), contend.getBrandName(), contend.getId(), submitter, submitterId);
}
}
return ResponseResult.success();
}
@Override
......@@ -436,27 +436,6 @@ public class ProjectServiceImpl implements ProjectService {
.collect(Collectors.groupingBy(MarkerTag::getGroupName));
}
// public JSONObject getUserInfoAndProjectConfig(User user) {
// long start = System.currentTimeMillis();
// JSONObject result = new JSONObject();
// JSONObject userJson = new JSONObject();
// userJson.put("id", user.getId());
// userJson.put("nick", user.getNickname());
// userJson.put("username", user.getUsername());
// userJson.put("projectId", UserThreadLocal.getProjectId());
// userJson.put("roleId", UserThreadLocal.getRoleId());
// userJson.put("expiredTime",userProject.getExpiredTime());
// //userJson.put("exportAmount",userProject.getExportAmount());
// Project project = projectMapper.selectByPrimaryKey(userProject.getProjectId());
// userJson.put("projectName", project.getProjectName());
// userJson.put("endTime", project.getLastTime());
// userJson.put("isBackUp", project.getIsBackup());
// result.put("user", userJson);
// result.put("config", this.getConfig(userProject.getProjectId()));
// log.info("获取项目配置信息时间：{}", System.currentTimeMillis() - start);
// return result;
// }
protected List<JSONObject> getProjectListByUser(User user, boolean retainExpired) {
List<Project> lists;
Map<String, Date> roleExpired = new HashMap<>();
......@@ -476,6 +455,7 @@ public class ProjectServiceImpl implements ProjectService {
json.put("name", project.getProjectName());
json.put("group", project.getBrandLinkedGroup());
json.put("avatarUrl", project.getAvatarUrl());
json.put("permission", getPermissionList(project));
if (retainExpired) {
Date expiredTime = roleExpired.get(project.getId());
if (null == expiredTime || expiredTime.after(new Date())) {
......@@ -502,5 +482,18 @@ public class ProjectServiceImpl implements ProjectService {
return project;
}
/**
* 获取权限列表
*
* @param project
* @return
*/
private List<Map<String, Object>> getPermissionList(Project project) {
List<Map<String, Object>> permissionList = new ArrayList<>();
Map<String, Object> originPermission = new HashMap<>();
originPermission.put("origin", project.getModuleShowList().contains(2));
permissionList.add(originPermission);
return permissionList;
}
}
src/main/java/com/zhiwei/brandkbs2/service/impl/ProjectWarnServiceImpl.java
......@@ -649,7 +649,7 @@ public class ProjectWarnServiceImpl implements ProjectWarnService {
project = projectService.getProjectById(projectId);
log.info("获取渠道参与预警开始,project:{}", project.getProjectName());
EsClientDao.SearchHelper helper = EsClientDao.createSearchHelper();
BoolQueryBuilder postFilter = EsQueryTools.assembleCacheMapsQuery(project.getId(), project.getBrandLinkedGroupId(), "0");
BoolQueryBuilder postFilter = EsQueryTools.assembleCacheMapsQuery(project.getId(), "0");
// 时间(标注时间)
postFilter.must(QueryBuilders.rangeQuery("mtime").gte(start).lt(end));
// 自定义渠道
......
src/main/java/com/zhiwei/brandkbs2/service/impl/ReportServiceImpl.java
......@@ -387,17 +387,16 @@ public class ReportServiceImpl implements ReportService {
Long lastStartTimeStr = startTime - (endTime - startTime);
String projectId = report.getProjectId();
String contendId = "0";
String linkedGroupId = projectService.getProjectByContendId(projectId, contendId).getBrandLinkedGroupId();
JSONObject result = new JSONObject();
result.put("title", report.getTitle());
result.put("startTime", startTime);
result.put("endTime", endTime);
result.put("status", true);
result.put("type", ReportTypeEnum.getInstanceByState(report.getType()).getDescribe());
result.put("brandSummary", this.getPcBrandSummary(startTime, endTime, lastStartTimeStr, projectId, linkedGroupId, contendId));
result.put("brandSummary", this.getPcBrandSummary(startTime, endTime, lastStartTimeStr, projectId, contendId));
Pair<Boolean, List<Map<String, Long>>> pair = Tools.getCutList(startTime, endTime);
result.put("timeType", pair.getLeft() ? "day" : "hour");
result.put("brandSpread", this.getPcBrandSpread(pair.getRight(), projectId, linkedGroupId, contendId));
result.put("brandSpread", this.getPcBrandSpread(pair.getRight(), projectId, contendId));
result.put("brandEvent", this.getPcBrandEvent(startTime, endTime, projectId, contendId));
List<JSONObject> contendCompare = report.getContends().stream().map(brandContendId -> {
JSONObject json = new JSONObject();
......@@ -496,13 +495,13 @@ public class ReportServiceImpl implements ReportService {
result.put("curNeuPro", curMonthTotal == 0 ? 0 : curNeutralTotal * 1.0 / curMonthTotal);
result.put("curNegPro", curMonthTotal == 0 ? 0 : curNegativeTotal * 1.0 / curMonthTotal);
result.put("posCompare", lastPositiveTotal == 0 ? 0 : (curPositiveTotal - lastPositiveTotal) * 1.0 / lastPositiveTotal);
List<JSONObject> platforms = markDataService.getMarkPlatformProportion(startTime, endTime, true);
List<JSONObject> platforms = markDataService.getMarkPlatformProportion(projectId, startTime, endTime, true);
result.put("maxPlatform", platforms.stream().max(Comparator.comparing(o -> o.getLong("count"))).get().getString("platform"));
result.put("minPlatform", platforms.stream().min(Comparator.comparing(o -> o.getLong("count"))).get().getString("platform"));
List<JSONObject> posPlatforms = markDataService.getMarkPlatformProportion(startTime, endTime, EmotionEnum.POSITIVE.getName(),true);
List<JSONObject> posPlatforms = markDataService.getMarkPlatformProportion(projectId, startTime, endTime, EmotionEnum.POSITIVE.getName(), true);
result.put("maxPosPlatform", posPlatforms.stream().max(Comparator.comparing(o -> o.getLong("count"))).get().getString("platform"));
List<JSONObject> neuPlatforms = markDataService.getMarkPlatformProportion(startTime, endTime, EmotionEnum.NEUTRAL.getName(),true);
List<JSONObject> negPlatforms = markDataService.getMarkPlatformProportion(startTime, endTime, EmotionEnum.NEGATIVE.getName(),true);
List<JSONObject> neuPlatforms = markDataService.getMarkPlatformProportion(projectId, startTime, endTime, EmotionEnum.NEUTRAL.getName(), true);
List<JSONObject> negPlatforms = markDataService.getMarkPlatformProportion(projectId, startTime, endTime, EmotionEnum.NEGATIVE.getName(), true);
result.put("maxNegPlatform", negPlatforms.stream().max(Comparator.comparing(o -> o.getLong("count"))).get().getString("platform"));
platforms.forEach(platform -> {
String platformStr = platform.getString("platform");
......@@ -573,11 +572,10 @@ public class ReportServiceImpl implements ReportService {
result.put("eventTotal", eventTotal);
result.put("posEventPro", eventTotal == 0 ? 0 : posEventTotal * 1.0 / eventTotal);
//获取最热稿件相关信息
List<Map.Entry<String, Integer>> resList = markDataService.getMarkTopTitle(start, end, EmotionEnum.ALL.getName(), projectId, null,
Constant.PRIMARY_CONTEND_ID, 1);
List<Map.Entry<String, Integer>> resList = markDataService.getMarkTopTitle(start, end, EmotionEnum.ALL.getName(), projectId, Constant.PRIMARY_CONTEND_ID, 1);
if (CollectionUtils.isNotEmpty(resList)) {
Map.Entry<String, Integer> resMap = resList.get(0);
BaseMap firstArticle = markDataService.getFirstArticle(start, end, resMap.getKey(), projectId, null, Constant.PRIMARY_CONTEND_ID);
BaseMap firstArticle = markDataService.getFirstArticle(start, end, resMap.getKey(), projectId, Constant.PRIMARY_CONTEND_ID);
result.put("topArticle", firstArticle.getTitle());
result.put("similarArticleCount", resMap.getValue());
}
......@@ -681,7 +679,7 @@ public class ReportServiceImpl implements ReportService {
* @param projectId 项目ID
* @return 总览页面数据
*/
private JSONObject getPcBrandSummary(Long startTime, Long endTime, Long lastStartTime, String projectId, String linkedGroupId, String contendId) throws IOException {
private JSONObject getPcBrandSummary(Long startTime, Long endTime, Long lastStartTime, String projectId, String contendId) throws IOException {
JSONObject result = new JSONObject();
//本月舆情总量数据
long curTotal = markDataService.getYuqingMarkCount(startTime, endTime, EmotionEnum.ALL.getName(), projectId, contendId);
......@@ -699,8 +697,8 @@ public class ReportServiceImpl implements ReportService {
List<com.zhiwei.middleware.event.pojo.entity.Event> topPosEventList =
eventMiddlewareDao.getEventsByTotalChannelVolumeTop(startTime, endTime, EmotionEnum.POSITIVE.getName(), projectId, projectId, 3);
if (CollectionUtils.isEmpty(topPosEventList)) {
List<Map.Entry<String, Integer>> topPosArticleList = markDataService.getMarkTopTitle(startTime, endTime, EmotionEnum.POSITIVE.getName(), projectId, linkedGroupId, contendId, 3);
result.put("topPosSummary", this.getTopArticlesMsg(startTime, endTime, projectId, linkedGroupId, contendId, topPosArticleList));
List<Map.Entry<String, Integer>> topPosArticleList = markDataService.getMarkTopTitle(startTime, endTime, EmotionEnum.POSITIVE.getName(), projectId, contendId, 3);
result.put("topPosSummary", this.getTopArticlesMsg(startTime, endTime, projectId, contendId, topPosArticleList));
} else {
result.put("topPosSummary", this.getTopEventMsg(topPosEventList, projectId));
}
......@@ -717,8 +715,8 @@ public class ReportServiceImpl implements ReportService {
List<com.zhiwei.middleware.event.pojo.entity.Event> topNeuEventList =
eventMiddlewareDao.getEventsByTotalChannelVolumeTop(startTime, endTime, EmotionEnum.NEUTRAL.getName(), projectId, projectId, 4);
if (CollectionUtils.isEmpty(topNeuEventList)) {
List<Map.Entry<String, Integer>> topNeuArticleList = markDataService.getMarkTopTitle(startTime, endTime, EmotionEnum.NEUTRAL.getName(), projectId, linkedGroupId, contendId, 4);
result.put("topNeuSummary", this.getTopArticlesMsg(startTime, endTime, projectId, linkedGroupId, contendId, topNeuArticleList));
List<Map.Entry<String, Integer>> topNeuArticleList = markDataService.getMarkTopTitle(startTime, endTime, EmotionEnum.NEUTRAL.getName(), projectId, contendId, 4);
result.put("topNeuSummary", this.getTopArticlesMsg(startTime, endTime, projectId, contendId, topNeuArticleList));
} else {
result.put("topNeuSummary", this.getTopEventMsg(topNeuEventList, projectId));
}
......@@ -735,8 +733,8 @@ public class ReportServiceImpl implements ReportService {
List<com.zhiwei.middleware.event.pojo.entity.Event> topNegEventList =
eventMiddlewareDao.getEventsByTotalChannelVolumeTop(startTime, endTime, EmotionEnum.NEGATIVE.getName(), projectId, projectId, 4);
if (CollectionUtils.isEmpty(topNegEventList)) {
List<Map.Entry<String, Integer>> topNegArticleList = markDataService.getMarkTopTitle(startTime, endTime, EmotionEnum.NEGATIVE.getName(), projectId, linkedGroupId, contendId, 4);
result.put("topNegSummary", this.getTopArticlesMsg(startTime, endTime, projectId, linkedGroupId, contendId, topNegArticleList));
List<Map.Entry<String, Integer>> topNegArticleList = markDataService.getMarkTopTitle(startTime, endTime, EmotionEnum.NEGATIVE.getName(), projectId, contendId, 4);
result.put("topNegSummary", this.getTopArticlesMsg(startTime, endTime, projectId, contendId, topNegArticleList));
} else {
result.put("topNegSummary", this.getTopEventMsg(topNegEventList, projectId));
}
......@@ -754,7 +752,7 @@ public class ReportServiceImpl implements ReportService {
* @param projectId 项目ID
* @return 传播分析页面数据
*/
private JSONObject getPcBrandSpread(List<Map<String, Long>> dayList, String projectId, String linkedGroupId, String contendId) throws IOException {
private JSONObject getPcBrandSpread(List<Map<String, Long>> dayList, String projectId, String contendId) throws IOException {
JSONObject result = new JSONObject();
List<JSONObject> lineList = new ArrayList<>(dayList.size());
//获取传播趋势
......@@ -785,10 +783,10 @@ public class ReportServiceImpl implements ReportService {
Long startTime = dateList.get(0);
Long endTime = DateUtils.addDays(new Date(startTime), 1).getTime();
//获取最热稿件相关信息
List<Map.Entry<String, Integer>> resList = markDataService.getMarkTopTitle(startTime, endTime, EmotionEnum.ALL.getName(), projectId, linkedGroupId, contendId, 1);
List<Map.Entry<String, Integer>> resList = markDataService.getMarkTopTitle(startTime, endTime, EmotionEnum.ALL.getName(), projectId, contendId, 1);
if (CollectionUtils.isNotEmpty(resList)) {
Map.Entry<String, Integer> entry = resList.get(0);
BaseMap baseMap = markDataService.getFirstArticle(startTime, endTime, entry.getKey(), projectId, linkedGroupId, contendId);
BaseMap baseMap = markDataService.getFirstArticle(startTime, endTime, entry.getKey(), projectId, contendId);
result.put("topArticle", baseMap.getTitle());
result.put("topArticleUrl", baseMap.getUrl());
result.put("topArticleEmotion", baseMap.getEmotion());
......@@ -854,12 +852,12 @@ public class ReportServiceImpl implements ReportService {
* @param topArticleList top稿件集合
* @return top稿件信息
*/
private List<JSONObject> getTopArticlesMsg(Long startTime, Long endTime, String projectId, String linkedGroupId, String contendId, List<Map.Entry<String, Integer>> topArticleList) {
private List<JSONObject> getTopArticlesMsg(Long startTime, Long endTime, String projectId, String contendId, List<Map.Entry<String, Integer>> topArticleList) {
return topArticleList.stream().map(entry -> {
JSONObject topPosArticle = null;
try {
String aggTitle = entry.getKey();
BaseMap firstArticle = markDataService.getFirstArticle(startTime, endTime, aggTitle, projectId, linkedGroupId, contendId);
BaseMap firstArticle = markDataService.getFirstArticle(startTime, endTime, aggTitle, projectId, contendId);
topPosArticle = new JSONObject();
topPosArticle.put("title", firstArticle.getTitle());
topPosArticle.put("url", firstArticle.getUrl());
......
src/main/java/com/zhiwei/brandkbs2/util/Tools.java
......@@ -19,6 +19,7 @@ import com.zhiwei.brandkbs2.pojo.BaseMap;
import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform;
import com.zhiwei.qbjc.bean.pojo.common.Tag;
import com.zhiwei.qbjc.bean.tools.BeanTools;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang3.StringUtils;
......@@ -41,9 +42,12 @@ import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel;
import java.text.ParseException;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import static com.zhiwei.brandkbs2.common.GenericAttribute.ES_MARK_CACHE_MAPS;
import static com.zhiwei.brandkbs2.config.Constant.*;
......@@ -959,6 +963,16 @@ public class Tools {
return new JSONObject();
}
public static JSONObject getHotEsHitMap(Map<String, Object> esMap, String hitKey) {
List<Map<String, Object>> cacheMaps = (List<Map<String, Object>>) esMap.get("monitor_rule_cache_maps");
for (Map<String, Object> cacheMap : cacheMaps) {
if (hitKey.equals(cacheMap.get("project"))) {
return new JSONObject(cacheMap);
}
}
return new JSONObject();
}
public static JSONObject getBrandkbsHitMapWithProjectId(Map<String, Object> esMap, String projectId) {
List<Map<String, Object>> cacheMaps = (List<Map<String, Object>>) esMap.get(GenericAttribute.ES_BRANDKBS_CACHE_MAPS);
for (Map<String, Object> cacheMap : cacheMaps) {
......@@ -1025,4 +1039,124 @@ public class Tools {
public static boolean isNullOrUndefined(Object o) {
return (isEmpty(o) || Objects.equals("undefined", o) || Objects.equals("null", o));
}
/**
* 正则表达式匹配结果
*
* @param line
* @param pattern
* @return List<String>
*/
public static List<String> patternMatchFind(String line, String pattern) {
List<String> res = new ArrayList<>();
// 创建 Pattern 对象
Pattern r = Pattern.compile(pattern);
// 现在创建 matcher 对象
Matcher m = r.matcher(line);
while (m.find()) {
res.add(m.group());
}
return res;
}
/**
* gzip压缩字符串
*
* @param primStr
* @return
*/
public static String gzip(String primStr) {
if (primStr == null || primStr.length() == 0) {
return primStr;
}
ByteArrayOutputStream out = new ByteArrayOutputStream();
GZIPOutputStream gzip = null;
try {
gzip = new GZIPOutputStream(out);
gzip.write(primStr.getBytes());
} catch (IOException e) {
e.printStackTrace();
} finally {
if (gzip != null) {
try {
gzip.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return new String(new Base64().encode(out.toByteArray()));
}
/**
* 使用gzip进行解压缩
*
* @param compressedStr
* @return
*/
public static String gunzip(String compressedStr) {
if (compressedStr == null) {
return null;
}
ByteArrayOutputStream out = new ByteArrayOutputStream();
ByteArrayInputStream in = null;
GZIPInputStream ginzip = null;
byte[] compressed = null;
String decompressed = null;
try {
compressed = new Base64().decode(compressedStr);
in = new ByteArrayInputStream(compressed);
ginzip = new GZIPInputStream(in);
byte[] buffer = new byte[102400];
int offset = -1;
while ((offset = ginzip.read(buffer)) != -1) {
out.write(buffer, 0, offset);
}
decompressed = out.toString();
} catch (IOException e) {
e.printStackTrace();
} finally {
if (ginzip != null) {
try {
ginzip.close();
} catch (IOException e) {
}
}
if (in != null) {
try {
in.close();
} catch (IOException e) {
}
}
if (out != null) {
try {
out.close();
} catch (IOException e) {
}
}
}
return decompressed;
}
/**
* 判断标签是否合法
* @param mtag
* @return
*/
public static boolean isIllegalTag(String mtag) {
// 非空
if (isEmpty(mtag)) {
return false;
}
String pattern = ",[\\d]+\\=";
List<String> mtags = Tools.isEmpty(mtag) ? new ArrayList<>() : Tools.patternMatchFind(mtag, pattern);
// 无重复标签组
long count = mtags.stream().distinct().count();
return count != mtags.size();
}
}
\ No newline at end of file
src/main/resources/application-dev.properties
......@@ -38,6 +38,10 @@ es.httpClusterNodes=192.168.0.130:9200:qbjc-back:yuqing.zhiweidata.com,192.168.0
#es.password=3vh65l$i6qQA
es.username=joker
es.password=jokerdevops
es.clusterNodesHotES=192.168.0.51:9400
es.clusterNameHotES=hot-new-es
es.usernameHotES=elastic
es.passwordHotES=qWxZRW42OHkuOhmF5AXX
es.index.test = false
#channel-index
......@@ -53,6 +57,8 @@ mark.provider.group=zhiwei-mark
#\u4E8B\u4EF6\u4E2D\u95F4\u4EF6
event.registry.address=zookeeper://192.168.0.203:2182?backup=192.168.0.104:2182,192.168.0.105:2182
event.provider.group=zhiwei-event
#\u53BB\u91CD\u4E2D\u95F4\u4EF6
filter.provider.group=zhiwei-bloom-filter
#\u7528\u6237\u4E2D\u5FC3
auth.center.client.consumer.group=zhiwei-auth
auth.center.client.registry.address=zookeeper://192.168.0.203:2182?backup=192.168.0.104:2182,192.168.0.105:2182
......
src/main/resources/application-local.properties
......@@ -40,6 +40,10 @@ es.httpClusterNodes=202.107.192.94:1443:brandkbs2:3vh65l$i6qQA
#es.password=3vh65l$i6qQA
es.username=joker
es.password=jokerdevops
es.clusterNodesHotES=202.107.192.94:29400
es.clusterNameHotES=hot-new-es
es.usernameHotES=elastic
es.passwordHotES=qWxZRW42OHkuOhmF5AXX
es.index.test = false
#channel-index
......@@ -55,6 +59,8 @@ mark.provider.group=zhiwei-mark-test_liuyu
#\u4E8B\u4EF6\u4E2D\u95F4\u4EF6
event.registry.address=zookeeper://192.168.0.11:2181?backup=192.168.0.30:2181,192.168.0.35:2181
event.provider.group=zhiwei-event-ygd
#\u53BB\u91CD\u4E2D\u95F4\u4EF6
filter.provider.group=local-filter
#\u7528\u6237\u4E2D\u5FC3
auth.center.client.consumer.group=zhiwei-auth-dev-liuyu
auth.center.client.registry.address=zookeeper://192.168.0.11:2181?backup=192.168.0.30:2181,192.168.0.35:2181
......
src/main/resources/application-prod.properties
......@@ -38,6 +38,10 @@ es.httpClusterNodes=192.168.0.130:9200:qbjc-back:yuqing.zhiweidata.com,192.168.0
#es.password=3vh65l$i6qQA
es.username=joker
es.password=jokerdevops
es.clusterNodesHotES=192.168.0.51:9400
es.clusterNameHotES=hot-new-es
es.usernameHotES=elastic
es.passwordHotES=qWxZRW42OHkuOhmF5AXX
es.index.test = false
#channel-index
......@@ -53,6 +57,8 @@ mark.provider.group=zhiwei-mark
#\u4E8B\u4EF6\u4E2D\u95F4\u4EF6
event.registry.address=zookeeper://192.168.0.203:2182?backup=192.168.0.104:2182,192.168.0.105:2182
event.provider.group=zhiwei-event
#\u53BB\u91CD\u4E2D\u95F4\u4EF6
filter.provider.group=zhiwei-bloom-filter
#\u7528\u6237\u4E2D\u5FC3
auth.center.client.consumer.group=zhiwei-auth
auth.center.client.registry.address=zookeeper://192.168.0.203:2182?backup=192.168.0.104:2182,192.168.0.105:2182
......
src/test/java/com/zhiwei/brandkbs2/BrandkbsIndexCreateTest.java
package com.zhiwei.brandkbs2;
import com.zhiwei.brandkbs2.es.ChannelEsDao;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.client.RequestOptions;
......@@ -30,9 +31,48 @@ public class BrandkbsIndexCreateTest {
public void test1() throws IOException {
String channelIndex = "brandkbs2_channel_record_2023";
// String brandkbsIndex = "brandkbs2_2022";
putChannelIndex(channelIndex);
// putChannelIndex(channelIndex);
// putBrandkbsIndex("brandkbs2_2018","brandkbs2_2019","brandkbs2_2020","brandkbs2_2021");
// putBrandkbsIndex("brandkbs2_2017");
putChannelCopyIndex(ChannelEsDao.CHANNEL_COPY_ES_INDEX_NAME);
}
public void putChannelCopyIndex(String... indexes) throws IOException {
for (String index : indexes) {
CreateIndexRequest createIndexRequest = new CreateIndexRequest(index);
// 设置分片和副本
createIndexRequest.settings(Settings.builder().put("number_of_shards", "7").put("number_of_replicas", "1"));
Map<String, Object> properties = new HashMap<>();
builder(properties, "text", "id", true);
builder(properties, "text", "project_id", true);
builder(properties, "text", "contend_id", true);
builder(properties, "keyword", "platform", false);
builder(properties, "keyword", "real_source", false);
builder(properties, "keyword", "source", false);
builder(properties, "text", "fid", true);
builder(properties, "integer", "emotion", false);
builder(properties, "float", "emotion_index", false);
builder(properties, "long", "c_time", false);
builder(properties, "long", "last_time", false);
builder(properties, "text", "avatar_url", false);
builder(properties, "long", "article_count", false);
builder(properties, "long", "event_count", false);
builder(properties, "text", "experience_level", true);
builder(properties, "boolean", "show", false);
builder(properties, "boolean", "is_collect", false);
builder(properties, "long", "collect_time", false);
builder(properties, "float", "influence", false);
Map<String, Object> mapping = new HashMap<>();
mapping.put("properties", properties);
createIndexRequest.mapping(mapping);
//设置超时时间 分钟
createIndexRequest.setTimeout(TimeValue.timeValueMinutes(1));
//设置主节点超时时间
createIndexRequest.setMasterTimeout(TimeValue.timeValueMinutes(1));
esClient.indices().create(createIndexRequest, RequestOptions.DEFAULT);
log.info("索引:{}已创建", index);
}
}
public void putChannelIndex(String... indexes) throws IOException {
......
src/test/java/com/zhiwei/brandkbs2/CopyChannelFromMongo2Es.java
......@@ -65,7 +65,7 @@ public class CopyChannelFromMongo2Es {
while (mongoQueryUtil.hasNextPage()) {
List<Channel> list = channelDao.findList(mongoQueryUtil.getNextPageQuery());
List<CompletableFuture<Map<String, Object>>> futureList = list.stream().map(channel -> CompletableFuture.supplyAsync(() -> {
JSONObject markData = markDataService.getLastMarkData(channel.getProjectId(), "-1", channel.getContendId(), channel.getPlatform(), channel.getRealSource(), channel.getSource());
JSONObject markData = markDataService.getLastMarkData(channel.getProjectId(), channel.getContendId(), channel.getPlatform(), channel.getRealSource(), channel.getSource());
if (null != markData) {
channel.setLastArticle(markData.toJSONString());
}
......
src/test/java/com/zhiwei/brandkbs2/MarkDataServiceTest.java
package com.zhiwei.brandkbs2;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.enmus.EmotionEnum;
import com.zhiwei.brandkbs2.pojo.BaseMap;
import com.zhiwei.brandkbs2.pojo.Event;
......@@ -43,7 +44,7 @@ public class MarkDataServiceTest {
@Test
public void getYuqingMarkAggreeProgressTest(){
String yuqingMarkAggreeProgress = markDataService.getYuqingMarkAggreeProgress("2e555f743f454ff29fc35bda2668c72c");
String yuqingMarkAggreeProgress = markDataService.getAggreeProgress("2e555f743f454ff29fc35bda2668c72c");
System.out.println(yuqingMarkAggreeProgress);
}
......@@ -90,7 +91,7 @@ public class MarkDataServiceTest {
@Test
public void getMarkPlatformProportion(){
List<JSONObject> markPlatformProportion = markDataService.getMarkPlatformProportion(1658073600000L, 1661937298000L, false);
List<JSONObject> markPlatformProportion = markDataService.getMarkPlatformProportion(Constant.DEFAULT_PROJECT_ID,1658073600000L, 1661937298000L, false);
for (JSONObject jsonObject : markPlatformProportion) {
System.out.println(jsonObject);
}
......@@ -107,7 +108,7 @@ public class MarkDataServiceTest {
@Test
public void getMarkTopTitle() throws IOException {
List<Map.Entry<String, Integer>> markTopTitle =
markDataService.getMarkTopTitle(1658073600000L, 1661937298000L, null, "62beadd1bbf8eb20f96d2f2e", "136", "62e1f113b746930599b616a0", 10);
markDataService.getMarkTopTitle(1658073600000L, 1661937298000L, "62beadd1bbf8eb20f96d2f2e", "136", "62e1f113b746930599b616a0", 10);
for (Map.Entry<String, Integer> stringIntegerEntry : markTopTitle) {
System.out.println(stringIntegerEntry);
}
......@@ -116,7 +117,7 @@ public class MarkDataServiceTest {
@Test
public void getFirstArticleTest() throws IOException {
BaseMap article = markDataService.getFirstArticle(1658073600000L, 1661727700000L, "飞鹤杯小小朗读者活动北上广津等八位公益推广大使来啦",
"62beadd1bbf8eb20f96d2f2e", "136", "62e1f113b746930599b616a0");
"62beadd1bbf8eb20f96d2f2e", "136");
System.out.println(article.getContent());
}
......@@ -136,7 +137,7 @@ public class MarkDataServiceTest {
@Test
public void searchMarkDataByTimeTest(){
List<JSONObject> jsonObjects =
markDataService.searchMarkDataByTime("62beadd1bbf8eb20f96d2f2d", "23", "62e1f113b746930599b6167e", 1658073600000L, 1661727700000L);
markDataService.searchMarkDataByTime("62beadd1bbf8eb20f96d2f2d","62e1f113b746930599b6167e", 1658073600000L, 1661727700000L);
for (JSONObject jsonObject : jsonObjects) {
System.out.println(jsonObject);
}
......@@ -206,12 +207,12 @@ public class MarkDataServiceTest {
@Test
public void getEsTopSourceTest() throws IOException {
List<Map<String, Object>> esTopSource = markDataService.getEsTopSource(1663516800000L, 1666108800000L, "62beadd1bbf8eb20f96d2f1a", "25", "0",
List<Map<String, Object>> esTopSource = markDataService.getEsTopSource(1663516800000L, 1666108800000L, "62beadd1bbf8eb20f96d2f1a", "0",
EmotionEnum.POSITIVE.getName(), 3);
for (Map<String, Object> stringObjectMap : esTopSource) {
System.out.println(stringObjectMap.entrySet());
}
List<Map<String, Object>> esTopSource2 = markDataService.getEsTopSource(1663516800000L, 1666108800000L, "62beadd1bbf8eb20f96d2f1a", "25", "0",
List<Map<String, Object>> esTopSource2 = markDataService.getEsTopSource(1663516800000L, 1666108800000L, "62beadd1bbf8eb20f96d2f1a", "0",
EmotionEnum.NEGATIVE.getName(), 3);
for (Map<String, Object> stringObjectMap : esTopSource2) {
System.out.println(stringObjectMap.entrySet());
......
src/test/java/com/zhiwei/brandkbs2/Test.java
package com.zhiwei.brandkbs2;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.JSON;
import com.hankcs.hanlp.dictionary.CustomDictionary;
import com.hankcs.hanlp.dictionary.stopword.CoreStopWordDictionary;
import com.zhiwei.brandkbs2.util.Tools;
import com.zhiwei.brandkbs2.pojo.dto.ExportAppYuqingDTO;
import org.apache.commons.lang3.StringUtils;
import java.io.UnsupportedEncodingException;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Objects;
/**
* @ClassName: Test
......@@ -15,44 +20,76 @@ import java.io.UnsupportedEncodingException;
*/
public class Test {
public static final String PROJECT_WARN_NEW_CASE = "BRANDKBS:NEW_CASE:";
private static final String SEPARATOR = ":";
static{
CustomDictionary.add("猪肉供应");
CoreStopWordDictionary.add("保障");
}
public static void main(String[] args) throws UnsupportedEncodingException {
System.out.println(JSONObject.toJSONString(Tools.parseToDays(1658512800000L,1659031200000L)));
// List<Term> results= HanLP.segment("国家发改委：猪肉供应有保障 猪肉价格不具备大幅上涨基础");
// for(Term term: CoreStopWordDictionary.apply(results)){
//// if(CoreStopWordDictionary.shouldInclude(term)){
//// System.out.println(term);
//// }else{
// System.err.println(term);
//// }
//
// }
// String token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJwYXlsb2FkIjoie1widXNlcklkXCI6XCIyMFwiLFwia2V5XCI6XCIyMFwiLFwibmlja05hbWVcIjpcIuayiOWQm-adsFwiLFwic2VydmljZVwiOlwiXCJ9IiwiZXhwIjoxNjUzMDExNjcwLCJpYXQiOjE2NTI0MDY4MTB9.jcVXxeZkayc6-Aiq8cyYc1uyq4ugji6FdWQXCCp4M2o";
// String token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJwYXlsb2FkIjoiXCLlk4Hop4FcIiIsImV4cCI6NDc2MjgyMjEzMiwiaWF0IjoxNjUyNDIyMDcyfQ.DXQ8yKgfsCMjhT0xniZeWCMv4syqIoDvztU4QWsd-Fg";
// String t = JwtUtil.unsign(token, String.class);
// System.out.println(JSONObject.toJSONString(t));
// String test = URLEncoder.encode("测试", "utf-8");
// System.out.println(test);
// String projectId = "1";
// List<UserRole> list = new ArrayList<>();
// UserDTO userDto = new UserDTO();
// userDto.setProjectId(projectId);
// userDto.setRoleId(1);
// list.add(UserRole.createFromUserDto(userDto));
//
// list.stream().filter(userRole -> userRole.getProjectId().equals(projectId)).findAny().ifPresent(role -> {
// role.setRoleId(2);
// });
// System.out.println(JSONObject.toJSONString(list));
public static String newCrisisCaseListKey() {
return generateRedisKey(PROJECT_WARN_NEW_CASE, "LIST");
}
private static String generateRedisKey(String... keys) {
Objects.requireNonNull(keys);
boolean contains = keys[0].endsWith(":");
StringBuilder sb = new StringBuilder(keys[0]);
for (int i = 1; i < keys.length; i++) {
if (1 == i && contains) {
sb.append(keys[i]);
} else {
sb.append(SEPARATOR).append(keys[i]);
}
}
return sb.toString();
}
public static void main(String[] args) {
List<ExportAppYuqingDTO> list = new ArrayList<>();
for (int i = 0; i < 2; i++) {
ExportAppYuqingDTO dto = new ExportAppYuqingDTO();
dto.setTitle("真的是离大谱啊真的是离大谱啊真的是离大谱啊真的是离大谱啊真的是离大谱啊真的是离大谱啊真的是离大谱啊真的是离大谱啊真的是离大谱啊");
dto.setSource("JJ" + i);
dto.setTime(new Date(System.currentTimeMillis()));
list.add(dto);
}
// beforeTest(ExportAppYuqingDTO.class);
test(ExportAppYuqingDTO.class, list);
for (ExportAppYuqingDTO dto : list) {
System.out.println(JSON.toJSONString(dto));
}
}
private static <T> void beforeTest(Class<T> clazz){
for (Field field : clazz.getDeclaredFields()) {
// String格式截取前32767位
if (field.getGenericType().toString().equals(String.class.toString())) {
field.setAccessible(true);
}
}
}
private static <T> void test(Class<T> clazz, List<T> datas) {
List<Field> stringFields = new ArrayList<>();
// 记录需要设置的部分
for (Field field : clazz.getDeclaredFields()) {
// String格式截取前32767位
if (field.getGenericType().toString().equals(String.class.toString())) {
field.setAccessible(true);
stringFields.add(field);
}
}
datas.forEach(data -> {
for (Field field : stringFields) {
try {
field.set(data, StringUtils.substring((String) (field.get(data)), 0, 10));
} catch (IllegalAccessException e) {
e.printStackTrace();
}
}
});
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment