Commit b1bcbde8 by shenjunjie

Merge branch 'feature' into 'dev'

Feature

See merge request !454
parents 97354d9e a03528a4
......@@ -47,6 +47,7 @@ public class GenericAttribute {
* es gid
**/
public static final String ES_GID = "gid";
public static final String ES_MGID = "mgid";
public static final String ES_PROJECT_ID = "project_id";
public static final String ES_CONTEND_ID = "contend_id";
public static final String ES_CHANNEL_FID = "fid";
......@@ -88,6 +89,7 @@ public class GenericAttribute {
* es mtime
**/
public static final String ES_MTIME = "mtime";
public static final String ES_STIME = "stime";
/**
* es mtag
**/
......
......@@ -5,6 +5,7 @@ import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.pojo.ChannelTag;
import com.zhiwei.brandkbs2.pojo.HighlightWord;
import com.zhiwei.brandkbs2.pojo.Project;
import com.zhiwei.brandkbs2.service.ChannelService;
import com.zhiwei.brandkbs2.service.SystemInfoService;
import com.zhiwei.brandkbs2.util.Tools;
import com.zhiwei.middleware.automaticmark.graphs.Graphs;
......@@ -46,6 +47,9 @@ public class GlobalPojo {
@Resource(name = "highlightWordDao")
private HighlightWordDao highlightWordDao;
@Resource(name = "channelServiceImpl")
private ChannelService channelService;
/**
* 监测系统平台
**/
......@@ -82,6 +86,8 @@ public class GlobalPojo {
public static Map<String, Graphs> PROJECT_GRAPHS = new HashMap<>();
public static Map<String, JSONObject> PROJECT_EMOTION_CHANNEL_DATA = new HashMap<>();
public static final List<String> PERMANENT_PLATFORM_NAMES = Arrays.asList("网媒", "微博", "微信", "今日头条");
public static final String ELSE_PLATFORM_NAME = "其他自媒体";
......@@ -119,9 +125,10 @@ public class GlobalPojo {
MEDIA_TYPE = systemInfoService.getMediaTypes();
PROJECT_MAP = systemInfoService.getProjects();
YU_QING_PROJECTS = systemInfoService.getYuQingProjects();
PROJECT_EMOTION_CHANNEL_DATA = channelService.getProjectEmotionChannelListData();
updateHighlightGraphs();
log.info("{}-获取PLATFORMS-size:{},TAGS-size:{},LINKED_GROUP_ID_TAGS:{},CHANNEL_TAGS:{},MEDIA_TYPE:{},PROJECT_MAP:{},YUQING-PROJECTS-size:{}", logMsg, PLATFORMS.size(), TAGS.size(),
LINKED_GROUP_ID_TAGS.size(), CHANNEL_TAGS.size(), MEDIA_TYPE.size(), PROJECT_MAP.size(), YU_QING_PROJECTS.size());
log.info("{}-获取PLATFORMS-size:{},TAGS-size:{},LINKED_GROUP_ID_TAGS:{},CHANNEL_TAGS:{},MEDIA_TYPE:{},PROJECT_MAP:{},YUQING-PROJECTS-size:{},PROJECT_EMOTION_CHANNEL_DATA-size:{}", logMsg, PLATFORMS.size(), TAGS.size(),
LINKED_GROUP_ID_TAGS.size(), CHANNEL_TAGS.size(), MEDIA_TYPE.size(), PROJECT_MAP.size(), YU_QING_PROJECTS.size(), PROJECT_EMOTION_CHANNEL_DATA.size());
} catch (Exception e) {
log.info("{}-获取缓存值异常", logMsg, e);
}
......
......@@ -102,6 +102,17 @@ public class RedisKeyPrefix {
public static final String NON_MANUAL_PROJECT_MARK_MAX_GID = "BRANDKBS:NON_MANUAL:PROJECT:MARK:MAX_GID:";
/**
* 新舆情分析页面相关缓存
*/
public static final String YUQING_ANALYZE_PROJECT_AVG_COUNT = "BRANDKBS:YUQING:ANALYZE:AMOUNT:AVG:";
public static final String YUQING_ANALYZE_EMOTION_DISTRIBUTION_AVG = "BRANDKBS:YUQING:ANALYZE:EMOTION:AVG:";
public static final String YUQING_ANALYZE_PLATFORM_AVG_COUNT = "BRANDKBS:YUQING:ANALYZE:PLATFORM:AVG:";
public static final String YUQING_ANALYZE_HIGH_WORD = "BRANDKBS:YUQING:ANALYZE:HIGH:WORD:";
public static String projectWarnHotTopKeyAll(String projectId, String type) {
return RedisKeyPrefix.generateRedisKey(RedisKeyPrefix.PROJECT_WARN_HOT_TOP, projectId, Tools.concat(type, "*"));
}
......
......@@ -54,6 +54,16 @@ public class CommonController extends BaseController {
}
}
@ApiOperation("获取平台类型及id")
@GetMapping("/get/platform-id")
public ResponseResult getPlatformWithId() {
try {
return ResponseResult.success(commonService.getQbjcPlatform("id", "name"));
} catch (Exception e) {
return ResponseResult.failure(e.getMessage());
}
}
@ApiOperation("获取当前用户拥有的所有项目及品牌列表")
@GetMapping("/user/getUserAllProjects")
@Auth(role = RoleEnum.CUSTOMER)
......
......@@ -2,10 +2,7 @@ package com.zhiwei.brandkbs2.controller;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.external.BrandkbsChannelConfig;
import com.zhiwei.brandkbs2.pojo.external.BrandkbsHotEventConfig;
import com.zhiwei.brandkbs2.pojo.external.BrandkbsHotTopConfig;
import com.zhiwei.brandkbs2.pojo.external.BrandkbsYuQingConfig;
import com.zhiwei.brandkbs2.pojo.external.*;
import com.zhiwei.brandkbs2.pojo.vo.CrisisCaseWarnVO;
import com.zhiwei.brandkbs2.service.ProjectService;
import com.zhiwei.brandkbs2.service.ProjectWarnService;
......@@ -58,12 +55,21 @@ public class InterfaceController {
@ApiOperation("获取舆情动态-预警结果")
@PostMapping("/warn/yuqing")
@Deprecated
public ResponseResult getYuqingWaring(@RequestBody JSONObject json) {
String projectId = json.getString("projectId");
BrandkbsYuQingConfig config = json.getObject("config", BrandkbsYuQingConfig.class);
return projectWarnService.getYuqingWaring(projectId, config);
}
@ApiOperation("获取舆情动态-预警结果")
@PostMapping("/warn/yuqingNew")
public ResponseResult getYuqingWaringNew(@RequestBody JSONObject json) {
String projectId = json.getString("projectId");
BrandkbsYuQingConfigNew config = json.getObject("config", BrandkbsYuQingConfigNew.class);
return projectWarnService.getYuqingWaringNew(projectId, config);
}
@ApiOperation("获取渠道参与-预警结果")
@PostMapping("/warn/channel")
public ResponseResult getChannelWaring(@RequestBody JSONObject json) {
......@@ -113,4 +119,19 @@ public class InterfaceController {
List<String> list = JSONObject.parseObject(texts).getJSONArray("texts").toJavaList(String.class);
return ResponseResult.success(textUtil.getHighWordsJson(list, 30));
}
@ApiOperation("获取用户项目列表分页")
@GetMapping("/middleware/user-project-page-data")
public ResponseResult getUserProjectPageData(@RequestParam(value = "current") int current,
@RequestParam(value = "pageSize") int pageSize,
@RequestParam(value = "userId", required = false) String userId,
@RequestParam(value = "keyword", required = false) String keyword) {
return projectService.getUserProjectPageData(userId, keyword, current, pageSize);
}
@ApiOperation("获取用户项目列表")
@GetMapping("/middleware/user-project-list")
public ResponseResult getUserProjectList(String userId) {
return projectService.getUserProject(userId);
}
}
......@@ -440,6 +440,91 @@ public class AppArticleController extends BaseController {
return ResponseResult.success(markDataService.getNonManualMarkAggreeList(markSearchDTO));
}
@ApiOperation("获取方案列表")
@GetMapping("/analyze/plan/list")
public ResponseResult getNonManualPlanList(){
return ResponseResult.success(markDataService.getNonManualPlanList());
}
@ApiOperation("新-舆情分析-舆情总量")
@GetMapping("/analyze/amount")
public ResponseResult getYuqingAmount(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "planId", required = false) String planId) {
return ResponseResult.success(markDataService.getYuqingAmount(startTime, endTime, planId));
}
@ApiOperation("新-舆情分析-情感分布")
@GetMapping("/analyze/emotion")
public ResponseResult getYuqingEmotionDistribution(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "planId", required = false) String planId) {
return ResponseResult.success(markDataService.getYuqingEmotionDistribution(startTime, endTime, planId));
}
@ApiOperation("新-舆情分析-重点平台")
@GetMapping("/analyze/important-platform")
public ResponseResult getImportantPlatformPercentage(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "planId", required = false) String planId) {
return ResponseResult.success(markDataService.getImportantPlatformPercentage(startTime, endTime, planId));
}
@ApiOperation("新-舆情分析-平台占比")
@GetMapping("/analyze/platform-percent")
public ResponseResult getPlatformPercentage(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "planId", required = false) String planId) {
return ResponseResult.success(markDataService.getPlatformPercentage(startTime, endTime, planId));
}
@ApiOperation("新-舆情分析-舆情走势图")
@GetMapping("/analyze/tendency")
public ResponseResult getSpreadTendency(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "planId", required = false) String planId) {
return ResponseResult.success(markDataService.getSpreadTendency(startTime, endTime, planId));
}
@ApiOperation("新-舆情分析-活跃渠道")
@GetMapping("/analyze/active-channel")
public ResponseResult getActiveChannels(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "planId", required = false) String planId) {
return ResponseResult.success(markDataService.getActiveChannels(startTime, endTime, planId));
}
@ApiOperation("新-舆情分析-ip分布")
@GetMapping("/analyze/ip-located")
public ResponseResult getArticleIpLocated(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "planId", required = false) String planId,
@RequestParam(value = "size") int size) {
return ResponseResult.success(markDataService.getArticleIpLocated(startTime, endTime, planId, size));
}
@ApiOperation("新-舆情分析-词云")
@GetMapping("/analyze/high-word")
public ResponseResult getHighWord(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "planId", required = false) String planId) {
return ResponseResult.success(markDataService.getHighWord(startTime, endTime, planId, true));
}
@ApiOperation("新-舆情分析-高频标题")
@GetMapping("/analyze/frequent-title")
public ResponseResult getLastNews(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime,
@RequestParam(value = "planId", required = false) String planId) {
return ResponseResult.success(markDataService.getLastNews(startTime, endTime, planId, 5, true));
}
@ApiOperation("新-舆情分析-活跃渠道、ip分布、词云详情页面,社媒平台发文")
@PostMapping("/analyze/mark-data")
public ResponseResult getYuqingAnalyzeDetail(@RequestBody MarkSearchDTO markSearchDTO) {
return ResponseResult.success(markDataService.getYuqingAnalyzeDetail(markSearchDTO));
}
private boolean checkMTagIllegal(StringBuilder mtag) {
List<MarkerTag> hitTags = projectService.getProjectById(UserThreadLocal.getProjectId()).getHitTags();
if (!Tools.isEmpty(hitTags)) {
......
......@@ -277,7 +277,7 @@ public class AppChannelController extends BaseController {
@RequestParam(value = "endTime", required = false) Long endTime,
@RequestParam(value = "sorter", defaultValue = "{\"index\":\"descend\"}") String sorter,
@RequestParam(value = "pageSize", defaultValue = "50") int size) {
return ResponseResult.success(channelService.getPositiveChannelList(contendId, startTime, endTime, size, sorter, true));
return ResponseResult.success(channelService.getPositiveChannelList(contendId, startTime, endTime, size, null, true));
}
@ApiImplicitParams({@ApiImplicitParam(name = "contendId", value = "品牌id", defaultValue = "0", paramType = "query", dataType = "string"),
......@@ -294,7 +294,7 @@ public class AppChannelController extends BaseController {
@RequestParam(value = "endTime", required = false) Long endTime,
@RequestParam(value = "sorter", defaultValue = "{\"index\":\"descend\"}") String sorter,
@RequestParam(value = "pageSize", defaultValue = "50") int size) {
return ResponseResult.success(channelService.getNegativeChannelList(contendId, startTime, endTime, size, sorter, true));
return ResponseResult.success(channelService.getNegativeChannelList(contendId, startTime, endTime, size, null, true));
}
@ApiOperation("渠道库-渠道申请")
......
......@@ -31,6 +31,7 @@ import javax.annotation.Resource;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.Map;
import java.util.Objects;
/**
......@@ -93,6 +94,9 @@ public class AppWarnController extends BaseController {
@Value("${warn.taskSwitch.url}")
private String warnTaskSwitchUrl;
@Value("${warn.push.ticket.url}")
private String warnPushTicketUrl;
@ApiOperation("情报预警-推送任务获取")
@GetMapping("/project")
public ResponseResult getPushTaskId() {
......@@ -108,7 +112,13 @@ public class AppWarnController extends BaseController {
@PostMapping("/project")
public ResponseResult postPushTaskId(@RequestBody JSONObject json) {
try {
return pushTaskId(null, json.getString("planName"), HttpMethod.POST);
ResponseResult responseResult = pushTaskId(null, json.getString("planName"), HttpMethod.POST);
JSONObject body = (JSONObject) responseResult.getData();
if (!body.getBoolean("status") && Objects.isNull(body.getJSONObject("data"))){
return ResponseResult.failure("超过方案数量上限,新增失败");
}
JSONObject data = body.getJSONObject("data");
return ResponseResult.success(data.getString("id"));
} catch (Exception e) {
log.error("情报预警-推送任务新增失败", e);
return ResponseResult.failure("情报预警-推送任务新增失败");
......@@ -119,7 +129,8 @@ public class AppWarnController extends BaseController {
@PutMapping("/project")
public ResponseResult putPushTaskId(@RequestBody JSONObject json) {
try {
return pushTaskId(json.getString("taskId"), json.getString("planName"), HttpMethod.PUT);
pushTaskId(json.getString("taskId"), json.getString("planName"), HttpMethod.PUT);
return ResponseResult.success();
} catch (Exception e) {
log.error("情报预警-推送任务修改失败", e);
return ResponseResult.failure("情报预警-推送任务修改失败");
......@@ -305,6 +316,27 @@ public class AppWarnController extends BaseController {
}
}
@ApiOperation("情报预警-获取自动化ticket")
@GetMapping("/push/ticket")
public ResponseResult getTicket() {
try {
String token = request.getHeader(jwtKey);
// 请求头参数设置
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
headers.add("Push-Token", token);
HttpEntity<String> request = new HttpEntity<>(headers);
String ticket = restTemplate.exchange(warnPushTicketUrl, HttpMethod.GET, request, JSONObject.class)
.getBody().getJSONObject("data").getString("ticket");
return ResponseResult.success(ticket);
} catch (Exception e) {
log.error("情报预警-获取自动化ticket", e);
return ResponseResult.failure("情报预警-获取自动化ticket");
}
}
private ResponseResult pushTaskId(String taskId, String planName, HttpMethod httpMethod) throws UnsupportedEncodingException {
String projectId = UserThreadLocal.getProjectId();
String projectName = projectService.getProjectById(projectId).getProjectName();
......@@ -323,8 +355,7 @@ public class AppWarnController extends BaseController {
}
if (HttpMethod.DELETE == httpMethod) {
request = new HttpEntity<>(headers);
return ResponseResult.success(restTemplate.exchange(warnProjectUrl + "/"+taskId, httpMethod, request, JSONObject.class).getBody().getJSONArray(
"data"));
return ResponseResult.success(restTemplate.exchange(warnProjectUrl + "/"+taskId, httpMethod, request, JSONObject.class).getBody().getJSONArray("data"));
}
JSONObject paramMap = new JSONObject();
paramMap.put("planName", planName);
......@@ -335,7 +366,8 @@ public class AppWarnController extends BaseController {
paramMap.put("projectName", projectName);
}
request = new HttpEntity<>(paramMap, headers);
return ResponseResult.success(restTemplate.exchange(warnProjectUrl, httpMethod, request, JSONObject.class).getBody().getJSONArray("data"));
JSONObject body = restTemplate.exchange(warnProjectUrl, httpMethod, request, JSONObject.class).getBody();
return ResponseResult.success(body);
}
}
......@@ -26,6 +26,8 @@ import java.util.ArrayList;
import java.util.Calendar;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
/**
......@@ -74,22 +76,30 @@ public class ChannelEsDao extends EsClientDao {
public void upsertChannelRecord(List<ChannelRecord> channelRecords) {
String index = getChannelRecordIndexes().get(0);
BulkRequest bulkRequest = new BulkRequest();
Long startTime = null;
Long endTime = null;
for (List<ChannelRecord> records : ListUtils.partition(channelRecords, 100)) {
// 打印统计时间
AtomicLong atomicStartTime = new AtomicLong(-1);
AtomicLong atomicEndTime = new AtomicLong(-1);
BulkRequest bulkRequest = new BulkRequest();
for (ChannelRecord record : records) {
startTime = null == startTime ? record.getRangeStartTime() : Math.min(startTime, record.getRangeStartTime());
endTime = null == endTime ? record.getRangeEndTime() : Math.max(endTime, record.getRangeEndTime());
bulkRequest.add(new IndexRequest(index).id(record.getEsId()).source(record.toEsMap()));
bulkRequest.add(createChannelRecordIndexRequest(record, index, atomicStartTime, atomicEndTime));
}
BulkResponse bulkResponse = retryTemplate.execute(context -> {
try {
return channelEsClient.bulk(bulkRequest, RequestOptions.DEFAULT);
} catch (Exception ignored) {
return null;
}
});
BulkResponse bulkResponse;
try {
bulkResponse = retryTemplate.execute(context -> {
try {
return channelEsClient.bulk(bulkRequest, RequestOptions.DEFAULT);
} catch (Exception e) {
log.info("upsertRecord批量操作失败,尝试重试第{}次-", context.getRetryCount() + 1, e);
return null;
}
});
} catch (Exception e) {
// 重试三次后失败拆分channelRecords
bulkResponse = upsertChannelRecordLimit(records, index, 10);
}
Long startTime = atomicStartTime.get();
Long endTime = atomicEndTime.get();
if (null == bulkResponse) {
log.error("upsertRecord批量操作重试后失败,index:{},rangeTime:{}", index, startTime + "-" + endTime);
} else if (bulkResponse.hasFailures()) {
......@@ -101,6 +111,46 @@ public class ChannelEsDao extends EsClientDao {
}
}
private BulkResponse upsertChannelRecordLimit(List<ChannelRecord> records, String index, int limit) {
AtomicBoolean res = new AtomicBoolean(true);
BulkResponse bulkResponse = null;
// 重试三次后失败拆分channelRecords
for (List<ChannelRecord> minRecords : ListUtils.partition(records, limit)) {
BulkRequest minBulkRequest = new BulkRequest();
for (ChannelRecord minRecord : minRecords) {
minBulkRequest.add(createChannelRecordIndexRequest(minRecord, index, null, null));
}
bulkResponse = retryTemplate.execute(context -> {
try {
return channelEsClient.bulk(minBulkRequest, RequestOptions.DEFAULT);
} catch (Exception ex) {
res.set(false);
log.info("upsertRecord批量操作失败后二次重试,尝试重试第{}次-", context.getRetryCount() + 1, ex);
return null;
}
});
}
return !res.get() ? null : bulkResponse;
}
private IndexRequest createChannelRecordIndexRequest(ChannelRecord record, String index, AtomicLong startTime, AtomicLong endTime) {
if (null != startTime) {
if (-1 == startTime.get()) {
startTime.set(record.getRangeStartTime());
} else {
startTime.set(Math.min(startTime.get(), record.getRangeStartTime()));
}
}
if (null != endTime) {
if (-1 == endTime.get()) {
endTime.set(record.getRangeEndTime());
} else {
endTime.set(Math.max(endTime.get(), record.getRangeEndTime()));
}
}
return new IndexRequest(index).id(record.getEsId()).source(record.toEsMap());
}
public void batchInsert(List<Map<String, Object>> insertList) {
retryTemplate.execute(context -> {
try {
......
......@@ -6,6 +6,7 @@ import com.zhiwei.brandkbs2.common.GlobalPojo;
import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.enmus.ImportantChannelEnum;
import com.zhiwei.brandkbs2.pojo.external.BrandkbsYuQingConfig;
import com.zhiwei.brandkbs2.pojo.external.BrandkbsYuQingConfigNew;
import com.zhiwei.pushlog.tools.Tools;
import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform;
import org.apache.commons.collections4.CollectionUtils;
......@@ -161,6 +162,19 @@ public class EsQueryTools {
return boolQueryBuilder;
}
public static BoolQueryBuilder assembleCacheMapsQueryWithProject(String projectId, BrandkbsYuQingConfigNew config) {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
String key = concat(projectId, Constant.PRIMARY_CONTEND_ID);
BoolQueryBuilder nestedBoolBuilder = QueryBuilders.boolQuery();
// 必要条件
nestedBoolBuilder.must(QueryBuilders.termQuery("brandkbs_cache_maps.key.keyword", key));
// 或需要添加其余nested字段
boolQueryBuilder.must(cacheMapsNestedQuery(nestedBoolBuilder));
// sensitiveChannel
addSensitiveChannel(config, boolQueryBuilder);
return boolQueryBuilder;
}
public static BoolQueryBuilder assembleCacheMapsQueryExcludePrimaryId(String projectId){
BoolQueryBuilder query = QueryBuilders.boolQuery();
query.must(cacheMapsNestedQuery(QueryBuilders.termQuery("brandkbs_cache_maps.project_id.keyword", projectId)));
......@@ -276,11 +290,8 @@ public class EsQueryTools {
public static BoolQueryBuilder assembleForward2Query(boolean isForward) {
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
// 需要转发数据,平台必须为微博
if (isForward) {
BoolQueryBuilder mustWeiboCondition = QueryBuilders.boolQuery();
queryBuilder.should(mustWeiboCondition.must(QueryBuilders.termQuery("platform_id", "5d02236e6395002a7c380b79")));
queryBuilder.should(QueryBuilders.termQuery("is_forward", true));
queryBuilder.must(QueryBuilders.termQuery("is_forward", true));
}else {
queryBuilder.mustNot(QueryBuilders.termQuery("is_forward", true));
}
......@@ -295,7 +306,19 @@ public class EsQueryTools {
*/
public static BoolQueryBuilder assembleC2Query(Integer dataType) {
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
queryBuilder.should(QueryBuilders.termQuery("c2", dataType << 23));
queryBuilder.should(QueryBuilders.termQuery(GenericAttribute.ES_C2, dataType << 23));
return queryBuilder;
}
/**
* ip查询
*
* @param ip ip地址
* @return
*/
public static BoolQueryBuilder assembleIpQuery(String ip) {
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
queryBuilder.should(QueryBuilders.termQuery("ip_location.keyword", ip));
return queryBuilder;
}
......@@ -392,6 +415,25 @@ public class EsQueryTools {
boolQueryBuilder.must(hitBoolQuery);
}
public static void addSensitiveChannel(BrandkbsYuQingConfigNew config, BoolQueryBuilder boolQueryBuilder) {
BoolQueryBuilder hitBoolQuery = QueryBuilders.boolQuery();
if (!Tools.isEmpty(config.getPoliticsLevels())) {
List<String> hitList = config.getPoliticsLevels();
if (config.getPoliticsLevels().contains("全部")) {
hitList = ChannelType.POLITICS_LEVELS;
}
hitList.forEach(politicsLevel -> hitBoolQuery.should(QueryBuilders.termQuery(GenericAttribute.ES_SENSITIVE_CHANNEL + "." + GenericAttribute.ES_POLITICS_LEVEL + ".keyword", politicsLevel)));
}
if (!Tools.isEmpty(config.getFields())) {
List<String> hitList = config.getFields();
if (config.getFields().contains("全部")) {
hitList = ChannelType.COMMON_FIELDS;
}
hitList.forEach(field -> hitBoolQuery.should(QueryBuilders.termQuery(GenericAttribute.ES_SENSITIVE_CHANNEL + "." + GenericAttribute.ES_FIELD + ".keyword", field)));
}
boolQueryBuilder.must(hitBoolQuery);
}
public static void addSensitiveChannel(String politicsLevel, String field, String region, String mainBodyType, BoolQueryBuilder boolQueryBuilder) {
if (null != politicsLevel) {
boolQueryBuilder.must(QueryBuilders.termQuery(GenericAttribute.ES_SENSITIVE_CHANNEL + "." + GenericAttribute.ES_POLITICS_LEVEL + ".keyword", politicsLevel));
......
......@@ -103,12 +103,14 @@ public class ChannelIndex extends AbstractBaseMongo {
return res;
}
List<Map<String, Object>> cacheMaps = (List<Map<String, Object>>) sourceAsMap.get(GenericAttribute.ES_BRANDKBS_CACHE_MAPS);
List<String> manualProjectIds = GlobalPojo.PROJECT_MAP.entrySet().stream().filter(entry -> entry.getValue().isManual()).map(Map.Entry::getKey).collect(Collectors.toList());
// 非人工项目
List<String> nonManualProjectIds =
GlobalPojo.PROJECT_MAP.entrySet().stream().filter(entry ->!entry.getValue().isManual()).map(Map.Entry::getKey).collect(Collectors.toList());
for (Map<String, Object> cacheMap : cacheMaps) {
String projectId = String.valueOf(cacheMap.get("project_id"));
String contendId = String.valueOf(cacheMap.get("contend_id"));
// 剔除非人工项目
if(manualProjectIds.contains(projectId)){
if(nonManualProjectIds.contains(projectId)){
continue;
}
ChannelIndex channelIndex = new ChannelIndex(projectId, contendId, messagePlatform.getName(), realSource, source);
......@@ -198,11 +200,21 @@ public class ChannelIndex extends AbstractBaseMongo {
}
public static List<Article> filterArticles(Long startTime, Long endTime, List<Article> articles) {
return filterArticles(startTime, endTime, articles, null);
}
public static List<Article> filterArticles(Long startTime, Long endTime, List<Article> articles, Integer emotion) {
// 去除不符合时间段数据
articles = articles.stream().filter(article -> Tools.hitTimeRange(startTime, endTime, article.getTime())).collect(Collectors.toList());
// 去重并保留最近标注时间
Map<String, ChannelIndex.Article> setMap = new HashMap<>();
for (ChannelIndex.Article article : articles) {
if (null != emotion) {
// 文章类型与其不一致
if (emotion != EmotionEnum.ALL.getState() && emotion != article.getEmotion()) {
continue;
}
}
setMap.compute(article.getId(), (k, v) -> {
// 旧值为null或标注时间更新
if (null == v || article.getMtime() > v.getMtime()) {
......
......@@ -102,7 +102,7 @@ public class MarkFlowEntity implements Serializable {
int c2 = tJson.getIntValue(GenericAttribute.ES_C2);
// 微博平台头像url拼接
String weiboHeadUrl = "https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/weibo-avatar-url?url=";
if (1020 == tJson.getIntValue(GenericAttribute.ES_C4) && !tJson.getString("avatar_url").contains(weiboHeadUrl)){
if (1020 == tJson.getIntValue(GenericAttribute.ES_C4) && Objects.nonNull(tJson.getString("avatar_url")) && !tJson.getString("avatar_url").contains(weiboHeadUrl)){
tJson.put("avatar_url", weiboHeadUrl + tJson.getString("avatar_url"));
}
switch (ClassB.TypeB.fromEncode(c2)) {
......
......@@ -181,7 +181,18 @@ public class MarkSearchDTO {
@ApiModelProperty(value = "数据类型(1:长文本, 2:短文本, 3:问答, 5:视频)")
private Integer dataType;
/**
* gid用于未读已读筛选,仅非人工项目
* ip地址
*/
@ApiModelProperty(value = "ip地址")
private String ip;
/**
* gid用于未读已读筛选,非人工项目舆情数据用
*/
@ApiModelProperty(value = "gid")
private Long gid;
/**
* gid限制 舆情分析页面滚动翻页用
*/
@ApiModelProperty(value = "gid限制")
private Long pageGid;
}
......@@ -5,6 +5,7 @@ import lombok.Data;
import java.util.List;
@Data
@Deprecated
public class BrandkbsYuQingConfig {
private String configName;
......
package com.zhiwei.brandkbs2.pojo.external;
import lombok.Data;
import java.util.List;
@Data
public class BrandkbsYuQingConfigNew {
private String configName;
/**
* 关键字命中
*/
private String keyword;
/**
* 自定义渠道
*/
private String channel;
/**
* 行政级别
*/
private List<String> politicsLevels;
/**
* 是否转发
*/
private List<Boolean> primary;
/**
* 领域
*/
private List<String> fields;
/**
* 本品舆情
*/
private List<String> markerTags;
/**
* pushInterval 推送间隔
*/
private Long pushInterval;
/**
* distinctFilter 去重过滤
*/
private boolean distinctFilter;
}
......@@ -45,6 +45,7 @@ public class PageVO<T> {
pageVO.setPages(pages);
pageVO.setSize(size);
pageVO.setHasNext(pageNum < pages);
pageVO.setPageSize(size);
return pageVO;
}
......
......@@ -163,7 +163,7 @@ public class ProjectVO {
project.setPositiveChannelParams((this.getPositiveChannelParams()));
project.setBlackChannelGroup(this.getBlackChannelGroup());
project.setShow(true);
project.setStart(false);
project.setStart(true);
project.setManual(true);
project.setCTime(time.getTime());
project.setUTime(time.getTime());
......@@ -189,7 +189,7 @@ public class ProjectVO {
// 非人工项目
project.setManual(false);
project.setShow(true);
project.setStart(false);
project.setStart(true);
project.setCTime(System.currentTimeMillis());
project.setUTime(System.currentTimeMillis());
return project;
......
......@@ -12,7 +12,9 @@ import com.zhiwei.brandkbs2.pojo.vo.ChannelListVO;
import com.zhiwei.brandkbs2.pojo.vo.ChannelVO;
import com.zhiwei.brandkbs2.pojo.vo.PageVO;
import java.io.IOException;
import java.util.List;
import java.util.Map;
/**
* @ClassName: ChannelService
......@@ -300,4 +302,10 @@ public interface ChannelService {
JSONObject getMobileSpreadingTend(String channelId,String type);
/**
* 获取项目正面、负面渠道数据
* @return
* @throws IOException
*/
Map<String, JSONObject> getProjectEmotionChannelListData() throws IOException;
}
......@@ -436,6 +436,12 @@ public interface MarkDataService {
PageVO<JSONObject> getNonManualProjectPlanList();
/**
* 获取方案列表
* @return
*/
List<JSONObject> getNonManualPlanList();
/**
* 获取方案设置-方案列表-昨日,今日数据消耗量
* @return
*/
......@@ -511,4 +517,121 @@ public interface MarkDataService {
* @return
*/
List<JSONObject> getNonManualMarkCountList();
/**
* 计算近一年舆情总量项目日均
* @param projectId
* @param planId
* @return
* @throws IOException
*/
void countYuqingAmountAvg(Long startTime, Long endTime, String projectId, String planId) throws IOException;
/**
* 计算近一年情感分布均值
* @param projectId
* @param planId
* @throws IOException
*/
void countEmotionDistributionAvg(Long startTime, Long endTime, String projectId, String planId) throws IOException;
/**
* 计算项目近一年重点平台均值
* @param projectId
* @param planId
* @throws IOException
*/
void countImportantPlatformPercentageAvg(Long startTime, Long endTime, String projectId, String planId) throws IOException;
/**
* 新-舆情分析-舆情总量
* @param startTime
* @param endTime
* @param planId
* @return
* @throws IOException
*/
JSONObject getYuqingAmount(Long startTime, Long endTime, String planId);
/**
* 新-舆情分析-情感分布
* @param startTime
* @param endTime
* @param planId
* @return
* @throws IOException
*/
JSONObject getYuqingEmotionDistribution(Long startTime, Long endTime, String planId);
/**
* 新-舆情分析-重点平台
* @param startTime
* @param endTime
* @param planId
* @return
*/
JSONObject getImportantPlatformPercentage(Long startTime, Long endTime, String planId);
/**
* 新-舆情分析-平台占比
* @param startTime
* @param endTime
* @param planId
* @return
*/
List<JSONObject> getPlatformPercentage(Long startTime, Long endTime, String planId);
/**
* 新-舆情分析-舆情走势图
* @param startTime
* @param endTime
* @param planId
* @return
*/
JSONObject getSpreadTendency(Long startTime, Long endTime, String planId);
/**
* 新-舆情分析-活跃渠道
* @param startTime
* @param endTime
* @param planId
* @return
*/
List<JSONObject> getActiveChannels(Long startTime, Long endTime, String planId);
/**
* 新-舆情分析-ip分布
* @param startTime
* @param endTime
* @param planId
* @return
*/
List<JSONObject> getArticleIpLocated(Long startTime, Long endTime, String planId, int size);
/**
* 新-舆情分析-活跃渠道、ip分布、词云详情页面
* @param dto
* @return
*/
PageVO<MarkFlowEntity> getYuqingAnalyzeDetail(MarkSearchDTO dto);
/**
* 新-舆情分析-词云
* @param startTime
* @param endTime
* @param planId
* @return
*/
List<JSONObject> getHighWord(Long startTime, Long endTime, String planId, boolean cache);
/**
* 新-舆情分析-高频标题
* @param startTime
* @param endTime
* @param planId
* @param size
* @param include
* @return
*/
List<JSONObject> getLastNews(Long startTime, Long endTime, String planId, int size, boolean include);
}
......@@ -193,4 +193,21 @@ public interface ProjectService {
* @return json
*/
JSONObject nonManualProjectOverview(String project);
/**
* 获取用户拥有权限的项目
* @param userId
* @return
*/
ResponseResult getUserProject(String userId);
/**
* 分页获取用户拥有权限的项目,支持项目名关键词模糊
* @param userId
* @param keyword 项目名模糊查询关键词
* @param page
* @param pageSize
* @return
*/
ResponseResult getUserProjectPageData(String userId, String keyword, int page, int pageSize);
}
package com.zhiwei.brandkbs2.service;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.external.BrandkbsChannelConfig;
import com.zhiwei.brandkbs2.pojo.external.BrandkbsHotEventConfig;
import com.zhiwei.brandkbs2.pojo.external.BrandkbsHotTopConfig;
import com.zhiwei.brandkbs2.pojo.external.BrandkbsYuQingConfig;
import com.zhiwei.brandkbs2.pojo.external.*;
import com.zhiwei.brandkbs2.pojo.vo.CrisisCaseWarnVO;
public interface ProjectWarnService {
......@@ -34,6 +31,14 @@ public interface ProjectWarnService {
ResponseResult getYuqingWaring(String projectId, BrandkbsYuQingConfig brandkbsYuQingConfig);
/**
* 获取舆情动态预警-新
* @param projectId
* @param brandkbsYuQingConfigNew
* @return
*/
ResponseResult getYuqingWaringNew(String projectId, BrandkbsYuQingConfigNew brandkbsYuQingConfigNew);
/**
* 获取渠道参与预警
* @param projectId
* @param brandkbsChannelConfig
......
......@@ -42,4 +42,14 @@ public interface TaskService{
* 事件相关更新
*/
void eventUpdate();
/**
* 计算项目舆情总量、情感分布、重点平台均值
*/
void calculateProjectAvg();
/**
* 生成舆情分析词云缓存
*/
void yuqingAnalyzeHighWordCache();
}
......@@ -45,8 +45,11 @@ import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.RangeQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.BucketOrder;
import org.elasticsearch.search.aggregations.bucket.terms.ParsedLongTerms;
import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.aggregations.metrics.Sum;
......@@ -99,6 +102,9 @@ public class ChannelServiceImpl implements ChannelService {
@Resource(name = "channelTagDao")
ChannelTagDao channelTagDao;
@Resource(name = "projectDao")
private ProjectDao projectDao;
@Resource(name = "qbjcPojoDao")
private QbjcPojoDao qbjcPojoDao;
......@@ -510,15 +516,14 @@ public class ChannelServiceImpl implements ChannelService {
// 过滤掉不符合时间条件的数据并排序
Map<String, List<ChannelRecord>> channelRecords = keyMap.values().stream().map(pair -> {
ChannelRecord channelRecord = pair.getRight();
// 情感过滤
if (emotion == EmotionEnum.ALL.getState() || emotion == channelRecord.getEmotion()) {
List<ChannelIndex.Article> articles = ChannelIndex.Record.filterArticles(startTime, endTime,
channelRecord.getRecord().getArticles());
articles.sort(Comparator.comparingLong(ChannelIndex.Article::getTime).reversed());
channelRecord.getRecord().setArticles(articles);
return channelRecord;
List<ChannelIndex.Article> articles = ChannelIndex.Record.filterArticles(startTime, endTime,
channelRecord.getRecord().getArticles(), emotion);
if (0 == articles.size()) {
return null;
}
return null;
articles.sort(Comparator.comparingLong(ChannelIndex.Article::getTime).reversed());
channelRecord.getRecord().setArticles(articles);
return channelRecord;
}).filter(Objects::nonNull).collect(Collectors.groupingBy(ChannelRecord::getPlatform));
for (String platformName : PLATFORMS) {
List<ChannelRecord> channelRecordList = channelRecords.getOrDefault(platformName, Collections.emptyList()).stream().limit(size).collect(Collectors.toList());
......@@ -569,6 +574,7 @@ public class ChannelServiceImpl implements ChannelService {
return resList;
}
@Deprecated
private List<ChannelListVO> getEmotionList(String projectId, String contendId, String platform, String keyword, String sorter,
Long startTime, Long endTime, int size, int emotion, boolean cache) throws IOException {
List<ChannelListVO> resList = new ArrayList<>();
......@@ -1216,6 +1222,39 @@ public class ChannelServiceImpl implements ChannelService {
return result;
}
@Override
public Map<String, JSONObject> getProjectEmotionChannelListData() throws IOException {
int size = projectDao.findList(new Query()).size();
Map<String, JSONObject> res = new HashMap<>();
// query
BoolQueryBuilder query = QueryBuilders.boolQuery();
// contendId
query.must(QueryBuilders.termQuery("contend_id.keyword", Constant.PRIMARY_CONTEND_ID));
// emotion
query.mustNot(QueryBuilders.termQuery("emotion", EmotionEnum.NEUTRAL.getState()));
query.mustNot(QueryBuilders.termQuery("emotion", EmotionEnum.UNDEFINED.getState()));
// agg
TermsAggregationBuilder projectAgg = AggregationBuilders.terms("projectAgg").field("project_id.keyword").size(size);
TermsAggregationBuilder emotionAgg = AggregationBuilders.terms("emotionAgg").field("emotion");
// response
SearchResponse searchResponse = esClientDao.searchResponse(new String[]{"brandkbs2_channel_copy"}, null, query,
projectAgg.subAggregation(emotionAgg), null, null, 0, 0, null);
Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
ParsedStringTerms projectTeamAgg = (ParsedStringTerms) aggMap.get("projectAgg");
List<? extends Terms.Bucket> buckets = projectTeamAgg.getBuckets();
buckets.forEach(bucket -> {
JSONObject jsonObject = new JSONObject();
Map<String, Aggregation> map = bucket.getAggregations().asMap();
ParsedLongTerms emotionTeamAgg = (ParsedLongTerms) map.get("emotionAgg");
List<? extends Terms.Bucket> list = emotionTeamAgg.getBuckets();
for (Terms.Bucket eBucket : list) {
jsonObject.put(EmotionEnum.state2Name(eBucket.getKeyAsNumber().intValue()), eBucket.getDocCount());
}
res.put(bucket.getKeyAsString(), jsonObject);
});
return res;
}
private BoolQueryBuilder getChannelListQuery(String projectId, String contendId, String keyword,
List<String> platforms, List<Integer> emotions, List<String> mediaTypes, Integer[] articlesCount) {
BoolQueryBuilder postFilter = QueryBuilders.boolQuery();
......
......@@ -131,6 +131,14 @@ public class EsSearchServiceImpl implements EsSearchService {
if (StringUtils.isNotEmpty(dto.getHostKeyword())) {
postFilter.must(EsQueryTools.assembleFiledKeywordQuery("host", dto.getHostKeyword()));
}
// ip
if (StringUtils.isNotEmpty(dto.getIp())){
postFilter.must(EsQueryTools.assembleIpQuery(dto.getIp()));
}
// mgid限制 舆情分析页面滚动翻页用
if (Objects.nonNull(dto.getPageGid())){
postFilter.must(QueryBuilders.rangeQuery(GenericAttribute.ES_MGID).lt(dto.getPageGid()));
}
helper.setPostFilter(postFilter);
// helper.setQuery(query);
// sort
......@@ -315,6 +323,14 @@ public class EsSearchServiceImpl implements EsSearchService {
if (Objects.nonNull(dto.getDataType())){
postFilter.must(EsQueryTools.assembleC2Query(dto.getDataType()));
}
// ip
if (StringUtils.isNotEmpty(dto.getIp())){
postFilter.must(EsQueryTools.assembleIpQuery(dto.getIp()));
}
// mgid限制 舆情分析页面滚动翻页用
if (Objects.nonNull(dto.getPageGid())){
postFilter.must(QueryBuilders.rangeQuery(GenericAttribute.ES_MGID).lt(dto.getPageGid()));
}
helper.setPostFilter(postFilter);
// sort
FieldSortBuilder sort = null;
......
......@@ -378,8 +378,10 @@ public class IndexServiceImpl implements IndexService {
long normalCount = markDataService.getYuqingMarkCount(startTime, endTime, EmotionEnum.ALL.getName(), projectId, contendId);
//获取时间范围内总正面稿件数
long positiveCount = markDataService.getYuqingMarkCount(startTime, endTime, EmotionEnum.POSITIVE.getName(), projectId, contendId);
//获取时间范围内总中性稿件数
long neutralCount = markDataService.getYuqingMarkCount(startTime, endTime, EmotionEnum.NEUTRAL.getName(), projectId, contendId);
double reputation = normalCount == 0 ? 0d : 100 * positiveCount / (double) normalCount;
double neutralPercent = normalCount == 0 ? 0d : 100 * neutralCount / (double) normalCount;
SimpleDateFormat sdf;
if ("year".equals(type)) {
type = "年";
......@@ -393,9 +395,11 @@ public class IndexServiceImpl implements IndexService {
}
String date = spreadResult.stream().max(Comparator.comparing(json -> json.getIntValue("normalCount"))).map(json -> sdf.format(json.getDate("time"))).get();
if (!flag) {
return "近一月,传播高峰为" + date + ",全网品牌相关新闻传播总量达" + normalCount + "篇,正面内容占比为" + String.format("%.1f", reputation) + "%。";
return "近一月,传播高峰为" + date + ",全网品牌相关新闻传播总量达" + normalCount + "篇,正面内容占比为" + String.format("%.1f", reputation) + "%,中性内容占比为" + String.format("%.1f"
, neutralPercent) + "%。";
} else {
return "近" + spreadResult.size() + type + "内,传播高峰为" + date + ",全网品牌相关新闻传播总量达" + normalCount + "篇,正面内容占比为" + String.format("%.1f", reputation) + "%。";
return "近" + spreadResult.size() + type + "内,传播高峰为" + date + ",全网品牌相关新闻传播总量达" + normalCount + "篇,正面内容占比为" + String.format("%.1f", reputation) +
"%,中性内容占比为" + String.format("%.1f", neutralPercent) + "%。";
}
}
......
......@@ -6,6 +6,7 @@ import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.common.GlobalPojo;
import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.dao.*;
import com.zhiwei.brandkbs2.enmus.EmotionEnum;
import com.zhiwei.brandkbs2.enmus.response.ProjectCodeEnum;
import com.zhiwei.brandkbs2.exception.ExceptionCast;
import com.zhiwei.brandkbs2.model.CommonCodeEnum;
......@@ -22,6 +23,7 @@ import com.zhiwei.middleware.event.pojo.dto.EventTagRelatedDTO;
import com.zhiwei.middleware.event.pojo.entity.BrandkbsBasicInfo;
import com.zhiwei.middleware.mark.vo.MarkerTag;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Value;
......@@ -498,6 +500,9 @@ public class ProjectServiceImpl implements ProjectService {
}
Project project = ProjectVO.createNonManualProject(projectVO);
projectDao.insertOneWithoutId(project);
// 绑定事件标签,默认已绑定情感标签
eventMiddlewareDao.bindBrandkbs(project.getBrandLinkedGroup(), project.getBrandLinkedGroupId(), Collections.emptyList(),
project.getProjectName(), project.getId(), project.getBrandName(), project.getId(), null, null);
return ResponseResult.success();
}
......@@ -585,6 +590,51 @@ public class ProjectServiceImpl implements ProjectService {
return json;
}
@Override
public ResponseResult getUserProject(String userId) {
User user = userDao.findOneById(userId);
Query query = new Query();
if (!user.isSuperAdmin()){
List<String> projectIds = user.getRoles().stream().map(UserRole::getProjectId).collect(Collectors.toList());
query.addCriteria(Criteria.where("_id").in(projectIds));
}
List<Project> projectList = projectDao.findList(query);
List<JSONObject> res = projectList.stream().map(project -> {
JSONObject jsonObject = new JSONObject();
jsonObject.put("id", project.getId());
jsonObject.put("name", project.getProjectName());
return jsonObject;
}).collect(Collectors.toList());
return ResponseResult.success(res);
}
@Override
public ResponseResult getUserProjectPageData(String userId, String keyword, int page, int pageSize) {
Query query = new Query();
projectDao.addSort(query, "{\"_id\":\"asc\"}");
// 项目名关键词模糊
if (StringUtils.isNotBlank(keyword)){
projectDao.addKeywordFuzz(query, keyword, "projectName");
}
// 用户拥有权限的项目
if (StringUtils.isNotBlank(userId) && !userDao.findOneById(userId).isSuperAdmin()){
List<String> projectIds = userDao.findOneById(userId).getRoles().stream().map(UserRole::getProjectId).collect(Collectors.toList());
query.addCriteria(Criteria.where("_id").in(projectIds));
}
long total = projectDao.count(query);
int skipCount = (page - 1) * pageSize;
query.limit(pageSize);
query.skip(skipCount);
List<Project> projectList = projectDao.findList(query);
List<JSONObject> resList = projectList.stream().map(project -> {
JSONObject result = new JSONObject();
result.put("id", project.getId());
result.put("projectName", project.getProjectName());
return result;
}).collect(Collectors.toList());
return ResponseResult.success(PageVO.createPageVo(total, page, pageSize, resList));
}
/**
* 获取舆情对应项目的情感标签
* @param brandName
......@@ -662,6 +712,12 @@ public class ProjectServiceImpl implements ProjectService {
Map<String, Object> originPermission = new HashMap<>();
originPermission.put("origin", Objects.nonNull(project.getModuleShowList()) && project.getModuleShowList().contains(2));
permissionList.add(originPermission);
// 友好渠道榜、敏感渠道榜是否有数据
Map<String, Object> channelPermission = new HashMap<>();
JSONObject jsonObject = GlobalPojo.PROJECT_EMOTION_CHANNEL_DATA.get(project.getId());
channelPermission.put("positiveChannel", Objects.nonNull(jsonObject) && Objects.nonNull(jsonObject.getLong(EmotionEnum.POSITIVE.getName())) && 0 != jsonObject.getLong(EmotionEnum.POSITIVE.getName()));
channelPermission.put("negativeChannel", Objects.nonNull(jsonObject) && Objects.nonNull(jsonObject.getLong(EmotionEnum.NEGATIVE.getName())) && 0 != jsonObject.getLong(EmotionEnum.NEGATIVE.getName()));
permissionList.add(channelPermission);
return permissionList;
}
......
......@@ -170,12 +170,19 @@ public class ProjectWarnServiceImpl implements ProjectWarnService {
public ResponseResult getProjectWarnCriteriaDefault(String projectId, String type) {
switch (type) {
case "舆情动态":
BrandkbsYuQingConfig config = new BrandkbsYuQingConfig();
// BrandkbsYuQingConfig config = new BrandkbsYuQingConfig();
// config.setPoliticsLevels(Collections.singletonList("全部"));
// config.setFields(Collections.singletonList("全部"));
// config.setPrimary(Collections.singletonList(Boolean.TRUE));
// config.setMarkerTags(Arrays.asList("正面", "负面", "中性"));
// config.setDuplicate(Boolean.FALSE);
// config.setPushInterval(10 * ONE_MINUTE);
BrandkbsYuQingConfigNew config = new BrandkbsYuQingConfigNew();
config.setPoliticsLevels(Collections.singletonList("央级"));
config.setFields(Collections.singletonList("财经"));
config.setPrimary(Collections.singletonList(Boolean.TRUE));
config.setMarkerTags(Collections.singletonList("负面"));
config.setDuplicate(Boolean.FALSE);
config.setFields(Collections.singletonList("全部"));
config.setPrimary(Collections.singletonList(Boolean.FALSE));
config.setMarkerTags(Arrays.asList("正面", "负面", "中性"));
config.setDistinctFilter(true);
config.setPushInterval(10 * ONE_MINUTE);
return ResponseResult.success(config);
case "渠道参与":
......@@ -213,6 +220,89 @@ public class ProjectWarnServiceImpl implements ProjectWarnService {
return ResponseResult.success();
}
private BrandkbsWarnTemplate brandkbsWarnTemplate4Yuqing(BrandkbsYuQingConfigNew config, List<BaseMap> datas, long start, long end, AbstractProject project) {
if (datas.isEmpty()) {
return null;
}
int firstCount = datas.size();
// key1
String key1 = "【品见】舆情动态-" + project.getProjectName();
// key2
// String key2Header = "全部";
String key2Header = "";
StringBuilder mediaChannel = new StringBuilder();
List<String> politicsLevels = config.getPoliticsLevels();
List<String> fields = config.getFields();
if(Tools.isEmpty(politicsLevels)){
key2Header += "【全部级别】";
}else if(politicsLevels.contains("全部")){
key2Header += "【重点级别】";
}else{
politicsLevels.forEach(politicsLevel -> mediaChannel.append(politicsLevel).append("、"));
}
if (Tools.isEmpty(fields)) {
if (0 != key2Header.length()) {
key2Header += "、";
key2Header += "【全部领域】,";
} else {
key2Header += "【全部领域】";
}
} else if (fields.contains("全部")) {
if (0 != key2Header.length()) {
key2Header += "、";
key2Header += "【重点领域】,";
} else {
key2Header += "【重点领域】";
}
} else {
fields.forEach(field -> mediaChannel.append(field).append("、"));
}
if (0 != mediaChannel.length()) {
if (0 != key2Header.length()) {
key2Header += "、";
}
// 替换“全部” 并去掉末位的、
key2Header += mediaChannel.deleteCharAt(mediaChannel.length() - 1).toString();
key2Header += "媒体,";
}
StringBuilder dataType = new StringBuilder();
config.getMarkerTags().forEach(emotion -> dataType.append(emotion).append(" "));
// dataType.append("(本品)");
// 如果有竞品
// if (CollectionUtils.isNotEmpty(config.getContends())) {
// dataType.append(",");
// config.getContends().forEach(contend -> dataType.append(projectService.getProjectByContendId(project.getId(), contend).getBrandName()).append(" "));
// dataType.append("(竞品)");
// }
String key2 = key2Header + dataType;
// key3
String key3 = Constant.SPEC_MINUTE_FORMAT.format(start) + " ~ " + Constant.SPEC_MINUTE_FORMAT.format(end);
// key4
String key4;
StringBuilder keyBuilder4 = new StringBuilder();
for (int i = 0; i < Math.min(datas.size(), 2); i++) {
BaseMap baseMap = datas.get(i);
String platform = baseMap.getPlatform();
String source = baseMap.getSource();
String title = baseMap.getTitle();
keyBuilder4.append(platform).append("-").append(source).append("\r\n");
// 最后一条
if (datas.size() == i + 1) {
keyBuilder4.append(i + 1).append("、").append(title);
} else {
keyBuilder4.append(i + 1).append("、").append(title).append("\r\n");
}
}
if (datas.size() > 2) {
key4 = keyBuilder4.append("...").toString();
} else {
key4 = keyBuilder4.toString();
}
List<BaseMapCompound> collect = datas.stream().map(baseMap -> BaseMapCompound.createFromBaseMap(baseMap, project.getId())).collect(Collectors.toList());
return new BrandkbsWarnTemplate(firstCount, key1, key2, key3, key4, new BrandkbsYuQingWarn(collect));
}
@Deprecated
private BrandkbsWarnTemplate brandkbsWarnTemplate4Yuqing(BrandkbsYuQingConfig config, List<BaseMap> datas, long start, long end, AbstractProject project) {
if (datas.isEmpty()) {
return null;
......@@ -613,6 +703,7 @@ public class ProjectWarnServiceImpl implements ProjectWarnService {
}
@Override
@Deprecated
public ResponseResult getYuqingWaring(String projectId, BrandkbsYuQingConfig config) {
Project project = null;
long end = System.currentTimeMillis();
......@@ -671,6 +762,60 @@ public class ProjectWarnServiceImpl implements ProjectWarnService {
}
@Override
public ResponseResult getYuqingWaringNew(String projectId, BrandkbsYuQingConfigNew config) {
Project project;
long end = System.currentTimeMillis();
long start = end - config.getPushInterval();
try {
project = projectService.getProjectById(projectId);
log.info("获取新舆情动态预警开始,project:{}", project.getProjectName());
EsClientDao.SearchHelper helper = EsClientDao.createSearchHelper();
// 发声渠道
BoolQueryBuilder postFilter = EsQueryTools.assembleCacheMapsQueryWithProject(projectId, config);
// 时间(标注时间)
postFilter.must(QueryBuilders.rangeQuery("mtime").gte(start).lt(end));
// 文章类型
if (CollectionUtils.isNotEmpty(config.getPrimary()) && config.getPrimary().size() == 1) {
postFilter.must(EsQueryTools.assembleForward2Query(config.getPrimary().get(0)));
}
// 舆情调性
BoolQueryBuilder sourceBuilder = QueryBuilders.boolQuery();
if (CollectionUtils.isNotEmpty(config.getMarkerTags())) {
config.getMarkerTags().forEach(tagName -> sourceBuilder.should(QueryBuilders.termQuery("brandkbs_mark_cache_maps.name.keyword", tagName)));
}
postFilter.must(sourceBuilder);
// 自定义渠道
if (StringUtils.isNotBlank(config.getChannel())){
postFilter.must(EsQueryTools.assembleSourceQuery(config.getChannel()));
}
helper.setPostFilter(postFilter);
SearchHits searchHits = esClientDao.searchHits(helper);
log.info("获取新舆情动态预警-es搜索到数据:{}条,剩余关键词筛选未进行,project:{}", searchHits.getHits().length, project.getProjectName());
List<BaseMap> datas = Arrays.stream(searchHits.getHits()).map(hit -> {
Map<String, Object> sourceMap = hit.getSourceAsMap();
BaseMap baseMap = Tools.getBaseFromEsMap(sourceMap);
// 关键词命中
if (!Tools.isContains(config.getKeyword(), baseMap.getTitle() + baseMap.getContent())) {
return null;
}
baseMap.setTitle(baseMap.getTitleNullOptionalContent());
baseMap.subStringContentTop();
return baseMap;
}).filter(Objects::nonNull).collect(Collectors.toList());
// 开启去重
if (Boolean.TRUE.equals(config.isDistinctFilter())) {
datas = duplicateBaseMapList(datas);
}
log.info("获取舆情动态预警结束,project:{}", project.getProjectName());
// 封装返回值
return ResponseResult.success(brandkbsWarnTemplate4Yuqing(config, datas, start, end, project));
} catch (Exception e) {
log.info("获取舆情动态预警失败,projectId:{},start:{},end:{}", projectId, start, end, e);
}
return ResponseResult.failure("获取舆情动态预警失败");
}
@Override
public ResponseResult getChannelWaring(String projectId, BrandkbsChannelConfig config) {
Project project = null;
long end = System.currentTimeMillis();
......
......@@ -3,14 +3,13 @@ package com.zhiwei.brandkbs2.service.impl;
import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.common.GlobalPojo;
import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.dao.AggreeResultDao;
import com.zhiwei.brandkbs2.dao.BrandkbsTaskDao;
import com.zhiwei.brandkbs2.dao.ChannelDao;
import com.zhiwei.brandkbs2.dao.ReportSettingsDao;
import com.zhiwei.brandkbs2.dao.*;
import com.zhiwei.brandkbs2.enmus.ReportTypeEnum;
import com.zhiwei.brandkbs2.es.ChannelEsDao;
import com.zhiwei.brandkbs2.es.EsClientDao;
import com.zhiwei.brandkbs2.exception.ExceptionCast;
import com.zhiwei.brandkbs2.listener.ApplicationProjectListener;
import com.zhiwei.brandkbs2.model.CommonCodeEnum;
import com.zhiwei.brandkbs2.pojo.*;
import com.zhiwei.brandkbs2.service.*;
import com.zhiwei.brandkbs2.util.Tools;
......@@ -19,6 +18,8 @@ import org.apache.commons.lang3.tuple.Pair;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ElasticsearchStatusException;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.stereotype.Service;
......@@ -28,6 +29,7 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
/**
......@@ -61,6 +63,9 @@ public class TaskServiceImpl implements TaskService {
@Resource(name = "aggreeResultDaoImpl")
AggreeResultDao aggreeResultDao;
@Resource(name = "nonManualProjectPlanDao")
NonManualProjectPlanDao nonManualProjectPlanDao;
@Resource(name = "brandkbsTaskServiceImpl")
BrandkbsTaskService brandkbsTaskService;
......@@ -82,6 +87,9 @@ public class TaskServiceImpl implements TaskService {
@Resource(name = "customEventServiceImpl")
CustomEventService customEventService;
@Resource(name = "markDataServiceImpl")
MarkDataService markDataService;
@Resource(name = "taskServiceExecutor")
ThreadPoolTaskExecutor taskServiceExecutor;
......@@ -155,10 +163,10 @@ public class TaskServiceImpl implements TaskService {
channelService.getActiveChannelList(Constant.PRIMARY_CONTEND_ID, times[0], times[1], 50, false);
// 友好渠道榜
channelService.getPositiveList(Constant.PRIMARY_CONTEND_ID, null, null, sorter, times[0], times[1], 50, false);
channelService.getPositiveChannelList(Constant.PRIMARY_CONTEND_ID, times[0], times[1], 50, sorter, false);
channelService.getPositiveChannelList(Constant.PRIMARY_CONTEND_ID, times[0], times[1], 50, null, false);
// 敏感渠道榜
channelService.getNegativeList(Constant.PRIMARY_CONTEND_ID, null, null, sorter, times[0], times[1], 50, false);
channelService.getNegativeChannelList(Constant.PRIMARY_CONTEND_ID, times[0], times[1], 50, sorter, false);
channelService.getNegativeChannelList(Constant.PRIMARY_CONTEND_ID, times[0], times[1], 50, null, false);
});
log.info("项目:{}-渠道榜单缓存已完成:{}个", project.getProjectName(), total.incrementAndGet());
return null;
......@@ -324,6 +332,56 @@ public class TaskServiceImpl implements TaskService {
}
}
@Override
public void calculateProjectAvg() {
AtomicLong total = new AtomicLong();
GlobalPojo.PROJECT_MAP.forEach((projectId, project) ->{
try {
// 取近一年
long endTime = System.currentTimeMillis();
long startTime = endTime - Constant.ONE_YEAR;
// 项目创建时间在近一年内则取项目创建时间
startTime = project.getCTime() > startTime ? project.getCTime() : startTime;
// 同时计算非人工项目各方案均值
if (!project.isManual()) {
List<NonManualProjectPlan> plans = nonManualProjectPlanDao.findList(new Query(Criteria.where("projectId").is(projectId)));
for (NonManualProjectPlan plan : plans) {
markDataService.countYuqingAmountAvg(startTime, endTime, projectId, plan.getId());
markDataService.countEmotionDistributionAvg(startTime, endTime, projectId, plan.getId());
markDataService.countImportantPlatformPercentageAvg(startTime, endTime, projectId, plan.getId());
}
}
markDataService.countYuqingAmountAvg(startTime, endTime, projectId, null);
markDataService.countEmotionDistributionAvg(startTime, endTime, projectId, null);
markDataService.countImportantPlatformPercentageAvg(startTime, endTime, projectId, null);
log.info("项目:{}-均值计算已完成:{}个", project.getProjectName(), total.incrementAndGet());
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "calculateProjectAvg异常-projectId:" + projectId, e);
}
});
}
@Override
public void yuqingAnalyzeHighWordCache() {
AtomicInteger total = new AtomicInteger();
Long[] time = commonService.getTimeRangeMonth();
List<CompletableFuture<Object>> projectFutures = GlobalPojo.PROJECT_MAP.values().stream().map(project -> CompletableFuture.supplyAsync(() -> {
UserThreadLocal.set(new UserInfo().setProjectId(project.getId()));
markDataService.getHighWord(time[0], time[1], null, false);
log.info("项目:{}-{}-词云缓存已完成:{}个", project.getProjectName(), project.getId(), total.incrementAndGet());
return null;
}, cacheServiceExecutor)).collect(Collectors.toList());
AtomicInteger total2 = new AtomicInteger();
List<CompletableFuture<Object>> planFutures = nonManualProjectPlanDao.findList(new Query()).stream().map(plan -> CompletableFuture.supplyAsync(() -> {
UserThreadLocal.set(new UserInfo().setProjectId(plan.getProjectId()));
markDataService.getHighWord(time[0], time[1], plan.getId(), false);
log.info("方案:{}-{}-词云缓存已完成:{}个", plan.getName(), plan.getId(), total2.incrementAndGet());
return null;
}, cacheServiceExecutor)).collect(Collectors.toList());
projectFutures.addAll(planFutures);
CompletableFuture.allOf(projectFutures.toArray(new CompletableFuture[0])).join();
}
private boolean reportSendByProject(Project project) {
boolean flag = false;
// 扫描setting信息并生成对应报告
......
......@@ -45,6 +45,7 @@ public class ControlCenter {
try {
taskService.messageFlowCache();
taskService.customEventCache();
taskService.yuqingAnalyzeHighWordCache();
} catch (Exception e) {
log.error("定时按天缓存数据-出错", e);
} finally {
......@@ -91,4 +92,16 @@ public class ControlCenter {
}
}
@Async("scheduledExecutor")
@Scheduled(cron = "0 0 3 ? * SUN")
public void calculateProjectAvg() {
log.info("定时每周计算项目均值-启动");
try {
taskService.calculateProjectAvg();
} catch (Exception e) {
log.error("定时按周计算项目均值-出错", e);
} finally {
log.info("定时按周计算项目均值-结束");
}
}
}
......@@ -58,16 +58,17 @@ public class IndexUtil {
start = truncDate(start, pattern);
end = truncDate(end, pattern);
List<String> list = Lists.newArrayList();
Period period = new Period(start.getTime(), end.getTime(), PeriodType.months());
Period monthPeriod = new Period(start.getTime(), end.getTime(), PeriodType.months());
Period yearPeriod = new Period(start.getTime(), end.getTime(), PeriodType.years());
switch (pattern) {
case "yyyy": {
for (int i = 0; i <= period.getYears(); i++) {
for (int i = 0; i <= yearPeriod.getYears(); i++) {
list.add(index + "_" + DF_yyyy.format(DateUtils.addYears(start, i)));
}
break;
}
case "yyyyMM": {
for (int i = 0; i <= period.getMonths(); i++) {
for (int i = 0; i <= monthPeriod.getMonths(); i++) {
list.add(index + "_" + DF_yyyyMM.format(DateUtils.addMonths(start, i)));
}
break;
......
......@@ -7,6 +7,7 @@ import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
......@@ -82,6 +83,34 @@ public class RedisUtil {
return RedisKeyPrefix.NON_MANUAL_PROJECT_MARK_MAX_GID + Tools.concat(projectId, planId, userId);
}
public static String getYuqingAnalyzeProjectAvgCountKey(String projectId, String contendId, String planId){
if (Objects.isNull(planId)){
return RedisKeyPrefix.YUQING_ANALYZE_PROJECT_AVG_COUNT + Tools.concat(projectId, contendId);
}
return RedisKeyPrefix.YUQING_ANALYZE_PROJECT_AVG_COUNT + Tools.concat(projectId, contendId, planId);
}
public static String getYuqingAnalyzeEmotionDistributionAvgKey(String projectId, String contendId, String emotion, String planId){
if (Objects.isNull(planId)){
return RedisKeyPrefix.YUQING_ANALYZE_EMOTION_DISTRIBUTION_AVG + Tools.concat(projectId, contendId, emotion);
}
return RedisKeyPrefix.YUQING_ANALYZE_EMOTION_DISTRIBUTION_AVG + Tools.concat(projectId, contendId, emotion, planId);
}
public static String getYuqingAnalyzePlatformAvgCountKey(String projectId, String contendId, String planId){
if (Objects.isNull(planId)){
return RedisKeyPrefix.YUQING_ANALYZE_PLATFORM_AVG_COUNT + Tools.concat(projectId, contendId);
}
return RedisKeyPrefix.YUQING_ANALYZE_PLATFORM_AVG_COUNT + Tools.concat(projectId, contendId, planId);
}
public static String getYuqingAnalyzeHighWordKey(String projectId, String contendId, String planId, Long startTime, Long endTime){
if (Objects.isNull(planId)){
return RedisKeyPrefix.YUQING_ANALYZE_HIGH_WORD + Tools.concat(projectId, contendId, startTime, endTime);
}
return RedisKeyPrefix.YUQING_ANALYZE_HIGH_WORD + Tools.concat(projectId, contendId, planId, startTime, endTime);
}
public void setExpire(String key, String value, long timeout, TimeUnit unit) {
stringRedisTemplate.opsForValue().set(key, value, timeout, unit);
}
......
......@@ -117,6 +117,7 @@ warn.hotEvent.url=https://auto-push.zhiweidata.com/qbjc/brandkbsPush/interface/b
warn.hotTop.url=https://auto-push.zhiweidata.com/qbjc/brandkbsPush/interface/brandkbs/config/hotTop/{1}
warn.yuQing.url=https://auto-push.zhiweidata.com/qbjc/brandkbsPush/interface/brandkbs/config/yuQing/{1}
warn.taskSwitch.url=https://auto-push.zhiweidata.com/qbjc/brandkbsPush/interface/brandkbs/enable/used
warn.push.ticket.url=https://auto-push.zhiweidata.com/qbjc/pushTask/interface/task/token/ticket
#\u9884\u8B66\u5916\u90E8\u63A5\u53E3
ef.external.filterNew.url=https://ef.zhiweidata.com/external/filterNew.do?firstTypes={1}&start={2}&end={3}
hot.search.url=https://hotsearch-manage.zhiweidata.com/hotsearch/hotSearch/findNewHotSearch?type={1}
......
......@@ -123,6 +123,7 @@ warn.hotEvent.url=http://192.168.0.225:11003/qbjc/brandkbsPush/interface/brandkb
warn.hotTop.url=http://192.168.0.225:11003/qbjc/brandkbsPush/interface/brandkbs/config/hotTop/{1}
warn.yuQing.url=http://192.168.0.225:11003/qbjc/brandkbsPush/interface/brandkbs/config/yuQing/{1}
warn.taskSwitch.url=http://192.168.0.225:11003/qbjc/brandkbsPush/interface/brandkbs/enable/used
warn.push.ticket.url=http://192.168.0.225:11003/qbjc/pushTask/interface/task/token/ticket
#\u9884\u8B66\u5916\u90E8\u63A5\u53E3
ef.external.filterNew.url=https://ef.zhiweidata.com/external/filterNew.do?firstTypes={1}&start={2}&end={3}
hot.search.url=https://hotsearch-manage.zhiweidata.com/hotsearch/hotSearch/findNewHotSearch?type={1}
......
......@@ -117,6 +117,7 @@ warn.hotEvent.url=https://auto-push.zhiweidata.com/qbjc/brandkbsPush/interface/b
warn.hotTop.url=https://auto-push.zhiweidata.com/qbjc/brandkbsPush/interface/brandkbs/config/hotTop/{1}
warn.yuQing.url=https://auto-push.zhiweidata.com/qbjc/brandkbsPush/interface/brandkbs/config/yuQing/{1}
warn.taskSwitch.url=https://auto-push.zhiweidata.com/qbjc/brandkbsPush/interface/brandkbs/enable/used
warn.push.ticket.url=https://auto-push.zhiweidata.com/qbjc/pushTask/interface/task/token/ticket
#\u9884\u8B66\u5916\u90E8\u63A5\u53E3
ef.external.filterNew.url=https://ef.zhiweidata.com/external/filterNew.do?firstTypes={1}&start={2}&end={3}
hot.search.url=https://hotsearch-manage.zhiweidata.com/hotsearch/hotSearch/findNewHotSearch?type={1}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment