Skip to content

B
brandkbs2
Commit 2f399fac by shenjunjie
Options

Merge branch 'release' into 'master' 

Release

See merge request !429
parents db2182e6 829098dd
Showing with 1733 additions and 79 deletions
src/main/java/com/zhiwei/brandkbs2/auth/AuthAspect.java
......@@ -88,7 +88,8 @@ public class AuthAspect {
}
UserThreadLocal.set(userInfo);
}
} catch (Exception ignore) {
} catch (Exception e) {
log.info("around-通过token设置userInfo出错,", e);
} finally {
proceed = joinPoint.proceed();
UserThreadLocal.clear();
......
src/main/java/com/zhiwei/brandkbs2/common/GenericAttribute.java
......@@ -15,8 +15,8 @@ public class GenericAttribute {
* es index
*/
public static final String ES_INDEX_PRE = "brandkbs2_";
// public static final String ES_INDEX_TEST = "brandkbs2_test";
public static final String ES_INDEX_TEST = "brandkbs2_2022";
public static final String ES_INDEX_TEST = "brandkbs2_test";
// public static final String ES_INDEX_TEST = "brandkbs2_2022";
public static final String ES_CHANNEL_INDEX_TEST = "brandkbs2_channel_record_test";
public static final String ES_CHANNEL_INDEX_PREFIX = "brandkbs2_channel_record_";
/**
......@@ -145,6 +145,7 @@ public class GenericAttribute {
public static final String EXPORT_AMOUNT = "exportAmount";
public static final String EXPIRED_TIME = "expiredTime";
public static final String PHONE_NUMBER = "phoneNumber";
public static final String IS_MANUAL = "isManual";
// public enum ChannelParam{
// 负面稿件数("negativeArticles"),
......
src/main/java/com/zhiwei/brandkbs2/common/RedisKeyPrefix.java
......@@ -100,6 +100,8 @@ public class RedisKeyPrefix {
*/
public static final String TOOLSET_ARTICLE_SUMMARY_LIMIT = "BRANDKBS:TOOLSET:ARTICLE_SUMMARY:LIMIT:";
public static final String NON_MANUAL_PROJECT_MARK_MAX_GID = "BRANDKBS:NON_MANUAL:PROJECT:MARK:MAX_GID:";
public static String projectWarnHotTopKeyAll(String projectId, String type) {
return RedisKeyPrefix.generateRedisKey(RedisKeyPrefix.PROJECT_WARN_HOT_TOP, projectId, Tools.concat(type, "*"));
}
......
src/main/java/com/zhiwei/brandkbs2/config/Constant.java
......@@ -13,6 +13,7 @@ import java.text.ParseException;
public class Constant {
public static final Long EIGHT_HOUR = 8 * 60 * 60 * 1000L;
public static final Long ONE_HOUR = 1 * 60 * 60 * 1000L;
public static final Long ONE_DAY = 24 * 60 * 60 * 1000L;
public static final Long ONE_WEEK = 7 * 24 * 60 * 60 * 1000L;
public static final Long ONE_MONTH = 30 * 24 * 60 * 60 * 1000L;
......
src/main/java/com/zhiwei/brandkbs2/controller/admin/EventController.java
......@@ -41,7 +41,7 @@ import java.util.Set;
* @date: 2022-05-16 14:30
*/
@RestController
@RequestMapping("/admin/event")
//@RequestMapping("/admin/event")
@Api(tags = "事件管理", description = "提供事件的增、删、改、查等功能")
@Auth(role = RoleEnum.COMMON_ADMIN)
@Deprecated
......
src/main/java/com/zhiwei/brandkbs2/controller/admin/ProjectController.java
......@@ -262,4 +262,50 @@ public class ProjectController extends BaseController {
noticeInfoService.deleteNoticeInfo(id);
return ResponseResult.success();
}
@ApiOperation("添加非人工项目")
@PostMapping("/non-manual/add")
public ResponseResult addNonManualProject(@RequestBody ProjectVO projectVO) {
return ProjectService.addNonManualProject(projectVO);
}
@ApiOperation("修改非人工项目")
@PutMapping("/non-manual/update")
public ResponseResult updateNonManualProject(@RequestBody ProjectVO projectVO) {
return ProjectService.updateNonManualProject(projectVO);
}
@ApiOperation("查询所有非人工项目")
@ApiImplicitParams({@ApiImplicitParam(name = "page", value = "页码", required = false, defaultValue = "1", paramType = "query", dataType = "int"),
@ApiImplicitParam(name = "size", value = "每页记录数", required = false, defaultValue = "10", paramType = "query", dataType = "int"),
@ApiImplicitParam(name = "sorter", value = "排序", required = false, defaultValue = "{\"cTime\":\"descend\"}", paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "keyword", value = "搜索关键字", required = false, defaultValue = "", paramType = "query", dataType = "string")})
@GetMapping("/non-manual/list")
public ResponseResult findNonManualProjectList(@RequestParam(value = "page", defaultValue = "1") int page,
@RequestParam(value = "size", defaultValue = "10") int size,
@RequestParam(value = "sorter", defaultValue = "{\"cTime\":\"descend\"}") String sorter,
@RequestParam(value = "keyword", defaultValue = "") String keyword) {
return ResponseResult.success(ProjectService.findNonManualProjectList(page, size, sorter, keyword));
}
@ApiOperation("非人工项目-维护记录列表")
@GetMapping("/non-manual/maintain/list")
public ResponseResult findNonManualProjectMaintainList(@RequestParam(value = "page", defaultValue = "1") int page,
@RequestParam(value = "size", defaultValue = "10") int size,
@RequestParam(value = "projectId") String projectId) {
return ResponseResult.success(ProjectService.findNonManualProjectMaintainList(projectId, page, size));
}
@ApiOperation("非人工项目-维护记录数据总览")
@GetMapping("/non-manual/maintain/overview")
public ResponseResult interactionOverview(@RequestParam(value = "projectId") String projectId) {
return ResponseResult.success(ProjectService.nonManualProjectOverview(projectId));
}
@ApiOperation("非人工项目-维护记录-剩余数据量调整")
@PutMapping("/non-manual/maintain/adjust")
public ResponseResult adjustInteractionRecord(@ApiParam(name = "json:{count:调整数值, projectId:项目id)") @RequestBody JSONObject json) {
ProjectService.adjustNonManualProjectDataBalance(json.getString("projectId"), json.getInteger("count"));
return ResponseResult.success();
}
}
src/main/java/com/zhiwei/brandkbs2/controller/app/AppArticleController.java
......@@ -9,17 +9,11 @@ import com.zhiwei.brandkbs2.controller.BaseController;
import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
import com.zhiwei.brandkbs2.enmus.RoleEnum;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.dto.ExportAppYuqingDTO;
import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
import com.zhiwei.brandkbs2.pojo.dto.ReportDTO;
import com.zhiwei.brandkbs2.pojo.dto.ReportSearchDTO;
import com.zhiwei.brandkbs2.pojo.dto.*;
import com.zhiwei.brandkbs2.service.*;
import com.zhiwei.brandkbs2.util.Tools;
import com.zhiwei.middleware.mark.vo.MarkerTag;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiImplicitParams;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.*;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
......@@ -242,7 +236,7 @@ public class AppArticleController extends BaseController {
@RequestParam(value = "endTime", required = false) Long endTime,
@RequestParam(value = "page", defaultValue = "1") Integer page,
@RequestParam(value = "pageSize", defaultValue = "10")Integer pageSize,
@RequestParam(value = "keyword", required = false) String keyword) {
@RequestParam(value = "keyword", required = false) String keyword) {
String linkedGroupId = projectService.getProjectById(UserThreadLocal.getProjectId()).getBrandLinkedGroupId();
ResponseEntity<JSONObject> responseEntity = restTemplate.getForEntity(briefReportUrl + "?startTime={1}&endTime={2}&page={3}&pageSize={4}&projectId={5" +
"}&keyword={6}", JSONObject.class, startTime, endTime, page, pageSize, linkedGroupId, keyword);
......@@ -321,6 +315,132 @@ public class AppArticleController extends BaseController {
return ResponseResult.success(markDataService.interactionUpdatePrediction(markSearchDTO));
}
@ApiOperation("非人工项目-方案设置-新建方案")
@PostMapping("/non-manual/plan/add")
public ResponseResult addNonManualProjectPlan(@RequestBody NonManualProjectPlanDTO dto){
return markDataService.addNonManualProjectPlan(dto);
}
@ApiOperation("非人工项目-方案设置-删除方案")
@DeleteMapping("/non-manual/plan/delete/{id}")
public ResponseResult deleteNonManualProjectPlan(@PathVariable String id) {
markDataService.deleteNonManualProjectPlan(id);
return ResponseResult.success();
}
@ApiOperation("非人工项目-方案设置-获取方案详情")
@GetMapping("/non-manual/plan/{id}")
public ResponseResult getNonManualProjectPlanInfo(@PathVariable String id) {
return ResponseResult.success(markDataService.getNonManualProjectPlanInfo(id));
}
@ApiOperation("非人工项目-方案设置-编辑方案")
@PostMapping("/non-manual/plan/update")
public ResponseResult updateNonManualProjectPlan(@RequestBody NonManualProjectPlanDTO dto){
return markDataService.updateNonManualProjectPlan(dto);
}
@ApiOperation("非人工项目-方案设置-数据量预估")
@PutMapping("/non-manual/plan/prediction")
public ResponseResult getPlanDataPrediction(@ApiParam(name = "json:{keyword:关键词, filterWord:去噪词)") @RequestBody JSONObject json){
return ResponseResult.success(markDataService.getPlanDataPrediction(json.getString("keyword"), json.getString("filterWord")));
}
@ApiOperation("非人工项目-方案设置-方案列表")
@GetMapping("/non-manual/plan/list")
public ResponseResult getNonManualProjectPlanList(){
return ResponseResult.success(markDataService.getNonManualProjectPlanList());
}
@ApiOperation("非人工项目-方案设置-昨日，今日数据消耗量")
@GetMapping("/non-manual/plan/count")
public ResponseResult getProjectPlanConsume(){
return ResponseResult.success(markDataService.getProjectPlanConsume());
}
@ApiOperation("非人工项目-方案设置-获取总数据消耗图谱")
@GetMapping("/non-manual/plan/total-graph")
public ResponseResult getTotalDataConsumeGraph(@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime){
return ResponseResult.success(markDataService.getTotalDataConsumeGraph(startTime, endTime));
}
@ApiOperation("非人工项目-方案设置-获取时间范围数据消耗图谱")
@GetMapping("/non-manual/plan/time-graph")
public ResponseResult getTimeDataConsumeGraph(@RequestParam(value = "planId") String planId,
@RequestParam(value = "startTime") Long startTime,
@RequestParam(value = "endTime") Long endTime){
return ResponseResult.success(markDataService.getTimeDataConsumeGraph(planId, startTime, endTime));
}
@ApiOperation("非人工项目-舆情数据-舆情数据筛选条件")
@GetMapping("/non-manual/mark/criteria")
public ResponseResult getNonManualMarkCriteria(){
return ResponseResult.success(markDataService.getNonManualMarkCriteria());
}
@ApiOperation("非人工项目-舆情数据-舆情数据列表")
@PostMapping("/non-manual/mark/list")
public ResponseResult getNonManualMarkList(@RequestBody MarkSearchDTO markSearchDTO){
return ResponseResult.success(markDataService.getNonManualMarkList(markSearchDTO));
}
@ApiOperation("非人工项目-舆情数据-单条纠错")
@PostMapping("/non-manual/mark/tag")
public ResponseResult updateNonManualMarkTag(@RequestBody JSONObject json) {
String zip = json.getString("zip");
String linkedGroupId = json.getString("linkedGroupId");
StringBuilder mtag = new StringBuilder(json.getString("mtag"));
// 带上项目配置的必要标注标签
if (checkMTagIllegal(mtag)){
return ResponseResult.failure("标注标签出现同组标签，请勿连点标注");
}
markDataService.addMtag(zip, linkedGroupId, mtag.toString());
return ResponseResult.success();
}
@ApiOperation("非人工项目-舆情数据-互动量更新")
@PostMapping("/non-manual/mark/interaction-update")
@LogRecord(description = "非人工项目舆情库-有效舆情互动量更新")
@DownloadTask(taskName = "舆情库互动量更新结果下载", description = "舆情库互动量更新结果", entity = true)
public ResponseResult nonManualMarkInteractionUpdate(@RequestBody MarkSearchDTO markSearchDTO){
return markDataService.nonManualMarkInteractionUpdate(markSearchDTO);
}
@ApiOperation("非人工项目-舆情库互动量更新-数据量预估")
@PostMapping("/non-manual/mark/interaction-prediction")
public ResponseResult nonManualInteractionUpdatePrediction(@RequestBody MarkSearchDTO markSearchDTO){
return ResponseResult.success(markDataService.nonManualInteractionUpdatePrediction(markSearchDTO));
}
@ApiOperation("非人工项目-舆情数据-统计各方案数据量")
@GetMapping("/non-manual/mark/count")
public ResponseResult countNonManualMarkData(){
return ResponseResult.success(markDataService.getNonManualMarkCountList());
}
@ApiOperation("非人工项目-舆情数据-记录gid")
@PutMapping("/non-manual/mark/gid")
public ResponseResult adjustMaxGid(@ApiParam(name = "json:{planId:方案id, maxGid:值)") @RequestBody JSONObject json) {
markDataService.adjustMaxGid(json.getString("planId"), json.getLong("maxGid"));
return ResponseResult.success();
}
@ApiOperation("非人工项目-舆情数据-生成聚合")
@PostMapping("/non-manual/mark/aggree")
public ResponseResult generateNonManualMarkAggreeList(@ApiParam(name = "info:{planId:方案id, startTime:开始时间, endTime:结束时间)") @RequestBody JSONObject info) {
String planId = info.getString("planId");
Long startTime = info.getLong("startTime");
Long endTime = info.getLong("endTime");
return ResponseResult.success(markDataService.generateNonManualMarkAggreeList(planId, startTime, endTime));
}
@ApiOperation("非人工项目-舆情数据-获取聚合结果")
@PostMapping("/non-manual/mark/aggree/list")
public ResponseResult getNonManualMarkAggreeList(@RequestBody MarkSearchDTO markSearchDTO) {
return ResponseResult.success(markDataService.getNonManualMarkAggreeList(markSearchDTO));
}
private boolean checkMTagIllegal(StringBuilder mtag) {
List<MarkerTag> hitTags = projectService.getProjectById(UserThreadLocal.getProjectId()).getHitTags();
if (!Tools.isEmpty(hitTags)) {
......
src/main/java/com/zhiwei/brandkbs2/controller/app/AppDownloadController.java
......@@ -336,6 +336,19 @@ public class AppDownloadController extends BaseController {
return ResponseResult.success(filePath);
}
@ApiOperation("非人工项目-舆情库-舆情数据下载")
@PostMapping(value = "/yuqing/non-manual/mark")
@LogRecord(description = "非人工项目-舆情库-舆情导出", values = {"startTime", "endTime", "customTags", "field", "keyword", "politicsLevel", "mainBodyType", "platforms", "region", "tags"}, entity = true, arguments = true)
@DownloadTask(taskName = "舆情库舆情数据下载", description = "舆情库舆情数据")
public ResponseResult downloadNonManualMarkList(@RequestBody MarkSearchDTO markSearchDTO) {
Pair<String, List<ExportAppYuqingDTO>> stringListPair = markDataService.downloadNonManualMarkList(markSearchDTO);
// excel写入至指定路径
String projectName = projectService.getProjectById(UserThreadLocal.getProjectId()).getProjectName();
String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), stringListPair.getLeft() + "_舆情列表数据");
EasyExcelUtil.write(filePath, "sheet1", ExportAppYuqingDTO.class, stringListPair.getRight());
return ResponseResult.success(filePath);
}
@ApiOperation("渠道库文章列表")
@ApiImplicitParams({
@ApiImplicitParam(name = "startTime", value = "开始时间", required = true, paramType = "query", dataType = "long"),
......
src/main/java/com/zhiwei/brandkbs2/controller/app/AppEventController.java
......@@ -71,34 +71,39 @@ public class AppEventController extends BaseController {
}
@ApiOperation("前台事件库-搜索条件")
@GetMapping("/search/criteria")
// @GetMapping("/search/criteria")
@Deprecated
public ResponseResult getEventsSearchCriteria() {
return ResponseResult.success(eventService.getEventsSearchCriteria());
}
@ApiOperation("前台事件库-品牌事件库")
@PostMapping("/list")
// @PostMapping("/list")
@Deprecated
public ResponseResult getEventList(@RequestBody EventSearchDTO eventSearchDTO) {
return ResponseResult.success(eventService.getEventList(eventSearchDTO));
}
@ApiOperation("前台事件库-事件详情-基础信息")
@GetMapping("/info/base/{id}")
// @GetMapping("/info/base/{id}")
@Auth(role = RoleEnum.NO_AUTHORISE)
@Deprecated
public ResponseResult getEventBaseInfo(@PathVariable String id) {
return ResponseResult.success(eventService.getEventBaseInfo(id));
}
@ApiOperation("前台事件库-事件详情-传播趋势图")
@GetMapping("/info/dissemination-trends/{id}")
// @GetMapping("/info/dissemination-trends/{id}")
@Auth(role = RoleEnum.NO_AUTHORISE)
@Deprecated
public ResponseResult getEventDisseminationTrends(@PathVariable String id, @RequestParam(value = "type", defaultValue = "小时") String type) {
return ResponseResult.success(eventService.getEventDisseminationTrends(id, type));
}
@ApiOperation("前台事件库-事件详情-渠道发声")
@GetMapping("/info/channel-voices/{id}")
// @GetMapping("/info/channel-voices/{id}")
@Auth(role = RoleEnum.NO_AUTHORISE)
@Deprecated
public ResponseResult getEventChannelVoices(@PathVariable String id,
@RequestParam(value = "type", defaultValue = "重要渠道") String type,
@RequestParam(value = "page", defaultValue = "1") int page,
......@@ -108,8 +113,9 @@ public class AppEventController extends BaseController {
}
@ApiOperation("前台事件库-事件详情-热门文章分析")
@GetMapping("/info/top-articles-analysis/{id}")
// @GetMapping("/info/top-articles-analysis/{id}")
@Auth(role = RoleEnum.NO_AUTHORISE)
@Deprecated
public ResponseResult getEventTopArticlesAnalysis(@PathVariable String id,
@RequestParam(value = "type", defaultValue = "按时间") String type,
@RequestParam(value = "emotion", defaultValue = "全部") String emotion,
......
src/main/java/com/zhiwei/brandkbs2/dao/NonManualProjectDataUpdateRecordDao.java 0 → 100644
package com.zhiwei.brandkbs2.dao;
import com.zhiwei.brandkbs2.pojo.NonManualProjectDataUpdateRecord;
import com.zhiwei.brandkbs2.pojo.vo.LineVO;
import java.util.List;
/**
* @ClassName: BytedanceCustomEventUpdateTaskDataDao
* @Description BytedanceCustomEventUpdateTaskDataDao
* @author: cjz
* @date: 2023-11-10 16:30
*/
public interface NonManualProjectDataUpdateRecordDao extends BaseMongoDao<NonManualProjectDataUpdateRecord>{
/**
* 数据总览
* @return list
*/
List<LineVO> aggregateDayLastRecord(String projectId);
}
src/main/java/com/zhiwei/brandkbs2/dao/NonManualProjectPlanDao.java 0 → 100644
package com.zhiwei.brandkbs2.dao;
import com.zhiwei.brandkbs2.pojo.NonManualProjectPlan;
/**
* @ClassName: NonManualProjectPlanDao
* @Description NonManualProjectPlanDao
* @author: cjz
* @date: 2023-11-16 10:30
*/
public interface NonManualProjectPlanDao extends BaseMongoDao<NonManualProjectPlan> {
}
src/main/java/com/zhiwei/brandkbs2/dao/impl/NonManualProjectDataUpdateRecordDaoImpl.java 0 → 100644
package com.zhiwei.brandkbs2.dao.impl;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.dao.NonManualProjectDataUpdateRecordDao;
import com.zhiwei.brandkbs2.pojo.NonManualProjectDataUpdateRecord;
import com.zhiwei.brandkbs2.pojo.vo.LineVO;
import org.springframework.data.domain.Sort;
import org.springframework.data.mongodb.core.aggregation.Aggregation;
import org.springframework.data.mongodb.core.aggregation.AggregationOperation;
import org.springframework.data.mongodb.core.aggregation.AggregationResults;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;
/**
* @ClassName: BytedanceCustomEventUpdateTaskDataDaoImpl
* @Description BytedanceCustomEventUpdateTaskDataDaoImpl
* @author: cjz
* @date: 2023-11-10 16:31
*/
@Component("nonManualProjectDataUpdateRecordDao")
public class NonManualProjectDataUpdateRecordDaoImpl extends BaseMongoDaoImpl<NonManualProjectDataUpdateRecord> implements NonManualProjectDataUpdateRecordDao {
private static final String COLLECTION_NAME = "brandkbs_project_data_update_record";
public NonManualProjectDataUpdateRecordDaoImpl() {
super(COLLECTION_NAME);
}
@Override
public List<LineVO> aggregateDayLastRecord(String projectId) {
List<LineVO> res = new ArrayList<>();
Criteria criteria = Criteria.where("projectId").is(projectId);
List<AggregationOperation> operations = new ArrayList<>();
// operations区分先后顺序
operations.add(Aggregation.match(criteria));
operations.add(Aggregation.project("currentCount").and("cTime").substring(0, 10).as("cTime"));
operations.add(Aggregation.group("cTime").last("currentCount").as("count"));
operations.add(Aggregation.sort(Sort.by(Sort.Order.asc("_id"))));
Aggregation aggregation = Aggregation.newAggregation(operations);
// 优先小的年份
AggregationResults<JSONObject> aggregateResult = mongoTemplate.aggregate(aggregation, COLLECTION_NAME, JSONObject.class);
List<JSONObject> mappedResults = aggregateResult.getMappedResults();
for (JSONObject mappedResult : mappedResults) {
res.add(new LineVO(mappedResult.getLong("count"), mappedResult.getDate("_id").getTime()));
}
return res;
}
}
src/main/java/com/zhiwei/brandkbs2/dao/impl/NonManualProjectPlanDaoImpl.java 0 → 100644
package com.zhiwei.brandkbs2.dao.impl;
import com.zhiwei.brandkbs2.dao.NonManualProjectPlanDao;
import com.zhiwei.brandkbs2.pojo.NonManualProjectPlan;
import org.springframework.stereotype.Component;
/**
* @ClassName: NonManualProjectPlanDaoImpl
* @Description NonManualProjectPlanDaoImpl
* @author: cjz
* @date: 2023-11-16 10:31
*/
@Component("nonManualProjectPlanDao")
public class NonManualProjectPlanDaoImpl extends BaseMongoDaoImpl<NonManualProjectPlan> implements NonManualProjectPlanDao {
private static final String COLLECTION_NAME = "brandkbs_non_manual_project_plan";
public NonManualProjectPlanDaoImpl() {
super(COLLECTION_NAME);
}
}
src/main/java/com/zhiwei/brandkbs2/es/ChannelEsDao.java
......@@ -86,7 +86,7 @@ public class ChannelEsDao extends EsClientDao {
BulkResponse bulkResponse = retryTemplate.execute(context -> {
try {
return channelEsClient.bulk(bulkRequest, RequestOptions.DEFAULT);
} catch (IOException ignored) {
} catch (Exception ignored) {
return null;
}
});
......
src/main/java/com/zhiwei/brandkbs2/es/EsQueryTools.java
......@@ -104,8 +104,12 @@ public class EsQueryTools {
return assembleCacheMapsQuery(projectId, contendId, politicsLevel, field, region, mainBodyType, null);
}
public static BoolQueryBuilder assembleCacheMapsQuery(String projectId,String contendId, String politicsLevel, String field, String region,
String mainBodyType,Integer channelEmotion) {
public static BoolQueryBuilder assembleCacheMapsQuery(String projectId, String planId, String politicsLevel) {
return assembleCacheMapsQuery(projectId, planId, politicsLevel, null, null, null, null);
}
public static BoolQueryBuilder assembleCacheMapsQuery(String projectId, String contendId, String politicsLevel, String field, String region,
String mainBodyType, Integer channelEmotion) {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
String key = concat(projectId, contendId);
BoolQueryBuilder nestedBoolBuilder = QueryBuilders.boolQuery();
......@@ -121,6 +125,22 @@ public class EsQueryTools {
return boolQueryBuilder;
}
public static BoolQueryBuilder assembleCacheMapsPlanQuery(String projectId, String planId) {
return assembleCacheMapsPlanQuery(projectId, planId, null);
}
public static BoolQueryBuilder assembleCacheMapsPlanQuery(String projectId, String planId, String politicsLevel) {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
String key = concat(projectId, planId);
BoolQueryBuilder nestedBoolBuilder = QueryBuilders.boolQuery();
// 必要条件
nestedBoolBuilder.must(QueryBuilders.termQuery("brandkbs_cache_maps.plan_key.keyword", key));
boolQueryBuilder.must(cacheMapsNestedQuery(nestedBoolBuilder));
// sensitiveChannel
addSensitiveChannel(politicsLevel, null, null, null, boolQueryBuilder);
return boolQueryBuilder;
}
public static BoolQueryBuilder assembleCacheMapsQueryWithProject(String projectId, BrandkbsYuQingConfig config) {
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
BoolQueryBuilder nestedBoolBuilder = QueryBuilders.boolQuery();
......@@ -254,6 +274,31 @@ public class EsQueryTools {
return queryBuilder;
}
public static BoolQueryBuilder assembleForward2Query(boolean isForward) {
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
// 需要转发数据，平台必须为微博
if (isForward) {
BoolQueryBuilder mustWeiboCondition = QueryBuilders.boolQuery();
queryBuilder.should(mustWeiboCondition.must(QueryBuilders.termQuery("platform_id", "5d02236e6395002a7c380b79")));
queryBuilder.should(QueryBuilders.termQuery("is_forward", true));
}else {
queryBuilder.mustNot(QueryBuilders.termQuery("is_forward", true));
}
return queryBuilder;
}
/**
* c2查询
*
* @param dataType 数据类型（1:长文本, 2:短文本, 3:问答, 5:视频）
* @return
*/
public static BoolQueryBuilder assembleC2Query(Integer dataType) {
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
queryBuilder.should(QueryBuilders.termQuery("c2", dataType << 23));
return queryBuilder;
}
/**
* 字段不拆封,多字段 同关键词
*
......@@ -349,8 +394,7 @@ public class EsQueryTools {
public static void addSensitiveChannel(String politicsLevel, String field, String region, String mainBodyType, BoolQueryBuilder boolQueryBuilder) {
if (null != politicsLevel) {
boolQueryBuilder.must(QueryBuilders.termQuery(GenericAttribute.ES_SENSITIVE_CHANNEL + "." + GenericAttribute.ES_POLITICS_LEVEL + ".keyword",
politicsLevel));
boolQueryBuilder.must(QueryBuilders.termQuery(GenericAttribute.ES_SENSITIVE_CHANNEL + "." + GenericAttribute.ES_POLITICS_LEVEL + ".keyword", politicsLevel));
}
if (null != mainBodyType) {
boolQueryBuilder.must(QueryBuilders.matchPhraseQuery(GenericAttribute.ES_SENSITIVE_CHANNEL + "." + GenericAttribute.ES_MAIN_BODY_TYPE + ".keyword", mainBodyType));
......
src/main/java/com/zhiwei/brandkbs2/pojo/ChannelIndex.java
......@@ -103,9 +103,14 @@ public class ChannelIndex extends AbstractBaseMongo {
return res;
}
List<Map<String, Object>> cacheMaps = (List<Map<String, Object>>) sourceAsMap.get(GenericAttribute.ES_BRANDKBS_CACHE_MAPS);
List<String> manualProjectIds = GlobalPojo.PROJECT_MAP.entrySet().stream().filter(entry -> entry.getValue().isManual()).map(Map.Entry::getKey).collect(Collectors.toList());
for (Map<String, Object> cacheMap : cacheMaps) {
String projectId = String.valueOf(cacheMap.get("project_id"));
String contendId = String.valueOf(cacheMap.get("contend_id"));
// 剔除非人工项目
if(manualProjectIds.contains(projectId)){
continue;
}
ChannelIndex channelIndex = new ChannelIndex(projectId, contendId, messagePlatform.getName(), realSource, source);
// 默认主品牌
channelIndex.setContendId(String.valueOf(0));
......
src/main/java/com/zhiwei/brandkbs2/pojo/Event.java
......@@ -19,6 +19,7 @@ import java.util.Objects;
@Getter
@Setter
@Document(collection = "brandkbs_event")
@Deprecated
public class Event extends AbstractBaseMongo {
/**
......
src/main/java/com/zhiwei/brandkbs2/pojo/EventData.java
......@@ -23,6 +23,7 @@ import java.util.Optional;
*/
@Getter
@Setter
@Deprecated
public class EventData extends AbstractBaseMongo {
/**
......
src/main/java/com/zhiwei/brandkbs2/pojo/EventDisseminationTrend.java
......@@ -19,6 +19,7 @@ import java.util.Map;
@Getter
@Setter
@Document(collection = "brandkbs_event_dissemination_trend")
@Deprecated
public class EventDisseminationTrend extends AbstractBaseMongo {
/**
* 负面稿件传播
......
src/main/java/com/zhiwei/brandkbs2/pojo/EventTopArticlesAnalysis.java
......@@ -12,6 +12,7 @@ import org.springframework.data.mongodb.core.mapping.Document;
@Getter
@Setter
@Document(collection = "brandkbs_event_top_articles_analysis")
@Deprecated
public class EventTopArticlesAnalysis extends AbstractBaseMongo {
/**
......
src/main/java/com/zhiwei/brandkbs2/pojo/MarkFlowEntity.java
......@@ -100,6 +100,11 @@ public class MarkFlowEntity implements Serializable {
tJson.put(GenericAttribute.ES_CNAME, "采集来源缺失");
}
int c2 = tJson.getIntValue(GenericAttribute.ES_C2);
// 微博平台头像url拼接
String weiboHeadUrl = "https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/weibo-avatar-url?url=";
if (1020 == tJson.getIntValue(GenericAttribute.ES_C4) && !tJson.getString("avatar_url").contains(weiboHeadUrl)){
tJson.put("avatar_url", weiboHeadUrl + tJson.getString("avatar_url"));
}
switch (ClassB.TypeB.fromEncode(c2)) {
case COMPLETE:
if (!Tools.isEmpty(tJson.get("ind_full_text"))) {
......
src/main/java/com/zhiwei/brandkbs2/pojo/NonManualProjectDataUpdateRecord.java 0 → 100644
package com.zhiwei.brandkbs2.pojo;
import lombok.Getter;
import lombok.Setter;
import java.util.Date;
/**
* @ClassName: NonManualProjectDataUpdateRecord
* @Description 非人工项目剩余数据量调整记录
* @author: cjz
* @date: 2023-11-10 15:53
*/
@Getter
@Setter
public class NonManualProjectDataUpdateRecord extends AbstractBaseMongo {
/**
* 项目id
*/
private String projectId;
/**
* 上一次记录数(本次调整前剩余次数)
*/
private int lastCount;
/**
* 本次记录数(本次调整后剩余次数)
*/
private int currentCount;
/**
* 提交人
*/
private String submitter;
/**
* 创建时间(Date类型方便聚合查询)
*/
private Date cTime;
public NonManualProjectDataUpdateRecord(String projectId, int lastCount, int currentCount, String submitter){
this.projectId = projectId;
this.lastCount = lastCount;
this.currentCount = currentCount;
this.submitter = submitter;
this.cTime = new Date();
}
}
src/main/java/com/zhiwei/brandkbs2/pojo/NonManualProjectPlan.java 0 → 100644
package com.zhiwei.brandkbs2.pojo;
import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.pojo.dto.NonManualProjectPlanDTO;
import com.zhiwei.brandkbs2.util.Tools;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import java.util.List;
/**
* @ClassName: NonManualProjectPlan
* @Description 非人工项目方案
* @author: cjz
* @date: 2023-11-15 14:53
*/
@Getter
@Setter
@NoArgsConstructor
public class NonManualProjectPlan extends AbstractBaseMongo{
/**
* 项目id
*/
private String projectId;
/**
* 方案名
*/
private String name;
/**
* 关键词
*/
private String keyword;
/**
* 去噪词
*/
private String filterWord;
/**
* 方案类型 （永久/临时）
*/
private Boolean lasting;
/**
* 方案结束时间
*/
private Long endTime;
/**
* 数据范围 （原创，转发，ocr）
*/
private List<String> type;
/**
* 创建时间
*/
private Long cTime;
/**
* 修改时间
*/
private Long uTime;
/**
* 提交人
*/
private String submitter;
public NonManualProjectPlan(NonManualProjectPlanDTO dto){
this.id = dto.getId();
this.projectId = UserThreadLocal.getProjectId();
this.name = dto.getName();
this.keyword = Tools.canonicalKeyword(dto.getKeyword());
this.filterWord = Tools.canonicalKeyword(dto.getFilterWord());
this.lasting = dto.getLasting();
this.endTime = dto.getEndTime();
this.type = dto.getType();
this.cTime = System.currentTimeMillis();
this.uTime = System.currentTimeMillis();
this.submitter = UserThreadLocal.getNickname();
}
}
src/main/java/com/zhiwei/brandkbs2/pojo/Project.java
......@@ -75,6 +75,11 @@ public class Project extends AbstractProject {
private boolean isStart;
/**
* 是否为人工项目
*/
private boolean isManual;
/**
* 创建时间
*/
private Long cTime;
......@@ -100,6 +105,11 @@ public class Project extends AbstractProject {
private int interactionBalance;
/**
* 剩余数据量（仅非人工项目）
*/
private int dataBalance;
/**
* 项目主品牌配置信息转换
*
* @return 项目对象
......
src/main/java/com/zhiwei/brandkbs2/pojo/dto/MarkSearchDTO.java
......@@ -160,4 +160,28 @@ public class MarkSearchDTO {
@ApiModelProperty(value = "屏蔽词")
private String shieldWord;
/**
* 方案id,仅非人工项目
*/
@ApiModelProperty(value = "方案id")
private String planId;
/**
* 内容类型（原创/转发）,仅非人工项目
*/
@ApiModelProperty(value = "内容类型（原创/转发）")
private Boolean forward;
/**
* 信息浏览（已读/未读）,仅非人工项目
*/
@ApiModelProperty(value = "信息浏览（已读/未读）")
private Boolean read;
/**
* 数据类型（长文本/短文本/问答/视频）,仅非人工项目
*/
@ApiModelProperty(value = "数据类型（1:长文本, 2:短文本, 3:问答, 5:视频）")
private Integer dataType;
/**
* gid用于未读已读筛选,仅非人工项目
*/
private Long gid;
}
src/main/java/com/zhiwei/brandkbs2/pojo/dto/NonManualProjectPlanDTO.java 0 → 100644
package com.zhiwei.brandkbs2.pojo.dto;
import com.zhiwei.brandkbs2.pojo.NonManualProjectPlan;
import io.swagger.annotations.ApiModelProperty;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import java.util.List;
/**
* @ClassName: NonManualProjectPlan
* @Description 非人工项目方案传输
* @author: cjz
* @date: 2023-11-16 9:53
*/
@Getter
@Setter
@AllArgsConstructor
@NoArgsConstructor
public class NonManualProjectPlanDTO {
/**
* 主键ID
*/
@ApiModelProperty("项目主键ID")
private String id;
/**
* 方案名
*/
private String name;
/**
* 关键词
*/
private String keyword;
/**
* 去噪词
*/
private String filterWord;
/**
* 方案类型 （永久/临时）
*/
private Boolean lasting;
/**
* 方案结束时间
*/
private Long endTime;
/**
* 数据范围 （原创，转发，ocr）
*/
private List<String> type;
public NonManualProjectPlanDTO(NonManualProjectPlan plan){
this.id = plan.getId();
this.name = plan.getName();
this.keyword = plan.getKeyword();
this.filterWord = plan.getFilterWord();
this.lasting = plan.getLasting();
this.endTime = plan.getEndTime();
this.type = plan.getType();
}
}
src/main/java/com/zhiwei/brandkbs2/pojo/vo/ProjectVO.java
......@@ -9,12 +9,10 @@ import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import lombok.ToString;
import org.apache.commons.collections4.CollectionUtils;
import org.bson.types.ObjectId;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.stream.Collectors;
/**
......@@ -166,11 +164,37 @@ public class ProjectVO {
project.setBlackChannelGroup(this.getBlackChannelGroup());
project.setShow(true);
project.setStart(false);
project.setManual(true);
project.setCTime(time.getTime());
project.setUTime(time.getTime());
return project;
}
public static Project createNonManualProject(ProjectVO vo){
Project project = new Project();
project.setId(vo.getId());
project.setProjectName(vo.getProjectName());
project.setBrandName(vo.getBrandName());
project.setBrandLinkedGroup(vo.getBrandLinkedGroup());
project.setBrandLinkedGroupId(vo.getBrandLinkedGroupId());
project.setImportTime(vo.getImportTime().getTime());
if (null == vo.getAvatarUrl()) {
project.setAvatarUrl(Constant.DEFAULT_PROJECT_URL);
} else {
project.setAvatarUrl(vo.getAvatarUrl());
}
project.setToolsetShowList(Arrays.asList("articleSummary", "interactionUpdate", "highWord"));
// 新建项目时默认数据量50w
project.setDataBalance(500000);
// 非人工项目
project.setManual(false);
project.setShow(true);
project.setStart(false);
project.setCTime(System.currentTimeMillis());
project.setUTime(System.currentTimeMillis());
return project;
}
// private static Map<String, Double> parse2NegativeChannelParmasMap(List<Double> negativeChannel) {
// if (4 != negativeChannel.size()) {
// ExceptionCast.cast(CommonCodeEnum.INVALID_PARAM.message("不友好渠道传参异常"));
......
src/main/java/com/zhiwei/brandkbs2/service/EsSearchService.java
......@@ -44,6 +44,15 @@ public interface EsSearchService {
Pair<SearchHits[], Map<String, Long>> searchOriginHitsAndCount(MarkSearchDTO dto, boolean scroll) throws IOException;
/**
* 搜索非人工项目数据
*
* @param dto 搜索实体类
* @param scroll 滚动查询
* @return 搜索结果
*/
Pair<SearchHits[], Map<String, Long>> searchNonManualHitsAndCount(MarkSearchDTO dto, boolean scroll) throws IOException;
/**
* 查舆情舆情库-平台进量（声量）统计
* @param dto 搜索实体类
* @return
......@@ -58,4 +67,12 @@ public interface EsSearchService {
* @throws IOException
*/
JSONObject countOriginPlatform(MarkSearchDTO dto) throws IOException;
/**
* 非人工项目舆情库方案-平台进量（声量）统计
* @param dto 搜索实体类
* @return
* @throws IOException
*/
JSONObject countNonManualPlatform(MarkSearchDTO dto) throws IOException;
}
src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java
......@@ -9,6 +9,8 @@ import com.zhiwei.brandkbs2.pojo.dto.ExportAppYuqingDTO;
import com.zhiwei.brandkbs2.pojo.dto.ExportSearchWholeDTO;
import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
import com.zhiwei.brandkbs2.pojo.dto.SearchFilterDTO;
import com.zhiwei.brandkbs2.pojo.dto.NonManualProjectPlanDTO;
import com.zhiwei.brandkbs2.pojo.vo.LineVO;
import com.zhiwei.brandkbs2.pojo.vo.PageVO;
import org.apache.commons.lang3.tuple.Pair;
import org.elasticsearch.index.query.BoolQueryBuilder;
......@@ -392,4 +394,121 @@ public interface MarkDataService {
* @return
*/
JSONObject countOriginPlatform(MarkSearchDTO dto);
/**
* 新建非人工项目方案
* @param dto
* @return
*/
ResponseResult addNonManualProjectPlan(NonManualProjectPlanDTO dto);
/**
* 删除非人工项目方案
* @param id
*/
void deleteNonManualProjectPlan(String id);
/**
* 获取非人工项目方案详情
* @param id
*/
NonManualProjectPlanDTO getNonManualProjectPlanInfo(String id);
/**
* 编辑非人工项目方案
* @param dto
* @return
*/
ResponseResult updateNonManualProjectPlan(NonManualProjectPlanDTO dto);
/**
* 非人工项目方案-预估数据量
* @param keyword 关键词
* @param filterWord 去噪词
* @return
*/
Integer getPlanDataPrediction(String keyword, String filterWord);
/**
* 获取方案设置-方案列表
* @return
*/
PageVO<JSONObject> getNonManualProjectPlanList();
/**
* 获取方案设置-方案列表-昨日，今日数据消耗量
* @return
*/
JSONObject getProjectPlanConsume();
/**
* 获取时间范围数据消耗图谱
* @return
*/
List<LineVO> getTimeDataConsumeGraph(String planId, Long startTime, Long endTime);
/**
* 获取总数据消耗图谱
* @return
*/
List<LineVO> getTotalDataConsumeGraph(Long startTime, Long endTime);
/**
* 获取非人工项目舆情数据筛选条件
* @return
*/
JSONObject getNonManualMarkCriteria();
/**
* 获取非人工项目舆情数据列表
* @param markSearchDTO 标注数据搜索传输类
* @return
*/
PageVO<MarkFlowEntity> getNonManualMarkList(MarkSearchDTO markSearchDTO);
/**
* 非人工项目舆情数据列表-互动量更新
* @param dto 标注数据搜索传输类
* @return
*/
ResponseResult nonManualMarkInteractionUpdate(MarkSearchDTO dto);
/**
* 非人工项目工具库-舆情库互动量更新预估
* @param dto 标注数据搜索传输类
* @return
*/
Long nonManualInteractionUpdatePrediction(MarkSearchDTO dto);
/**
* @param markSearchDTO 标注数据搜索传输类
* @return 返回结果
*/
Pair<String, List<ExportAppYuqingDTO>> downloadNonManualMarkList(MarkSearchDTO markSearchDTO);
/**
* 保存/更新gid
* @param gid
*/
void adjustMaxGid(String planId, Long gid);
/**
* 生成聚合列表并返回id
*
* @return 聚合id
*/
String generateNonManualMarkAggreeList(String planId, Long startTime, Long endTime);
/**
* 获取非人工项目聚合结果
* @param dto
* @return
*/
PageVO<MarkFlowEntity> getNonManualMarkAggreeList(MarkSearchDTO dto);
/**
* 非人工项目舆情数据列表-方案列表与各方案数据量统计
* @return
*/
List<JSONObject> getNonManualMarkCountList();
}
src/main/java/com/zhiwei/brandkbs2/service/MarkFlowService.java
......@@ -33,6 +33,14 @@ public interface MarkFlowService {
JSONObject createMarkFlowInfo4Origin(MarkInfoSource markInfoSource);
/**
* 创建标注消息流实体
*
* @param markInfoSource
* @return
*/
JSONObject createNonManualMarkFlowInfo(MarkInfoSource markInfoSource);
/**
* 匹配标注标签值
* @param mes
* @param linkedGroupId
......@@ -72,6 +80,8 @@ public interface MarkFlowService {
private String projectId;
private String contendId;
private String linkedGroupId;
private String planId;
private Long gid;
public MarkInfoSource(JSONObject json, String projectId, String contendId) {
this.json = json;
......@@ -83,6 +93,14 @@ public interface MarkFlowService {
this.json = json;
this.linkedGroupId = linkedGroupId;
}
public MarkInfoSource(JSONObject json, String projectId, String contendId, String planId, Long gid) {
this.json = json;
this.projectId = projectId;
this.contendId = contendId;
this.planId = planId;
this.gid = gid;
}
}
}
src/main/java/com/zhiwei/brandkbs2/service/ProjectService.java
......@@ -149,4 +149,48 @@ public interface ProjectService {
* @return
*/
PageVO<JSONObject> getToolsetBytedanceCustomInfo(int page, int pageSize, String type);
/**
* 添加非人工项目
*
* @param projectVO 项目配置展示对象
*/
ResponseResult addNonManualProject(ProjectVO projectVO);
/**
* 修改非人工项目信息
* @param projectVO 项目配置展示对象
* @return
*/
ResponseResult updateNonManualProject(ProjectVO projectVO);
/**
* 查询非人工项目列表
* @param page 页码
* @param size 页码大小
* @param keyword 关键词
* @return
*/
PageVO<JSONObject> findNonManualProjectList(int page, int size, String sorter, String keyword);
/**
* 非人工项目剩余数据量调整记录列表
*
* @param page 页码
* @param pageSize 页码大小
* @return PageVO
*/
PageVO<JSONObject> findNonManualProjectMaintainList(String projectId, int page, int pageSize);
/**
* 非人工项目剩余数据量调整
* @param currentCount 调整后数值
*/
void adjustNonManualProjectDataBalance(String projectId, Integer currentCount);
/**
* 非人工项目剩余数据调整记录总览
* @return json
*/
JSONObject nonManualProjectOverview(String project);
}
src/main/java/com/zhiwei/brandkbs2/service/impl/EsSearchServiceImpl.java
......@@ -58,15 +58,23 @@ public class EsSearchServiceImpl implements EsSearchService {
}
@Override
public Pair<SearchHits[], Map<String, Long>> searchNonManualHitsAndCount(MarkSearchDTO dto, boolean scroll) throws IOException {
return searchHitsAndCount(dto, this::createNonManualSearchHelper, esClientDao, scroll);
}
@Override
public JSONObject countMarkPlatform(MarkSearchDTO dto) throws IOException {
EsClientDao.SearchHelper helper = createSearchHelper(dto);
return platformCount(helper.getPostFilter(), esClientDao);
return platformCount(dto, this::createSearchHelper, esClientDao);
}
@Override
public JSONObject countOriginPlatform(MarkSearchDTO dto) throws IOException {
EsClientDao.SearchHelper helper = createOriginSearchHelper(dto);
return platformCount(helper.getPostFilter(), hotEsDao);
return platformCount(dto, this::createOriginSearchHelper, hotEsDao);
}
@Override
public JSONObject countNonManualPlatform(MarkSearchDTO dto) throws IOException {
return platformCount(dto, this::createNonManualSearchHelper, esClientDao);
}
private Pair<SearchHits[], Map<String, Long>> searchHitsAndCount(MarkSearchDTO dto, ArticleFunction.CreateSearchHelper createSearchHelper,
......@@ -259,16 +267,93 @@ public class EsSearchServiceImpl implements EsSearchService {
return helper;
}
private EsClientDao.SearchHelper createNonManualSearchHelper(MarkSearchDTO dto) {
EsClientDao.SearchHelper helper = EsClientDao.createSearchHelper();
// projectId、contendId
String projectId = dto.getProjectId();
String planId = dto.getPlanId();
// PostFilter 后置过滤器
BoolQueryBuilder postFilter = EsQueryTools.assembleCacheMapsPlanQuery(projectId, planId, dto.getPoliticsLevel());
// time
postFilter.must(QueryBuilders.rangeQuery(dto.getTimeType()).gte(dto.getStartTime()).lt(dto.getEndTime()));
// platform
if (CollectionUtils.isNotEmpty(dto.getPlatforms())) {
postFilter.must(EsQueryTools.assemblePlatformQuery(Tools.getPlatformByIds(dto.getPlatforms())));
}
// tags
if (CollectionUtils.isNotEmpty(dto.getTags())) {
postFilter.must(EsQueryTools.assembleTagQuery(dto.getTags()));
}
// Query 查询条件
String[] fieldSearch = "仅标题".equals(dto.getSearchField()) ? new String[]{GenericAttribute.ES_IND_TITLE} : new String[]{GenericAttribute.ES_IND_FULL_TEXT};
// keyword
if (StringUtils.isNotEmpty(dto.getKeyword())) {
// Query
postFilter.must(EsQueryTools.assembleNormalKeywordQuery(dto.getKeyword(), fieldSearch));
}
// sourceKeyword
if (StringUtils.isNotEmpty(dto.getSourceKeyword())) {
postFilter.must(EsQueryTools.assembleSourceQuery(dto.getSourceKeyword()));
}
// hostKeyword
if (StringUtils.isNotEmpty(dto.getHostKeyword())) {
postFilter.must(EsQueryTools.assembleFiledKeywordQuery("host", dto.getHostKeyword()));
}
// forward
if (Objects.nonNull(dto.getForward())){
postFilter.must(EsQueryTools.assembleForward2Query(dto.getForward()));
}
// gid 已读 未读
if (Objects.nonNull(dto.getRead())){
if (dto.getRead()){
postFilter.must(QueryBuilders.rangeQuery(GenericAttribute.ES_GID).lte(dto.getGid()));
}else {
postFilter.must(QueryBuilders.rangeQuery(GenericAttribute.ES_GID).gt(dto.getGid()));
}
}
// dataType
if (Objects.nonNull(dto.getDataType())){
postFilter.must(EsQueryTools.assembleC2Query(dto.getDataType()));
}
helper.setPostFilter(postFilter);
// sort
FieldSortBuilder sort = null;
if (null != dto.getSorter()) {
for (Map.Entry<String, Object> entry : dto.getSorter().entrySet()) {
String key = entry.getKey();
if (key.equals("influence")) {
key = "channel_influence";
} else if (key.equals("followers")) {
key = "channel_followers";
}
if (null == entry.getValue() || entry.getValue().toString().contains("desc")) {
sort = SortBuilders.fieldSort(key).order(SortOrder.DESC);
} else {
sort = SortBuilders.fieldSort(key).order(SortOrder.ASC);
}
}
}
helper.setSort(sort);
// from size
if (null != dto.getPageSize()) {
helper.setFrom((dto.getPage() - 1) * dto.getPageSize());
helper.setSize(dto.getPageSize());
}
// HighlightBuilder ???
return helper;
}
/**
* 平台进量统计
*
* @return
*/
private JSONObject platformCount(BoolQueryBuilder postFilter, EsClientDao esClientDao) throws IOException {
private JSONObject platformCount(MarkSearchDTO dto, ArticleFunction.CreateSearchHelper createSearchHelper, EsClientDao esClientDao) throws IOException {
EsClientDao.SearchHelper helper = createSearchHelper.create(dto);
List<JSONObject> list = new ArrayList<>();
//平台计量统计 仅第一页且无平台限制
for (MessagePlatform platform : GlobalPojo.PLATFORMS.stream().filter(messagePlatform -> !"外媒".equals(messagePlatform.getName())).collect(Collectors.toList())) {
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery().must(postFilter).must(EsQueryTools.assemblePlatformQuery(Collections.singletonList(platform)));
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery().must(helper.getPostFilter()).must(EsQueryTools.assemblePlatformQuery(Collections.singletonList(platform)));
Long count = esClientDao.count(esClientDao.getIndexes(), queryBuilder, null);
JSONObject jsonObject = new JSONObject();
jsonObject.put("id", platform.getId());
......@@ -282,5 +367,4 @@ public class EsSearchServiceImpl implements EsSearchService {
res.put("total", total);
return res;
}
}
src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java
......@@ -16,6 +16,7 @@ import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.dao.AggreeResultDao;
import com.zhiwei.brandkbs2.dao.ChannelDao;
import com.zhiwei.brandkbs2.dao.HighlightWordDao;
import com.zhiwei.brandkbs2.dao.NonManualProjectPlanDao;
import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
import com.zhiwei.brandkbs2.easyexcel.dto.ExportYuqingInteractionUpdateDTO;
import com.zhiwei.brandkbs2.enmus.ChannelEmotion;
......@@ -30,14 +31,8 @@ import com.zhiwei.brandkbs2.listener.ApplicationProjectListener;
import com.zhiwei.brandkbs2.model.CommonCodeEnum;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.*;
import com.zhiwei.brandkbs2.pojo.dto.ExportAppYuqingDTO;
import com.zhiwei.brandkbs2.pojo.dto.ExportSearchWholeDTO;
import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
import com.zhiwei.brandkbs2.pojo.dto.SearchFilterDTO;
import com.zhiwei.brandkbs2.pojo.vo.CustomTagVo;
import com.zhiwei.brandkbs2.pojo.vo.LineVO;
import com.zhiwei.brandkbs2.pojo.vo.PageVO;
import com.zhiwei.brandkbs2.pojo.vo.ProjectVO;
import com.zhiwei.brandkbs2.pojo.dto.*;
import com.zhiwei.brandkbs2.pojo.vo.*;
import com.zhiwei.brandkbs2.service.*;
import com.zhiwei.brandkbs2.util.MongoUtil;
import com.zhiwei.brandkbs2.util.RedisUtil;
......@@ -156,6 +151,9 @@ public class MarkDataServiceImpl implements MarkDataService {
@Resource(name = "highlightWordDao")
HighlightWordDao highlightWordDao;
@Resource(name = "nonManualProjectPlanDao")
NonManualProjectPlanDao nonManualProjectPlanDao;
@Resource(name = "toolsetServiceImpl")
private ToolsetService toolsetService;
......@@ -622,6 +620,13 @@ public class MarkDataServiceImpl implements MarkDataService {
projectService.getProjectVOById(projectId).getContendList().stream().filter(contend -> contend.getId().equals(markSearchDTO.getContendId())).findFirst();
contendOptional.ifPresent(contend -> markSearchDTO.setLinkedGroupId(contend.getBrandLinkedGroupId()));
}
// gid
String gid = redisUtil.get(RedisUtil.getNonManualProjectMarkMaxGidKey(UserThreadLocal.getProjectId(), markSearchDTO.getPlanId(), UserThreadLocal.getUserId()));
if (Objects.isNull(gid)) {
markSearchDTO.setGid(-1L);
}else {
markSearchDTO.setGid(Long.parseLong(gid));
}
}
private List<List<JSONObject>> aggregation(SearchHits[] searchHitsList) {
......@@ -974,7 +979,6 @@ public class MarkDataServiceImpl implements MarkDataService {
primaryPlatformsProportion.stream().max(Comparator.comparing(o -> o.getLong("count"))).orElseGet(JSONObject::new);
result.put("priMaxPlatform", primaryMaxPlatformProportion.getString("platform"));
result.put("priMaxPlatformPro", primaryMaxPlatformProportion.getDoubleValue("proportion"));
// 竞品的名字
String brandName = projectService.getProjectByContendId(dto.getProjectId(), dto.getContendId()).getBrandName();
result.put("conName", brandName);
// 获取时间段竞品在每个平台传播量
......@@ -1001,7 +1005,6 @@ public class MarkDataServiceImpl implements MarkDataService {
@Override
public JSONObject getContendSpreadAnalyze(Long startTime, Long endTime, String projectId, String contendId, int hotArticleSize, boolean cache) throws IOException {
ProjectVO project = projectService.getProjectVOById(projectId);
// 竞品的名字
AbstractProject projectByContendId = projectService.getProjectByContendId(projectId, contendId);
String brandName = projectByContendId.getBrandName();
// 缓存
......@@ -1083,7 +1086,6 @@ public class MarkDataServiceImpl implements MarkDataService {
}
// 封装竞品信息结果
// 竞品名
result.put("conName", brandName);
// 获取竞品稿件传播平均值
double avgContendSpreadNormalCount = contendSpread.stream().mapToLong(LineVO::getCount).average().orElse(0);
......@@ -1157,7 +1159,7 @@ public class MarkDataServiceImpl implements MarkDataService {
JSONObject data = getSearchWholeOpinionJson(dto);
JSONObject json = packageExternalDataSourceResult(data);
// 仅当第一页时统计平台进量（声量）
if (1 == dto.getPage()){
if (1 == dto.getPage()) {
json.put("info", getExternalDataSourcePlatformCount(dto));
}
return Pair.of(json, json.getInteger("pageLimit"));
......@@ -1173,10 +1175,11 @@ public class MarkDataServiceImpl implements MarkDataService {
/**
* 获取外部数据源搜索结果json
*
* @param dto
* @return
*/
private JSONObject getSearchWholeOpinionJson(SearchFilterDTO dto){
private JSONObject getSearchWholeOpinionJson(SearchFilterDTO dto) {
// 组装请求参数
JSONObject taskEntity = packageExternalDataSourceParams(dto);
ResponseEntity<JSONObject> responseEntity = restTemplate.postForEntity(addIStarShineKSDataUrl, taskEntity, JSONObject.class);
......@@ -1186,10 +1189,11 @@ public class MarkDataServiceImpl implements MarkDataService {
/**
* 外部数据源组装请求参数
*
* @param dto
* @return
*/
private JSONObject packageExternalDataSourceParams(SearchFilterDTO dto){
private JSONObject packageExternalDataSourceParams(SearchFilterDTO dto) {
JSONObject params = externalTransform(dto);
JSONObject taskEntity = new JSONObject();
taskEntity.put("appId", "6183571e0d710000f6003a12"); // 应用id, 由张志伟提供给使用者
......@@ -1226,6 +1230,7 @@ public class MarkDataServiceImpl implements MarkDataService {
/**
* 舆情库数据结果组装成标准分页返回格式
*
* @param responseEntity
* @return
*/
......@@ -1286,6 +1291,7 @@ public class MarkDataServiceImpl implements MarkDataService {
/**
* 获取外部数据源搜索结果JSONObject
*
* @param taskId
* @return
*/
......@@ -1308,19 +1314,20 @@ public class MarkDataServiceImpl implements MarkDataService {
/**
* 将外部数据源JSONObject组装成标准的返回结果
*
* @param data
* @return
*/
private JSONObject packageExternalDataSourceResult(JSONObject data){
private JSONObject packageExternalDataSourceResult(JSONObject data) {
JSONObject jsonObject = new JSONObject();
List<JSONObject> resultList = new ArrayList<>();
JSONObject task = data.getJSONObject("task");
Integer receiveCount = task.getInteger("receiveCount");
Integer searchCount = task.getInteger("searchCount");
int totalPage;
if (0 == receiveCount || 0 == searchCount){
if (0 == receiveCount || 0 == searchCount) {
totalPage = 0;
}else {
} else {
totalPage = searchCount % receiveCount == 0 ? searchCount / receiveCount : (searchCount / receiveCount + 1);
}
List<JSONObject> listArray = data.getJSONArray("data").toJavaList(JSONObject.class);
......@@ -1348,10 +1355,11 @@ public class MarkDataServiceImpl implements MarkDataService {
/**
* 获取实时采集（外部数据源）平台进量（声量）
*
* @param dto
* @return
*/
private JSONObject getExternalDataSourcePlatformCount(SearchFilterDTO dto){
private JSONObject getExternalDataSourcePlatformCount(SearchFilterDTO dto) {
JSONObject res = new JSONObject();
List<JSONObject> list = new ArrayList<>();
List<String> platforms = Arrays.asList("App", "博客", "平媒", "新闻", "微博", "微信", "论坛", "视频", "短视频");
......@@ -1383,7 +1391,7 @@ public class MarkDataServiceImpl implements MarkDataService {
return ClassB.TypeB.COMPLETE;
}
private String getSourceExternal(JSONObject tJson, ClassB.TypeB typeB){
private String getSourceExternal(JSONObject tJson, ClassB.TypeB typeB) {
String source = null;
switch (typeB) {
case COMPLETE:
......@@ -1608,9 +1616,24 @@ public class MarkDataServiceImpl implements MarkDataService {
}
// 互动量更新剩余可用次数
int interactionBalance = projectService.getProjectById(UserThreadLocal.getProjectId()).getInteractionBalance();
if (interactionBalance - list.size() < 0){
if (interactionBalance - list.size() < 0) {
return ResponseResult.failure("剩余可用次数不足");
}
List<ExportYuqingInteractionUpdateDTO> exportList = markInteractionUpdate(list);
// excel输出到指定路径
String projectName = projectService.getProjectById(UserThreadLocal.getProjectId()).getProjectName();
String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), "舆情库互动量更新结果");
EasyExcelUtil.write(filePath, "sheet1", ExportYuqingInteractionUpdateDTO.class, exportList);
JSONObject res = new JSONObject();
// 记录使用情况
extraService.decreaseInteractionRecord(InteractionUpdateRecord.UsedType.yuqing, list.size());
res.put("filePath", filePath);
res.put("remainingCount", interactionBalance - list.size());
// 记录使用
return ResponseResult.success(res);
}
private List<ExportYuqingInteractionUpdateDTO> markInteractionUpdate(List<BaseMap> list){
Map<String, String> map = list.stream().collect(Collectors.toMap(BaseMap::getUrl, BaseMap::getPlatform, (v1, v2) -> v1));
// 分割map，list里每个map.size为100，目前品见最大链接限制为100，见知微tapd-wiki《互动量更新中间件使用》说明
List<Map<String, String>> mapList = Tools.splitMap(map, 100);
......@@ -1631,20 +1654,9 @@ public class MarkDataServiceImpl implements MarkDataService {
}
Map<Object, JSONObject> resultMap = result.stream().collect(Collectors.toMap(jsonObject -> jsonObject.get("url"), o -> o));
Map<String, BaseMap> baseMap = list.stream().collect(Collectors.toMap(BaseMap::getUrl, o -> o, (v1, v2) -> v2));
List<ExportYuqingInteractionUpdateDTO> exportList = list.stream().map(BaseMap::getUrl)
return list.stream().map(BaseMap::getUrl)
.map(url -> new ExportYuqingInteractionUpdateDTO(id.incrementAndGet(), baseMap, resultMap, url))
.collect(Collectors.toList());
// excel输出到指定路径
String projectName = projectService.getProjectById(UserThreadLocal.getProjectId()).getProjectName();
String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), "舆情库互动量更新结果");
EasyExcelUtil.write(filePath, "sheet1", ExportYuqingInteractionUpdateDTO.class, exportList);
JSONObject res = new JSONObject();
// 记录使用情况
extraService.decreaseInteractionRecord(InteractionUpdateRecord.UsedType.yuqing, list.size());
res.put("filePath", filePath);
res.put("remainingCount", interactionBalance - list.size());
// 记录使用
return ResponseResult.success(res);
}
@Override
......@@ -1660,7 +1672,7 @@ public class MarkDataServiceImpl implements MarkDataService {
postFilter.must(EsQueryTools.assemblePlatformQuery(Tools.getPlatformByIds(dto.getPlatforms())));
}
return esClientDao.count(postFilter);
}catch (Exception e){
} catch (Exception e) {
ExceptionCast.cast(CommonCodeEnum.FAIL, "es查询异常");
}
return null;
......@@ -1671,7 +1683,7 @@ public class MarkDataServiceImpl implements MarkDataService {
defaultMarkSearch(dto);
try {
return esSearchService.countMarkPlatform(dto);
}catch (Exception e){
} catch (Exception e) {
ExceptionCast.cast(CommonCodeEnum.FAIL, "es查询异常", e);
}
return null;
......@@ -1688,12 +1700,470 @@ public class MarkDataServiceImpl implements MarkDataService {
defaultMarkSearch(dto);
try {
return esSearchService.countOriginPlatform(dto);
}catch (Exception e){
} catch (Exception e) {
ExceptionCast.cast(CommonCodeEnum.FAIL, "es查询异常", e);
}
return null;
}
@Override
public ResponseResult addNonManualProjectPlan(NonManualProjectPlanDTO dto) {
if (nonManualProjectPlanDao.count(new Query(Criteria.where("projectId").is(UserThreadLocal.getProjectId()))) >= 5) {
return ResponseResult.failure("超过方案数量上限5");
}
NonManualProjectPlan plan = new NonManualProjectPlan(dto);
nonManualProjectPlanDao.insertOne(plan);
return ResponseResult.success();
}
@Override
public void deleteNonManualProjectPlan(String id) {
nonManualProjectPlanDao.deleteOneById(id);
}
@Override
public NonManualProjectPlanDTO getNonManualProjectPlanInfo(String id) {
return new NonManualProjectPlanDTO(nonManualProjectPlanDao.findOneById(id));
}
@Override
public ResponseResult updateNonManualProjectPlan(NonManualProjectPlanDTO dto) {
NonManualProjectPlan existPlan = nonManualProjectPlanDao.findOneById(dto.getId());
NonManualProjectPlan plan = new NonManualProjectPlan(dto);
plan.setCTime(existPlan.getCTime());
nonManualProjectPlanDao.updateOne(plan);
return ResponseResult.success();
}
@Override
public Integer getPlanDataPrediction(String keyword, String filterWord) {
JSONObject data = getSearchWholeOpinionJson(createSearchFilterDTO(keyword, filterWord));
JSONObject task = data.getJSONObject("task");
return task.getInteger("searchCount");
}
private SearchFilterDTO createSearchFilterDTO(String keyword, String filterWord){
SearchFilterDTO dto = new SearchFilterDTO();
dto.setSearch(keyword);
dto.setFilterWords(filterWord);
// 预估一天
dto.setEndTime(System.currentTimeMillis());
dto.setStartTime(dto.getEndTime() - Constant.ONE_DAY);
dto.setPage(1);
dto.setPageSize(20);
return dto;
}
@Override
public PageVO<JSONObject> getNonManualProjectPlanList() {
Query query = new Query(Criteria.where("projectId").is(UserThreadLocal.getProjectId()));
List<NonManualProjectPlan> planList = nonManualProjectPlanDao.findList(query);
long total = nonManualProjectPlanDao.count(query);
List<JSONObject> collect = planList.stream().map(plan -> {
JSONObject jsonObject = new JSONObject();
jsonObject.put("id", plan.getId());
jsonObject.put("name", plan.getName());
jsonObject.put("keyword", plan.getKeyword());
jsonObject.put("keywordCount", getWordCount(plan.getKeyword()));
jsonObject.put("filterWord", plan.getFilterWord());
jsonObject.put("filterWordCount", getWordCount(plan.getFilterWord()));
jsonObject.put("lasting", plan.getLasting());
jsonObject.put("endTime", plan.getEndTime());
jsonObject.put("isEnd", !plan.getLasting() && Objects.nonNull(plan.getEndTime()) && System.currentTimeMillis() > plan.getEndTime());
return jsonObject;
}).collect(Collectors.toList());
return PageVO.createPageVo(total, 1, 5, collect);
}
/**
* 获取关键词/去噪词数量
* @param word
* @return
*/
private int getWordCount(String word){
if (Objects.isNull(word)){
return 0;
}
int count = 0;
for (String s : word.split("\\|")) {
count += s.split(" ").length;
}
return count;
}
@Override
public JSONObject getProjectPlanConsume() {
Calendar calendar = Calendar.getInstance();
calendar.add(Calendar.DAY_OF_YEAR, -1);
calendar.set(Calendar.HOUR_OF_DAY, 0);
calendar.set(Calendar.SECOND, 0);
calendar.set(Calendar.MINUTE, 0);
calendar.set(Calendar.MILLISECOND, 0);
long yesterdayStartTime = calendar.getTimeInMillis();
calendar.add(Calendar.DAY_OF_YEAR, 1);
long yesterdayEndTime = calendar.getTimeInMillis();
JSONObject jsonObject = new JSONObject();
try {
jsonObject.put("yesterdayCount", getProjectPlanConsume(yesterdayStartTime, yesterdayEndTime));
jsonObject.put("todayCount", getProjectPlanConsume(yesterdayEndTime, System.currentTimeMillis()));
jsonObject.put("dataBalance", projectService.getProjectById(UserThreadLocal.getProjectId()).getDataBalance());
}catch (Exception e){
log.error("获取昨日今日数据消耗量出错", e);
}
return jsonObject;
}
private long getProjectPlanConsume(Long startTime, Long endTime) throws IOException {
String projectId = UserThreadLocal.getProjectId();
BoolQueryBuilder query = projectContendIdQuery(projectId, Constant.PRIMARY_CONTEND_ID);
// stime
query.must(QueryBuilders.rangeQuery("stime").gte(startTime).lt(endTime));
return esClientDao.count(query);
}
@Override
public JSONObject getNonManualMarkCriteria() {
String projectId = UserThreadLocal.getProjectId();
String linkedGroupId = projectService.getProjectVOById(projectId).getBrandLinkedGroupId();
JSONObject result = new JSONObject();
// 搜索时间
result.put("times", Arrays.asList("今天", "24小时", "三天"));
// 平台
result.put("platformList", commonService.getQbjcPlatform("id", "name"));
// 情感标签
result.put("emotionList", commonService.getEmotionTagsWithSort(projectId, linkedGroupId).stream().map(markerTag -> {
JSONObject json = new JSONObject();
json.put("uniqueId", markerTag.getUniqueId());
json.put("name", markerTag.getName());
return json;
}).collect(Collectors.toList()));
// 行政级别
result.put("politicsLevels", ChannelType.channelOption.getJSONArray("politicsLevels"));
// 内容类型
result.put("forward", Arrays.asList("原创", "转发"));
// 数据类型
List<JSONObject> dataType = new ArrayList<>();
List<String> list = Arrays.asList("长文本", "短文本", "问答", "图片", "视频");
for (int i = 0; i < list.size(); i++) {
JSONObject jsonObject = new JSONObject();
jsonObject.put("id", i + 1);
jsonObject.put("name", list.get(i));
dataType.add(jsonObject);
}
result.put("dataType", dataType.stream().filter(jsonObject -> !jsonObject.getString("name").equals("图片")));
// 信息浏览
result.put("read", Arrays.asList("已读", "未读"));
// 自定义标签列表
List<CustomTagVo> customTagList = customTagService.findCustomTagList(1, 1).getList();
if (CollectionUtils.isNotEmpty(customTagList)) {
result.put("customTag", customTagList.get(0).generateSearchSonTagMaps());
}
return result;
}
@Override
public PageVO<MarkFlowEntity> getNonManualMarkList(MarkSearchDTO markSearchDTO) {
try {
//设置默认的时间、页码、排序方式等
defaultMarkSearch(markSearchDTO);
// 搜索es数据
Pair<SearchHits[], Map<String, Long>> hitsAndCounts = esSearchService.searchNonManualHitsAndCount(markSearchDTO, false);
// 总量
long total = hitsAndCounts.getLeft()[0].getTotalHits().value;
// 消息列表
List<MarkFlowEntity> flowEntityList = getNonManualMarkFlowEntity(markSearchDTO, hitsAndCounts.getLeft()[0]);
JSONObject jsonObject = new JSONObject();
// 统计平台进量（声量）
if (1 == markSearchDTO.getPage()) {
jsonObject.put("counts", esSearchService.countNonManualPlatform(markSearchDTO));
}
// 获取最大的gid
jsonObject.put("maxGid", getMaxGid(markSearchDTO));
// 返回分页结果
return PageVO.createPageVo(total, markSearchDTO.getPage(), markSearchDTO.getPageSize(), flowEntityList).setInfo(jsonObject);
} catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.FAIL, "es查询异常");
}
return null;
}
@Override
public List<LineVO> getTimeDataConsumeGraph(String planId, Long startTime, Long endTime) {
try {
return getDataConsumeGraph(planId, startTime, endTime);
}catch (Exception e){
log.error("获取时间数据消耗量图谱出错", e);
return Collections.singletonList(new LineVO(0L, 0L));
}
}
@Override
public List<LineVO> getTotalDataConsumeGraph(Long startTime, Long endTime) {
try {
return getDataConsumeGraph(null, startTime, endTime);
}catch (Exception e){
log.error("获取总数据消耗量图谱出错", e);
return Collections.singletonList(new LineVO(0L, 0L));
}
}
private LineVO getDataConsumeGraph2(String planId, Long startTime, Long endTime) throws IOException {
// project plan query
String projectId = UserThreadLocal.getProjectId();
BoolQueryBuilder query = EsQueryTools.assembleCacheMapsPlanQuery(projectId, planId);
if (Objects.isNull(planId)){
query = projectContendIdQuery(projectId, Constant.PRIMARY_CONTEND_ID);
}
// stime
query.must(QueryBuilders.rangeQuery("stime").gte(startTime).lt(endTime));
return new LineVO(esClientDao.count(query), startTime);
}
/**
* 获取非人工项目时间段内舆情数据图
* @param planId
* @param startTime
* @param endTime
* @return
* @throws IOException
*/
private List<LineVO> getDataConsumeGraph(String planId, Long startTime, Long endTime) throws IOException{
List<LineVO> res = new ArrayList<>();
String[] indexes = esClientDao.getIndexes();
DateHistogramAggregationBuilder daysAggregationBuilder;
// 天级以小时为颗粒度，周、月级以天作为颗粒度
if (endTime - startTime <= Constant.ONE_DAY){
daysAggregationBuilder = AggregationBuilders.dateHistogram("timeAgg").field("stime").calendarInterval(DateHistogramInterval.HOUR);
}else {
daysAggregationBuilder = AggregationBuilders.dateHistogram("timeAgg").field("stime").calendarInterval(DateHistogramInterval.DAY);
}
String projectId = UserThreadLocal.getProjectId();
// project plan query
BoolQueryBuilder query = EsQueryTools.assembleCacheMapsPlanQuery(projectId, planId);
if (Objects.isNull(planId)){
query = projectContendIdQuery(projectId, Constant.PRIMARY_CONTEND_ID);
}
// stime
query.must(QueryBuilders.rangeQuery("stime").gte(startTime).lt(endTime));
// response
SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, query, daysAggregationBuilder, null, null, 0, 0, null);
Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
ParsedDateHistogram teamAgg = (ParsedDateHistogram) aggMap.get("timeAgg");
for (Histogram.Bucket bucket : teamAgg.getBuckets()) {
long time = Long.parseLong(bucket.getKeyAsString());
long count = bucket.getDocCount();
res.add(new LineVO(count, time));
}
return CompleteLine(res, startTime, endTime);
}
/**
* 补全图谱
* @param lines
* @param startTime
* @param endTime
* @return
*/
private List<LineVO> CompleteLine(List<LineVO> lines, Long startTime, Long endTime){
int size = endTime - startTime > 7 * Constant.ONE_DAY ? 30 : (endTime - startTime <= Constant.ONE_DAY ? 24 : 7);
long timeRange = 7 == size || 30 == size ? Constant.ONE_DAY : Constant.ONE_HOUR;
if (lines.size() < size){
// 无数据，按时间段全部补为0
if (CollectionUtils.isEmpty(lines)){
for (int i = 0; i < size; i++) {
lines.add(new LineVO(0L, startTime));
startTime = startTime + timeRange;
if (startTime >= endTime || lines.size() >= size){
break;
}
}
return lines;
}else {
// 往前补0
LineVO startLine = lines.get(0);
Long startLineTime = startLine.getDate();
for (int i = 0; i < size; i++) {
startLineTime = startLineTime - timeRange;
if (startLineTime <= startTime || lines.size() >= size){
break;
}
lines.add(0, new LineVO(0L, startLineTime));
}
// 往后补0
LineVO endLine = lines.get(lines.size() - 1);
Long endLineTime = endLine.getDate();
for (int i = 0; i < size; i++) {
endLineTime = endLineTime + timeRange;
if (endLineTime >= endTime || lines.size() >= size){
break;
}
lines.add(new LineVO(0L, endLineTime));
}
}
}
return lines;
}
@Override
public ResponseResult nonManualMarkInteractionUpdate(MarkSearchDTO dto) {
List<BaseMap> list = new ArrayList<>();
dto.setProjectId(UserThreadLocal.getProjectId());
try {
SearchHits[] searchHits = esSearchService.searchNonManualHitsAndCount(dto, true).getLeft();
for (SearchHits searchHit : searchHits) {
List<BaseMap> baseMaps = Arrays.stream(searchHit.getHits()).map(SearchHit::getSourceAsMap).map(Tools::getBaseFromEsMap).collect(Collectors.toList());
list.addAll(baseMaps);
}
} catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.FAIL, "es查询异常");
}
// 互动量更新剩余可用次数
int interactionBalance = projectService.getProjectById(UserThreadLocal.getProjectId()).getInteractionBalance();
if (interactionBalance - list.size() < 0) {
return ResponseResult.failure("剩余可用次数不足");
}
List<ExportYuqingInteractionUpdateDTO> exportList = markInteractionUpdate(list);
// excel输出到指定路径
String projectName = projectService.getProjectById(UserThreadLocal.getProjectId()).getProjectName();
String filePath = EasyExcelUtil.generateExcelFilePath(brandkbsFilePath, projectName, UserThreadLocal.getNickname(), "舆情库互动量更新结果");
EasyExcelUtil.write(filePath, "sheet1", ExportYuqingInteractionUpdateDTO.class, exportList);
JSONObject res = new JSONObject();
// 记录使用情况
extraService.decreaseInteractionRecord(InteractionUpdateRecord.UsedType.yuqing, list.size());
res.put("filePath", filePath);
res.put("remainingCount", interactionBalance - list.size());
return ResponseResult.success(res);
}
@Override
public Long nonManualInteractionUpdatePrediction(MarkSearchDTO dto) {
try {
String projectId = UserThreadLocal.getProjectId();
String planId = dto.getPlanId();
BoolQueryBuilder postFilter = EsQueryTools.assembleCacheMapsPlanQuery(projectId, planId);
// time
postFilter.must(QueryBuilders.rangeQuery(dto.getTimeType()).gte(dto.getStartTime()).lt(dto.getEndTime()));
// platform
if (CollectionUtils.isNotEmpty(dto.getPlatforms())) {
postFilter.must(EsQueryTools.assemblePlatformQuery(Tools.getPlatformByIds(dto.getPlatforms())));
}
return esClientDao.count(postFilter);
} catch (Exception e) {
ExceptionCast.cast(CommonCodeEnum.FAIL, "es查询异常");
}
return null;
}
@Override
public Pair<String, List<ExportAppYuqingDTO>> downloadNonManualMarkList(MarkSearchDTO markSearchDTO) {
return downloadList(markSearchDTO, esSearchService::searchNonManualHitsAndCount);
}
@Override
public void adjustMaxGid(String planId, Long gid) {
if (Objects.isNull(gid) || -1 == gid){
return;
}
String key = RedisUtil.getNonManualProjectMarkMaxGidKey(UserThreadLocal.getProjectId(), planId, UserThreadLocal.getUserId());
redisUtil.set(key, String.valueOf(gid));
}
@Override
public String generateNonManualMarkAggreeList(String planId, Long startTime, Long endTime) {
return generateAggreeList(UserThreadLocal.getProjectId(), planId, startTime, endTime, esSearchService::searchNonManualHitsAndCount);
}
@Override
public PageVO<MarkFlowEntity> getNonManualMarkAggreeList(MarkSearchDTO dto) {
try {
dto.setGid(-1L);
PageVO<MarkFlowEntity> aggreeList = getAggreeList(dto, markFlowService::createNonManualMarkFlowInfo, false);
JSONObject jsonObject = new JSONObject();
// 统计平台进量（声量）
if (1 == dto.getPage()) {
jsonObject.put("counts", esSearchService.countNonManualPlatform(dto));
}
// 获取最大的gid
jsonObject.put("maxGid", getMaxGid(dto));
aggreeList.setInfo(jsonObject);
return aggreeList;
}catch (Exception e){
log.error("获取非人工项目舆情列表聚合结果出错", e);
return new PageVO<>();
}
}
@Override
public List<JSONObject> getNonManualMarkCountList() {
try {
List<JSONObject> res = new ArrayList<>();
String projectId = UserThreadLocal.getProjectId();
List<NonManualProjectPlan> planList = nonManualProjectPlanDao.findList(new Query(Criteria.where("projectId").is(projectId)));
long startTime = Constant.DF_yyyyMMdd.parse(Constant.DF_yyyyMMdd.format(new Date())).getTime();
long endTime = System.currentTimeMillis();
for (NonManualProjectPlan plan : planList) {
JSONObject jsonObject = new JSONObject();
jsonObject.put("id", plan.getId());
jsonObject.put("name", plan.getName());
BoolQueryBuilder query = EsQueryTools.assembleCacheMapsPlanQuery(projectId, plan.getId());
query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
jsonObject.put("count", esClientDao.count(query));
res.add(jsonObject);
}
return res;
}catch (Exception e){
log.error("获取非人工项目舆情列表数据量统计出错", e);
return Collections.emptyList();
}
}
/**
* 获取最大的gid
* @param dto
* @return
* @throws IOException
*/
private Long getMaxGid(MarkSearchDTO dto) throws IOException {
EsClientDao.SearchHelper helper = EsClientDao.createSearchHelper();
BoolQueryBuilder postFilter = EsQueryTools.assembleCacheMapsPlanQuery(dto.getProjectId(), dto.getPlanId());
helper.setPostFilter(postFilter);
helper.setSort(SortBuilders.fieldSort(GenericAttribute.ES_GID).order(SortOrder.DESC));
helper.setFrom(null);
helper.setSize(1);
SearchResponse response = esClientDao.searchResponse(helper);
SearchHits hits = response.getHits();
if (0 == hits.getTotalHits().value) {
return -1L;
}
return Long.parseLong(hits.getAt(0).getSourceAsMap().get(GenericAttribute.ES_GID) + "");
}
private String generateAggreeList(String projectId, String planId, Long startTime, Long endTime, ArticleFunction.SearchHitsAndCount searchHitsAndCount) {
String uuid = Tools.getUUID();
MarkSearchDTO markSearchDTO = defaultMarkSearch4Aggree(startTime, endTime);
markSearchDTO.setPlanId(planId);
ApplicationProjectListener.getThreadPool().execute(() -> {
String redisKey = RedisUtil.getAggreeCacheKey(uuid, projectId);
try {
redisUtil.setExpire(redisKey, "0");
log.info("taskId:{},聚合任务开始", uuid);
// 搜索es数据
Pair<SearchHits[], Map<String, Long>> hitsAndCounts = searchHitsAndCount.search(markSearchDTO, true);
log.info("taskId:{},聚合前数据:{}条", uuid, hitsAndCounts.getLeft()[0].getTotalHits().value);
// 聚合分组后数据
List<List<JSONObject>> groupList = aggregation(hitsAndCounts.getLeft());
log.info("taskId:{},聚合完成,聚合组:{}个", uuid, groupList.size());
// 入mongo缓存
cacheAggreeResult(groupList, uuid, projectId);
redisUtil.setExpire(redisKey, "100");
} catch (Exception e) {
log.error("generateAggreeList-", e);
redisUtil.remove(redisKey);
}
});
return uuid;
}
private static JSONObject getTitleAndUrl(Map<String, Object> map) {
JSONObject json = new JSONObject();
BaseMap baseMap = Tools.getBaseFromEsMap(map);
......@@ -1996,6 +2466,38 @@ public class MarkDataServiceImpl implements MarkDataService {
return textList;
}
private List<MarkFlowEntity> getNonManualMarkFlowEntity(MarkSearchDTO markSearchDTO, SearchHits searchHits) {
String projectId = markSearchDTO.getProjectId();
String contendId = markSearchDTO.getContendId();
String planId = markSearchDTO.getPlanId();
Long gid = markSearchDTO.getGid();
// 重复消息折叠
if (markSearchDTO.isFold()) {
// 根据旧顺序排序
List<String> sortList = Arrays.stream(searchHits.getHits()).map(hit -> hit.getSourceAsMap().get("id") + "").collect(Collectors.toList());
Map<String, List<Map<String, Object>>> groupMap = Arrays.stream(searchHits.getHits()).map(SearchHit::getSourceAsMap).collect(Collectors.groupingBy(map -> {
BaseMap baseMap = Tools.getBaseFromEsMap(map);
String title = baseMap.getTitleNullOptionalContent();
return Tools.filterSpecialCharacter(title) + baseMap.getTypeB().encode();
}));
Map<String, List<Map<String, Object>>> collect = groupMap.values().stream().map(list -> list.stream().sorted(Comparator.comparingLong(o -> Long.parseLong(o.get("time") + ""))).collect(Collectors.toList())).collect(Collectors.toMap(list -> list.get(0).get("id") + "", list -> list));
return sortList.stream().map(id -> {
List<Map<String, Object>> maps = collect.get(id);
if (null != maps) {
return MarkFlowEntity.getFoldInstance(maps.stream().map(map -> getNonManualMarkFlowEntity(map, projectId, contendId, planId, gid)).collect(Collectors.toList()));
}
return null;
}).filter(Objects::nonNull).collect(Collectors.toList());
}
return Arrays.stream(searchHits.getHits()).map(hit -> getNonManualMarkFlowEntity(hit.getSourceAsMap(), projectId, contendId, planId, gid)).collect(Collectors.toList());
}
private MarkFlowEntity getNonManualMarkFlowEntity(Map<String, Object> map, String projectId, String contendId, String planId, Long gid) {
MarkFlowEntity instance = new MarkFlowEntity(new JSONObject(map));
instance.setInfo(markFlowService.createNonManualMarkFlowInfo(new MarkFlowService.MarkInfoSource(new JSONObject(map), projectId, contendId, planId, gid)));
return instance;
}
private List<MarkFlowEntity> getMarkFlowEntity(MarkSearchDTO markSearchDTO, SearchHits searchHits) {
String projectId = markSearchDTO.getProjectId();
String contendId = markSearchDTO.getContendId();
......@@ -2106,7 +2608,8 @@ public class MarkDataServiceImpl implements MarkDataService {
for (AggreeResult aggreeResult : fatherList) {
List<MarkFlowEntity> markFlowEntities = new ArrayList<>();
MarkFlowEntity instance = new MarkFlowEntity(aggreeResult.getData(), dto.getLinkedGroupId());
instance.setInfo(createMarkFlowInfo.create(new MarkFlowService.MarkInfoSource(aggreeResult.getData(), dto.getProjectId(), dto.getContendId(), dto.getLinkedGroupId())));
instance.setInfo(createMarkFlowInfo.create(new MarkFlowService.MarkInfoSource(aggreeResult.getData(), dto.getProjectId(), dto.getContendId(),
dto.getLinkedGroupId(), dto.getPlanId(), dto.getGid())));
markFlowEntities.add(instance);
// 添加父标题集下的子标题集
resList.add(MarkFlowEntity.getAggreeInstance(markFlowEntities, aggreeResult, sonDetails));
......
src/main/java/com/zhiwei/brandkbs2/service/impl/MarkFlowServiceImpl.java
......@@ -80,6 +80,23 @@ public class MarkFlowServiceImpl implements MarkFlowService {
return resultInfo;
}
@Override
public JSONObject createNonManualMarkFlowInfo(MarkInfoSource markInfoSource) {
JSONObject json = markInfoSource.getJson();
JSONObject resultInfo = createInfoWithEmotion(json);
resultInfo.put("sourceDetails", getSourceDetails(markInfoSource));
String zip = Tools.gzip(JSON.toJSONString(json));
resultInfo.put("zip", zip);
// 每条数据高亮词
resultInfo.put("highLightWord", getHighLightWords(json, markInfoSource.getProjectId(), markInfoSource.getPlanId()));
// 词频
resultInfo.put("keywordCount", getKeywordCount(json, markInfoSource.getProjectId(), markInfoSource.getPlanId()));
// 已读
Long esGid = json.getLong(GenericAttribute.ES_GID);
resultInfo.put("read", markInfoSource.getGid() >= esGid);
return resultInfo;
}
/**
* 消息实体批量处理标注信息
*
......@@ -193,6 +210,55 @@ public class MarkFlowServiceImpl implements MarkFlowService {
return info;
}
private Map<String, Integer> getKeywordCount(JSONObject json, String projectId, String planId){
List<String> keywords = getKeywords(json, projectId, planId);
Map<String, Integer> res = new HashMap<>();
for (String keyword : keywords) {
if (Objects.isNull(keyword)){
continue;
}
String[] manyKeywordAndFrequency = keyword.split(",");
for (String keywordAndFrequency : manyKeywordAndFrequency) {
String[] split = keywordAndFrequency.split(":");
String word = split[0];
Integer frequency = Integer.valueOf(split[1]);
res.put(word, frequency);
}
}
return res;
}
private List<String> getHighLightWords(JSONObject json, String projectId, String planId){
List<String> keywords = getKeywords(json, projectId, planId);
List<String> res = new ArrayList<>();
for (String keyword : keywords) {
if (Objects.isNull(keyword)){
continue;
}
String[] manyKeywordAndFrequency = keyword.split(",");
for (String keywordAndFrequency : manyKeywordAndFrequency) {
String[] split = keywordAndFrequency.split(":");
String words = split[0];
if (Objects.nonNull(words)){
String[] word = words.split(" ");
res.addAll(new ArrayList<>(Arrays.asList(word)));
}
}
}
return res.stream().distinct().collect(Collectors.toList());
}
private List<String> getKeywords(JSONObject json, String projectId, String planId){
List<String> keywords = new ArrayList<>();
List<JSONObject> brandkbsCacheMaps = json.getJSONArray("brandkbs_cache_maps").toJavaList(JSONObject.class);
for (JSONObject brandkbsCacheMap : brandkbsCacheMaps) {
if (Objects.equals(brandkbsCacheMap.getString("plan_key"), Tools.concat(projectId, planId))){
keywords.add(brandkbsCacheMap.getString("keyword"));
}
}
return keywords;
}
private JSONObject getSourceDetails(MarkInfoSource markInfoSource) {
JSONObject sourceDetails = new JSONObject();
JSONObject tJson = markInfoSource.getJson();
......
src/main/java/com/zhiwei/brandkbs2/service/impl/ProjectServiceImpl.java
......@@ -10,10 +10,7 @@ import com.zhiwei.brandkbs2.enmus.response.ProjectCodeEnum;
import com.zhiwei.brandkbs2.exception.ExceptionCast;
import com.zhiwei.brandkbs2.model.CommonCodeEnum;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.AbstractProject;
import com.zhiwei.brandkbs2.pojo.Contend;
import com.zhiwei.brandkbs2.pojo.Project;
import com.zhiwei.brandkbs2.pojo.User;
import com.zhiwei.brandkbs2.pojo.*;
import com.zhiwei.brandkbs2.pojo.vo.PageVO;
import com.zhiwei.brandkbs2.pojo.vo.ProjectVO;
import com.zhiwei.brandkbs2.service.ProjectService;
......@@ -77,6 +74,12 @@ public class ProjectServiceImpl implements ProjectService {
@Resource(name = "BytedanceCustomPlatformWeightDao")
private BytedanceCustomPlatformWeightDao bytedanceCustomPlatformWeightDao;
@Resource(name = "nonManualProjectDataUpdateRecordDao")
private NonManualProjectDataUpdateRecordDao nonManualProjectDataUpdateRecordDao;
@Resource(name = "nonManualProjectPlanDao")
NonManualProjectPlanDao nonManualProjectPlanDao;
@Value("${jwt.key}")
private String jwtKey;
......@@ -88,13 +91,13 @@ public class ProjectServiceImpl implements ProjectService {
@Override
public ResponseResult addProject(ProjectVO projectVO) {
if (!Tools.isEmpty(projectVO.getHitKeywords()) && projectVO.getModuleShowList().contains(2)) {
return ResponseResult.failure("关键字和原始数据配置异常");
}
if (Objects.isNull(projectVO)) {
//抛出非法参数异常
ExceptionCast.cast(CommonCodeEnum.INVALID_PARAM);
}
if (!Tools.isEmpty(projectVO.getHitKeywords()) && projectVO.getModuleShowList().contains(2)) {
return ResponseResult.failure("关键字和原始数据配置异常");
}
Project existsProject = projectDao.findOne("projectName", projectVO.getProjectName());
if (Objects.nonNull(existsProject)) {
//抛出项目名称已存在异常
......@@ -127,6 +130,7 @@ public class ProjectServiceImpl implements ProjectService {
@Override
public PageVO<JSONObject> findProjectList(int page, int size, String keyword) {
Query query = new Query();
query.addCriteria(Criteria.where("isManual").is(true));
projectDao.addKeywordFuzz(query, keyword, "projectName");
long count = projectDao.count(query);
projectDao.addSort(query, "{\"cTime\":\"descend\"}");
......@@ -238,6 +242,12 @@ public class ProjectServiceImpl implements ProjectService {
// });
//删除项目
projectDao.deleteOneById(pid);
// 删除非人工项目维护记录和方案
if (!project.isManual()){
Query query = new Query(Criteria.where("projectId").is(pid));
nonManualProjectPlanDao.deleteOneByQuery(query);
nonManualProjectDataUpdateRecordDao.deleteOneByQuery(query);
}
}
@Override
......@@ -475,6 +485,106 @@ public class ProjectServiceImpl implements ProjectService {
}
}
@Override
public ResponseResult addNonManualProject(ProjectVO projectVO) {
if (Objects.isNull(projectVO)) {
//抛出非法参数异常
ExceptionCast.cast(CommonCodeEnum.INVALID_PARAM);
}
Project existsProject = projectDao.findOne("projectName", projectVO.getProjectName());
if (Objects.nonNull(existsProject)) {
//抛出项目名称已存在异常
return ResponseResult.failure(ProjectCodeEnum.PROJECT_EXISTSNAME_ERROR.message());
}
Project project = ProjectVO.createNonManualProject(projectVO);
projectDao.insertOneWithoutId(project);
return ResponseResult.success();
}
@Override
public ResponseResult updateNonManualProject(ProjectVO projectVO) {
Project existsProject = projectDao.findOneById(projectVO.getId());
if (Objects.isNull(existsProject)) {
//抛出非法参数异常
ExceptionCast.cast(CommonCodeEnum.INVALID_PARAM);
}
Project existsNameProject = projectDao.findOne("projectName", projectVO.getProjectName());
//如果该项目的名字已存在且不是本身
if (Objects.nonNull(existsNameProject) && !projectVO.getId().equals(existsNameProject.getId())) {
//抛出项目名已存在异常
ExceptionCast.cast(ProjectCodeEnum.PROJECT_EXISTSNAME_ERROR);
}
Project project = ProjectVO.createNonManualProject(projectVO);
project.setCTime(existsProject.getCTime());
project.setShow(existsProject.isShow());
project.setDataBalance(existsProject.getDataBalance());
project.setStart(existsProject.isStart());
projectDao.updateOne(project);
return ResponseResult.success();
}
@Override
public PageVO<JSONObject> findNonManualProjectList(int page, int size, String sorter, String keyword) {
Query query = new Query();
query.addCriteria(Criteria.where("isManual").is(false));
projectDao.addKeywordFuzz(query, keyword, "projectName");
long count = projectDao.count(query);
projectDao.addSort(query, sorter);
mongoUtil.start(page, size, query);
List<Project> projectList = projectDao.findList(query);
List<JSONObject> resList = projectList.stream().map(project -> {
JSONObject result = new JSONObject();
result.put("id", project.getId());
result.put("importTime", project.getImportTime());
result.put("cTime", project.getCTime());
result.put("projectName", project.getProjectName());
result.put("brandName", project.getBrandName());
result.put("isStart", project.isStart());
result.put("isShow", project.isShow());
result.put("dataBalance", project.getDataBalance());
return result;
}).collect(Collectors.toList());
MongoUtil.PageHelper<JSONObject> pageHelper = mongoUtil.pageHelper(count, resList);
return PageVO.createPageVo(pageHelper, resList);
}
@Override
public PageVO<JSONObject> findNonManualProjectMaintainList(String projectId, int page, int pageSize) {
Query query = Query.query(Criteria.where("projectId").is(projectId));
nonManualProjectDataUpdateRecordDao.addSort(query, "{\"cTime\":\"descend\"}");
long total = nonManualProjectDataUpdateRecordDao.count(query);
mongoUtil.start(page, pageSize, query);
List<JSONObject> collect = nonManualProjectDataUpdateRecordDao.findList(query).stream().map(record -> {
JSONObject json = new JSONObject();
json.put("lastCount", record.getLastCount());
json.put("currentCount", record.getCurrentCount());
json.put("cTime", record.getCTime().getTime());
json.put("submitter", record.getSubmitter());
return json;
}).collect(Collectors.toList());
return PageVO.createPageVo(total, page, pageSize, collect);
}
@Override
public void adjustNonManualProjectDataBalance(String projectId, Integer currentCount) {
if (null == currentCount) {
return;
}
Update update = Update.update("dataBalance", currentCount);
Project project = projectDao.findOneById(projectId);
projectDao.updateOneByIdWithField(projectId, update);
NonManualProjectDataUpdateRecord record = new NonManualProjectDataUpdateRecord(projectId, project.getDataBalance(), currentCount, UserThreadLocal.getNickname());
nonManualProjectDataUpdateRecordDao.insertOne(record);
}
@Override
public JSONObject nonManualProjectOverview(String projectId) {
JSONObject json = new JSONObject();
json.put("balance", projectDao.findOneById(projectId).getDataBalance());
json.put("trendList", nonManualProjectDataUpdateRecordDao.aggregateDayLastRecord(projectId));
return json;
}
/**
* 获取舆情对应项目的情感标签
* @param brandName
......@@ -512,6 +622,7 @@ public class ProjectServiceImpl implements ProjectService {
json.put("group", project.getBrandLinkedGroup());
json.put("avatarUrl", project.getAvatarUrl());
json.put("permission", getPermissionList(project));
json.put("isManual", project.isManual());
Date expiredTime = roleExpired.get(project.getId());
if (null == expiredTime || expiredTime.after(new Date())) {
json.put("expired", false);
......@@ -549,7 +660,7 @@ public class ProjectServiceImpl implements ProjectService {
private List<Map<String, Object>> getPermissionList(Project project) {
List<Map<String, Object>> permissionList = new ArrayList<>();
Map<String, Object> originPermission = new HashMap<>();
originPermission.put("origin", project.getModuleShowList().contains(2));
originPermission.put("origin", Objects.nonNull(project.getModuleShowList()) && project.getModuleShowList().contains(2));
permissionList.add(originPermission);
return permissionList;
}
......
src/main/java/com/zhiwei/brandkbs2/util/RedisUtil.java
......@@ -78,6 +78,10 @@ public class RedisUtil {
return RedisKeyPrefix.TOOLSET_ARTICLE_SUMMARY_LIMIT + projectId;
}
public static String getNonManualProjectMarkMaxGidKey(String projectId, String planId, String userId){
return RedisKeyPrefix.NON_MANUAL_PROJECT_MARK_MAX_GID + Tools.concat(projectId, planId, userId);
}
public void setExpire(String key, String value, long timeout, TimeUnit unit) {
stringRedisTemplate.opsForValue().set(key, value, timeout, unit);
}
......
src/main/java/com/zhiwei/brandkbs2/util/Tools.java
......@@ -1245,4 +1245,96 @@ public class Tools {
return null;
}
}
/**
* 是否为null
*
* @param @param obj
* @param @return 设定文件
* @return boolean 返回类型
* @Title: isNull
* @Description: 是否为null
*/
public static boolean isNull(Object obj) {
return !nonNull(obj);
}
/**
* 关键词规范 空格且丨或
*
* @param str
* @return String
*/
public static String canonicalKeyword(String str) {
if (isNull(str)) {
return null;
}
// 回车、换行处理
str = str.replaceAll("(\r\n|\r|\n|\n\r)", "|");
// 空格处理
str = checkAndUniteStringBlankOnce(str);
// 前后空格去除
str = str.trim();
// 多余|去除
str = checkAndUniteStringVerticalOnce(str);
// 前后|去除
str = trimSpecial(str, '|');
return str;
}
/**
* 特殊trim
*
* @param str
* @param c
* @return String
*/
public static String trimSpecial(String str, char c) {
char[] chars = str.toCharArray();
int len = chars.length;
int st = 0;
while ((st < len) && (chars[st] == c)) {
st++;
}
while ((st < len) && (chars[len - 1] == c)) {
len--;
}
return (st > 0) && (len < chars.length) ? str.substring(st, len) : str;
}
/**
* 空格处理（多空格-》1，特殊空格去除）
*
* @param str
* @return
*/
public static String checkAndUniteStringBlankOnce(String str) {
//全角空格替换
if (str.split("[\\u3000]+").length > 1) {
str = str.replaceAll("[\\u3000]+", " ");
}
//不间断空格替换
if (str.split("[\\u00A0]+").length > 1) {
str = str.replaceAll("[\\u00A0]+", " ");
}
// XSL中的空格
if (str.split("(\\u0026\\u0023\\u0031\\u0036\\u0030\\u003b)+").length > 1) {
str = str.replaceAll("(\\u0026\\u0023\\u0031\\u0036\\u0030\\u003b)+", " ");
}
return Arrays.stream(str.split("[\\u0020]+")).filter(StringUtils::isNotBlank).collect(Collectors.joining(" "));
}
/**
* 丨处理
*
* @param str
* @return
*/
public static String checkAndUniteStringVerticalOnce(String str) {
//丨替换
if (str.split("[\\u4e28]+").length > 1) {
str = str.replaceAll("[\\u4e28]+", "|");
}
return Arrays.stream(str.split("[\\u007c]+")).filter(StringUtils::isNotBlank).collect(Collectors.joining("|"));
}
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment