Skip to content

B
brandkbs2
Commit f36f6f14 by shenjunjie
Options

Merge branch 'feature' into 'master' 

Feature

See merge request !6
parents 3e110360 54f7db66
Showing with 1407 additions and 66 deletions
src/main/java/com/zhiwei/brandkbs2/common/GenericAttribute.java
......@@ -56,6 +56,9 @@ public class GenericAttribute {
public static final String ES_MARK_CACHE_MAPS = "mark_cache_maps";
public static final String LINKED_GROUP_ID = "linkedGroupId";
// public static final String ES_CACHE_MAP_PROJECT = "brandkbs_cache_maps.projectId.keyword";
// public static final String ES_CACHE_MAP_LINKED_GROUP_ID = "brandkbs_cache_maps.linkedGroupId.keyword";
// public static final String PLATFORM = "platform";
/**
......
src/main/java/com/zhiwei/brandkbs2/common/RedisKeyPrefix.java
......@@ -15,25 +15,28 @@ public class RedisKeyPrefix {
private RedisKeyPrefix() {
}
public static final String MARK_ANALYZE_SUMMARY = "BRANDKBS:MARK:ANALYZE:SUMMARY:";
public static final String ARTICLE_SPREAD = "BRANDKBS:MARK:ANALYZE:SPREAD:";
private static final String SEPARATOR = ":";
/**
* 稿件数据上传缓存key
*/
private static final String EVENT_DATA_UPLOAD_PROGRESS = "brandkbs:event:eventData:progress:";
private static final String EVENT_DATA_UPLOAD_PROGRESS = "BRANDKBS:EVENT:EVENT_DATA:PROGRESS:";
/**
* 舆情事件导入进展
*/
private static final String YUQING_PROGRESS = "brandkbs:event:yuqingImport:progress:";
private static final String YUQING_PROGRESS = "BRANDKBS:EVENT:YUQING_IMPORT:PROGRESS:";
/**
* 舆情事件分析进度
*/
private static final String EVENT_ANALYSIS_PROGRESS = "brandkbs:event:analysis:progress:";
private static final String EVENT_ANALYZE_PROGRESS = "BRANDKBS:EVENT:ANALYZE:PROGRESS:";
public static String eventAnalysisProgress(String eventId) {
return RedisKeyPrefix.generateRedisKey(RedisKeyPrefix.EVENT_ANALYSIS_PROGRESS, UserThreadLocal.getProjectId(), eventId);
return RedisKeyPrefix.generateRedisKey(RedisKeyPrefix.EVENT_ANALYZE_PROGRESS, UserThreadLocal.getProjectId(), eventId);
}
public static String yuqingProgressKey(String linkedGroupId) {
......
src/main/java/com/zhiwei/brandkbs2/config/Constant.java 0 → 100644
package com.zhiwei.brandkbs2.config;
/**
* @ClassName: Constant
* @Description 常量
* @author: sjj
* @date: 2022-07-05 14:20
*/
public class Constant {
public static final String DAY_PATTERN = "yyyy-MM-dd";
/**
* 自定义fid分隔符号
*/
public static final String DEFAULT_SEPARATOR = "_";
}
src/main/java/com/zhiwei/brandkbs2/controller/AppArticleController.java
......@@ -6,24 +6,27 @@ import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
import com.zhiwei.brandkbs2.service.IMarkDataService;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiImplicitParams;
import io.swagger.annotations.ApiOperation;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import javax.annotation.Resource;
/**
* @author sjj
* @ClassName AppArticleController
* @Description 提供前台稿件相关信息展示
* @author sjj
* @date 2022-06-20 09:27
*/
@RestController
@RequestMapping("/app/yuqing")
@Api(tags = "前台稿件展示接口", description = "提供前台稿件相关信息展示")
@Auth(role = RoleEnum.CUSTOMER)
public class AppArticleController extends BaseController{
public class AppArticleController extends BaseController {
@Resource(name = "markDataServiceImpl")
IMarkDataService markDataService;
......@@ -34,6 +37,20 @@ public class AppArticleController extends BaseController{
return ResponseResult.success(markDataService.getYuqingMarkList(markSearchDTO));
}
@ApiOperation("舆情标注数据搜索条件")
@GetMapping("/list/mark/criteria")
public ResponseResult getYuqingMark(@RequestParam(required = false) String linkedGroupId) {
return ResponseResult.success(markDataService.getYuqingMarkCriteria(linkedGroupId));
}
@ApiOperation("舆情标注数据提要")
@ApiImplicitParams({
@ApiImplicitParam(name = "startTime", value = "开始时间", paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long")
})
@GetMapping("/analyze/summary")
public ResponseResult getArticleSummary(@RequestParam(value = "startTime", required = false) Long startTime,
@RequestParam(value = "endTime", required = false) Long endTime) {
return ResponseResult.success(markDataService.getAnalyzeSummary(startTime, endTime, true));
}
}
src/main/java/com/zhiwei/brandkbs2/controller/CommonController.java
package com.zhiwei.brandkbs2.controller;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.auth.Auth;
import com.zhiwei.brandkbs2.enmus.RoleEnum;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.service.ICommonService;
import com.zhiwei.brandkbs2.service.IProjectService;
import com.zhiwei.middleware.mark.pojo.enums.TagField;
import com.zhiwei.middleware.mark.vo.MarkerTag;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiOperation;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpEntity;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.client.RestTemplate;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
/**
......@@ -38,12 +31,6 @@ import java.util.stream.Collectors;
@Auth(role = RoleEnum.COMMON_ADMIN)
public class CommonController extends BaseController {
@Value("${qbjc.platform.url}")
private String qbjcPlatformUrl;
@Autowired
private RestTemplate restTemplate;
@Resource(name = "commonServiceImpl")
ICommonService commonService;
......@@ -63,8 +50,7 @@ public class CommonController extends BaseController {
@GetMapping("/get/platform")
public ResponseResult getPlatform() {
try {
HttpEntity<JSONObject> entity = restTemplate.getForEntity(qbjcPlatformUrl, JSONObject.class);
return ResponseResult.success(Objects.requireNonNull(entity.getBody()).getJSONArray("data").toJavaList(JSONObject.class).stream().map(json -> json.getString("name")).collect(Collectors.toList()));
return ResponseResult.success(commonService.getQbjcPlatformNames());
} catch (Exception e) {
return ResponseResult.failure(e.getMessage());
}
......
src/main/java/com/zhiwei/brandkbs2/controller/admin/CustomTagController.java 0 → 100644
package com.zhiwei.brandkbs2.controller.admin;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.auth.Auth;
import com.zhiwei.brandkbs2.controller.BaseController;
import com.zhiwei.brandkbs2.enmus.RoleEnum;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.Behavior;
import com.zhiwei.brandkbs2.service.IBehaviorService;
import com.zhiwei.brandkbs2.service.ICustomTagService;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiImplicitParams;
import io.swagger.annotations.ApiOperation;
import org.springframework.web.bind.annotation.*;
import springfox.documentation.annotations.ApiIgnore;
import javax.annotation.Resource;
import java.util.List;
import static com.alibaba.fastjson.JSON.parseArray;
/**
* @author sjj
* @version 1.0
* @description 自定义标签管理接口
* @date 2022年3月24日15:17:57
*/
@RestController
@RequestMapping("/admin/custom")
@Api(tags = "自定义标签管理接口", description = "提供自定义标签的增、删、改、查")
@Auth(role = RoleEnum.COMMON_ADMIN)
public class CustomTagController extends BaseController {
@Resource(name = "customTagServiceImpl")
private ICustomTagService customTagService;
@Resource(name = "behaviorServiceImpl")
private IBehaviorService behaviorService;
private static final Behavior.Operation OPERATION = new Behavior.Operation("自定义标签管理", true);
@ApiOperation("查询所有自定义标签")
@ApiImplicitParams({@ApiImplicitParam(name = "page", value = "页码", defaultValue = "1", paramType = "query", dataType = "int"), @ApiImplicitParam(name = "size", value = "每页记录数", defaultValue = "10", paramType = "query", dataType = "int")})
@GetMapping("/list")
public ResponseResult findCustomTagList(@RequestParam(value = "page", defaultValue = "1") int page, @RequestParam(value = "size", defaultValue = "10") int size) {
return ResponseResult.success(customTagService.findCustomTagList(page, size));
}
@ApiOperation("添加自定义标签")
@ApiImplicitParams({@ApiImplicitParam(name = "tagName", value = "标签名", required = true, paramType = "body", dataType = "string"), @ApiImplicitParam(name = "sonTagNames", value = "子标签列表", required = true, paramType = "body", dataType = "list")})
@PostMapping("/add")
public ResponseResult addCustomTag(@ApiIgnore @RequestBody JSONObject json) {
customTagService.addCustomTag(json.getString("tagName"), parseArray(json.getString("sonTagNames"), String.class));
// 添加用户行为
behaviorService.pushBehavior(OPERATION, "添加自定义标签", request);
return ResponseResult.success();
}
@ApiOperation("修改自定义标签")
@ApiImplicitParams({@ApiImplicitParam(name = "tagName", value = "标签名", required = true, paramType = "body", dataType = "string"), @ApiImplicitParam(name = "sonTagNames", value = "子标签列表", required = true, paramType = "body", dataType = "list"), @ApiImplicitParam(name = "tagId", value = "标签id", required = true, paramType = "body", dataType = "string")})
@PutMapping("/update")
public ResponseResult updateCustomTag(@ApiIgnore @RequestBody JSONObject json) {
String tagName = json.getString("tagName");
List<String> sonTagNames = parseArray(json.getString("sonTagNames"), String.class);
String tagId = json.getString("tagId");
customTagService.updateCustomTag(tagName, sonTagNames, tagId);
// 添加用户行为
behaviorService.pushBehavior(OPERATION, "修改自定义标签", request);
return ResponseResult.success();
}
@ApiOperation("删除自定义标签")
@ApiImplicitParam(name = "tagId", value = "标签id", required = true, paramType = "path", dataType = "string")
@DeleteMapping("/delete/{tagId}")
@Auth(role = RoleEnum.SUPER_ADMIN)
public ResponseResult deleteCustomTag(@PathVariable String tagId) {
customTagService.deleteCustomTagByTagId(tagId);
// 添加用户行为
behaviorService.pushBehavior(OPERATION, "删除自定义标签", request);
return ResponseResult.success();
}
}
src/main/java/com/zhiwei/brandkbs2/dao/ICustomTagDao.java 0 → 100644
package com.zhiwei.brandkbs2.dao;
import com.zhiwei.brandkbs2.pojo.CustomTag;
import java.util.List;
/**
* @ClassName: ICustomTagDao
* @Description ICustomTagDao
* @author: sjj
* @date: 2022-07-04 13:43
*/
public interface ICustomTagDao extends IBaseMongoDao<CustomTag> {
/**
* 查询所有自定义标签（按创建时间降序）
*
* @param pid 项目ID
* @return 所有自定义标签
*/
List<CustomTag> findCustomTagList(String pid);
/**
* 通过tagId返回自定义标签
*
* @param tagId 标签ID
* @param projectId 项目ID
* @return List<CustomTag>
*/
List<CustomTag> findCustomTagListByTagId(String tagId, String projectId);
/**
* 查询在用的tag标签量
*
* @return 在用tag标签量
*/
int findCustomTagCountByProjectId(String pid);
/**
* 逻辑删除自定义标签
*
* @param tagIds 标签ID
*/
void deleteCustomTagByTagIds(List<String> tagIds);
/**
* 逻辑删除自定义标签
*
* @param ids 子标签ID
*/
void deleteCustomTagByIds(List<String> ids);
}
src/main/java/com/zhiwei/brandkbs2/dao/impl/CustomTagDaoImpl.java 0 → 100644
package com.zhiwei.brandkbs2.dao.impl;
import com.zhiwei.brandkbs2.dao.ICustomTagDao;
import com.zhiwei.brandkbs2.pojo.CustomTag;
import org.springframework.stereotype.Component;
import java.util.Collections;
import java.util.List;
/**
* @ClassName: CustomTagDaoImpl
* @Description CustomTagDaoImpl
* @author: sjj
* @date: 2022-07-04 13:46
*/
@Component("customTagDao")
public class CustomTagDaoImpl extends BaseMongoDaoImpl<CustomTag> implements ICustomTagDao {
private static final String COLLECTION_NAME = "brandkbs_custom_tag";
public CustomTagDaoImpl() {
super(COLLECTION_NAME);
}
@Override
public List<CustomTag> findCustomTagList(String pid) {
return Collections.emptyList();
}
@Override
public List<CustomTag> findCustomTagListByTagId(String tagId, String projectId) {
return Collections.emptyList();
}
@Override
public int findCustomTagCountByProjectId(String pid) {
return 0;
}
@Override
public void deleteCustomTagByTagIds(List<String> tagIds) {
}
@Override
public void deleteCustomTagByIds(List<String> ids) {
}
}
src/main/java/com/zhiwei/brandkbs2/enmus/EmotionEnum.java
......@@ -12,25 +12,26 @@ public enum EmotionEnum {
/**
* 全部
*/
ALL(-1),
ALL(-1, "全部"),
/**
* 未定义
*/
UNDEFINED(0),
UNDEFINED(0, "未定义"),
/**
* 正面的
*/
POSITIVE(1),
POSITIVE(1, "正面"),
/**
* 中性的
*/
NEUTRAL(2),
NEUTRAL(2, "中性"),
/**
* 负面的
*/
NEGATIVE(3);
NEGATIVE(3, "负面");
private final int state;
private final String name;
public static final String EMOTION_LABEL_KEY = "情感倾向";
public static final String POSITIVE_LABEL = "正面";
......@@ -41,14 +42,19 @@ public enum EmotionEnum {
public static final String SPECIAL_TS_LABEL = "投诉";
public static final String SPECIAL_NEGATIVE_LABEL = "关联负面";
EmotionEnum(int state) {
EmotionEnum(int state, String name) {
this.state = state;
this.name = name;
}
public int getState() {
return state;
}
public String getName() {
return name;
}
/**
* 获取事件数据情感倾向
*
......@@ -92,13 +98,13 @@ public enum EmotionEnum {
int emotion = EmotionEnum.NEUTRAL.getState();
if (Objects.nonNull(value)) {
switch (value) {
case "正面":
case POSITIVE_LABEL:
emotion = EmotionEnum.POSITIVE.getState();
break;
case "负面":
case NEGATIVE_LABEL:
emotion = EmotionEnum.NEGATIVE.getState();
break;
case "中性":
case NEUTRAL_LABEL:
default:
break;
}
......
src/main/java/com/zhiwei/brandkbs2/enmus/ImportantChannelEnum.java 0 → 100644
package com.zhiwei.brandkbs2.enmus;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
/**
* @author lxj
* @version 1.0
* @description 重要渠道枚举类
* @date 2019/9/6 14:08
*/
public enum ImportantChannelEnum {
/**
* 央级
*/
YANGJI("央级"),
/**
* 门户
*/
MENGHU("门户"),
/**
* 新闻
*/
XINWEN("新闻"),
/**
* 地方
*/
DIFANG("地方"),
/**
* 财经
*/
CAIJING("财经"),
/**
* 科技
*/
KEJI("科技"),
/**
* 其他
*/
QITA("其他");
private String state;
ImportantChannelEnum(String state) {
this.state = state;
}
public static List<String> getAllTagExceptSpec() {
return Arrays.stream(ImportantChannelEnum.values())
.map(ImportantChannelEnum::getState)
.filter(tag -> !tag.equals(ImportantChannelEnum.XINWEN.getState()) && !tag.equals(ImportantChannelEnum.DIFANG.getState()))
.collect(Collectors.toList());
}
public String getState() {
return state;
}
}
src/main/java/com/zhiwei/brandkbs2/enmus/response/CustomTagCodeEnum.java 0 → 100644
package com.zhiwei.brandkbs2.enmus.response;
import com.zhiwei.brandkbs2.model.ResultCode;
/**
* @author sjj
* @version 1.0
* @description 自定义标签错误状态码及信息
* @date 2022年3月25日17:43:44
*/
public enum CustomTagCodeEnum implements ResultCode {
/**
* 自定义标签已达上限
*/
CUSTOM_TAG_LIMIT_ERROR(false, 3001, "自定义标签已达上限！", 200),
/**
* 子标签名错误
*/
SON_TAG_NAME_ERROR(false, 3002, "子标签名错误！", 200);
/**
* 操作是否成功
*/
final boolean success;
/**
* 操作代码
*/
final int code;
/**
* 提示信息
*/
final String message;
/**
* 聚合状态码
*/
final int aggCode;
CustomTagCodeEnum(boolean success, int code, String message, int aggCode) {
this.success = success;
this.code = code;
this.message = message;
this.aggCode = aggCode;
}
@Override
public boolean success() {
return success;
}
@Override
public int code() {
return code;
}
@Override
public String message() {
return message;
}
@Override
public int aggCode() {
return aggCode;
}
}
src/main/java/com/zhiwei/brandkbs2/es/EsClientDao.java
......@@ -23,6 +23,7 @@ import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.RangeQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.sort.FieldSortBuilder;
......@@ -44,7 +45,7 @@ import java.util.stream.Collectors;
* @date: 2022-06-10 14:38
*/
@Component("esClientDao")
public class EsClientDao{
public class EsClientDao {
private static final Logger log = LogManager.getLogger(EsClientDao.class);
private static final FastDateFormat DF = FastDateFormat.getInstance("yyyy-MM-dd HH:mm:ss");
private static final String[] CHANNEL_RECORD_FETCH_SOURCE = new String[]{"c5", "foreign", "real_source", "source", "mtime", "time", "brandkbs_cache_maps"};
......@@ -141,9 +142,7 @@ public class EsClientDao{
*/
private List<Map<String, Object>> searchScroll(SearchSourceBuilder searchSourceBuilder) throws IOException {
List<Map<String, Object>> res = new ArrayList<>();
SearchResponse searchResponse = esClient.search(
new SearchRequest(getIndexes()).source(searchSourceBuilder).scroll(TIME_VALUE),
RequestOptions.DEFAULT);
SearchResponse searchResponse = esClient.search(new SearchRequest(getIndexes()).source(searchSourceBuilder).scroll(TIME_VALUE), RequestOptions.DEFAULT);
while (true) {
if (0 == searchResponse.getHits().getHits().length) {
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
......@@ -169,8 +168,7 @@ public class EsClientDao{
*/
private List<Map<String, Object>> searchScroll(QueryBuilder queryBuilder, int size, String[] fetchSource) throws IOException {
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
sourceBuilder.query(queryBuilder).size(size)
.fetchSource(fetchSource, null);
sourceBuilder.query(queryBuilder).size(size).fetchSource(fetchSource, null);
return searchScroll(sourceBuilder);
}
......@@ -185,8 +183,7 @@ public class EsClientDao{
if (null == v) {
v = new ChannelIndex.ChannelRecord();
}
return v.mergeRecord(new ChannelIndex.ChannelRecord((long) result.get(GenericAttribute.ES_TIME), String.valueOf(result.get(
"id"))));
return v.mergeRecord(new ChannelIndex.ChannelRecord((long) result.get(GenericAttribute.ES_TIME), String.valueOf(result.get("id"))));
});
}
}
......@@ -229,8 +226,12 @@ public class EsClientDao{
// return builder;
// }
public SearchHits searchHits(String[] indexes, QueryBuilder postFilter, QueryBuilder query, FieldSortBuilder sort, int from, int size,
HighlightBuilder highlighter) throws IOException {
public SearchHits searchHits(String[] indexes, QueryBuilder postFilter, QueryBuilder query, FieldSortBuilder sort, int from, int size, HighlightBuilder highlighter) throws IOException {
return searchResponse(indexes, postFilter, query, null, sort, from, size, highlighter).getHits();
}
public SearchResponse searchResponse(String[] indexes, QueryBuilder postFilter, QueryBuilder query, AggregationBuilder aggregationBuilder,
FieldSortBuilder sort, int from, int size, HighlightBuilder highlighter) throws IOException {
SearchRequest searchRequest = new SearchRequest();
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchRequest.indices(indexes);
......@@ -240,6 +241,9 @@ public class EsClientDao{
if (!Objects.isNull(query)) {
searchSourceBuilder.query(query);
}
if (!Objects.isNull(aggregationBuilder)) {
searchSourceBuilder.aggregation(aggregationBuilder);
}
if (!Objects.isNull(sort)) {
searchSourceBuilder.sort(sort);
}
......@@ -253,26 +257,21 @@ public class EsClientDao{
searchSourceBuilder.highlighter(highlighter);
}
searchRequest.source(searchSourceBuilder);
SearchResponse response = retryTemplate.execute(
context -> esClient.search(searchRequest, RequestOptions.DEFAULT)
);
return response.getHits();
return retryTemplate.execute(context -> esClient.search(searchRequest, RequestOptions.DEFAULT));
}
public Long count(String[] indexes, QueryBuilder postFilter, QueryBuilder query) throws IOException {
CountRequest countRequest = new CountRequest();
BoolQueryBuilder countQuery = QueryBuilders.boolQuery();
if(!Objects.isNull(query)){
if (!Objects.isNull(query)) {
countQuery.must(query);
}
if(!Objects.isNull(postFilter)){
if (!Objects.isNull(postFilter)) {
countQuery.must(postFilter);
}
countRequest.indices(indexes);
countRequest.query(countQuery);
CountResponse response = retryTemplate.execute(
context -> esClient.count(countRequest, RequestOptions.DEFAULT)
);
CountResponse response = retryTemplate.execute(context -> esClient.count(countRequest, RequestOptions.DEFAULT));
return response.getCount();
}
......
src/main/java/com/zhiwei/brandkbs2/pojo/CustomTag.java 0 → 100644
package com.zhiwei.brandkbs2.pojo;
import io.swagger.annotations.ApiModelProperty;
import lombok.Getter;
import lombok.Setter;
import org.springframework.data.mongodb.core.mapping.Document;
import java.util.Date;
/**
* @author sjj
* @version 1.0
* @description 自定义标签实体类
* @date 2022年3月22日17:44:21
*/
@Setter
@Getter
@Document("brandkbs_custom_tag")
public class CustomTag extends AbstractBaseMongo {
/**
* 标签ID
*/
@ApiModelProperty("标签ID")
private Integer tagId;
/**
* 标签名
*/
@ApiModelProperty("标签名")
private String tagName;
/**
* 子标签名
*/
@ApiModelProperty("子标签名")
private String sonTagName;
/**
* 创建时间
*/
@ApiModelProperty("创建时间")
private Date cTime;
/**
* 修改时间
*/
@ApiModelProperty("修改时间")
private Date uTime;
/**
* 项目ID
*/
@ApiModelProperty("项目ID")
private String projectId;
/**
* 启用开关
*/
@ApiModelProperty("是否下线")
private boolean offline = false;
// /**
// * 将标签数据转换为标签展示信息
// *
// * @param label 标签对象
// * @return 标签展示类对象
// */
// public static LabelVO creatLabelVO(Label label) {
// LabelVO labelVO = new LabelVO();
// labelVO.setId(label.getId());
// labelVO.setLabelName(label.getLabelName());
// labelVO.setType(label.getType());
// labelVO.setCTime(label.getCTime());
// labelVO.setUTime(label.getUTime());
// labelVO.setProjectId(label.getProjectId());
// return labelVO;
// }
}
src/main/java/com/zhiwei/brandkbs2/pojo/vo/CustomTagVo.java 0 → 100644
package com.zhiwei.brandkbs2.pojo.vo;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.ToString;
import java.util.*;
import java.util.stream.Collectors;
/**
* @author sjj
* @version 1.0
* @description 自定义标签数据展示
* @date 2022年3月24日17:11:22
*/
@Data
@ToString
@AllArgsConstructor
public class CustomTagVo {
/**
* 标签名称
*/
private String tagName;
/**
* 标签id
*/
private int tagId;
/**
* 子标签列表
*/
private List<String> sonTagNames;
/**
* 搜索标签列表
*/
private List<Map<String, Object>> searchSonTagMaps;
/**
* 标注标签列表
*/
private List<Map<String, Object>> markSonTagMaps;
/**
* 创建时间
*/
private Date ctime;
public CustomTagVo(String tagName, int tagId, List<String> sonTagNames, List<Map<String, Object>> markSonTagMaps,
Date ctime) {
super();
this.tagName = tagName;
this.tagId = tagId;
this.sonTagNames = sonTagNames;
this.markSonTagMaps = markSonTagMaps;
this.ctime = ctime;
}
public CustomTagVo generateSearchSonTagMaps() {
searchSonTagMaps = new ArrayList<>();
searchSonTagMaps.add(CommonCustomTag.ALL.getEsMap());
searchSonTagMaps.addAll(new ArrayList<>(markSonTagMaps));
searchSonTagMaps.add(CommonCustomTag.UN_TAG.getEsMap());
return this;
}
public enum CommonCustomTag {
ALL(0, "全部"), UN_TAG(-1, "未标注");
private int id;
private String sonTagName;
public int getId() {
return id;
}
public String getSonTagName() {
return sonTagName;
}
public Map<String, Object> getEsMap() {
Map<String, Object> map = new HashMap<>(2);
map.put("id", id);
map.put("sonTagName", sonTagName);
return map;
}
CommonCustomTag(int id, String sonTagName) {
this.id = id;
this.sonTagName = sonTagName;
}
public static List<String> names() {
return Arrays.stream(CommonCustomTag.values()).map(CommonCustomTag::getSonTagName)
.collect(Collectors.toList());
}
}
}
src/main/java/com/zhiwei/brandkbs2/service/ICommonService.java
package com.zhiwei.brandkbs2.service;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.middleware.mark.vo.MarkerTag;
import com.zhiwei.middleware.mark.vo.TagSearch;
......@@ -30,4 +31,16 @@ public interface ICommonService {
* @return markTags
*/
List<MarkerTag> getQbjcTagsByGroupName(String groupName, TagSearch... tagSearches);
/**
* 获取qbjcPlatform名称
* @return 平台名称
*/
List<String> getQbjcPlatformNames();
/**
* 获取qbjcPlatform
* @return 平台
*/
List<JSONObject> getQbjcPlatform(String... includeFields);
}
src/main/java/com/zhiwei/brandkbs2/service/ICustomTagService.java 0 → 100644
package com.zhiwei.brandkbs2.service;
import com.zhiwei.brandkbs2.pojo.vo.CustomTagVo;
import com.zhiwei.brandkbs2.pojo.vo.PageVO;
import java.util.List;
/**
* @author sjj
* @version 1.0
* @description 自定义标签业务接口
* @date 2022年3月24日15:20:16
*/
public interface ICustomTagService {
/**
* 分页查询该项目所有自定义标签数据
*
* @param page 页码
* @param size 大小
* @return 所有标签数据
*/
PageVO<CustomTagVo> findCustomTagList(int page, int size);
/**
* 添加自定义标签
*
* @param tagName 标签名
* @param sonTagNames 子标签列表
*/
void addCustomTag(String tagName, List<String> sonTagNames);
/**
* 修改自定义标签
*
* @param tagName 标签名
* @param sonTagNames 子标签列表
* @param tagId 标签id
*/
void updateCustomTag(String tagName, List<String> sonTagNames, String tagId);
/**
* 删除自定义标签
*
* @param tagId 标签ID
*/
void deleteCustomTagByTagId(String tagId);
/**
* 标注自定义标签
*
* @param id 数据ID
* @param tagId 标签ID
*/
void customTagMark(String id, String tagId);
}
src/main/java/com/zhiwei/brandkbs2/service/IMarkDataService.java
package com.zhiwei.brandkbs2.service;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.pojo.MarkFlowEntity;
import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
import com.zhiwei.brandkbs2.pojo.vo.PageVO;
......@@ -18,4 +19,19 @@ public interface IMarkDataService {
*/
PageVO<MarkFlowEntity> getYuqingMarkList(MarkSearchDTO markSearchDTO);
/**
* 舆情标注数据搜索条件
*
*/
JSONObject getYuqingMarkCriteria(String linkedGroup);
/**
* 舆情标注数据提要信息
*
* @param startTime 开始时间
* @param endTime 结束时间
* @param cache 是否启用缓存
* @return 舆情消息流提要信息
*/
JSONObject getAnalyzeSummary(Long startTime, Long endTime, boolean cache);
}
src/main/java/com/zhiwei/brandkbs2/service/impl/CommonServiceImpl.java
package com.zhiwei.brandkbs2.service.impl;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.service.ICommonService;
import com.zhiwei.brandkbs2.service.IProjectService;
import com.zhiwei.middleware.mark.pojo.enums.TagField;
......@@ -7,14 +8,17 @@ import com.zhiwei.middleware.mark.service.MarkerClient;
import com.zhiwei.middleware.mark.vo.MarkerTag;
import com.zhiwei.middleware.mark.vo.TagSearch;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpEntity;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
/**
* @ClassName: CommonServiceImpl
......@@ -25,12 +29,18 @@ import java.util.Objects;
@Service("commonServiceImpl")
public class CommonServiceImpl implements ICommonService {
@Autowired
private MarkerClient markClient;
@Value("${qbjc.platform.url}")
private String qbjcPlatformUrl;
@Resource(name = "projectServiceImpl")
private IProjectService projectService;
@Autowired
private MarkerClient markClient;
@Autowired
private RestTemplate restTemplate;
@Override
public List<MarkerTag> getQbjcTags(String linkedGroupId, TagSearch... tagSearches) {
Objects.requireNonNull(linkedGroupId);
......@@ -58,4 +68,26 @@ public class CommonServiceImpl implements ICommonService {
return markClient.getTagsByQuery(res.toArray(new TagSearch[0]));
}
@Override
public List<String> getQbjcPlatformNames() {
HttpEntity<JSONObject> entity = restTemplate.getForEntity(qbjcPlatformUrl, JSONObject.class);
return Objects.requireNonNull(entity.getBody()).getJSONArray("data").toJavaList(JSONObject.class).stream().map(json -> json.getString("name"))
.collect(Collectors.toList());
}
@Override
public List<JSONObject> getQbjcPlatform(String... includeFields) {
HttpEntity<JSONObject> entity = restTemplate.getForEntity(qbjcPlatformUrl, JSONObject.class);
return Objects.requireNonNull(entity.getBody()).getJSONArray("data").toJavaList(JSONObject.class).stream().map(json -> {
if (null == includeFields) {
return json;
}
JSONObject res = new JSONObject();
for (String field : includeFields) {
res.put(field, json.get(field));
}
return res;
}).collect(Collectors.toList());
}
}
src/main/java/com/zhiwei/brandkbs2/service/impl/CustomTagServiceImpl.java 0 → 100644
package com.zhiwei.brandkbs2.service.impl;
import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.dao.ICustomTagDao;
import com.zhiwei.brandkbs2.enmus.response.CustomTagCodeEnum;
import com.zhiwei.brandkbs2.es.EsClientDao;
import com.zhiwei.brandkbs2.exception.ExceptionCast;
import com.zhiwei.brandkbs2.model.CommonCodeEnum;
import com.zhiwei.brandkbs2.pojo.CustomTag;
import com.zhiwei.brandkbs2.pojo.vo.CustomTagVo;
import com.zhiwei.brandkbs2.pojo.vo.PageVO;
import com.zhiwei.brandkbs2.service.ICustomTagService;
import org.apache.commons.collections.CollectionUtils;
import org.elasticsearch.action.DocWriteResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;
/**
* @author sjj
* @version 1.0
* @description 标签业务实现
* @date 2022年3月24日18:22:53
*/
@Service
public class CustomTagServiceImpl implements ICustomTagService {
@Resource(name = "customTagDao")
private ICustomTagDao customTagDao;
@Resource(name = "esClient")
private RestHighLevelClient esClient;
@Resource(name = "esClientDao")
private EsClientDao esClientDao;
private final Object mLock = new Object();
public static final String CUSTOM_TAG_MAP = "custom_tag_map";
private static final int CUSTOM_TAG_LIMIT = 1;
@Override
public PageVO<CustomTagVo> findCustomTagList(int page, int size) {
if (page < 1) {
page = 1;
}
if (size < 0 || size > 50) {
size = 10;
}
List<CustomTag> customTagList = customTagDao.findCustomTagList(UserThreadLocal.getProjectId());
Map<Integer, List<CustomTag>> tagIdList = customTagList.stream()
.collect(Collectors.groupingBy(CustomTag::getTagId));
List<CustomTagVo> returnList = new ArrayList<>();
for (Map.Entry<Integer, List<CustomTag>> entry : tagIdList.entrySet()) {
int tagId = entry.getKey();
List<CustomTag> list = entry.getValue();
CustomTag example = list.get(0);
List<Map<String, Object>> sonTagMaps = new ArrayList<>(list.size());
for (CustomTag tag : list) {
Map<String, Object> sonTagMap = new HashMap<>();
sonTagMap.put("sonTagName", tag.getSonTagName());
sonTagMap.put("id", tag.getId());
sonTagMaps.add(sonTagMap);
}
returnList.add(new CustomTagVo(example.getTagName(), tagId,
list.stream().map(CustomTag::getSonTagName).collect(Collectors.toList()), sonTagMaps,
example.getCTime()));
}
return PageVO.createPageVo(customTagList.size(), page, size, returnList);
}
@Override
public void addCustomTag(String tagName, List<String> sonTagNames) {
String projectId = UserThreadLocal.getProjectId();
if (null == tagName || CollectionUtils.isEmpty(sonTagNames)) {
// 抛出非法传参异常
ExceptionCast.cast(CommonCodeEnum.INVALID_PARAM);
}
if (CUSTOM_TAG_LIMIT <= findCustomTagCountByProjectId(projectId)) {
// 标签已达上限
ExceptionCast.cast(CustomTagCodeEnum.CUSTOM_TAG_LIMIT_ERROR);
}
if (!CollectionUtils.intersection(CustomTagVo.CommonCustomTag.names(), sonTagNames).isEmpty()) {
// 不得使用关键字命名
ExceptionCast.cast(CustomTagCodeEnum.SON_TAG_NAME_ERROR);
}
synchronized (mLock) {
Date now = new Date();
sonTagNames.forEach(sonTagName -> insertOne(tagName, sonTagName, now, projectId));
}
}
@Override
public void updateCustomTag(String tagName, List<String> sonTagNames, String tagId) {
String projectId = UserThreadLocal.getProjectId();
if (!CollectionUtils.intersection(CustomTagVo.CommonCustomTag.names(), sonTagNames).isEmpty()) {
// 不得使用关键字命名
ExceptionCast.cast(CustomTagCodeEnum.SON_TAG_NAME_ERROR);
}
List<CustomTag> tagList = customTagDao.findCustomTagListByTagId(tagId, projectId);
// 根据标签ID无法找到对应list
if (tagList.isEmpty()) {
ExceptionCast.cast(CommonCodeEnum.INVALID_PARAM);
return;
}
// 根据标签ID找到对应list
Map<String, String> originMaps = tagList.stream()
.collect(Collectors.toMap(CustomTag::getId, CustomTag::getSonTagName));
String originTagName = tagList.get(0).getTagName();
Date now = new Date();
sonTagNames.forEach(newTagName -> {
// 新旧标签名不一致或新的子标签名
if (!originTagName.equals(tagName) || !originMaps.containsValue(newTagName)) {
insertOne(tagName, newTagName, now, projectId);
}
// 新旧标签名一致,从需要删除的集合中剔除
if (originTagName.equals(tagName) && originMaps.containsValue(newTagName)) {
originMaps.values().removeAll(Collections.singletonList(newTagName));
}
});
if (!originMaps.isEmpty()) {
// 删除旧标签
deleteCustomTagByIds(new ArrayList<>(originMaps.keySet()));
}
}
@Override
public void deleteCustomTagByTagId(String tagId) {
if (null == tagId) {
// 抛出非法传参异常
ExceptionCast.cast(CommonCodeEnum.INVALID_PARAM);
}
// 逻辑删除该标签
List<String> deleteList = new ArrayList<>();
deleteList.add(tagId);
deleteCustomTagByTagIds(deleteList);
}
@Override
public void customTagMark(String id, String customId) {
// System.out.println("自定义标注接口暂时关闭,id:" + id + ",customTagId:" + customId + ",projectId:" + projectId);
try {
String projectId = UserThreadLocal.getProjectId();
// 重置标签
if (null == customId) {
reSetEsCustomTag(id);
return;
}
CustomTag record = new CustomTag();
record.setId(customId);
CustomTag example = customTagDao.findOneById(customId);
if (null == example) {
// 抛出非法传参异常
ExceptionCast.cast(CommonCodeEnum.INVALID_PARAM);
} else {
updateEsCustomTag(id, example);
}
} catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.SERVER_ERROR);
}
}
private void reSetEsCustomTag(String id) throws IOException {
// 依次遍历使用库
for (String index : esClientDao.getIndexes()) {
UpdateRequest request = new UpdateRequest(index, id);
Map<String, Object> update = new HashMap<>(1);
Map<String, Object> inner = new HashMap<>(1);
inner.put("id", CustomTagVo.CommonCustomTag.UN_TAG.getId());
update.put(CUSTOM_TAG_MAP, inner);
request.doc(update);
if (esClient.update(request, RequestOptions.DEFAULT).getResult().equals(DocWriteResponse.Result.UPDATED)) {
break;
}
}
}
private void updateEsCustomTag(String id, CustomTag customTag) throws IOException {
// 依次遍历使用库
for (String index : esClientDao.getIndexes()) {
UpdateRequest request = new UpdateRequest(index, id);
Map<String, Object> update = new HashMap<>(1);
Map<String, Object> inner = new HashMap<>(4);
inner.put("id", customTag.getId());
inner.put("tag_id", customTag.getTagId());
inner.put("tag_name", customTag.getTagName());
inner.put("son_tag_name", customTag.getSonTagName());
update.put(CUSTOM_TAG_MAP, inner);
request.doc(update);
if (esClient.update(request, RequestOptions.DEFAULT).getResult().equals(DocWriteResponse.Result.UPDATED)) {
break;
}
}
}
private void deleteCustomTagByTagIds(List<String> tagIds) {
if (CollectionUtils.isEmpty(tagIds)) {
// 抛出非法传参异常
ExceptionCast.cast(CommonCodeEnum.INVALID_PARAM);
}
customTagDao.deleteCustomTagByTagIds(tagIds);
}
private void deleteCustomTagByIds(List<String> ids) {
if (CollectionUtils.isEmpty(ids)) {
// 抛出非法传参异常
ExceptionCast.cast(CommonCodeEnum.INVALID_PARAM);
}
customTagDao.deleteCustomTagByIds(ids);
}
private int findCustomTagCountByProjectId(String projectId) {
return customTagDao.findCustomTagCountByProjectId(projectId);
}
private void insertOne(String tagName, String sonTagName, Date time, String projectId) {
CustomTag customTag = new CustomTag();
customTag.setTagName(tagName);
customTag.setSonTagName(sonTagName);
customTag.setCTime(time);
customTag.setUTime(time);
customTag.setProjectId(projectId);
customTag.setOffline(false);
customTagDao.insertOne(customTag);
}
}
src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java
package com.zhiwei.brandkbs2.service.impl;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.ImmutableMap;
import com.sun.org.apache.bcel.internal.generic.RETURN;
import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.common.GlobalPojo;
import com.zhiwei.brandkbs2.common.RedisKeyPrefix;
import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.enmus.EmotionEnum;
import com.zhiwei.brandkbs2.enmus.ImportantChannelEnum;
import com.zhiwei.brandkbs2.es.EsClientDao;
import com.zhiwei.brandkbs2.es.EsQueryTools;
import com.zhiwei.brandkbs2.exception.ExceptionCast;
import com.zhiwei.brandkbs2.model.CommonCodeEnum;
import com.zhiwei.brandkbs2.pojo.MarkFlowEntity;
import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
import com.zhiwei.brandkbs2.pojo.vo.CustomTagVo;
import com.zhiwei.brandkbs2.pojo.vo.PageVO;
import com.zhiwei.brandkbs2.pojo.vo.ProjectVO;
import com.zhiwei.brandkbs2.service.ICommonService;
import com.zhiwei.brandkbs2.service.ICustomTagService;
import com.zhiwei.brandkbs2.service.IMarkDataService;
import com.zhiwei.brandkbs2.service.IProjectService;
import com.zhiwei.brandkbs2.util.AggreeUtil;
import com.zhiwei.brandkbs2.util.Tools;
import com.zhiwei.middleware.mark.pojo.enums.TagField;
import com.zhiwei.qbjc.bean.pojo.common.MessagePlatform;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.script.Script;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.BucketOrder;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.bucket.histogram.ParsedDateHistogram;
import org.elasticsearch.search.aggregations.bucket.terms.ParsedLongTerms;
import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.FieldSortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import static com.zhiwei.brandkbs2.config.Constant.DAY_PATTERN;
/**
* @ClassName: MarkDataServiceImpl
* @Description
......@@ -49,6 +83,17 @@ public class MarkDataServiceImpl implements IMarkDataService {
@Resource(name = "esClientDao")
private EsClientDao esClientDao;
@Resource(name = "commonServiceImpl")
private ICommonService commonService;
@Resource(name = "customTagServiceImpl")
private ICustomTagService customTagService;
@Autowired
private StringRedisTemplate stringRedisTemplate;
private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat(Constant.DAY_PATTERN);
@Override
public PageVO<MarkFlowEntity> getYuqingMarkList(MarkSearchDTO markSearchDTO) {
try {
......@@ -75,6 +120,88 @@ public class MarkDataServiceImpl implements IMarkDataService {
return null;
}
@Override
public JSONObject getYuqingMarkCriteria(String linkedGroupId) {
if (null == linkedGroupId) {
linkedGroupId = projectService.getProjectVOById(UserThreadLocal.getProjectId()).getBrandLinkedGroupId();
}
JSONObject result = new JSONObject();
Date endDate = new Date();
// 搜索时间
result.put("times", criteriaTime(endDate));
// 平台
result.put("platformList", commonService.getQbjcPlatform("id", "name"));
// 情感标签
result.put("emotionList", commonService.getQbjcTags(linkedGroupId, TagField.GROUP_NAME.is("情感倾向")).stream().map(markerTag -> {
JSONObject json = new JSONObject();
json.put("uniqueId", markerTag.getUniqueId());
json.put("name", markerTag.getName());
return json;
}).collect(Collectors.toList()));
//重要发声方
result.put("mediaTypeList", ImportantChannelEnum.getAllTagExceptSpec());
// 自定义标签列表
List<CustomTagVo> customTagList = customTagService.findCustomTagList(1, 1).getList();
if (CollectionUtils.isNotEmpty(customTagList)) {
result.put("customTag", customTagList.get(0).generateSearchSonTagMaps());
}
return result;
}
@Override
public JSONObject getAnalyzeSummary(Long startTime, Long endTime, boolean cache) {
try {
String projectId = UserThreadLocal.getProjectId();
String linkedGroupId = projectService.getProjectVOById(projectId).getBrandLinkedGroupId();
if (Objects.isNull(startTime) || Objects.isNull(endTime)) {
endTime = DateUtils.addDays(Tools.truncDate(new Date(), DAY_PATTERN), 1).getTime();
startTime = DateUtils.addMonths(new Date(endTime), -1).getTime();
}
String redisKey = RedisKeyPrefix.MARK_ANALYZE_SUMMARY + Tools.concat(projectId, startTime, endTime);
String resultStr;
// 返回缓存
if (cache && StringUtils.isNotEmpty(resultStr = stringRedisTemplate.opsForValue().get(redisKey))) {
return JSON.parseObject(resultStr);
}
// 返回值
JSONObject result = new JSONObject();
// 添加各情感发文数统计
emotionAnalyzeAdd(startTime, endTime, projectId, linkedGroupId, result);
// 获取标注数据传播趋势
List<JSONObject> markSpread = getMarkSpread(startTime, endTime, projectId, true);
markSpread.stream().max(Comparator.comparing(o -> o.getInteger("posNormalCount"))).ifPresent(e -> result.put("posNormalMaxTime", e.getDate("time")));
// 获取所有平台稿件倾向稿件数量信息
List<JSONObject> platformsCount = getPlatformMarkEmotionCount(startTime, endTime, projectId, linkedGroupId);
platformsCount.stream().max(Comparator.comparing(o -> o.getInteger(EmotionEnum.POSITIVE.toString()))).ifPresent(e -> result.put("posTopPlatform",
e.getString("platform")));
// String posTopArticleTitle = this.getArticleTopTitle(startTime, endTime, EmotionEnum.POSITIVE.getState(), pid, Constant.PRIMARY_CONTENDID, 1).get(0).getKey();
// if (StringUtils.isNotEmpty(posTopArticleTitle)) {
// String posTopArtTitle = this.selectFirstNormalByAggTitleAndTimeAndProjectIdAndContendId
// (posTopArticleTitle, startTimeStr, endTimeStr, pid, Constant.PRIMARY_CONTENDID).getTitle();
// result.put("posTopArtTitle", posTopArtTitle.length() <= 30 ? posTopArtTitle : posTopArtTitle.substring(0, 30));
// }
// result.put("negNormalMaxTime", articleSpread.stream().max(Comparator.comparing(o -> o.getInteger("negNormalCount"))).get().getDate("time"));
// result.put("negTopPlatform", platformsCount.stream().max(Comparator.comparing(o -> o.getInteger(EmotionEnum.NEGATIVE.toString()))).get().getString("platform"));
// String negTopArticleTitle = this.getArticleTopTitle(startTime, endTime, EmotionEnum.NEGATIVE.getState(), pid, Constant.PRIMARY_CONTENDID, 1).get(0).getKey();
// if (StringUtils.isNotEmpty(negTopArticleTitle)) {
// String negTopArtTitle = this.selectFirstNormalByAggTitleAndTimeAndProjectIdAndContendId
// (negTopArticleTitle, startTimeStr, endTimeStr, pid, Constant.PRIMARY_CONTENDID).getTitle();
// result.put("negTopArtTitle", negTopArtTitle.length() <= 30 ? negTopArtTitle : negTopArtTitle.substring(0, 30));
// }
// List<JSONObject> articlePlatformProportion = this.getArticlePlatformProportion(startTime, endTime, pid, true);
// result.put("platformRank", articlePlatformProportion.stream()
// .filter(articlePlatform -> articlePlatform.getLongValue("count") > 0)
// .sorted((o1, o2) -> o2.getLong("count").compareTo(o1.getLong("count")))
// .collect(Collectors.toList()));
// stringRedisTemplate.opsForValue().set(redisKey, JSON.toJSONString(result), 1, TimeUnit.HOURS);
// return result;
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
private void defaultMarkSearch(MarkSearchDTO markSearchDTO) {
// 时间范围默认一个月
if (Objects.isNull(markSearchDTO.getStartTime()) || Objects.isNull(markSearchDTO.getEndTime())) {
......@@ -157,4 +284,290 @@ public class MarkDataServiceImpl implements IMarkDataService {
return AggreeUtil.sortByTimeAsc(kResult, jsonList, uniqueField);
}
private List<JSONObject> criteriaTime(Date endDate) {
List<JSONObject> times = new ArrayList<>();
for (String criteriaTime : Arrays.asList("今天", "24小时", "三天", "七天", "近30天")) {
JSONObject time = new JSONObject();
switch (criteriaTime) {
case "今天":
time.put("name", criteriaTime);
time.put("startTime", Tools.truncDate(endDate, DAY_PATTERN).getTime());
time.put("endTime", endDate.getTime());
times.add(time);
break;
case "24小时":
time.put("name", criteriaTime);
time.put("startTime", DateUtils.addHours(endDate, -24).getTime());
time.put("endTime", endDate.getTime());
times.add(time);
break;
case "三天":
time.put("name", criteriaTime);
time.put("startTime", DateUtils.addDays(Tools.truncDate(endDate, DAY_PATTERN), -2).getTime());
time.put("endTime", endDate.getTime());
times.add(time);
break;
case "七天":
time.put("name", criteriaTime);
time.put("startTime", DateUtils.addDays(Tools.truncDate(endDate, DAY_PATTERN), -6).getTime());
time.put("endTime", endDate.getTime());
times.add(time);
break;
case "近30天":
time.put("name", criteriaTime);
time.put("startTime", DateUtils.addDays(Tools.truncDate(endDate, DAY_PATTERN), -29).getTime());
time.put("endTime", endDate.getTime());
times.add(time);
break;
}
}
return times;
}
private void emotionAnalyzeAdd(long startTime, long endTime, String projectId, String linkedGroupId, JSONObject result) throws IOException {
long lastStartTime = startTime - (endTime - startTime);
List<String> emotionList = Arrays.asList(EmotionEnum.POSITIVE.getName(), EmotionEnum.NEGATIVE.getName(), EmotionEnum.NEUTRAL.getName());
List<JSONObject> curArticleEmotions = getMarkEmotionsCount(startTime, endTime, null, null, projectId, linkedGroupId, emotionList);
int curPosCount = curArticleEmotions.get(0).getIntValue("count");
int curNegCount = curArticleEmotions.get(1).getIntValue("count");
int curNeuCount = curArticleEmotions.get(2).getIntValue("count");
List<JSONObject> lastArticleEmotions = getMarkEmotionsCount(lastStartTime, startTime, null, null, projectId, linkedGroupId, emotionList);
int lastPosCount = lastArticleEmotions.get(0).getIntValue("count");
int lastNegCount = lastArticleEmotions.get(1).getIntValue("count");
int lastNeuCount = lastArticleEmotions.get(2).getIntValue("count");
int curArticleTotal = curPosCount + curNegCount + curNeuCount;
int oldArticleTotal = lastPosCount + lastNegCount + lastNeuCount;
result.put("startTime", startTime);
result.put("endTime", endTime);
result.put("articlesCount", curArticleTotal);
result.put("compare", oldArticleTotal == 0 ? 0d : curArticleTotal / (double) oldArticleTotal);
result.put("posProportion", curPosCount * 1.0 / curArticleTotal);
result.put("negProportion", curNegCount * 1.0 / curArticleTotal);
result.put("posCompare", lastPosCount == 0 ? 0d : (curPosCount - lastPosCount) * 1.0 / lastPosCount);
}
/**
* 获取所有稿件倾向稿件数量信息
*
* @param startTime 开始时间
* @param endTime 结束时间
* @param keyword 关键词
* @param searchField 检索字段
* @param projectId 项目ID
* @param linkedGroupId 关联组id
* @param emotionList 情感倾向集合
* @return 所有稿件倾向稿件数量信息
*/
private List<JSONObject> getMarkEmotionsCount(Long startTime, Long endTime, String keyword, String searchField, String projectId, String linkedGroupId, List<String> emotionList) throws IOException {
// 获取索引
String[] indexes = esClientDao.getIndexes();
// 聚合查询
TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("count").field("mark_cache_maps.name.keyword");
// query
BoolQueryBuilder query = QueryBuilders.boolQuery();
// keyword
if (StringUtils.isNotEmpty(keyword)) {
String[] fieldSearch = "标题".equals(searchField) ? new String[]{GenericAttribute.ES_IND_TITLE}
: new String[]{GenericAttribute.ES_IND_FULL_TEXT};
query.must(EsQueryTools.assembleNormalKeywordQuery(keyword, fieldSearch));
}
// postFilter
BoolQueryBuilder postFilter = QueryBuilders.boolQuery();
postFilter.must(QueryBuilders.termQuery("brandkbs_cache_maps.linkedGroupId.keyword", linkedGroupId))
.must(QueryBuilders.termQuery("brandkbs_cache_maps.projectId.keyword", projectId));
// time range
postFilter.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
List<JSONObject> resultList = new ArrayList<>(emotionList.size());
Map<String, Aggregation> aggMap = esClientDao.searchResponse(indexes, postFilter, query, aggregationBuilder, null, 0, 0, null).getAggregations().asMap();
ParsedLongTerms teamAgg = (ParsedLongTerms) aggMap.get("count");
List<? extends Terms.Bucket> buckets = teamAgg.getBuckets();
for (String emotion : emotionList) {
List<? extends Terms.Bucket> res = buckets.stream().filter(bucket -> bucket.getKeyAsString().equals(emotion)).collect(Collectors.toList());
JSONObject emotionResult = new JSONObject();
emotionResult.put("emotion", emotion);
emotionResult.put("count", CollectionUtils.isEmpty(res) ? 0 : res.get(0).getDocCount());
resultList.add(emotionResult);
}
return resultList;
}
public List<JSONObject> getMarkSpread(Long startTime, Long endTime, String projectId, boolean cache) throws IOException {
ProjectVO project = projectService.getProjectVOById(projectId);
String linkedGroupId = project.getBrandLinkedGroupId();
if (Objects.isNull(startTime) || Objects.isNull(endTime)) {
endTime = DateUtils.addDays(Tools.truncDate(new Date(), Constant.DAY_PATTERN), 1).getTime();
startTime = DateUtils.addMonths(new Date(endTime), -1).getTime();
}
String redisKey = RedisKeyPrefix.ARTICLE_SPREAD + Tools.concat(projectId, startTime, endTime);
String resultStr;
if (cache && StringUtils.isNotEmpty(resultStr = stringRedisTemplate.opsForValue().get(redisKey))) {
return JSON.parseArray(resultStr, JSONObject.class);
}
List<JSONObject> lineList = new ArrayList<>();
List<JSONObject> result = getDayMarkEmotionCount(startTime, endTime, projectId, linkedGroupId);
result.forEach(day -> {
JSONObject line = new JSONObject();
line.put("normalCount", day.getIntValue("total"));
line.put("posNormalCount", day.getIntValue(EmotionEnum.POSITIVE.toString()));
line.put("neuNormalCount", day.getIntValue(EmotionEnum.NEUTRAL.toString()));
line.put("negNormalCount", day.getIntValue(EmotionEnum.NEGATIVE.toString()));
line.put("time", day.getLongValue("date"));
lineList.add(line);
});
stringRedisTemplate.opsForValue().set(redisKey, JSON.toJSONString(lineList), 1, TimeUnit.HOURS);
return lineList;
}
/**
* 获取时间段某情感数据最多的标题
*
* @param startTime 开始时间
* @param endTime 结束时间
* @param emotion 情感倾向名称
* @param projectId 项目id
* @param linkedGroupId 关联组id
* @param size 分页大小
* @return 最热标题
*/
public List<Map.Entry<String, Integer>> getMarkTopTitle(Long startTime, Long endTime, String emotion, String projectId, String linkedGroupId, int size) throws IOException {
// 索引
String[] indexes = esClientDao.getIndexes();
//搜索源构建对象
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
// TODO script使用
Script script = new Script("params._source.ind_full_title");
TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("titles").script(script).order(BucketOrder.count(false)).size(60000);
// postFilter
BoolQueryBuilder postFilter = projectLinkedGroupQuery(projectId, linkedGroupId);
postFilter.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime))
// 过滤微博
.mustNot(QueryBuilders.termQuery("platform_id", "5d02236e6395002a7c380b79"));
if (emotion != null) {
postFilter.must(QueryBuilders.termQuery("mark_cache_maps.name.keyword", emotion));
}
Map<String, Integer> result = new HashMap<>();
SearchResponse searchResponse = esClientDao.searchResponse(indexes, postFilter, null, aggregationBuilder, null, 0, 0, null);
Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
ParsedStringTerms teamAgg = (ParsedStringTerms) aggMap.get("titles");
List<? extends Terms.Bucket> buckets = teamAgg.getBuckets();
buckets.forEach(bucket -> {
String title = bucket.getKeyAsString();
String aggTitle = Tools.filterSpecialCharacter(title);
int num = (int) bucket.getDocCount();
result.merge(aggTitle, num, Integer::sum);
});
// 去掉 “分享一篇文章” 的标题
return result.entrySet().stream().filter(data -> !"分享一篇文章".equals(data.getKey())).sorted(Map.Entry.comparingByValue(Comparator.reverseOrder()))
.limit(size).collect(Collectors.toList());
}
/**
* 获取每日稿件倾向稿件数量信息
*
* @param startTime 开始时间
* @param endTime 结束时间
* @param projectId 项目id
* @param linkedGroupId 关联组id
* @return 所有稿件倾向稿件数量信息
*/
private List<JSONObject> getDayMarkEmotionCount(Long startTime, Long endTime, String projectId, String linkedGroupId) throws IOException {
List<JSONObject> ResultList = new ArrayList<>();
// 索引
String[] indexes = esClientDao.getIndexes();
// ??? TODO 为什么是16h offset
DateHistogramAggregationBuilder daysAggregationBuilder = AggregationBuilders.dateHistogram("dayAgg").field("time")
.calendarInterval(DateHistogramInterval.DAY);
TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("count").field("mark_cache_maps.name.keyword");
// postFilter
BoolQueryBuilder postFilter = QueryBuilders.boolQuery();
postFilter.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime))
.must(QueryBuilders.termQuery("brandkbs_cache_maps.projectId.keyword", projectId))
.must(QueryBuilders.termQuery("brandkbs_cache_maps.linkedGroupId.keyword", linkedGroupId));
// response
SearchResponse searchResponse = esClientDao.searchResponse(indexes, postFilter, null, aggregationBuilder, null, 0, 0, null);
Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
ParsedDateHistogram dayAggTeam = (ParsedDateHistogram) aggMap.get("dayAgg");
List<? extends Histogram.Bucket> buckets = dayAggTeam.getBuckets();
buckets.forEach(bucket -> {
JSONObject result = new JSONObject();
Map<String, Aggregation> map = bucket.getAggregations().asMap();
ParsedLongTerms countTeam = (ParsedLongTerms) map.get("count");
List<? extends Terms.Bucket> list = countTeam.getBuckets();
result.put("date", bucket.getKeyAsString());
result.put("total", bucket.getDocCount());
result.put(EmotionEnum.POSITIVE.getName(), 0);
result.put(EmotionEnum.NEUTRAL.getName(), 0);
result.put(EmotionEnum.NEGATIVE.getName(), 0);
emotionCount(result, list);
ResultList.add(result);
});
return ResultList;
}
/**
* 获取所有平台稿件倾向稿件数量信息
*
* @param startTime 开始时间
* @param endTime 结束时间
* @param projectId 项目ID
* @param linkedGroupId 关联组id
* @return 所有稿件倾向稿件数量信息
*/
private List<JSONObject> getPlatformMarkEmotionCount(Long startTime, Long endTime, String projectId, String linkedGroupId) throws IOException {
List<JSONObject> ResultList = new ArrayList<>();
// 索引
String[] indexes = esClientDao.getIndexes();
// 聚合请求
TermsAggregationBuilder platformAggregationBuilder = AggregationBuilders.terms("platform_count").field("platform_id").order(BucketOrder.count(false));
TermsAggregationBuilder emotionAggregationBuilder = AggregationBuilders.terms("emotion_count").field("mark_cache_maps.name.keyword");
// postFilter
BoolQueryBuilder postFilter = QueryBuilders.boolQuery();
postFilter.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime))
.must(QueryBuilders.termQuery("brandkbs_cache_maps.projectId.keyword", projectId))
.must(QueryBuilders.termQuery("brandkbs_cache_maps.linkedGroupId.keyword", linkedGroupId));
SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, postFilter,
platformAggregationBuilder.subAggregation(emotionAggregationBuilder), null, 0, 0, null);
Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
ParsedStringTerms teamAgg = (ParsedStringTerms) aggMap.get("platform_count");
List<? extends Terms.Bucket> buckets = teamAgg.getBuckets();
buckets.forEach(bucket -> {
JSONObject result = new JSONObject();
Map<String, Aggregation> map = bucket.getAggregations().asMap();
ParsedLongTerms countTeam = (ParsedLongTerms) map.get("emotion_count");
List<? extends Terms.Bucket> list = countTeam.getBuckets();
result.put("platform", bucket.getKeyAsString());
result.put("total", bucket.getDocCount());
result.put(EmotionEnum.POSITIVE.toString(), 0);
result.put(EmotionEnum.NEUTRAL.toString(), 0);
result.put(EmotionEnum.NEGATIVE.toString(), 0);
emotionCount(result, list);
ResultList.add(result);
});
return ResultList;
}
private void emotionCount(JSONObject result, List<? extends Terms.Bucket> list) {
list.forEach(data -> {
String emotion = data.getKeyAsString();
if (EmotionEnum.POSITIVE.getName().equals(emotion)) {
result.put(EmotionEnum.POSITIVE.getName(), data.getDocCount());
} else if (EmotionEnum.NEUTRAL.getName().equals(emotion)) {
result.put(EmotionEnum.NEUTRAL.getName(), data.getDocCount());
} else if (EmotionEnum.NEGATIVE.getName().equals(emotion)) {
result.put(EmotionEnum.NEGATIVE.getName(), data.getDocCount());
}
});
}
private BoolQueryBuilder projectLinkedGroupQuery(String projectId, String linkedGroupId) {
return QueryBuilders.boolQuery().must(QueryBuilders.termQuery("brandkbs_cache_maps.projectId.keyword", projectId))
.must(QueryBuilders.termQuery("brandkbs_cache_maps.linkedGroupId.keyword", linkedGroupId));
}
}
\ No newline at end of file
src/main/java/com/zhiwei/brandkbs2/util/Tools.java
......@@ -10,6 +10,7 @@ import com.zhiwei.base.entity.subclass.QAText;
import com.zhiwei.base.entity.subclass.Video;
import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.common.GlobalPojo;
import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.easyexcel.dto.UploadKeywordDTO;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.BaseMap;
......@@ -418,6 +419,60 @@ public class Tools {
return false;
}
/**
* 日期取整，去掉时分秒
*
* @param date
* @return Date
* @Title: truncDate
*/
public static Date truncDate(Date date, String pattern) {
Date result = null;
if (date != null) {
Calendar c = Calendar.getInstance();
c.setTime(date);
if (!pattern.contains("yyyy")) {
c.set(Calendar.YEAR, 1970);
}
if (!pattern.contains("MM")) {
c.set(Calendar.MONTH, 0);
}
if (!pattern.contains("dd")) {
c.set(Calendar.DAY_OF_MONTH, 1);
}
if (!pattern.contains("HH")) {
c.set(Calendar.HOUR_OF_DAY, 0);
}
if (!pattern.contains("mm")) {
c.set(Calendar.MINUTE, 0);
}
if (!pattern.contains("ss")) {
c.set(Calendar.SECOND, 0);
}
if (!pattern.contains("SSS")) {
c.set(Calendar.MILLISECOND, 0);
}
result = c.getTime();
}
return result;
}
/**
* 拼接字符串,以_隔开
*
* @param objects 可变参数
* @return 拼接后的字符串
*/
public static String concat(Object... objects) {
StringBuilder sb = new StringBuilder();
for (Object obj : objects) {
sb.append(obj).append(Constant.DEFAULT_SEPARATOR);
}
String resultStr = sb.toString();
return resultStr.substring(0, resultStr.length() - 1);
}
// public static String getEmotionByMtag(String mtag, String group) {
// if (StringUtils.isEmpty(mtag)) {
// return null;
......
src/main/resources/application-dev.properties
......@@ -29,7 +29,7 @@ secondary.uri=mongodb://qbjcuser:qbjc1q2w3e4r@192.168.0.150:27017,192.168.0.151:
es.clusterNodes=192.168.0.130:9200
es.clusterName=zhiweidata-new-es
es.httpClusterNodes=192.168.0.130:9200:qbjc-back:yuqing.zhiweidata.com,192.168.0.51:9400:elastic:qWxZRW42OHkuOhmF5AXX
es.username=brandkbs2
es.username=brandkbs2_test
es.password=3vh65l$i6qQA
es.index.test = true
......
src/main/resources/application-local.properties
......@@ -23,15 +23,16 @@ mongo.autoConnectRetry=true
mongo.socketKeepAlive=true
mongo.socketTimeout=120000
mongo.slaveOk=true
primary.uri=mongodb://rsync:rsync1q2w3e4r@115.236.59.88:30001/brandkbs2?authSource=admin
#primary.uri=mongodb://rsync:rsync1q2w3e4r@115.236.59.88:30001/brandkbs2?authSource=admin
primary.uri=mongodb://brandkbs2:3vh65l$i6qQA@202.107.192.94:17152/brandkbs2?authSource=admin
secondary.uri=mongodb://qbjcuser:qbjc1q2w3e4r@202.107.192.94:17152/qbjc?authSource=admin
#es
es.clusterNodes=127.0.0.1:9200
es.clusterName=my-application
es.httpClusterNodes=127.0.0.1:9300
es.username=
es.password=
es.clusterNodes=202.107.192.94:1443
es.clusterName=zhiweidata-new-es
es.httpClusterNodes=202.107.192.94:1443:brandkbs2:3vh65l$i6qQA
es.username=brandkbs2_test
es.password=3vh65l$i6qQA
es.index.test = true
#channel-index
......
src/main/resources/application-prod.properties
......@@ -23,14 +23,14 @@ mongo.autoConnectRetry=true
mongo.socketKeepAlive=true
mongo.socketTimeout=120000
mongo.slaveOk=true
primary.uri=mongodb://brandkbs2:3vh65l$i6qQA@192.168.0.150:27017,192.168.0.151:27017,192.168.0.152:27017/qbjc?authSource=admin
secondary.uri=mongodb://qbjcuser:qbjc1q2w3e4r@192.168.0.150:27017,192.168.0.151:27017,192.168.0.152:27017/qbjc?authSource=admin
primary.uri=mongodb://brandkbs2:3vh65l$i6qQA@192.168.0.150:27017,192.168.0.151:27017,192.168.0.152:27017/brandkbs2?authSource=admin
secondary.uri=mongodb://qbjcuser:asSADf5ffs@192.168.0.150:27017,192.168.0.151:27017,192.168.0.152:27017/qbjc?authSource=admin
#es
es.clusterNodes=192.168.0.130:9200
es.clusterName=zhiweidata-new-es
es.httpClusterNodes=192.168.0.130:9200:qbjc-back:yuqing.zhiweidata.com,192.168.0.51:9400:elastic:qWxZRW42OHkuOhmF5AXX
es.username=brandkbs2
es.username=brandkbs2_test
es.password=3vh65l$i6qQA
es.index.test = true
......
src/main/resources/application.properties
server.port=8888
spring.profiles.active=local
spring.profiles.active=pro
spring.flyway.encoding=UTF-8
server.servlet.context-path=/brandkbs2
\ No newline at end of file
server.servlet.context-path=/brandkbs
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment