Skip to content

B
brandkbs2
Commit b68d9b1f by shenjunjie
Options

舆情库聚合接口完成

parent 9683fdbc
Showing with 582 additions and 90 deletions
src/main/java/com/zhiwei/brandkbs2/common/GenericAttribute.java
......@@ -29,6 +29,13 @@ public class GenericAttribute {
*/
public static final String ES_FOLLOWERS_NUM = "followers_num";
/**
* es media_type
*/
public static final String ES_MEDIA_TYPE = "media_type";
// ss platform_id
// TODO 待修改为 platform_id
public static final String ES_PLATFORM_ID= "platformId";
/**
* es rootSource
**/
public static final String ES_ROOT_SOURCE = "root_source";
......
src/main/java/com/zhiwei/brandkbs2/common/RedisKeyPrefix.java
......@@ -25,6 +25,7 @@ public class RedisKeyPrefix {
public static final String MARK_HIGH_WORD = "BRANDKBS:MARK:HIGH_WORD:";
public static final String REDIS_SYSTEM_COPY = "BRANDKBS:SYSTEM:COPY:";
public static final String AGGREE_RESULT_CACHE = "BRANDKBS:AGGREE_CACHE:";
/**
* 项目简报报相关缓存KEY
......
src/main/java/com/zhiwei/brandkbs2/controller/admin/EventController.java
......@@ -47,7 +47,7 @@ import java.util.Set;
public class EventController extends BaseController {
@Resource(name = "eventServiceImpl")
private EventService EventService;
private EventService eventService;
@Resource(name = "behaviorServiceImpl")
private BehaviorService behaviorService;
......@@ -64,13 +64,13 @@ public class EventController extends BaseController {
@ApiImplicitParam(name = "linkedGroupId", value = "关联项目组id", paramType = "query", dataType = "string")
@GetMapping("/yq/tag")
public ResponseResult searchCriteria(@RequestParam(value = "linkedGroupId") String linkedGroupId) {
return ResponseResult.success(EventService.findEventTagListAll(linkedGroupId));
return ResponseResult.success(eventService.findEventTagListAll(linkedGroupId));
}
@ApiOperation("分页查询舆情事件列表")
@PostMapping("/yq/list")
public ResponseResult findYqEventList(@RequestBody YqEventSearchVO yqEventSearchVO) {
PageVO<JSONObject> yqEventList = EventService.findYqEventList(yqEventSearchVO);
PageVO<JSONObject> yqEventList = eventService.findYqEventList(yqEventSearchVO);
return ResponseResult.success(yqEventList);
}
......@@ -87,27 +87,27 @@ public class EventController extends BaseController {
@RequestParam(value = "linkedGroupId") String linkedGroupId,
@RequestParam(value = "keyword", defaultValue = "") String keyword,
@RequestParam(value = "sorter", defaultValue = "{\"startTime\":\"descend\"}") String sorter) {
return ResponseResult.success(EventService.findEventList(page, size, linkedGroupId, keyword, sorter));
return ResponseResult.success(eventService.findEventList(page, size, linkedGroupId, keyword, sorter));
}
@ApiOperation("获取单个事件信息")
@ApiImplicitParam(name = "eventId", value = "事件ID", required = true, paramType = "path", dataType = "string")
@GetMapping("/get/{eventId}")
public ResponseResult getEvent(@PathVariable("eventId") String eventId) {
return ResponseResult.success(EventService.getEventJSONByEventId(eventId));
return ResponseResult.success(eventService.getEventJSONByEventId(eventId));
}
@ApiOperation("获取事件数据列表")
@ApiImplicitParams({@ApiImplicitParam(name = "page", value = "页码", required = false, defaultValue = "1", paramType = "query", dataType = "int"), @ApiImplicitParam(name = "size", value = "每页记录数", required = false, defaultValue = "10", paramType = "query", dataType = "int"), @ApiImplicitParam(name = "eventId", value = "事件ID", required = true, paramType = "query", dataType = "string")})
@GetMapping("/data/get")
public ResponseResult getEventDatas(@RequestParam(value = "page", defaultValue = "1") int page, @RequestParam(value = "size", defaultValue = "10") int size, @RequestParam(value = "eventId") String eventId) {
return ResponseResult.success(EventService.getEventDatas(page, size, eventId));
return ResponseResult.success(eventService.getEventDatas(page, size, eventId));
}
@ApiOperation("修改事件信息")
@PutMapping("/update")
public ResponseResult updateEvent(@RequestBody EventVO eventVO) {
EventService.updateEvent(eventVO);
eventService.updateEvent(eventVO);
behaviorService.pushBehavior(OPERATION, "修改事件信息:" + eventVO.getId(), request);
return ResponseResult.success();
}
......@@ -115,7 +115,7 @@ public class EventController extends BaseController {
@ApiOperation("修改事件首发信息")
@PutMapping("/updateFirst")
public ResponseResult updateEventFirst(@RequestBody EventDataDTO eventDataDTO) {
EventService.updateEventFirst(eventDataDTO);
eventService.updateEventFirst(eventDataDTO);
behaviorService.pushBehavior(OPERATION, "修改事件首发信息:" + eventDataDTO.getEventId(), request);
return ResponseResult.success();
}
......@@ -124,7 +124,7 @@ public class EventController extends BaseController {
@ApiImplicitParam(name = "eventIds", value = "事件ID集合", paramType = "body", dataType = "list")
@PostMapping("/analyze")
public ResponseResult analysisEvents(@RequestBody JSONObject info) {
EventService.analysisEvents(info.getJSONArray("eventIds").toJavaList(String.class));
eventService.analysisEvents(info.getJSONArray("eventIds").toJavaList(String.class));
behaviorService.pushBehavior(OPERATION, "批量更新事件", request);
return ResponseResult.success();
}
......@@ -133,7 +133,7 @@ public class EventController extends BaseController {
@ApiImplicitParam(name = "eventId", value = "事件ID", required = true, paramType = "path", dataType = "string")
@PutMapping("/end/{eventId}")
public ResponseResult endEvent(@PathVariable("eventId") String eventId) {
EventService.endEvent(eventId);
eventService.endEvent(eventId);
behaviorService.pushBehavior(OPERATION, "结束事件:" + eventId, request);
return ResponseResult.success();
}
......@@ -142,7 +142,7 @@ public class EventController extends BaseController {
@ApiImplicitParam(name = "eventId", value = "事件ID", required = true, paramType = "path", dataType = "string")
@DeleteMapping("/delete/{eventId}")
public ResponseResult deleteEvent(@PathVariable("eventId") String eventId) {
String title = EventService.deleteEvent(eventId);
String title = eventService.deleteEvent(eventId);
behaviorService.pushBehavior(OPERATION, "删除事件:" + title, request);
return ResponseResult.success();
}
......@@ -153,7 +153,7 @@ public class EventController extends BaseController {
public ResponseResult deleteEventData(@RequestBody JSONObject info) {
String id = info.getString("id");
String eventId = info.getString("eventId");
String title = EventService.deleteEventData(id, eventId);
String title = eventService.deleteEventData(id, eventId);
behaviorService.pushBehavior(OPERATION, "删除事件单条数据,eventId:" + eventId + ",数据id:" + id + ",事件标题:" + title, request);
return ResponseResult.success();
}
......@@ -172,14 +172,14 @@ public class EventController extends BaseController {
@ApiOperation("事件上传进度获取")
@GetMapping("data/upload/progress/{ticket}")
public ResponseResult getEventDataUploadProgress(@PathVariable String ticket) {
return ResponseResult.success(EventService.getEventDataUploadProgress(ticket));
return ResponseResult.success(eventService.getEventDataUploadProgress(ticket));
}
@ApiOperation("事件数据上传进度获取")
@ApiImplicitParams(@ApiImplicitParam(name = "eventIds", value = "事件id列表", paramType = "body", dataType = "list"))
@PostMapping("analyze/progress")
public ResponseResult getEventAnalyzeProgress(@RequestBody JSONObject info) {
return ResponseResult.success(EventService.getEventAnalyzeProgress(info.getJSONArray("eventIds").toJavaList(String.class)));
return ResponseResult.success(eventService.getEventAnalyzeProgress(info.getJSONArray("eventIds").toJavaList(String.class)));
}
@ApiOperation("事件上传模板下载")
......@@ -202,7 +202,7 @@ public class EventController extends BaseController {
@ApiImplicitParam(name = "linkedGroupId", value = "关联项目组id", required = true, paramType = "query", dataType = "string")
@GetMapping("/download")
public ResponseResult downloadEvents(@RequestParam("linkedGroupId") String linkedGroupId) {
Pair<String, List<ExportEventDTO>> result = EventService.downloadEvents(linkedGroupId);
Pair<String, List<ExportEventDTO>> result = eventService.downloadEvents(linkedGroupId);
String fileName = result.getLeft();
EasyExcelUtil.download(fileName + "_事件列表数据", fileName, ExportEventDTO.class, result.getRight(), response);
return ResponseResult.success();
......@@ -212,7 +212,7 @@ public class EventController extends BaseController {
@ApiImplicitParam(name = "eventId", value = "事件ID", required = true, paramType = "query", dataType = "string")
@GetMapping("/data/download")
public ResponseResult downloadEventDatas(@RequestParam("eventId") String eventId) {
Pair<String, List<ExportEventDataDTO>> result = EventService.downloadEventDatas(eventId);
Pair<String, List<ExportEventDataDTO>> result = eventService.downloadEventDatas(eventId);
String title = result.getLeft();
EasyExcelUtil.download(title, title, ExportEventDataDTO.class, result.getRight(), response);
return ResponseResult.success();
......@@ -224,7 +224,7 @@ public class EventController extends BaseController {
public ResponseResult addEventsByYq(@RequestBody JSONObject info) {
String linkedGroupId = info.getString("linkedGroupId");
List<YqEventDTO> yqEventList = info.getJSONArray("list").toJavaList(YqEventDTO.class);
EventService.addYqEvents(linkedGroupId, yqEventList);
eventService.addYqEvents(linkedGroupId, yqEventList);
behaviorService.pushBehavior(OPERATION, "批量导入舆情事件数据", request);
return ResponseResult.success();
}
......@@ -232,7 +232,7 @@ public class EventController extends BaseController {
@ApiOperation("按搜索条件全部导入舆情事件数据")
@PutMapping("/upload/yqAll")
public ResponseResult addEventAllByYq(@RequestBody YqEventSearchVO yqEventSearchVO) {
EventService.addYqEventAll(yqEventSearchVO);
eventService.addYqEventAll(yqEventSearchVO);
behaviorService.pushBehavior(OPERATION, "全部导入舆情事件数据", request);
return ResponseResult.success();
}
......@@ -242,7 +242,7 @@ public class EventController extends BaseController {
@PostMapping(value = "/upload/file", headers = "content-type=multipart/form-data")
@Auth(role = RoleEnum.SUPER_ADMIN)
public ResponseResult addEventsByFile(@RequestParam(value = "linkedGroupId") String linkedGroupId, @RequestParam("fileUrl") String fileUrl) {
EventService.addFileEvents(linkedGroupId, fileUrl);
eventService.addFileEvents(linkedGroupId, fileUrl);
behaviorService.pushBehavior(OPERATION, "文件上传事件", request);
return ResponseResult.success();
}
......@@ -252,7 +252,7 @@ public class EventController extends BaseController {
"linkedGroupId", value = "关联项目id", paramType = "form", dataType = "string")})
@PostMapping(value = "/data/upload", headers = "content-type=multipart/form-data")
public ResponseResult uploadEventDatas(@RequestParam("linkedGroupId") String linkedGroupId, @RequestParam("file") MultipartFile file) {
EventService.uploadEventDatas(linkedGroupId, file);
eventService.uploadEventDatas(linkedGroupId, file);
behaviorService.pushBehavior(OPERATION, "事件数据上传", request);
return ResponseResult.success();
}
......@@ -261,13 +261,13 @@ public class EventController extends BaseController {
@ApiImplicitParams({@ApiImplicitParam(name = "page", value = "页码", required = false, defaultValue = "1", paramType = "query", dataType = "int"), @ApiImplicitParam(name = "size", value = "每页记录数", required = false, defaultValue = "10", paramType = "query", dataType = "int"), @ApiImplicitParam(name = "linkedGroupId", value = "关联性项目组id", required = true, paramType = "query", dataType = "string"), @ApiImplicitParam(name = "tagGroupName", value = "标签组名", required = true, paramType = "query", dataType = "string")})
@GetMapping("/tag/list")
public ResponseResult findEventTagList(@RequestParam(value = "page", defaultValue = "1") int page, @RequestParam(value = "size", defaultValue = "10") int size, @RequestParam(value = "linkedGroupId") String linkedGroupId, @RequestParam(value = "tagGroupName") String tagGroupName) {
return ResponseResult.success(EventService.findEventTagList(page, size, linkedGroupId, tagGroupName));
return ResponseResult.success(eventService.findEventTagList(page, size, linkedGroupId, tagGroupName));
}
@ApiOperation("查询所有事件标签组名")
@GetMapping("/tag/groupName")
public ResponseResult findEventTagGroupName(@RequestParam(value = "linkedGroupId") String linkedGroupId) {
return ResponseResult.success(EventService.findEventTagGroupName(linkedGroupId));
return ResponseResult.success(eventService.findEventTagGroupName(linkedGroupId));
}
}
src/main/java/com/zhiwei/brandkbs2/controller/app/AppArticleController.java
......@@ -2,8 +2,10 @@ package com.zhiwei.brandkbs2.controller.app;
import com.zhiwei.brandkbs2.auth.Auth;
import com.zhiwei.brandkbs2.controller.BaseController;
import com.zhiwei.brandkbs2.easyexcel.EasyExcelUtil;
import com.zhiwei.brandkbs2.enmus.RoleEnum;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.dto.ExportAppYuqingDTO;
import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
import com.zhiwei.brandkbs2.pojo.dto.ReportDTO;
import com.zhiwei.brandkbs2.pojo.dto.ReportSearchDTO;
......@@ -14,9 +16,11 @@ import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiImplicitParams;
import io.swagger.annotations.ApiOperation;
import org.apache.commons.lang3.tuple.Pair;
import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource;
import java.util.List;
/**
* @author sjj
......@@ -46,17 +50,38 @@ public class AppArticleController extends BaseController {
}
@ApiOperation("舆情列表-生成聚合")
@PostMapping("/mark/agree")
@PostMapping("/mark/aggree")
public ResponseResult generateYuqingMarkAggreeList(@RequestBody MarkSearchDTO markSearchDTO) {
return ResponseResult.success(markDataService.generateYuqingMarkAggreeList(markSearchDTO));
}
@ApiOperation("舆情列表-聚合进度查询")
@GetMapping("/mark/aggree/progress/{aggreeId}")
public ResponseResult getYuqingMarkAggreeProgress(@PathVariable String aggreeId) {
return ResponseResult.success(markDataService.getYuqingMarkAggreeProgress(aggreeId));
}
@ApiOperation("舆情列表-获取聚合结果")
@PostMapping("/mark/aggree/list")
public ResponseResult getYuqingMarkAggreeList(@RequestBody MarkSearchDTO markSearchDTO) {
return ResponseResult.success(markDataService.getYuqingMarkAggreeList(markSearchDTO));
}
@ApiOperation("舆情列表-搜索条件")
@GetMapping("/mark/list/criteria")
public ResponseResult getYuqingMark(@RequestParam(required = false) String linkedGroupId) {
return ResponseResult.success(markDataService.getYuqingMarkCriteria(linkedGroupId));
}
@ApiOperation("舆情导出")
@PostMapping(value = "mark/list/export")
public ResponseResult exportYuqingMarkList(@RequestBody MarkSearchDTO markSearchDTO) {
Pair<String, List<ExportAppYuqingDTO>> stringListPair = markDataService.downloadYuqingMarkList(markSearchDTO);
EasyExcelUtil.download(stringListPair.getLeft() + "_舆情列表数据", "sheet1", ExportAppYuqingDTO.class, stringListPair.getRight(), response);
return ResponseResult.success();
}
@ApiOperation("舆情分析-舆情概览")
@ApiImplicitParams({@ApiImplicitParam(name = "startTime", value = "开始时间", paramType = "query", dataType = "long"), @ApiImplicitParam(name = "endTime", value = "结束时间", paramType = "query", dataType = "long")})
@GetMapping("/analyze/summary")
......
src/main/java/com/zhiwei/brandkbs2/controller/app/GlobalController.java
......@@ -19,7 +19,7 @@ import javax.annotation.Resource;
*/
@RestController
@RequestMapping("/app/global")
@Api(tags = "全局通用接口")
@Api(tags = "全局通用接口", description = "提供全局通用相关")
@Auth(role = RoleEnum.CUSTOMER)
public class GlobalController extends BaseController {
......
src/main/java/com/zhiwei/brandkbs2/dao/AggreeResultDao.java 0 → 100644
package com.zhiwei.brandkbs2.dao;
import com.zhiwei.brandkbs2.pojo.AggreeResult;
/**
* @ClassName: AggreeResultDao
* @Description AggreeResultDao
* @author: sjj
* @date: 2022-07-18 09:47
*/
public interface AggreeResultDao extends BaseMongoDao<AggreeResult>{
}
src/main/java/com/zhiwei/brandkbs2/dao/impl/AggreeResultDaoImpl.java 0 → 100644
package com.zhiwei.brandkbs2.dao.impl;
import com.zhiwei.brandkbs2.dao.AggreeResultDao;
import com.zhiwei.brandkbs2.pojo.AggreeResult;
import org.springframework.stereotype.Service;
/**
* @ClassName: AggreeResultDaoImpl
* @Description AggreeResultDaoImpl
* @author: sjj
* @date: 2022-07-18 09:50
*/
@Service("aggreeResultDaoImpl")
public class AggreeResultDaoImpl extends BaseMongoDaoImpl<AggreeResult> implements AggreeResultDao {
private static final String COLLECTION_PREFIX = "brandkbs_aggree_result";
public AggreeResultDaoImpl() {
super(COLLECTION_PREFIX);
}
}
src/main/java/com/zhiwei/brandkbs2/dao/impl/UserProjectOldDaoImpl.java
......@@ -6,8 +6,6 @@ import lombok.Setter;
import org.springframework.data.mongodb.core.mapping.Document;
import org.springframework.stereotype.Component;
import java.util.Date;
/**
* @ClassName: UserProjectOldImpl
* @Description 用户项目关联旧表
......@@ -28,7 +26,7 @@ public class UserProjectOldDaoImpl extends BaseMongoDaoImpl<UserProjectOldDaoImp
public static class UserProjectOld extends AbstractBaseMongo{
String userId;
Integer roleId;
Date expiredTime;
Long expiredTime;
Integer exportAmount;
String projectName;
}
......
src/main/java/com/zhiwei/brandkbs2/easyexcel/listener/EventFileListener.java
......@@ -65,6 +65,6 @@ public class EventFileListener extends AnalysisEventListener<UploadEventDTO> {
* 保存更新事件信息逻辑
*/
private void addFileEvent() {
datas.forEach(eventDTO -> EventService.addFileEvent(projectId, linkedGroupId, eventDTO));
datas.forEach(eventDTO -> eventService.addFileEvent(projectId, linkedGroupId, eventDTO));
}
}
src/main/java/com/zhiwei/brandkbs2/pojo/AggreeResult.java 0 → 100644
package com.zhiwei.brandkbs2.pojo;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.util.Tools;
import lombok.Getter;
import lombok.Setter;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
* @ClassName: AggreeResult
* @Description 聚合实体
* @author: sjj
* @date: 2022-07-18 09:41
*/
@Setter
@Getter
public class AggreeResult extends AbstractBaseMongo {
/**
* 任务id
*/
private String taskId;
/**
* 创建时间
*/
private Long cTime;
/**
* 原数据
*/
private JSONObject data;
/**
* 聚合标题
*/
private String aggreeTitle;
/**
* 发文时间
*/
private Long time;
/**
* 平台id
*/
private String platformId;
/**
* 重要发声方
*/
private List<String> mediaTypes;
/**
* 情感倾向(舆情调性)
*/
private List<String> tags;
/**
* 聚合量
*/
private Integer aggreeSize;
/**
* 父聚合id
*/
private String fatherId;
/**
* 是否是父聚合
*/
private boolean isFather;
public static AggreeResult createFather(JSONObject json, String taskId, Integer aggreeSize) {
AggreeResult aggreeResult = getInstance(json, taskId);
aggreeResult.setFather(true);
aggreeResult.setAggreeSize(aggreeSize);
aggreeResult.setCTime(System.currentTimeMillis());
return aggreeResult;
}
public static AggreeResult createSon(JSONObject json, String taskId, String fatherId, String fatherTitle) {
AggreeResult aggreeResult = getInstance(json, taskId);
aggreeResult.setFatherId(fatherId);
// 使用父聚合标题
aggreeResult.setAggreeTitle(fatherTitle);
aggreeResult.setCTime(System.currentTimeMillis());
return aggreeResult;
}
private static AggreeResult getInstance(JSONObject json, String taskId) {
AggreeResult aggreeResult = new AggreeResult();
aggreeResult.setTaskId(taskId);
aggreeResult.setData(json);
BaseMap baseMap = Tools.getBaseFromEsMap(json);
aggreeResult.setTime(baseMap.getTime());
aggreeResult.setPlatformId(json.getString(GenericAttribute.ES_PLATFORM_ID));
String mediaType = json.getString(GenericAttribute.ES_MEDIA_TYPE);
if (StringUtils.isNotEmpty(mediaType)) {
aggreeResult.setMediaTypes(Arrays.asList(mediaType.split(",")));
}
aggreeResult.setTags(parseMatg(json.getString(GenericAttribute.ES_MTAG)));
aggreeResult.setAggreeTitle(baseMap.getTitleNullOptionalContent());
return aggreeResult;
}
private static List<String> parseMatg(String mtag) {
if (StringUtils.isEmpty(mtag)) {
return null;
}
List<String> resList = new ArrayList<>();
String[] uniques = mtag.substring(1).split(",");
for (String unique : uniques) {
resList.add("," + unique);
}
return resList;
}
}
src/main/java/com/zhiwei/brandkbs2/pojo/MarkFlowEntity.java
......@@ -15,6 +15,7 @@ import lombok.Setter;
import org.apache.commons.lang3.ObjectUtils;
import org.elasticsearch.search.SearchHit;
import java.io.Serializable;
import java.util.List;
import java.util.Objects;
......@@ -28,7 +29,7 @@ import java.util.Objects;
@Getter
@Setter
@NoArgsConstructor
public class MarkFlowEntity {
public class MarkFlowEntity implements Serializable {
/**
* 消息类型
......@@ -47,16 +48,29 @@ public class MarkFlowEntity {
public static MarkFlowEntity getFoldInstance(List<MarkFlowEntity> entities) {
// 首条数据为主
MarkFlowEntity entity = entities.get(0);
MarkFlowEntity entity = Tools.copyByJson(entities.get(0), MarkFlowEntity.class);
if (entities.size() > 1) {
entity.getType().put("fold", true);
entity.getType().put("size", entities.size());
// info内容
if (null == entity.getInfo()) {
entity.setInfo(new JSONObject());
}
entity.getInfo().put("foldInfo", entities);
}
return entity;
}
public static MarkFlowEntity getAggreeInstance(List<MarkFlowEntity> entities,int size) {
// 首条数据为主
MarkFlowEntity entity = Tools.copyByJson(entities.get(0), MarkFlowEntity.class);
if (size > 1) {
entity.getType().put("fold", true);
entity.getType().put("size",size);
}
return entity;
}
/**
* Constructor
*
......
src/main/java/com/zhiwei/brandkbs2/pojo/UserRole.java
......@@ -3,6 +3,7 @@ package com.zhiwei.brandkbs2.pojo;
import com.zhiwei.brandkbs2.enmus.RoleEnum;
import com.zhiwei.brandkbs2.pojo.dto.UserDTO;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import org.apache.commons.lang3.time.DateUtils;
......@@ -17,6 +18,7 @@ import java.util.Date;
*/
@Setter
@Getter
@NoArgsConstructor
public class UserRole {
// 默认绑定阿里项目
......@@ -51,10 +53,10 @@ public class UserRole {
*/
private Integer exportAmount;
public UserRole(String projectId, int roleId, Date expiredTime, Integer exportAmount) {
public UserRole(String projectId, int roleId, Long expiredTime, Integer exportAmount) {
this.projectId = projectId;
this.roleId = roleId;
this.expiredTime = expiredTime.getTime();
this.expiredTime = expiredTime;
this.exportAmount = exportAmount;
}
......@@ -63,7 +65,7 @@ public class UserRole {
if (null != userDTO.getExportAmount() && userDTO.getExportAmount() < MAX_EXPORT_LIMIT) {
exportAmount = userDTO.getExportAmount();
}
Date expiredTime = null;
Long expiredTime = null;
//当创建用户为客户时，添加过期时间
if (userDTO.getRoleId() == RoleEnum.CUSTOMER.getState()) {
Date date = new Date(userDTO.getExpiredTime());
......@@ -72,13 +74,13 @@ public class UserRole {
calendar.set(Calendar.HOUR_OF_DAY, 23);
calendar.set(Calendar.MINUTE, 59);
calendar.set(Calendar.SECOND, 59);
expiredTime = calendar.getTime();
expiredTime = calendar.getTime().getTime();
}
return new UserRole(userDTO.getProjectId(), userDTO.getRoleId(), expiredTime, exportAmount);
}
public static UserRole defaultUserRole(Date now) {
return new UserRole(DEFAULT_PROJECT_ID, DEFAULT_ROLE_ID, DateUtils.addDays(now, 30), DEFAULT_EXPORT_LIMIT);
return new UserRole(DEFAULT_PROJECT_ID, DEFAULT_ROLE_ID, DateUtils.addDays(now, 30).getTime(), DEFAULT_EXPORT_LIMIT);
}
}
src/main/java/com/zhiwei/brandkbs2/pojo/dto/ExportAppYuqingDTO.java 0 → 100644
package com.zhiwei.brandkbs2.pojo.dto;
import com.alibaba.excel.annotation.ExcelProperty;
import com.alibaba.excel.annotation.write.style.ColumnWidth;
import com.zhiwei.brandkbs2.pojo.BaseMap;
import com.zhiwei.brandkbs2.util.Tools;
import lombok.Data;
import lombok.ToString;
import java.util.Date;
import java.util.Map;
/**
* 舆情导出
*/
@Data
@ToString
public class ExportAppYuqingDTO {
@ColumnWidth(20)
@ExcelProperty("时间")
private Date time;
@ColumnWidth(20)
@ExcelProperty("平台")
private String platform;
@ColumnWidth(20)
@ExcelProperty("渠道")
private String source;
@ColumnWidth(50)
@ExcelProperty("标题")
private String title;
@ColumnWidth(50)
@ExcelProperty("文本")
private String content;
@ColumnWidth(50)
@ExcelProperty("地址")
private String url;
@ColumnWidth(25)
@ExcelProperty("情感调性")
private String emotion;
public static ExportAppYuqingDTO createFromEsMap(Map<String, Object> map) {
ExportAppYuqingDTO dto = new ExportAppYuqingDTO();
BaseMap baseFromEsMap = Tools.getBaseFromEsMap(map);
dto.setTime(new Date(baseFromEsMap.getTime()));
dto.setPlatform(baseFromEsMap.getPlatform());
dto.setSource(baseFromEsMap.getSource());
dto.setTitle(baseFromEsMap.getTitle());
dto.setContent(baseFromEsMap.getContent());
dto.setUrl(baseFromEsMap.getUrl());
dto.setEmotion(baseFromEsMap.getEmotion());
return dto;
}
}
src/main/java/com/zhiwei/brandkbs2/pojo/dto/MarkSearchDTO.java
......@@ -12,6 +12,12 @@ import java.util.List;
@ToString
@ApiModel("标注数据搜索传输类")
public class MarkSearchDTO {
/**
* 聚合id
*/
@ApiModelProperty("聚合id")
private String aggreeId;
/**
* 页码
*/
......@@ -25,6 +31,8 @@ public class MarkSearchDTO {
private int pageSize;
// postFilter
@ApiModelProperty(value = "项目id")
private String projectId;
/**
* 关联项目组id
*/
......
src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java
......@@ -2,8 +2,10 @@ package com.zhiwei.brandkbs2.service;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.brandkbs2.pojo.MarkFlowEntity;
import com.zhiwei.brandkbs2.pojo.dto.ExportAppYuqingDTO;
import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
import com.zhiwei.brandkbs2.pojo.vo.PageVO;
import org.apache.commons.lang3.tuple.Pair;
import java.util.List;
......@@ -22,6 +24,12 @@ public interface MarkDataService {
PageVO<MarkFlowEntity> getYuqingMarkList(MarkSearchDTO markSearchDTO);
/**
* @param markSearchDTO 标注数据搜索传输类
* @return 返回结果
*/
Pair<String,List<ExportAppYuqingDTO>> downloadYuqingMarkList(MarkSearchDTO markSearchDTO);
/**
* 生成聚合列表并返回id
*
* @return 聚合id
......@@ -29,6 +37,21 @@ public interface MarkDataService {
String generateYuqingMarkAggreeList(MarkSearchDTO markSearchDTO);
/**
* 获取聚合进度结果
* @param id 聚合id
* @return 进度值
*/
String getYuqingMarkAggreeProgress(String id);
/**
* 获取聚合结果列表
*
* @param markSearchDTO 搜索实体
* @return 标注消息流列表
*/
PageVO<MarkFlowEntity> getYuqingMarkAggreeList(MarkSearchDTO markSearchDTO);
/**
* 舆情标注数据搜索条件
*/
JSONObject getYuqingMarkCriteria(String linkedGroup);
......
src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java
......@@ -8,6 +8,7 @@ import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.common.GlobalPojo;
import com.zhiwei.brandkbs2.common.RedisKeyPrefix;
import com.zhiwei.brandkbs2.dao.AggreeResultDao;
import com.zhiwei.brandkbs2.enmus.EmotionEnum;
import com.zhiwei.brandkbs2.enmus.ImportantChannelEnum;
import com.zhiwei.brandkbs2.es.EsClientDao;
......@@ -15,13 +16,14 @@ import com.zhiwei.brandkbs2.es.EsQueryTools;
import com.zhiwei.brandkbs2.exception.ExceptionCast;
import com.zhiwei.brandkbs2.listener.ApplicationProjectListener;
import com.zhiwei.brandkbs2.model.CommonCodeEnum;
import com.zhiwei.brandkbs2.pojo.BaseMap;
import com.zhiwei.brandkbs2.pojo.MarkFlowEntity;
import com.zhiwei.brandkbs2.pojo.*;
import com.zhiwei.brandkbs2.pojo.dto.ExportAppYuqingDTO;
import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
import com.zhiwei.brandkbs2.pojo.vo.CustomTagVo;
import com.zhiwei.brandkbs2.pojo.vo.PageVO;
import com.zhiwei.brandkbs2.pojo.vo.ProjectVO;
import com.zhiwei.brandkbs2.service.*;
import com.zhiwei.brandkbs2.util.MongoUtil;
import com.zhiwei.brandkbs2.util.RedisUtil;
import com.zhiwei.brandkbs2.util.TextUtil;
import com.zhiwei.brandkbs2.util.Tools;
......@@ -52,6 +54,8 @@ import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilde
import org.elasticsearch.search.sort.FieldSortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
......@@ -88,19 +92,25 @@ public class MarkDataServiceImpl implements MarkDataService {
@Resource(name = "markFlowServiceImpl")
MarkFlowService markFlowService;
@Resource(name = "aggreeResultDaoImpl")
AggreeResultDao aggreeResultDao;
@Resource(name = "redisUtil")
RedisUtil redisUtil;
@Resource(name = "mongoUtil")
MongoUtil mongoUtil;
@Override
public PageVO<MarkFlowEntity> getYuqingMarkList(MarkSearchDTO markSearchDTO) {
try {
defaultMarkSearch(markSearchDTO);
// 搜索es数据
Pair<SearchHits, Map<String, Long>> hitsAndCounts = searchMarkHitsAndCount(markSearchDTO);
Pair<SearchHits[], Map<String, Long>> hitsAndCounts = searchMarkHitsAndCount(markSearchDTO, false);
// 总量
long total = hitsAndCounts.getLeft().getTotalHits().value > 10000 ? 10000 : hitsAndCounts.getLeft().getTotalHits().value;
long total = hitsAndCounts.getLeft()[0].getTotalHits().value > 10000 ? 10000 : hitsAndCounts.getLeft()[0].getTotalHits().value;
// 消息列表
List<MarkFlowEntity> flowEntityList = getMarkFlowEntity(markSearchDTO, hitsAndCounts.getLeft());
List<MarkFlowEntity> flowEntityList = getMarkFlowEntity(markSearchDTO, hitsAndCounts.getLeft()[0]);
// 返回分页结果并设置平台count
return PageVO.createPageVo(total, markSearchDTO.getPage(), markSearchDTO.getPageSize(), flowEntityList)
// 各平台计量
......@@ -112,25 +122,159 @@ public class MarkDataServiceImpl implements MarkDataService {
}
@Override
public Pair<String, List<ExportAppYuqingDTO>> downloadYuqingMarkList(MarkSearchDTO markSearchDTO) {
try {
ProjectVO project = projectService.getProjectVOById(UserThreadLocal.getProjectId());
List<ExportAppYuqingDTO> returnList = new ArrayList<>();
Pair<SearchHits[], Map<String, Long>> hitsAndCounts = searchMarkHitsAndCount(markSearchDTO, false);
for (SearchHits searchHits : hitsAndCounts.getLeft()) {
List<ExportAppYuqingDTO> collect = Arrays.stream(searchHits.getHits()).map(SearchHit::getSourceAsMap).map(ExportAppYuqingDTO::createFromEsMap).collect(Collectors.toList());
returnList.addAll(collect);
}
return Pair.of(project.getBrandName() + "_" + markSearchDTO.getStartTime() + "_" + markSearchDTO.getEndTime(), returnList);
} catch (IOException e) {
ExceptionCast.cast(CommonCodeEnum.FAIL.message("es检索异常"));
}
return null;
}
@Override
public String generateYuqingMarkAggreeList(MarkSearchDTO markSearchDTO) {
String uuid = Tools.getUUID();
String projectId = UserThreadLocal.getProjectId();
defaultMarkSearch4Aggree(markSearchDTO);
ApplicationProjectListener.getThreadPool().execute(() -> {
String redisKey = RedisUtil.getAggreeCacheKey(uuid, projectId);
try {
defaultMarkSearch(markSearchDTO);
redisUtil.setExpire(redisKey, "0");
// 搜索es数据
Pair<SearchHits, Map<String, Long>> hitsAndCounts = searchMarkHitsAndCount(markSearchDTO);
Pair<SearchHits[], Map<String, Long>> hitsAndCounts = searchMarkHitsAndCount(markSearchDTO, true);
log.info("taskId:{},聚合前数据:{}条", uuid, hitsAndCounts.getLeft()[0].getTotalHits().value);
// 聚合分组后数据
List<List<JSONObject>> groupList = aggregation(hitsAndCounts.getLeft(), "id");
log.info("taskId:{},聚合完成,聚合组:{}个", uuid, groupList.size());
// 入mongo缓存
cacheAggreeResult(groupList, uuid);
redisUtil.setExpire(redisKey, "100");
} catch (Exception e) {
log.error("generateYuqingMarkAggreeList-", e);
redisUtil.remove(redisKey);
}
});
return uuid;
}
@Override
public String getYuqingMarkAggreeProgress(String id) {
String redisKey = RedisUtil.getAggreeCacheKey(id, UserThreadLocal.getProjectId());
return redisUtil.get(redisKey);
}
@Override
public PageVO<MarkFlowEntity> getYuqingMarkAggreeList(MarkSearchDTO dto) {
if (null == dto.getAggreeId()) {
ExceptionCast.cast(CommonCodeEnum.INVALID_PARAM.message("聚合id不得为空"));
}
defaultMarkSearch4Aggree(dto);
Query query = assembleAggreeQuery(dto);
// 量查询
long count = aggreeResultDao.count(query);
// 开启分页
mongoUtil.start(dto.getPage(), dto.getPageSize(), query);
// 排序
aggreeResultDao.addSort(query, "{\"aggreeSize\":\"descend\"}");
// 返回结果
List<MarkFlowEntity> resList = new ArrayList<>();
List<AggreeResult> fatherList = aggreeResultDao.findList(query);
// List<String> fatherIds = fatherList.stream().map(AggreeResult::getFatherId).collect(Collectors.toList());
// Map<String, List<AggreeResult>> fatherMap = new HashMap<>();
// for (AggreeResult aggreeResult : aggreeResultDao.findList(Query.query(Criteria.where("fatherId").in(fatherIds)))) {
// fatherMap.compute(aggreeResult.getFatherId(), (k, v) -> {
// if (null == v) {
// v = new ArrayList<>();
// }
// v.add(aggreeResult);
// return v;
// });
// }
// Map<String, List<AggreeResult>> fatherMap = aggreeResultDao.findList(Query.query(Criteria.where("fatherId").in(fatherIds))).stream().collect(Collectors.groupingBy(AggreeResult::getFatherId));
for (AggreeResult aggreeResult : fatherList) {
List<MarkFlowEntity> markFlowEntities = new ArrayList<>();
MarkFlowEntity instance = new MarkFlowEntity(aggreeResult.getData());
instance.setInfo(markFlowService.createMarkFlowInfo(aggreeResult.getData(), dto.getProjectId(), dto.getLinkedGroupId()));
markFlowEntities.add(instance);
// 添加父标题集下的子标题集
// markFlowEntities.addAll(fatherMap.get(aggreeResult.getFatherId()).stream().map(sonResult -> new MarkFlowEntity(sonResult.getData())).collect(Collectors.toList()));
resList.add(MarkFlowEntity.getAggreeInstance(markFlowEntities, aggreeResult.getAggreeSize()));
}
return PageVO.createPageVo(count, dto.getPage(), dto.getPageSize(), resList);
}
private Query assembleAggreeQuery(MarkSearchDTO dto) {
Query query = new Query();
query.addCriteria(Criteria.where("taskId").is(dto.getAggreeId()));
// 平台
if (CollectionUtils.isNotEmpty(dto.getPlatforms())) {
Criteria orCriteria = new Criteria();
List<Criteria> platformCriteria = new ArrayList<>();
for (String platformId : dto.getPlatforms()) {
platformCriteria.add(Criteria.where("platformId").is(platformId));
}
query.addCriteria(orCriteria.orOperator(platformCriteria.toArray(new Criteria[0])));
}
// 媒体类型
if (CollectionUtils.isNotEmpty(dto.getMediaTypes())) {
Criteria orCriteria = new Criteria();
List<Criteria> mediaCriteria = new ArrayList<>();
for (String mediaType : dto.getMediaTypes()) {
mediaCriteria.add(Criteria.where("mediaTypes").is(mediaType));
}
query.addCriteria(orCriteria.orOperator(mediaCriteria.toArray(new Criteria[0])));
}
// 情感调性
if (CollectionUtils.isNotEmpty(dto.getTags())) {
Criteria orCriteria = new Criteria();
List<Criteria> tagCriteria = new ArrayList<>();
for (String tag : dto.getTags()) {
tagCriteria.add(Criteria.where("tags").is(tag));
}
query.addCriteria((orCriteria.orOperator(tagCriteria.toArray(new Criteria[0]))));
}
// 关键词模糊查询
aggreeResultDao.addKeywordFuzz(query, dto.getKeyword(), "aggreeTitle");
return query;
}
private void cacheAggreeResult(List<List<JSONObject>> groupList, String taskId) {
int count = 0;
int percent;
for (List<JSONObject> list : groupList) {
String fatherId = null;
String fatherTitle = null;
for (int i = 0; i < list.size(); i++) {
JSONObject json = list.get(i);
AggreeResult aggreeResult;
// 父聚合处理
if (i == 0) {
aggreeResult = AggreeResult.createFather(json, taskId, list.size());
aggreeResultDao.insertOneWithoutId(aggreeResult);
fatherId = aggreeResult.getId();
fatherTitle = aggreeResult.getAggreeTitle();
} else {
aggreeResult = AggreeResult.createSon(json, taskId, fatherId, fatherTitle);
aggreeResultDao.insertOneWithoutId(aggreeResult);
}
}
if ((percent = (++count / list.size())) % (10 / list.size()) == 0) {
log.info("taskId:{},fatherId:{} 入库完毕,进度:{}%", taskId, fatherId, percent);
}
}
}
@Override
public JSONObject getYuqingMarkCriteria(String linkedGroupId) {
if (null == linkedGroupId) {
linkedGroupId = projectService.getProjectVOById(UserThreadLocal.getProjectId()).getBrandLinkedGroupId();
......@@ -183,8 +327,7 @@ public class MarkDataServiceImpl implements MarkDataService {
// 获取所有平台稿件倾向稿件数量信息
List<JSONObject> platformsCount = getPlatformMarkEmotionCount(startTime, endTime, projectId, linkedGroupId);
// 正面舆情峰值平台
platformsCount.stream().max(Comparator.comparing(o -> o.getInteger(EmotionEnum.POSITIVE.getName()))).ifPresent(e -> result.put("posTopPlatform",
e.getString("platform_id")));
platformsCount.stream().max(Comparator.comparing(o -> o.getInteger(EmotionEnum.POSITIVE.getName()))).ifPresent(e -> result.put("posTopPlatform", e.getString("platform_id")));
// 负面舆情峰值平台
platformsCount.stream().max(Comparator.comparing(o -> o.getInteger(EmotionEnum.NEGATIVE.getName()))).ifPresent(e -> result.put("negTopPlatform", e.getString("platform_id")));
// 获取时间段正面情感数据最多的标题
......@@ -278,11 +421,7 @@ public class MarkDataServiceImpl implements MarkDataService {
for (JSONObject result : channelEmotions) {
int channelEmotion = result.getIntValue("channelEmotion");
long channelEmotionCount = result.getLongValue("count");
List<String> mediaTypeList = Arrays.asList(
ImportantChannelEnum.YANGJI.getState(),
ImportantChannelEnum.KEJI.getState(),
ImportantChannelEnum.CAIJING.getState(),
ImportantChannelEnum.QITA.getState());
List<String> mediaTypeList = Arrays.asList(ImportantChannelEnum.YANGJI.getState(), ImportantChannelEnum.KEJI.getState(), ImportantChannelEnum.CAIJING.getState(), ImportantChannelEnum.QITA.getState());
List<JSONObject> mediaTypeResultList = new ArrayList<>(mediaTypeList.size());
for (String mediaType : mediaTypeList) {
BoolQueryBuilder boolQueryBuilder = projectLinkedGroupQuery(projectId, linkedGroupId);
......@@ -291,9 +430,7 @@ public class MarkDataServiceImpl implements MarkDataService {
.must(QueryBuilders.termQuery("channel_emotion", channelEmotion));
// TODO 其他类别标签
if (ImportantChannelEnum.QITA.getState().equals(mediaType)) {
boolQueryBuilder.mustNot(QueryBuilders.matchQuery("tag", ImportantChannelEnum.YANGJI.getState()))
.mustNot(QueryBuilders.matchQuery("tag", ImportantChannelEnum.KEJI.getState()))
.mustNot(QueryBuilders.matchQuery("tag", ImportantChannelEnum.CAIJING.getState()));
boolQueryBuilder.mustNot(QueryBuilders.matchQuery("tag", ImportantChannelEnum.YANGJI.getState())).mustNot(QueryBuilders.matchQuery("tag", ImportantChannelEnum.KEJI.getState())).mustNot(QueryBuilders.matchQuery("tag", ImportantChannelEnum.CAIJING.getState()));
} else {
boolQueryBuilder.must(QueryBuilders.matchQuery("tag", mediaType));
}
......@@ -349,6 +486,16 @@ public class MarkDataServiceImpl implements MarkDataService {
return null;
}
private void defaultMarkSearch4Aggree(MarkSearchDTO markSearchDTO) {
// 时间范围默认一星期
if (Objects.isNull(markSearchDTO.getStartTime()) || Objects.isNull(markSearchDTO.getEndTime())) {
Date now = new Date();
markSearchDTO.setEndTime(now.getTime());
markSearchDTO.setStartTime(DateUtils.addDays(now, -7).getTime());
}
defaultMarkSearch(markSearchDTO);
}
private void defaultMarkSearch(MarkSearchDTO markSearchDTO) {
// 时间范围默认一个月
if (Objects.isNull(markSearchDTO.getStartTime()) || Objects.isNull(markSearchDTO.getEndTime())) {
......@@ -369,19 +516,22 @@ public class MarkDataServiceImpl implements MarkDataService {
sorter.put("time", "descend");
markSearchDTO.setSorter(sorter);
}
String projectId = UserThreadLocal.getProjectId();
markSearchDTO.setProjectId(projectId);
String linkedGroupId = markSearchDTO.getLinkedGroupId();
if (null == linkedGroupId) {
linkedGroupId = projectService.getProjectVOById(projectId).getBrandLinkedGroupId();
markSearchDTO.setLinkedGroupId(linkedGroupId);
}
}
private Pair<SearchHits, Map<String, Long>> searchMarkHitsAndCount(MarkSearchDTO dto) throws IOException {
// 获取索引
String[] indexes = esClientDao.getIndexes();
// PostFilter 后置过滤器
BoolQueryBuilder postFilter = QueryBuilders.boolQuery();
private EsClientDao.SearchHelper createSearchHelperByMarkSearchDTO(MarkSearchDTO dto) {
EsClientDao.SearchHelper helper = EsClientDao.createSearchHelper();
// linkedGroupId projectId
String projectId = dto.getProjectId();
String linkedGroupId = dto.getLinkedGroupId();
if (null == linkedGroupId) {
linkedGroupId = projectService.getProjectVOById(UserThreadLocal.getProjectId()).getBrandLinkedGroupId();
}
postFilter.must(QueryBuilders.termQuery("brandkbs_cache_maps.linkedGroupId.keyword", linkedGroupId)).must(QueryBuilders.termQuery("brandkbs_cache_maps.projectId.keyword", UserThreadLocal.getProjectId()));
// PostFilter 后置过滤器
BoolQueryBuilder postFilter = projectLinkedGroupQuery(projectId, linkedGroupId);
// time
postFilter.must(QueryBuilders.rangeQuery(dto.getTimeType()).lte(dto.getStartTime()).lt(dto.getEndTime()));
// platform
......@@ -397,6 +547,7 @@ public class MarkDataServiceImpl implements MarkDataService {
if (CollectionUtils.isNotEmpty(dto.getCustomTags())) {
postFilter.must(EsQueryTools.assembleTagQuery(dto.getCustomTags()));
}
helper.setPostFilter(postFilter);
// Query 查询条件
BoolQueryBuilder query = QueryBuilders.boolQuery();
// keyword
......@@ -409,6 +560,7 @@ public class MarkDataServiceImpl implements MarkDataService {
if (StringUtils.isNotEmpty(dto.getSourceKeyword())) {
query.must(QueryBuilders.matchQuery(GenericAttribute.ES_SOURCE, dto.getSourceKeyword()));
}
helper.setQuery(query);
// sort
FieldSortBuilder sort = null;
for (Map.Entry<String, Object> entry : dto.getSorter().entrySet()) {
......@@ -422,28 +574,46 @@ public class MarkDataServiceImpl implements MarkDataService {
sort = SortBuilders.fieldSort(entry.getKey()).order(SortOrder.ASC);
}
}
helper.setSort(sort);
// from size
int from = (dto.getPage() - 1) * dto.getPageSize();
helper.setFrom((dto.getPage() - 1) * dto.getPageSize());
helper.setSize(dto.getPageSize());
// HighlightBuilder ???
SearchHits searchHits = esClientDao.searchHits(indexes, postFilter, query, null, sort, from, dto.getPageSize(), null);
return helper;
}
private Pair<SearchHits[], Map<String, Long>> searchMarkHitsAndCount(MarkSearchDTO dto, boolean aggree) throws IOException {
EsClientDao.SearchHelper helper = createSearchHelperByMarkSearchDTO(dto);
if (aggree) {
helper.setSize(0);
// TODO 本地调试不滚动查询
// List<SearchResponse> searchResponses = esClientDao.searchScrollResponse(helper);
List<SearchResponse> searchResponses = Collections.singletonList(esClientDao.searchResponse(helper));
return Pair.of(searchResponses.stream().map(SearchResponse::getHits).toArray(SearchHits[]::new), null);
}
SearchHits searchHits = esClientDao.searchHits(helper);
//平台计量统计 仅第一页且无平台限制
Map<String, Long> counts = new HashMap<>();
if (1 == dto.getPage() && CollectionUtils.isEmpty(dto.getPlatforms())) {
for (MessagePlatform platform : GlobalPojo.PLATFORMS) {
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery().must(query).must(EsQueryTools.assemblePlatformQuery(Collections.singletonList(platform)));
Long count = esClientDao.count(indexes, postFilter, queryBuilder);
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery().must(helper.getQuery()).must(EsQueryTools.assemblePlatformQuery(Collections.singletonList(platform)));
Long count = esClientDao.count(esClientDao.getIndexes(), helper.getPostFilter(), queryBuilder);
counts.put(platform.getName(), count);
}
}
return Pair.of(searchHits, counts);
return Pair.of(new SearchHits[]{searchHits}, counts);
}
private List<List<JSONObject>> aggregation(SearchHits searchHits, String uniqueField) {
List<JSONObject> jsonList = Arrays.stream(searchHits.getHits()).map(hit -> new JSONObject(hit.getSourceAsMap())).collect(Collectors.toList());
Map<String, String> idTitle = jsonList.stream().collect(Collectors.toMap(json -> json.getString(uniqueField),
json -> Tools.getBaseFromEsMap(json).getTitleNullOptionalContent()));
private List<List<JSONObject>> aggregation(SearchHits[] searchHitsList, String uniqueField) {
List<SearchHit> hitList = new ArrayList<>();
for (SearchHits searchHits : searchHitsList) {
hitList.addAll(Arrays.stream(searchHits.getHits()).collect(Collectors.toList()));
}
Map<String, String> idTitle = hitList.stream().collect(Collectors.toMap(SearchHit::getId, hit -> Tools.getBaseFromEsMap(hit.getSourceAsMap()).getTitleNullOptionalContent()));
List<List<String>> kResult = TextUtil.getKResult(idTitle);
return TextUtil.restoreAndSort(kResult, jsonList, uniqueField);
// return TextUtil.restoreThenSort(kResult, hitList.stream().map(hit -> new JSONObject(hit.getSourceAsMap())).collect(Collectors.toList()),
// uniqueField);
return TextUtil.restoreThenSort(kResult, hitList);
}
// private List<JSONObject> criteriaTime(Date endDate) {
......@@ -536,8 +706,7 @@ public class MarkDataServiceImpl implements MarkDataService {
query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
List<JSONObject> resultList = new ArrayList<>(emotionList.size());
Map<String, Aggregation> aggMap =
esClientDao.searchResponse(indexes, null, query, aggregationBuilder, null, null, 0, 0, null).getAggregations().asMap();
Map<String, Aggregation> aggMap = esClientDao.searchResponse(indexes, null, query, aggregationBuilder, null, null, 0, 0, null).getAggregations().asMap();
ParsedStringTerms teamAgg = (ParsedStringTerms) aggMap.get("count");
List<? extends Terms.Bucket> buckets = teamAgg.getBuckets();
for (String emotion : emotionList) {
......@@ -635,14 +804,11 @@ public class MarkDataServiceImpl implements MarkDataService {
TermsAggregationBuilder emotionAggregationBuilder = AggregationBuilders.terms("count").field("mark_cache_maps.name.keyword");
// query
BoolQueryBuilder query = QueryBuilders.boolQuery();
query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime)).
must(QueryBuilders.termQuery("brandkbs_cache_maps.projectId.keyword", projectId)).
must(QueryBuilders.termQuery("brandkbs_cache_maps.linkedGroupId.keyword", linkedGroupId));
query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime)).must(QueryBuilders.termQuery("brandkbs_cache_maps.projectId.keyword", projectId)).must(QueryBuilders.termQuery("brandkbs_cache_maps.linkedGroupId.keyword", linkedGroupId));
// 情感倾向限制
query.must(QueryBuilders.termQuery("mark_cache_maps.group_name.keyword", "情感倾向"));
// response
SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, query,
daysAggregationBuilder.subAggregation(emotionAggregationBuilder), null, null, 0, 0, null);
SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, query, daysAggregationBuilder.subAggregation(emotionAggregationBuilder), null, null, 0, 0, null);
Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
ParsedDateHistogram dayAggTeam = (ParsedDateHistogram) aggMap.get("dayAgg");
List<? extends Histogram.Bucket> buckets = dayAggTeam.getBuckets();
......@@ -680,8 +846,7 @@ public class MarkDataServiceImpl implements MarkDataService {
query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
// 情感倾向限制
query.must(QueryBuilders.termQuery("mark_cache_maps.group_name.keyword", "情感倾向"));
SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, query,
platformAggregationBuilder.subAggregation(emotionAggregationBuilder), null, null, 0, 0, null);
SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, query, platformAggregationBuilder.subAggregation(emotionAggregationBuilder), null, null, 0, 0, null);
Map<String, Aggregation> aggMap = searchResponse.getAggregations().asMap();
ParsedStringTerms teamAgg = (ParsedStringTerms) aggMap.get("platform_count");
List<? extends Terms.Bucket> buckets = teamAgg.getBuckets();
......@@ -723,8 +888,7 @@ public class MarkDataServiceImpl implements MarkDataService {
List<JSONObject> platformList = commonService.getQbjcPlatform("id", "name");
List<JSONObject> platformsCount = getPlatformsCount(startTime, endTime, null, null, projectId, linkedGroupId, platformList);
long articlesCount = platformsCount.stream().mapToLong(platform -> platform.getLongValue("count")).sum();
List<JSONObject> resultList = platformsCount.stream().peek(platform -> platform.put("proportion",
0 == articlesCount ? 0 : platform.getLongValue("count") * 1.0 / articlesCount)).collect(Collectors.toList());
List<JSONObject> resultList = platformsCount.stream().peek(platform -> platform.put("proportion", 0 == articlesCount ? 0 : platform.getLongValue("count") * 1.0 / articlesCount)).collect(Collectors.toList());
redisUtil.setExpire(redisKey, JSON.toJSONString(resultList), 1, TimeUnit.HOURS);
return resultList;
}
......@@ -777,8 +941,7 @@ public class MarkDataServiceImpl implements MarkDataService {
*
* @return 所有稿件倾向稿件数量信息
*/
private List<JSONObject> getChannelEmotionsCount(Long startTime, Long endTime, String keyword, String searchField, String projectId, String linkedGroupId,
List<String> emotionList) {
private List<JSONObject> getChannelEmotionsCount(Long startTime, Long endTime, String keyword, String searchField, String projectId, String linkedGroupId, List<String> emotionList) {
return Collections.emptyList();
}
......
src/main/java/com/zhiwei/brandkbs2/service/impl/ReportServiceImpl.java
......@@ -88,7 +88,7 @@ public class ReportServiceImpl implements ReportService {
@Override
public List<Report> getCustomReportByStatus(String projectId, boolean status) {
Criteria criteria = Criteria.where("projectId").is(projectId).and("status").is(status).is("type").is(ReportTypeEnum.CUSTOM.getState());
Criteria criteria = Criteria.where("projectId").is(projectId).and("status").is(status).and("type").is(ReportTypeEnum.CUSTOM.getState());
return reportDao.findList(Query.query(criteria));
}
......@@ -158,7 +158,7 @@ public class ReportServiceImpl implements ReportService {
}
JSONObject all = new JSONObject();
all.put("name", "不限");
all.put("count",collect.values().stream().mapToInt(v -> v).sum());
all.put("count", collect.values().stream().mapToInt(v -> v).sum());
result.add(0, all);
return result;
}
......@@ -220,7 +220,18 @@ public class ReportServiceImpl implements ReportService {
@Override
public PageVO<JSONObject> findReportList(ReportSearchDTO reportSearch) {
Query query = new Query();
Criteria criteria = new Criteria();
if (null != reportSearch.getStartTime()) {
criteria.and("startTime").gte(reportSearch.getStartTime());
}
if (null != reportSearch.getEndTime()) {
criteria.and("endTime").lt(reportSearch.getEndTime());
}
if (StringUtils.isNotEmpty(reportSearch.getType())) {
criteria.and("type").is(reportSearch.getType());
}
Query query = new Query(criteria);
// 添加关键字查询并排序
long count = reportDao.count(query, reportSearch.getKeyword(), new String[]{"title"});
mongoUtil.start(reportSearch.getPage(), reportSearch.getPageSize(), query);
// 添加排序
......
src/main/java/com/zhiwei/brandkbs2/util/RedisUtil.java
......@@ -23,6 +23,10 @@ public class RedisUtil {
return RedisKeyPrefix.REDIS_SYSTEM_COPY + "SHOT_PAGE:" + projectId + "-" + id;
}
public static String getAggreeCacheKey(String id, String projectId) {
return RedisKeyPrefix.AGGREE_RESULT_CACHE + projectId + ":" + id;
}
public void setExpire(String key, String value, long timeout, TimeUnit unit) {
stringRedisTemplate.opsForValue().set(key, value, timeout, unit);
}
......@@ -35,4 +39,8 @@ public class RedisUtil {
return stringRedisTemplate.opsForValue().get(key);
}
public void remove(String key) {
setExpire(key, "-1", 1, TimeUnit.SECONDS);
}
}
src/main/java/com/zhiwei/brandkbs2/util/TextUtil.java
......@@ -13,6 +13,7 @@ import com.zhiwei.nlp.common.AggreeConfig;
import com.zhiwei.nlp.vo.KResult;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.search.SearchHit;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.Resource;
import org.springframework.stereotype.Component;
......@@ -67,7 +68,7 @@ public class TextUtil {
return AggreeBootStarter.getKResult(listMap, LIMIT).stream().map(KResult::getDataPoints).collect(Collectors.toList());
}
public static <T> List<List<JSONObject>> restoreAndSort(List<List<T>> kResultList, List<JSONObject> sourceList, String tField) {
public static <T> List<List<JSONObject>> restoreThenSort(List<List<T>> kResultList, List<JSONObject> sourceList, String tField) {
ImmutableMap<T, JSONObject> idMap = Maps.uniqueIndex(sourceList, json -> (T) json.get(tField));
// 还原数据
return kResultList.stream().map(gList ->
......@@ -75,6 +76,14 @@ public class TextUtil {
).collect(Collectors.toList());
}
public static List<List<JSONObject>> restoreThenSort(List<List<String>> kResultList, List<SearchHit> sourceList) {
ImmutableMap<String, SearchHit> idMap = Maps.uniqueIndex(sourceList, SearchHit::getId);
// 还原数据
return kResultList.stream().map(gList ->
gList.stream().map(title -> new JSONObject(idMap.get(title).getSourceAsMap())).sorted(Comparator.comparingLong(o -> o.getLong(DEFAULT_SORT_FIELD))).collect(Collectors.toList())
).collect(Collectors.toList());
}
public static Map<String, Integer> getHighWords(List<String> texts) {
return getHighWords(texts, null);
}
......
src/main/java/com/zhiwei/brandkbs2/util/Tools.java
......@@ -2,6 +2,7 @@ package com.zhiwei.brandkbs2.util;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.read.listener.PageReadListener;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.base.category.ClassCodec;
import com.zhiwei.base.entity.subclass.CompleteText;
......@@ -554,4 +555,8 @@ public class Tools {
return calendar.getTime().getTime();
}
public static <T> T copyByJson(T obj, Class<T> clazz) {
return JSON.parseObject(JSON.toJSONString(obj), clazz);
}
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment