Skip to content

B
brandkbs2
Commit 6744d566 by shenjunjie
Options

Merge branch 'feature' into 'dev' 

Feature

See merge request !500
parents b8dea545 e865e06d
Showing with 183 additions and 41 deletions
src/main/java/com/zhiwei/brandkbs2/controller/admin/ProjectController.java
......@@ -308,4 +308,10 @@ public class ProjectController extends BaseController {
ProjectService.adjustNonManualProjectDataBalance(json.getString("projectId"), json.getInteger("count"));
return ResponseResult.success();
}
@ApiOperation("非人工项目-数据源切换")
@PutMapping("/non-manual/data-source/{id}")
public ResponseResult switchExternalDataSource(@PathVariable String id){
return ProjectService.switchExternalDataSource(id);
}
}
src/main/java/com/zhiwei/brandkbs2/controller/app/AppArticleController.java
......@@ -259,7 +259,7 @@ public class AppArticleController extends BaseController {
@ApiOperation("舆情简报-全部报告列表")
@GetMapping("/report/list")
@LogRecord(description = "舆情库-舆情简报")
@LogRecord(description = "舆情库-其他简报")
public ResponseResult findReportList(ReportSearchDTO reportSearch) {
return ResponseResult.success(reportService.findReportList(reportSearch));
}
......@@ -540,6 +540,7 @@ public class AppArticleController extends BaseController {
@ApiOperation("舆情简报-每日一报列表")
@GetMapping("/daily-report/list")
@LogRecord(description = "舆情简报-每日一报")
public ResponseResult getDailyReportList(@RequestParam(value = "page", defaultValue = "1") Integer page,
@RequestParam(value = "pageSize", defaultValue = "10")Integer pageSize) {
return ResponseResult.success(markDataService.getDailyReportList(page, pageSize));
......
src/main/java/com/zhiwei/brandkbs2/es/EsQueryTools.java
......@@ -211,10 +211,26 @@ public class EsQueryTools {
return channelBoolQueryBuilder;
}
/**
* source模糊匹配
* @param sourceKeyword
* @return
*/
public static BoolQueryBuilder assembleSourceQuery(String sourceKeyword) {
return assembleFiledKeywordQuery("source", sourceKeyword);
}
/**
* source精确匹配
* @param sourceKeyword
* @return
*/
public static BoolQueryBuilder assembleSourceKeywordQuery(String sourceKeyword) {
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
queryBuilder.should(QueryBuilders.termQuery("source", sourceKeyword));
return queryBuilder;
}
public static void platformMustNot(BoolQueryBuilder boolQueryBuilder, String... platformNames) {
if (null == platformNames) {
return;
......
src/main/java/com/zhiwei/brandkbs2/pojo/Project.java
......@@ -111,6 +111,11 @@ public class Project extends AbstractProject {
private int dataBalance;
/**
* 仅非人工项目，数据源：true:商业数据（星光）|false:内部数据（底层数据）
*/
private Boolean externalDataSource;
/**
* 项目主品牌配置信息转换
*
* @return 项目对象
......
src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java
......@@ -400,12 +400,26 @@ public interface MarkDataService {
ResponseResult addNonManualProjectPlan(NonManualProjectPlanDTO dto);
/**
* 添加非人工项目方案订阅任务
* @param dto
* @return
*/
String addIStarShineSubjectTask(NonManualProjectPlanDTO dto);
/**
* 删除非人工项目方案
* @param id
*/
ResponseResult deleteNonManualProjectPlan(String id);
/**
* 删除非人工项目方案的数据订阅任务
* @param id
* @return
*/
ResponseResult deleteIStarShineSubjectTask(String id);
/**
* 获取非人工项目方案详情
* @param id
*/
......
src/main/java/com/zhiwei/brandkbs2/service/ProjectService.java
......@@ -210,4 +210,10 @@ public interface ProjectService {
* @return
*/
ResponseResult getUserProjectPageData(String userId, String keyword, int page, int pageSize);
/**
* 切换非人工项目数据源
* @param projectId
*/
ResponseResult switchExternalDataSource(String projectId);
}
src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java
......@@ -1900,13 +1900,21 @@ public class MarkDataServiceImpl implements MarkDataService {
@Override
public ResponseResult deleteNonManualProjectPlan(String id) {
ResponseResult responseResult = deleteIStarShineSubjectTask(id);
if (responseResult.isStatus()){
nonManualProjectPlanDao.deleteOneById(id);
}
return responseResult;
}
@Override
public ResponseResult deleteIStarShineSubjectTask(String id) {
String iStarShineSubjectTaskId = nonManualProjectPlanDao.findOneById(id).getIStarShineSubjectTaskId();
JSONObject body = restTemplate.getForEntity(deleteIStarShineSubjectTaskUrl, JSONObject.class, iStarShineSubjectTaskId, UserThreadLocal.getNickname()).getBody();
if (Objects.isNull(body) || body.isEmpty() || !body.getBoolean("status")){
log.info(id + "-删除快搜订阅任务失败,接口返回信息-" + body);
log.error(id + "-删除快搜订阅任务失败,接口返回信息-" + body);
return ResponseResult.failure("删除失败");
}
nonManualProjectPlanDao.deleteOneById(id);
return ResponseResult.success();
}
......@@ -1935,7 +1943,8 @@ public class MarkDataServiceImpl implements MarkDataService {
* @param dto
* @return
*/
private String addIStarShineSubjectTask(NonManualProjectPlanDTO dto){
@Override
public String addIStarShineSubjectTask(NonManualProjectPlanDTO dto){
JSONObject param = new JSONObject();
param.put("creatUser", UserThreadLocal.getNickname());
param.put("groupId", UserThreadLocal.getProjectId());
......@@ -1954,12 +1963,12 @@ public class MarkDataServiceImpl implements MarkDataService {
HttpEntity<String> request = new HttpEntity<>(param.toJSONString(), httpHeaders);
JSONObject body = restTemplate.postForEntity(addIStarShineSubjectTaskUrl, request, JSONObject.class).getBody();
if (Objects.isNull(body) || body.isEmpty() || !body.getBoolean("status")){
log.info(JSONObject.toJSONString(dto) + "-添加快搜订阅任务失败" + body);
log.error(JSONObject.toJSONString(dto) + "-添加快搜订阅任务失败" + body);
return null;
}
JSONObject data = body.getJSONObject("data");
if (Objects.isNull(data) || data.isEmpty()){
log.info(JSONObject.toJSONString(dto) + "-添加快搜订阅任务失败" + body);
log.error(JSONObject.toJSONString(dto) + "-添加快搜订阅任务失败" + body);
return null;
}
return data.getString("id");
......@@ -2509,13 +2518,11 @@ public class MarkDataServiceImpl implements MarkDataService {
JSONObject res = new JSONObject();
try {
List<JSONObject> list = new ArrayList<>();
List<String> importantPlatforms = new ArrayList<>();
String projectId = UserThreadLocal.getProjectId();
dto.setProjectId(projectId);
for (String platformNames : Arrays.asList("微博", "微信", "抖音", "小红书", "今日头条", "网媒")) {
importantPlatforms.add(GlobalPojo.getPlatformIdByName(platformNames));
if (CollectionUtils.isEmpty(dto.getPlatforms())) {
dto.setPlatforms(Stream.of("微博", "微信", "抖音", "小红书", "今日头条", "网媒").map(GlobalPojo::getPlatformIdByName).collect(Collectors.toList()));
}
dto.setPlatforms(importantPlatforms);
Long total = getYuqingAnalyzeCount(dto);
// 平台聚合
SearchResponse searchResponse = platformAggSearchResponse(dto);
......@@ -2580,8 +2587,10 @@ public class MarkDataServiceImpl implements MarkDataService {
* @throws IOException
*/
private List<JSONObject> getPlatformPercentage(MarkSearchDTO dto) throws IOException {
List<String> platforms = commonService.getQbjcPlatform().stream().map(jsonObject -> jsonObject.getString("id")).collect(Collectors.toList());
dto.setPlatforms(platforms);
if (CollectionUtils.isEmpty(dto.getPlatforms())) {
List<String> platforms = commonService.getQbjcPlatform().stream().map(jsonObject -> jsonObject.getString("id")).collect(Collectors.toList());
dto.setPlatforms(platforms);
}
List<JSONObject> list = new ArrayList<>();
// total
dto.setProjectId(UserThreadLocal.getProjectId());
......@@ -2774,7 +2783,7 @@ public class MarkDataServiceImpl implements MarkDataService {
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "新舆情分析getActiveChannels异常-", e);
}
return list.stream().limit(20).collect(Collectors.toList());
return list.stream().limit(size).collect(Collectors.toList());
}
@Override
......@@ -2815,19 +2824,46 @@ public class MarkDataServiceImpl implements MarkDataService {
@Override
public PageVO<MarkFlowEntity> getYuqingAnalyzeDetail(MarkSearchDTO dto) {
//设置默认的时间、页码、排序方式等
defaultMarkSearch(dto);
PageVO<MarkFlowEntity> yuqingMarkList;
if (Objects.isNull(dto.getPlanId())){
yuqingMarkList = getYuqingMarkList(dto);
}else {
yuqingMarkList = getNonManualMarkList(dto);
try {
//设置默认的时间、页码、排序方式等
defaultMarkSearch(dto);
String[] indexes = esClientDao.getIndexes();
BoolQueryBuilder postFilter = yuqingAnalyzeQuery(dto);
FieldSortBuilder sort = null;
if (null != dto.getSorter()) {
for (Map.Entry<String, Object> entry : dto.getSorter().entrySet()) {
String key = entry.getKey();
if (null == entry.getValue() || entry.getValue().toString().contains("desc")) {
sort = SortBuilders.fieldSort(key).order(SortOrder.DESC);
} else {
sort = SortBuilders.fieldSort(key).order(SortOrder.ASC);
}
}
}
int from = (dto.getPage() - 1) * dto.getPageSize();
int size = dto.getPageSize();
SearchHits hits = esClientDao.searchHits(indexes, postFilter, null, null, sort, from, size, null);
// 总量
long total = hits.getTotalHits().value;
// 消息列表
List<MarkFlowEntity> flowEntityList;
if (Objects.nonNull(dto.getPlanId())) {
flowEntityList = getNonManualMarkFlowEntity(dto, hits);
} else {
flowEntityList = getMarkFlowEntity(dto, hits);
}
JSONObject jsonObject = new JSONObject();
// 高亮关键词
jsonObject.put("highlightWord", GlobalPojo.HIGHLIGHT_MAP.get(UserThreadLocal.getProjectId()));
// 列表数据最小mgid
MarkFlowEntity markFlowEntity = flowEntityList.stream()
.min(Comparator.comparingLong(entity -> JSONObject.parseObject(JSONObject.toJSONString(entity.getData())).getLongValue("mgid"))).orElse(null);
jsonObject.put("pageGid", Objects.isNull(markFlowEntity) ? Long.MAX_VALUE : JSONObject.parseObject(JSONObject.toJSONString(markFlowEntity.getData())).getLongValue("mgid"));
return PageVO.createPageVo(total, dto.getPage(), dto.getPageSize(), flowEntityList).setInfo(jsonObject);
}catch (Exception e){
ExceptionCast.cast(CommonCodeEnum.FAIL, "新舆情分析详情页getYuqingAnalyzeDetail异常-", e);
}
// 列表数据最小mgid
MarkFlowEntity markFlowEntity = yuqingMarkList.getList().stream()
.min(Comparator.comparingLong(entity -> JSONObject.parseObject(JSONObject.toJSONString(entity.getData())).getLongValue("mgid"))).orElse(null);
yuqingMarkList.getInfo().put("pageGid", Objects.isNull(markFlowEntity) ? Long.MAX_VALUE : JSONObject.parseObject(JSONObject.toJSONString(markFlowEntity.getData())).getLongValue("mgid"));
return yuqingMarkList;
return PageVO.createPageVo(0, dto.getPage(), dto.getPageSize(), Collections.emptyList());
}
@Override
......@@ -2939,10 +2975,11 @@ public class MarkDataServiceImpl implements MarkDataService {
String projectId = UserThreadLocal.getProjectId();
dto.setProjectId(projectId);
String linkedGroupId = projectService.getProjectById(projectId).getBrandLinkedGroupId();
String aggTitle = dto.getAggTitle();
CompletableFuture<JSONObject> firstArticleFuture = CompletableFuture.supplyAsync(() -> {
try {
// 最早发布
BaseMap firstArticle = getAnalyzeFirstArticle(dto, null);
BaseMap firstArticle = getAnalyzeFirstArticle(dto, aggTitle);
// 匹配重要渠道
JSONObject firstSensitiveChannel = channelService.matchYuQingSensitiveChannel(linkedGroupId, firstArticle.getSource());
JSONObject firstArticleJson = JSONObject.parseObject(JSONObject.toJSONString(firstArticle));
......@@ -2955,7 +2992,7 @@ public class MarkDataServiceImpl implements MarkDataService {
CompletableFuture<JSONObject> lastArticleFuture = CompletableFuture.supplyAsync(() -> {
try {
// 最新发布
BaseMap lastArticle = getAnalyzeLastArticle(dto, null);
BaseMap lastArticle = getAnalyzeLastArticle(dto, aggTitle);
// 匹配重要渠道
JSONObject lastSensitiveChannel = channelService.matchYuQingSensitiveChannel(linkedGroupId, lastArticle.getSource());
JSONObject lastArticleJson = JSONObject.parseObject(JSONObject.toJSONString(lastArticle));
......@@ -2993,7 +3030,7 @@ public class MarkDataServiceImpl implements MarkDataService {
dto.setProjectId(UserThreadLocal.getProjectId());
BoolQueryBuilder query = yuqingAnalyzeQuery(dto);
DateHistogramAggregationBuilder daysAggregationBuilder =
AggregationBuilders.dateHistogram("timeAgg").field("time").calendarInterval(DateHistogramInterval.DAY);
AggregationBuilders.dateHistogram("timeAgg").field("time").calendarInterval(DateHistogramInterval.DAY).offset("16h");
TermsAggregationBuilder sourceAggregationBuilder = AggregationBuilders.terms("sourceAgg").field("source").size(10000);
// response
SearchResponse searchResponse = esClientDao.searchResponse(indexes, null, query,
......@@ -3066,17 +3103,15 @@ public class MarkDataServiceImpl implements MarkDataService {
String[] indexes = esClientDao.getIndexes();
dto.setProjectId(UserThreadLocal.getProjectId());
// query
if (Objects.nonNull(dto.getPlatforms()) && Objects.nonNull(dto.getPlatforms().get(0))){
dto.setPlatforms(Collections.singletonList(GlobalPojo.getPlatformIdByName(dto.getPlatforms().get(0))));
}
BoolQueryBuilder query = yuqingAnalyzeQuery(dto);
if (Objects.equals("其他", dto.getPlatforms().get(0))){
List<String> platforms = getPlatformPercentage(dto).stream().map(json -> json.getString("platform")).collect(Collectors.toList());
List<String> allPlatforms = GlobalPojo.PLATFORMS.stream().map(MessagePlatform::getName).collect(Collectors.toList());
allPlatforms.removeAll(platforms);
dto.setPlatforms(allPlatforms.stream().map(GlobalPojo::getPlatformIdByName).collect(Collectors.toList()));
query = yuqingAnalyzeQuery(dto);
}else {
dto.setPlatforms(Collections.singletonList(GlobalPojo.getPlatformIdByName(dto.getPlatforms().get(0))));
}
BoolQueryBuilder query = yuqingAnalyzeQuery(dto);
// sort
FieldSortBuilder sort = SortBuilders.fieldSort("time").order(SortOrder.DESC);
// response
......@@ -3190,9 +3225,12 @@ public class MarkDataServiceImpl implements MarkDataService {
private Map<String, List<LineVO>> getPlatformSpreadTendency(MarkSearchDTO dto) throws IOException{
Map<String, List<LineVO>> res = new HashMap<>(4);
// 平台
List<String> platforms = new ArrayList<>();
for (String platformNames : Arrays.asList("微博", "微信", "抖音", "小红书")) {
platforms.add(GlobalPojo.getPlatformIdByName(platformNames));
// List<String> platforms = new ArrayList<>();
// for (String platformNames : Arrays.asList("微博", "微信", "抖音", "小红书")) {
// platforms.add(GlobalPojo.getPlatformIdByName(platformNames));
// }
if (CollectionUtils.isEmpty(dto.getPlatforms())){
dto.setPlatforms(Stream.of("微博", "微信", "抖音", "小红书").map(GlobalPojo::getPlatformIdByName).collect(Collectors.toList()));
}
String[] indexes = esClientDao.getIndexes();
// 聚合请求
......@@ -3204,7 +3242,6 @@ public class MarkDataServiceImpl implements MarkDataService {
}else {
daysAggregationBuilder = AggregationBuilders.dateHistogram("timeAgg").field("time").calendarInterval(DateHistogramInterval.DAY);
}
dto.setPlatforms(platforms);
// query
BoolQueryBuilder query = yuqingAnalyzeQuery(dto);
// response
......@@ -3334,7 +3371,7 @@ public class MarkDataServiceImpl implements MarkDataService {
// 新增channelEmotion
BoolQueryBuilder postFilter = EsQueryTools.assembleCacheMapsQuery(projectId, contendId, dto.getPoliticsLevel(), dto.getField(), dto.getRegion(), dto.getMainBodyType(), dto.getChannelEmotion());
if (Objects.nonNull(dto.getPlanId())){
postFilter = EsQueryTools.assembleCacheMapsPlanQuery(projectId, dto.getPlanId());
postFilter = EsQueryTools.assembleCacheMapsPlanQuery(projectId, dto.getPlanId(), dto.getPoliticsLevel());
}
// time
postFilter.must(QueryBuilders.rangeQuery("time").gte(dto.getStartTime()).lt(dto.getEndTime()));
......@@ -3367,7 +3404,7 @@ public class MarkDataServiceImpl implements MarkDataService {
}
// sourceKeyword
if (StringUtils.isNotEmpty(dto.getSourceKeyword())) {
postFilter.must(EsQueryTools.assembleSourceQuery(dto.getSourceKeyword()));
postFilter.must(EsQueryTools.assembleSourceKeywordQuery(dto.getSourceKeyword()));
}
// hostKeyword
if (StringUtils.isNotEmpty(dto.getHostKeyword())) {
......@@ -3385,6 +3422,9 @@ public class MarkDataServiceImpl implements MarkDataService {
if (Objects.nonNull(aggTitle)){
postFilter.must(QueryBuilders.termQuery("agg_title.keyword", aggTitle));
}
if (Objects.isNull(aggTitle) && Objects.nonNull(dto.getAggTitle())){
postFilter.must(QueryBuilders.termQuery("agg_title.keyword", dto.getAggTitle()));
}
return postFilter;
}
......@@ -3910,5 +3950,4 @@ public class MarkDataServiceImpl implements MarkDataService {
}
return null;
}
}
\ No newline at end of file
src/main/java/com/zhiwei/brandkbs2/service/impl/ProjectServiceImpl.java
......@@ -12,8 +12,10 @@ import com.zhiwei.brandkbs2.exception.ExceptionCast;
import com.zhiwei.brandkbs2.model.CommonCodeEnum;
import com.zhiwei.brandkbs2.model.ResponseResult;
import com.zhiwei.brandkbs2.pojo.*;
import com.zhiwei.brandkbs2.pojo.dto.NonManualProjectPlanDTO;
import com.zhiwei.brandkbs2.pojo.vo.PageVO;
import com.zhiwei.brandkbs2.pojo.vo.ProjectVO;
import com.zhiwei.brandkbs2.service.MarkDataService;
import com.zhiwei.brandkbs2.service.ProjectService;
import com.zhiwei.brandkbs2.service.UserService;
import com.zhiwei.brandkbs2.util.MongoUtil;
......@@ -64,6 +66,9 @@ public class ProjectServiceImpl implements ProjectService {
@Resource(name = "userServiceImpl")
private UserService userService;
@Resource(name = "markDataServiceImpl")
private MarkDataService markDataService;
@Resource(name = "eventMiddlewareDao")
private EventMiddlewareDao eventMiddlewareDao;
......@@ -499,6 +504,8 @@ public class ProjectServiceImpl implements ProjectService {
return ResponseResult.failure(ProjectCodeEnum.PROJECT_EXISTSNAME_ERROR.message());
}
Project project = ProjectVO.createNonManualProject(projectVO);
// 添加时默认商业数据源
project.setExternalDataSource(true);
projectDao.insertOneWithoutId(project);
// 绑定事件标签，默认已绑定情感标签
eventMiddlewareDao.bindBrandkbs(project.getBrandLinkedGroup(), project.getBrandLinkedGroupId(), Collections.emptyList(),
......@@ -547,6 +554,7 @@ public class ProjectServiceImpl implements ProjectService {
result.put("isStart", project.isStart());
result.put("isShow", project.isShow());
result.put("dataBalance", project.getDataBalance());
result.put("externalDataSource", project.getExternalDataSource());
return result;
}).collect(Collectors.toList());
MongoUtil.PageHelper<JSONObject> pageHelper = mongoUtil.pageHelper(count, resList);
......@@ -635,6 +643,54 @@ public class ProjectServiceImpl implements ProjectService {
return ResponseResult.success(PageVO.createPageVo(total, page, pageSize, resList));
}
@Override
public ResponseResult switchExternalDataSource(String projectId) {
Project project = projectDao.findOneById(projectId);
if (Objects.isNull(project)) {
return ResponseResult.failure("项目不存在");
}
List<NonManualProjectPlan> planList = nonManualProjectPlanDao.findList(new Query(Criteria.where("projectId").is(project.getId())));
// 若商业数据切换为内部数据，此时同时删除数据订阅任务
if (project.getExternalDataSource()){
for (NonManualProjectPlan plan : planList) {
markDataService.deleteIStarShineSubjectTask(plan.getId());
}
}
// 若内部数据切换为商业数据，此时进行数据预估同时创建商业数据订阅任务
if (!project.getExternalDataSource()){
// 数据预估
for (NonManualProjectPlan plan : planList) {
Integer dataCount = markDataService.getPlanDataPrediction(plan.getKeyword(), plan.getFilterWord());
if (dataCount > 10000){
return ResponseResult.failure("方案：" + plan.getName() + ",预估数据量超过10000条/天,切换失败");
}
}
List<NonManualProjectPlan> successList = new ArrayList<>(planList.size());
// 添加商业数据订阅任务
for (NonManualProjectPlan plan : planList) {
String taskId = markDataService.addIStarShineSubjectTask(Tools.convertMap(plan, NonManualProjectPlanDTO.class));
if (Objects.isNull(taskId)){
for (NonManualProjectPlan successPlan : successList) {
markDataService.deleteIStarShineSubjectTask(successPlan.getId());
successPlan.setIStarShineSubjectTaskId(null);
successPlan.setUTime(System.currentTimeMillis());
nonManualProjectPlanDao.updateOne(successPlan);
}
return ResponseResult.failure("切换失败");
}
plan.setIStarShineSubjectTaskId(taskId);
plan.setUTime(System.currentTimeMillis());
nonManualProjectPlanDao.updateOne(plan);
successList.add(plan);
}
}
Update update = new Update();
update.set("uTime", System.currentTimeMillis());
update.set("externalDataSource", !project.getExternalDataSource());
projectDao.updateOneByIdWithField(projectId, update);
return ResponseResult.success();
}
/**
* 获取舆情对应项目的情感标签
* @param brandName
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment