Commit 90c9a6a5 by shenjunjie

Merge branch 'feature' into 'dev'

Feature

See merge request !520
parents 615eabd6 fd799dfb
......@@ -22,6 +22,11 @@ public class GenericAttribute {
// public static final String ES_INDEX_TEST = "brandkbs2_2022";
public static final String ES_CHANNEL_INDEX_TEST = "brandkbs2_channel_record_test";
public static final String ES_CHANNEL_INDEX_PREFIX = "brandkbs2_channel_record_";
public static final String ES_COMPLETE_TEXT_INDEX_PRE = "complete_text_";
public static final String ES_INCOMPLETE_TEXT_INDEX_PRE = "incomplete_text_";
public static final String ES_QA_TEXT_INDEX_PRE = "qa_text_";
public static final String ES_VIDEO_INDEX_PRE = "video_";
/**
* es ind_title
**/
......
......@@ -34,6 +34,8 @@ public class Constant {
public static final FastDateFormat ONLY_MONTH_FORMAT = FastDateFormat.getInstance("M月");
public static final FastDateFormat DF_yyyyMMdd = FastDateFormat.getInstance("yyyyMMdd");
public static final FastDateFormat DF_yyyyMM = FastDateFormat.getInstance("yyyyMM");
/**
* 自定义fid分隔符号
*/
......
......@@ -89,15 +89,17 @@ public class BehaviorController extends BaseController {
@ApiImplicitParam(name = "pageSize", value = "每页记录数", defaultValue = "10", paramType = "query", dataType = "int"),
@ApiImplicitParam(name = "startTime", value = "开始时间", required = true, paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "endTime", value = "结束时间", required = true, paramType = "query", dataType = "long"),
@ApiImplicitParam(name = "keyword", value = "关键词",defaultValue = "", paramType = "query", dataType = "string")
@ApiImplicitParam(name = "keyword", value = "关键词",defaultValue = "", paramType = "query", dataType = "string"),
@ApiImplicitParam(name = "roleId", value = "用户权限",defaultValue = "", paramType = "query", dataType = "int")
})
@GetMapping("/logRecord/list")
public ResponseResult findLogRecordList(@RequestParam(value = "page", defaultValue = "1") int page,
@RequestParam(value = "pageSize", defaultValue = "10") int pageSize,
@RequestParam("startTime") long startTime,
@RequestParam("endTime") long endTime,
@RequestParam(value = "keyword", defaultValue = "") String keyword) {
return ResponseResult.success(behaviorService.findLogRecordList(page, pageSize, startTime, endTime, keyword));
@RequestParam(value = "pageSize", defaultValue = "10") int pageSize,
@RequestParam("startTime") long startTime,
@RequestParam("endTime") long endTime,
@RequestParam(value = "keyword", defaultValue = "") String keyword,
@RequestParam(value = "roleId", required = false) Integer roleId) {
return ResponseResult.success(behaviorService.findLogRecordList(page, pageSize, startTime, endTime, keyword, roleId));
}
@ApiOperation("下载用户操作记录列表")
......
......@@ -553,6 +553,36 @@ public class AppArticleController extends BaseController {
return ResponseResult.success(markDataService.getDailyReportDetail(id));
}
@ApiOperation("原发溯源-首发信息")
@PostMapping("/search-whole/origin/first-article")
public ResponseResult getWholeSearchFirstArticle(@RequestBody MarkSearchDTO dto) {
return ResponseResult.success(markDataService.getWholeSearchFirstArticle(dto));
}
@ApiOperation("原发溯源-媒体参与")
@PostMapping("/search-whole/origin/media-info")
public ResponseResult getWholeSearchMediaParticipation(@RequestBody MarkSearchDTO dto) {
return ResponseResult.success(markDataService.getWholeSearchMediaParticipation(dto));
}
@ApiOperation("原发溯源-发布节点")
@PostMapping("/search-whole/origin/article-point")
public ResponseResult getWholeSearchArticlePoint(@RequestBody MarkSearchDTO dto) {
return ResponseResult.success(markDataService.getWholeSearchArticlePoint(dto));
}
@ApiOperation("原发溯源-平台分布")
@PostMapping("/search-whole/origin/platform-percent")
public ResponseResult getWholeSearchPlatformPercentage(@RequestBody MarkSearchDTO dto) {
return ResponseResult.success(markDataService.getWholeSearchPlatformPercentage(dto));
}
@ApiOperation("原发溯源-发文列表")
@PostMapping("/search-whole/origin/articles")
public ResponseResult getWholeSearchArticleList(@RequestBody MarkSearchDTO dto) {
return ResponseResult.success(markDataService.getWholeSearchArticleList(dto));
}
private boolean checkMTagIllegal(StringBuilder mtag) {
List<MarkerTag> hitTags = projectService.getProjectById(UserThreadLocal.getProjectId()).getHitTags();
if (!Tools.isEmpty(hitTags)) {
......
......@@ -7,6 +7,7 @@ import com.zhiwei.brandkbs2.pojo.ChannelIndex;
import com.zhiwei.brandkbs2.util.Tools;
import lombok.Getter;
import lombok.Setter;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.commons.lang3.time.FastDateFormat;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.logging.log4j.LogManager;
......@@ -29,6 +30,8 @@ import org.elasticsearch.search.aggregations.AggregationBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.sort.FieldSortBuilder;
import org.joda.time.Period;
import org.joda.time.PeriodType;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.retry.support.RetryTemplate;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
......@@ -195,6 +198,10 @@ public class EsClientDao {
return getIndexList().toArray(new String[0]);
}
public String[] getAllIndexes(Long startTime, Long endTime) {
return getAllIndexList(startTime, endTime).toArray(new String[0]);
}
protected RestHighLevelClient getEsClient() {
return esClient;
}
......@@ -214,6 +221,25 @@ public class EsClientDao {
return res;
}
public List<String> getAllIndexList(Long start, Long end) {
List<String> list = new ArrayList<>();
Date endTime = new Date(end);
Date startTime = new Date(start);
String pattern = "yyyyMM";
startTime = Tools.truncDate(startTime, pattern);
endTime = Tools.truncDate(endTime, pattern);
Period period = new Period(startTime.getTime(), endTime.getTime(), PeriodType.months());
int mouths = period.getMonths();
for (int i = 0; i <= mouths; i++) {
String suffix = Constant.DF_yyyyMM.format(DateUtils.addMonths(startTime, i));
list.add(GenericAttribute.ES_COMPLETE_TEXT_INDEX_PRE + suffix);
list.add(GenericAttribute.ES_INCOMPLETE_TEXT_INDEX_PRE + suffix);
list.add(GenericAttribute.ES_QA_TEXT_INDEX_PRE + suffix);
list.add(GenericAttribute.ES_VIDEO_INDEX_PRE + suffix);
}
return list;
}
// private SearchSourceBuilder addSort(SearchSourceBuilder builder, String sorter) {
// if (StringUtils.isNotEmpty(sorter)) {
// for (Map.Entry<String, Object> entry : JSONObject.parseObject(sorter).entrySet()) {
......
......@@ -65,7 +65,7 @@ public interface BehaviorService {
* @param nickname 用户昵称
* @return 用户行为列表
*/
PageVO<UserLogRecord> findLogRecordList(int page, int size, long startTime, long endTime, String nickname);
PageVO<UserLogRecord> findLogRecordList(int page, int size, long startTime, long endTime, String nickname, Integer roleId);
/**
* 下载用户操作记录列表
......
......@@ -715,4 +715,39 @@ public interface MarkDataService {
* @return
*/
DailyReport getDailyReportDetail(String id);
/**
* 获取原发溯源-首发信息
* @param dto 全网搜数据搜索传输类
* @return
*/
MarkFlowEntity getWholeSearchFirstArticle(MarkSearchDTO dto);
/**
* 获取原发溯源-平台分布
* @param dto 全网搜数据搜索传输类
* @return
*/
List<JSONObject> getWholeSearchMediaParticipation(MarkSearchDTO dto);
/**
* 获取原发溯源-发布节点
* @param dto 全网搜数据搜索传输类
* @return
*/
List<JSONObject> getWholeSearchArticlePoint(MarkSearchDTO dto);
/**
* 获取原发溯源-平台分布
* @param dto 全网搜数据搜索传输类
* @return
*/
List<JSONObject> getWholeSearchPlatformPercentage(MarkSearchDTO dto);
/**
* 获取原发溯源-发文列表
* @param dto 全网搜数据搜索传输类
* @return
*/
PageVO<JSONObject> getWholeSearchArticleList(MarkSearchDTO dto);
}
......@@ -3,6 +3,7 @@ package com.zhiwei.brandkbs2.service;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.base.entity.subclass.mark.MarkInfo;
import com.zhiwei.brandkbs2.pojo.MarkFlowEntity;
import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
import com.zhiwei.brandkbs2.pojo.dto.TagFilterDTO;
import lombok.AllArgsConstructor;
import lombok.Data;
......@@ -74,6 +75,14 @@ public interface MarkFlowService {
*/
MarkFlowEntity getShotPageFromCache(String id);
/**
* 获取大库数据实体
* @param json
* @param dto
* @return
*/
MarkFlowEntity createAllIndexFlowInfo(JSONObject json, MarkSearchDTO dto);
@Data
@AllArgsConstructor
class MarkInfoSource {
......
......@@ -181,7 +181,7 @@ public class BehaviorServiceImpl implements BehaviorService {
}
@Override
public PageVO<UserLogRecord> findLogRecordList(int page, int size, long startTime, long endTime, String nickname) {
public PageVO<UserLogRecord> findLogRecordList(int page, int size, long startTime, long endTime, String nickname, Integer roleId) {
Date start = new Date(startTime);
Date end = new Date(endTime);
String projectId = UserThreadLocal.getProjectId();
......@@ -193,6 +193,9 @@ public class BehaviorServiceImpl implements BehaviorService {
if (StringUtils.isNotEmpty(nickname)) {
userLogRecordDao.addKeywordFuzz(query, nickname, "nickname");
}
if (Objects.nonNull(roleId)){
query.addCriteria(Criteria.where("roleId").is(roleId));
}
long count = userLogRecordDao.count(query, collectionNames);
List<UserLogRecord> behaviorList = new ArrayList<>();
int pageSizeFlag = size;
......
......@@ -10,14 +10,14 @@ import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import com.zhiwei.brandkbs2.common.GenericAttribute;
import com.zhiwei.brandkbs2.common.GlobalPojo;
import com.zhiwei.brandkbs2.config.Constant;
import com.zhiwei.brandkbs2.dao.HighlightWordDao;
import com.zhiwei.brandkbs2.enmus.ChannelEmotion;
import com.zhiwei.brandkbs2.exception.ExceptionCast;
import com.zhiwei.brandkbs2.model.CommonCodeEnum;
import com.zhiwei.brandkbs2.pojo.MarkFlowEntity;
import com.zhiwei.brandkbs2.pojo.SensitiveChannel;
import com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO;
import com.zhiwei.brandkbs2.pojo.dto.TagFilterDTO;
import com.zhiwei.brandkbs2.service.MarkFlowService;
import com.zhiwei.brandkbs2.service.TagFilterService;
import com.zhiwei.brandkbs2.util.RedisUtil;
import com.zhiwei.brandkbs2.util.Tools;
import com.zhiwei.middleware.automaticmark.vo.Keyword;
......@@ -27,6 +27,7 @@ import com.zhiwei.middleware.mark.vo.QueryResult;
import com.zhiwei.qbjc.bean.pojo.common.Tag;
import com.zhiwei.qbjc.bean.tools.BeanTools;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.stereotype.Service;
......@@ -197,6 +198,63 @@ public class MarkFlowServiceImpl implements MarkFlowService {
return JSON.parseObject(data, MarkFlowEntity.class);
}
@Override
public MarkFlowEntity createAllIndexFlowInfo(JSONObject json, MarkSearchDTO dto) {
MarkFlowEntity markFlowEntity = new MarkFlowEntity(json);
JSONObject info = new JSONObject();
info.put("sourceDetails", getSourceDetails(json));
info.put("startTime", dto.getStartTime());
info.put("endTime", dto.getEndTime());
info.put("highlightWordMap", getHighlightWordMap(UserThreadLocal.getProjectId(), json));
markFlowEntity.setInfo(info);
return markFlowEntity;
}
/**
* 底层大库数据SourceDetails提取
* @param tJson
* @return
*/
private JSONObject getSourceDetails(JSONObject tJson){
JSONObject sourceDetails = new JSONObject();
String source = tJson.getString(GenericAttribute.ES_SOURCE);
String rootSource = tJson.getString(GenericAttribute.ES_ROOT_SOURCE);
// 是否原创
if (tJson.containsKey(GenericAttribute.ES_ROOT_SOURCE) && (rootSource.contains(source) || source.contains(rootSource))) {
sourceDetails.put("rootPublish", "原创");
}
// C4,realSource提取展示
sourceDetails.put("clientFrom", Tools.isEmpty(tJson.getString(GenericAttribute.ES_REAL_SOURCE)) ? ClassD.TypeD.fromEncode(tJson.getIntValue(GenericAttribute.ES_C4)).title() : tJson.getString(GenericAttribute.ES_REAL_SOURCE));
// platform
sourceDetails.put("platform", BeanTools.filterPlatform(GlobalPojo.PLATFORMS, tJson.getIntValue(GenericAttribute.ES_C5), tJson.getIntValue(GenericAttribute.ES_FOREIGN)).getName());
// source
sourceDetails.put("source", source);
// 粉丝量提取
long followersNum = tJson.getLongValue("followers_num");
if (followersNum > 0) {
sourceDetails.put("followersNum", followersNum);
}
// 渠道标签
SensitiveChannel sensitiveChannel = GlobalPojo.COMMON_SENSITIVE_CHANNEL.get(source);
if (Objects.nonNull(sensitiveChannel)) {
List<String> channelTagStr = new ArrayList<>(4);
if (StringUtils.isNotBlank(sensitiveChannel.getPoliticsLevel())){
channelTagStr.add(sensitiveChannel.getPoliticsLevel());
}
if (StringUtils.isNotBlank(sensitiveChannel.getMainBodyType())){
channelTagStr.add(sensitiveChannel.getMainBodyType());
}
if (StringUtils.isNotBlank(sensitiveChannel.getField())){
channelTagStr.add(sensitiveChannel.getField());
}
if (StringUtils.isNotBlank(sensitiveChannel.getRegion())){
channelTagStr.add(sensitiveChannel.getRegion());
}
sourceDetails.put("channelTag", StringUtils.join(channelTagStr, "|"));
}
return sourceDetails;
}
/**
* 获取自定义标签
* @param markInfoSource
......
......@@ -33,6 +33,7 @@ import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.context.request.RequestContextHolder;
import org.springframework.web.context.request.ServletRequestAttributes;
......@@ -90,12 +91,18 @@ public class ProjectServiceImpl implements ProjectService {
@Value("${jwt.key}")
private String jwtKey;
@Value("${istarshine.subject.updateTaskStatus.url}")
private String updateIStarShineSubjectTaskStatusUrl;
@Value("${brandkbs.file.url}")
private String brandkbsFileUrl;
@Value("${brandkbs.image.url}")
private String brandkbsImageUrl;
@Resource(name = "restTemplate")
private RestTemplate restTemplate;
@Override
public ResponseResult addProject(ProjectVO projectVO) {
if (Objects.isNull(projectVO)) {
......@@ -284,12 +291,30 @@ public class ProjectServiceImpl implements ProjectService {
if (Objects.isNull(project)) {
ExceptionCast.cast(CommonCodeEnum.INVALID_PARAM, "项目不存在");
}
// 对开启了商业数据订阅的非人工项目
if (!project.isManual() && project.getExternalDataSource()){
List<NonManualProjectPlan> planList = nonManualProjectPlanDao.findList(new Query(Criteria.where("projectId").is(project.getId())));
// 启动->终止:暂停商业数据订阅任务 终止->启动:启动商业数据订阅任务
planList.forEach(plan -> updateStarShineSubjectTaskStatus(plan.getIStarShineSubjectTaskId()));
}
Update update = new Update();
update.set("uTime", System.currentTimeMillis());
update.set("isStart", !project.isStart());
projectDao.updateOneByIdWithField(pid, update);
}
/**
* 修改快搜任务状态
* @param taskId
* @return
*/
private void updateStarShineSubjectTaskStatus(String taskId){
JSONObject body = restTemplate.getForEntity(updateIStarShineSubjectTaskStatusUrl, JSONObject.class, taskId, UserThreadLocal.getNickname()).getBody();
if (Objects.isNull(body) || body.isEmpty() || !body.getBoolean("status")){
log.error("快搜任务id:" + taskId + "-修改快搜订阅任务状态失败,接口返回信息-" + body);
}
}
@Override
public ResponseResult getUserAllProjects() {
User user = userDao.findOneById(UserThreadLocal.getUserId());
......@@ -676,6 +701,9 @@ public class ProjectServiceImpl implements ProjectService {
List<NonManualProjectPlan> successList = new ArrayList<>(planList.size());
// 添加商业数据订阅任务
for (NonManualProjectPlan plan : planList) {
if (CollectionUtils.isEmpty(plan.getType())){
plan.setType(new ArrayList<>(Arrays.asList("原创", "转发", "OCR")));
}
String taskId = markDataService.addIStarShineSubjectTask(Tools.convertMap(plan, NonManualProjectPlanDTO.class));
if (Objects.isNull(taskId)){
for (NonManualProjectPlan successPlan : successList) {
......
......@@ -132,4 +132,5 @@ toolset.interactionResult.url=https://zhiwei-middleware.zhiweidata.com/interact/
#\u5916\u90E8\u6570\u636E-\u5FEB\u641C\u8BA2\u9605\u670D\u52A1\u63A5\u53E3
istarshine.subject.addTask.url=https://istarshine-service.zhiweidata.com/istarshine/subject/addIStarShineSubjectTask
istarshine.subject.deleteTask.url=https://istarshine-service.zhiweidata.com/istarshine/subject/deleteIStarShineSubjectTask?taskId={1}&updateUserName={2}
istarshine.subject.updateTask.url=https://istarshine-service.zhiweidata.com/istarshine/subject/updateIStarShineSubjectTask
\ No newline at end of file
istarshine.subject.updateTask.url=https://istarshine-service.zhiweidata.com/istarshine/subject/updateIStarShineSubjectTask
istarshine.subject.updateTaskStatus.url=https://istarshine-service.zhiweidata.com/istarshine/subject/updateIStarShineSubjectTaskStatus?taskId={1}&updateUserName={2}
\ No newline at end of file
......@@ -112,7 +112,7 @@ ef.checkCaptcha.url=https://ef.zhiweidata.com/captchaApi/checkeCaptcha.do.do?id=
#\u5168\u7F51\u641C\u5916\u90E8\u63A5\u53E3
istarshine.addIStarShineKSData.url=https://istarshine-service.zhiweidata.com/istarshine/ks/addIStarShineKSData
istarshine.getIStarShineKSInfoData.url=https://istarshine-service.zhiweidata.com/istarshine/ks/getIStarShineKSInfoData?taskId={1}&appId=6183571e0d710000f6003a12
whole.search.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/search/yuqing
whole.search.url=https://yuqing.test.zhiweidata.com/qbjcbackPhoenix/interface/middleware/search/yuqing
whole.searchCriteria.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/search/pt
whole.extraParam.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/search/extra/param
brief.report.url=http://192.168.0.79:11000/qbjcbackPhoenix/interface/middleware/brief-report-web
......@@ -142,4 +142,5 @@ toolset.interactionResult.url=https://zhiwei-middleware.zhiweidata.com/interact/
#\u5916\u90E8\u6570\u636E-\u5FEB\u641C\u8BA2\u9605\u670D\u52A1\u63A5\u53E3
istarshine.subject.addTask.url=https://istarshine-service.zhiweidata.com/istarshine/subject/addIStarShineSubjectTask
istarshine.subject.deleteTask.url=https://istarshine-service.zhiweidata.com/istarshine/subject/deleteIStarShineSubjectTask?taskId={1}&updateUserName={2}
istarshine.subject.updateTask.url=https://istarshine-service.zhiweidata.com/istarshine/subject/updateIStarShineSubjectTask
\ No newline at end of file
istarshine.subject.updateTask.url=https://istarshine-service.zhiweidata.com/istarshine/subject/updateIStarShineSubjectTask
istarshine.subject.updateTaskStatus.url=https://istarshine-service.zhiweidata.com/istarshine/subject/updateIStarShineSubjectTaskStatus?taskId={1}&updateUserName={2}
\ No newline at end of file
......@@ -102,7 +102,7 @@ ef.checkCaptcha.url=https://ef.zhiweidata.com/captchaApi/checkeCaptcha.do.do?id=
#\u5168\u7F51\u641C\u5916\u90E8\u63A5\u53E3
istarshine.addIStarShineKSData.url=https://istarshine-service.zhiweidata.com/istarshine/ks/addIStarShineKSData
istarshine.getIStarShineKSInfoData.url=https://istarshine-service.zhiweidata.com/istarshine/ks/getIStarShineKSInfoData?taskId={1}&appId=6183571e0d710000f6003a12
whole.search.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/search/yuqing
whole.search.url=https://yuqing.test.zhiweidata.com/qbjcbackPhoenix/interface/middleware/search/yuqing
whole.searchCriteria.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/search/pt
whole.extraParam.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/search/extra/param
brief.report.url=https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/brief-report-web
......@@ -132,4 +132,5 @@ toolset.interactionResult.url=https://zhiwei-middleware.zhiweidata.com/interact/
#\u5916\u90E8\u6570\u636E-\u5FEB\u641C\u8BA2\u9605\u670D\u52A1\u63A5\u53E3
istarshine.subject.addTask.url=https://istarshine-service.zhiweidata.com/istarshine/subject/addIStarShineSubjectTask
istarshine.subject.deleteTask.url=https://istarshine-service.zhiweidata.com/istarshine/subject/deleteIStarShineSubjectTask?taskId={1}&updateUserName={2}
istarshine.subject.updateTask.url=https://istarshine-service.zhiweidata.com/istarshine/subject/updateIStarShineSubjectTask
\ No newline at end of file
istarshine.subject.updateTask.url=https://istarshine-service.zhiweidata.com/istarshine/subject/updateIStarShineSubjectTask
istarshine.subject.updateTaskStatus.url=https://istarshine-service.zhiweidata.com/istarshine/subject/updateIStarShineSubjectTaskStatus?taskId={1}&updateUserName={2}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment