Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
B
brandkbs2
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
shenjunjie
brandkbs2
Commits
90c9a6a5
Commit
90c9a6a5
authored
Jun 03, 2024
by
shenjunjie
Browse files
Options
Browse Files
Download
Plain Diff
Merge branch 'feature' into 'dev'
Feature See merge request
!520
parents
615eabd6
fd799dfb
Show whitespace changes
Inline
Side-by-side
Showing
15 changed files
with
554 additions
and
22 deletions
+554
-22
src/main/java/com/zhiwei/brandkbs2/common/GenericAttribute.java
+5
-0
src/main/java/com/zhiwei/brandkbs2/config/Constant.java
+2
-0
src/main/java/com/zhiwei/brandkbs2/controller/admin/BehaviorController.java
+5
-3
src/main/java/com/zhiwei/brandkbs2/controller/app/AppArticleController.java
+30
-0
src/main/java/com/zhiwei/brandkbs2/es/EsClientDao.java
+26
-0
src/main/java/com/zhiwei/brandkbs2/service/BehaviorService.java
+1
-1
src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java
+35
-0
src/main/java/com/zhiwei/brandkbs2/service/MarkFlowService.java
+9
-0
src/main/java/com/zhiwei/brandkbs2/service/impl/BehaviorServiceImpl.java
+4
-1
src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java
+341
-13
src/main/java/com/zhiwei/brandkbs2/service/impl/MarkFlowServiceImpl.java
+60
-2
src/main/java/com/zhiwei/brandkbs2/service/impl/ProjectServiceImpl.java
+28
-0
src/main/resources/application-dev.properties
+2
-0
src/main/resources/application-local.properties
+3
-1
src/main/resources/application-prod.properties
+3
-1
No files found.
src/main/java/com/zhiwei/brandkbs2/common/GenericAttribute.java
View file @
90c9a6a5
...
...
@@ -22,6 +22,11 @@ public class GenericAttribute {
// public static final String ES_INDEX_TEST = "brandkbs2_2022";
public
static
final
String
ES_CHANNEL_INDEX_TEST
=
"brandkbs2_channel_record_test"
;
public
static
final
String
ES_CHANNEL_INDEX_PREFIX
=
"brandkbs2_channel_record_"
;
public
static
final
String
ES_COMPLETE_TEXT_INDEX_PRE
=
"complete_text_"
;
public
static
final
String
ES_INCOMPLETE_TEXT_INDEX_PRE
=
"incomplete_text_"
;
public
static
final
String
ES_QA_TEXT_INDEX_PRE
=
"qa_text_"
;
public
static
final
String
ES_VIDEO_INDEX_PRE
=
"video_"
;
/**
* es ind_title
**/
...
...
src/main/java/com/zhiwei/brandkbs2/config/Constant.java
View file @
90c9a6a5
...
...
@@ -34,6 +34,8 @@ public class Constant {
public
static
final
FastDateFormat
ONLY_MONTH_FORMAT
=
FastDateFormat
.
getInstance
(
"M月"
);
public
static
final
FastDateFormat
DF_yyyyMMdd
=
FastDateFormat
.
getInstance
(
"yyyyMMdd"
);
public
static
final
FastDateFormat
DF_yyyyMM
=
FastDateFormat
.
getInstance
(
"yyyyMM"
);
/**
* 自定义fid分隔符号
*/
...
...
src/main/java/com/zhiwei/brandkbs2/controller/admin/BehaviorController.java
View file @
90c9a6a5
...
...
@@ -89,15 +89,17 @@ public class BehaviorController extends BaseController {
@ApiImplicitParam
(
name
=
"pageSize"
,
value
=
"每页记录数"
,
defaultValue
=
"10"
,
paramType
=
"query"
,
dataType
=
"int"
),
@ApiImplicitParam
(
name
=
"startTime"
,
value
=
"开始时间"
,
required
=
true
,
paramType
=
"query"
,
dataType
=
"long"
),
@ApiImplicitParam
(
name
=
"endTime"
,
value
=
"结束时间"
,
required
=
true
,
paramType
=
"query"
,
dataType
=
"long"
),
@ApiImplicitParam
(
name
=
"keyword"
,
value
=
"关键词"
,
defaultValue
=
""
,
paramType
=
"query"
,
dataType
=
"string"
)
@ApiImplicitParam
(
name
=
"keyword"
,
value
=
"关键词"
,
defaultValue
=
""
,
paramType
=
"query"
,
dataType
=
"string"
),
@ApiImplicitParam
(
name
=
"roleId"
,
value
=
"用户权限"
,
defaultValue
=
""
,
paramType
=
"query"
,
dataType
=
"int"
)
})
@GetMapping
(
"/logRecord/list"
)
public
ResponseResult
findLogRecordList
(
@RequestParam
(
value
=
"page"
,
defaultValue
=
"1"
)
int
page
,
@RequestParam
(
value
=
"pageSize"
,
defaultValue
=
"10"
)
int
pageSize
,
@RequestParam
(
"startTime"
)
long
startTime
,
@RequestParam
(
"endTime"
)
long
endTime
,
@RequestParam
(
value
=
"keyword"
,
defaultValue
=
""
)
String
keyword
)
{
return
ResponseResult
.
success
(
behaviorService
.
findLogRecordList
(
page
,
pageSize
,
startTime
,
endTime
,
keyword
));
@RequestParam
(
value
=
"keyword"
,
defaultValue
=
""
)
String
keyword
,
@RequestParam
(
value
=
"roleId"
,
required
=
false
)
Integer
roleId
)
{
return
ResponseResult
.
success
(
behaviorService
.
findLogRecordList
(
page
,
pageSize
,
startTime
,
endTime
,
keyword
,
roleId
));
}
@ApiOperation
(
"下载用户操作记录列表"
)
...
...
src/main/java/com/zhiwei/brandkbs2/controller/app/AppArticleController.java
View file @
90c9a6a5
...
...
@@ -553,6 +553,36 @@ public class AppArticleController extends BaseController {
return
ResponseResult
.
success
(
markDataService
.
getDailyReportDetail
(
id
));
}
@ApiOperation
(
"原发溯源-首发信息"
)
@PostMapping
(
"/search-whole/origin/first-article"
)
public
ResponseResult
getWholeSearchFirstArticle
(
@RequestBody
MarkSearchDTO
dto
)
{
return
ResponseResult
.
success
(
markDataService
.
getWholeSearchFirstArticle
(
dto
));
}
@ApiOperation
(
"原发溯源-媒体参与"
)
@PostMapping
(
"/search-whole/origin/media-info"
)
public
ResponseResult
getWholeSearchMediaParticipation
(
@RequestBody
MarkSearchDTO
dto
)
{
return
ResponseResult
.
success
(
markDataService
.
getWholeSearchMediaParticipation
(
dto
));
}
@ApiOperation
(
"原发溯源-发布节点"
)
@PostMapping
(
"/search-whole/origin/article-point"
)
public
ResponseResult
getWholeSearchArticlePoint
(
@RequestBody
MarkSearchDTO
dto
)
{
return
ResponseResult
.
success
(
markDataService
.
getWholeSearchArticlePoint
(
dto
));
}
@ApiOperation
(
"原发溯源-平台分布"
)
@PostMapping
(
"/search-whole/origin/platform-percent"
)
public
ResponseResult
getWholeSearchPlatformPercentage
(
@RequestBody
MarkSearchDTO
dto
)
{
return
ResponseResult
.
success
(
markDataService
.
getWholeSearchPlatformPercentage
(
dto
));
}
@ApiOperation
(
"原发溯源-发文列表"
)
@PostMapping
(
"/search-whole/origin/articles"
)
public
ResponseResult
getWholeSearchArticleList
(
@RequestBody
MarkSearchDTO
dto
)
{
return
ResponseResult
.
success
(
markDataService
.
getWholeSearchArticleList
(
dto
));
}
private
boolean
checkMTagIllegal
(
StringBuilder
mtag
)
{
List
<
MarkerTag
>
hitTags
=
projectService
.
getProjectById
(
UserThreadLocal
.
getProjectId
()).
getHitTags
();
if
(!
Tools
.
isEmpty
(
hitTags
))
{
...
...
src/main/java/com/zhiwei/brandkbs2/es/EsClientDao.java
View file @
90c9a6a5
...
...
@@ -7,6 +7,7 @@ import com.zhiwei.brandkbs2.pojo.ChannelIndex;
import
com.zhiwei.brandkbs2.util.Tools
;
import
lombok.Getter
;
import
lombok.Setter
;
import
org.apache.commons.lang3.time.DateUtils
;
import
org.apache.commons.lang3.time.FastDateFormat
;
import
org.apache.commons.lang3.tuple.Pair
;
import
org.apache.logging.log4j.LogManager
;
...
...
@@ -29,6 +30,8 @@ import org.elasticsearch.search.aggregations.AggregationBuilder;
import
org.elasticsearch.search.builder.SearchSourceBuilder
;
import
org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder
;
import
org.elasticsearch.search.sort.FieldSortBuilder
;
import
org.joda.time.Period
;
import
org.joda.time.PeriodType
;
import
org.springframework.beans.factory.annotation.Value
;
import
org.springframework.retry.support.RetryTemplate
;
import
org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor
;
...
...
@@ -195,6 +198,10 @@ public class EsClientDao {
return
getIndexList
().
toArray
(
new
String
[
0
]);
}
public
String
[]
getAllIndexes
(
Long
startTime
,
Long
endTime
)
{
return
getAllIndexList
(
startTime
,
endTime
).
toArray
(
new
String
[
0
]);
}
protected
RestHighLevelClient
getEsClient
()
{
return
esClient
;
}
...
...
@@ -214,6 +221,25 @@ public class EsClientDao {
return
res
;
}
public
List
<
String
>
getAllIndexList
(
Long
start
,
Long
end
)
{
List
<
String
>
list
=
new
ArrayList
<>();
Date
endTime
=
new
Date
(
end
);
Date
startTime
=
new
Date
(
start
);
String
pattern
=
"yyyyMM"
;
startTime
=
Tools
.
truncDate
(
startTime
,
pattern
);
endTime
=
Tools
.
truncDate
(
endTime
,
pattern
);
Period
period
=
new
Period
(
startTime
.
getTime
(),
endTime
.
getTime
(),
PeriodType
.
months
());
int
mouths
=
period
.
getMonths
();
for
(
int
i
=
0
;
i
<=
mouths
;
i
++)
{
String
suffix
=
Constant
.
DF_yyyyMM
.
format
(
DateUtils
.
addMonths
(
startTime
,
i
));
list
.
add
(
GenericAttribute
.
ES_COMPLETE_TEXT_INDEX_PRE
+
suffix
);
list
.
add
(
GenericAttribute
.
ES_INCOMPLETE_TEXT_INDEX_PRE
+
suffix
);
list
.
add
(
GenericAttribute
.
ES_QA_TEXT_INDEX_PRE
+
suffix
);
list
.
add
(
GenericAttribute
.
ES_VIDEO_INDEX_PRE
+
suffix
);
}
return
list
;
}
// private SearchSourceBuilder addSort(SearchSourceBuilder builder, String sorter) {
// if (StringUtils.isNotEmpty(sorter)) {
// for (Map.Entry<String, Object> entry : JSONObject.parseObject(sorter).entrySet()) {
...
...
src/main/java/com/zhiwei/brandkbs2/service/BehaviorService.java
View file @
90c9a6a5
...
...
@@ -65,7 +65,7 @@ public interface BehaviorService {
* @param nickname 用户昵称
* @return 用户行为列表
*/
PageVO
<
UserLogRecord
>
findLogRecordList
(
int
page
,
int
size
,
long
startTime
,
long
endTime
,
String
nickname
);
PageVO
<
UserLogRecord
>
findLogRecordList
(
int
page
,
int
size
,
long
startTime
,
long
endTime
,
String
nickname
,
Integer
roleId
);
/**
* 下载用户操作记录列表
...
...
src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java
View file @
90c9a6a5
...
...
@@ -715,4 +715,39 @@ public interface MarkDataService {
* @return
*/
DailyReport
getDailyReportDetail
(
String
id
);
/**
* 获取原发溯源-首发信息
* @param dto 全网搜数据搜索传输类
* @return
*/
MarkFlowEntity
getWholeSearchFirstArticle
(
MarkSearchDTO
dto
);
/**
* 获取原发溯源-平台分布
* @param dto 全网搜数据搜索传输类
* @return
*/
List
<
JSONObject
>
getWholeSearchMediaParticipation
(
MarkSearchDTO
dto
);
/**
* 获取原发溯源-发布节点
* @param dto 全网搜数据搜索传输类
* @return
*/
List
<
JSONObject
>
getWholeSearchArticlePoint
(
MarkSearchDTO
dto
);
/**
* 获取原发溯源-平台分布
* @param dto 全网搜数据搜索传输类
* @return
*/
List
<
JSONObject
>
getWholeSearchPlatformPercentage
(
MarkSearchDTO
dto
);
/**
* 获取原发溯源-发文列表
* @param dto 全网搜数据搜索传输类
* @return
*/
PageVO
<
JSONObject
>
getWholeSearchArticleList
(
MarkSearchDTO
dto
);
}
src/main/java/com/zhiwei/brandkbs2/service/MarkFlowService.java
View file @
90c9a6a5
...
...
@@ -3,6 +3,7 @@ package com.zhiwei.brandkbs2.service;
import
com.alibaba.fastjson.JSONObject
;
import
com.zhiwei.base.entity.subclass.mark.MarkInfo
;
import
com.zhiwei.brandkbs2.pojo.MarkFlowEntity
;
import
com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO
;
import
com.zhiwei.brandkbs2.pojo.dto.TagFilterDTO
;
import
lombok.AllArgsConstructor
;
import
lombok.Data
;
...
...
@@ -74,6 +75,14 @@ public interface MarkFlowService {
*/
MarkFlowEntity
getShotPageFromCache
(
String
id
);
/**
* 获取大库数据实体
* @param json
* @param dto
* @return
*/
MarkFlowEntity
createAllIndexFlowInfo
(
JSONObject
json
,
MarkSearchDTO
dto
);
@Data
@AllArgsConstructor
class
MarkInfoSource
{
...
...
src/main/java/com/zhiwei/brandkbs2/service/impl/BehaviorServiceImpl.java
View file @
90c9a6a5
...
...
@@ -181,7 +181,7 @@ public class BehaviorServiceImpl implements BehaviorService {
}
@Override
public
PageVO
<
UserLogRecord
>
findLogRecordList
(
int
page
,
int
size
,
long
startTime
,
long
endTime
,
String
nickname
)
{
public
PageVO
<
UserLogRecord
>
findLogRecordList
(
int
page
,
int
size
,
long
startTime
,
long
endTime
,
String
nickname
,
Integer
roleId
)
{
Date
start
=
new
Date
(
startTime
);
Date
end
=
new
Date
(
endTime
);
String
projectId
=
UserThreadLocal
.
getProjectId
();
...
...
@@ -193,6 +193,9 @@ public class BehaviorServiceImpl implements BehaviorService {
if
(
StringUtils
.
isNotEmpty
(
nickname
))
{
userLogRecordDao
.
addKeywordFuzz
(
query
,
nickname
,
"nickname"
);
}
if
(
Objects
.
nonNull
(
roleId
)){
query
.
addCriteria
(
Criteria
.
where
(
"roleId"
).
is
(
roleId
));
}
long
count
=
userLogRecordDao
.
count
(
query
,
collectionNames
);
List
<
UserLogRecord
>
behaviorList
=
new
ArrayList
<>();
int
pageSizeFlag
=
size
;
...
...
src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java
View file @
90c9a6a5
...
...
@@ -36,6 +36,7 @@ import com.zhiwei.brandkbs2.util.RedisUtil;
import
com.zhiwei.brandkbs2.util.TextUtil
;
import
com.zhiwei.brandkbs2.util.Tools
;
import
com.zhiwei.qbjc.bean.pojo.common.MessagePlatform
;
import
com.zhiwei.qbjc.bean.tools.BeanTools
;
import
org.apache.commons.collections4.CollectionUtils
;
import
org.apache.commons.collections4.ListUtils
;
import
org.apache.commons.lang3.StringUtils
;
...
...
@@ -1914,6 +1915,10 @@ public class MarkDataServiceImpl implements MarkDataService {
NonManualProjectPlan
plan
=
new
NonManualProjectPlan
(
dto
);
Project
project
=
projectService
.
getProjectById
(
projectId
);
if
(
project
.
getExternalDataSource
())
{
// 未填默认选全部
if
(
CollectionUtils
.
isEmpty
(
dto
.
getType
())){
dto
.
setType
(
new
ArrayList
<>(
Arrays
.
asList
(
"原创"
,
"转发"
,
"OCR"
)));
}
// 同时添加快搜订阅任务
String
iStarShineSubjectTaskId
=
addIStarShineSubjectTask
(
dto
);
if
(
Objects
.
isNull
(
iStarShineSubjectTaskId
))
{
...
...
@@ -1966,6 +1971,10 @@ public class MarkDataServiceImpl implements MarkDataService {
Project
project
=
projectService
.
getProjectById
(
UserThreadLocal
.
getProjectId
());
// 商业数据源同时修改快搜订阅任务
if
(
project
.
getExternalDataSource
()){
// 未填默认选全部
if
(
CollectionUtils
.
isEmpty
(
dto
.
getType
())){
dto
.
setType
(
new
ArrayList
<>(
Arrays
.
asList
(
"原创"
,
"转发"
,
"OCR"
)));
}
ResponseResult
responseResult
=
updateIStarShineSubjectTask
(
dto
);
if
(!
responseResult
.
isStatus
()){
return
responseResult
;
...
...
@@ -1995,7 +2004,21 @@ public class MarkDataServiceImpl implements MarkDataService {
if
(!
dto
.
getLasting
()
&&
Objects
.
nonNull
(
dto
.
getEndTime
()))
{
param
.
put
(
"stopTime"
,
dto
.
getEndTime
());
}
// 只选ocr
if
(
dto
.
getType
().
size
()
==
1
&&
dto
.
getType
().
contains
(
"OCR"
)){
param
.
put
(
"ocrType"
,
2
);
}
// 未选ocr
if
(!
dto
.
getType
().
contains
(
"OCR"
)){
param
.
put
(
"filterWtype"
,
String
.
join
(
","
,
dto
.
getType
()));
param
.
put
(
"ocrType"
,
1
);
}
// 选了ocr+其他
if
(
dto
.
getType
().
contains
(
"OCR"
)
&&
dto
.
getType
().
size
()
!=
1
){
dto
.
getType
().
remove
(
"OCR"
);
param
.
put
(
"filterWtype"
,
String
.
join
(
","
,
dto
.
getType
()));
param
.
put
(
"ocrType"
,
3
);
}
HttpHeaders
httpHeaders
=
new
HttpHeaders
();
httpHeaders
.
setContentType
(
MediaType
.
APPLICATION_JSON
);
HttpEntity
<
String
>
request
=
new
HttpEntity
<>(
param
.
toJSONString
(),
httpHeaders
);
...
...
@@ -2028,7 +2051,21 @@ public class MarkDataServiceImpl implements MarkDataService {
if
(!
dto
.
getLasting
()
&&
Objects
.
nonNull
(
dto
.
getEndTime
()))
{
param
.
put
(
"stopTime"
,
dto
.
getEndTime
());
}
// 只选ocr
if
(
dto
.
getType
().
size
()
==
1
&&
dto
.
getType
().
contains
(
"OCR"
)){
param
.
put
(
"ocrType"
,
2
);
}
// 未选ocr
if
(!
dto
.
getType
().
contains
(
"OCR"
)){
param
.
put
(
"filterWtype"
,
String
.
join
(
","
,
dto
.
getType
()));
param
.
put
(
"ocrType"
,
1
);
}
// 选了ocr+其他
if
(
dto
.
getType
().
contains
(
"OCR"
)
&&
dto
.
getType
().
size
()
!=
1
){
dto
.
getType
().
remove
(
"OCR"
);
param
.
put
(
"filterWtype"
,
String
.
join
(
","
,
dto
.
getType
()));
param
.
put
(
"ocrType"
,
3
);
}
param
.
put
(
"taskName"
,
dto
.
getName
()
+
"_"
+
System
.
currentTimeMillis
());
param
.
put
(
"groupName"
,
projectService
.
getProjectById
(
UserThreadLocal
.
getProjectId
()).
getProjectName
());
param
.
put
(
"isKsSubject"
,
true
);
...
...
@@ -2647,12 +2684,12 @@ public class MarkDataServiceImpl implements MarkDataService {
*/
private
List
<
JSONObject
>
getPlatformPercentage
(
MarkSearchDTO
dto
)
throws
IOException
{
if
(
CollectionUtils
.
isEmpty
(
dto
.
getPlatforms
()))
{
List
<
String
>
platforms
=
commonService
.
getQbjcPlatform
().
stream
().
map
(
jsonObject
->
jsonObject
.
getString
(
"id"
)
).
collect
(
Collectors
.
toList
());
List
<
String
>
platforms
=
GlobalPojo
.
PLATFORMS
.
stream
().
map
(
MessagePlatform:
:
getId
).
collect
(
Collectors
.
toList
());
dto
.
setPlatforms
(
platforms
);
}
List
<
JSONObject
>
list
=
new
ArrayList
<>();
// total
d
to
.
setProjectId
(
UserThreadLocal
.
getProjectId
()
);
d
efaultMarkSearch
(
dto
);
Long
total
=
getYuqingAnalyzeCount
(
dto
);
// 平台聚合
SearchResponse
searchResponse
=
platformAggSearchResponse
(
dto
);
...
...
@@ -2707,11 +2744,15 @@ public class MarkDataServiceImpl implements MarkDataService {
Pair
<
String
,
List
<
LineVO
>>
spreadTendency
=
getEmotionSpreadTendency
(
dto
,
null
);
// 负面趋势图
Pair
<
String
,
List
<
LineVO
>>
negativeSpreadTendency
=
getEmotionSpreadTendency
(
dto
,
EmotionEnum
.
NEGATIVE
.
getName
());
// 正面趋势图
Pair
<
String
,
List
<
LineVO
>>
positiveSpreadTendency
=
getEmotionSpreadTendency
(
dto
,
EmotionEnum
.
POSITIVE
.
getName
());
Map
<
String
,
BaseMap
>
baseMaps
=
new
HashMap
<>(
2
);
BaseMap
totalBaseMaps
=
null
;
BaseMap
negativeBaseMaps
=
null
;
if
(
Objects
.
nonNull
(
negativeSpreadTendency
.
getLeft
())
||
Objects
.
nonNull
(
spreadTendency
.
getLeft
()))
{
CompletableFuture
.
allOf
(
Stream
.
of
(
spreadTendency
.
getLeft
(),
negativeSpreadTendency
.
getLeft
()).
filter
(
Objects:
:
nonNull
).
map
(
aggTitle
->
CompletableFuture
.
runAsync
(()
->
{
BaseMap
positiveBaseMaps
=
null
;
if
(
Objects
.
nonNull
(
negativeSpreadTendency
.
getLeft
())
||
Objects
.
nonNull
(
spreadTendency
.
getLeft
())
||
Objects
.
nonNull
(
positiveSpreadTendency
.
getLeft
()))
{
CompletableFuture
.
allOf
(
Stream
.
of
(
spreadTendency
.
getLeft
(),
negativeSpreadTendency
.
getLeft
(),
positiveSpreadTendency
.
getLeft
())
.
filter
(
Objects:
:
nonNull
).
map
(
aggTitle
->
CompletableFuture
.
runAsync
(()
->
{
try
{
baseMaps
.
put
(
aggTitle
,
getAnalyzeFirstArticle
(
dto
,
aggTitle
));
}
catch
(
IOException
ignored
)
{
...
...
@@ -2723,7 +2764,10 @@ public class MarkDataServiceImpl implements MarkDataService {
totalBaseMaps
=
getWeiboFirstArticle
(
endTime
,
spreadTendency
.
getRight
(),
planId
,
projectId
,
null
);
}
if
(
Objects
.
isNull
(
negativeSpreadTendency
.
getLeft
())
&&
CollectionUtils
.
isNotEmpty
(
negativeSpreadTendency
.
getRight
())){
negativeBaseMaps
=
getWeiboFirstArticle
(
endTime
,
spreadTendency
.
getRight
(),
planId
,
projectId
,
EmotionEnum
.
NEGATIVE
.
getName
());
negativeBaseMaps
=
getWeiboFirstArticle
(
endTime
,
negativeSpreadTendency
.
getRight
(),
planId
,
projectId
,
EmotionEnum
.
NEGATIVE
.
getName
());
}
if
(
Objects
.
isNull
(
positiveSpreadTendency
.
getLeft
())
&&
CollectionUtils
.
isNotEmpty
(
positiveSpreadTendency
.
getRight
())){
positiveBaseMaps
=
getWeiboFirstArticle
(
endTime
,
positiveSpreadTendency
.
getRight
(),
planId
,
projectId
,
EmotionEnum
.
POSITIVE
.
getName
());
}
// 最高点
BaseMap
baseMap
=
Objects
.
isNull
(
baseMaps
.
get
(
spreadTendency
.
getLeft
()))
?
totalBaseMaps
:
baseMaps
.
get
(
spreadTendency
.
getLeft
());
...
...
@@ -2739,11 +2783,20 @@ public class MarkDataServiceImpl implements MarkDataService {
negativeHighestJson
.
put
(
"title"
,
Tools
.
filterUselessTitle
(
negativeBaseMap
.
getTitle
())
?
StringUtils
.
substring
(
negativeBaseMap
.
getContent
(),
0
,
15
)
:
negativeBaseMap
.
getTitle
());
negativeHighestJson
.
put
(
"url"
,
negativeBaseMap
.
getUrl
());
}
// 正面最高点
BaseMap
positiveBaseMap
=
Objects
.
isNull
(
baseMaps
.
get
(
positiveSpreadTendency
.
getLeft
()))
?
positiveBaseMaps
:
baseMaps
.
get
(
positiveSpreadTendency
.
getLeft
());
JSONObject
positiveHighestJson
=
new
JSONObject
();
if
(
Objects
.
nonNull
(
positiveBaseMap
))
{
positiveHighestJson
.
put
(
"title"
,
Tools
.
filterUselessTitle
(
positiveBaseMap
.
getTitle
())
?
StringUtils
.
substring
(
positiveBaseMap
.
getContent
(),
0
,
15
)
:
positiveBaseMap
.
getTitle
());
positiveHighestJson
.
put
(
"url"
,
positiveBaseMap
.
getUrl
());
}
res
.
putAll
(
platformSpreadTendency
);
res
.
put
(
"总量"
,
spreadTendency
.
getRight
());
res
.
put
(
"负面"
,
negativeSpreadTendency
.
getRight
());
res
.
put
(
"正面"
,
positiveSpreadTendency
.
getRight
());
res
.
put
(
"highest"
,
highestJson
);
res
.
put
(
"negativeHighest"
,
negativeHighestJson
);
res
.
put
(
"positiveHighest"
,
positiveHighestJson
);
}
catch
(
Exception
e
){
ExceptionCast
.
cast
(
CommonCodeEnum
.
FAIL
,
"新舆情分析getSpreadTendency异常-"
,
e
);
}
...
...
@@ -3010,6 +3063,7 @@ public class MarkDataServiceImpl implements MarkDataService {
json
.
put
(
"content"
,
firstArticle
.
getContent
());
json
.
put
(
"url"
,
firstArticle
.
getUrl
());
json
.
put
(
"realSource"
,
firstArticle
.
getRealSource
());
json
.
put
(
"source"
,
firstArticle
.
getSource
());
json
.
put
(
"emotion"
,
firstArticle
.
getEmotion
());
json
.
put
(
"time"
,
firstArticle
.
getTime
());
}
catch
(
IOException
ignored
)
{
...
...
@@ -3085,9 +3139,9 @@ public class MarkDataServiceImpl implements MarkDataService {
public
List
<
JSONObject
>
getAggTitleArticlePoints
(
MarkSearchDTO
dto
)
{
List
<
JSONObject
>
jsonObject
=
new
ArrayList
<>();
try
{
defaultMarkSearch
(
dto
);
String
[]
indexes
=
esClientDao
.
getIndexes
();
// query
dto
.
setProjectId
(
UserThreadLocal
.
getProjectId
());
BoolQueryBuilder
query
=
yuqingAnalyzeQuery
(
dto
);
DateHistogramAggregationBuilder
daysAggregationBuilder
=
AggregationBuilders
.
dateHistogram
(
"timeAgg"
).
field
(
"time"
).
calendarInterval
(
DateHistogramInterval
.
DAY
).
offset
(
"16h"
);
...
...
@@ -3157,18 +3211,19 @@ public class MarkDataServiceImpl implements MarkDataService {
@Override
public
PageVO
<
JSONObject
>
getArticleList
(
MarkSearchDTO
dto
)
{
int
page
=
Objects
.
isNull
(
dto
.
getPage
())
?
1
:
dto
.
getPage
();
int
pageSize
=
Objects
.
isNull
(
dto
.
getPageSize
())
?
10
:
dto
.
getPageSize
();
try
{
defaultMarkSearch
(
dto
);
String
[]
indexes
=
esClientDao
.
getIndexes
();
dto
.
setProjectId
(
UserThreadLocal
.
getProjectId
());
int
page
=
dto
.
getPage
();
int
pageSize
=
dto
.
getPageSize
();
// query
if
(
Objects
.
equals
(
"其他"
,
dto
.
getPlatforms
().
get
(
0
))){
List
<
String
>
platforms
=
getPlatformPercentage
(
dto
).
stream
().
map
(
json
->
json
.
getString
(
"platform"
)).
collect
(
Collectors
.
toList
());
List
<
String
>
allPlatforms
=
GlobalPojo
.
PLATFORMS
.
stream
().
map
(
MessagePlatform:
:
getName
).
collect
(
Collectors
.
toList
());
allPlatforms
.
removeAll
(
platforms
);
dto
.
setPlatforms
(
allPlatforms
.
stream
().
map
(
GlobalPojo:
:
getPlatformIdByName
).
collect
(
Collectors
.
toList
()));
}
else
{
}
if
(!
Objects
.
equals
(
"全部"
,
dto
.
getPlatforms
().
get
(
0
))){
dto
.
setPlatforms
(
Collections
.
singletonList
(
GlobalPojo
.
getPlatformIdByName
(
dto
.
getPlatforms
().
get
(
0
))));
}
BoolQueryBuilder
query
=
yuqingAnalyzeQuery
(
dto
);
...
...
@@ -3182,6 +3237,7 @@ public class MarkDataServiceImpl implements MarkDataService {
JSONObject
jsonObject
=
new
JSONObject
();
jsonObject
.
put
(
"source"
,
baseMap
.
getSource
());
jsonObject
.
put
(
"platform"
,
baseMap
.
getPlatform
());
jsonObject
.
put
(
"realSource"
,
baseMap
.
getRealSource
());
jsonObject
.
put
(
"title"
,
baseMap
.
getTitle
());
jsonObject
.
put
(
"time"
,
baseMap
.
getTime
());
jsonObject
.
put
(
"url"
,
baseMap
.
getUrl
());
...
...
@@ -3191,7 +3247,7 @@ public class MarkDataServiceImpl implements MarkDataService {
}
catch
(
Exception
e
){
ExceptionCast
.
cast
(
CommonCodeEnum
.
FAIL
,
"新舆情分析getArticleList异常-"
,
e
);
}
return
PageVO
.
createPageVo
(
0
,
page
,
pageSize
,
Collections
.
emptyList
())
;
return
null
;
}
@Override
...
...
@@ -3241,6 +3297,272 @@ public class MarkDataServiceImpl implements MarkDataService {
return
dailyReportDao
.
findOneById
(
id
);
}
@Override
public
MarkFlowEntity
getWholeSearchFirstArticle
(
MarkSearchDTO
dto
)
{
try
{
BoolQueryBuilder
query
=
getAllIndexSearchOriginQuery
(
dto
);
String
[]
indexes
=
esClientDao
.
getAllIndexes
(
dto
.
getStartTime
(),
dto
.
getEndTime
());
// sort
FieldSortBuilder
sort
=
new
FieldSortBuilder
(
"time"
).
order
(
SortOrder
.
ASC
);
// hits
SearchHits
hits
=
esClientDao
.
searchHits
(
indexes
,
query
,
null
,
null
,
sort
,
0
,
1
,
null
);
if
(
0
==
hits
.
getTotalHits
().
value
)
{
return
null
;
}
Map
<
String
,
Object
>
hit
=
hits
.
getAt
(
0
).
getSourceAsMap
();
JSONObject
tJson
=
new
JSONObject
(
hit
);
return
markFlowService
.
createAllIndexFlowInfo
(
tJson
,
dto
);
}
catch
(
Exception
e
){
ExceptionCast
.
cast
(
CommonCodeEnum
.
FAIL
,
"全网搜原发溯源getWholeSearchFirstArticle异常-"
,
e
);
}
return
null
;
}
@Override
public
List
<
JSONObject
>
getWholeSearchMediaParticipation
(
MarkSearchDTO
dto
)
{
List
<
JSONObject
>
res
=
new
ArrayList
<>();
try
{
BoolQueryBuilder
query
=
getAllIndexSearchOriginQuery
(
dto
);
String
[]
indexes
=
esClientDao
.
getAllIndexes
(
dto
.
getStartTime
(),
dto
.
getEndTime
());
// sort
FieldSortBuilder
sort
=
new
FieldSortBuilder
(
"time"
).
order
(
SortOrder
.
ASC
);
// ipAgg
TermsAggregationBuilder
provinceAggregationBuilder
=
AggregationBuilders
.
terms
(
"ipAgg"
).
field
(
"ip_location"
).
size
(
6
);
// response
SearchResponse
searchResponse
=
esClientDao
.
searchResponse
(
indexes
,
null
,
query
,
provinceAggregationBuilder
,
null
,
sort
,
0
,
1000
,
null
);
// hits
SearchHits
hits
=
searchResponse
.
getHits
();
JSONObject
firstJson
=
new
JSONObject
(
hits
.
getAt
(
0
).
getSourceAsMap
());
Map
<
String
,
Integer
>
platformCount
=
new
HashMap
<>();
Map
<
String
,
Integer
>
fieldCount
=
new
HashMap
<>();
int
centralLevelCount
=
0
;
int
provincialLevelCount
=
0
;
int
prefectureLevelCount
=
0
;
// 重要渠道
Map
<
String
,
SensitiveChannel
>
sensitiveChannelMap
=
GlobalPojo
.
COMMON_SENSITIVE_CHANNEL
;
for
(
SearchHit
hit
:
hits
.
getHits
())
{
JSONObject
hitJson
=
new
JSONObject
(
hit
.
getSourceAsMap
());
String
platform
=
BeanTools
.
filterPlatform
(
GlobalPojo
.
PLATFORMS
,
hitJson
.
getIntValue
(
GenericAttribute
.
ES_C5
),
hitJson
.
getIntValue
(
GenericAttribute
.
ES_FOREIGN
)).
getName
();
// 平台统计
platformCount
.
compute
(
platform
,
(
k
,
v
)
->
Objects
.
isNull
(
v
)
?
1
:
++
v
);
String
source
=
hitJson
.
getString
(
GenericAttribute
.
ES_SOURCE
);
SensitiveChannel
sensitiveChannel
=
sensitiveChannelMap
.
get
(
source
);
if
(
Objects
.
isNull
(
sensitiveChannel
)){
continue
;
}
// 重要渠道领域统计
if
(
StringUtils
.
isNotBlank
(
sensitiveChannel
.
getField
()))
{
fieldCount
.
compute
(
sensitiveChannel
.
getField
(),
(
k
,
v
)
->
Objects
.
isNull
(
v
)
?
1
:
++
v
);
}
if
(
StringUtils
.
isNotBlank
(
sensitiveChannel
.
getPoliticsLevel
())){
if
(
Objects
.
equals
(
"央级"
,
sensitiveChannel
.
getPoliticsLevel
())){
centralLevelCount
++;
}
if
(
Objects
.
equals
(
"省级"
,
sensitiveChannel
.
getPoliticsLevel
())){
provincialLevelCount
++;
}
if
(
Objects
.
equals
(
"市级"
,
sensitiveChannel
.
getPoliticsLevel
())){
prefectureLevelCount
++;
}
}
}
// 首发平台
JSONObject
firstPlatform
=
new
JSONObject
();
firstPlatform
.
put
(
"name"
,
"首发平台"
);
firstPlatform
.
put
(
"value"
,
BeanTools
.
filterPlatform
(
GlobalPojo
.
PLATFORMS
,
firstJson
.
getIntValue
(
GenericAttribute
.
ES_C5
),
firstJson
.
getIntValue
(
GenericAttribute
.
ES_FOREIGN
)).
getName
());
res
.
add
(
firstPlatform
);
// 爆发平台
Set
<
String
>
maxPlatforms
=
Tools
.
sortMap
(
platformCount
,
1
).
keySet
();
JSONObject
maxPlatform
=
new
JSONObject
();
maxPlatform
.
put
(
"name"
,
"爆发平台"
);
maxPlatform
.
put
(
"value"
,
maxPlatforms
.
toArray
()[
0
]);
res
.
add
(
maxPlatform
);
// 传播量
JSONObject
count
=
new
JSONObject
();
count
.
put
(
"name"
,
"传播量"
);
count
.
put
(
"value"
,
hits
.
getHits
().
length
);
res
.
add
(
count
);
// 媒体领域
JSONObject
mediaField
=
new
JSONObject
();
mediaField
.
put
(
"name"
,
"媒体领域"
);
mediaField
.
put
(
"value"
,
Tools
.
sortMap
(
fieldCount
,
6
).
keySet
());
res
.
add
(
mediaField
);
// 央级
JSONObject
centralLevel
=
new
JSONObject
();
centralLevel
.
put
(
"name"
,
"央级"
);
centralLevel
.
put
(
"value"
,
centralLevelCount
);
res
.
add
(
centralLevel
);
// 省级
JSONObject
provincialLevel
=
new
JSONObject
();
provincialLevel
.
put
(
"name"
,
"省级"
);
provincialLevel
.
put
(
"value"
,
provincialLevelCount
);
res
.
add
(
provincialLevel
);
// 地级
JSONObject
prefectureLevel
=
new
JSONObject
();
prefectureLevel
.
put
(
"name"
,
"地级"
);
prefectureLevel
.
put
(
"value"
,
prefectureLevelCount
);
res
.
add
(
prefectureLevel
);
// 媒体地域
Map
<
String
,
Aggregation
>
provinceMap
=
searchResponse
.
getAggregations
().
asMap
();
ParsedStringTerms
provinceCountTeam
=
(
ParsedStringTerms
)
provinceMap
.
get
(
"ipAgg"
);
List
<?
extends
Terms
.
Bucket
>
provinceBuckets
=
provinceCountTeam
.
getBuckets
();
JSONObject
mediaProvince
=
new
JSONObject
();
mediaProvince
.
put
(
"name"
,
"媒体地域"
);
mediaProvince
.
put
(
"value"
,
provinceBuckets
.
stream
().
map
(
MultiBucketsAggregation
.
Bucket
::
getKeyAsString
).
collect
(
Collectors
.
toList
()));
res
.
add
(
mediaProvince
);
return
res
;
}
catch
(
Exception
e
){
ExceptionCast
.
cast
(
CommonCodeEnum
.
FAIL
,
"全网搜原发溯源getWholeSearchMediaParticipation异常-"
,
e
);
}
return
res
;
}
@Override
public
List
<
JSONObject
>
getWholeSearchArticlePoint
(
MarkSearchDTO
dto
)
{
List
<
JSONObject
>
jsonObjects
=
new
ArrayList
<>();
try
{
BoolQueryBuilder
query
=
getAllIndexSearchOriginQuery
(
dto
);
String
[]
indexes
=
esClientDao
.
getAllIndexes
(
dto
.
getStartTime
(),
dto
.
getEndTime
());
// aggregation
DateHistogramAggregationBuilder
daysAggregationBuilder
=
AggregationBuilders
.
dateHistogram
(
"timeAgg"
).
field
(
"time"
).
calendarInterval
(
DateHistogramInterval
.
DAY
).
offset
(
"16h"
);
TermsAggregationBuilder
sourceAggregationBuilder
=
AggregationBuilders
.
terms
(
"sourceAgg"
).
field
(
"source"
).
size
(
10000
);
// response
SearchResponse
searchResponse
=
esClientDao
.
searchResponse
(
indexes
,
null
,
query
,
daysAggregationBuilder
.
subAggregation
(
sourceAggregationBuilder
),
null
,
null
,
0
,
0
,
null
);
Map
<
String
,
Aggregation
>
aggMap
=
searchResponse
.
getAggregations
().
asMap
();
ParsedDateHistogram
teamAgg
=
(
ParsedDateHistogram
)
aggMap
.
get
(
"timeAgg"
);
List
<?
extends
Histogram
.
Bucket
>
buckets
=
teamAgg
.
getBuckets
();
// 重要渠道
Map
<
String
,
SensitiveChannel
>
sensitiveChannelMap
=
GlobalPojo
.
COMMON_SENSITIVE_CHANNEL
;
buckets
.
forEach
(
bucket
->
{
JSONObject
point
=
new
JSONObject
();
point
.
put
(
"time"
,
Long
.
parseLong
(
bucket
.
getKeyAsString
()));
// 发文渠道
Map
<
String
,
Aggregation
>
sourceMap
=
bucket
.
getAggregations
().
asMap
();
ParsedStringTerms
countTeam
=
(
ParsedStringTerms
)
sourceMap
.
get
(
"sourceAgg"
);
List
<?
extends
Terms
.
Bucket
>
sourceBuckets
=
countTeam
.
getBuckets
();
List
<
JSONObject
>
sources
=
new
ArrayList
<>();
sourceBuckets
.
forEach
(
sourceBucket
->
{
JSONObject
source
=
new
JSONObject
();
source
.
put
(
"name"
,
sourceBucket
.
getKeyAsString
());
// 重要渠道判断
source
.
put
(
"isImportant"
,
sensitiveChannelMap
.
containsKey
(
sourceBucket
.
getKeyAsString
()));
sources
.
add
(
source
);
});
point
.
put
(
"sources"
,
sources
);
point
.
put
(
"count"
,
sources
.
size
());
jsonObjects
.
add
(
point
);
});
}
catch
(
Exception
e
){
ExceptionCast
.
cast
(
CommonCodeEnum
.
FAIL
,
"全网搜原发溯源发布节点getWholeSearchArticlePoint异常-"
,
e
);
}
return
jsonObjects
;
}
@Override
public
List
<
JSONObject
>
getWholeSearchPlatformPercentage
(
MarkSearchDTO
dto
)
{
try
{
BoolQueryBuilder
query
=
getAllIndexSearchOriginQuery
(
dto
);
String
[]
indexes
=
esClientDao
.
getAllIndexes
(
dto
.
getStartTime
(),
dto
.
getEndTime
());
Long
total
=
esClientDao
.
count
(
indexes
,
query
,
null
);
// 聚合条件
Script
script
=
new
Script
(
"doc['c5'].value +'_' +doc['foreign'].value"
);
TermsAggregationBuilder
aggregationBuilder
=
AggregationBuilders
.
terms
(
"agg"
).
script
(
script
).
size
(
1000
);
// response
SearchResponse
searchResponse
=
esClientDao
.
searchResponse
(
indexes
,
null
,
query
,
aggregationBuilder
,
null
,
null
,
0
,
0
,
null
);
// buckets
Map
<
String
,
Aggregation
>
aggregationMap
=
searchResponse
.
getAggregations
().
asMap
();
ParsedStringTerms
terms
=
(
ParsedStringTerms
)
aggregationMap
.
get
(
"agg"
);
List
<?
extends
Terms
.
Bucket
>
buckets
=
terms
.
getBuckets
();
Map
<
String
,
Long
>
platformCount
=
new
HashMap
<>();
for
(
Terms
.
Bucket
bucket
:
buckets
)
{
String
[]
split
=
bucket
.
getKeyAsString
().
split
(
"_"
);
String
platform
=
BeanTools
.
filterPlatform
(
GlobalPojo
.
PLATFORMS
,
Integer
.
valueOf
(
split
[
0
]),
Integer
.
valueOf
(
split
[
1
])).
getName
();
if
(
StringUtils
.
isBlank
(
platform
)){
continue
;
}
platformCount
.
compute
(
platform
,
(
k
,
v
)
->
Objects
.
isNull
(
v
)
?
bucket
.
getDocCount
()
:
v
+
bucket
.
getDocCount
());
}
List
<
JSONObject
>
list
=
new
ArrayList
<>();
for
(
Map
.
Entry
<
String
,
Long
>
entry
:
platformCount
.
entrySet
())
{
JSONObject
platformResult
=
new
JSONObject
();
platformResult
.
put
(
"platform"
,
entry
.
getKey
());
platformResult
.
put
(
"percent"
,
0
==
total
?
0
d
:
entry
.
getValue
()
/
(
double
)
total
);
list
.
add
(
platformResult
);
}
// 取占比最高的前9
List
<
JSONObject
>
res
=
list
.
stream
()
.
sorted
(
Comparator
.
comparingDouble
((
JSONObject
jsonObject
)
->
jsonObject
.
getDoubleValue
(
"percent"
))
.
reversed
()).
limit
(
9
).
collect
(
Collectors
.
toList
());
// 第10个算做其他平台
JSONObject
other
=
new
JSONObject
();
other
.
put
(
"platform"
,
"其他"
);
other
.
put
(
"percent"
,
1
-
res
.
stream
().
mapToDouble
(
count
->
count
.
getDoubleValue
(
"percent"
)).
sum
());
res
.
add
(
other
);
return
res
;
}
catch
(
Exception
e
){
ExceptionCast
.
cast
(
CommonCodeEnum
.
FAIL
,
"全网搜原发溯源平台分布getWholeSearchPlatformPercentage异常-"
,
e
);
}
return
null
;
}
@Override
public
PageVO
<
JSONObject
>
getWholeSearchArticleList
(
MarkSearchDTO
dto
)
{
try
{
defaultMarkSearch
(
dto
);
// keyword
BoolQueryBuilder
keywordQuery
=
QueryBuilders
.
boolQuery
();
keywordQuery
.
must
(
EsQueryTools
.
assembleNormalKeywordQuery
(
dto
.
getKeyword
(),
new
String
[]{
GenericAttribute
.
ES_IND_FULL_TEXT
}));
BoolQueryBuilder
query
=
QueryBuilders
.
boolQuery
().
must
(
keywordQuery
);
// platform query
if
(
Objects
.
equals
(
"其他"
,
dto
.
getPlatforms
().
get
(
0
))){
List
<
String
>
platforms
=
getWholeSearchPlatformPercentage
(
dto
).
stream
().
map
(
json
->
json
.
getString
(
"platform"
)).
collect
(
Collectors
.
toList
());
query
.
must
(
EsQueryTools
.
assemblePlatformQuery
(
GlobalPojo
.
PLATFORMS
.
stream
()
.
filter
(
platform
->
!
platforms
.
contains
(
platform
.
getName
())).
collect
(
Collectors
.
toList
())));
}
if
(!
Objects
.
equals
(
"全部"
,
dto
.
getPlatforms
().
get
(
0
))){
query
.
must
(
EsQueryTools
.
assemblePlatformQuery
(
GlobalPojo
.
PLATFORMS
.
stream
()
.
filter
(
platform
->
Objects
.
equals
(
dto
.
getPlatforms
().
get
(
0
),
platform
.
getName
())).
collect
(
Collectors
.
toList
())));
}
int
page
=
dto
.
getPage
();
int
pageSize
=
dto
.
getPageSize
();
String
[]
indexes
=
esClientDao
.
getAllIndexes
(
dto
.
getStartTime
(),
dto
.
getEndTime
());
FieldSortBuilder
sort
=
new
FieldSortBuilder
(
"time"
).
order
(
SortOrder
.
ASC
);
SearchHits
hits
=
esClientDao
.
searchHits
(
indexes
,
query
,
null
,
null
,
sort
,
(
page
-
1
)
*
pageSize
,
pageSize
,
null
);
List
<
JSONObject
>
list
=
new
ArrayList
<>();
for
(
SearchHit
hit
:
hits
)
{
JSONObject
hitJson
=
new
JSONObject
(
hit
.
getSourceAsMap
());
JSONObject
data
=
JSONObject
.
parseObject
(
JSONObject
.
toJSONString
(
new
MarkFlowEntity
(
hitJson
).
getData
()));
JSONObject
jsonObject
=
new
JSONObject
();
jsonObject
.
put
(
"platform"
,
BeanTools
.
filterPlatform
(
GlobalPojo
.
PLATFORMS
,
hitJson
.
getIntValue
(
GenericAttribute
.
ES_C5
),
hitJson
.
getIntValue
(
GenericAttribute
.
ES_FOREIGN
)).
getName
());
jsonObject
.
put
(
"source"
,
hitJson
.
getString
(
GenericAttribute
.
ES_SOURCE
));
jsonObject
.
put
(
"realSource"
,
hitJson
.
getString
(
GenericAttribute
.
ES_REAL_SOURCE
));
jsonObject
.
put
(
"time"
,
hitJson
.
getLongValue
(
"time"
));
String
title
=
StringUtils
.
isNotBlank
(
Tools
.
getTitleWithContent
(
data
.
getString
(
"title"
),
data
.
getString
(
"content"
)))
?
Tools
.
getTitleWithContent
(
data
.
getString
(
"title"
),
data
.
getString
(
"content"
))
:
Tools
.
getTitleWithContent
(
data
.
getString
(
"questionTitle"
),
data
.
getString
(
"questionContent"
));
jsonObject
.
put
(
"title"
,
title
);
list
.
add
(
jsonObject
);
}
return
PageVO
.
createPageVo
(
hits
.
getHits
().
length
,
page
,
pageSize
,
list
);
}
catch
(
Exception
e
){
ExceptionCast
.
cast
(
CommonCodeEnum
.
FAIL
,
"全网搜原发溯源文章列表getWholeSearchArticleList异常-"
,
e
);
}
return
null
;
}
private
BoolQueryBuilder
getAllIndexSearchOriginQuery
(
MarkSearchDTO
dto
)
{
dto
.
setStartTime
(
dto
.
getEndTime
()
-
Constant
.
ONE_MONTH
);
// keyword
BoolQueryBuilder
keywordQuery
=
QueryBuilders
.
boolQuery
();
keywordQuery
.
must
(
QueryBuilders
.
matchPhraseQuery
(
GenericAttribute
.
ES_IND_FULL_TEXT
,
dto
.
getKeyword
()));
// platform
BoolQueryBuilder
platformQuery
=
EsQueryTools
.
assemblePlatformQuery
(
GlobalPojo
.
PLATFORMS
);
return
QueryBuilders
.
boolQuery
().
must
(
keywordQuery
).
must
(
platformQuery
)
.
must
(
QueryBuilders
.
rangeQuery
(
"time"
).
gte
(
dto
.
getStartTime
()).
lte
(
dto
.
getEndTime
()));
}
private
JSONObject
getDailyReportSummary
(
JSONObject
yuqingAmount
,
JSONObject
yuqingEmotion
,
JSONObject
platformPercentage
){
JSONObject
res
=
new
JSONObject
();
long
total
=
yuqingAmount
.
getLongValue
(
"total"
);
...
...
@@ -3432,8 +3754,14 @@ public class MarkDataServiceImpl implements MarkDataService {
if
(
Objects
.
equals
(
"视频"
,
dto
.
getSearchType
())){
postFilter
.
must
(
QueryBuilders
.
existsQuery
(
GenericAttribute
.
ES_VIDEO_URLS
));
}
// time
postFilter
.
must
(
QueryBuilders
.
rangeQuery
(
"time"
).
gte
(
dto
.
getStartTime
()).
lt
(
dto
.
getEndTime
()));
// startTime
if
(
Objects
.
nonNull
(
dto
.
getStartTime
()))
{
postFilter
.
must
(
QueryBuilders
.
rangeQuery
(
"time"
).
gte
(
dto
.
getStartTime
()));
}
// endTime
if
(
Objects
.
nonNull
(
dto
.
getEndTime
()))
{
postFilter
.
must
(
QueryBuilders
.
rangeQuery
(
"time"
).
lt
(
dto
.
getEndTime
()));
}
// platform
if
(
CollectionUtils
.
isNotEmpty
(
dto
.
getPlatforms
()))
{
postFilter
.
must
(
EsQueryTools
.
assemblePlatformQuery
(
Tools
.
getPlatformByIds
(
dto
.
getPlatforms
())));
...
...
src/main/java/com/zhiwei/brandkbs2/service/impl/MarkFlowServiceImpl.java
View file @
90c9a6a5
...
...
@@ -10,14 +10,14 @@ import com.zhiwei.brandkbs2.auth.UserThreadLocal;
import
com.zhiwei.brandkbs2.common.GenericAttribute
;
import
com.zhiwei.brandkbs2.common.GlobalPojo
;
import
com.zhiwei.brandkbs2.config.Constant
;
import
com.zhiwei.brandkbs2.dao.HighlightWordDao
;
import
com.zhiwei.brandkbs2.enmus.ChannelEmotion
;
import
com.zhiwei.brandkbs2.exception.ExceptionCast
;
import
com.zhiwei.brandkbs2.model.CommonCodeEnum
;
import
com.zhiwei.brandkbs2.pojo.MarkFlowEntity
;
import
com.zhiwei.brandkbs2.pojo.SensitiveChannel
;
import
com.zhiwei.brandkbs2.pojo.dto.MarkSearchDTO
;
import
com.zhiwei.brandkbs2.pojo.dto.TagFilterDTO
;
import
com.zhiwei.brandkbs2.service.MarkFlowService
;
import
com.zhiwei.brandkbs2.service.TagFilterService
;
import
com.zhiwei.brandkbs2.util.RedisUtil
;
import
com.zhiwei.brandkbs2.util.Tools
;
import
com.zhiwei.middleware.automaticmark.vo.Keyword
;
...
...
@@ -27,6 +27,7 @@ import com.zhiwei.middleware.mark.vo.QueryResult;
import
com.zhiwei.qbjc.bean.pojo.common.Tag
;
import
com.zhiwei.qbjc.bean.tools.BeanTools
;
import
org.apache.commons.collections4.CollectionUtils
;
import
org.apache.commons.lang3.StringUtils
;
import
org.apache.logging.log4j.LogManager
;
import
org.apache.logging.log4j.Logger
;
import
org.springframework.stereotype.Service
;
...
...
@@ -197,6 +198,63 @@ public class MarkFlowServiceImpl implements MarkFlowService {
return
JSON
.
parseObject
(
data
,
MarkFlowEntity
.
class
);
}
@Override
public
MarkFlowEntity
createAllIndexFlowInfo
(
JSONObject
json
,
MarkSearchDTO
dto
)
{
MarkFlowEntity
markFlowEntity
=
new
MarkFlowEntity
(
json
);
JSONObject
info
=
new
JSONObject
();
info
.
put
(
"sourceDetails"
,
getSourceDetails
(
json
));
info
.
put
(
"startTime"
,
dto
.
getStartTime
());
info
.
put
(
"endTime"
,
dto
.
getEndTime
());
info
.
put
(
"highlightWordMap"
,
getHighlightWordMap
(
UserThreadLocal
.
getProjectId
(),
json
));
markFlowEntity
.
setInfo
(
info
);
return
markFlowEntity
;
}
/**
* 底层大库数据SourceDetails提取
* @param tJson
* @return
*/
private
JSONObject
getSourceDetails
(
JSONObject
tJson
){
JSONObject
sourceDetails
=
new
JSONObject
();
String
source
=
tJson
.
getString
(
GenericAttribute
.
ES_SOURCE
);
String
rootSource
=
tJson
.
getString
(
GenericAttribute
.
ES_ROOT_SOURCE
);
// 是否原创
if
(
tJson
.
containsKey
(
GenericAttribute
.
ES_ROOT_SOURCE
)
&&
(
rootSource
.
contains
(
source
)
||
source
.
contains
(
rootSource
)))
{
sourceDetails
.
put
(
"rootPublish"
,
"原创"
);
}
// C4,realSource提取展示
sourceDetails
.
put
(
"clientFrom"
,
Tools
.
isEmpty
(
tJson
.
getString
(
GenericAttribute
.
ES_REAL_SOURCE
))
?
ClassD
.
TypeD
.
fromEncode
(
tJson
.
getIntValue
(
GenericAttribute
.
ES_C4
)).
title
()
:
tJson
.
getString
(
GenericAttribute
.
ES_REAL_SOURCE
));
// platform
sourceDetails
.
put
(
"platform"
,
BeanTools
.
filterPlatform
(
GlobalPojo
.
PLATFORMS
,
tJson
.
getIntValue
(
GenericAttribute
.
ES_C5
),
tJson
.
getIntValue
(
GenericAttribute
.
ES_FOREIGN
)).
getName
());
// source
sourceDetails
.
put
(
"source"
,
source
);
// 粉丝量提取
long
followersNum
=
tJson
.
getLongValue
(
"followers_num"
);
if
(
followersNum
>
0
)
{
sourceDetails
.
put
(
"followersNum"
,
followersNum
);
}
// 渠道标签
SensitiveChannel
sensitiveChannel
=
GlobalPojo
.
COMMON_SENSITIVE_CHANNEL
.
get
(
source
);
if
(
Objects
.
nonNull
(
sensitiveChannel
))
{
List
<
String
>
channelTagStr
=
new
ArrayList
<>(
4
);
if
(
StringUtils
.
isNotBlank
(
sensitiveChannel
.
getPoliticsLevel
())){
channelTagStr
.
add
(
sensitiveChannel
.
getPoliticsLevel
());
}
if
(
StringUtils
.
isNotBlank
(
sensitiveChannel
.
getMainBodyType
())){
channelTagStr
.
add
(
sensitiveChannel
.
getMainBodyType
());
}
if
(
StringUtils
.
isNotBlank
(
sensitiveChannel
.
getField
())){
channelTagStr
.
add
(
sensitiveChannel
.
getField
());
}
if
(
StringUtils
.
isNotBlank
(
sensitiveChannel
.
getRegion
())){
channelTagStr
.
add
(
sensitiveChannel
.
getRegion
());
}
sourceDetails
.
put
(
"channelTag"
,
StringUtils
.
join
(
channelTagStr
,
"|"
));
}
return
sourceDetails
;
}
/**
* 获取自定义标签
* @param markInfoSource
...
...
src/main/java/com/zhiwei/brandkbs2/service/impl/ProjectServiceImpl.java
View file @
90c9a6a5
...
...
@@ -33,6 +33,7 @@ import org.springframework.data.mongodb.core.query.Criteria;
import
org.springframework.data.mongodb.core.query.Query
;
import
org.springframework.data.mongodb.core.query.Update
;
import
org.springframework.stereotype.Service
;
import
org.springframework.web.client.RestTemplate
;
import
org.springframework.web.context.request.RequestContextHolder
;
import
org.springframework.web.context.request.ServletRequestAttributes
;
...
...
@@ -90,12 +91,18 @@ public class ProjectServiceImpl implements ProjectService {
@Value
(
"${jwt.key}"
)
private
String
jwtKey
;
@Value
(
"${istarshine.subject.updateTaskStatus.url}"
)
private
String
updateIStarShineSubjectTaskStatusUrl
;
@Value
(
"${brandkbs.file.url}"
)
private
String
brandkbsFileUrl
;
@Value
(
"${brandkbs.image.url}"
)
private
String
brandkbsImageUrl
;
@Resource
(
name
=
"restTemplate"
)
private
RestTemplate
restTemplate
;
@Override
public
ResponseResult
addProject
(
ProjectVO
projectVO
)
{
if
(
Objects
.
isNull
(
projectVO
))
{
...
...
@@ -284,12 +291,30 @@ public class ProjectServiceImpl implements ProjectService {
if
(
Objects
.
isNull
(
project
))
{
ExceptionCast
.
cast
(
CommonCodeEnum
.
INVALID_PARAM
,
"项目不存在"
);
}
// 对开启了商业数据订阅的非人工项目
if
(!
project
.
isManual
()
&&
project
.
getExternalDataSource
()){
List
<
NonManualProjectPlan
>
planList
=
nonManualProjectPlanDao
.
findList
(
new
Query
(
Criteria
.
where
(
"projectId"
).
is
(
project
.
getId
())));
// 启动->终止:暂停商业数据订阅任务 终止->启动:启动商业数据订阅任务
planList
.
forEach
(
plan
->
updateStarShineSubjectTaskStatus
(
plan
.
getIStarShineSubjectTaskId
()));
}
Update
update
=
new
Update
();
update
.
set
(
"uTime"
,
System
.
currentTimeMillis
());
update
.
set
(
"isStart"
,
!
project
.
isStart
());
projectDao
.
updateOneByIdWithField
(
pid
,
update
);
}
/**
* 修改快搜任务状态
* @param taskId
* @return
*/
private
void
updateStarShineSubjectTaskStatus
(
String
taskId
){
JSONObject
body
=
restTemplate
.
getForEntity
(
updateIStarShineSubjectTaskStatusUrl
,
JSONObject
.
class
,
taskId
,
UserThreadLocal
.
getNickname
()).
getBody
();
if
(
Objects
.
isNull
(
body
)
||
body
.
isEmpty
()
||
!
body
.
getBoolean
(
"status"
)){
log
.
error
(
"快搜任务id:"
+
taskId
+
"-修改快搜订阅任务状态失败,接口返回信息-"
+
body
);
}
}
@Override
public
ResponseResult
getUserAllProjects
()
{
User
user
=
userDao
.
findOneById
(
UserThreadLocal
.
getUserId
());
...
...
@@ -676,6 +701,9 @@ public class ProjectServiceImpl implements ProjectService {
List
<
NonManualProjectPlan
>
successList
=
new
ArrayList
<>(
planList
.
size
());
// 添加商业数据订阅任务
for
(
NonManualProjectPlan
plan
:
planList
)
{
if
(
CollectionUtils
.
isEmpty
(
plan
.
getType
())){
plan
.
setType
(
new
ArrayList
<>(
Arrays
.
asList
(
"原创"
,
"转发"
,
"OCR"
)));
}
String
taskId
=
markDataService
.
addIStarShineSubjectTask
(
Tools
.
convertMap
(
plan
,
NonManualProjectPlanDTO
.
class
));
if
(
Objects
.
isNull
(
taskId
)){
for
(
NonManualProjectPlan
successPlan
:
successList
)
{
...
...
src/main/resources/application-dev.properties
View file @
90c9a6a5
...
...
@@ -133,3 +133,4 @@ toolset.interactionResult.url=https://zhiwei-middleware.zhiweidata.com/interact/
istarshine.subject.addTask.url
=
https://istarshine-service.zhiweidata.com/istarshine/subject/addIStarShineSubjectTask
istarshine.subject.deleteTask.url
=
https://istarshine-service.zhiweidata.com/istarshine/subject/deleteIStarShineSubjectTask?taskId={1}&updateUserName={2}
istarshine.subject.updateTask.url
=
https://istarshine-service.zhiweidata.com/istarshine/subject/updateIStarShineSubjectTask
istarshine.subject.updateTaskStatus.url
=
https://istarshine-service.zhiweidata.com/istarshine/subject/updateIStarShineSubjectTaskStatus?taskId={1}&updateUserName={2}
\ No newline at end of file
src/main/resources/application-local.properties
View file @
90c9a6a5
...
...
@@ -112,7 +112,7 @@ ef.checkCaptcha.url=https://ef.zhiweidata.com/captchaApi/checkeCaptcha.do.do?id=
#\u5168\u7F51\u641C\u5916\u90E8\u63A5\u53E3
istarshine.addIStarShineKSData.url
=
https://istarshine-service.zhiweidata.com/istarshine/ks/addIStarShineKSData
istarshine.getIStarShineKSInfoData.url
=
https://istarshine-service.zhiweidata.com/istarshine/ks/getIStarShineKSInfoData?taskId={1}&appId=6183571e0d710000f6003a12
whole.search.url
=
https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/search/yuqing
whole.search.url
=
https://yuqing.
test.
zhiweidata.com/qbjcbackPhoenix/interface/middleware/search/yuqing
whole.searchCriteria.url
=
https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/search/pt
whole.extraParam.url
=
https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/search/extra/param
brief.report.url
=
http://192.168.0.79:11000/qbjcbackPhoenix/interface/middleware/brief-report-web
...
...
@@ -143,3 +143,4 @@ toolset.interactionResult.url=https://zhiwei-middleware.zhiweidata.com/interact/
istarshine.subject.addTask.url
=
https://istarshine-service.zhiweidata.com/istarshine/subject/addIStarShineSubjectTask
istarshine.subject.deleteTask.url
=
https://istarshine-service.zhiweidata.com/istarshine/subject/deleteIStarShineSubjectTask?taskId={1}&updateUserName={2}
istarshine.subject.updateTask.url
=
https://istarshine-service.zhiweidata.com/istarshine/subject/updateIStarShineSubjectTask
istarshine.subject.updateTaskStatus.url
=
https://istarshine-service.zhiweidata.com/istarshine/subject/updateIStarShineSubjectTaskStatus?taskId={1}&updateUserName={2}
\ No newline at end of file
src/main/resources/application-prod.properties
View file @
90c9a6a5
...
...
@@ -102,7 +102,7 @@ ef.checkCaptcha.url=https://ef.zhiweidata.com/captchaApi/checkeCaptcha.do.do?id=
#\u5168\u7F51\u641C\u5916\u90E8\u63A5\u53E3
istarshine.addIStarShineKSData.url
=
https://istarshine-service.zhiweidata.com/istarshine/ks/addIStarShineKSData
istarshine.getIStarShineKSInfoData.url
=
https://istarshine-service.zhiweidata.com/istarshine/ks/getIStarShineKSInfoData?taskId={1}&appId=6183571e0d710000f6003a12
whole.search.url
=
https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/search/yuqing
whole.search.url
=
https://yuqing.
test.
zhiweidata.com/qbjcbackPhoenix/interface/middleware/search/yuqing
whole.searchCriteria.url
=
https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/search/pt
whole.extraParam.url
=
https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/search/extra/param
brief.report.url
=
https://yuqing.zhiweidata.com/qbjcbackPhoenix/interface/middleware/brief-report-web
...
...
@@ -133,3 +133,4 @@ toolset.interactionResult.url=https://zhiwei-middleware.zhiweidata.com/interact/
istarshine.subject.addTask.url
=
https://istarshine-service.zhiweidata.com/istarshine/subject/addIStarShineSubjectTask
istarshine.subject.deleteTask.url
=
https://istarshine-service.zhiweidata.com/istarshine/subject/deleteIStarShineSubjectTask?taskId={1}&updateUserName={2}
istarshine.subject.updateTask.url
=
https://istarshine-service.zhiweidata.com/istarshine/subject/updateIStarShineSubjectTask
istarshine.subject.updateTaskStatus.url
=
https://istarshine-service.zhiweidata.com/istarshine/subject/updateIStarShineSubjectTaskStatus?taskId={1}&updateUserName={2}
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment