Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
B
brandkbs2
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
shenjunjie
brandkbs2
Commits
18f92bf9
Commit
18f92bf9
authored
Jul 08, 2024
by
shenjunjie
Browse files
Options
Browse Files
Download
Plain Diff
Merge branch 'release' into 'master'
Release See merge request
!548
parents
09feb611
7bed0bf0
Hide whitespace changes
Inline
Side-by-side
Showing
36 changed files
with
1208 additions
and
190 deletions
+1208
-190
src/main/java/com/zhiwei/brandkbs2/common/RedisKeyPrefix.java
+6
-0
src/main/java/com/zhiwei/brandkbs2/controller/admin/BaseModuleController.java
+39
-8
src/main/java/com/zhiwei/brandkbs2/controller/admin/ExtraController.java
+13
-0
src/main/java/com/zhiwei/brandkbs2/controller/app/AppArticleController.java
+61
-0
src/main/java/com/zhiwei/brandkbs2/dao/XiaohongshuRecordDao.java
+24
-0
src/main/java/com/zhiwei/brandkbs2/dao/XiaohongshuWordDao.java
+12
-0
src/main/java/com/zhiwei/brandkbs2/dao/impl/XiaohongshuRecordDaoImpl.java
+56
-0
src/main/java/com/zhiwei/brandkbs2/dao/impl/XiaohongshuWordDaoImpl.java
+20
-0
src/main/java/com/zhiwei/brandkbs2/es/EsClientDao.java
+24
-0
src/main/java/com/zhiwei/brandkbs2/es/EsQueryTools.java
+4
-2
src/main/java/com/zhiwei/brandkbs2/pojo/AbstractBaseWord.java
+5
-3
src/main/java/com/zhiwei/brandkbs2/pojo/HighWord.java
+4
-0
src/main/java/com/zhiwei/brandkbs2/pojo/HighlightWord.java
+4
-0
src/main/java/com/zhiwei/brandkbs2/pojo/Project.java
+2
-3
src/main/java/com/zhiwei/brandkbs2/pojo/XiaohongshuRecord.java
+46
-0
src/main/java/com/zhiwei/brandkbs2/pojo/XiaohongshuWord.java
+25
-0
src/main/java/com/zhiwei/brandkbs2/pojo/dto/MarkSearchDTO.java
+7
-3
src/main/java/com/zhiwei/brandkbs2/pojo/vo/ProjectVO.java
+3
-3
src/main/java/com/zhiwei/brandkbs2/service/ExtraService.java
+8
-0
src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java
+79
-0
src/main/java/com/zhiwei/brandkbs2/service/TaskService.java
+10
-0
src/main/java/com/zhiwei/brandkbs2/service/ToolsetService.java
+2
-0
src/main/java/com/zhiwei/brandkbs2/service/XiaohongshuWordService.java
+24
-0
src/main/java/com/zhiwei/brandkbs2/service/impl/EsSearchServiceImpl.java
+1
-1
src/main/java/com/zhiwei/brandkbs2/service/impl/ExtraServiceImpl.java
+19
-2
src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java
+448
-137
src/main/java/com/zhiwei/brandkbs2/service/impl/ProjectServiceImpl.java
+10
-4
src/main/java/com/zhiwei/brandkbs2/service/impl/TableWordServiceImpl.java
+2
-2
src/main/java/com/zhiwei/brandkbs2/service/impl/TaskServiceImpl.java
+118
-0
src/main/java/com/zhiwei/brandkbs2/service/impl/ToolsetServiceImpl.java
+1
-9
src/main/java/com/zhiwei/brandkbs2/service/impl/XiaohongshuWordServiceImpl.java
+81
-0
src/main/java/com/zhiwei/brandkbs2/task/ControlCenter.java
+26
-0
src/main/java/com/zhiwei/brandkbs2/util/RedisUtil.java
+18
-12
src/main/resources/application-dev.properties
+2
-0
src/main/resources/application-local.properties
+2
-1
src/main/resources/application-prod.properties
+2
-0
No files found.
src/main/java/com/zhiwei/brandkbs2/common/RedisKeyPrefix.java
View file @
18f92bf9
...
@@ -109,12 +109,18 @@ public class RedisKeyPrefix {
...
@@ -109,12 +109,18 @@ public class RedisKeyPrefix {
*/
*/
public
static
final
String
YUQING_ANALYZE_PROJECT_AVG_COUNT
=
"BRANDKBS:YUQING:ANALYZE:AMOUNT:AVG:"
;
public
static
final
String
YUQING_ANALYZE_PROJECT_AVG_COUNT
=
"BRANDKBS:YUQING:ANALYZE:AMOUNT:AVG:"
;
public
static
final
String
CUSTOM_YUQING_ANALYZE_AVG_COUNT
=
"BRANDKBS:CUSTOM:YUQING:ANALYZE:AMOUNT:AVG:"
;
public
static
final
String
YUQING_ANALYZE_EMOTION_DISTRIBUTION_AVG
=
"BRANDKBS:YUQING:ANALYZE:EMOTION:AVG:"
;
public
static
final
String
YUQING_ANALYZE_EMOTION_DISTRIBUTION_AVG
=
"BRANDKBS:YUQING:ANALYZE:EMOTION:AVG:"
;
public
static
final
String
CUSTOM_YUQING_ANALYZE_EMOTION_DISTRIBUTION_AVG
=
"BRANDKBS:CUSTOM:YUQING:ANALYZE:EMOTION:AVG:"
;
public
static
final
String
YUQING_ANALYZE_PLATFORM_AVG_COUNT
=
"BRANDKBS:YUQING:ANALYZE:PLATFORM:AVG:"
;
public
static
final
String
YUQING_ANALYZE_PLATFORM_AVG_COUNT
=
"BRANDKBS:YUQING:ANALYZE:PLATFORM:AVG:"
;
public
static
final
String
YUQING_ANALYZE_HIGH_WORD
=
"BRANDKBS:YUQING:ANALYZE:HIGH:WORD:"
;
public
static
final
String
YUQING_ANALYZE_HIGH_WORD
=
"BRANDKBS:YUQING:ANALYZE:HIGH:WORD:"
;
public
static
final
String
CUSTOM_YUQING_ANALYZE_HIGH_WORD
=
"BRANDKBS:CUSTOM:YUQING:ANALYZE:HIGH:WORD:"
;
public
static
final
String
SEARCH_KEYWORD
=
"BRANDKBS:SEARCH:KEYWORD:"
;
public
static
final
String
SEARCH_KEYWORD
=
"BRANDKBS:SEARCH:KEYWORD:"
;
public
static
String
projectWarnHotTopKeyAll
(
String
projectId
,
String
type
)
{
public
static
String
projectWarnHotTopKeyAll
(
String
projectId
,
String
type
)
{
...
...
src/main/java/com/zhiwei/brandkbs2/controller/admin/BaseModuleController.java
View file @
18f92bf9
...
@@ -10,15 +10,9 @@ import com.zhiwei.brandkbs2.enmus.RoleEnum;
...
@@ -10,15 +10,9 @@ import com.zhiwei.brandkbs2.enmus.RoleEnum;
import
com.zhiwei.brandkbs2.model.ResponseResult
;
import
com.zhiwei.brandkbs2.model.ResponseResult
;
import
com.zhiwei.brandkbs2.pojo.dto.TagFilterDTO
;
import
com.zhiwei.brandkbs2.pojo.dto.TagFilterDTO
;
import
com.zhiwei.brandkbs2.pojo.vo.ProjectVO
;
import
com.zhiwei.brandkbs2.pojo.vo.ProjectVO
;
import
com.zhiwei.brandkbs2.service.HighWordService
;
import
com.zhiwei.brandkbs2.service.*
;
import
com.zhiwei.brandkbs2.service.HighlightWordService
;
import
com.zhiwei.brandkbs2.service.ProjectService
;
import
com.zhiwei.brandkbs2.service.TagFilterService
;
import
com.zhiwei.brandkbs2.util.Tools
;
import
com.zhiwei.brandkbs2.util.Tools
;
import
io.swagger.annotations.Api
;
import
io.swagger.annotations.*
;
import
io.swagger.annotations.ApiImplicitParam
;
import
io.swagger.annotations.ApiImplicitParams
;
import
io.swagger.annotations.ApiOperation
;
import
org.springframework.web.bind.annotation.*
;
import
org.springframework.web.bind.annotation.*
;
import
org.springframework.web.multipart.MultipartFile
;
import
org.springframework.web.multipart.MultipartFile
;
...
@@ -46,6 +40,9 @@ public class BaseModuleController extends BaseController {
...
@@ -46,6 +40,9 @@ public class BaseModuleController extends BaseController {
@Resource
(
name
=
"highlightWordServiceImpl"
)
@Resource
(
name
=
"highlightWordServiceImpl"
)
HighlightWordService
highlightWordService
;
HighlightWordService
highlightWordService
;
@Resource
(
name
=
"xiaohongshuWordServiceImpl"
)
XiaohongshuWordService
xiaohongshuWordService
;
@Resource
(
name
=
"projectServiceImpl"
)
@Resource
(
name
=
"projectServiceImpl"
)
ProjectService
projectService
;
ProjectService
projectService
;
...
@@ -146,4 +143,38 @@ public class BaseModuleController extends BaseController {
...
@@ -146,4 +143,38 @@ public class BaseModuleController extends BaseController {
return
ResponseResult
.
success
();
return
ResponseResult
.
success
();
}
}
@ApiOperation
(
"获取舆情分析小红书关键词(小红书定制模块)"
)
@ApiImplicitParams
({
@ApiImplicitParam
(
name
=
"page"
,
value
=
"页码"
,
defaultValue
=
"1"
,
paramType
=
"query"
,
dataType
=
"int"
),
@ApiImplicitParam
(
name
=
"size"
,
value
=
"每页记录数"
,
defaultValue
=
"10"
,
paramType
=
"query"
,
dataType
=
"int"
),
@ApiImplicitParam
(
name
=
"keyword"
,
value
=
"搜索关键字"
,
paramType
=
"query"
,
dataType
=
"string"
)})
@GetMapping
(
"/xiaohongshuWord/list"
)
public
ResponseResult
findXiaohongshuWordList
(
@RequestParam
(
value
=
"page"
,
defaultValue
=
"1"
)
int
page
,
@RequestParam
(
value
=
"pageSize"
,
defaultValue
=
"10"
)
int
pageSize
,
@RequestParam
(
value
=
"keyword"
,
defaultValue
=
""
)
String
keyword
)
{
return
ResponseResult
.
success
(
xiaohongshuWordService
.
findWord
(
page
,
pageSize
,
keyword
));
}
@ApiOperation
(
"新增舆情分析小红书关键词(小红书定制模块)"
)
@PostMapping
(
"/xiaohongshuWord/add"
)
public
ResponseResult
addXiaohongshuWordList
(
@ApiParam
(
name
=
"info"
,
value
=
"{keyword:关键词, planName:方案名)"
)
@RequestBody
JSONObject
info
)
{
String
keyword
=
info
.
getString
(
"keyword"
);
String
planName
=
info
.
getString
(
"planName"
);
return
xiaohongshuWordService
.
addWord
(
keyword
,
planName
,
UserThreadLocal
.
getProjectId
(),
UserThreadLocal
.
getNickname
());
}
@ApiOperation
(
"删除舆情分析小红书关键词(小红书定制模块)"
)
@PostMapping
(
"/xiaohongshuWord/delete/{id}"
)
public
ResponseResult
deleteXiaohongshuWord
(
@PathVariable
String
id
)
{
xiaohongshuWordService
.
deleteWord
(
id
);
return
ResponseResult
.
success
();
}
@ApiOperation
(
"更新舆情分析小红书关键词(小红书定制模块)"
)
@PostMapping
(
"/xiaohongshuWord/update"
)
public
ResponseResult
updateXiaohongshuWordList
(
@ApiParam
(
name
=
"info"
,
value
=
"{keyword:关键词, planName:方案名)"
)
@RequestBody
JSONObject
info
)
{
String
id
=
info
.
getString
(
"id"
);
String
keyword
=
info
.
getString
(
"keyword"
);
String
planName
=
info
.
getString
(
"planName"
);
xiaohongshuWordService
.
updateWord
(
id
,
keyword
,
planName
,
UserThreadLocal
.
getProjectId
(),
UserThreadLocal
.
getNickname
());
return
ResponseResult
.
success
();
}
}
}
src/main/java/com/zhiwei/brandkbs2/controller/admin/ExtraController.java
View file @
18f92bf9
...
@@ -119,4 +119,17 @@ public class ExtraController extends BaseController {
...
@@ -119,4 +119,17 @@ public class ExtraController extends BaseController {
extraService
.
adjustInteractionRecord
(
json
.
getInteger
(
"count"
));
extraService
.
adjustInteractionRecord
(
json
.
getInteger
(
"count"
));
return
ResponseResult
.
success
();
return
ResponseResult
.
success
();
}
}
@ApiOperation
(
"互动量更新-舆情分析-数据更新用量(小红书定制模块)"
)
@ApiImplicitParams
({
@ApiImplicitParam
(
name
=
"startTime"
,
value
=
"起始时间"
,
paramType
=
"query"
,
dataType
=
"long"
),
@ApiImplicitParam
(
name
=
"endTime"
,
value
=
"结束时间"
,
paramType
=
"query"
,
dataType
=
"long"
),
@ApiImplicitParam
(
name
=
"timeType"
,
defaultValue
=
"week"
,
value
=
"时间类型字段(week/month)"
,
paramType
=
"query"
,
dataType
=
"string"
)}
)
@GetMapping
(
"/xiaohongshuRecord"
)
public
ResponseResult
findXiaohongshuRecordList
(
@RequestParam
(
value
=
"startTime"
)
long
startTime
,
@RequestParam
(
value
=
"endTime"
)
long
endTime
,
@RequestParam
(
value
=
"timeType"
,
defaultValue
=
"week"
)
String
timeType
)
{
return
ResponseResult
.
success
(
extraService
.
xiaohongshuRecordLog
(
startTime
,
endTime
,
timeType
));
}
}
}
src/main/java/com/zhiwei/brandkbs2/controller/app/AppArticleController.java
View file @
18f92bf9
...
@@ -590,6 +590,67 @@ public class AppArticleController extends BaseController {
...
@@ -590,6 +590,67 @@ public class AppArticleController extends BaseController {
return
ResponseResult
.
success
(
markDataService
.
expandOriginRange
(
dto
));
return
ResponseResult
.
success
(
markDataService
.
expandOriginRange
(
dto
));
}
}
@ApiOperation
(
"定制小红书平台舆情分析-舆情总量"
)
@PostMapping
(
"/analyze/custom-xhs/amount"
)
// @LogRecord(description = "舆情库-舆情分析")
public
ResponseResult
getXhsCustomYuqingAmount
(
@RequestBody
MarkSearchDTO
markSearchDTO
)
{
return
ResponseResult
.
success
(
markDataService
.
getXhsCustomYuqingAmount
(
markSearchDTO
));
}
@ApiOperation
(
"定制小红书平台舆情分析-情感分布"
)
@PostMapping
(
"/analyze/custom-xhs/emotion"
)
public
ResponseResult
getXhsCustomYuqingEmotionDistribution
(
@RequestBody
MarkSearchDTO
markSearchDTO
)
{
return
ResponseResult
.
success
(
markDataService
.
getXhsCustomYuqingEmotionDistribution
(
markSearchDTO
));
}
@ApiOperation
(
"定制小红书平台舆情分析-互动量指标"
)
@PostMapping
(
"/analyze/custom-xhs/interaction-count"
)
public
ResponseResult
getXhsCustomYuqingInteractionCount
(
@RequestBody
MarkSearchDTO
markSearchDTO
)
{
return
ResponseResult
.
success
(
markDataService
.
getXhsCustomYuqingInteractionCount
(
markSearchDTO
));
}
@ApiOperation
(
"定制小红书平台舆情分析-舆情走势图"
)
@PostMapping
(
"/analyze/custom-xhs/tendency"
)
public
ResponseResult
getXhsCustomSpreadTendency
(
@RequestBody
MarkSearchDTO
markSearchDTO
)
{
return
ResponseResult
.
success
(
markDataService
.
getXhsCustomSpreadTendency
(
markSearchDTO
));
}
@ApiOperation
(
"定制小红书平台舆情分析-高互动量发文"
)
@PostMapping
(
"/analyze/custom-xhs/interaction-article"
)
public
ResponseResult
getXhsCustomInteractionArticle
(
@RequestBody
MarkSearchDTO
markSearchDTO
)
{
return
ResponseResult
.
success
(
markDataService
.
getXhsCustomInteractionArticle
(
markSearchDTO
));
}
@ApiOperation
(
"定制小红书平台舆情分析-负面舆情"
)
@PostMapping
(
"/analyze/custom-xhs/negative-article"
)
public
ResponseResult
getXhsCustomNegativeArticle
(
@RequestBody
MarkSearchDTO
markSearchDTO
)
{
return
ResponseResult
.
success
(
markDataService
.
getXhsCustomNegativeArticle
(
markSearchDTO
));
}
@ApiOperation
(
"定制小红书平台舆情分析-ip分布、词云详情页舆情数据"
)
@PostMapping
(
"/analyze/custom-xhs/mark-data"
)
public
ResponseResult
getXhsCustomArticleDetail
(
@RequestBody
MarkSearchDTO
markSearchDTO
)
{
return
ResponseResult
.
success
(
markDataService
.
getXhsCustomArticleDetail
(
markSearchDTO
));
}
@ApiOperation
(
"定制小红书平台舆情分析-ip分布"
)
@PostMapping
(
"/analyze/custom-xhs/ip-located"
)
public
ResponseResult
getXhsCustomArticleIpLocated
(
@RequestBody
MarkSearchDTO
markSearchDTO
)
{
return
ResponseResult
.
success
(
markDataService
.
getXhsCustomArticleIpLocated
(
markSearchDTO
));
}
@ApiOperation
(
"定制小红书平台舆情分析-词云"
)
@PostMapping
(
"/analyze/custom-xhs/high-word"
)
public
ResponseResult
getXhsCustomAnalyzeHighWord
(
@RequestBody
MarkSearchDTO
markSearchDTO
)
{
return
ResponseResult
.
success
(
markDataService
.
getXhsCustomAnalyzeHighWord
(
markSearchDTO
,
true
));
}
@ApiOperation
(
"定制小红书平台舆情分析-小红书实时热门词云"
)
@PostMapping
(
"/analyze/custom-xhs/timely-high-word"
)
public
ResponseResult
getXhsCustomTimelyHighWord
(
@RequestBody
MarkSearchDTO
markSearchDTO
)
{
return
ResponseResult
.
success
(
markDataService
.
getXhsCustomTimelyHighWord
(
markSearchDTO
));
}
private
boolean
checkMTagIllegal
(
StringBuilder
mtag
)
{
private
boolean
checkMTagIllegal
(
StringBuilder
mtag
)
{
List
<
MarkerTag
>
hitTags
=
projectService
.
getProjectById
(
UserThreadLocal
.
getProjectId
()).
getHitTags
();
List
<
MarkerTag
>
hitTags
=
projectService
.
getProjectById
(
UserThreadLocal
.
getProjectId
()).
getHitTags
();
if
(!
Tools
.
isEmpty
(
hitTags
))
{
if
(!
Tools
.
isEmpty
(
hitTags
))
{
...
...
src/main/java/com/zhiwei/brandkbs2/dao/XiaohongshuRecordDao.java
0 → 100644
View file @
18f92bf9
package
com
.
zhiwei
.
brandkbs2
.
dao
;
import
com.zhiwei.brandkbs2.pojo.XiaohongshuRecord
;
import
com.zhiwei.brandkbs2.pojo.vo.LineVO
;
import
java.util.List
;
/**
* @ClassName: XiaohongshuRecordDao
* @Description XiaohongshuRecordDao
* @author: sjj
* @date: 2024-06-28 17:41
*/
public
interface
XiaohongshuRecordDao
extends
BaseMongoDao
<
XiaohongshuRecord
>
{
/**
* @param startTime 起始时间
* @param endTime 结束时间
* @param timeType 时间类型
* @return list
*/
List
<
LineVO
>
aggregateDayLastRecord
(
long
startTime
,
long
endTime
,
String
timeType
);
}
src/main/java/com/zhiwei/brandkbs2/dao/XiaohongshuWordDao.java
0 → 100644
View file @
18f92bf9
package
com
.
zhiwei
.
brandkbs2
.
dao
;
import
com.zhiwei.brandkbs2.pojo.XiaohongshuWord
;
/**
* @ClassName: XiaohongshuWordDao
* @Description XiaohongshuWordDao
* @author: sjj
* @date: 2024-06-28 14:08
*/
public
interface
XiaohongshuWordDao
extends
BaseMongoDao
<
XiaohongshuWord
>{
}
src/main/java/com/zhiwei/brandkbs2/dao/impl/XiaohongshuRecordDaoImpl.java
0 → 100644
View file @
18f92bf9
package
com
.
zhiwei
.
brandkbs2
.
dao
.
impl
;
import
com.alibaba.fastjson.JSONObject
;
import
com.zhiwei.brandkbs2.auth.UserThreadLocal
;
import
com.zhiwei.brandkbs2.dao.XiaohongshuRecordDao
;
import
com.zhiwei.brandkbs2.pojo.XiaohongshuRecord
;
import
com.zhiwei.brandkbs2.pojo.vo.LineVO
;
import
org.springframework.data.domain.Sort
;
import
org.springframework.data.mongodb.core.aggregation.Aggregation
;
import
org.springframework.data.mongodb.core.aggregation.AggregationOperation
;
import
org.springframework.data.mongodb.core.aggregation.AggregationResults
;
import
org.springframework.data.mongodb.core.query.Criteria
;
import
org.springframework.stereotype.Component
;
import
java.util.ArrayList
;
import
java.util.List
;
/**
* @ClassName: XiaohongshuRecordDaoImpl
* @Description XiaohongshuRecordDaoImpl
* @author: sjj
* @date: 2024-06-28 17:43
*/
@Component
(
"xiaohongshuRecordDao"
)
public
class
XiaohongshuRecordDaoImpl
extends
BaseMongoDaoImpl
<
XiaohongshuRecord
>
implements
XiaohongshuRecordDao
{
private
static
final
String
COLLECTION_NAME
=
"brandkbs_xiaohongshu_record"
;
public
XiaohongshuRecordDaoImpl
()
{
super
(
COLLECTION_NAME
);
}
@Override
public
List
<
LineVO
>
aggregateDayLastRecord
(
long
startTime
,
long
endTime
,
String
timeType
)
{
// 2024-06-19 前10位,代表天级;则前7位代表月级
int
norOfChars
=
10
;
if
(
timeType
.
equals
(
"month"
))
{
norOfChars
=
7
;
}
List
<
LineVO
>
res
=
new
ArrayList
<>();
Criteria
criteria
=
Criteria
.
where
(
"projectId"
).
is
(
UserThreadLocal
.
getProjectId
());
List
<
AggregationOperation
>
operations
=
new
ArrayList
<>();
// operations区分先后顺序
operations
.
add
(
Aggregation
.
match
(
criteria
));
operations
.
add
(
Aggregation
.
project
(
"usedCount"
).
and
(
"cTime"
).
substring
(
0
,
norOfChars
).
as
(
"cTime"
));
operations
.
add
(
Aggregation
.
group
(
"cTime"
).
sum
(
"usedCount"
).
as
(
"count"
));
operations
.
add
(
Aggregation
.
sort
(
Sort
.
by
(
Sort
.
Order
.
asc
(
"_id"
))));
Aggregation
aggregation
=
Aggregation
.
newAggregation
(
operations
);
// 聚合结果
AggregationResults
<
JSONObject
>
aggregateResult
=
mongoTemplate
.
aggregate
(
aggregation
,
COLLECTION_NAME
,
JSONObject
.
class
);
List
<
JSONObject
>
mappedResults
=
aggregateResult
.
getMappedResults
();
for
(
JSONObject
mappedResult
:
mappedResults
)
{
res
.
add
(
new
LineVO
(
mappedResult
.
getLong
(
"count"
),
mappedResult
.
getDate
(
"_id"
).
getTime
()));
}
return
res
;
}
}
src/main/java/com/zhiwei/brandkbs2/dao/impl/XiaohongshuWordDaoImpl.java
0 → 100644
View file @
18f92bf9
package
com
.
zhiwei
.
brandkbs2
.
dao
.
impl
;
import
com.zhiwei.brandkbs2.dao.XiaohongshuWordDao
;
import
com.zhiwei.brandkbs2.pojo.XiaohongshuWord
;
import
org.springframework.stereotype.Component
;
/**
* @ClassName: XiaohongshuWordDaoImpl
* @Description XiaohongshuWordDaoImpl
* @author: sjj
* @date: 2024年6月28日16:02:03
*/
@Component
(
"xiaohongshuWordDao"
)
public
class
XiaohongshuWordDaoImpl
extends
BaseMongoDaoImpl
<
XiaohongshuWord
>
implements
XiaohongshuWordDao
{
private
static
final
String
COLLECTION_NAME
=
"brandkbs_xiaohongshu_word"
;
public
XiaohongshuWordDaoImpl
()
{
super
(
COLLECTION_NAME
);
}
}
src/main/java/com/zhiwei/brandkbs2/es/EsClientDao.java
View file @
18f92bf9
...
@@ -7,15 +7,18 @@ import com.zhiwei.brandkbs2.pojo.ChannelIndex;
...
@@ -7,15 +7,18 @@ import com.zhiwei.brandkbs2.pojo.ChannelIndex;
import
com.zhiwei.brandkbs2.util.Tools
;
import
com.zhiwei.brandkbs2.util.Tools
;
import
lombok.Getter
;
import
lombok.Getter
;
import
lombok.Setter
;
import
lombok.Setter
;
import
org.apache.commons.collections4.CollectionUtils
;
import
org.apache.commons.lang3.time.DateUtils
;
import
org.apache.commons.lang3.time.DateUtils
;
import
org.apache.commons.lang3.time.FastDateFormat
;
import
org.apache.commons.lang3.time.FastDateFormat
;
import
org.apache.commons.lang3.tuple.Pair
;
import
org.apache.commons.lang3.tuple.Pair
;
import
org.apache.logging.log4j.LogManager
;
import
org.apache.logging.log4j.LogManager
;
import
org.apache.logging.log4j.Logger
;
import
org.apache.logging.log4j.Logger
;
import
org.elasticsearch.action.bulk.BulkRequest
;
import
org.elasticsearch.action.search.ClearScrollRequest
;
import
org.elasticsearch.action.search.ClearScrollRequest
;
import
org.elasticsearch.action.search.SearchRequest
;
import
org.elasticsearch.action.search.SearchRequest
;
import
org.elasticsearch.action.search.SearchResponse
;
import
org.elasticsearch.action.search.SearchResponse
;
import
org.elasticsearch.action.search.SearchScrollRequest
;
import
org.elasticsearch.action.search.SearchScrollRequest
;
import
org.elasticsearch.action.update.UpdateRequest
;
import
org.elasticsearch.client.RequestOptions
;
import
org.elasticsearch.client.RequestOptions
;
import
org.elasticsearch.client.RestHighLevelClient
;
import
org.elasticsearch.client.RestHighLevelClient
;
import
org.elasticsearch.client.core.CountRequest
;
import
org.elasticsearch.client.core.CountRequest
;
...
@@ -202,6 +205,27 @@ public class EsClientDao {
...
@@ -202,6 +205,27 @@ public class EsClientDao {
return
getAllIndexList
(
startTime
,
endTime
).
toArray
(
new
String
[
0
]);
return
getAllIndexList
(
startTime
,
endTime
).
toArray
(
new
String
[
0
]);
}
}
public
void
batchUpdate
(
List
<
Pair
<
String
,
Map
<
String
,
Object
>>>
updateList
)
{
if
(
CollectionUtils
.
isEmpty
(
updateList
)){
return
;
}
retryTemplate
.
execute
(
context
->
{
try
{
BulkRequest
bulkRequest
=
new
BulkRequest
();
for
(
Pair
<
String
,
Map
<
String
,
Object
>>
pair
:
updateList
)
{
Map
<
String
,
Object
>
map
=
pair
.
getRight
();
String
index
=
pair
.
getLeft
();
bulkRequest
.
add
(
new
UpdateRequest
().
index
(
index
).
id
(
map
.
get
(
"id"
)
+
""
).
doc
(
map
));
}
esClient
.
bulk
(
bulkRequest
,
RequestOptions
.
DEFAULT
);
log
.
info
(
"定制化舆情分析-互动量更新结果数据-批量更新es数据【{}】条"
,
updateList
.
size
());
}
catch
(
Exception
e
)
{
e
.
printStackTrace
();
}
return
null
;
});
}
protected
RestHighLevelClient
getEsClient
()
{
protected
RestHighLevelClient
getEsClient
()
{
return
esClient
;
return
esClient
;
}
}
...
...
src/main/java/com/zhiwei/brandkbs2/es/EsQueryTools.java
View file @
18f92bf9
...
@@ -341,9 +341,11 @@ public class EsQueryTools {
...
@@ -341,9 +341,11 @@ public class EsQueryTools {
* @param dataType 数据类型(1:长文本, 2:短文本, 3:问答, 5:视频)
* @param dataType 数据类型(1:长文本, 2:短文本, 3:问答, 5:视频)
* @return
* @return
*/
*/
public
static
BoolQueryBuilder
assembleC2Query
(
Integer
dataType
)
{
public
static
BoolQueryBuilder
assembleC2Query
(
List
<
Integer
>
dataType
)
{
BoolQueryBuilder
queryBuilder
=
QueryBuilders
.
boolQuery
();
BoolQueryBuilder
queryBuilder
=
QueryBuilders
.
boolQuery
();
queryBuilder
.
should
(
QueryBuilders
.
termQuery
(
GenericAttribute
.
ES_C2
,
dataType
<<
23
));
for
(
Integer
type
:
dataType
)
{
queryBuilder
.
should
(
QueryBuilders
.
termQuery
(
GenericAttribute
.
ES_C2
,
type
<<
23
));
}
return
queryBuilder
;
return
queryBuilder
;
}
}
...
...
src/main/java/com/zhiwei/brandkbs2/pojo/AbstractBaseWord.java
View file @
18f92bf9
...
@@ -36,16 +36,18 @@ public abstract class AbstractBaseWord extends AbstractBaseMongo {
...
@@ -36,16 +36,18 @@ public abstract class AbstractBaseWord extends AbstractBaseMongo {
*/
*/
private
String
projectId
;
private
String
projectId
;
public
static
<
T
>
List
<
T
>
createFromList
(
List
<
String
>
keywords
,
String
projectId
,
String
submitter
)
{
abstract
AbstractBaseWord
newInstance
();
public
<
T
extends
AbstractBaseWord
>
List
<
T
>
createFromList
(
List
<
String
>
keywords
,
String
projectId
,
String
submitter
)
{
List
<
T
>
res
=
new
ArrayList
<>();
List
<
T
>
res
=
new
ArrayList
<>();
Long
now
=
System
.
currentTimeMillis
();
Long
now
=
System
.
currentTimeMillis
();
for
(
String
keyword
:
keywords
)
{
for
(
String
keyword
:
keywords
)
{
AbstractBaseWord
word
=
new
HighWord
();
T
word
=
(
T
)
newInstance
();
word
.
setKeyword
(
keyword
);
word
.
setKeyword
(
keyword
);
word
.
setCTime
(
now
);
word
.
setCTime
(
now
);
word
.
setProjectId
(
projectId
);
word
.
setProjectId
(
projectId
);
word
.
setSubmitter
(
submitter
);
word
.
setSubmitter
(
submitter
);
res
.
add
(
(
T
)
word
);
res
.
add
(
word
);
}
}
return
res
;
return
res
;
}
}
...
...
src/main/java/com/zhiwei/brandkbs2/pojo/HighWord.java
View file @
18f92bf9
...
@@ -14,4 +14,8 @@ import org.springframework.data.mongodb.core.mapping.Document;
...
@@ -14,4 +14,8 @@ import org.springframework.data.mongodb.core.mapping.Document;
@Setter
@Setter
@Document
(
collection
=
"brandkbs_high_word"
)
@Document
(
collection
=
"brandkbs_high_word"
)
public
class
HighWord
extends
AbstractBaseWord
{
public
class
HighWord
extends
AbstractBaseWord
{
@Override
AbstractBaseWord
newInstance
()
{
return
new
HighWord
();
}
}
}
src/main/java/com/zhiwei/brandkbs2/pojo/HighlightWord.java
View file @
18f92bf9
...
@@ -14,4 +14,8 @@ import org.springframework.data.mongodb.core.mapping.Document;
...
@@ -14,4 +14,8 @@ import org.springframework.data.mongodb.core.mapping.Document;
@Setter
@Setter
@Document
(
collection
=
"brandkbs_highlight_word"
)
@Document
(
collection
=
"brandkbs_highlight_word"
)
public
class
HighlightWord
extends
AbstractBaseWord
{
public
class
HighlightWord
extends
AbstractBaseWord
{
@Override
AbstractBaseWord
newInstance
()
{
return
new
HighlightWord
();
}
}
}
src/main/java/com/zhiwei/brandkbs2/pojo/Project.java
View file @
18f92bf9
...
@@ -34,10 +34,9 @@ public class Project extends AbstractProject {
...
@@ -34,10 +34,9 @@ public class Project extends AbstractProject {
private
List
<
Contend
>
contendList
;
private
List
<
Contend
>
contendList
;
/**
/**
* 模块展示列表(
1:标注库 2:大库
)
* 模块展示列表(
xiaohongshu:达达项目定制化
)
*/
*/
@Deprecated
private
List
<
String
>
moduleShowList
;
private
List
<
Integer
>
moduleShowList
;
/**
/**
* 工具库功能入口展示列表(目前有:摘要提取,互动量更新,词云,字节影响力计算,其中前三者默认开启)
* 工具库功能入口展示列表(目前有:摘要提取,互动量更新,词云,字节影响力计算,其中前三者默认开启)
...
...
src/main/java/com/zhiwei/brandkbs2/pojo/XiaohongshuRecord.java
0 → 100644
View file @
18f92bf9
package
com
.
zhiwei
.
brandkbs2
.
pojo
;
import
lombok.Getter
;
import
lombok.Setter
;
import
java.util.Date
;
/**
* @ClassName: XiaohongshuRecord
* @Description 小红书定制模块更新互动量记录
* @author: sjj
* @date: 2024年6月28日17:24:30
*/
@Getter
@Setter
public
class
XiaohongshuRecord
extends
AbstractBaseMongo
{
/**
* 项目id
*/
private
String
projectId
;
/**
* 本次使用数值
*/
private
int
usedCount
;
/**
* 提交人
*/
private
String
submitter
;
/**
* 创建时间(Date类型方便聚合查询)
*/
private
Date
cTime
;
public
static
XiaohongshuRecord
createUsedRecord
(
String
projectId
,
int
usedCount
,
String
submitter
)
{
XiaohongshuRecord
record
=
new
XiaohongshuRecord
();
record
.
projectId
=
projectId
;
record
.
usedCount
=
usedCount
;
record
.
submitter
=
submitter
;
record
.
cTime
=
new
Date
();
return
record
;
}
}
src/main/java/com/zhiwei/brandkbs2/pojo/XiaohongshuWord.java
0 → 100644
View file @
18f92bf9
package
com
.
zhiwei
.
brandkbs2
.
pojo
;
import
lombok.Getter
;
import
lombok.Setter
;
import
org.springframework.data.mongodb.core.mapping.Document
;
/**
* @ClassName: XiaohongshuWord
* @Description 小红书定制配置词
* @author: sjj
* @date: 2024-06-28 14:08
*/
@Getter
@Setter
@Document
(
collection
=
"brandkbs_xiaohongshu_word"
)
public
class
XiaohongshuWord
extends
AbstractBaseWord
{
private
String
planName
;
@Override
AbstractBaseWord
newInstance
()
{
return
new
XiaohongshuWord
();
}
}
src/main/java/com/zhiwei/brandkbs2/pojo/dto/MarkSearchDTO.java
View file @
18f92bf9
...
@@ -183,13 +183,17 @@ public class MarkSearchDTO {
...
@@ -183,13 +183,17 @@ public class MarkSearchDTO {
*/
*/
@ApiModelProperty
(
value
=
"屏蔽词检索字段:全文|标题"
)
@ApiModelProperty
(
value
=
"屏蔽词检索字段:全文|标题"
)
private
String
filterType
;
private
String
filterType
;
/**
/**
* 方案id,仅非人工项目
*
非人工项目-
方案id,仅非人工项目
*/
*/
@ApiModelProperty
(
value
=
"方案id"
)
@ApiModelProperty
(
value
=
"方案id"
)
private
String
planId
;
private
String
planId
;
/**
/**
* 定制化舆情分析页面-方案id
*/
@ApiModelProperty
(
value
=
"定制化舆情分析页面方案id"
)
private
String
customPlanId
;
/**
* 内容类型(原创/转发),仅非人工项目
* 内容类型(原创/转发),仅非人工项目
*/
*/
@ApiModelProperty
(
value
=
"内容类型(原创/转发)"
)
@ApiModelProperty
(
value
=
"内容类型(原创/转发)"
)
...
@@ -203,7 +207,7 @@ public class MarkSearchDTO {
...
@@ -203,7 +207,7 @@ public class MarkSearchDTO {
* 数据类型(长文本/短文本/问答/视频),仅非人工项目
* 数据类型(长文本/短文本/问答/视频),仅非人工项目
*/
*/
@ApiModelProperty
(
value
=
"数据类型(1:长文本, 2:短文本, 3:问答, 5:视频)"
)
@ApiModelProperty
(
value
=
"数据类型(1:长文本, 2:短文本, 3:问答, 5:视频)"
)
private
Integer
dataType
;
private
List
<
Integer
>
dataType
;
/**
/**
* ip地址
* ip地址
*/
*/
...
...
src/main/java/com/zhiwei/brandkbs2/pojo/vo/ProjectVO.java
View file @
18f92bf9
...
@@ -88,10 +88,10 @@ public class ProjectVO {
...
@@ -88,10 +88,10 @@ public class ProjectVO {
private
List
<
Contend
>
contendList
;
private
List
<
Contend
>
contendList
;
/**
/**
* 模块展示列表(
1:标注库 2:大库
)
* 模块展示列表(
xiaohongshu:达达项目定制化
)
*/
*/
@ApiModelProperty
(
"
舆情数据源(模块展示列表(1:标注库 2:大库))
"
)
@ApiModelProperty
(
"
xiaohongshu:达达项目定制化
"
)
private
List
<
Integer
>
moduleShowList
;
private
List
<
String
>
moduleShowList
;
/**
/**
* 工具库功能入口展示列表(目前有:摘要提取,互动量更新,词云,字节影响力计算,其中前三者默认开启)
* 工具库功能入口展示列表(目前有:摘要提取,互动量更新,词云,字节影响力计算,其中前三者默认开启)
...
...
src/main/java/com/zhiwei/brandkbs2/service/ExtraService.java
View file @
18f92bf9
...
@@ -115,4 +115,12 @@ public interface ExtraService {
...
@@ -115,4 +115,12 @@ public interface ExtraService {
* @param usedCount 使用次数
* @param usedCount 使用次数
*/
*/
void
decreaseInteractionRecord
(
InteractionUpdateRecord
.
UsedType
usedType
,
Integer
usedCount
);
void
decreaseInteractionRecord
(
InteractionUpdateRecord
.
UsedType
usedType
,
Integer
usedCount
);
/**
* 数据总览
*
* @return json
*/
JSONObject
xiaohongshuRecordLog
(
long
startTime
,
long
endTime
,
String
timeType
);
}
}
src/main/java/com/zhiwei/brandkbs2/service/MarkDataService.java
View file @
18f92bf9
...
@@ -545,6 +545,8 @@ public interface MarkDataService {
...
@@ -545,6 +545,8 @@ public interface MarkDataService {
*/
*/
void
countYuqingAmountAvg
(
Long
startTime
,
Long
endTime
,
String
projectId
,
String
planId
)
throws
IOException
;
void
countYuqingAmountAvg
(
Long
startTime
,
Long
endTime
,
String
projectId
,
String
planId
)
throws
IOException
;
void
countCustomYuqingAmountAvg
(
Long
startTime
,
Long
endTime
,
String
projectId
,
String
customPlanId
)
throws
IOException
;
/**
/**
* 计算近一年情感分布均值
* 计算近一年情感分布均值
* @param projectId
* @param projectId
...
@@ -553,6 +555,8 @@ public interface MarkDataService {
...
@@ -553,6 +555,8 @@ public interface MarkDataService {
*/
*/
void
countEmotionDistributionAvg
(
Long
startTime
,
Long
endTime
,
String
projectId
,
String
planId
)
throws
IOException
;
void
countEmotionDistributionAvg
(
Long
startTime
,
Long
endTime
,
String
projectId
,
String
planId
)
throws
IOException
;
void
countCustomEmotionDistributionAvg
(
Long
startTime
,
Long
endTime
,
String
projectId
,
String
customPlanId
)
throws
IOException
;
/**
/**
* 计算项目近一年重点平台均值
* 计算项目近一年重点平台均值
* @param projectId
* @param projectId
...
@@ -636,6 +640,81 @@ public interface MarkDataService {
...
@@ -636,6 +640,81 @@ public interface MarkDataService {
List
<
JSONObject
>
getSearchAnalyzeHighWord
(
MarkSearchDTO
dto
);
List
<
JSONObject
>
getSearchAnalyzeHighWord
(
MarkSearchDTO
dto
);
/**
/**
* 达达项目定制小红书平台舆情分析-舆情总量
* @param dto 标注数据搜索传输类
* @return
* @throws IOException
*/
JSONObject
getXhsCustomYuqingAmount
(
MarkSearchDTO
dto
);
/**
* 达达项目定制小红书平台舆情分析-情感分布
* @param dto 标注数据搜索传输类
* @return
* @throws IOException
*/
JSONObject
getXhsCustomYuqingEmotionDistribution
(
MarkSearchDTO
dto
);
/**
* 达达项目定制小红书平台舆情分析-互动量指标
* @param dto 标注数据搜索传输类
* @return
* @throws IOException
*/
JSONObject
getXhsCustomYuqingInteractionCount
(
MarkSearchDTO
dto
);
/**
* 达达项目定制小红书平台舆情分析-舆情走势图
* @param dto 标注数据搜索传输类
* @return
*/
JSONObject
getXhsCustomSpreadTendency
(
MarkSearchDTO
dto
);
/**
* 达达项目定制小红书平台舆情分析-高互动量发文
* @param dto 标注数据搜索传输类
* @return
*/
List
<
MarkFlowEntity
>
getXhsCustomInteractionArticle
(
MarkSearchDTO
dto
);
/**
* 达达项目定制小红书平台舆情分析-负面舆情
* @param dto 标注数据搜索传输类
* @return
*/
PageVO
<
MarkFlowEntity
>
getXhsCustomNegativeArticle
(
MarkSearchDTO
dto
);
/**
* 达达项目定制小红书平台舆情分析-ip分布、词云详情页舆情数据
* @param dto 标注数据搜索传输类
* @return
*/
PageVO
<
MarkFlowEntity
>
getXhsCustomArticleDetail
(
MarkSearchDTO
dto
);
/**
* 达达项目定制小红书平台舆情分析-ip分布
* @param dto 标注数据搜索传输类
* @return
*/
List
<
JSONObject
>
getXhsCustomArticleIpLocated
(
MarkSearchDTO
dto
);
/**
* 达达项目定制小红书平台舆情分析-词云
* @param dto 标注数据搜索传输类
* @return
*/
List
<
JSONObject
>
getXhsCustomAnalyzeHighWord
(
MarkSearchDTO
dto
,
boolean
cache
);
void
cacheXhsCustomHighWord
(
Long
startTime
,
Long
endTime
,
String
customPlanId
);
/**
* 达达项目定制小红书平台舆情分析-小红书实时热门词云
* @param dto 标注数据搜索传输类
* @return
*/
List
<
JSONObject
>
getXhsCustomTimelyHighWord
(
MarkSearchDTO
dto
);
/**
* 新-舆情分析-高频标题
* 新-舆情分析-高频标题
* @param startTime
* @param startTime
* @param endTime
* @param endTime
...
...
src/main/java/com/zhiwei/brandkbs2/service/TaskService.java
View file @
18f92bf9
...
@@ -62,4 +62,14 @@ public interface TaskService{
...
@@ -62,4 +62,14 @@ public interface TaskService{
* 生成每日一报
* 生成每日一报
*/
*/
void
generateDailyReport
();
void
generateDailyReport
();
/**
* 每月互动量更新 时间范围近六个月
*/
void
monthlyCustomXhsInteractionUpdate
();
/**
* 每日互动量更新 时间范围近10天
*/
void
dailyCustomXhsInteractionUpdate
();
}
}
src/main/java/com/zhiwei/brandkbs2/service/ToolsetService.java
View file @
18f92bf9
...
@@ -42,6 +42,8 @@ public interface ToolsetService {
...
@@ -42,6 +42,8 @@ public interface ToolsetService {
*/
*/
ResponseResult
urlInteractionUpdate
(
MultipartFile
file
);
ResponseResult
urlInteractionUpdate
(
MultipartFile
file
);
List
<
JSONObject
>
urlInteractionUpdate
(
List
<
String
>
urls
);
/**
/**
* 获取互动量更新剩余可用次数
* 获取互动量更新剩余可用次数
* @return
* @return
...
...
src/main/java/com/zhiwei/brandkbs2/service/XiaohongshuWordService.java
0 → 100644
View file @
18f92bf9
package
com
.
zhiwei
.
brandkbs2
.
service
;
import
com.zhiwei.brandkbs2.model.ResponseResult
;
/**
* @ClassName: XiaohongshuWordService
* @Description 小红书定制模块关键词接口
* @author: sjj
* @date: 2024-06-28 10:09
*/
public
interface
XiaohongshuWordService
extends
TableWordService
{
/**
* 添加关键词
* @param keyword 关键词
* @param planName 方案名
* @param projectId 项目id
* @param submitter 提交人
* @return
*/
ResponseResult
addWord
(
String
keyword
,
String
planName
,
String
projectId
,
String
submitter
);
void
updateWord
(
String
id
,
String
keyword
,
String
planName
,
String
projectId
,
String
submitter
);
}
src/main/java/com/zhiwei/brandkbs2/service/impl/EsSearchServiceImpl.java
View file @
18f92bf9
...
@@ -334,7 +334,7 @@ public class EsSearchServiceImpl implements EsSearchService {
...
@@ -334,7 +334,7 @@ public class EsSearchServiceImpl implements EsSearchService {
}
}
}
}
// dataType
// dataType
if
(
Objects
.
nonNull
(
dto
.
getDataType
())){
if
(
CollectionUtils
.
isNotEmpty
(
dto
.
getDataType
())){
postFilter
.
must
(
EsQueryTools
.
assembleC2Query
(
dto
.
getDataType
()));
postFilter
.
must
(
EsQueryTools
.
assembleC2Query
(
dto
.
getDataType
()));
}
}
// ip
// ip
...
...
src/main/java/com/zhiwei/brandkbs2/service/impl/ExtraServiceImpl.java
View file @
18f92bf9
...
@@ -6,9 +6,11 @@ import com.zhiwei.brandkbs2.config.Constant;
...
@@ -6,9 +6,11 @@ import com.zhiwei.brandkbs2.config.Constant;
import
com.zhiwei.brandkbs2.dao.InteractionUpdateRecordDao
;
import
com.zhiwei.brandkbs2.dao.InteractionUpdateRecordDao
;
import
com.zhiwei.brandkbs2.dao.ProjectDao
;
import
com.zhiwei.brandkbs2.dao.ProjectDao
;
import
com.zhiwei.brandkbs2.dao.WholeSearchRecordDao
;
import
com.zhiwei.brandkbs2.dao.WholeSearchRecordDao
;
import
com.zhiwei.brandkbs2.
pojo.Project
;
import
com.zhiwei.brandkbs2.
dao.XiaohongshuRecordDao
;
import
com.zhiwei.brandkbs2.pojo.InteractionUpdateRecord
;
import
com.zhiwei.brandkbs2.pojo.InteractionUpdateRecord
;
import
com.zhiwei.brandkbs2.pojo.Project
;
import
com.zhiwei.brandkbs2.pojo.WholeSearchRecord
;
import
com.zhiwei.brandkbs2.pojo.WholeSearchRecord
;
import
com.zhiwei.brandkbs2.pojo.vo.LineVO
;
import
com.zhiwei.brandkbs2.pojo.vo.PageVO
;
import
com.zhiwei.brandkbs2.pojo.vo.PageVO
;
import
com.zhiwei.brandkbs2.service.ExtraService
;
import
com.zhiwei.brandkbs2.service.ExtraService
;
import
com.zhiwei.brandkbs2.util.MongoUtil
;
import
com.zhiwei.brandkbs2.util.MongoUtil
;
...
@@ -20,7 +22,6 @@ import org.springframework.stereotype.Service;
...
@@ -20,7 +22,6 @@ import org.springframework.stereotype.Service;
import
javax.annotation.Resource
;
import
javax.annotation.Resource
;
import
java.util.Date
;
import
java.util.Date
;
import
java.util.List
;
import
java.util.List
;
import
java.util.Objects
;
import
java.util.stream.Collectors
;
import
java.util.stream.Collectors
;
/**
/**
...
@@ -38,6 +39,9 @@ public class ExtraServiceImpl implements ExtraService {
...
@@ -38,6 +39,9 @@ public class ExtraServiceImpl implements ExtraService {
@Resource
(
name
=
"interactionUpdateRecordDao"
)
@Resource
(
name
=
"interactionUpdateRecordDao"
)
private
InteractionUpdateRecordDao
interactionUpdateRecordDao
;
private
InteractionUpdateRecordDao
interactionUpdateRecordDao
;
@Resource
(
name
=
"xiaohongshuRecordDao"
)
private
XiaohongshuRecordDao
xiaohongshuRecordDao
;
@Resource
(
name
=
"projectDao"
)
@Resource
(
name
=
"projectDao"
)
private
ProjectDao
projectDao
;
private
ProjectDao
projectDao
;
...
@@ -204,6 +208,19 @@ public class ExtraServiceImpl implements ExtraService {
...
@@ -204,6 +208,19 @@ public class ExtraServiceImpl implements ExtraService {
interactionUpdateRecordDao
.
insertOne
(
record
,
interactionUpdateRecordDao
.
generateCollectionName
());
interactionUpdateRecordDao
.
insertOne
(
record
,
interactionUpdateRecordDao
.
generateCollectionName
());
}
}
@Override
public
JSONObject
xiaohongshuRecordLog
(
long
startTime
,
long
endTime
,
String
timeType
)
{
JSONObject
json
=
new
JSONObject
();
Query
query
=
new
Query
();
query
.
addCriteria
(
Criteria
.
where
(
"projectId"
).
is
(
UserThreadLocal
.
getProjectId
()));
query
.
addCriteria
(
Criteria
.
where
(
"cTime"
).
gte
(
new
Date
(
startTime
)).
lt
(
new
Date
(
endTime
)));
List
<
LineVO
>
lineList
=
xiaohongshuRecordDao
.
aggregateDayLastRecord
(
startTime
,
endTime
,
timeType
);
long
sum
=
lineList
.
stream
().
mapToLong
(
LineVO:
:
getCount
).
sum
();
json
.
put
(
"count"
,
sum
);
json
.
put
(
"trendList"
,
lineList
);
return
json
;
}
private
List
<
JSONObject
>
interactionUsedList
(
Query
query
,
String
...
collectionNames
)
{
private
List
<
JSONObject
>
interactionUsedList
(
Query
query
,
String
...
collectionNames
)
{
return
interactionUpdateRecordDao
.
findList
(
query
,
collectionNames
).
stream
().
map
(
record
->
{
return
interactionUpdateRecordDao
.
findList
(
query
,
collectionNames
).
stream
().
map
(
record
->
{
JSONObject
json
=
new
JSONObject
();
JSONObject
json
=
new
JSONObject
();
...
...
src/main/java/com/zhiwei/brandkbs2/service/impl/MarkDataServiceImpl.java
View file @
18f92bf9
...
@@ -64,6 +64,8 @@ import org.elasticsearch.search.aggregations.bucket.histogram.ParsedDateHistogra
...
@@ -64,6 +64,8 @@ import org.elasticsearch.search.aggregations.bucket.histogram.ParsedDateHistogra
import
org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms
;
import
org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms
;
import
org.elasticsearch.search.aggregations.bucket.terms.Terms
;
import
org.elasticsearch.search.aggregations.bucket.terms.Terms
;
import
org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder
;
import
org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder
;
import
org.elasticsearch.search.aggregations.metrics.ParsedSum
;
import
org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder
;
import
org.elasticsearch.search.sort.FieldSortBuilder
;
import
org.elasticsearch.search.sort.FieldSortBuilder
;
import
org.elasticsearch.search.sort.SortBuilders
;
import
org.elasticsearch.search.sort.SortBuilders
;
import
org.elasticsearch.search.sort.SortOrder
;
import
org.elasticsearch.search.sort.SortOrder
;
...
@@ -73,10 +75,8 @@ import org.springframework.beans.factory.annotation.Value;
...
@@ -73,10 +75,8 @@ import org.springframework.beans.factory.annotation.Value;
import
org.springframework.data.mongodb.core.query.Criteria
;
import
org.springframework.data.mongodb.core.query.Criteria
;
import
org.springframework.data.mongodb.core.query.Query
;
import
org.springframework.data.mongodb.core.query.Query
;
import
org.springframework.data.mongodb.core.query.Update
;
import
org.springframework.data.mongodb.core.query.Update
;
import
org.springframework.http.HttpEntity
;
import
org.springframework.http.*
;
import
org.springframework.http.HttpHeaders
;
import
org.springframework.http.MediaType
;
import
org.springframework.http.MediaType
;
import
org.springframework.http.ResponseEntity
;
import
org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor
;
import
org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor
;
import
org.springframework.stereotype.Service
;
import
org.springframework.stereotype.Service
;
import
org.springframework.web.client.RestTemplate
;
import
org.springframework.web.client.RestTemplate
;
...
@@ -106,6 +106,8 @@ public class MarkDataServiceImpl implements MarkDataService {
...
@@ -106,6 +106,8 @@ public class MarkDataServiceImpl implements MarkDataService {
private
static
final
String
ORIGIN_SEARCH_PATTERN
=
"[\\p{P}+~$`^=|<>~`$^+=|<>¥×\\s\u200B\u200C\u200D\u00A0\u0020\u3000\\d]"
;
private
static
final
String
ORIGIN_SEARCH_PATTERN
=
"[\\p{P}+~$`^=|<>~`$^+=|<>¥×\\s\u200B\u200C\u200D\u00A0\u0020\u3000\\d]"
;
private
static
final
String
XHS_PLATFORM_ID
=
"6433c2251701316728003be4"
;
@Value
(
"${istarshine.addIStarShineKSData.url}"
)
@Value
(
"${istarshine.addIStarShineKSData.url}"
)
private
String
addIStarShineKSDataUrl
;
private
String
addIStarShineKSDataUrl
;
...
@@ -124,6 +126,15 @@ public class MarkDataServiceImpl implements MarkDataService {
...
@@ -124,6 +126,15 @@ public class MarkDataServiceImpl implements MarkDataService {
@Value
(
"${istarshine.subject.updateTask.url}"
)
@Value
(
"${istarshine.subject.updateTask.url}"
)
private
String
updateIStarShineSubjectTaskUrl
;
private
String
updateIStarShineSubjectTaskUrl
;
@Value
(
"${trends.nearYearWords.url}"
)
private
String
nearYearWordsUrl
;
@Value
(
"${trend.nearYearWords.token}"
)
private
String
nearYearWordsToken
;
@Value
(
"${trends.longTimeInListSearchByInner.url}"
)
private
String
trendsListUrl
;
@Resource
(
name
=
"restTemplate"
)
@Resource
(
name
=
"restTemplate"
)
private
RestTemplate
restTemplate
;
private
RestTemplate
restTemplate
;
...
@@ -136,6 +147,9 @@ public class MarkDataServiceImpl implements MarkDataService {
...
@@ -136,6 +147,9 @@ public class MarkDataServiceImpl implements MarkDataService {
@Resource
(
name
=
"hotEsDao"
)
@Resource
(
name
=
"hotEsDao"
)
private
EsClientDao
hotEsDao
;
private
EsClientDao
hotEsDao
;
@Resource
(
name
=
"xiaohongshuWordDao"
)
private
XiaohongshuWordDao
xiaohongshuWordDao
;
@Resource
(
name
=
"commonServiceImpl"
)
@Resource
(
name
=
"commonServiceImpl"
)
private
CommonService
commonService
;
private
CommonService
commonService
;
...
@@ -1021,7 +1035,7 @@ public class MarkDataServiceImpl implements MarkDataService {
...
@@ -1021,7 +1035,7 @@ public class MarkDataServiceImpl implements MarkDataService {
private
BaseMap
getArticleSort
(
MarkSearchDTO
dto
,
String
sorter
,
String
aggTitle
)
throws
IOException
{
private
BaseMap
getArticleSort
(
MarkSearchDTO
dto
,
String
sorter
,
String
aggTitle
)
throws
IOException
{
// 索引
// 索引
String
[]
indexes
=
esClientDao
.
getIndexes
();
String
[]
indexes
=
esClientDao
.
getIndexes
();
BoolQueryBuilder
query
=
yuqingAnalyzeQuery
(
dto
,
aggTitle
);
BoolQueryBuilder
query
=
yuqingAnalyzeQuery
(
dto
,
null
,
aggTitle
);
//sort
//sort
FieldSortBuilder
sort
=
new
FieldSortBuilder
(
"time"
).
order
(
SortOrder
.
ASC
);
FieldSortBuilder
sort
=
new
FieldSortBuilder
(
"time"
).
order
(
SortOrder
.
ASC
);
JSONObject
sortJson
=
JSONObject
.
parseObject
(
sorter
);
JSONObject
sortJson
=
JSONObject
.
parseObject
(
sorter
);
...
@@ -2479,15 +2493,10 @@ public class MarkDataServiceImpl implements MarkDataService {
...
@@ -2479,15 +2493,10 @@ public class MarkDataServiceImpl implements MarkDataService {
List
<
JSONObject
>
res
=
new
ArrayList
<>();
List
<
JSONObject
>
res
=
new
ArrayList
<>();
String
projectId
=
UserThreadLocal
.
getProjectId
();
String
projectId
=
UserThreadLocal
.
getProjectId
();
List
<
NonManualProjectPlan
>
planList
=
nonManualProjectPlanDao
.
findList
(
new
Query
(
Criteria
.
where
(
"projectId"
).
is
(
projectId
)));
List
<
NonManualProjectPlan
>
planList
=
nonManualProjectPlanDao
.
findList
(
new
Query
(
Criteria
.
where
(
"projectId"
).
is
(
projectId
)));
// long startTime = Constant.DF_yyyyMMdd.parse(Constant.DF_yyyyMMdd.format(new Date())).getTime();
// long endTime = System.currentTimeMillis();
for
(
NonManualProjectPlan
plan
:
planList
)
{
for
(
NonManualProjectPlan
plan
:
planList
)
{
JSONObject
jsonObject
=
new
JSONObject
();
JSONObject
jsonObject
=
new
JSONObject
();
jsonObject
.
put
(
"id"
,
plan
.
getId
());
jsonObject
.
put
(
"id"
,
plan
.
getId
());
jsonObject
.
put
(
"name"
,
plan
.
getName
());
jsonObject
.
put
(
"name"
,
plan
.
getName
());
// BoolQueryBuilder query = EsQueryTools.assembleCacheMapsPlanQuery(projectId, plan.getId());
// query.must(QueryBuilders.rangeQuery("time").gte(startTime).lt(endTime));
// jsonObject.put("count", esClientDao.count(query));
res
.
add
(
jsonObject
);
res
.
add
(
jsonObject
);
}
}
return
res
;
return
res
;
...
@@ -2499,29 +2508,47 @@ public class MarkDataServiceImpl implements MarkDataService {
...
@@ -2499,29 +2508,47 @@ public class MarkDataServiceImpl implements MarkDataService {
@Override
@Override
public
void
countYuqingAmountAvg
(
Long
startTime
,
Long
endTime
,
String
projectId
,
String
planId
)
throws
IOException
{
public
void
countYuqingAmountAvg
(
Long
startTime
,
Long
endTime
,
String
projectId
,
String
planId
)
throws
IOException
{
Long
total
=
getYuqingAnalyzeCount
(
startTime
,
endTime
,
projectId
,
planId
,
null
,
null
);
countYuqingAmountAvg
(
startTime
,
endTime
,
projectId
,
planId
,
null
,
null
,
false
);
}
@Override
public
void
countCustomYuqingAmountAvg
(
Long
startTime
,
Long
endTime
,
String
projectId
,
String
customPlanId
)
throws
IOException
{
countYuqingAmountAvg
(
startTime
,
endTime
,
projectId
,
null
,
customPlanId
,
Collections
.
singletonList
(
"小红书"
),
true
);
}
private
void
countYuqingAmountAvg
(
Long
startTime
,
Long
endTime
,
String
projectId
,
String
planId
,
String
customPlanId
,
List
<
String
>
platforms
,
boolean
isCustom
)
throws
IOException
{
String
projectYuqingCountAvgKey
=
RedisUtil
.
getYuqingAnalyzeProjectAvgCountKey
(
projectId
,
Constant
.
PRIMARY_CONTEND_ID
,
planId
,
customPlanId
,
isCustom
);
Long
total
=
getTaskYuqingAnalyzeCount
(
startTime
,
endTime
,
projectId
,
planId
,
customPlanId
,
null
,
platforms
);
int
days
=
new
Period
(
startTime
,
endTime
,
PeriodType
.
days
()).
getDays
();
int
days
=
new
Period
(
startTime
,
endTime
,
PeriodType
.
days
()).
getDays
();
days
=
0
==
days
?
1
:
days
;
days
=
0
==
days
?
1
:
days
;
// avg
// avg
double
avg
=
Objects
.
isNull
(
total
)
||
0
==
total
?
0
d
:
total
/
(
double
)
days
;
double
avg
=
Objects
.
isNull
(
total
)
||
0
==
total
?
0
d
:
total
/
(
double
)
days
;
String
projectYuqingCountAvgKey
=
RedisUtil
.
getYuqingAnalyzeProjectAvgCountKey
(
projectId
,
Constant
.
PRIMARY_CONTEND_ID
,
planId
);
redisUtil
.
set
(
projectYuqingCountAvgKey
,
String
.
valueOf
(
avg
));
redisUtil
.
set
(
projectYuqingCountAvgKey
,
String
.
valueOf
(
avg
));
}
}
@Override
@Override
public
void
countEmotionDistributionAvg
(
Long
startTime
,
Long
endTime
,
String
projectId
,
String
planId
)
throws
IOException
{
public
void
countEmotionDistributionAvg
(
Long
startTime
,
Long
endTime
,
String
projectId
,
String
planId
)
throws
IOException
{
countEmotionDistributionAvg
(
startTime
,
endTime
,
projectId
,
planId
,
null
,
null
,
false
);
}
@Override
public
void
countCustomEmotionDistributionAvg
(
Long
startTime
,
Long
endTime
,
String
projectId
,
String
customPlanId
)
throws
IOException
{
countEmotionDistributionAvg
(
startTime
,
endTime
,
projectId
,
null
,
customPlanId
,
Collections
.
singletonList
(
"小红书"
),
true
);
}
private
void
countEmotionDistributionAvg
(
Long
startTime
,
Long
endTime
,
String
projectId
,
String
planId
,
String
customPlanId
,
List
<
String
>
platforms
,
boolean
isCustom
)
throws
IOException
{
// key
String
positiveKey
=
RedisUtil
.
getYuqingAnalyzeEmotionDistributionAvgKey
(
projectId
,
Constant
.
PRIMARY_CONTEND_ID
,
EmotionEnum
.
POSITIVE
.
getName
(),
planId
,
customPlanId
,
isCustom
);
String
neutralKey
=
RedisUtil
.
getYuqingAnalyzeEmotionDistributionAvgKey
(
projectId
,
Constant
.
PRIMARY_CONTEND_ID
,
EmotionEnum
.
NEUTRAL
.
getName
(),
planId
,
customPlanId
,
isCustom
);
String
negativeKey
=
RedisUtil
.
getYuqingAnalyzeEmotionDistributionAvgKey
(
projectId
,
Constant
.
PRIMARY_CONTEND_ID
,
EmotionEnum
.
NEGATIVE
.
getName
(),
planId
,
customPlanId
,
isCustom
);
// 舆情总量
// 舆情总量
Long
total
=
get
YuqingAnalyzeCount
(
startTime
,
endTime
,
projectId
,
planId
,
null
,
null
);
Long
total
=
get
TaskYuqingAnalyzeCount
(
startTime
,
endTime
,
projectId
,
planId
,
customPlanId
,
null
,
platforms
);
// 正面舆情
// 正面舆情
long
positiveCount
=
get
YuqingAnalyzeCount
(
startTime
,
endTime
,
projectId
,
planId
,
EmotionEnum
.
POSITIVE
.
getName
(),
null
);
long
positiveCount
=
get
TaskYuqingAnalyzeCount
(
startTime
,
endTime
,
projectId
,
planId
,
customPlanId
,
EmotionEnum
.
POSITIVE
.
getName
(),
platforms
);
// 中性舆情
// 中性舆情
long
neutralCount
=
get
YuqingAnalyzeCount
(
startTime
,
endTime
,
projectId
,
planId
,
EmotionEnum
.
NEUTRAL
.
getName
(),
null
);
long
neutralCount
=
get
TaskYuqingAnalyzeCount
(
startTime
,
endTime
,
projectId
,
planId
,
customPlanId
,
EmotionEnum
.
NEUTRAL
.
getName
(),
platforms
);
// 负面舆情
// 负面舆情
long
negativeCount
=
getYuqingAnalyzeCount
(
startTime
,
endTime
,
projectId
,
planId
,
EmotionEnum
.
NEGATIVE
.
getName
(),
null
);
long
negativeCount
=
getTaskYuqingAnalyzeCount
(
startTime
,
endTime
,
projectId
,
planId
,
customPlanId
,
EmotionEnum
.
NEGATIVE
.
getName
(),
platforms
);
// key
String
positiveKey
=
RedisUtil
.
getYuqingAnalyzeEmotionDistributionAvgKey
(
projectId
,
Constant
.
PRIMARY_CONTEND_ID
,
EmotionEnum
.
POSITIVE
.
getName
(),
planId
);
String
neutralKey
=
RedisUtil
.
getYuqingAnalyzeEmotionDistributionAvgKey
(
projectId
,
Constant
.
PRIMARY_CONTEND_ID
,
EmotionEnum
.
NEUTRAL
.
getName
(),
planId
);
String
negativeKey
=
RedisUtil
.
getYuqingAnalyzeEmotionDistributionAvgKey
(
projectId
,
Constant
.
PRIMARY_CONTEND_ID
,
EmotionEnum
.
NEGATIVE
.
getName
(),
planId
);
// avg
// avg
double
positiveAvg
=
Objects
.
isNull
(
total
)
||
0
==
total
?
0
d
:
positiveCount
/
(
double
)
total
;
double
positiveAvg
=
Objects
.
isNull
(
total
)
||
0
==
total
?
0
d
:
positiveCount
/
(
double
)
total
;
double
neutralAvg
=
Objects
.
isNull
(
total
)
||
0
==
total
?
0
d
:
neutralCount
/
(
double
)
total
;
double
neutralAvg
=
Objects
.
isNull
(
total
)
||
0
==
total
?
0
d
:
neutralCount
/
(
double
)
total
;
...
@@ -2533,7 +2560,7 @@ public class MarkDataServiceImpl implements MarkDataService {
...
@@ -2533,7 +2560,7 @@ public class MarkDataServiceImpl implements MarkDataService {
@Override
@Override
public
void
countImportantPlatformPercentageAvg
(
Long
startTime
,
Long
endTime
,
String
projectId
,
String
planId
)
throws
IOException
{
public
void
countImportantPlatformPercentageAvg
(
Long
startTime
,
Long
endTime
,
String
projectId
,
String
planId
)
throws
IOException
{
Long
total
=
get
YuqingAnalyzeCount
(
startTime
,
endTime
,
projectId
,
planId
,
null
,
Arrays
.
asList
(
"微博"
,
"微信"
,
"抖音"
,
"小红书"
,
"今日头条"
,
"网媒"
));
Long
total
=
get
TaskYuqingAnalyzeCount
(
startTime
,
endTime
,
projectId
,
planId
,
null
,
null
,
Arrays
.
asList
(
"微博"
,
"微信"
,
"抖音"
,
"小红书"
,
"今日头条"
,
"网媒"
));
String
key
=
RedisUtil
.
getYuqingAnalyzePlatformAvgCountKey
(
projectId
,
Constant
.
PRIMARY_CONTEND_ID
,
planId
);
String
key
=
RedisUtil
.
getYuqingAnalyzePlatformAvgCountKey
(
projectId
,
Constant
.
PRIMARY_CONTEND_ID
,
planId
);
int
days
=
new
Period
(
startTime
,
endTime
,
PeriodType
.
days
()).
getDays
();
int
days
=
new
Period
(
startTime
,
endTime
,
PeriodType
.
days
()).
getDays
();
days
=
0
==
days
?
1
:
days
;
days
=
0
==
days
?
1
:
days
;
...
@@ -2549,10 +2576,13 @@ public class MarkDataServiceImpl implements MarkDataService {
...
@@ -2549,10 +2576,13 @@ public class MarkDataServiceImpl implements MarkDataService {
String
planId
=
dto
.
getPlanId
();
String
planId
=
dto
.
getPlanId
();
dto
.
setProjectId
(
projectId
);
dto
.
setProjectId
(
projectId
);
// 舆情总量
// 舆情总量
long
total
=
getYuqingAnalyzeCount
(
dto
);
long
total
=
getYuqingAnalyzeCount
(
dto
,
null
);
jsonObject
.
put
(
"total"
,
total
);
jsonObject
.
put
(
"total"
,
total
);
// 项目日均
// 项目日均
String
projectYuqingCountAvgKey
=
RedisUtil
.
getYuqingAnalyzeProjectAvgCountKey
(
projectId
,
Constant
.
PRIMARY_CONTEND_ID
,
planId
);
String
projectYuqingCountAvgKey
=
RedisUtil
.
getYuqingAnalyzeProjectAvgCountKey
(
projectId
,
Constant
.
PRIMARY_CONTEND_ID
,
planId
,
null
,
false
);
if
(
Objects
.
isNull
(
redisUtil
.
get
(
projectYuqingCountAvgKey
)))
{
countYuqingAmountAvg
(
dto
.
getStartTime
(),
dto
.
getEndTime
(),
projectId
,
planId
);
}
double
projectYuqingAvgCount
=
Objects
.
isNull
(
redisUtil
.
get
(
projectYuqingCountAvgKey
))
?
0
d
:
Double
.
parseDouble
(
redisUtil
.
get
(
projectYuqingCountAvgKey
));
double
projectYuqingAvgCount
=
Objects
.
isNull
(
redisUtil
.
get
(
projectYuqingCountAvgKey
))
?
0
d
:
Double
.
parseDouble
(
redisUtil
.
get
(
projectYuqingCountAvgKey
));
jsonObject
.
put
(
"projectAvg"
,
projectYuqingAvgCount
);
jsonObject
.
put
(
"projectAvg"
,
projectYuqingAvgCount
);
// 本次日均
// 本次日均
...
@@ -2570,49 +2600,63 @@ public class MarkDataServiceImpl implements MarkDataService {
...
@@ -2570,49 +2600,63 @@ public class MarkDataServiceImpl implements MarkDataService {
public
JSONObject
getYuqingEmotionDistribution
(
MarkSearchDTO
dto
)
{
public
JSONObject
getYuqingEmotionDistribution
(
MarkSearchDTO
dto
)
{
JSONObject
jsonObject
=
new
JSONObject
();
JSONObject
jsonObject
=
new
JSONObject
();
try
{
try
{
String
projectId
=
UserThreadLocal
.
getProjectId
();
return
getEmotionDistribution
(
dto
,
false
);
String
planId
=
dto
.
getPlanId
();
dto
.
setProjectId
(
projectId
);
// 舆情总量
long
total
=
getYuqingAnalyzeCount
(
dto
);
// 正面舆情
long
positiveCount
=
getYuqingAnalyzeCount
(
dto
,
EmotionEnum
.
POSITIVE
.
getName
());
// 中性舆情
long
neutralCount
=
getYuqingAnalyzeCount
(
dto
,
EmotionEnum
.
NEUTRAL
.
getName
());
// 负面舆情
long
negativeCount
=
getYuqingAnalyzeCount
(
dto
,
EmotionEnum
.
NEGATIVE
.
getName
());
// 各情感倾向占比
double
positivePercentage
=
total
==
0
?
0
d
:
positiveCount
/
(
double
)
total
;
double
neutralPercentage
=
total
==
0
?
0
d
:
neutralCount
/
(
double
)
total
;
double
negativePercentage
=
total
==
0
?
0
d
:
negativeCount
/
(
double
)
total
;
// 项目情感倾向占比均值
String
positiveKey
=
RedisUtil
.
getYuqingAnalyzeEmotionDistributionAvgKey
(
projectId
,
Constant
.
PRIMARY_CONTEND_ID
,
EmotionEnum
.
POSITIVE
.
getName
(),
planId
);
String
neutralKey
=
RedisUtil
.
getYuqingAnalyzeEmotionDistributionAvgKey
(
projectId
,
Constant
.
PRIMARY_CONTEND_ID
,
EmotionEnum
.
NEUTRAL
.
getName
(),
planId
);
String
negativeKey
=
RedisUtil
.
getYuqingAnalyzeEmotionDistributionAvgKey
(
projectId
,
Constant
.
PRIMARY_CONTEND_ID
,
EmotionEnum
.
NEGATIVE
.
getName
(),
planId
);
double
positiveAvg
=
Objects
.
isNull
(
redisUtil
.
get
(
positiveKey
))
?
0
d
:
Double
.
parseDouble
(
redisUtil
.
get
(
positiveKey
));
double
neutralAvg
=
Objects
.
isNull
(
redisUtil
.
get
(
neutralKey
))
?
0
d
:
Double
.
parseDouble
(
redisUtil
.
get
(
neutralKey
));
double
negativeAvg
=
Objects
.
isNull
(
redisUtil
.
get
(
negativeKey
))
?
0
d
:
Double
.
parseDouble
(
redisUtil
.
get
(
negativeKey
));
JSONObject
positive
=
new
JSONObject
();
JSONObject
neutral
=
new
JSONObject
();
JSONObject
negative
=
new
JSONObject
();
positive
.
put
(
"positiveCount"
,
positiveCount
);
neutral
.
put
(
"neutralCount"
,
neutralCount
);
negative
.
put
(
"negativeCount"
,
negativeCount
);
positive
.
put
(
"positivePercent"
,
positivePercentage
);
neutral
.
put
(
"neutralPercent"
,
neutralPercentage
);
negative
.
put
(
"negativePercent"
,
negativePercentage
);
positive
.
put
(
"positiveAvg"
,
positiveAvg
);
neutral
.
put
(
"neutralAvg"
,
neutralAvg
);
negative
.
put
(
"negativeAvg"
,
negativeAvg
);
jsonObject
.
put
(
"positive"
,
positive
);
jsonObject
.
put
(
"neutral"
,
neutral
);
jsonObject
.
put
(
"negative"
,
negative
);
}
catch
(
Exception
e
){
}
catch
(
Exception
e
){
ExceptionCast
.
cast
(
CommonCodeEnum
.
FAIL
,
"getYuqingEmotionDistribution异常-"
,
e
);
ExceptionCast
.
cast
(
CommonCodeEnum
.
FAIL
,
"getYuqingEmotionDistribution异常-"
,
e
);
}
}
return
jsonObject
;
return
jsonObject
;
}
}
private
JSONObject
getEmotionDistribution
(
MarkSearchDTO
dto
,
boolean
isCustom
)
throws
IOException
{
JSONObject
jsonObject
=
new
JSONObject
();
String
projectId
=
UserThreadLocal
.
getProjectId
();
String
planId
=
dto
.
getPlanId
();
String
customPlanId
=
dto
.
getCustomPlanId
();
dto
.
setProjectId
(
projectId
);
// 舆情总量
long
total
=
getYuqingAnalyzeCount
(
dto
,
null
);
// 正面舆情
long
positiveCount
=
getYuqingAnalyzeCount
(
dto
,
EmotionEnum
.
POSITIVE
.
getName
());
// 中性舆情
long
neutralCount
=
getYuqingAnalyzeCount
(
dto
,
EmotionEnum
.
NEUTRAL
.
getName
());
// 负面舆情
long
negativeCount
=
getYuqingAnalyzeCount
(
dto
,
EmotionEnum
.
NEGATIVE
.
getName
());
// 各情感倾向占比
double
positivePercentage
=
total
==
0
?
0
d
:
positiveCount
/
(
double
)
total
;
double
neutralPercentage
=
total
==
0
?
0
d
:
neutralCount
/
(
double
)
total
;
double
negativePercentage
=
total
==
0
?
0
d
:
negativeCount
/
(
double
)
total
;
// 项目情感倾向占比均值
String
positiveKey
=
RedisUtil
.
getYuqingAnalyzeEmotionDistributionAvgKey
(
projectId
,
Constant
.
PRIMARY_CONTEND_ID
,
EmotionEnum
.
POSITIVE
.
getName
(),
planId
,
customPlanId
,
isCustom
);
String
neutralKey
=
RedisUtil
.
getYuqingAnalyzeEmotionDistributionAvgKey
(
projectId
,
Constant
.
PRIMARY_CONTEND_ID
,
EmotionEnum
.
NEUTRAL
.
getName
(),
planId
,
customPlanId
,
isCustom
);
String
negativeKey
=
RedisUtil
.
getYuqingAnalyzeEmotionDistributionAvgKey
(
projectId
,
Constant
.
PRIMARY_CONTEND_ID
,
EmotionEnum
.
NEGATIVE
.
getName
(),
planId
,
customPlanId
,
isCustom
);
if
(
Objects
.
isNull
(
redisUtil
.
get
(
positiveKey
))
||
Objects
.
isNull
(
redisUtil
.
get
(
neutralKey
))
||
Objects
.
isNull
(
redisUtil
.
get
(
negativeKey
))){
if
(
isCustom
){
countCustomEmotionDistributionAvg
(
dto
.
getStartTime
(),
dto
.
getEndTime
(),
projectId
,
customPlanId
);
}
else
{
countEmotionDistributionAvg
(
dto
.
getStartTime
(),
dto
.
getEndTime
(),
projectId
,
planId
);
}
}
double
positiveAvg
=
Objects
.
isNull
(
redisUtil
.
get
(
positiveKey
))
?
0
d
:
Double
.
parseDouble
(
redisUtil
.
get
(
positiveKey
));
double
neutralAvg
=
Objects
.
isNull
(
redisUtil
.
get
(
neutralKey
))
?
0
d
:
Double
.
parseDouble
(
redisUtil
.
get
(
neutralKey
));
double
negativeAvg
=
Objects
.
isNull
(
redisUtil
.
get
(
negativeKey
))
?
0
d
:
Double
.
parseDouble
(
redisUtil
.
get
(
negativeKey
));
JSONObject
positive
=
new
JSONObject
();
JSONObject
neutral
=
new
JSONObject
();
JSONObject
negative
=
new
JSONObject
();
positive
.
put
(
"positiveCount"
,
positiveCount
);
neutral
.
put
(
"neutralCount"
,
neutralCount
);
negative
.
put
(
"negativeCount"
,
negativeCount
);
positive
.
put
(
"positivePercent"
,
positivePercentage
);
neutral
.
put
(
"neutralPercent"
,
neutralPercentage
);
negative
.
put
(
"negativePercent"
,
negativePercentage
);
positive
.
put
(
"positiveAvg"
,
positiveAvg
);
neutral
.
put
(
"neutralAvg"
,
neutralAvg
);
negative
.
put
(
"negativeAvg"
,
negativeAvg
);
jsonObject
.
put
(
"positive"
,
positive
);
jsonObject
.
put
(
"neutral"
,
neutral
);
jsonObject
.
put
(
"negative"
,
negative
);
return
jsonObject
;
}
@Override
@Override
public
JSONObject
getImportantPlatformPercentage
(
MarkSearchDTO
dto
)
{
public
JSONObject
getImportantPlatformPercentage
(
MarkSearchDTO
dto
)
{
JSONObject
res
=
new
JSONObject
();
JSONObject
res
=
new
JSONObject
();
...
@@ -2623,7 +2667,7 @@ public class MarkDataServiceImpl implements MarkDataService {
...
@@ -2623,7 +2667,7 @@ public class MarkDataServiceImpl implements MarkDataService {
if
(
CollectionUtils
.
isEmpty
(
dto
.
getPlatforms
()))
{
if
(
CollectionUtils
.
isEmpty
(
dto
.
getPlatforms
()))
{
dto
.
setPlatforms
(
Stream
.
of
(
"微博"
,
"微信"
,
"抖音"
,
"小红书"
,
"今日头条"
,
"网媒"
).
map
(
GlobalPojo:
:
getPlatformIdByName
).
collect
(
Collectors
.
toList
()));
dto
.
setPlatforms
(
Stream
.
of
(
"微博"
,
"微信"
,
"抖音"
,
"小红书"
,
"今日头条"
,
"网媒"
).
map
(
GlobalPojo:
:
getPlatformIdByName
).
collect
(
Collectors
.
toList
()));
}
}
Long
total
=
getYuqingAnalyzeCount
(
dto
);
Long
total
=
getYuqingAnalyzeCount
(
dto
,
null
);
// 平台聚合
// 平台聚合
SearchResponse
searchResponse
=
platformAggSearchResponse
(
dto
);
SearchResponse
searchResponse
=
platformAggSearchResponse
(
dto
);
Map
<
String
,
Aggregation
>
aggMap
=
searchResponse
.
getAggregations
().
asMap
();
Map
<
String
,
Aggregation
>
aggMap
=
searchResponse
.
getAggregations
().
asMap
();
...
@@ -2645,6 +2689,9 @@ public class MarkDataServiceImpl implements MarkDataService {
...
@@ -2645,6 +2689,9 @@ public class MarkDataServiceImpl implements MarkDataService {
double
avg
=
total
==
0
?
0
d
:
total
/
(
double
)
days
;
double
avg
=
total
==
0
?
0
d
:
total
/
(
double
)
days
;
// 项目均值
// 项目均值
String
key
=
RedisUtil
.
getYuqingAnalyzePlatformAvgCountKey
(
projectId
,
Constant
.
PRIMARY_CONTEND_ID
,
dto
.
getPlanId
());
String
key
=
RedisUtil
.
getYuqingAnalyzePlatformAvgCountKey
(
projectId
,
Constant
.
PRIMARY_CONTEND_ID
,
dto
.
getPlanId
());
if
(
Objects
.
isNull
(
redisUtil
.
get
(
key
))){
countImportantPlatformPercentageAvg
(
dto
.
getStartTime
(),
dto
.
getEndTime
(),
projectId
,
dto
.
getPlanId
());
}
double
projectAvg
=
Objects
.
isNull
(
redisUtil
.
get
(
key
))
?
0
d
:
Double
.
parseDouble
(
redisUtil
.
get
(
key
));
double
projectAvg
=
Objects
.
isNull
(
redisUtil
.
get
(
key
))
?
0
d
:
Double
.
parseDouble
(
redisUtil
.
get
(
key
));
overview
.
put
(
"compare"
,
compare
(
avg
,
projectAvg
));
overview
.
put
(
"compare"
,
compare
(
avg
,
projectAvg
));
overview
.
put
(
"avg"
,
projectAvg
);
overview
.
put
(
"avg"
,
projectAvg
);
...
@@ -2694,7 +2741,7 @@ public class MarkDataServiceImpl implements MarkDataService {
...
@@ -2694,7 +2741,7 @@ public class MarkDataServiceImpl implements MarkDataService {
List
<
JSONObject
>
list
=
new
ArrayList
<>();
List
<
JSONObject
>
list
=
new
ArrayList
<>();
// total
// total
defaultMarkSearch
(
dto
);
defaultMarkSearch
(
dto
);
Long
total
=
getYuqingAnalyzeCount
(
dto
);
Long
total
=
getYuqingAnalyzeCount
(
dto
,
null
);
// 平台聚合
// 平台聚合
SearchResponse
searchResponse
=
platformAggSearchResponse
(
dto
);
SearchResponse
searchResponse
=
platformAggSearchResponse
(
dto
);
Map
<
String
,
Aggregation
>
aggMap
=
searchResponse
.
getAggregations
().
asMap
();
Map
<
String
,
Aggregation
>
aggMap
=
searchResponse
.
getAggregations
().
asMap
();
...
@@ -2742,14 +2789,15 @@ public class MarkDataServiceImpl implements MarkDataService {
...
@@ -2742,14 +2789,15 @@ public class MarkDataServiceImpl implements MarkDataService {
Long
endTime
=
dto
.
getEndTime
();
Long
endTime
=
dto
.
getEndTime
();
String
planId
=
dto
.
getPlanId
();
String
planId
=
dto
.
getPlanId
();
dto
.
setProjectId
(
projectId
);
dto
.
setProjectId
(
projectId
);
// 各平台趋势图
Map
<
String
,
List
<
LineVO
>>
platformSpreadTendency
=
getPlatformSpreadTendency
(
dto
);
// 总趋势图
// 总趋势图
Pair
<
String
,
List
<
LineVO
>>
spreadTendency
=
getEmotionSpreadTendency
(
dto
,
null
);
Pair
<
String
,
List
<
LineVO
>>
spreadTendency
=
getEmotionSpreadTendency
(
dto
,
null
);
// 负面趋势图
// 负面趋势图
Pair
<
String
,
List
<
LineVO
>>
negativeSpreadTendency
=
getEmotionSpreadTendency
(
dto
,
EmotionEnum
.
NEGATIVE
.
getName
());
Pair
<
String
,
List
<
LineVO
>>
negativeSpreadTendency
=
getEmotionSpreadTendency
(
dto
,
EmotionEnum
.
NEGATIVE
.
getName
());
// 正面趋势图
// 正面趋势图
Pair
<
String
,
List
<
LineVO
>>
positiveSpreadTendency
=
getEmotionSpreadTendency
(
dto
,
EmotionEnum
.
POSITIVE
.
getName
());
Pair
<
String
,
List
<
LineVO
>>
positiveSpreadTendency
=
getEmotionSpreadTendency
(
dto
,
EmotionEnum
.
POSITIVE
.
getName
());
// 各平台趋势图
Map
<
String
,
List
<
LineVO
>>
platformSpreadTendency
=
getPlatformSpreadTendency
(
dto
);
dto
.
setPlatforms
(
null
);
Map
<
String
,
BaseMap
>
baseMaps
=
new
HashMap
<>(
2
);
Map
<
String
,
BaseMap
>
baseMaps
=
new
HashMap
<>(
2
);
BaseMap
totalBaseMaps
=
null
;
BaseMap
totalBaseMaps
=
null
;
BaseMap
negativeBaseMaps
=
null
;
BaseMap
negativeBaseMaps
=
null
;
...
@@ -2943,45 +2991,49 @@ public class MarkDataServiceImpl implements MarkDataService {
...
@@ -2943,45 +2991,49 @@ public class MarkDataServiceImpl implements MarkDataService {
try
{
try
{
//设置默认的时间、页码、排序方式等
//设置默认的时间、页码、排序方式等
defaultMarkSearch
(
dto
);
defaultMarkSearch
(
dto
);
String
[]
indexes
=
esClientDao
.
getIndexes
();
return
getYuqingAnalyzeDetail
(
dto
,
null
);
BoolQueryBuilder
postFilter
=
yuqingAnalyzeQuery
(
dto
);
FieldSortBuilder
sort
=
null
;
if
(
null
!=
dto
.
getSorter
())
{
for
(
Map
.
Entry
<
String
,
Object
>
entry
:
dto
.
getSorter
().
entrySet
())
{
String
key
=
entry
.
getKey
();
if
(
null
==
entry
.
getValue
()
||
entry
.
getValue
().
toString
().
contains
(
"desc"
))
{
sort
=
SortBuilders
.
fieldSort
(
key
).
order
(
SortOrder
.
DESC
);
}
else
{
sort
=
SortBuilders
.
fieldSort
(
key
).
order
(
SortOrder
.
ASC
);
}
}
}
int
from
=
(
dto
.
getPage
()
-
1
)
*
dto
.
getPageSize
();
int
size
=
dto
.
getPageSize
();
SearchHits
hits
=
esClientDao
.
searchHits
(
indexes
,
postFilter
,
null
,
null
,
sort
,
from
,
size
,
null
);
// 总量
long
total
=
hits
.
getTotalHits
().
value
;
// 消息列表
List
<
MarkFlowEntity
>
flowEntityList
;
if
(
Objects
.
nonNull
(
dto
.
getPlanId
()))
{
flowEntityList
=
getNonManualMarkFlowEntity
(
dto
,
hits
);
}
else
{
flowEntityList
=
getMarkFlowEntity
(
dto
,
hits
);
}
JSONObject
jsonObject
=
new
JSONObject
();
// 高亮关键词
jsonObject
.
put
(
"highlightWord"
,
GlobalPojo
.
HIGHLIGHT_MAP
.
get
(
UserThreadLocal
.
getProjectId
()));
// 列表数据最小mgid
MarkFlowEntity
markFlowEntity
=
flowEntityList
.
stream
()
.
min
(
Comparator
.
comparingLong
(
entity
->
JSONObject
.
parseObject
(
JSONObject
.
toJSONString
(
entity
.
getData
())).
getLongValue
(
"mgid"
))).
orElse
(
null
);
jsonObject
.
put
(
"pageGid"
,
Objects
.
isNull
(
markFlowEntity
)
?
Long
.
MAX_VALUE
:
JSONObject
.
parseObject
(
JSONObject
.
toJSONString
(
markFlowEntity
.
getData
())).
getLongValue
(
"mgid"
));
return
PageVO
.
createPageVo
(
total
,
dto
.
getPage
(),
dto
.
getPageSize
(),
flowEntityList
).
setInfo
(
jsonObject
);
}
catch
(
Exception
e
){
}
catch
(
Exception
e
){
ExceptionCast
.
cast
(
CommonCodeEnum
.
FAIL
,
"新舆情分析详情页getYuqingAnalyzeDetail异常-"
,
e
);
ExceptionCast
.
cast
(
CommonCodeEnum
.
FAIL
,
"新舆情分析详情页getYuqingAnalyzeDetail异常-"
,
e
);
}
}
return
PageVO
.
createPageVo
(
0
,
dto
.
getPage
(),
dto
.
getPageSize
(),
Collections
.
emptyList
());
return
PageVO
.
createPageVo
(
0
,
dto
.
getPage
(),
dto
.
getPageSize
(),
Collections
.
emptyList
());
}
}
private
PageVO
<
MarkFlowEntity
>
getYuqingAnalyzeDetail
(
MarkSearchDTO
dto
,
String
emotion
)
throws
IOException
{
String
[]
indexes
=
esClientDao
.
getIndexes
();
BoolQueryBuilder
postFilter
=
yuqingAnalyzeQuery
(
dto
,
emotion
,
null
);
FieldSortBuilder
sort
=
null
;
if
(
null
!=
dto
.
getSorter
())
{
for
(
Map
.
Entry
<
String
,
Object
>
entry
:
dto
.
getSorter
().
entrySet
())
{
String
key
=
entry
.
getKey
();
if
(
null
==
entry
.
getValue
()
||
entry
.
getValue
().
toString
().
contains
(
"desc"
))
{
sort
=
SortBuilders
.
fieldSort
(
key
).
order
(
SortOrder
.
DESC
);
}
else
{
sort
=
SortBuilders
.
fieldSort
(
key
).
order
(
SortOrder
.
ASC
);
}
}
}
int
from
=
(
dto
.
getPage
()
-
1
)
*
dto
.
getPageSize
();
int
size
=
dto
.
getPageSize
();
SearchHits
hits
=
esClientDao
.
searchHits
(
indexes
,
postFilter
,
null
,
null
,
sort
,
from
,
size
,
null
);
// 总量
long
total
=
hits
.
getTotalHits
().
value
;
// 消息列表
List
<
MarkFlowEntity
>
flowEntityList
;
if
(
Objects
.
nonNull
(
dto
.
getPlanId
()))
{
flowEntityList
=
getNonManualMarkFlowEntity
(
dto
,
hits
);
}
else
{
flowEntityList
=
getMarkFlowEntity
(
dto
,
hits
);
}
JSONObject
jsonObject
=
new
JSONObject
();
// 高亮关键词
jsonObject
.
put
(
"highlightWord"
,
GlobalPojo
.
HIGHLIGHT_MAP
.
get
(
UserThreadLocal
.
getProjectId
()));
// 列表数据最小mgid
MarkFlowEntity
markFlowEntity
=
flowEntityList
.
stream
()
.
min
(
Comparator
.
comparingLong
(
entity
->
JSONObject
.
parseObject
(
JSONObject
.
toJSONString
(
entity
.
getData
())).
getLongValue
(
"mgid"
))).
orElse
(
null
);
jsonObject
.
put
(
"pageGid"
,
Objects
.
isNull
(
markFlowEntity
)
?
Long
.
MAX_VALUE
:
JSONObject
.
parseObject
(
JSONObject
.
toJSONString
(
markFlowEntity
.
getData
())).
getLongValue
(
"mgid"
));
return
PageVO
.
createPageVo
(
total
,
dto
.
getPage
(),
dto
.
getPageSize
(),
flowEntityList
).
setInfo
(
jsonObject
);
}
@Override
@Override
public
void
cacheHighWord
(
Long
startTime
,
Long
endTime
,
String
planId
,
boolean
cache
)
{
public
void
cacheHighWord
(
Long
startTime
,
Long
endTime
,
String
planId
,
boolean
cache
)
{
getAnalyzeHighWord
(
convertedToMarkSearch
(
null
,
startTime
,
endTime
,
planId
),
cache
);
getAnalyzeHighWord
(
convertedToMarkSearch
(
null
,
startTime
,
endTime
,
planId
),
cache
);
...
@@ -2991,7 +3043,7 @@ public class MarkDataServiceImpl implements MarkDataService {
...
@@ -2991,7 +3043,7 @@ public class MarkDataServiceImpl implements MarkDataService {
public
List
<
JSONObject
>
getAnalyzeHighWord
(
MarkSearchDTO
dto
,
boolean
cache
)
{
public
List
<
JSONObject
>
getAnalyzeHighWord
(
MarkSearchDTO
dto
,
boolean
cache
)
{
String
projectId
=
UserThreadLocal
.
getProjectId
();
String
projectId
=
UserThreadLocal
.
getProjectId
();
dto
.
setProjectId
(
projectId
);
dto
.
setProjectId
(
projectId
);
String
redisKey
=
RedisUtil
.
getYuqingAnalyzeHighWordKey
(
projectId
,
Constant
.
PRIMARY_CONTEND_ID
,
dto
.
getPlanId
(),
dto
.
getStartTime
(),
dto
.
getEndTime
());
String
redisKey
=
RedisUtil
.
getYuqingAnalyzeHighWordKey
(
projectId
,
Constant
.
PRIMARY_CONTEND_ID
,
dto
.
getPlanId
(),
dto
.
getStartTime
(),
dto
.
getEndTime
()
,
null
,
false
);
String
resultStr
;
String
resultStr
;
// 返回缓存
// 返回缓存
if
(
cache
&&
StringUtils
.
isNotEmpty
(
resultStr
=
redisUtil
.
get
(
redisKey
)))
{
if
(
cache
&&
StringUtils
.
isNotEmpty
(
resultStr
=
redisUtil
.
get
(
redisKey
)))
{
...
@@ -3019,6 +3071,288 @@ public class MarkDataServiceImpl implements MarkDataService {
...
@@ -3019,6 +3071,288 @@ public class MarkDataServiceImpl implements MarkDataService {
return
highWord
;
return
highWord
;
}
}
@Override
public
JSONObject
getXhsCustomYuqingAmount
(
MarkSearchDTO
dto
)
{
JSONObject
jsonObject
=
new
JSONObject
();
try
{
defaultXhsCustomSearchDTO
(
dto
);
// 舆情总量
long
total
=
getYuqingAnalyzeCount
(
dto
,
null
);
jsonObject
.
put
(
"total"
,
total
);
// 视频舆情总量
dto
.
setDataType
(
Collections
.
singletonList
(
5
));
long
videoTotal
=
getYuqingAnalyzeCount
(
dto
,
null
);
jsonObject
.
put
(
"videoTotal"
,
videoTotal
);
// 图文舆情总量
long
textTotal
=
total
-
videoTotal
;
jsonObject
.
put
(
"textTotal"
,
textTotal
);
// 项目日均
String
projectYuqingCountAvgKey
=
RedisUtil
.
getYuqingAnalyzeProjectAvgCountKey
(
UserThreadLocal
.
getProjectId
(),
Constant
.
PRIMARY_CONTEND_ID
,
null
,
dto
.
getCustomPlanId
(),
true
);
if
(
Objects
.
isNull
(
redisUtil
.
get
(
projectYuqingCountAvgKey
))){
countCustomYuqingAmountAvg
(
dto
.
getStartTime
(),
dto
.
getEndTime
(),
UserThreadLocal
.
getProjectId
(),
dto
.
getCustomPlanId
());
}
double
projectYuqingAvgCount
=
Objects
.
isNull
(
redisUtil
.
get
(
projectYuqingCountAvgKey
))
?
0
d
:
Double
.
parseDouble
(
redisUtil
.
get
(
projectYuqingCountAvgKey
));
jsonObject
.
put
(
"projectAvg"
,
projectYuqingAvgCount
);
// 本次日均
int
days
=
new
Period
(
dto
.
getStartTime
(),
dto
.
getEndTime
(),
PeriodType
.
days
()).
getDays
();
days
=
0
==
days
?
1
:
days
;
double
yuqingAvgCount
=
total
==
0
?
0
d
:
total
/
(
double
)
days
;
jsonObject
.
put
(
"compare"
,
compare
(
yuqingAvgCount
,
projectYuqingAvgCount
));
}
catch
(
Exception
e
){
ExceptionCast
.
cast
(
CommonCodeEnum
.
FAIL
,
"小红书定制舆情分析getYuqingAmount异常-"
,
e
);
}
return
jsonObject
;
}
@Override
public
JSONObject
getXhsCustomYuqingEmotionDistribution
(
MarkSearchDTO
dto
)
{
try
{
defaultXhsCustomSearchDTO
(
dto
);
return
getEmotionDistribution
(
dto
,
true
);
}
catch
(
Exception
e
){
ExceptionCast
.
cast
(
CommonCodeEnum
.
FAIL
,
"小红书定制舆情分析getXhsCustomYuqingEmotionDistribution异常-"
,
e
);
}
return
null
;
}
@Override
public
JSONObject
getXhsCustomYuqingInteractionCount
(
MarkSearchDTO
dto
)
{
JSONObject
jsonObject
=
new
JSONObject
();
try
{
defaultXhsCustomSearchDTO
(
dto
);
// 索引
String
[]
indexes
=
esClientDao
.
getIndexes
();
// query
BoolQueryBuilder
query
=
yuqingAnalyzeQuery
(
dto
);
Map
<
String
,
SumAggregationBuilder
>
sumAggregationMap
=
new
HashMap
<>();
sumAggregationMap
.
put
(
"likeNum"
,
AggregationBuilders
.
sum
(
"likeNum"
).
field
(
"like_num"
));
sumAggregationMap
.
put
(
"favouritesNum"
,
AggregationBuilders
.
sum
(
"favouritesNum"
).
field
(
"favourites_num"
));
sumAggregationMap
.
put
(
"commentNum"
,
AggregationBuilders
.
sum
(
"commentNum"
).
field
(
"comment_num"
));
for
(
Map
.
Entry
<
String
,
SumAggregationBuilder
>
entry
:
sumAggregationMap
.
entrySet
())
{
String
key
=
entry
.
getKey
();
// response
SearchResponse
searchResponse
=
esClientDao
.
searchResponse
(
indexes
,
null
,
query
,
entry
.
getValue
(),
null
,
null
,
0
,
0
,
null
);
Map
<
String
,
Aggregation
>
aggMap
=
searchResponse
.
getAggregations
().
asMap
();
ParsedSum
teamAgg
=
(
ParsedSum
)
aggMap
.
get
(
key
);
jsonObject
.
put
(
key
,
teamAgg
.
getValue
());
}
}
catch
(
Exception
e
){
ExceptionCast
.
cast
(
CommonCodeEnum
.
FAIL
,
"小红书定制舆情分析getXhsCustomYuqingInteractionCount异常-"
,
e
);
}
return
jsonObject
;
}
@Override
public
JSONObject
getXhsCustomSpreadTendency
(
MarkSearchDTO
dto
)
{
JSONObject
res
=
new
JSONObject
();
try
{
defaultXhsCustomSearchDTO
(
dto
);
// 总趋势图
Pair
<
String
,
List
<
LineVO
>>
spreadTendency
=
getEmotionSpreadTendency
(
dto
,
null
);
// 负面趋势图
Pair
<
String
,
List
<
LineVO
>>
negativeSpreadTendency
=
getEmotionSpreadTendency
(
dto
,
EmotionEnum
.
NEGATIVE
.
getName
());
// 视频趋势图
dto
.
setDataType
(
Collections
.
singletonList
(
5
));
Pair
<
String
,
List
<
LineVO
>>
videoSpreadTendency
=
getEmotionSpreadTendency
(
dto
,
null
);
// 图文趋势图
dto
.
setDataType
(
Arrays
.
asList
(
1
,
2
,
3
));
Pair
<
String
,
List
<
LineVO
>>
textSpreadTendency
=
getEmotionSpreadTendency
(
dto
,
null
);
dto
.
setDataType
(
null
);
Map
<
String
,
BaseMap
>
baseMaps
=
new
HashMap
<>(
2
);
if
(
Objects
.
nonNull
(
negativeSpreadTendency
.
getLeft
())
||
Objects
.
nonNull
(
spreadTendency
.
getLeft
()))
{
CompletableFuture
.
allOf
(
Stream
.
of
(
spreadTendency
.
getLeft
(),
negativeSpreadTendency
.
getLeft
())
.
filter
(
Objects:
:
nonNull
).
map
(
aggTitle
->
CompletableFuture
.
runAsync
(()
->
{
try
{
baseMaps
.
put
(
aggTitle
,
getAnalyzeFirstArticle
(
dto
,
aggTitle
));
}
catch
(
IOException
ignored
)
{
}
},
executor
)).
toArray
(
CompletableFuture
[]::
new
)).
join
();
}
// 最高点
BaseMap
baseMap
=
baseMaps
.
get
(
spreadTendency
.
getLeft
());
JSONObject
highestJson
=
new
JSONObject
();
if
(
Objects
.
nonNull
(
baseMap
))
{
highestJson
.
put
(
"title"
,
Tools
.
filterUselessTitle
(
baseMap
.
getTitle
())
?
StringUtils
.
substring
(
baseMap
.
getContent
(),
0
,
15
)
:
baseMap
.
getTitle
());
highestJson
.
put
(
"url"
,
baseMap
.
getUrl
());
}
// 负面最高点
BaseMap
negativeBaseMap
=
baseMaps
.
get
(
negativeSpreadTendency
.
getLeft
());
JSONObject
negativeHighestJson
=
new
JSONObject
();
if
(
Objects
.
nonNull
(
negativeBaseMap
))
{
negativeHighestJson
.
put
(
"title"
,
Tools
.
filterUselessTitle
(
negativeBaseMap
.
getTitle
())
?
StringUtils
.
substring
(
negativeBaseMap
.
getContent
(),
0
,
15
)
:
negativeBaseMap
.
getTitle
());
negativeHighestJson
.
put
(
"url"
,
negativeBaseMap
.
getUrl
());
}
res
.
put
(
"总量"
,
spreadTendency
.
getRight
());
res
.
put
(
"负面"
,
negativeSpreadTendency
.
getRight
());
res
.
put
(
"图文"
,
textSpreadTendency
.
getRight
());
res
.
put
(
"视频"
,
videoSpreadTendency
.
getRight
());
res
.
put
(
"highest"
,
highestJson
);
res
.
put
(
"negativeHighest"
,
negativeHighestJson
);
}
catch
(
Exception
e
){
ExceptionCast
.
cast
(
CommonCodeEnum
.
FAIL
,
"小红书定制舆情分析getXhsCustomSpreadTendency异常-"
,
e
);
}
return
res
;
}
@Override
public
List
<
MarkFlowEntity
>
getXhsCustomInteractionArticle
(
MarkSearchDTO
dto
)
{
try
{
defaultXhsCustomSearchDTO
(
dto
);
// 索引
String
[]
indexes
=
esClientDao
.
getIndexes
();
BoolQueryBuilder
query
=
yuqingAnalyzeQuery
(
dto
);
// sort
FieldSortBuilder
sort
=
SortBuilders
.
fieldSort
(
"like_num"
).
order
(
SortOrder
.
DESC
);
if
(
null
!=
dto
.
getSorter
())
{
for
(
Map
.
Entry
<
String
,
Object
>
entry
:
dto
.
getSorter
().
entrySet
())
{
String
key
=
entry
.
getKey
();
switch
(
key
)
{
case
"likeNum"
:
key
=
"like_num"
;
break
;
case
"favouritesNum"
:
key
=
"favourites_num"
;
break
;
case
"commentNum"
:
key
=
"comment_num"
;
break
;
case
"followersNum"
:
key
=
"followers_num"
;
break
;
}
if
(
null
==
entry
.
getValue
()
||
entry
.
getValue
().
toString
().
contains
(
"desc"
))
{
sort
=
SortBuilders
.
fieldSort
(
key
).
order
(
SortOrder
.
DESC
);
}
else
{
sort
=
SortBuilders
.
fieldSort
(
key
).
order
(
SortOrder
.
ASC
);
}
}
}
SearchHits
hits
=
esClientDao
.
searchHits
(
indexes
,
query
,
null
,
null
,
sort
,
0
,
50
,
null
);
return
getMarkFlowEntity
(
dto
,
hits
);
}
catch
(
Exception
e
){
ExceptionCast
.
cast
(
CommonCodeEnum
.
FAIL
,
"小红书定制舆情分析getXhsCustomInteractionArticle异常-"
,
e
);
}
return
Collections
.
emptyList
();
}
@Override
public
PageVO
<
MarkFlowEntity
>
getXhsCustomNegativeArticle
(
MarkSearchDTO
dto
)
{
try
{
defaultXhsCustomSearchDTO
(
dto
);
return
getYuqingAnalyzeDetail
(
dto
,
EmotionEnum
.
NEGATIVE
.
getName
());
}
catch
(
Exception
e
){
ExceptionCast
.
cast
(
CommonCodeEnum
.
FAIL
,
"小红书定制舆情分析getXhsCustomNegativeArticle异常-"
,
e
);
}
return
null
;
}
@Override
public
PageVO
<
MarkFlowEntity
>
getXhsCustomArticleDetail
(
MarkSearchDTO
dto
)
{
try
{
defaultXhsCustomSearchDTO
(
dto
);
return
getYuqingAnalyzeDetail
(
dto
,
null
);
}
catch
(
Exception
e
){
ExceptionCast
.
cast
(
CommonCodeEnum
.
FAIL
,
"小红书定制舆情分析getXhsCustomNegativeArticle异常-"
,
e
);
}
return
null
;
}
@Override
public
List
<
JSONObject
>
getXhsCustomArticleIpLocated
(
MarkSearchDTO
dto
)
{
defaultXhsCustomSearchDTO
(
dto
);
return
getArticleIpLocated
(
dto
);
}
@Override
public
List
<
JSONObject
>
getXhsCustomAnalyzeHighWord
(
MarkSearchDTO
dto
,
boolean
cache
)
{
defaultXhsCustomSearchDTO
(
dto
);
String
redisKey
=
RedisUtil
.
getYuqingAnalyzeHighWordKey
(
UserThreadLocal
.
getProjectId
(),
Constant
.
PRIMARY_CONTEND_ID
,
null
,
dto
.
getStartTime
(),
dto
.
getEndTime
(),
dto
.
getCustomPlanId
(),
true
);
String
resultStr
;
// 返回缓存
if
(
cache
&&
StringUtils
.
isNotEmpty
(
resultStr
=
redisUtil
.
get
(
redisKey
)))
{
return
JSONArray
.
parseArray
(
resultStr
,
JSONObject
.
class
);
}
List
<
JSONObject
>
res
=
generateHighWord
(
dto
);
redisUtil
.
setExpire
(
redisKey
,
JSONArray
.
toJSONString
(
res
));
return
res
;
}
@Override
public
void
cacheXhsCustomHighWord
(
Long
startTime
,
Long
endTime
,
String
customPlanId
)
{
MarkSearchDTO
dto
=
new
MarkSearchDTO
();
dto
.
setStartTime
(
startTime
);
dto
.
setEndTime
(
endTime
);
dto
.
setCustomPlanId
(
customPlanId
);
getXhsCustomAnalyzeHighWord
(
dto
,
false
);
}
@Override
public
List
<
JSONObject
>
getXhsCustomTimelyHighWord
(
MarkSearchDTO
dto
)
{
// 小红书实时热点榜
ResponseEntity
<
JSONObject
>
hotResponse
=
restTemplate
.
getForEntity
(
trendsListUrl
,
JSONObject
.
class
,
"realTime"
,
"little-red-book"
,
null
);
List
<
JSONObject
>
hotList
=
Objects
.
requireNonNull
(
hotResponse
.
getBody
()).
getJSONArray
(
"data"
).
toJavaList
(
JSONObject
.
class
);
// 热议词
HttpHeaders
headers
=
new
HttpHeaders
();
headers
.
setContentType
(
MediaType
.
APPLICATION_JSON
);
headers
.
add
(
"token"
,
nearYearWordsToken
);
HttpEntity
<
String
>
request
=
new
HttpEntity
<>(
headers
);
ResponseEntity
<
JSONObject
>
wordResponse
=
restTemplate
.
exchange
(
nearYearWordsUrl
,
HttpMethod
.
GET
,
request
,
JSONObject
.
class
);
List
<
JSONObject
>
hotWords
=
Objects
.
requireNonNull
(
wordResponse
.
getBody
()).
getJSONObject
(
"data"
).
getJSONArray
(
"data"
).
toJavaList
(
JSONObject
.
class
);
List
<
JSONObject
>
matchedWords
=
new
ArrayList
<>(
10
);
for
(
int
i
=
0
;
i
<
hotList
.
size
();
i
++)
{
String
name
=
hotList
.
get
(
i
).
getString
(
"name"
);
for
(
JSONObject
word
:
hotWords
)
{
if
(
name
.
contains
(
word
.
getString
(
"name"
))){
if
(
matchedWords
.
size
()
>=
10
){
break
;
}
JSONObject
jsonObject
=
new
JSONObject
();
jsonObject
.
put
(
"word"
,
word
.
getString
(
"name"
));
jsonObject
.
put
(
"rank"
,
i
+
1
);
jsonObject
.
put
(
"id"
,
word
.
getString
(
"id"
));
matchedWords
.
add
(
jsonObject
);
}
}
}
double
sumRank
=
matchedWords
.
stream
().
limit
(
10
).
mapToDouble
(
json
->
getRankPoint
(
json
.
getIntValue
(
"rank"
))).
sum
();
matchedWords
.
forEach
(
json
->
json
.
put
(
"proportion"
,
getRankPoint
(
json
.
getIntValue
(
"rank"
))
/
sumRank
));
return
matchedWords
;
}
private
int
getRankPoint
(
int
rank
){
switch
(
rank
){
case
1
:
return
10
;
case
2
:
return
9
;
case
3
:
return
8
;
case
4
:
return
7
;
case
5
:
return
6
;
case
6
:
return
5
;
case
7
:
return
4
;
case
8
:
return
3
;
case
9
:
return
2
;
case
10
:
return
1
;
default
:
return
0
;
}
}
private
void
defaultXhsCustomSearchDTO
(
MarkSearchDTO
dto
){
dto
.
setProjectId
(
UserThreadLocal
.
getProjectId
());
dto
.
setPlatforms
(
Collections
.
singletonList
(
XHS_PLATFORM_ID
));
}
private
List
<
JSONObject
>
generateHighWord
(
MarkSearchDTO
dto
){
private
List
<
JSONObject
>
generateHighWord
(
MarkSearchDTO
dto
){
List
<
JSONObject
>
res
=
new
ArrayList
<>();
List
<
JSONObject
>
res
=
new
ArrayList
<>();
try
{
try
{
...
@@ -3608,29 +3942,6 @@ public class MarkDataServiceImpl implements MarkDataService {
...
@@ -3608,29 +3942,6 @@ public class MarkDataServiceImpl implements MarkDataService {
// 关键词 数字符号替换后重组query
// 关键词 数字符号替换后重组query
keywordQuery
.
should
(
EsQueryTools
.
assembleNormalKeywordQuery
(
Tools
.
canonicalKeyword
(
dto
.
getKeyword
().
replaceAll
(
ORIGIN_SEARCH_PATTERN
,
" "
)),
fieldSearch
));
keywordQuery
.
should
(
EsQueryTools
.
assembleNormalKeywordQuery
(
Tools
.
canonicalKeyword
(
dto
.
getKeyword
().
replaceAll
(
ORIGIN_SEARCH_PATTERN
,
" "
)),
fieldSearch
));
}
}
// String keyword = dto.getKeyword();
// // 特殊符号、数字替换空格
// String processedKeyword = Tools.canonicalKeyword(keyword.replaceAll(ORIGIN_SEARCH_PATTERN, " "));
//// BoolQueryBuilder keywordQuery = QueryBuilders.boolQuery();
// // keyword条件一: 空格为且条件
// if (StringUtils.isNotEmpty(processedKeyword)) {
// BoolQueryBuilder keywordQuery1 = QueryBuilders.boolQuery();
// String[] words = processedKeyword.split(" +");
// for (int i = 0; i < words.length; i++) {
// BoolQueryBuilder wordQuery = QueryBuilders.boolQuery();
// if (i != words.length - 1) {
// keywordQuery1.must(wordQuery.must(QueryBuilders.matchPhraseQuery(GenericAttribute.ES_IND_FULL_TEXT, words[i])));
// } else {
// keywordQuery1.must(wordQuery.must(QueryBuilders.matchPhrasePrefixQuery(GenericAttribute.ES_IND_FULL_TEXT, words[i])));
// }
// }
// keywordQuery.should(keywordQuery1);
// }
// // keyword条件二
// BoolQueryBuilder keywordQuery2 = QueryBuilders.boolQuery();
// keywordQuery2.must(QueryBuilders.matchPhrasePrefixQuery(GenericAttribute.ES_IND_FULL_TEXT, keyword));
// // 两个keyword条件满足一个即可
// keywordQuery.should(keywordQuery2).minimumShouldMatch(1);
query
.
must
(
keywordQuery
);
query
.
must
(
keywordQuery
);
// platform
// platform
BoolQueryBuilder
platformQuery
=
EsQueryTools
.
assemblePlatformQuery
(
GlobalPojo
.
PLATFORMS
);
BoolQueryBuilder
platformQuery
=
EsQueryTools
.
assemblePlatformQuery
(
GlobalPojo
.
PLATFORMS
);
...
@@ -3734,7 +4045,7 @@ public class MarkDataServiceImpl implements MarkDataService {
...
@@ -3734,7 +4045,7 @@ public class MarkDataServiceImpl implements MarkDataService {
daysAggregationBuilder
=
AggregationBuilders
.
dateHistogram
(
"timeAgg"
).
field
(
"time"
).
calendarInterval
(
DateHistogramInterval
.
DAY
);
daysAggregationBuilder
=
AggregationBuilders
.
dateHistogram
(
"timeAgg"
).
field
(
"time"
).
calendarInterval
(
DateHistogramInterval
.
DAY
);
}
}
TermsAggregationBuilder
titleAggregationBuilder
=
AggregationBuilders
.
terms
(
"titleAgg"
).
field
(
"agg_title.keyword"
).
order
(
BucketOrder
.
count
(
false
));
TermsAggregationBuilder
titleAggregationBuilder
=
AggregationBuilders
.
terms
(
"titleAgg"
).
field
(
"agg_title.keyword"
).
order
(
BucketOrder
.
count
(
false
));
dto
.
setPlatforms
(
null
);
//
dto.setPlatforms(null);
// query
// query
BoolQueryBuilder
query
=
yuqingAnalyzeQuery
(
dto
,
emotion
,
null
);
BoolQueryBuilder
query
=
yuqingAnalyzeQuery
(
dto
,
emotion
,
null
);
// response
// response
...
@@ -3762,22 +4073,12 @@ public class MarkDataServiceImpl implements MarkDataService {
...
@@ -3762,22 +4073,12 @@ public class MarkDataServiceImpl implements MarkDataService {
return
Pair
.
of
(
maxAggTitle
,
res
);
return
Pair
.
of
(
maxAggTitle
,
res
);
}
}
private
Long
getYuqingAnalyzeCount
(
MarkSearchDTO
dto
)
throws
IOException
{
BoolQueryBuilder
query
=
yuqingAnalyzeQuery
(
dto
,
null
,
null
);
return
esClientDao
.
count
(
query
);
}
private
Long
getYuqingAnalyzeCount
(
MarkSearchDTO
dto
,
String
emotion
)
throws
IOException
{
private
Long
getYuqingAnalyzeCount
(
MarkSearchDTO
dto
,
String
emotion
)
throws
IOException
{
BoolQueryBuilder
query
=
yuqingAnalyzeQuery
(
dto
,
emotion
,
null
);
BoolQueryBuilder
query
=
yuqingAnalyzeQuery
(
dto
,
emotion
,
null
);
return
esClientDao
.
count
(
query
);
return
esClientDao
.
count
(
query
);
}
}
private
Long
getYuqingAnalyzeCount
(
Long
startTime
,
Long
endTime
,
String
projectId
,
String
planId
,
String
emotion
,
List
<
String
>
platforms
)
throws
IOException
{
private
Long
getTaskYuqingAnalyzeCount
(
Long
startTime
,
Long
endTime
,
String
projectId
,
String
planId
,
String
customPlanId
,
String
emotion
,
List
<
String
>
platforms
)
throws
IOException
{
BoolQueryBuilder
query
=
yuqingAnalyzeQuery
(
startTime
,
endTime
,
projectId
,
planId
,
emotion
,
platforms
);
return
esClientDao
.
count
(
query
);
}
private
BoolQueryBuilder
yuqingAnalyzeQuery
(
Long
startTime
,
Long
endTime
,
String
projectId
,
String
planId
,
String
emotion
,
List
<
String
>
platforms
){
// project plan query
// project plan query
BoolQueryBuilder
query
=
EsQueryTools
.
assembleCacheMapsPlanQuery
(
projectId
,
planId
);
BoolQueryBuilder
query
=
EsQueryTools
.
assembleCacheMapsPlanQuery
(
projectId
,
planId
);
if
(
Objects
.
isNull
(
planId
)){
if
(
Objects
.
isNull
(
planId
)){
...
@@ -3804,17 +4105,18 @@ public class MarkDataServiceImpl implements MarkDataService {
...
@@ -3804,17 +4105,18 @@ public class MarkDataServiceImpl implements MarkDataService {
if
(
Objects
.
nonNull
(
endTime
))
{
if
(
Objects
.
nonNull
(
endTime
))
{
query
.
must
(
QueryBuilders
.
rangeQuery
(
"time"
).
lt
(
endTime
));
query
.
must
(
QueryBuilders
.
rangeQuery
(
"time"
).
lt
(
endTime
));
}
}
return
query
;
// 定制化舆情分析方案关键词
if
(
Objects
.
nonNull
(
customPlanId
)){
String
planKeyword
=
xiaohongshuWordDao
.
findOneById
(
customPlanId
).
getKeyword
();
query
.
must
(
EsQueryTools
.
assembleNormalKeywordQuery
(
planKeyword
,
new
String
[]{
GenericAttribute
.
ES_IND_FULL_TEXT
}));
}
return
esClientDao
.
count
(
query
);
}
}
private
BoolQueryBuilder
yuqingAnalyzeQuery
(
MarkSearchDTO
dto
){
private
BoolQueryBuilder
yuqingAnalyzeQuery
(
MarkSearchDTO
dto
){
return
yuqingAnalyzeQuery
(
dto
,
null
,
null
);
return
yuqingAnalyzeQuery
(
dto
,
null
,
null
);
}
}
private
BoolQueryBuilder
yuqingAnalyzeQuery
(
MarkSearchDTO
dto
,
String
aggTitle
){
return
yuqingAnalyzeQuery
(
dto
,
null
,
aggTitle
);
}
private
BoolQueryBuilder
yuqingAnalyzeQuery
(
MarkSearchDTO
dto
,
String
emotion
,
String
aggTitle
){
private
BoolQueryBuilder
yuqingAnalyzeQuery
(
MarkSearchDTO
dto
,
String
emotion
,
String
aggTitle
){
// projectId、contendId
// projectId、contendId
String
projectId
=
dto
.
getProjectId
();
String
projectId
=
dto
.
getProjectId
();
...
@@ -3892,6 +4194,15 @@ public class MarkDataServiceImpl implements MarkDataService {
...
@@ -3892,6 +4194,15 @@ public class MarkDataServiceImpl implements MarkDataService {
if
(
Objects
.
isNull
(
aggTitle
)
&&
Objects
.
nonNull
(
dto
.
getAggTitle
())){
if
(
Objects
.
isNull
(
aggTitle
)
&&
Objects
.
nonNull
(
dto
.
getAggTitle
())){
postFilter
.
must
(
QueryBuilders
.
termQuery
(
"agg_title.keyword"
,
dto
.
getAggTitle
()));
postFilter
.
must
(
QueryBuilders
.
termQuery
(
"agg_title.keyword"
,
dto
.
getAggTitle
()));
}
}
// dataType
if
(
CollectionUtils
.
isNotEmpty
(
dto
.
getDataType
())){
postFilter
.
must
(
EsQueryTools
.
assembleC2Query
(
dto
.
getDataType
()));
}
// 定制化舆情分析方案关键词
if
(
Objects
.
nonNull
(
dto
.
getCustomPlanId
())){
String
planKeyword
=
xiaohongshuWordDao
.
findOneById
(
dto
.
getCustomPlanId
()).
getKeyword
();
postFilter
.
must
(
EsQueryTools
.
assembleNormalKeywordQuery
(
planKeyword
,
new
String
[]{
GenericAttribute
.
ES_IND_FULL_TEXT
}));
}
return
postFilter
;
return
postFilter
;
}
}
...
...
src/main/java/com/zhiwei/brandkbs2/service/impl/ProjectServiceImpl.java
View file @
18f92bf9
...
@@ -574,7 +574,11 @@ public class ProjectServiceImpl implements ProjectService {
...
@@ -574,7 +574,11 @@ public class ProjectServiceImpl implements ProjectService {
query
.
addCriteria
(
Criteria
.
where
(
"isManual"
).
is
(
false
));
query
.
addCriteria
(
Criteria
.
where
(
"isManual"
).
is
(
false
));
projectDao
.
addKeywordFuzz
(
query
,
keyword
,
"projectName"
);
projectDao
.
addKeywordFuzz
(
query
,
keyword
,
"projectName"
);
long
count
=
projectDao
.
count
(
query
);
long
count
=
projectDao
.
count
(
query
);
projectDao
.
addSort
(
query
,
sorter
);
if
(
Objects
.
isNull
(
sorter
)
||
Objects
.
isNull
(
JSONObject
.
parseObject
(
sorter
))
||
JSONObject
.
parseObject
(
sorter
).
isEmpty
()){
projectDao
.
addSort
(
query
,
"{\"cTime\":\"descend\"}"
);
}
else
{
projectDao
.
addSort
(
query
,
sorter
);
}
mongoUtil
.
start
(
page
,
size
,
query
);
mongoUtil
.
start
(
page
,
size
,
query
);
List
<
Project
>
projectList
=
projectDao
.
findList
(
query
);
List
<
Project
>
projectList
=
projectDao
.
findList
(
query
);
List
<
JSONObject
>
resList
=
projectList
.
stream
().
map
(
project
->
{
List
<
JSONObject
>
resList
=
projectList
.
stream
().
map
(
project
->
{
...
@@ -801,9 +805,11 @@ public class ProjectServiceImpl implements ProjectService {
...
@@ -801,9 +805,11 @@ public class ProjectServiceImpl implements ProjectService {
*/
*/
private
List
<
Map
<
String
,
Object
>>
getPermissionList
(
Project
project
)
{
private
List
<
Map
<
String
,
Object
>>
getPermissionList
(
Project
project
)
{
List
<
Map
<
String
,
Object
>>
permissionList
=
new
ArrayList
<>();
List
<
Map
<
String
,
Object
>>
permissionList
=
new
ArrayList
<>();
Map
<
String
,
Object
>
originPermission
=
new
HashMap
<>();
// 定制化模块权限
originPermission
.
put
(
"origin"
,
Objects
.
nonNull
(
project
.
getModuleShowList
())
&&
project
.
getModuleShowList
().
contains
(
2
));
List
<
String
>
moduleShowList
=
project
.
getModuleShowList
();
permissionList
.
add
(
originPermission
);
Map
<
String
,
Object
>
customPermission
=
new
HashMap
<>();
customPermission
.
put
(
"xiaohongshu"
,
Objects
.
nonNull
(
moduleShowList
)
&&
moduleShowList
.
contains
(
"xiaohongshu"
));
permissionList
.
add
(
customPermission
);
// 友好渠道榜、敏感渠道榜是否有数据
// 友好渠道榜、敏感渠道榜是否有数据
Map
<
String
,
Object
>
channelPermission
=
new
HashMap
<>();
Map
<
String
,
Object
>
channelPermission
=
new
HashMap
<>();
JSONObject
jsonObject
=
GlobalPojo
.
PROJECT_EMOTION_CHANNEL_DATA
.
get
(
project
.
getId
());
JSONObject
jsonObject
=
GlobalPojo
.
PROJECT_EMOTION_CHANNEL_DATA
.
get
(
project
.
getId
());
...
...
src/main/java/com/zhiwei/brandkbs2/service/impl/TableWordServiceImpl.java
View file @
18f92bf9
...
@@ -22,7 +22,7 @@ import java.util.stream.Collectors;
...
@@ -22,7 +22,7 @@ import java.util.stream.Collectors;
* @author: sjj
* @author: sjj
* @date: 2023-05-25 14:35
* @date: 2023-05-25 14:35
*/
*/
public
class
TableWordServiceImpl
implements
TableWordService
{
public
class
TableWordServiceImpl
implements
TableWordService
{
final
BaseMongoDao
<?
extends
AbstractBaseWord
>
baseMongoDao
;
final
BaseMongoDao
<?
extends
AbstractBaseWord
>
baseMongoDao
;
final
MongoUtil
mongoUtil
;
final
MongoUtil
mongoUtil
;
...
@@ -60,7 +60,7 @@ public class TableWordServiceImpl implements TableWordService{
...
@@ -60,7 +60,7 @@ public class TableWordServiceImpl implements TableWordService{
if
(
newList
.
size
()
+
existList
.
size
()
>
100
)
{
if
(
newList
.
size
()
+
existList
.
size
()
>
100
)
{
return
ResponseResult
.
failure
(
"关键词添加超过上限值100"
);
return
ResponseResult
.
failure
(
"关键词添加超过上限值100"
);
}
}
baseMongoDao
.
insertMany
(
HighWord
.
createFromList
(
newList
,
projectId
,
submitter
));
baseMongoDao
.
insertMany
(
new
HighWord
()
.
createFromList
(
newList
,
projectId
,
submitter
));
return
ResponseResult
.
success
();
return
ResponseResult
.
success
();
}
}
...
...
src/main/java/com/zhiwei/brandkbs2/service/impl/TaskServiceImpl.java
View file @
18f92bf9
package
com
.
zhiwei
.
brandkbs2
.
service
.
impl
;
package
com
.
zhiwei
.
brandkbs2
.
service
.
impl
;
import
com.alibaba.fastjson.JSONObject
;
import
com.zhiwei.brandkbs2.auth.UserThreadLocal
;
import
com.zhiwei.brandkbs2.auth.UserThreadLocal
;
import
com.zhiwei.brandkbs2.common.GenericAttribute
;
import
com.zhiwei.brandkbs2.common.GlobalPojo
;
import
com.zhiwei.brandkbs2.common.GlobalPojo
;
import
com.zhiwei.brandkbs2.config.Constant
;
import
com.zhiwei.brandkbs2.config.Constant
;
import
com.zhiwei.brandkbs2.dao.*
;
import
com.zhiwei.brandkbs2.dao.*
;
import
com.zhiwei.brandkbs2.enmus.ReportTypeEnum
;
import
com.zhiwei.brandkbs2.enmus.ReportTypeEnum
;
import
com.zhiwei.brandkbs2.es.ChannelEsDao
;
import
com.zhiwei.brandkbs2.es.ChannelEsDao
;
import
com.zhiwei.brandkbs2.es.EsClientDao
;
import
com.zhiwei.brandkbs2.es.EsClientDao
;
import
com.zhiwei.brandkbs2.es.EsQueryTools
;
import
com.zhiwei.brandkbs2.exception.ExceptionCast
;
import
com.zhiwei.brandkbs2.exception.ExceptionCast
;
import
com.zhiwei.brandkbs2.listener.ApplicationProjectListener
;
import
com.zhiwei.brandkbs2.listener.ApplicationProjectListener
;
import
com.zhiwei.brandkbs2.model.CommonCodeEnum
;
import
com.zhiwei.brandkbs2.model.CommonCodeEnum
;
import
com.zhiwei.brandkbs2.pojo.*
;
import
com.zhiwei.brandkbs2.pojo.*
;
import
com.zhiwei.brandkbs2.service.*
;
import
com.zhiwei.brandkbs2.service.*
;
import
com.zhiwei.brandkbs2.util.Tools
;
import
com.zhiwei.brandkbs2.util.Tools
;
import
org.apache.commons.collections4.CollectionUtils
;
import
org.apache.commons.collections4.ListUtils
;
import
org.apache.commons.collections4.ListUtils
;
import
org.apache.commons.lang3.tuple.Pair
;
import
org.apache.commons.lang3.tuple.Pair
;
import
org.apache.logging.log4j.LogManager
;
import
org.apache.logging.log4j.LogManager
;
import
org.apache.logging.log4j.Logger
;
import
org.apache.logging.log4j.Logger
;
import
org.elasticsearch.ElasticsearchStatusException
;
import
org.elasticsearch.ElasticsearchStatusException
;
import
org.elasticsearch.index.query.BoolQueryBuilder
;
import
org.elasticsearch.index.query.QueryBuilders
;
import
org.springframework.data.mongodb.core.query.Criteria
;
import
org.springframework.data.mongodb.core.query.Criteria
;
import
org.springframework.data.mongodb.core.query.Query
;
import
org.springframework.data.mongodb.core.query.Query
;
import
org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor
;
import
org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor
;
import
org.springframework.stereotype.Service
;
import
org.springframework.stereotype.Service
;
import
javax.annotation.Resource
;
import
javax.annotation.Resource
;
import
java.io.IOException
;
import
java.util.*
;
import
java.util.*
;
import
java.util.concurrent.CompletableFuture
;
import
java.util.concurrent.CompletableFuture
;
import
java.util.concurrent.ConcurrentHashMap
;
import
java.util.concurrent.ConcurrentHashMap
;
...
@@ -66,6 +73,12 @@ public class TaskServiceImpl implements TaskService {
...
@@ -66,6 +73,12 @@ public class TaskServiceImpl implements TaskService {
@Resource
(
name
=
"nonManualProjectPlanDao"
)
@Resource
(
name
=
"nonManualProjectPlanDao"
)
NonManualProjectPlanDao
nonManualProjectPlanDao
;
NonManualProjectPlanDao
nonManualProjectPlanDao
;
@Resource
(
name
=
"xiaohongshuWordDao"
)
private
XiaohongshuWordDao
xiaohongshuWordDao
;
@Resource
(
name
=
"xiaohongshuRecordDao"
)
private
XiaohongshuRecordDao
xiaohongshuRecordDao
;
@Resource
(
name
=
"brandkbsTaskServiceImpl"
)
@Resource
(
name
=
"brandkbsTaskServiceImpl"
)
BrandkbsTaskService
brandkbsTaskService
;
BrandkbsTaskService
brandkbsTaskService
;
...
@@ -90,6 +103,9 @@ public class TaskServiceImpl implements TaskService {
...
@@ -90,6 +103,9 @@ public class TaskServiceImpl implements TaskService {
@Resource
(
name
=
"markDataServiceImpl"
)
@Resource
(
name
=
"markDataServiceImpl"
)
MarkDataService
markDataService
;
MarkDataService
markDataService
;
@Resource
(
name
=
"toolsetServiceImpl"
)
private
ToolsetService
toolsetService
;
@Resource
(
name
=
"taskServiceExecutor"
)
@Resource
(
name
=
"taskServiceExecutor"
)
ThreadPoolTaskExecutor
taskServiceExecutor
;
ThreadPoolTaskExecutor
taskServiceExecutor
;
...
@@ -371,6 +387,16 @@ public class TaskServiceImpl implements TaskService {
...
@@ -371,6 +387,16 @@ public class TaskServiceImpl implements TaskService {
markDataService
.
countYuqingAmountAvg
(
startTime
,
endTime
,
projectId
,
null
);
markDataService
.
countYuqingAmountAvg
(
startTime
,
endTime
,
projectId
,
null
);
markDataService
.
countEmotionDistributionAvg
(
startTime
,
endTime
,
projectId
,
null
);
markDataService
.
countEmotionDistributionAvg
(
startTime
,
endTime
,
projectId
,
null
);
markDataService
.
countImportantPlatformPercentageAvg
(
startTime
,
endTime
,
projectId
,
null
);
markDataService
.
countImportantPlatformPercentageAvg
(
startTime
,
endTime
,
projectId
,
null
);
// 定制化舆情分析页面均值计算
if
(
CollectionUtils
.
isNotEmpty
(
project
.
getModuleShowList
())
&&
project
.
getModuleShowList
().
contains
(
"xiaohongshu"
))
{
markDataService
.
countCustomYuqingAmountAvg
(
startTime
,
endTime
,
projectId
,
null
);
markDataService
.
countCustomEmotionDistributionAvg
(
startTime
,
endTime
,
projectId
,
null
);
for
(
XiaohongshuWord
xiaohongshuWord
:
xiaohongshuWordDao
.
findList
(
new
Query
(
Criteria
.
where
(
"projectId"
).
is
(
projectId
))))
{
markDataService
.
countCustomYuqingAmountAvg
(
startTime
,
endTime
,
projectId
,
xiaohongshuWord
.
getId
());
markDataService
.
countCustomEmotionDistributionAvg
(
startTime
,
endTime
,
projectId
,
xiaohongshuWord
.
getId
());
}
log
.
info
(
"项目:{}-定制化分析页面均值计算已完成:{}个"
,
project
.
getProjectName
(),
total
.
incrementAndGet
());
}
log
.
info
(
"项目:{}-均值计算已完成:{}个"
,
project
.
getProjectName
(),
total
.
incrementAndGet
());
log
.
info
(
"项目:{}-均值计算已完成:{}个"
,
project
.
getProjectName
(),
total
.
incrementAndGet
());
}
catch
(
Exception
e
){
}
catch
(
Exception
e
){
ExceptionCast
.
cast
(
CommonCodeEnum
.
FAIL
,
"calculateProjectAvg异常-projectId:"
+
projectId
,
e
);
ExceptionCast
.
cast
(
CommonCodeEnum
.
FAIL
,
"calculateProjectAvg异常-projectId:"
+
projectId
,
e
);
...
@@ -385,6 +411,9 @@ public class TaskServiceImpl implements TaskService {
...
@@ -385,6 +411,9 @@ public class TaskServiceImpl implements TaskService {
List
<
CompletableFuture
<
Object
>>
projectFutures
=
GlobalPojo
.
PROJECT_MAP
.
values
().
stream
().
map
(
project
->
CompletableFuture
.
supplyAsync
(()
->
{
List
<
CompletableFuture
<
Object
>>
projectFutures
=
GlobalPojo
.
PROJECT_MAP
.
values
().
stream
().
map
(
project
->
CompletableFuture
.
supplyAsync
(()
->
{
UserThreadLocal
.
set
(
new
UserInfo
().
setProjectId
(
project
.
getId
()));
UserThreadLocal
.
set
(
new
UserInfo
().
setProjectId
(
project
.
getId
()));
markDataService
.
cacheHighWord
(
time
[
0
],
time
[
1
],
null
,
false
);
markDataService
.
cacheHighWord
(
time
[
0
],
time
[
1
],
null
,
false
);
if
(
CollectionUtils
.
isNotEmpty
(
project
.
getModuleShowList
())
&&
project
.
getModuleShowList
().
contains
(
"xiaohongshu"
)){
markDataService
.
cacheXhsCustomHighWord
(
time
[
0
],
time
[
1
],
null
);
}
log
.
info
(
"项目:{}-{}-词云缓存已完成:{}个"
,
project
.
getProjectName
(),
project
.
getId
(),
total
.
incrementAndGet
());
log
.
info
(
"项目:{}-{}-词云缓存已完成:{}个"
,
project
.
getProjectName
(),
project
.
getId
(),
total
.
incrementAndGet
());
return
null
;
return
null
;
},
cacheServiceExecutor
)).
collect
(
Collectors
.
toList
());
},
cacheServiceExecutor
)).
collect
(
Collectors
.
toList
());
...
@@ -395,7 +424,15 @@ public class TaskServiceImpl implements TaskService {
...
@@ -395,7 +424,15 @@ public class TaskServiceImpl implements TaskService {
log
.
info
(
"方案:{}-{}-词云缓存已完成:{}个"
,
plan
.
getName
(),
plan
.
getId
(),
total2
.
incrementAndGet
());
log
.
info
(
"方案:{}-{}-词云缓存已完成:{}个"
,
plan
.
getName
(),
plan
.
getId
(),
total2
.
incrementAndGet
());
return
null
;
return
null
;
},
cacheServiceExecutor
)).
collect
(
Collectors
.
toList
());
},
cacheServiceExecutor
)).
collect
(
Collectors
.
toList
());
AtomicInteger
total3
=
new
AtomicInteger
();
List
<
CompletableFuture
<
Object
>>
customPlanFutures
=
xiaohongshuWordDao
.
findList
(
new
Query
()).
stream
().
map
(
plan
->
CompletableFuture
.
supplyAsync
(()
->
{
UserThreadLocal
.
set
(
new
UserInfo
().
setProjectId
(
plan
.
getProjectId
()));
markDataService
.
cacheXhsCustomHighWord
(
time
[
0
],
time
[
1
],
plan
.
getId
());
log
.
info
(
"舆情分析定制化页面方案:{}-{}-词云缓存已完成:{}个"
,
plan
.
getPlanName
(),
plan
.
getId
(),
total3
.
incrementAndGet
());
return
null
;
},
cacheServiceExecutor
)).
collect
(
Collectors
.
toList
());
projectFutures
.
addAll
(
planFutures
);
projectFutures
.
addAll
(
planFutures
);
projectFutures
.
addAll
(
customPlanFutures
);
CompletableFuture
.
allOf
(
projectFutures
.
toArray
(
new
CompletableFuture
[
0
])).
join
();
CompletableFuture
.
allOf
(
projectFutures
.
toArray
(
new
CompletableFuture
[
0
])).
join
();
}
}
...
@@ -418,6 +455,87 @@ public class TaskServiceImpl implements TaskService {
...
@@ -418,6 +455,87 @@ public class TaskServiceImpl implements TaskService {
},
cacheServiceExecutor
)).
toArray
(
CompletableFuture
[]::
new
)).
join
();
},
cacheServiceExecutor
)).
toArray
(
CompletableFuture
[]::
new
)).
join
();
}
}
@Override
public
void
monthlyCustomXhsInteractionUpdate
()
{
// 近六个月
long
endTime
=
System
.
currentTimeMillis
();
long
startTime
=
endTime
-
Constant
.
ONE_MONTH
*
6
;
// 去除近10天区间,近10天为1天/次
endTime
=
endTime
-
10
*
Constant
.
ONE_DAY
;
List
<
Project
>
projects
=
GlobalPojo
.
PROJECT_MAP
.
values
().
stream
()
.
filter
(
project
->
CollectionUtils
.
isNotEmpty
(
project
.
getModuleShowList
())
&&
project
.
getModuleShowList
().
contains
(
"xiaohongshu"
)).
collect
(
Collectors
.
toList
());
for
(
Project
project
:
projects
)
{
try
{
List
<
Pair
<
String
,
Map
<
String
,
Object
>>>
res
=
customXhsInteractionUpdate
(
project
.
getId
(),
startTime
,
endTime
,
Collections
.
singletonList
(
"6433c2251701316728003be4"
));
esClientDao
.
batchUpdate
(
res
);
}
catch
(
Exception
e
){
log
.
error
(
"项目:{},定制化舆情分析-互动量更新出错-时间范围:{}-{}"
,
project
.
getProjectName
(),
startTime
,
endTime
);
}
}
}
@Override
public
void
dailyCustomXhsInteractionUpdate
()
{
// 近10天
long
endTime
=
System
.
currentTimeMillis
();
long
startTime
=
endTime
-
Constant
.
ONE_DAY
*
10
;
List
<
Project
>
projects
=
GlobalPojo
.
PROJECT_MAP
.
values
().
stream
()
.
filter
(
project
->
CollectionUtils
.
isNotEmpty
(
project
.
getModuleShowList
())
&&
project
.
getModuleShowList
().
contains
(
"xiaohongshu"
)).
collect
(
Collectors
.
toList
());
for
(
Project
project
:
projects
)
{
try
{
List
<
Pair
<
String
,
Map
<
String
,
Object
>>>
res
=
customXhsInteractionUpdate
(
project
.
getId
(),
startTime
,
endTime
,
Collections
.
singletonList
(
"6433c2251701316728003be4"
));
esClientDao
.
batchUpdate
(
res
);
}
catch
(
Exception
e
){
log
.
error
(
"项目:{},定制化舆情分析-互动量更新出错-时间范围:{}-{}-"
,
project
.
getProjectName
(),
startTime
,
endTime
,
e
);
}
}
}
private
List
<
Pair
<
String
,
Map
<
String
,
Object
>>>
customXhsInteractionUpdate
(
String
projectId
,
Long
startTime
,
Long
endTime
,
List
<
String
>
platforms
)
throws
IOException
{
List
<
Pair
<
String
,
Map
<
String
,
Object
>>>
res
=
new
ArrayList
<>();
BoolQueryBuilder
postFilter
=
EsQueryTools
.
assembleCacheMapsQuery
(
projectId
,
Constant
.
PRIMARY_CONTEND_ID
);
postFilter
.
must
(
QueryBuilders
.
rangeQuery
(
"time"
).
gte
(
startTime
).
lt
(
endTime
));
if
(
CollectionUtils
.
isNotEmpty
(
platforms
))
{
postFilter
.
must
(
EsQueryTools
.
assemblePlatformQuery
(
Tools
.
getPlatformByIds
(
platforms
)));
}
List
<
JSONObject
>
list
=
esClientDao
.
searchScroll
(
postFilter
,
10000
,
new
String
[]{
"id"
,
"like_num"
,
"favourites_num"
,
"comment_num"
,
"share_num"
,
"time"
,
"url"
});
List
<
String
>
urls
=
list
.
stream
().
map
(
json
->
json
.
getString
(
"url"
)).
distinct
().
collect
(
Collectors
.
toList
());
log
.
info
(
"项目:{}-定制化舆情分析-互动量更新-时间范围:{}-{},即将更新共{}条"
,
projectId
,
startTime
,
endTime
,
list
.
size
());
// 链接互动量更新
Map
<
String
,
JSONObject
>
updatedResMap
=
toolsetService
.
urlInteractionUpdate
(
urls
).
stream
().
collect
(
Collectors
.
toMap
(
json
->
json
.
getString
(
"url"
),
o
->
o
,
(
v1
,
v2
)
->
v1
));
int
failedCount
=
0
;
for
(
JSONObject
esJsonData
:
list
)
{
Map
<
String
,
Object
>
resJson
=
new
HashMap
<>();
JSONObject
updatedJson
=
updatedResMap
.
get
(
esJsonData
.
getString
(
"url"
));
if
(
Objects
.
isNull
(
updatedJson
)||
updatedJson
.
isEmpty
()
||
200
!=
updatedJson
.
getIntValue
(
"code"
)){
++
failedCount
;
continue
;
}
// 点赞、收藏、评论、分享
int
likeNum
=
Objects
.
nonNull
(
updatedJson
.
getInteger
(
"likeCount"
))
?
updatedJson
.
getIntValue
(
"likeCount"
)
:
Objects
.
nonNull
(
esJsonData
.
getInteger
(
"like_num"
))
?
esJsonData
.
getInteger
(
"like_num"
)
:
0
;
int
favouritesNum
=
Objects
.
nonNull
(
updatedJson
.
getInteger
(
"collectCount"
))
?
updatedJson
.
getIntValue
(
"collectCount"
)
:
Objects
.
nonNull
(
esJsonData
.
getInteger
(
"favourites_num"
))
?
esJsonData
.
getInteger
(
"favourites_num"
)
:
0
;
int
commentNum
=
Objects
.
nonNull
(
updatedJson
.
getInteger
(
"commentCount"
))
?
updatedJson
.
getIntValue
(
"commentCount"
)
:
Objects
.
nonNull
(
esJsonData
.
getInteger
(
"comment_num"
))
?
esJsonData
.
getInteger
(
"comment_num"
)
:
0
;
int
shareNum
=
Objects
.
nonNull
(
updatedJson
.
getInteger
(
"shareCount"
))
?
updatedJson
.
getIntValue
(
"shareCount"
)
:
Objects
.
nonNull
(
esJsonData
.
getInteger
(
"share_num"
))
?
esJsonData
.
getInteger
(
"share_num"
)
:
0
;
resJson
.
put
(
"id"
,
esJsonData
.
getString
(
"id"
));
resJson
.
put
(
"like_num"
,
likeNum
);
resJson
.
put
(
"favourites_num"
,
favouritesNum
);
resJson
.
put
(
"comment_num"
,
commentNum
);
resJson
.
put
(
"share_num"
,
shareNum
);
// 数据对应index
Calendar
date
=
Calendar
.
getInstance
();
date
.
setTime
(
new
Date
(
esJsonData
.
getLongValue
(
"time"
)));
String
index
=
GenericAttribute
.
ES_INDEX_PRE
+
date
.
get
(
Calendar
.
YEAR
);
res
.
add
(
Pair
.
of
(
index
,
resJson
));
}
xiaohongshuRecordDao
.
insertOne
(
XiaohongshuRecord
.
createUsedRecord
(
projectId
,
list
.
size
(),
null
));
log
.
info
(
"项目:{}-定制化舆情分析-互动量更新-时间范围:{}-{},本次更新共{}条,失败{}条"
,
projectId
,
startTime
,
endTime
,
list
.
size
(),
failedCount
);
return
res
;
}
private
boolean
reportSendByProject
(
Project
project
)
{
private
boolean
reportSendByProject
(
Project
project
)
{
boolean
flag
=
false
;
boolean
flag
=
false
;
// 扫描setting信息并生成对应报告
// 扫描setting信息并生成对应报告
...
...
src/main/java/com/zhiwei/brandkbs2/service/impl/ToolsetServiceImpl.java
View file @
18f92bf9
...
@@ -271,11 +271,7 @@ public class ToolsetServiceImpl implements ToolsetService {
...
@@ -271,11 +271,7 @@ public class ToolsetServiceImpl implements ToolsetService {
* @param urls 链接
* @param urls 链接
* @return
* @return
*/
*/
private
List
<
JSONObject
>
urlInteractionUpdate
(
List
<
String
>
urls
){
public
List
<
JSONObject
>
urlInteractionUpdate
(
List
<
String
>
urls
){
return
urlInteractionUpdate
(
urls
,
null
);
}
private
List
<
JSONObject
>
urlInteractionUpdate
(
List
<
String
>
urls
,
String
taskId
){
// 通过url获取域名进而获取任务类型
// 通过url获取域名进而获取任务类型
Map
<
String
,
String
>
map
=
new
HashMap
<>();
Map
<
String
,
String
>
map
=
new
HashMap
<>();
urls
.
forEach
(
url
->
map
.
compute
(
url
,
(
key
,
value
)
->
{
urls
.
forEach
(
url
->
map
.
compute
(
url
,
(
key
,
value
)
->
{
...
@@ -299,10 +295,6 @@ public class ToolsetServiceImpl implements ToolsetService {
...
@@ -299,10 +295,6 @@ public class ToolsetServiceImpl implements ToolsetService {
for
(
List
<
String
>
ids
:
ListUtils
.
partition
(
taskIdList
,
50
))
{
for
(
List
<
String
>
ids
:
ListUtils
.
partition
(
taskIdList
,
50
))
{
result
.
addAll
(
getInteractionResult
(
ids
));
result
.
addAll
(
getInteractionResult
(
ids
));
}
}
if
(
Objects
.
nonNull
(
taskId
))
{
// 更新处理进度
addProcessSchedule
(
taskId
,
taskIdList
.
size
());
}
}
}
return
result
;
return
result
;
}
}
...
...
src/main/java/com/zhiwei/brandkbs2/service/impl/XiaohongshuWordServiceImpl.java
0 → 100644
View file @
18f92bf9
package
com
.
zhiwei
.
brandkbs2
.
service
.
impl
;
import
com.alibaba.fastjson.JSONObject
;
import
com.zhiwei.brandkbs2.auth.UserThreadLocal
;
import
com.zhiwei.brandkbs2.dao.XiaohongshuWordDao
;
import
com.zhiwei.brandkbs2.model.ResponseResult
;
import
com.zhiwei.brandkbs2.pojo.XiaohongshuWord
;
import
com.zhiwei.brandkbs2.pojo.vo.PageVO
;
import
com.zhiwei.brandkbs2.service.XiaohongshuWordService
;
import
com.zhiwei.brandkbs2.util.MongoUtil
;
import
org.springframework.beans.factory.annotation.Qualifier
;
import
org.springframework.data.mongodb.core.query.Criteria
;
import
org.springframework.data.mongodb.core.query.Query
;
import
org.springframework.stereotype.Service
;
import
java.util.List
;
import
java.util.stream.Collectors
;
/**
* @ClassName: XiaohongshuWordServiceImpl
* @Description 小红书定制模块关键词接口实现类
* @author: sjj
* @date: 2024-06-28 10:11
*/
@Service
(
"xiaohongshuWordServiceImpl"
)
public
class
XiaohongshuWordServiceImpl
extends
TableWordServiceImpl
implements
XiaohongshuWordService
{
final
XiaohongshuWordDao
xiaohongshuWordDao
;
public
XiaohongshuWordServiceImpl
(
@Qualifier
(
"xiaohongshuWordDao"
)
XiaohongshuWordDao
xiaohongshuWordDao
,
@Qualifier
(
"mongoUtil"
)
MongoUtil
mongoUtil
)
{
super
(
xiaohongshuWordDao
,
mongoUtil
);
this
.
xiaohongshuWordDao
=
xiaohongshuWordDao
;
}
@Override
public
PageVO
<
JSONObject
>
findWord
(
int
page
,
int
size
,
String
keyword
)
{
String
projectId
=
UserThreadLocal
.
getProjectId
();
Query
query
=
Query
.
query
(
Criteria
.
where
(
"projectId"
).
is
(
projectId
));
long
count
=
xiaohongshuWordDao
.
count
(
query
);
mongoUtil
.
start
(
page
,
size
,
query
);
List
<
XiaohongshuWord
>
xiaohongshuWords
=
xiaohongshuWordDao
.
findListByKeywordFuzz
(
query
,
keyword
,
new
String
[]{
"keyword"
});
List
<
JSONObject
>
resList
=
xiaohongshuWords
.
stream
().
map
(
xiaohongshuWord
->
{
JSONObject
result
=
new
JSONObject
();
result
.
put
(
"id"
,
xiaohongshuWord
.
getId
());
result
.
put
(
"keyword"
,
xiaohongshuWord
.
getKeyword
());
result
.
put
(
"cTime"
,
xiaohongshuWord
.
getCTime
());
result
.
put
(
"submitter"
,
xiaohongshuWord
.
getSubmitter
());
result
.
put
(
"planName"
,
xiaohongshuWord
.
getPlanName
());
return
result
;
}).
collect
(
Collectors
.
toList
());
MongoUtil
.
PageHelper
<
JSONObject
>
pageHelper
=
mongoUtil
.
pageHelper
(
count
,
resList
);
return
PageVO
.
createPageVo
(
pageHelper
,
resList
);
}
@Override
public
ResponseResult
addWord
(
String
keyword
,
String
planName
,
String
projectId
,
String
submitter
)
{
long
count
=
xiaohongshuWordDao
.
count
(
Query
.
query
(
Criteria
.
where
(
"projectId"
).
is
(
projectId
)));
// 关键词10上限
if
(
count
>=
10
)
{
return
ResponseResult
.
failure
(
"新增方案添加超过上限值10"
);
}
XiaohongshuWord
xiaohongshuWord
=
new
XiaohongshuWord
();
xiaohongshuWord
.
setKeyword
(
keyword
);
xiaohongshuWord
.
setPlanName
(
planName
);
xiaohongshuWord
.
setProjectId
(
projectId
);
xiaohongshuWord
.
setSubmitter
(
submitter
);
xiaohongshuWord
.
setCTime
(
System
.
currentTimeMillis
());
xiaohongshuWordDao
.
insertOne
(
xiaohongshuWord
);
return
ResponseResult
.
success
();
}
@Override
public
void
updateWord
(
String
id
,
String
keyword
,
String
planName
,
String
projectId
,
String
submitter
)
{
XiaohongshuWord
xiaohongshuWord
=
xiaohongshuWordDao
.
findOneById
(
id
);
xiaohongshuWord
.
setKeyword
(
keyword
);
xiaohongshuWord
.
setPlanName
(
planName
);
xiaohongshuWord
.
setSubmitter
(
submitter
);
xiaohongshuWordDao
.
updateOne
(
xiaohongshuWord
);
}
}
src/main/java/com/zhiwei/brandkbs2/task/ControlCenter.java
View file @
18f92bf9
...
@@ -118,4 +118,30 @@ public class ControlCenter {
...
@@ -118,4 +118,30 @@ public class ControlCenter {
log
.
info
(
"定时生成每日一报-结束"
);
log
.
info
(
"定时生成每日一报-结束"
);
}
}
}
}
@Async
(
"scheduledExecutor"
)
@Scheduled
(
cron
=
"0 30 3 L * ?"
)
public
void
monthlyCustomXhsInteractionUpdate
()
{
log
.
info
(
"每月互动量更新-启动"
);
try
{
taskService
.
monthlyCustomXhsInteractionUpdate
();
}
catch
(
Exception
e
)
{
log
.
error
(
"每月互动量更新-出错"
,
e
);
}
finally
{
log
.
info
(
"每月互动量更新-结束"
);
}
}
@Async
(
"scheduledExecutor"
)
@Scheduled
(
cron
=
"0 0 5 * * ?"
)
public
void
dailyCustomXhsInteractionUpdate
()
{
log
.
info
(
"每日互动量更新-启动"
);
try
{
taskService
.
dailyCustomXhsInteractionUpdate
();
}
catch
(
Exception
e
)
{
log
.
error
(
"每日互动量更新-出错"
,
e
);
}
finally
{
log
.
info
(
"每日互动量更新-结束"
);
}
}
}
}
src/main/java/com/zhiwei/brandkbs2/util/RedisUtil.java
View file @
18f92bf9
...
@@ -91,18 +91,22 @@ public class RedisUtil {
...
@@ -91,18 +91,22 @@ public class RedisUtil {
return
RedisKeyPrefix
.
NON_MANUAL_PROJECT_MARK_MAX_GID
+
Tools
.
concat
(
projectId
,
planId
,
userId
);
return
RedisKeyPrefix
.
NON_MANUAL_PROJECT_MARK_MAX_GID
+
Tools
.
concat
(
projectId
,
planId
,
userId
);
}
}
public
static
String
getYuqingAnalyzeProjectAvgCountKey
(
String
projectId
,
String
contendId
,
String
planId
){
public
static
String
getYuqingAnalyzeProjectAvgCountKey
(
String
projectId
,
String
contendId
,
String
planId
,
String
customPlanId
,
boolean
isCustom
){
if
(
Objects
.
isNull
(
planId
)){
String
keyPrefix
=
isCustom
?
RedisKeyPrefix
.
CUSTOM_YUQING_ANALYZE_AVG_COUNT
:
RedisKeyPrefix
.
YUQING_ANALYZE_PROJECT_AVG_COUNT
;
return
RedisKeyPrefix
.
YUQING_ANALYZE_PROJECT_AVG_COUNT
+
Tools
.
concat
(
projectId
,
contendId
);
String
finalPlanId
=
isCustom
?
customPlanId
:
planId
;
if
(
Objects
.
isNull
(
finalPlanId
)){
return
keyPrefix
+
Tools
.
concat
(
projectId
,
contendId
);
}
}
return
RedisKeyPrefix
.
YUQING_ANALYZE_PROJECT_AVG_COUNT
+
Tools
.
concat
(
projectId
,
contendId
,
p
lanId
);
return
keyPrefix
+
Tools
.
concat
(
projectId
,
contendId
,
finalP
lanId
);
}
}
public
static
String
getYuqingAnalyzeEmotionDistributionAvgKey
(
String
projectId
,
String
contendId
,
String
emotion
,
String
planId
){
public
static
String
getYuqingAnalyzeEmotionDistributionAvgKey
(
String
projectId
,
String
contendId
,
String
emotion
,
String
planId
,
String
customPlanId
,
boolean
isCustom
){
if
(
Objects
.
isNull
(
planId
)){
String
keyPrefix
=
isCustom
?
RedisKeyPrefix
.
CUSTOM_YUQING_ANALYZE_EMOTION_DISTRIBUTION_AVG
:
RedisKeyPrefix
.
YUQING_ANALYZE_EMOTION_DISTRIBUTION_AVG
;
return
RedisKeyPrefix
.
YUQING_ANALYZE_EMOTION_DISTRIBUTION_AVG
+
Tools
.
concat
(
projectId
,
contendId
,
emotion
);
String
finalPlanId
=
isCustom
?
customPlanId
:
planId
;
if
(
Objects
.
isNull
(
finalPlanId
)){
return
keyPrefix
+
Tools
.
concat
(
projectId
,
contendId
,
emotion
);
}
}
return
RedisKeyPrefix
.
YUQING_ANALYZE_EMOTION_DISTRIBUTION_AVG
+
Tools
.
concat
(
projectId
,
contendId
,
emotion
,
p
lanId
);
return
keyPrefix
+
Tools
.
concat
(
projectId
,
contendId
,
emotion
,
finalP
lanId
);
}
}
public
static
String
getYuqingAnalyzePlatformAvgCountKey
(
String
projectId
,
String
contendId
,
String
planId
){
public
static
String
getYuqingAnalyzePlatformAvgCountKey
(
String
projectId
,
String
contendId
,
String
planId
){
...
@@ -112,11 +116,13 @@ public class RedisUtil {
...
@@ -112,11 +116,13 @@ public class RedisUtil {
return
RedisKeyPrefix
.
YUQING_ANALYZE_PLATFORM_AVG_COUNT
+
Tools
.
concat
(
projectId
,
contendId
,
planId
);
return
RedisKeyPrefix
.
YUQING_ANALYZE_PLATFORM_AVG_COUNT
+
Tools
.
concat
(
projectId
,
contendId
,
planId
);
}
}
public
static
String
getYuqingAnalyzeHighWordKey
(
String
projectId
,
String
contendId
,
String
planId
,
Long
startTime
,
Long
endTime
){
public
static
String
getYuqingAnalyzeHighWordKey
(
String
projectId
,
String
contendId
,
String
planId
,
Long
startTime
,
Long
endTime
,
String
customPlanId
,
boolean
isCustom
){
if
(
Objects
.
isNull
(
planId
)){
String
keyPrefix
=
isCustom
?
RedisKeyPrefix
.
CUSTOM_YUQING_ANALYZE_HIGH_WORD
:
RedisKeyPrefix
.
YUQING_ANALYZE_HIGH_WORD
;
return
RedisKeyPrefix
.
YUQING_ANALYZE_HIGH_WORD
+
Tools
.
concat
(
projectId
,
contendId
,
startTime
,
endTime
);
String
finalPlanId
=
isCustom
?
customPlanId
:
planId
;
if
(
Objects
.
isNull
(
finalPlanId
)){
return
keyPrefix
+
Tools
.
concat
(
projectId
,
contendId
,
startTime
,
endTime
);
}
}
return
RedisKeyPrefix
.
YUQING_ANALYZE_HIGH_WORD
+
Tools
.
concat
(
projectId
,
contendId
,
p
lanId
,
startTime
,
endTime
);
return
keyPrefix
+
Tools
.
concat
(
projectId
,
contendId
,
finalP
lanId
,
startTime
,
endTime
);
}
}
public
static
String
getSearchKeywordCacheKey
(
String
projectId
,
String
userId
,
String
searchType
){
public
static
String
getSearchKeywordCacheKey
(
String
projectId
,
String
userId
,
String
searchType
){
...
...
src/main/resources/application-dev.properties
View file @
18f92bf9
...
@@ -93,6 +93,8 @@ trends.longTimeInListSearch.url=https://trends.zhiweidata.com/hotSearchTrend/sea
...
@@ -93,6 +93,8 @@ trends.longTimeInListSearch.url=https://trends.zhiweidata.com/hotSearchTrend/sea
trends.getHotSearchFromEsInTimeAndTypeOrWord.url
=
https://hotsearch-manage.zhiweidata.com/hotsearch/hotSearch/getHotSearchFromEsInTimeAndTypeOrWord?limit={1}&page={2}&type={3}&word={4}&sort={5}&accurateWord={6}&maskWord={7}
trends.getHotSearchFromEsInTimeAndTypeOrWord.url
=
https://hotsearch-manage.zhiweidata.com/hotsearch/hotSearch/getHotSearchFromEsInTimeAndTypeOrWord?limit={1}&page={2}&type={3}&word={4}&sort={5}&accurateWord={6}&maskWord={7}
trends.queryHotSearchTrendInner.url
=
https://trends.zhiweidata.com/hotSearchTrend/inner/queryHotSearchTrendInner
trends.queryHotSearchTrendInner.url
=
https://trends.zhiweidata.com/hotSearchTrend/inner/queryHotSearchTrendInner
trends.getHotSearchSnapshot.url
=
https://hotsearch-manage.zhiweidata.com/hotsearch/hotSearch/getHotSearchSnapshot?type={1}&time={2}&rank={3}
trends.getHotSearchSnapshot.url
=
https://hotsearch-manage.zhiweidata.com/hotsearch/hotSearch/getHotSearchSnapshot?type={1}&time={2}&rank={3}
trend.nearYearWords.token
=
vo5ozWt861R125FMpEDCXxRkft3e5tVi3W4dG2hFLBViRFtGsmWtB5bx9MOlE6Vn
trends.nearYearWords.url
=
https://hotsearch-manage.zhiweidata.com/zhiwei-hot-words-web/brand/nearYearWords
#\u4E8B\u4EF6\u5E93\u5916\u90E8\u63A5\u53E3
#\u4E8B\u4EF6\u5E93\u5916\u90E8\u63A5\u53E3
ef.search.url
=
https://ef.zhiweidata.com/external/search.do?name={1}&page={2}
ef.search.url
=
https://ef.zhiweidata.com/external/search.do?name={1}&page={2}
ef.searchCriteria.url
=
https://ef.zhiweidata.com/index/getSearchKey.do
ef.searchCriteria.url
=
https://ef.zhiweidata.com/index/getSearchKey.do
...
...
src/main/resources/application-local.properties
View file @
18f92bf9
...
@@ -102,7 +102,8 @@ trends.longTimeInListSearch.url=https://trends.zhiweidata.com/hotSearchTrend/sea
...
@@ -102,7 +102,8 @@ trends.longTimeInListSearch.url=https://trends.zhiweidata.com/hotSearchTrend/sea
trends.getHotSearchFromEsInTimeAndTypeOrWord.url
=
https://hotsearch-manage.zhiweidata.com/hotsearch/hotSearch/getHotSearchFromEsInTimeAndTypeOrWord?limit={1}&page={2}&type={3}&word={4}&sort={5}&accurateWord={6}&maskWord={7}
trends.getHotSearchFromEsInTimeAndTypeOrWord.url
=
https://hotsearch-manage.zhiweidata.com/hotsearch/hotSearch/getHotSearchFromEsInTimeAndTypeOrWord?limit={1}&page={2}&type={3}&word={4}&sort={5}&accurateWord={6}&maskWord={7}
trends.queryHotSearchTrendInner.url
=
https://trends.zhiweidata.com/hotSearchTrend/inner/queryHotSearchTrendInner
trends.queryHotSearchTrendInner.url
=
https://trends.zhiweidata.com/hotSearchTrend/inner/queryHotSearchTrendInner
trends.getHotSearchSnapshot.url
=
https://hotsearch-manage.zhiweidata.com/hotsearch/hotSearch/getHotSearchSnapshot?type={1}&time={2}&rank={3}
trends.getHotSearchSnapshot.url
=
https://hotsearch-manage.zhiweidata.com/hotsearch/hotSearch/getHotSearchSnapshot?type={1}&time={2}&rank={3}
trend.nearYearWords.token
=
vo5ozWt861R125FMpEDCXxRkft3e5tVi3W4dG2hFLBViRFtGsmWtB5bx9MOlE6Vn
trends.nearYearWords.url
=
https://hotsearch-manage.zhiweidata.com/zhiwei-hot-words-web/brand/nearYearWords
#\u4E8B\u4EF6\u5E93\u5916\u90E8\u63A5\u53E3
#\u4E8B\u4EF6\u5E93\u5916\u90E8\u63A5\u53E3
ef.search.url
=
https://ef.zhiweidata.com/external/search.do?name={1}&page={2}&size={3}
ef.search.url
=
https://ef.zhiweidata.com/external/search.do?name={1}&page={2}&size={3}
ef.searchCriteria.url
=
https://ef.zhiweidata.com/index/getSearchKey.do
ef.searchCriteria.url
=
https://ef.zhiweidata.com/index/getSearchKey.do
...
...
src/main/resources/application-prod.properties
View file @
18f92bf9
...
@@ -93,6 +93,8 @@ trends.longTimeInListSearch.url=https://trends.zhiweidata.com/hotSearchTrend/sea
...
@@ -93,6 +93,8 @@ trends.longTimeInListSearch.url=https://trends.zhiweidata.com/hotSearchTrend/sea
trends.getHotSearchFromEsInTimeAndTypeOrWord.url
=
https://hotsearch-manage.zhiweidata.com/hotsearch/hotSearch/getHotSearchFromEsInTimeAndTypeOrWord?limit={1}&page={2}&type={3}&word={4}&sort={5}&accurateWord={6}&maskWord={7}
trends.getHotSearchFromEsInTimeAndTypeOrWord.url
=
https://hotsearch-manage.zhiweidata.com/hotsearch/hotSearch/getHotSearchFromEsInTimeAndTypeOrWord?limit={1}&page={2}&type={3}&word={4}&sort={5}&accurateWord={6}&maskWord={7}
trends.queryHotSearchTrendInner.url
=
https://trends.zhiweidata.com/hotSearchTrend/inner/queryHotSearchTrendInner
trends.queryHotSearchTrendInner.url
=
https://trends.zhiweidata.com/hotSearchTrend/inner/queryHotSearchTrendInner
trends.getHotSearchSnapshot.url
=
https://hotsearch-manage.zhiweidata.com/hotsearch/hotSearch/getHotSearchSnapshot?type={1}&time={2}&rank={3}
trends.getHotSearchSnapshot.url
=
https://hotsearch-manage.zhiweidata.com/hotsearch/hotSearch/getHotSearchSnapshot?type={1}&time={2}&rank={3}
trend.nearYearWords.token
=
vo5ozWt861R125FMpEDCXxRkft3e5tVi3W4dG2hFLBViRFtGsmWtB5bx9MOlE6Vn
trends.nearYearWords.url
=
https://hotsearch-manage.zhiweidata.com/zhiwei-hot-words-web/brand/nearYearWords
#\u4E8B\u4EF6\u5E93\u5916\u90E8\u63A5\u53E3
#\u4E8B\u4EF6\u5E93\u5916\u90E8\u63A5\u53E3
ef.search.url
=
https://ef.zhiweidata.com/external/search.do?name={1}&page={2}
ef.search.url
=
https://ef.zhiweidata.com/external/search.do?name={1}&page={2}
ef.searchCriteria.url
=
https://ef.zhiweidata.com/index/getSearchKey.do
ef.searchCriteria.url
=
https://ef.zhiweidata.com/index/getSearchKey.do
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment