Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
M
middleware-automatic
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
shenjunjie
middleware-automatic
Commits
2fedede7
Commit
2fedede7
authored
Apr 25, 2024
by
shentao
Browse files
Options
Browse Files
Download
Plain Diff
Merge branch 'feature' into 'release'
2024/4/25 指定标题不进行聚合、数字降低聚合相似度 See merge request
!90
parents
cb058d68
32462774
Hide whitespace changes
Inline
Side-by-side
Showing
18 changed files
with
115 additions
and
182 deletions
+115
-182
middleware-automatic-center-client-autoconfigure/pom.xml
+2
-2
middleware-automatic-center-client/pom.xml
+1
-1
middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/core/AutoMaticClient.java
+4
-0
middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/dubbo/service/AutoMaticService.java
+11
-0
middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/graphs/GraphsServer.java
+0
-11
middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/graphs/MonitorGraphsImpl.java
+0
-139
middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/pojo/TemplateTitleVo.java
+14
-1
middleware-automatic-center-server/pom.xml
+1
-1
middleware-automatic-center-server/src/main/java/com/zhiwei/middleware/automatic/server/dubbo/service/AutoMaticService.java
+11
-0
middleware-automatic-center-server/src/main/java/com/zhiwei/middleware/automatic/server/dubbo/service/impl/AutoMaticServiceImpl.java
+7
-2
middleware-automatic-center-server/src/main/java/com/zhiwei/middleware/automatic/server/service/TemplateTitleService.java
+1
-1
middleware-automatic-center-server/src/main/java/com/zhiwei/middleware/automatic/server/service/impl/TemplateTitleServiceImpl.java
+2
-4
middleware-automatic-center-server/src/main/java/com/zhiwei/middleware/automatic/server/util/Tools.java
+11
-4
middleware-automatic-center-son/pom.xml
+1
-1
middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/task/holder/TaskServiceHandler.java
+0
-1
middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/task/service/TaskServiceCommon.java
+29
-12
middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/task/service/TaskServiceTemplate.java
+7
-1
middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/util/Tools.java
+13
-1
No files found.
middleware-automatic-center-client-autoconfigure/pom.xml
View file @
2fedede7
...
...
@@ -11,13 +11,13 @@
</parent>
<artifactId>
middleware-automatic-center-autoconfigure
</artifactId>
<version>
1.0.0.
1
-SNAPSHOT
</version>
<version>
1.0.0.
3
-SNAPSHOT
</version>
<packaging>
jar
</packaging>
<properties>
<project.build.sourceEncoding>
UTF-8
</project.build.sourceEncoding>
<java.version>
1.8
</java.version>
<automatic.version>
1.0.0.
2
-SNAPSHOT
</automatic.version>
<automatic.version>
1.0.0.
3
-SNAPSHOT
</automatic.version>
</properties>
<dependencies>
...
...
middleware-automatic-center-client/pom.xml
View file @
2fedede7
...
...
@@ -11,7 +11,7 @@
</parent>
<artifactId>
middleware-automatic-center-client
</artifactId>
<version>
1.0.0.
2
-SNAPSHOT
</version>
<version>
1.0.0.
3
-SNAPSHOT
</version>
<properties>
<project.build.sourceEncoding>
UTF-8
</project.build.sourceEncoding>
...
...
middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/core/AutoMaticClient.java
View file @
2fedede7
...
...
@@ -53,6 +53,10 @@ public class AutoMaticClient {
return
autoMaticService
.
compareWithTemplateTileOL
(
project
,
title
);
}
public
Map
<
String
,
Object
>
compareWithTemplateTileOL
(
String
project
,
String
title
,
boolean
isUse
)
{
return
autoMaticService
.
compareWithTemplateTileOL
(
project
,
title
,
isUse
);
}
public
PageVO
<
JSONObject
>
getTemplateDaySumData
(
int
page
,
int
pageSize
,
String
templateId
)
{
return
autoMaticService
.
getTemplateDaySumData
(
page
,
pageSize
,
templateId
);
}
...
...
middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/dubbo/service/AutoMaticService.java
View file @
2fedede7
...
...
@@ -62,6 +62,17 @@ public interface AutoMaticService {
* @return 返回值
*/
Map
<
String
,
Object
>
compareWithTemplateTileOL
(
String
project
,
String
title
);
/**
* 根据项目组和标题在线匹配已有聚合标题 (使用中/已弃用)
*
* @param project 项目
* @param title 标题
* @param isUse 使用中
* @return 返回值
*/
Map
<
String
,
Object
>
compareWithTemplateTileOL
(
String
project
,
String
title
,
boolean
isUse
);
/**
* 分页查询模板声量统计详细数据
* @param templateId 模板id
...
...
middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/graphs/GraphsServer.java
deleted
100644 → 0
View file @
cb058d68
package
com
.
zhiwei
.
middleware
.
automatic
.
server
.
graphs
;
import
java.util.List
;
import
java.util.Map
;
public
interface
GraphsServer
<
T
,
O
>
{
void
addGraph
(
List
<
T
>
t
);
List
<
O
>
find
(
String
text
);
}
middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/graphs/MonitorGraphsImpl.java
deleted
100644 → 0
View file @
cb058d68
package
com
.
zhiwei
.
middleware
.
automatic
.
server
.
graphs
;
import
com.zhiwei.middleware.automatic.server.pojo.GroupTerm
;
import
com.zhiwei.middleware.automatic.server.pojo.MonitorKeyword
;
import
com.zhiwei.middleware.automatic.server.pojo.QbjcRuleMatchedInfo
;
import
org.apache.commons.lang3.StringUtils
;
import
java.util.*
;
import
java.util.stream.Collectors
;
public
class
MonitorGraphsImpl
implements
GraphsServer
<
MonitorKeyword
,
QbjcRuleMatchedInfo
>
{
/**
* 匹配图
*/
private
Graphs
graphs
;
/**
* 匹配图对应的绑定信息
*/
private
Map
<
String
,
List
<
GroupTerm
>>
terms
;
public
MonitorGraphsImpl
()
{
this
.
graphs
=
new
Graphs
();
this
.
terms
=
new
HashMap
<>();
}
@Override
public
void
addGraph
(
List
<
MonitorKeyword
>
monitorKeyword
)
{
Graphs
tempGraphs
=
new
Graphs
();
Map
<
String
,
List
<
GroupTerm
>>
tempTerms
=
new
HashMap
<>();
monitorKeyword
.
forEach
(
keyword
->
{
if
(
null
!=
keyword
.
getMonitorLevel
())
{
preGraphs
(
keyword
.
getKeywords
(),
tempGraphs
);
preTerms
(
keyword
,
tempTerms
);
}
});
graphs
=
tempGraphs
;
terms
=
tempTerms
;
}
@Override
public
List
<
QbjcRuleMatchedInfo
>
find
(
String
text
)
{
text
=
StringUtils
.
lowerCase
(
text
);
// 返回值
List
<
QbjcRuleMatchedInfo
>
res
=
new
ArrayList
<>();
// 匹配
List
<
Keyword
>
results
=
graphs
.
find
(
text
);
// 根据Keyword:key 统计分组
Map
<
String
,
List
<
Keyword
>>
kResults
=
results
.
stream
().
collect
(
Collectors
.
groupingBy
(
Keyword:
:
getKey
));
// 已统计列表,防止A&B 被统计两次
Set
<
String
>
hasMatched
=
new
HashSet
<>();
// 统计结果
Map
<
String
,
Integer
>
statis
=
graphs
.
change2Statistics
(
results
);
statis
.
forEach
((
keyword
,
rate
)
->
{
Set
<
Keyword
>
hitKeywords
=
new
HashSet
<>();
List
<
GroupTerm
>
list
=
terms
.
get
(
keyword
);
if
(
null
==
list
)
{
throw
new
IllegalStateException
(
"keyword不存在:"
+
keyword
);
}
// 可能符合的结果
list
.
forEach
(
groupTerm
->
{
MonitorKeyword
monitorKeyword
=
groupTerm
.
getMonitorKeyword
();
int
count
=
-
1
;
for
(
String
checkWord
:
groupTerm
.
getAndKeywords
())
{
// 已统计过,跳过该词组
if
(
hasMatched
.
contains
(
checkWord
))
{
count
=
-
1
;
break
;
}
int
current
=
statis
.
getOrDefault
(
checkWord
,
0
);
if
(
current
>
0
)
{
hitKeywords
.
addAll
(
kResults
.
get
(
checkWord
));
}
// 初次重置或大于最小值
count
=
(
count
==
-
1
||
current
<
count
)
?
current
:
count
;
}
// 大于0判定为命中
if
(
count
>
0
)
{
QbjcRuleMatchedInfo
ruleMatchedInfo
=
new
QbjcRuleMatchedInfo
();
// 设置基本信息
ruleMatchedInfo
.
setId
(
monitorKeyword
.
getId
());
ruleMatchedInfo
.
setProject
(
monitorKeyword
.
getProject
());
ruleMatchedInfo
.
setRuleType
(
QbjcRuleMatchedInfo
.
RuleType
.
getByName
(
monitorKeyword
.
getType
()));
ruleMatchedInfo
.
setChannels
(
monitorKeyword
.
getChannels
());
ruleMatchedInfo
.
setPlatforms
(
monitorKeyword
.
getPlatforms
());
ruleMatchedInfo
.
setMonitorLevel
(
monitorKeyword
.
getMonitorLevel
());
// 设置匹配关键词相关信息
List
<
QbjcRuleMatchedInfo
.
HitInfo
>
infos
=
new
ArrayList
<>();
infos
.
add
(
new
QbjcRuleMatchedInfo
.
HitInfo
(
hitKeywords
,
groupTerm
.
getFullName
(),
count
));
ruleMatchedInfo
.
setHitInfos
(
infos
);
res
.
add
(
ruleMatchedInfo
);
}
});
hasMatched
.
add
(
keyword
);
});
return
res
;
}
/**
* 添加词关联表
*/
private
void
preTerms
(
MonitorKeyword
monitorKeyword
,
Map
<
String
,
List
<
GroupTerm
>>
terms
)
{
List
<
String
>
usedKeywords
=
monitorKeyword
.
getKeywords
();
// 解析字词
usedKeywords
.
forEach
(
usedKeyword
->
{
usedKeyword
=
StringUtils
.
lowerCase
(
usedKeyword
);
// 拆分或逻辑(都可以作为主键)
String
[]
andStrs
=
usedKeyword
.
split
(
"\\|"
);
for
(
String
andStr
:
andStrs
)
{
String
[]
ands
=
andStr
.
trim
().
split
(
" +"
);
for
(
String
str
:
ands
)
{
// 初次创建
terms
.
putIfAbsent
(
str
,
new
ArrayList
<>());
// 添加对应节点
terms
.
get
(
str
).
add
(
new
GroupTerm
(
Arrays
.
asList
(
ands
),
usedKeyword
,
monitorKeyword
));
}
}
});
}
/**
* 添加关键字表
*
* @param usedKeywords void
*/
private
void
preGraphs
(
List
<
String
>
usedKeywords
,
Graphs
graphs
)
{
// 拆分关键词到子
usedKeywords
.
forEach
(
usedKeyword
->
{
usedKeyword
=
StringUtils
.
lowerCase
(
usedKeyword
);
String
[]
andStrs
=
usedKeyword
.
split
(
"\\|"
);
for
(
String
andStr
:
andStrs
)
{
for
(
String
str
:
andStr
.
trim
().
split
(
" +"
))
{
graphs
.
addGraph
(
str
);
}
}
});
}
}
middleware-automatic-center-client/src/main/java/com/zhiwei/middleware/automatic/server/pojo/TemplateTitleVo.java
View file @
2fedede7
...
...
@@ -7,6 +7,7 @@ import java.math.BigInteger;
import
java.security.MessageDigest
;
import
java.security.NoSuchAlgorithmException
;
import
java.util.Date
;
import
java.util.List
;
import
java.util.concurrent.atomic.AtomicLong
;
public
class
TemplateTitleVo
implements
Serializable
{
...
...
@@ -19,6 +20,9 @@ public class TemplateTitleVo implements Serializable {
private
long
daySum
;
private
String
mtag
;
private
String
url
;
private
List
<
String
>
textNumbers
;
private
TemplateStatus
status
;
public
String
getId
()
{
...
...
@@ -99,6 +103,14 @@ public class TemplateTitleVo implements Serializable {
this
.
url
=
url
;
}
public
List
<
String
>
getTextNumbers
()
{
return
textNumbers
;
}
public
void
setTextNumbers
(
List
<
String
>
textNumbers
)
{
this
.
textNumbers
=
textNumbers
;
}
public
TemplateStatus
getStatus
()
{
return
status
;
}
...
...
@@ -108,7 +120,7 @@ public class TemplateTitleVo implements Serializable {
}
public
TemplateTitleVo
(
String
templateTitle
,
String
mtag
,
String
url
)
{
public
TemplateTitleVo
(
String
templateTitle
,
String
mtag
,
String
url
,
List
<
String
>
textNumbers
)
{
this
.
updateTime
=
new
Date
();
this
.
createTime
=
new
Date
();
this
.
templateTitle
=
templateTitle
;
...
...
@@ -116,6 +128,7 @@ public class TemplateTitleVo implements Serializable {
this
.
daySum
=
0L
;
this
.
mtag
=
mtag
;
this
.
url
=
url
;
this
.
textNumbers
=
textNumbers
;
this
.
status
=
TemplateStatus
.
运行中
;
}
...
...
middleware-automatic-center-server/pom.xml
View file @
2fedede7
...
...
@@ -21,7 +21,7 @@
<qbjc-bean.version>
1.1.4.1-SNAPSHOT
</qbjc-bean.version>
<nlp-aggree.version>
0.0.5-SNAPSHOT
</nlp-aggree.version>
<dubbo-server.version>
2.7.4.1
</dubbo-server.version>
<automatic.version>
1.0.0.
2
-SNAPSHOT
</automatic.version>
<automatic.version>
1.0.0.
3
-SNAPSHOT
</automatic.version>
<base.version>
2.0.0-SNAPSHOT
</base.version>
</properties>
...
...
middleware-automatic-center-server/src/main/java/com/zhiwei/middleware/automatic/server/dubbo/service/AutoMaticService.java
View file @
2fedede7
...
...
@@ -69,6 +69,17 @@ public interface AutoMaticService {
*/
Map
<
String
,
Object
>
compareWithTemplateTileOL
(
String
project
,
String
title
);
/**
* 根据项目组和标题在线匹配已有聚合标题 (使用中/已弃用)
*
* @param project 项目
* @param title 标题
* @param isUse 使用中
* @return 返回值
*/
Map
<
String
,
Object
>
compareWithTemplateTileOL
(
String
project
,
String
title
,
boolean
isUse
);
/**
* 分页查询模板声量统计详细数据
* @param templateId 模板id
...
...
middleware-automatic-center-server/src/main/java/com/zhiwei/middleware/automatic/server/dubbo/service/impl/AutoMaticServiceImpl.java
View file @
2fedede7
...
...
@@ -68,7 +68,7 @@ public class AutoMaticServiceImpl implements AutoMaticService {
}
@Override
public
Map
<
String
,
Object
>
getTemplateTitleByProjectByLive
(
String
project
,
String
searchInfo
,
List
<
String
>
tags
,
long
startTime
,
public
Map
<
String
,
Object
>
getTemplateTitleByProjectByLive
(
String
project
,
String
searchInfo
,
List
<
String
>
tags
,
long
startTime
,
long
endTime
,
int
page
,
int
size
,
String
orderField
,
boolean
isAsc
)
{
return
templateTitleService
.
searchTemplateTitle
(
GenericAttribute
.
REDIS_MAP_KEY
,
project
,
searchInfo
,
tags
,
startTime
,
endTime
,
page
,
size
,
orderField
,
isAsc
);
}
...
...
@@ -86,7 +86,12 @@ public class AutoMaticServiceImpl implements AutoMaticService {
@Override
public
Map
<
String
,
Object
>
compareWithTemplateTileOL
(
String
project
,
String
title
)
{
return
templateTitleService
.
compareWithTemplateTileOL
(
project
,
title
);
return
templateTitleService
.
compareWithTemplateTileOL
(
project
,
title
,
true
);
}
@Override
public
Map
<
String
,
Object
>
compareWithTemplateTileOL
(
String
project
,
String
title
,
boolean
isUse
)
{
return
templateTitleService
.
compareWithTemplateTileOL
(
project
,
title
,
isUse
);
}
@Override
...
...
middleware-automatic-center-server/src/main/java/com/zhiwei/middleware/automatic/server/service/TemplateTitleService.java
View file @
2fedede7
...
...
@@ -56,7 +56,7 @@ public interface TemplateTitleService {
* @param title 标题
* @return 返回值
*/
Map
<
String
,
Object
>
compareWithTemplateTileOL
(
String
project
,
String
title
);
Map
<
String
,
Object
>
compareWithTemplateTileOL
(
String
project
,
String
title
,
boolean
isUse
);
/**
* 分页查询模板声量统计详细数据
...
...
middleware-automatic-center-server/src/main/java/com/zhiwei/middleware/automatic/server/service/impl/TemplateTitleServiceImpl.java
View file @
2fedede7
...
...
@@ -126,21 +126,20 @@ public class TemplateTitleServiceImpl implements TemplateTitleService {
}
@Override
public
Map
<
String
,
Object
>
compareWithTemplateTileOL
(
String
project
,
String
title
)
{
public
Map
<
String
,
Object
>
compareWithTemplateTileOL
(
String
project
,
String
title
,
boolean
isUse
)
{
Map
<
String
,
Object
>
res
=
new
HashMap
<>(
3
);
res
.
put
(
"isMatched"
,
false
);
if
(
StringUtils
.
isEmpty
(
title
)
||
StringUtils
.
isEmpty
(
project
))
{
res
.
put
(
"errorMessage"
,
"标题或项目为空"
);
return
res
;
}
Map
<
String
,
TemplateTitleVo
>
templateTitleVoMap
=
getTemplateTitleByProjectLive
(
project
);
Map
<
String
,
TemplateTitleVo
>
templateTitleVoMap
=
isUse
?
getTemplateTitleByProjectLive
(
project
)
:
getTemplateTitleByProjectLost
(
project
);
if
(
templateTitleVoMap
.
isEmpty
())
{
res
.
put
(
"errorMessage"
,
"该项目组未有聚合集"
);
return
res
;
}
TemplateTitleVo
titleVo
=
null
;
double
similarity
=
0.0
;
String
group
=
null
;
for
(
Map
.
Entry
<
String
,
TemplateTitleVo
>
entry
:
templateTitleVoMap
.
entrySet
())
{
if
(
TemplateStatus
.
已重置
==
entry
.
getValue
().
getStatus
())
{
continue
;
...
...
@@ -150,7 +149,6 @@ public class TemplateTitleServiceImpl implements TemplateTitleService {
if
(
currentSimilarity
>
GenericAttribute
.
SIMILAR_STANDARD
&&
currentSimilarity
>
similarity
)
{
similarity
=
currentSimilarity
;
titleVo
=
entry
.
getValue
();
group
=
entry
.
getKey
();
}
}
if
(
Objects
.
nonNull
(
titleVo
))
{
...
...
middleware-automatic-center-server/src/main/java/com/zhiwei/middleware/automatic/server/util/Tools.java
View file @
2fedede7
...
...
@@ -35,15 +35,22 @@ public class Tools {
* @return true:符合 false:不符合
*/
public
static
boolean
filterTag
(
List
<
String
>
tags
,
String
mtag
)
{
boolean
res
=
true
;
if
(
tags
!=
null
)
{
for
(
String
tag
:
tags
)
{
if
(
mtag
.
equals
(
Tools
.
sortTag
(
tag
)))
{
return
true
;
// if (mtag.equals(Tools.sortTag(tag))) {
// return true;
// }
if
(!
res
)
{
break
;
}
res
=
mtag
.
contains
(
tag
);
}
if
(
tags
.
isEmpty
()
&&
""
.
equals
(
mtag
))
{
return
true
;
}
return
tags
.
isEmpty
()
&&
""
.
equals
(
mtag
);
}
return
true
;
return
res
;
}
...
...
middleware-automatic-center-son/pom.xml
View file @
2fedede7
...
...
@@ -24,7 +24,7 @@
<qbjc-bean.version>
1.1.4.1-SNAPSHOT
</qbjc-bean.version>
<kafka.version>
2.4.1.RELEASE
</kafka.version>
<base.version>
2.0.0-SNAPSHOT
</base.version>
<automatic.version>
1.0.0.
2
-SNAPSHOT
</automatic.version>
<automatic.version>
1.0.0.
3
-SNAPSHOT
</automatic.version>
<marker.version>
1.2.3-SNAPSHOT
</marker.version>
<filter.version>
1.1.6-SNAPSHOT
</filter.version>
<nlp-aggree.version>
0.0.5-SNAPSHOT
</nlp-aggree.version>
...
...
middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/task/holder/TaskServiceHandler.java
View file @
2fedede7
package
com
.
zhiwei
.
middleware
.
automatic
.
son
.
task
.
holder
;
import
com.alibaba.fastjson.JSONObject
;
import
com.zhiwei.middleware.automatic.server.pojo.AutoTask
;
import
com.zhiwei.middleware.automatic.server.pojo.enums.TaskType
;
import
com.zhiwei.middleware.automatic.son.task.service.TaskService
;
...
...
middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/task/service/TaskServiceCommon.java
View file @
2fedede7
...
...
@@ -26,6 +26,8 @@ import org.springframework.beans.factory.annotation.Qualifier;
import
org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor
;
import
org.springframework.stereotype.Service
;
import
org.apache.commons.lang3.tuple.Pair
;
import
org.springframework.util.CollectionUtils
;
import
java.util.*
;
import
java.util.stream.Collectors
;
...
...
@@ -34,26 +36,22 @@ public class TaskServiceCommon extends BaseTaskTypePair<TaskServiceCommon.TaskCo
private
static
final
Logger
log
=
LogManager
.
getLogger
(
TaskServiceCommon
.
class
);
private
final
static
double
NUMBER_SIMILAR_STANDARD
=
0.29
;
private
final
RedissonUtil
redissonUtil
;
private
final
TemplateTitleService
templateTitleService
;
private
final
DubboHandler
dubboHandler
;
private
final
TemplateRecordDao
templateRecordDao
;
private
final
ThreadPoolTaskExecutor
autoMarkExecutor
;
private
final
static
String
COUNT_KEY
=
"count"
;
public
TaskServiceCommon
(
RedissonUtil
redissonUtil
,
TemplateTitleService
templateTitleService
,
DubboHandler
dubboHandler
,
TemplateRecordDao
templateRecordDao
,
@Qualifier
(
"autMarkExecutor"
)
ThreadPoolTaskExecutor
autoMarkExecutor
)
{
this
.
redissonUtil
=
redissonUtil
;
this
.
templateTitleService
=
templateTitleService
;
this
.
dubboHandler
=
dubboHandler
;
this
.
templateRecordDao
=
templateRecordDao
;
this
.
autoMarkExecutor
=
autoMarkExecutor
;
taskCache
(
TaskType
.
COMMON_TWO
,
this
::
getMultiAutoInfo
);
taskCache
(
TaskType
.
COMMON_ONE
,
this
::
getOneAutoInfo
);
...
...
@@ -154,7 +152,7 @@ public class TaskServiceCommon extends BaseTaskTypePair<TaskServiceCommon.TaskCo
* 项目自动标注
* @param groupMap 数据集 项目名分类
*/
p
rivate
void
projectAutoMark
(
Map
<
String
,
List
<
MarkInfo
>>
groupMap
)
{
p
ublic
void
projectAutoMark
(
Map
<
String
,
List
<
MarkInfo
>>
groupMap
)
{
for
(
Map
.
Entry
<
String
,
List
<
MarkInfo
>>
entry
:
groupMap
.
entrySet
())
{
try
{
Map
<
String
,
TemplateTitleVo
>
template
=
templateTitleService
.
getTemplateTitleByProjectLive
(
entry
.
getKey
());
...
...
@@ -263,9 +261,6 @@ public class TaskServiceCommon extends BaseTaskTypePair<TaskServiceCommon.TaskCo
// // 特征值记录
templateTitleService
.
insertTemplateRecord
(
new
TemplateRecord
(
templateTitleVo
.
getId
(),
title
,
id
,
getUrl
(
markInfo
),
getPlatform
(
markInfo
),
sourceObj
.
getString
(
"source"
),
sourceObj
.
getString
(
"real_source"
),
Tools
.
TIME_FORMAT
.
format
(
sourceObj
.
getLong
(
"time"
)),
updates
[
0
]));
// 测试环境临时添加,用于对比
// templateRecordDao.tempRecord(new TemplateTempRecord(templateTitleVo.getId(), templateTitleVo.getTemplateTitle(), group, getUrl(markInfo),
// id, templateTitleVo.getMtag()));
return
true
;
}
catch
(
Exception
e
)
{
log
.
error
(
"记录事件采集-标注数据特征值失败"
,
e
);
...
...
@@ -305,19 +300,41 @@ public class TaskServiceCommon extends BaseTaskTypePair<TaskServiceCommon.TaskCo
continue
;
}
String
aggreTitle
=
templateTitleVo
.
getTemplateTitle
();
// templateTitleVo.setTextNumbers(Tools.numberMatch(Tools.filterSymbol(aggreTitle)));
// 过滤掉标题里面的特殊符号
double
similar
=
CosineSimilarity
.
calculateTextSimWithBrand
(
aggreTitle
,
title
);
double
currentSimilar
=
similarMap
.
get
(
"similar"
)
!=
null
?
(
double
)
similarMap
.
get
(
"similar"
)
:
0.0
;
double
similarStandard
=
textNumberMatch
(
templateTitleVo
.
getTextNumbers
(),
title
)
?
GenericAttribute
.
SIMILAR_STANDARD
:
GenericAttribute
.
SIMILAR_STANDARD
-
NUMBER_SIMILAR_STANDARD
;
// 选取相似度最大的标注
if
(
similar
>=
GenericAttribute
.
SIMILAR_STANDARD
&&
similar
>
currentSimilar
)
{
if
(
similar
>=
similarStandard
&&
similar
>
currentSimilar
)
{
similarMap
.
put
(
"similar"
,
similar
);
similarMap
.
put
(
"aggreeTitle"
,
aggreTitle
);
}
if
(
similar
==
1.0
)
{
break
;
}
}
return
similarMap
;
}
public
static
MarkInfo
getMarkInfo
(
JSONObject
hit
)
{
private
boolean
textNumberMatch
(
List
<
String
>
textNumber
,
String
text
)
{
boolean
res
=
false
;
if
(
CollectionUtils
.
isEmpty
(
textNumber
))
{
return
res
;
}
List
<
String
>
numberMatch
=
Tools
.
numberMatch
(
text
);
if
(
CollectionUtils
.
isEmpty
(
numberMatch
)
||
numberMatch
.
size
()
!=
textNumber
.
size
())
{
return
res
;
}
for
(
String
number
:
textNumber
)
{
if
(!
numberMatch
.
contains
(
number
))
{
return
res
;
}
}
return
true
;
}
private
static
MarkInfo
getMarkInfo
(
JSONObject
hit
)
{
int
c2
=
Integer
.
parseInt
(
String
.
valueOf
(
hit
.
get
(
"c2"
)));
switch
(
ClassB
.
TypeB
.
fromEncode
(
c2
)){
case
COMPLETE:
...
...
middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/task/service/TaskServiceTemplate.java
View file @
2fedede7
...
...
@@ -43,6 +43,8 @@ public class TaskServiceTemplate extends BaseTaskTypePair<TaskServiceTemplate.Ta
private
final
Logger
log
=
LogManager
.
getLogger
(
TaskServiceTemplate
.
class
);
private
static
final
List
<
String
>
FILTER_TITLE
=
Arrays
.
asList
(
"Notitle"
);
private
final
TemplateTitleService
templateTitleService
;
private
final
EsDao
esDao
;
...
...
@@ -245,7 +247,11 @@ public class TaskServiceTemplate extends BaseTaskTypePair<TaskServiceTemplate.Ta
if
(
title
.
length
()
<
6
)
{
continue
;
}
TemplateTitleVo
templateTitleVo
=
new
TemplateTitleVo
(
title
,
tag
,
sourceList
.
get
(
result
.
getDataPoints
().
get
(
0
)).
getUrl
());
// 指定标题 不进行聚合
if
(
FILTER_TITLE
.
contains
(
title
))
{
continue
;
}
TemplateTitleVo
templateTitleVo
=
new
TemplateTitleVo
(
title
,
tag
,
sourceList
.
get
(
result
.
getDataPoints
().
get
(
0
)).
getUrl
(),
Tools
.
numberMatch
(
title
));
templateTitleVo
.
buildId
(
group
);
aggregationTitleTagMap
.
put
(
title
,
templateTitleVo
);
}
...
...
middleware-automatic-center-son/src/main/java/com/zhiwei/middleware/automatic/son/util/Tools.java
View file @
2fedede7
package
com
.
zhiwei
.
middleware
.
automatic
.
son
.
util
;
import
com.alibaba.fastjson.JSON
;
import
com.zhiwei.middleware.automatic.server.pojo.AutoTask
;
import
org.apache.commons.collections4.CollectionUtils
;
import
org.apache.commons.collections4.MapUtils
;
import
org.apache.commons.lang3.StringUtils
;
import
org.apache.commons.lang3.time.FastDateFormat
;
import
java.util.*
;
import
java.util.regex.Matcher
;
import
java.util.regex.Pattern
;
public
class
Tools
{
private
static
final
Pattern
SYMBOL_PATTERN
=
Pattern
.
compile
(
"[\\p{P}+~$`^=丨|<>~`$^+=|<>¥×\\s\u200B\u200C\u200D\u00A0\u0020\u3000]"
);
private
static
final
Pattern
NUMBER_PATTERN
=
Pattern
.
compile
(
"\\d+"
);
public
static
final
FastDateFormat
TIME_FORMAT
=
FastDateFormat
.
getInstance
(
"yyyy-MM-dd HH:mm:ss"
);
/**
* 是否为空,数据为空
...
...
@@ -116,4 +119,13 @@ public class Tools {
}
return
res
;
}
public
static
List
<
String
>
numberMatch
(
String
text
)
{
List
<
String
>
res
=
new
ArrayList
<>();
Matcher
matcher
=
NUMBER_PATTERN
.
matcher
(
text
);
while
(
matcher
.
find
())
{
res
.
add
(
matcher
.
group
());
}
return
res
;
}
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment