Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
M
messageflow
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
虞诚毅
messageflow
Commits
a95d6eb2
Commit
a95d6eb2
authored
May 10, 2018
by
shentao
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
2018/5/10 1.commonid增加,消息流调整2.关键词规则和噪音规则,调整适应
parent
35322d82
Hide whitespace changes
Inline
Side-by-side
Showing
14 changed files
with
364 additions
and
193 deletions
+364
-193
src/main/java/com/zhiwei/messageflow/bean/MediaMessage.java
+2
-0
src/main/java/com/zhiwei/messageflow/bean/VideoMessage.java
+2
-0
src/main/java/com/zhiwei/messageflow/bean/WeiboMessage.java
+1
-0
src/main/java/com/zhiwei/messageflow/bean/ZhihuMessage.java
+2
-0
src/main/java/com/zhiwei/messageflow/es/service/impl/NoiseProcessingServiceImpl.java
+315
-188
src/main/java/com/zhiwei/messageflow/mongo/bean/KeywordNew.java
+1
-0
src/main/java/com/zhiwei/messageflow/mongo/bean/NoiseRule.java
+3
-0
src/main/java/com/zhiwei/messageflow/mongo/dao/NoiseRuleDao.java
+8
-0
src/main/java/com/zhiwei/messageflow/mongo/dao/impl/KeywordNewDaoImpl.java
+1
-1
src/main/java/com/zhiwei/messageflow/mongo/dao/impl/NoiseRuleDaoImpl.java
+15
-1
src/main/java/com/zhiwei/messageflow/mongo/dao/impl/TrackRuleDaoImpl.java
+4
-3
src/main/resources/application.properties.local1
+0
-0
src/main/resources/redis.properties.local1
+0
-0
src/test/java/com/zhiwei/messageflow/MessageflowApplicationTests.java
+10
-0
No files found.
src/main/java/com/zhiwei/messageflow/bean/MediaMessage.java
View file @
a95d6eb2
...
@@ -33,4 +33,5 @@ public class MediaMessage {
...
@@ -33,4 +33,5 @@ public class MediaMessage {
private
String
rootSourcce
;
private
String
rootSourcce
;
private
Double
channelIndex
;
private
Double
channelIndex
;
private
String
crawlerV
;
private
String
crawlerV
;
private
Long
commonid
;
}
}
\ No newline at end of file
src/main/java/com/zhiwei/messageflow/bean/VideoMessage.java
View file @
a95d6eb2
...
@@ -21,4 +21,5 @@ public class VideoMessage {
...
@@ -21,4 +21,5 @@ public class VideoMessage {
private
String
time
;
private
String
time
;
private
String
uper
;
private
String
uper
;
private
String
type
;
private
String
type
;
private
Long
commonid
;
}
}
\ No newline at end of file
src/main/java/com/zhiwei/messageflow/bean/WeiboMessage.java
View file @
a95d6eb2
...
@@ -29,6 +29,7 @@ public class WeiboMessage {
...
@@ -29,6 +29,7 @@ public class WeiboMessage {
private
Long
rstime
;
private
Long
rstime
;
private
String
username
;
private
String
username
;
private
Integer
guanzhu
;
private
Integer
guanzhu
;
private
Long
commonid
;
private
String
rooturl
;
private
String
rooturl
;
private
String
roottext
;
private
String
roottext
;
...
...
src/main/java/com/zhiwei/messageflow/bean/ZhihuMessage.java
View file @
a95d6eb2
...
@@ -35,4 +35,5 @@ public class ZhihuMessage {
...
@@ -35,4 +35,5 @@ public class ZhihuMessage {
private
String
img
;
private
String
img
;
private
String
follow
;
private
String
follow
;
private
String
browse
;
private
String
browse
;
private
Long
commonid
;
}
}
\ No newline at end of file
src/main/java/com/zhiwei/messageflow/es/service/impl/NoiseProcessingServiceImpl.java
View file @
a95d6eb2
...
@@ -16,12 +16,16 @@ import com.zhiwei.messageflow.bean.ZhihuMessage;
...
@@ -16,12 +16,16 @@ import com.zhiwei.messageflow.bean.ZhihuMessage;
import
com.zhiwei.messageflow.es.service.HighLightFillingService
;
import
com.zhiwei.messageflow.es.service.HighLightFillingService
;
import
com.zhiwei.messageflow.es.service.NoiseProcessingService
;
import
com.zhiwei.messageflow.es.service.NoiseProcessingService
;
import
com.zhiwei.messageflow.mongo.bean.NoiseRule
;
import
com.zhiwei.messageflow.mongo.bean.NoiseRule
;
import
com.zhiwei.messageflow.mongo.dao.NoiseRuleDao
;
@Component
@Component
public
class
NoiseProcessingServiceImpl
implements
NoiseProcessingService
{
public
class
NoiseProcessingServiceImpl
implements
NoiseProcessingService
{
@Autowired
@Autowired
private
HighLightFillingService
highLightFillingService
;
private
HighLightFillingService
highLightFillingService
;
@Autowired
private
NoiseRuleDao
noiseRuleDao
;
@Override
@Override
public
List
<
WeiboMessage
>
weiboDenoising
(
List
<
NoiseRule
>
noiseRules
,
SearchHits
searchHits
,
String
platform
,
public
List
<
WeiboMessage
>
weiboDenoising
(
List
<
NoiseRule
>
noiseRules
,
SearchHits
searchHits
,
String
platform
,
...
@@ -49,52 +53,62 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
...
@@ -49,52 +53,62 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
/**
/**
* 关键词去噪
* 关键词去噪
*/
*/
String
kone
=
n
.
getKeyWordsInputOne
();
// if (n.getKwSearch().equals("标题")) {
//
String
ktwo
=
n
.
getKeyWordsInputTwo
();
// } else if (n.getKwSearch().equals("全文")) {
String
kone
=
n
.
getKeyWordsInputOne
();
if
(
n
.
getAndOr
().
equals
(
"且"
))
{
if
(
text
!=
null
&&
roottext
!=
null
String
ktwo
=
n
.
getKeyWordsInputTwo
();
&&
(
text
.
indexOf
(
kone
)
!=
-
1
||
roottext
.
indexOf
(
kone
)
!=
-
1
)
&&
(
text
.
indexOf
(
ktwo
)
!=
-
1
||
roottext
.
indexOf
(
ktwo
)
!=
-
1
))
{
if
(
n
.
getAndOr
().
equals
(
"且"
))
{
isnoise
=
true
;
if
(
text
!=
null
&&
roottext
!=
null
break
;
&&
(
text
.
indexOf
(
kone
)
!=
-
1
||
roottext
.
indexOf
(
kone
)
!=
-
1
)
}
else
if
(
text
!=
null
&&
roottext
==
null
&&
(
text
.
indexOf
(
kone
)
!=
-
1
)
&&
(
text
.
indexOf
(
ktwo
)
!=
-
1
||
roottext
.
indexOf
(
ktwo
)
!=
-
1
))
{
&&
(
text
.
indexOf
(
ktwo
)
!=
-
1
))
{
isnoise
=
true
;
isnoise
=
true
;
noiseRuleDao
.
addEffectNum
(
n
);
break
;
break
;
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
}
else
if
(
text
!=
null
&&
roottext
==
null
&&
(
text
.
indexOf
(
kone
)
!=
-
1
)
isnoise
=
true
;
&&
(
text
.
indexOf
(
ktwo
)
!=
-
1
))
{
break
;
isnoise
=
true
;
}
noiseRuleDao
.
addEffectNum
(
n
);
}
else
if
(
n
.
getAndOr
().
equals
(
"或"
))
{
break
;
if
(
text
!=
null
&&
roottext
!=
null
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
&&
((
text
.
indexOf
(
kone
)
!=
-
1
||
roottext
.
indexOf
(
kone
)
!=
-
1
)
isnoise
=
true
;
||
(
text
.
indexOf
(
ktwo
)
!=
-
1
||
roottext
.
indexOf
(
ktwo
)
!=
-
1
)))
{
break
;
}
isnoise
=
true
;
}
else
if
(
n
.
getAndOr
().
equals
(
"或"
))
{
break
;
if
(
text
!=
null
&&
roottext
!=
null
}
else
if
(
text
!=
null
&&
roottext
==
null
&&
((
text
.
indexOf
(
kone
)
!=
-
1
||
roottext
.
indexOf
(
kone
)
!=
-
1
)
&&
((
text
.
indexOf
(
kone
)
!=
-
1
)
||
(
text
.
indexOf
(
ktwo
)
!=
-
1
)))
{
||
(
text
.
indexOf
(
ktwo
)
!=
-
1
||
roottext
.
indexOf
(
ktwo
)
!=
-
1
)))
{
isnoise
=
true
;
break
;
isnoise
=
true
;
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
noiseRuleDao
.
addEffectNum
(
n
);
isnoise
=
true
;
break
;
break
;
}
else
if
(
text
!=
null
&&
roottext
==
null
}
&&
((
text
.
indexOf
(
kone
)
!=
-
1
)
||
(
text
.
indexOf
(
ktwo
)
!=
-
1
)))
{
}
else
{
isnoise
=
true
;
if
(
text
!=
null
&&
roottext
!=
null
noiseRuleDao
.
addEffectNum
(
n
);
&&
(
text
.
indexOf
(
kone
)
!=
-
1
||
roottext
.
indexOf
(
kone
)
!=
-
1
))
{
break
;
isnoise
=
true
;
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
break
;
isnoise
=
true
;
}
else
if
(
text
!=
null
&&
roottext
==
null
&&
(
text
.
indexOf
(
kone
)
!=
-
1
))
{
break
;
isnoise
=
true
;
}
break
;
}
else
{
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
if
(
text
!=
null
&&
roottext
!=
null
isnoise
=
true
;
&&
(
text
.
indexOf
(
kone
)
!=
-
1
||
roottext
.
indexOf
(
kone
)
!=
-
1
))
{
break
;
isnoise
=
true
;
noiseRuleDao
.
addEffectNum
(
n
);
break
;
}
else
if
(
text
!=
null
&&
roottext
==
null
&&
(
text
.
indexOf
(
kone
)
!=
-
1
))
{
isnoise
=
true
;
noiseRuleDao
.
addEffectNum
(
n
);
break
;
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
isnoise
=
true
;
break
;
}
}
}
}
//
}
}
else
if
((
n
.
getRuleType
().
equals
(
"channel"
))
}
else
if
((
n
.
getRuleType
().
equals
(
"channel"
))
&&
((
n
.
getChannelPt
().
equals
(
"微博"
)
||
n
.
getChannelPt
().
equals
(
"全部"
))))
{
&&
((
n
.
getChannelPt
().
equals
(
"微博"
)
||
n
.
getChannelPt
().
equals
(
"全部"
))))
{
/**
/**
...
@@ -106,6 +120,7 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
...
@@ -106,6 +120,7 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
source
=
source
.
replaceAll
(
" "
,
""
);
source
=
source
.
replaceAll
(
" "
,
""
);
if
(
channelQd
.
equals
(
source
))
{
if
(
channelQd
.
equals
(
source
))
{
isnoise
=
true
;
isnoise
=
true
;
noiseRuleDao
.
addEffectNum
(
n
);
break
;
break
;
}
}
}
}
...
@@ -154,56 +169,91 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
...
@@ -154,56 +169,91 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
* 关键词去噪
* 关键词去噪
*/
*/
// searchHit.getSource().get("text").toString().indexOf(str);
// searchHit.getSource().get("text").toString().indexOf(str);
String
kone
=
n
.
getKeyWordsInputOne
();
if
(
n
.
getKwSearch
().
equals
(
"标题"
))
{
String
ktwo
=
n
.
getKeyWordsInputTwo
();
String
kone
=
n
.
getKeyWordsInputOne
();
if
(
n
.
getAndOr
().
equals
(
"且"
))
{
if
(
text
!=
null
&&
roottext
!=
null
String
ktwo
=
n
.
getKeyWordsInputTwo
();
&&
(
text
.
indexOf
(
kone
)
!=
-
1
||
roottext
.
indexOf
(
kone
)
!=
-
1
)
&&
(
text
.
indexOf
(
ktwo
)
!=
-
1
||
roottext
.
indexOf
(
ktwo
)
!=
-
1
))
{
if
(
n
.
getAndOr
().
equals
(
"且"
))
{
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
if
(
text
!=
null
&&
(
text
.
indexOf
(
kone
)
!=
-
1
)
&&
(
text
.
indexOf
(
ktwo
)
!=
-
1
))
{
isnoise
=
true
;
isnoise
=
true
;
break
;
noiseRuleDao
.
addEffectNum
(
n
);
}
else
if
(
text
!=
null
&&
roottext
==
null
&&
(
text
.
indexOf
(
kone
)
!=
-
1
)
break
;
&&
(
text
.
indexOf
(
ktwo
)
!=
-
1
))
{
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise
=
true
;
isnoise
=
true
;
break
;
break
;
}
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
}
else
if
(
n
.
getAndOr
().
equals
(
"或"
))
{
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
if
(
text
!=
null
&&
((
text
.
indexOf
(
kone
)
!=
-
1
)
||
(
text
.
indexOf
(
ktwo
)
!=
-
1
)))
{
isnoise
=
true
;
isnoise
=
true
;
break
;
noiseRuleDao
.
addEffectNum
(
n
);
}
break
;
}
else
if
(
n
.
getAndOr
().
equals
(
"或"
))
{
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
if
(
text
!=
null
&&
roottext
!=
null
isnoise
=
true
;
&&
((
text
.
indexOf
(
kone
)
!=
-
1
||
roottext
.
indexOf
(
kone
)
!=
-
1
)
break
;
||
(
text
.
indexOf
(
ktwo
)
!=
-
1
||
roottext
.
indexOf
(
ktwo
)
!=
-
1
)))
{
}
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
}
else
{
isnoise
=
true
;
if
(
text
!=
null
&&
(
text
.
indexOf
(
kone
)
!=
-
1
))
{
break
;
isnoise
=
true
;
}
else
if
(
text
!=
null
&&
roottext
==
null
noiseRuleDao
.
addEffectNum
(
n
);
&&
((
text
.
indexOf
(
kone
)
!=
-
1
)
||
(
text
.
indexOf
(
ktwo
)
!=
-
1
)))
{
break
;
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
isnoise
=
true
;
isnoise
=
true
;
break
;
break
;
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
}
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise
=
true
;
break
;
}
}
}
else
{
}
else
if
(
n
.
getKwSearch
().
equals
(
"全文"
))
{
if
(
text
!=
null
&&
roottext
!=
null
String
kone
=
n
.
getKeyWordsInputOne
();
&&
(
text
.
indexOf
(
kone
)
!=
-
1
||
roottext
.
indexOf
(
kone
)
!=
-
1
))
{
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
String
ktwo
=
n
.
getKeyWordsInputTwo
();
isnoise
=
true
;
break
;
if
(
n
.
getAndOr
().
equals
(
"且"
))
{
}
else
if
(
text
!=
null
&&
roottext
==
null
&&
(
text
.
indexOf
(
kone
)
!=
-
1
))
{
if
(
text
!=
null
&&
roottext
!=
null
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
&&
(
text
.
indexOf
(
kone
)
!=
-
1
||
roottext
.
indexOf
(
kone
)
!=
-
1
)
isnoise
=
true
;
&&
(
text
.
indexOf
(
ktwo
)
!=
-
1
||
roottext
.
indexOf
(
ktwo
)
!=
-
1
))
{
break
;
isnoise
=
true
;
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
noiseRuleDao
.
addEffectNum
(
n
);
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
break
;
isnoise
=
true
;
}
else
if
(
text
!=
null
&&
roottext
==
null
&&
(
text
.
indexOf
(
kone
)
!=
-
1
)
break
;
&&
(
text
.
indexOf
(
ktwo
)
!=
-
1
))
{
isnoise
=
true
;
noiseRuleDao
.
addEffectNum
(
n
);
break
;
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
isnoise
=
true
;
break
;
}
}
else
if
(
n
.
getAndOr
().
equals
(
"或"
))
{
if
(
text
!=
null
&&
roottext
!=
null
&&
((
text
.
indexOf
(
kone
)
!=
-
1
||
roottext
.
indexOf
(
kone
)
!=
-
1
)
||
(
text
.
indexOf
(
ktwo
)
!=
-
1
||
roottext
.
indexOf
(
ktwo
)
!=
-
1
)))
{
isnoise
=
true
;
noiseRuleDao
.
addEffectNum
(
n
);
break
;
}
else
if
(
text
!=
null
&&
roottext
==
null
&&
((
text
.
indexOf
(
kone
)
!=
-
1
)
||
(
text
.
indexOf
(
ktwo
)
!=
-
1
)))
{
isnoise
=
true
;
noiseRuleDao
.
addEffectNum
(
n
);
break
;
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
isnoise
=
true
;
break
;
}
}
else
{
if
(
text
!=
null
&&
roottext
!=
null
&&
(
text
.
indexOf
(
kone
)
!=
-
1
||
roottext
.
indexOf
(
kone
)
!=
-
1
))
{
isnoise
=
true
;
noiseRuleDao
.
addEffectNum
(
n
);
break
;
}
else
if
(
text
!=
null
&&
roottext
==
null
&&
(
text
.
indexOf
(
kone
)
!=
-
1
))
{
isnoise
=
true
;
noiseRuleDao
.
addEffectNum
(
n
);
break
;
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
isnoise
=
true
;
break
;
}
}
}
}
}
}
else
if
((
n
.
getRuleType
().
equals
(
"channel"
))
}
else
if
((
n
.
getRuleType
().
equals
(
"channel"
))
...
@@ -217,6 +267,7 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
...
@@ -217,6 +267,7 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
source
=
source
.
replaceAll
(
" "
,
""
);
source
=
source
.
replaceAll
(
" "
,
""
);
if
(
channelQd
.
equals
(
source
))
{
if
(
channelQd
.
equals
(
source
))
{
isnoise
=
true
;
isnoise
=
true
;
noiseRuleDao
.
addEffectNum
(
n
);
break
;
break
;
}
}
}
}
...
@@ -263,51 +314,91 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
...
@@ -263,51 +314,91 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
/**
/**
* 关键词去噪
* 关键词去噪
*/
*/
// searchHit.getSource().get("text").toString().indexOf(str);
if
(
n
.
getKwSearch
().
equals
(
"标题"
))
{
String
kone
=
n
.
getKeyWordsInputOne
();
String
kone
=
n
.
getKeyWordsInputOne
();
String
ktwo
=
n
.
getKeyWordsInputTwo
();
if
(
n
.
getAndOr
().
equals
(
"且"
))
{
String
ktwo
=
n
.
getKeyWordsInputTwo
();
if
(
text
!=
null
&&
roottext
==
null
&&
(
text
.
indexOf
(
kone
)
!=
-
1
)
&&
(
text
.
indexOf
(
ktwo
)
!=
-
1
))
{
if
(
n
.
getAndOr
().
equals
(
"且"
))
{
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
if
(
text
!=
null
&&
(
text
.
indexOf
(
kone
)
!=
-
1
)
&&
(
text
.
indexOf
(
ktwo
)
!=
-
1
))
{
isnoise
=
true
;
isnoise
=
true
;
break
;
noiseRuleDao
.
addEffectNum
(
n
);
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
break
;
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
isnoise
=
true
;
isnoise
=
true
;
break
;
break
;
}
}
}
else
if
(
n
.
getAndOr
().
equals
(
"或"
))
{
}
else
if
(
n
.
getAndOr
().
equals
(
"或"
))
{
if
(
text
!=
null
&&
roottext
!=
null
if
(
text
!=
null
&&
((
text
.
indexOf
(
kone
)
!=
-
1
)
||
(
text
.
indexOf
(
ktwo
)
!=
-
1
)))
{
&&
((
text
.
indexOf
(
kone
)
!=
-
1
||
roottext
.
indexOf
(
kone
)
!=
-
1
)
isnoise
=
true
;
||
(
text
.
indexOf
(
ktwo
)
!=
-
1
||
roottext
.
indexOf
(
ktwo
)
!=
-
1
)))
{
noiseRuleDao
.
addEffectNum
(
n
);
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
break
;
isnoise
=
true
;
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
break
;
isnoise
=
true
;
}
else
if
(
text
!=
null
&&
roottext
==
null
break
;
&&
((
text
.
indexOf
(
kone
)
!=
-
1
)
||
(
text
.
indexOf
(
ktwo
)
!=
-
1
)))
{
}
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
}
else
{
isnoise
=
true
;
if
(
text
!=
null
&&
(
text
.
indexOf
(
kone
)
!=
-
1
))
{
break
;
isnoise
=
true
;
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
noiseRuleDao
.
addEffectNum
(
n
);
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
break
;
isnoise
=
true
;
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
break
;
isnoise
=
true
;
break
;
}
}
}
}
else
{
}
else
if
(
n
.
getKwSearch
().
equals
(
"全文"
))
{
if
(
text
!=
null
&&
roottext
!=
null
String
kone
=
n
.
getKeyWordsInputOne
();
&&
(
text
.
indexOf
(
kone
)
!=
-
1
||
roottext
.
indexOf
(
kone
)
!=
-
1
))
{
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
String
ktwo
=
n
.
getKeyWordsInputTwo
();
isnoise
=
true
;
break
;
if
(
n
.
getAndOr
().
equals
(
"且"
))
{
}
else
if
(
text
!=
null
&&
roottext
==
null
&&
(
text
.
indexOf
(
kone
)
!=
-
1
))
{
if
(
text
!=
null
&&
roottext
!=
null
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
&&
(
text
.
indexOf
(
kone
)
!=
-
1
||
roottext
.
indexOf
(
kone
)
!=
-
1
)
isnoise
=
true
;
&&
(
text
.
indexOf
(
ktwo
)
!=
-
1
||
roottext
.
indexOf
(
ktwo
)
!=
-
1
))
{
break
;
isnoise
=
true
;
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
noiseRuleDao
.
addEffectNum
(
n
);
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
break
;
isnoise
=
true
;
}
else
if
(
text
!=
null
&&
roottext
==
null
&&
(
text
.
indexOf
(
kone
)
!=
-
1
)
break
;
&&
(
text
.
indexOf
(
ktwo
)
!=
-
1
))
{
isnoise
=
true
;
noiseRuleDao
.
addEffectNum
(
n
);
break
;
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
isnoise
=
true
;
break
;
}
}
else
if
(
n
.
getAndOr
().
equals
(
"或"
))
{
if
(
text
!=
null
&&
roottext
!=
null
&&
((
text
.
indexOf
(
kone
)
!=
-
1
||
roottext
.
indexOf
(
kone
)
!=
-
1
)
||
(
text
.
indexOf
(
ktwo
)
!=
-
1
||
roottext
.
indexOf
(
ktwo
)
!=
-
1
)))
{
isnoise
=
true
;
noiseRuleDao
.
addEffectNum
(
n
);
break
;
}
else
if
(
text
!=
null
&&
roottext
==
null
&&
((
text
.
indexOf
(
kone
)
!=
-
1
)
||
(
text
.
indexOf
(
ktwo
)
!=
-
1
)))
{
isnoise
=
true
;
noiseRuleDao
.
addEffectNum
(
n
);
break
;
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
isnoise
=
true
;
break
;
}
}
else
{
if
(
text
!=
null
&&
roottext
!=
null
&&
(
text
.
indexOf
(
kone
)
!=
-
1
||
roottext
.
indexOf
(
kone
)
!=
-
1
))
{
isnoise
=
true
;
noiseRuleDao
.
addEffectNum
(
n
);
break
;
}
else
if
(
text
!=
null
&&
roottext
==
null
&&
(
text
.
indexOf
(
kone
)
!=
-
1
))
{
isnoise
=
true
;
noiseRuleDao
.
addEffectNum
(
n
);
break
;
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
isnoise
=
true
;
break
;
}
}
}
}
}
}
else
if
((
n
.
getRuleType
().
equals
(
"channel"
))
}
else
if
((
n
.
getRuleType
().
equals
(
"channel"
))
...
@@ -321,6 +412,7 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
...
@@ -321,6 +412,7 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
source
=
source
.
replaceAll
(
" "
,
""
);
source
=
source
.
replaceAll
(
" "
,
""
);
if
(
channelQd
.
equals
(
source
))
{
if
(
channelQd
.
equals
(
source
))
{
isnoise
=
true
;
isnoise
=
true
;
noiseRuleDao
.
addEffectNum
(
n
);
break
;
break
;
}
}
}
}
...
@@ -367,57 +459,91 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
...
@@ -367,57 +459,91 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
/**
/**
* 关键词去噪
* 关键词去噪
*/
*/
// searchHit.getSource().get("text").toString().indexOf(str);
if
(
n
.
getKwSearch
().
equals
(
"标题"
))
{
String
kone
=
n
.
getKeyWordsInputOne
();
String
kone
=
n
.
getKeyWordsInputOne
();
String
ktwo
=
n
.
getKeyWordsInputTwo
();
if
(
n
.
getAndOr
().
equals
(
"且"
))
{
String
ktwo
=
n
.
getKeyWordsInputTwo
();
if
(
text
!=
null
&&
roottext
!=
null
&&
(
text
.
indexOf
(
kone
)
!=
-
1
||
roottext
.
indexOf
(
kone
)
!=
-
1
)
if
(
n
.
getAndOr
().
equals
(
"且"
))
{
&&
(
text
.
indexOf
(
ktwo
)
!=
-
1
||
roottext
.
indexOf
(
ktwo
)
!=
-
1
))
{
if
(
text
!=
null
&&
(
text
.
indexOf
(
kone
)
!=
-
1
)
&&
(
text
.
indexOf
(
ktwo
)
!=
-
1
))
{
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise
=
true
;
isnoise
=
true
;
noiseRuleDao
.
addEffectNum
(
n
);
break
;
break
;
}
else
if
(
text
!=
null
&&
roottext
==
null
&&
(
text
.
indexOf
(
kone
)
!=
-
1
)
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
&&
(
text
.
indexOf
(
ktwo
)
!=
-
1
))
{
isnoise
=
true
;
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
break
;
isnoise
=
true
;
}
break
;
}
else
if
(
n
.
getAndOr
().
equals
(
"或"
))
{
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
if
(
text
!=
null
&&
((
text
.
indexOf
(
kone
)
!=
-
1
)
||
(
text
.
indexOf
(
ktwo
)
!=
-
1
)))
{
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise
=
true
;
isnoise
=
true
;
noiseRuleDao
.
addEffectNum
(
n
);
break
;
break
;
}
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
}
else
if
(
n
.
getAndOr
().
equals
(
"或"
))
{
isnoise
=
true
;
if
(
text
!=
null
&&
roottext
!=
null
break
;
&&
((
text
.
indexOf
(
kone
)
!=
-
1
||
roottext
.
indexOf
(
kone
)
!=
-
1
)
}
||
(
text
.
indexOf
(
ktwo
)
!=
-
1
||
roottext
.
indexOf
(
ktwo
)
!=
-
1
)))
{
}
else
{
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
if
(
text
!=
null
&&
(
text
.
indexOf
(
kone
)
!=
-
1
))
{
isnoise
=
true
;
isnoise
=
true
;
break
;
noiseRuleDao
.
addEffectNum
(
n
);
}
else
if
(
text
!=
null
&&
roottext
==
null
break
;
&&
((
text
.
indexOf
(
kone
)
!=
-
1
)
||
(
text
.
indexOf
(
ktwo
)
!=
-
1
)))
{
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise
=
true
;
isnoise
=
true
;
break
;
break
;
}
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
isnoise
=
true
;
break
;
}
}
}
else
{
}
else
if
(
n
.
getKwSearch
().
equals
(
"全文"
))
{
if
(
text
!=
null
&&
roottext
!=
null
String
kone
=
n
.
getKeyWordsInputOne
();
&&
(
text
.
indexOf
(
kone
)
!=
-
1
||
roottext
.
indexOf
(
kone
)
!=
-
1
))
{
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
String
ktwo
=
n
.
getKeyWordsInputTwo
();
isnoise
=
true
;
break
;
if
(
n
.
getAndOr
().
equals
(
"且"
))
{
}
else
if
(
text
!=
null
&&
roottext
==
null
&&
(
text
.
indexOf
(
kone
)
!=
-
1
))
{
if
(
text
!=
null
&&
roottext
!=
null
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
&&
(
text
.
indexOf
(
kone
)
!=
-
1
||
roottext
.
indexOf
(
kone
)
!=
-
1
)
isnoise
=
true
;
&&
(
text
.
indexOf
(
ktwo
)
!=
-
1
||
roottext
.
indexOf
(
ktwo
)
!=
-
1
))
{
break
;
isnoise
=
true
;
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
noiseRuleDao
.
addEffectNum
(
n
);
// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
break
;
isnoise
=
true
;
}
else
if
(
text
!=
null
&&
roottext
==
null
&&
(
text
.
indexOf
(
kone
)
!=
-
1
)
break
;
&&
(
text
.
indexOf
(
ktwo
)
!=
-
1
))
{
isnoise
=
true
;
noiseRuleDao
.
addEffectNum
(
n
);
break
;
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
isnoise
=
true
;
break
;
}
}
else
if
(
n
.
getAndOr
().
equals
(
"或"
))
{
if
(
text
!=
null
&&
roottext
!=
null
&&
((
text
.
indexOf
(
kone
)
!=
-
1
||
roottext
.
indexOf
(
kone
)
!=
-
1
)
||
(
text
.
indexOf
(
ktwo
)
!=
-
1
||
roottext
.
indexOf
(
ktwo
)
!=
-
1
)))
{
isnoise
=
true
;
noiseRuleDao
.
addEffectNum
(
n
);
break
;
}
else
if
(
text
!=
null
&&
roottext
==
null
&&
((
text
.
indexOf
(
kone
)
!=
-
1
)
||
(
text
.
indexOf
(
ktwo
)
!=
-
1
)))
{
isnoise
=
true
;
noiseRuleDao
.
addEffectNum
(
n
);
break
;
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
isnoise
=
true
;
break
;
}
}
else
{
if
(
text
!=
null
&&
roottext
!=
null
&&
(
text
.
indexOf
(
kone
)
!=
-
1
||
roottext
.
indexOf
(
kone
)
!=
-
1
))
{
isnoise
=
true
;
noiseRuleDao
.
addEffectNum
(
n
);
break
;
}
else
if
(
text
!=
null
&&
roottext
==
null
&&
(
text
.
indexOf
(
kone
)
!=
-
1
))
{
isnoise
=
true
;
noiseRuleDao
.
addEffectNum
(
n
);
break
;
}
else
if
(
text
==
null
&&
roottext
==
null
)
{
isnoise
=
true
;
break
;
}
}
}
}
}
}
else
if
((
n
.
getRuleType
().
equals
(
"channel"
))
}
else
if
((
n
.
getRuleType
().
equals
(
"channel"
))
...
@@ -431,6 +557,7 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
...
@@ -431,6 +557,7 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
source
=
source
.
replaceAll
(
" "
,
""
);
source
=
source
.
replaceAll
(
" "
,
""
);
if
(
channelQd
.
equals
(
source
))
{
if
(
channelQd
.
equals
(
source
))
{
isnoise
=
true
;
isnoise
=
true
;
noiseRuleDao
.
addEffectNum
(
n
);
break
;
break
;
}
}
}
}
...
...
src/main/java/com/zhiwei/messageflow/mongo/bean/KeywordNew.java
View file @
a95d6eb2
...
@@ -20,4 +20,5 @@ public class KeywordNew {
...
@@ -20,4 +20,5 @@ public class KeywordNew {
private
String
project
;
// 项目
private
String
project
;
// 项目
private
String
submitter
;
private
String
submitter
;
private
long
createAt
;
private
long
createAt
;
private
Boolean
isUsed
;
}
}
src/main/java/com/zhiwei/messageflow/mongo/bean/NoiseRule.java
View file @
a95d6eb2
...
@@ -24,4 +24,7 @@ public class NoiseRule {
...
@@ -24,4 +24,7 @@ public class NoiseRule {
private
String
submitter
;
private
String
submitter
;
private
String
channelPt
;
private
String
channelPt
;
private
String
channelQd
;
private
String
channelQd
;
private
String
kwSearch
;
private
Boolean
isUsed
;
private
Long
effectNum
;
}
}
src/main/java/com/zhiwei/messageflow/mongo/dao/NoiseRuleDao.java
View file @
a95d6eb2
...
@@ -12,4 +12,12 @@ public interface NoiseRuleDao {
...
@@ -12,4 +12,12 @@ public interface NoiseRuleDao {
* @return
* @return
*/
*/
List
<
NoiseRule
>
getNoiseRuleByProject
(
String
projectName
);
List
<
NoiseRule
>
getNoiseRuleByProject
(
String
projectName
);
/**
* 噪音规则加
* @Title: addEffectNum
* @Description: TODO(这里用一句话描述这个方法的作用)
* @param @param n 设定文件
* @return void 返回类型
*/
void
addEffectNum
(
NoiseRule
n
);
}
}
src/main/java/com/zhiwei/messageflow/mongo/dao/impl/KeywordNewDaoImpl.java
View file @
a95d6eb2
...
@@ -23,7 +23,7 @@ public class KeywordNewDaoImpl implements KeywordNewDao {
...
@@ -23,7 +23,7 @@ public class KeywordNewDaoImpl implements KeywordNewDao {
public
List
<
KeywordNew
>
getKeywordNewByProject
(
String
projectName
)
{
public
List
<
KeywordNew
>
getKeywordNewByProject
(
String
projectName
)
{
Query
query
=
new
Query
();
Query
query
=
new
Query
();
query
.
addCriteria
(
Criteria
.
where
(
"project"
).
is
(
projectName
));
query
.
addCriteria
(
Criteria
.
where
(
"project"
).
is
(
projectName
));
query
.
addCriteria
(
Criteria
.
where
(
"isUsed"
).
is
(
true
));
return
primaryMongoTemplate
.
find
(
query
,
KeywordNew
.
class
);
return
primaryMongoTemplate
.
find
(
query
,
KeywordNew
.
class
);
}
}
...
...
src/main/java/com/zhiwei/messageflow/mongo/dao/impl/NoiseRuleDaoImpl.java
View file @
a95d6eb2
...
@@ -7,6 +7,7 @@ import org.springframework.beans.factory.annotation.Qualifier;
...
@@ -7,6 +7,7 @@ import org.springframework.beans.factory.annotation.Qualifier;
import
org.springframework.data.mongodb.core.MongoTemplate
;
import
org.springframework.data.mongodb.core.MongoTemplate
;
import
org.springframework.data.mongodb.core.query.Criteria
;
import
org.springframework.data.mongodb.core.query.Criteria
;
import
org.springframework.data.mongodb.core.query.Query
;
import
org.springframework.data.mongodb.core.query.Query
;
import
org.springframework.data.mongodb.core.query.Update
;
import
org.springframework.stereotype.Component
;
import
org.springframework.stereotype.Component
;
import
com.zhiwei.messageflow.mongo.bean.NoiseRule
;
import
com.zhiwei.messageflow.mongo.bean.NoiseRule
;
...
@@ -23,8 +24,21 @@ public class NoiseRuleDaoImpl implements NoiseRuleDao {
...
@@ -23,8 +24,21 @@ public class NoiseRuleDaoImpl implements NoiseRuleDao {
public
List
<
NoiseRule
>
getNoiseRuleByProject
(
String
projectName
)
{
public
List
<
NoiseRule
>
getNoiseRuleByProject
(
String
projectName
)
{
Query
query
=
new
Query
();
Query
query
=
new
Query
();
query
.
addCriteria
(
Criteria
.
where
(
"project"
).
is
(
projectName
));
query
.
addCriteria
(
Criteria
.
where
(
"project"
).
is
(
projectName
));
query
.
addCriteria
(
Criteria
.
where
(
"isUsed"
).
is
(
true
));
return
primaryMongoTemplate
.
find
(
query
,
NoiseRule
.
class
);
return
primaryMongoTemplate
.
find
(
query
,
NoiseRule
.
class
);
}
}
@Override
public
void
addEffectNum
(
NoiseRule
n
)
{
Query
query
=
new
Query
();
query
.
addCriteria
(
Criteria
.
where
(
"_id"
).
is
(
n
.
get_id
()));
query
.
addCriteria
(
Criteria
.
where
(
"isUsed"
).
is
(
true
));
n
=
primaryMongoTemplate
.
findOne
(
query
,
NoiseRule
.
class
);
if
(
null
!=
n
)
{
query
=
Query
.
query
(
Criteria
.
where
(
"_id"
).
is
(
n
.
get_id
()));
Update
update
=
Update
.
update
(
"isUsed"
,
true
).
inc
(
"effectNum"
,
1
);
primaryMongoTemplate
.
updateMulti
(
query
,
update
,
NoiseRule
.
class
);
}
}
}
}
src/main/java/com/zhiwei/messageflow/mongo/dao/impl/TrackRuleDaoImpl.java
View file @
a95d6eb2
...
@@ -7,6 +7,7 @@ import org.springframework.beans.factory.annotation.Qualifier;
...
@@ -7,6 +7,7 @@ import org.springframework.beans.factory.annotation.Qualifier;
import
org.springframework.data.mongodb.core.MongoTemplate
;
import
org.springframework.data.mongodb.core.MongoTemplate
;
import
org.springframework.data.mongodb.core.query.Criteria
;
import
org.springframework.data.mongodb.core.query.Criteria
;
import
org.springframework.data.mongodb.core.query.Query
;
import
org.springframework.data.mongodb.core.query.Query
;
import
org.springframework.data.mongodb.core.query.Update
;
import
org.springframework.stereotype.Component
;
import
org.springframework.stereotype.Component
;
import
com.zhiwei.messageflow.mongo.bean.TrackRule
;
import
com.zhiwei.messageflow.mongo.bean.TrackRule
;
...
@@ -30,9 +31,9 @@ public class TrackRuleDaoImpl implements TrackRuleDao {
...
@@ -30,9 +31,9 @@ public class TrackRuleDaoImpl implements TrackRuleDao {
@Override
@Override
public
void
updateTrackrule
(
Long
trackRule_id
)
{
public
void
updateTrackrule
(
Long
trackRule_id
)
{
System
.
out
.
println
(
"预警状态更新"
);
//
System.out.println("预警状态更新");
//
primaryMongoTemplate.updateFirst(Query.query(Criteria.where("_id").is(trackRule_id)),
primaryMongoTemplate
.
updateFirst
(
Query
.
query
(
Criteria
.
where
(
"_id"
).
is
(
trackRule_id
)),
//
new Update().set("isWarn", true), TrackRule.class);
new
Update
().
set
(
"isWarn"
,
true
),
TrackRule
.
class
);
}
}
}
}
src/main/resources/application.properties.local
→
src/main/resources/application.properties.local
1
View file @
a95d6eb2
File moved
src/main/resources/redis.properties.local
→
src/main/resources/redis.properties.local
1
View file @
a95d6eb2
File moved
src/test/java/com/zhiwei/messageflow/MessageflowApplicationTests.java
View file @
a95d6eb2
...
@@ -2,15 +2,25 @@ package com.zhiwei.messageflow;
...
@@ -2,15 +2,25 @@ package com.zhiwei.messageflow;
import
org.junit.Test
;
import
org.junit.Test
;
import
org.junit.runner.RunWith
;
import
org.junit.runner.RunWith
;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.boot.test.context.SpringBootTest
;
import
org.springframework.boot.test.context.SpringBootTest
;
import
org.springframework.test.context.junit4.SpringRunner
;
import
org.springframework.test.context.junit4.SpringRunner
;
import
com.zhiwei.messageflow.mongo.bean.NoiseRule
;
import
com.zhiwei.messageflow.mongo.dao.NoiseRuleDao
;
@RunWith
(
SpringRunner
.
class
)
@RunWith
(
SpringRunner
.
class
)
@SpringBootTest
@SpringBootTest
public
class
MessageflowApplicationTests
{
public
class
MessageflowApplicationTests
{
@Autowired
private
NoiseRuleDao
noiseRuleDao
;
@Test
@Test
public
void
contextLoads
()
{
public
void
contextLoads
()
{
NoiseRule
n
=
new
NoiseRule
();
n
.
set_id
(
37
);
noiseRuleDao
.
addEffectNum
(
n
);
}
}
}
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment