Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
D
datamanage
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
shenjinzhu
datamanage
Commits
1f779ff3
Commit
1f779ff3
authored
May 18, 2018
by
admin
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
修改了检测的规则,并且把其他媒体加上了预警
parent
103ea399
Expand all
Show whitespace changes
Inline
Side-by-side
Showing
8 changed files
with
25 additions
and
12 deletions
+25
-12
.gitignore
+2
-0
log/error.log
+0
-0
src/main/java/com/zhiwei/manage/bean/NewsDelayed.java
+3
-1
src/main/java/com/zhiwei/manage/dao/MessageDaoImpl.java
+2
-2
src/main/java/com/zhiwei/manage/handle/MainThread.java
+9
-2
src/main/java/com/zhiwei/manage/handle/Task.java
+0
-0
src/main/java/com/zhiwei/manage/service/NewsServiceImpl.java
+1
-2
src/main/java/com/zhiwei/manage/util/ESUtil.java
+8
-5
No files found.
.gitignore
View file @
1f779ff3
...
...
@@ -6,3 +6,4 @@
.svn
.settings/
target/
/Jenkinsfile/
\ No newline at end of file
log/error.log
View file @
1f779ff3
This diff is collapsed.
Click to expand it.
src/main/java/com/zhiwei/manage/bean/NewsDelayed.java
View file @
1f779ff3
...
...
@@ -156,7 +156,9 @@ public class NewsDelayed implements Serializable {
return
"NewsDelayed [templateId="
+
templateId
+
", tempName="
+
tempName
+
", newsDelayed="
+
newsDelayed
+
", delayedInterval="
+
delayedInterval
+
", testCount="
+
testCount
+
", time="
+
time
+
", savetime="
+
savetime
+
", host="
+
host
+
", port="
+
port
+
", collection="
+
collection
+
", dBName="
+
dBName
+
", pt="
+
pt
+
", startNow="
+
startNow
+
", nextRun="
+
nextRun
+
"]"
;
+
", pt="
+
pt
+
", startNow="
+
startNow
+
", nextRun="
+
nextRun
+
", spyderInfoId="
+
spyderInfoId
+
"]"
;
}
}
src/main/java/com/zhiwei/manage/dao/MessageDaoImpl.java
View file @
1f779ff3
...
...
@@ -59,7 +59,7 @@ public class MessageDaoImpl implements MessageDao {
for
(
int
j
=
0
;
j
<
fields
.
length
;
j
++)
{
fields
[
j
].
setAccessible
(
true
);
try
{
if
(
fields
[
j
].
getName
().
equals
(
"templateId"
))
{
if
(
fields
[
j
].
getName
().
equals
(
"templateId"
)
||
fields
[
j
].
getName
().
equals
(
"tempName"
)
)
{
continue
;
}
if
(
fields
[
j
].
get
(
message
)
!=
null
)
...
...
@@ -69,7 +69,7 @@ public class MessageDaoImpl implements MessageDao {
e
.
printStackTrace
();
}
}
WriteResult
result
=
mongo
.
upsert
(
Query
.
query
(
Criteria
.
where
(
"
_id"
).
is
(
message
.
getMessageId
())),
update
,
WriteResult
result
=
mongo
.
upsert
(
Query
.
query
(
Criteria
.
where
(
"
tempName"
).
is
(
message
.
getTempName
())),
update
,
Message
.
class
);
if
(
result
.
getN
()
==
1
)
{
return
"修改成功"
;
...
...
src/main/java/com/zhiwei/manage/handle/MainThread.java
View file @
1f779ff3
...
...
@@ -48,7 +48,10 @@ public class MainThread extends Thread {
counts
.
put
(
"微博"
,
2000
);
counts
.
put
(
"微信"
,
3500
);
counts
.
put
(
"知乎"
,
300
);
// counts.put("平媒", 300);
counts
.
put
(
"平媒"
,
300
);
counts
.
put
(
"今日头条"
,
1000
);
counts
.
put
(
"贴吧"
,
250
);
counts
.
put
(
"论坛"
,
30
);
}
public
MainThread
()
{
...
...
@@ -183,7 +186,7 @@ public class MainThread extends Thread {
map
.
put
(
chan
.
getPt
(),
chan
.
getValue
());
try
{
ServerBean
ser
=
serverService
.
findServer
(
chan
.
getPt
());
count
=
es
.
getCountByPt
(
start
.
getTime
(),
end
.
getTime
(),
chan
.
getPt
(),
chan
.
getValue
(),
ser
);
count
=
es
.
getCountByPt
(
start
.
getTime
(),
end
.
getTime
(),
chan
.
getPt
(),
chan
.
getValue
(),
ser
);
//当日的数据量
System
.
out
.
println
(
count
);
Data
data
=
new
Data
();
data
.
setCount
(
count
);
...
...
@@ -191,11 +194,15 @@ public class MainThread extends Thread {
data
.
setTime
(
start
.
getTime
());
dataService
.
insert
(
data
);
log
.
info
(
sdf
.
format
(
start
.
getTime
())
+
"|"
+
chan
.
getPt
()
+
"|入库"
);
start
.
setTime
(
end
.
getTime
());
start
.
add
(
Calendar
.
HOUR
,
-
1
);
count
=
es
.
getCountByPt
(
start
.
getTime
(),
end
.
getTime
(),
chan
.
getPt
(),
chan
.
getValue
(),
ser
);
//小时的数据量
if
(
counts
.
get
(
chan
.
getPt
())
!=
null
)
if
(
count
<
counts
.
get
(
chan
.
getPt
()))
{
Messages
m
=
new
Messages
();
m
.
setTitle
(
"渠道低数据量预警{"
+
chan
.
getPt
()
+
"}"
);
m
.
setContent
(
"1小时内的数据量为:"
+
count
);
m
.
setDate
(
sdf
.
format
(
start
.
getTime
()));
for
(
String
mail
:
Config
.
getVal
(
"count_mail"
).
split
(
","
))
{
SendMailUtil
.
sendMessage
(
mail
,
m
);
}
...
...
src/main/java/com/zhiwei/manage/handle/Task.java
View file @
1f779ff3
This diff is collapsed.
Click to expand it.
src/main/java/com/zhiwei/manage/service/NewsServiceImpl.java
View file @
1f779ff3
...
...
@@ -16,7 +16,6 @@ public class NewsServiceImpl implements NewsService{
@Autowired
private
NewsDao
newsDao
;
private
ESUtil
es
=
new
ESUtil
();
@Override
public
List
<
News
>
findByNew
(
int
delayedCount
,
String
type
,
String
source
)
{
...
...
@@ -52,7 +51,7 @@ public class NewsServiceImpl implements NewsService{
@Override
public
List
<
Map
<
String
,
Object
>>
findNews
(
String
host
,
int
port
,
String
dbNamel
,
String
collection
,
int
count
,
String
id
,
String
pt
)
{
return
es
.
getDatasMap
(
host
,
port
,
dbNamel
,
collection
,
count
,
id
,
pt
);
return
ESUtil
.
getDatasMap
(
host
,
port
,
dbNamel
,
collection
,
count
,
id
,
pt
);
}
...
...
src/main/java/com/zhiwei/manage/util/ESUtil.java
View file @
1f779ff3
...
...
@@ -119,7 +119,8 @@ public class ESUtil {
// System.out.println(searchHits.getTotalHits());
SimpleDateFormat
sdf
=
new
SimpleDateFormat
(
"yyyy-MM-dd'T'HH:mm:ss.SSS Z"
);
for
(
Map
<
String
,
Object
>
map:
getDatasMap
(
"1.119.44.201"
,
9300
,
"Media-University"
,
"mediaspider*"
,
20
,
"UrlListDetail74ec93ff-2275-4f22-bb62-bae580f2d43b"
,
"网媒"
))
{
for
(
Map
<
String
,
Object
>
map:
getDatasMap
(
"1.119.44.201"
,
9300
,
"Media-University"
,
"mediaspider*"
,
20
,
"UrlListDetaildb4b9042-006f-4d09-a643-de379ac07e15"
,
"平媒"
))
{
System
.
out
.
println
(
map
.
get
(
"title"
));
String
date
=
map
.
get
(
"time"
).
toString
().
replace
(
"Z"
,
" UTC"
);
// System.out.println(map.get("time"));
...
...
@@ -144,20 +145,22 @@ public class ESUtil {
String
id
,
String
pt
)
{
List
<
Map
<
String
,
Object
>>
list
=
new
ArrayList
<>();
Client
esClient
=
null
;
Calendar
cal
=
Calendar
.
getInstance
();
cal
.
add
(
Calendar
.
DATE
,
-
7
);
try
{
if
(
DbDepot
.
esClients
.
get
(
pt
)
==
null
)
{
if
(
DbDepot
.
esClients
==
null
||
DbDepot
.
esClients
.
get
(
pt
)
==
null
)
{
esClient
=
new
ESClient
(
dbNamel
,
host
,
port
).
getClient
();
DbDepot
.
esClients
.
put
(
pt
,
esClient
);
//
DbDepot.esClients.put(pt, esClient);
}
else
{
esClient
=
(
Client
)
DbDepot
.
esClients
.
get
(
pt
);
}
// 搜索数据
SearchRequestBuilder
srb
=
esClient
.
prepareSearch
(
collection
);
srb
.
setPostFilter
(
QueryBuilders
.
rangeQuery
(
"time"
).
gte
(
getNeedTime
(
cal
)).
lte
(
getNeedTime
(
Calendar
.
getInstance
())));
BoolQueryBuilder
bb
=
new
BoolQueryBuilder
();
srb
.
setQuery
(
bb
);
srb
.
addSort
(
"time"
,
SortOrder
.
DESC
);
QueryBuilder
matchQuery
=
QueryBuilders
.
matchPhraseQuery
(
"spyderInfoId"
,
id
);
bb
.
should
(
matchQuery
);
srb
.
setQuery
(
bb
);
srb
.
setSize
(
count
);
SearchResponse
response
=
srb
.
execute
().
actionGet
();
SearchHits
searchHits
=
response
.
getHits
();
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment