Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
D
datamanage
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
shenjinzhu
datamanage
Commits
91f583d9
Commit
91f583d9
authored
Feb 09, 2018
by
shenjinzhu
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
更新操作
parent
ea51b12a
Show whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
28 additions
and
22 deletions
+28
-22
src/main/java/com/zhiwei/manage/dao/DataDaoImpl.java
+15
-7
src/main/java/com/zhiwei/manage/handle/MainThread.java
+11
-6
src/main/java/com/zhiwei/manage/servlet/TemplateController.java
+2
-9
No files found.
src/main/java/com/zhiwei/manage/dao/DataDaoImpl.java
View file @
91f583d9
...
@@ -27,6 +27,7 @@ import com.mongodb.DBCursor;
...
@@ -27,6 +27,7 @@ import com.mongodb.DBCursor;
import
com.mongodb.DBObject
;
import
com.mongodb.DBObject
;
import
com.mongodb.Mongo
;
import
com.mongodb.Mongo
;
import
com.mongodb.MongoClient
;
import
com.mongodb.MongoClient
;
import
com.mongodb.MongoClientOptions
;
import
com.mongodb.MongoCredential
;
import
com.mongodb.MongoCredential
;
import
com.mongodb.ServerAddress
;
import
com.mongodb.ServerAddress
;
import
com.mongodb.WriteResult
;
import
com.mongodb.WriteResult
;
...
@@ -141,7 +142,7 @@ public class DataDaoImpl implements DataDao {
...
@@ -141,7 +142,7 @@ public class DataDaoImpl implements DataDao {
int
year
=
c
.
get
(
Calendar
.
YEAR
);
int
year
=
c
.
get
(
Calendar
.
YEAR
);
Criteria
criteria
=
new
Criteria
();
Criteria
criteria
=
new
Criteria
();
criteria
.
and
(
"spyderInfoId"
).
is
(
SpId
);
criteria
.
and
(
"spyderInfoId"
).
is
(
SpId
);
criteria
.
and
(
"time"
).
gt
(
startTime
).
lt
(
endTime
);
criteria
.
and
(
"time"
).
gt
e
(
startTime
).
lt
(
endTime
);
return
mongo2
.
find
(
Query
.
query
(
criteria
),
EasyNews
.
class
,
return
mongo2
.
find
(
Query
.
query
(
criteria
),
EasyNews
.
class
,
"net_media_"
+
year
+
(
month
>
9
?
""
+
month
:
"0"
+
month
));
"net_media_"
+
year
+
(
month
>
9
?
""
+
month
:
"0"
+
month
));
}
}
...
@@ -263,12 +264,19 @@ public class DataDaoImpl implements DataDao {
...
@@ -263,12 +264,19 @@ public class DataDaoImpl implements DataDao {
public
static
void
main
(
String
[]
args
)
{
public
static
void
main
(
String
[]
args
)
{
// 连接到数据库
// 连接到数据库
ServerAddress
sa
=
new
ServerAddress
(
"115.236.59.91"
,
27017
);
MongoClientOptions
.
Builder
optionsBuilder
=
new
MongoClientOptions
.
Builder
();
MongoClientOptions
options
=
optionsBuilder
.
build
();
List
<
MongoCredential
>
credentials
=
new
ArrayList
<>();
MongoCredential
credential
=
MongoCredential
.
createCredential
(
"WX_datas"
,
"wx_datas"
,
"JDK_weixin66"
.
toCharArray
());
credentials
.
add
(
credential
);
List
<
ServerAddress
>
serverAddresses
=
new
ArrayList
<>();
serverAddresses
.
add
(
new
ServerAddress
(
"47.104.99.236"
,
Integer
.
valueOf
(
"2708"
)));
List
<
MongoCredential
>
mongoCredentialList
=
new
ArrayList
<
MongoCredential
>();
List
<
MongoCredential
>
mongoCredentialList
=
new
ArrayList
<
MongoCredential
>();
MongoClient
mongoClient
=
new
MongoClient
(
sa
,
mongoCredentialList
);
MongoClient
mongoClient
=
new
MongoClient
(
serverAddresses
,
credentials
,
options
);
MongoDatabase
mongo
=
mongoClient
.
getDatabase
(
"mediaspider"
);
MongoDatabase
mongo
=
mongoClient
.
getDatabase
(
"wx_datas"
);
MongoCollection
<
Document
>
col
=
mongo
.
getCollection
(
"net_media_201801"
);
MongoCollection
<
Document
>
col
=
mongo
.
getCollection
(
"wx_datas"
);
System
.
out
.
println
(
col
.
count
());
col
.
insertOne
(
new
Document
(
"k"
,
"bv"
));
}
}
@Override
@Override
...
@@ -391,7 +399,7 @@ public class DataDaoImpl implements DataDao {
...
@@ -391,7 +399,7 @@ public class DataDaoImpl implements DataDao {
int
year
=
c
.
get
(
Calendar
.
YEAR
);
int
year
=
c
.
get
(
Calendar
.
YEAR
);
int
month
=
c
.
get
(
Calendar
.
MONTH
)
+
1
;
int
month
=
c
.
get
(
Calendar
.
MONTH
)
+
1
;
return
(
int
)
mongo2
.
count
(
return
(
int
)
mongo2
.
count
(
Query
.
query
(
Criteria
.
where
(
"time"
).
gt
e
(
startTime
).
lt
(
endTime
).
and
(
"spyderInfoId"
).
is
(
syId
)),
Query
.
query
(
Criteria
.
where
(
"time"
).
gt
(
startTime
).
lt
(
endTime
).
and
(
"spyderInfoId"
).
is
(
syId
)),
EasyNews
.
class
,
"net_media_"
+
year
+
(
month
>
9
?
""
+
month
:
"0"
+
month
));
EasyNews
.
class
,
"net_media_"
+
year
+
(
month
>
9
?
""
+
month
:
"0"
+
month
));
}
}
...
...
src/main/java/com/zhiwei/manage/handle/MainThread.java
View file @
91f583d9
...
@@ -4,11 +4,13 @@ import java.text.ParseException;
...
@@ -4,11 +4,13 @@ import java.text.ParseException;
import
java.text.SimpleDateFormat
;
import
java.text.SimpleDateFormat
;
import
java.util.Calendar
;
import
java.util.Calendar
;
import
java.util.Date
;
import
java.util.Date
;
import
java.util.HashMap
;
import
java.util.Iterator
;
import
java.util.Iterator
;
import
java.util.List
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.Map
;
import
java.util.Set
;
import
java.util.Set
;
import
java.util.concurrent.ConcurrentHashMap
;
import
java.util.concurrent.ConcurrentHashMap
;
import
java.util.concurrent.LinkedBlockingQueue
;
import
java.util.concurrent.locks.Lock
;
import
java.util.concurrent.locks.Lock
;
import
java.util.concurrent.locks.ReentrantLock
;
import
java.util.concurrent.locks.ReentrantLock
;
...
@@ -136,9 +138,15 @@ public class MainThread extends Thread {
...
@@ -136,9 +138,15 @@ public class MainThread extends Thread {
}
}
}
}
public
static
boolean
delete
(
String
templateId
)
{
public
static
void
removeAll
()
{
allTmp
.
remove
(
templateId
);
allTmp
=
new
ConcurrentHashMap
<
String
,
Template
>();
return
true
;
mainMap
=
new
ConcurrentHashMap
<
String
,
Template
>();
DelayedThread
.
runMap
=
new
ConcurrentHashMap
<
String
,
NewsDelayed
>();
DelayedThread
.
queue
=
new
LinkedBlockingQueue
<
NewsDelayed
>(
500
);
FieldThread
.
runMap
=
new
ConcurrentHashMap
<
String
,
FieldIntegerity
>();
FieldThread
.
queue
=
new
LinkedBlockingQueue
<
FieldIntegerity
>(
500
);
PingThread
.
runMap
=
new
ConcurrentHashMap
<
String
,
PingUrl
>();
PingThread
.
queue
=
new
LinkedBlockingQueue
<
PingUrl
>(
500
);
}
}
@Override
@Override
...
@@ -158,9 +166,7 @@ public class MainThread extends Thread {
...
@@ -158,9 +166,7 @@ public class MainThread extends Thread {
Calendar
start
=
Calendar
.
getInstance
();
Calendar
start
=
Calendar
.
getInstance
();
if
(
start
.
get
(
Calendar
.
HOUR_OF_DAY
)
<
1
)
{
if
(
start
.
get
(
Calendar
.
HOUR_OF_DAY
)
<
1
)
{
Calendar
end
=
Calendar
.
getInstance
();
Calendar
end
=
Calendar
.
getInstance
();
Date
d
=
null
;
try
{
try
{
d
=
sdf
.
parse
(
today
);
// start.setTime(sdf2.parse(sdf2.format(start.getTime())));
// start.setTime(sdf2.parse(sdf2.format(start.getTime())));
// if (start.get(Calendar.HOUR_OF_DAY) > 1) {
// if (start.get(Calendar.HOUR_OF_DAY) > 1) {
// start.add(Calendar.HOUR_OF_DAY, -1);
// start.add(Calendar.HOUR_OF_DAY, -1);
...
@@ -170,7 +176,6 @@ public class MainThread extends Thread {
...
@@ -170,7 +176,6 @@ public class MainThread extends Thread {
// }
// }
start
.
add
(
Calendar
.
DATE
,
-
1
);
start
.
add
(
Calendar
.
DATE
,
-
1
);
end
.
setTime
(
sdf2
.
parse
(
today
));
end
.
setTime
(
sdf2
.
parse
(
today
));
end
.
add
(
Calendar
.
HOUR_OF_DAY
,
-
1
);
}
catch
(
ParseException
e2
)
{
}
catch
(
ParseException
e2
)
{
e2
.
printStackTrace
();
e2
.
printStackTrace
();
}
}
...
...
src/main/java/com/zhiwei/manage/servlet/TemplateController.java
View file @
91f583d9
...
@@ -46,14 +46,7 @@ public class TemplateController {
...
@@ -46,14 +46,7 @@ public class TemplateController {
@RequestMapping
(
value
=
"/stopAll"
,
produces
=
"application/json;charset=utf-8"
)
@RequestMapping
(
value
=
"/stopAll"
,
produces
=
"application/json;charset=utf-8"
)
@ResponseBody
@ResponseBody
public
String
stopAll
()
{
public
String
stopAll
()
{
Set
<
String
>
ids
=
MainThread
.
mainMap
.
keySet
();
MainThread
.
removeAll
();
for
(
String
id:
ids
)
{
try
{
MainThread
.
removeMap
(
id
);
}
catch
(
Exception
e
)
{
log
.
error
(
"停止失败{}"
,
e
);
}
}
JsonResult
result
=
new
JsonResult
(
true
,
200
,
"停止成功"
);
JsonResult
result
=
new
JsonResult
(
true
,
200
,
"停止成功"
);
return
JSON
.
toJSONString
(
result
);
return
JSON
.
toJSONString
(
result
);
}
}
...
@@ -96,7 +89,7 @@ public class TemplateController {
...
@@ -96,7 +89,7 @@ public class TemplateController {
String
results
=
""
;
String
results
=
""
;
for
(
String
l
:
templateId
)
{
for
(
String
l
:
templateId
)
{
isDelete
=
ts
.
delete
(
l
,
pt
);
isDelete
=
ts
.
delete
(
l
,
pt
);
MainThread
.
delete
(
l
);
MainThread
.
removeMap
(
l
);
results
+=
l
+
"\n"
;
results
+=
l
+
"\n"
;
}
}
JsonResult
result
=
new
JsonResult
(
true
,
200
,
results
+
"删除"
+
(
isDelete
==
false
?
"成功"
:
"失败"
));
JsonResult
result
=
new
JsonResult
(
true
,
200
,
results
+
"删除"
+
(
isDelete
==
false
?
"成功"
:
"失败"
));
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment