Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
S
searchhotcrawler
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
zhiwei
searchhotcrawler
Commits
c51af150
Commit
c51af150
authored
Dec 13, 2021
by
chenweitao
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
修复日志漏洞
parent
2e1ca3af
Hide whitespace changes
Inline
Side-by-side
Showing
16 changed files
with
119 additions
and
133 deletions
+119
-133
src/main/java/com/zhiwei/searchhotcrawler/cache/CacheListener.java
+1
-3
src/main/java/com/zhiwei/searchhotcrawler/crawler/BaiDuHotSearchCrawler.java
+1
-1
src/main/java/com/zhiwei/searchhotcrawler/crawler/DouyinHotSearchCrawler.java
+2
-3
src/main/java/com/zhiwei/searchhotcrawler/crawler/HotSearch36KrCrawler.java
+1
-1
src/main/java/com/zhiwei/searchhotcrawler/crawler/SougoHotSearchCrawler.java
+2
-3
src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiboSuperTopicCrawler.java
+1
-2
src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiboTopicCrawler.java
+1
-2
src/main/java/com/zhiwei/searchhotcrawler/crawler/ZhihuHotSearchCrawler.java
+1
-2
src/main/java/com/zhiwei/searchhotcrawler/test/HotSearch36KrCrawlerTest.java
+1
-1
src/main/java/com/zhiwei/searchhotcrawler/timer/BaiduHotSearchRun.java
+3
-6
src/main/java/com/zhiwei/searchhotcrawler/timer/DouyinHotSearchRun.java
+3
-4
src/main/java/com/zhiwei/searchhotcrawler/timer/SougoHotSearchRun.java
+3
-4
src/main/java/com/zhiwei/searchhotcrawler/timer/ZhihuHotSearchRun.java
+3
-4
src/main/java/com/zhiwei/searchhotcrawler/timer/quartz/GatherTimer.java
+76
-78
src/main/java/com/zhiwei/searchhotcrawler/util/TipsUtils.java
+4
-4
src/main/java/com/zhiwei/searchhotcrawler/util/WechatCodeUtil.java
+16
-15
No files found.
src/main/java/com/zhiwei/searchhotcrawler/cache/CacheListener.java
View file @
c51af150
package
com
.
zhiwei
.
searchhotcrawler
.
cache
;
package
com
.
zhiwei
.
searchhotcrawler
.
cache
;
import
lombok.extern.log4j.Log4j2
;
import
lombok.extern.log4j.Log4j2
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
com.zhiwei.tools.tools.ZhiWeiTools
;
import
com.zhiwei.tools.tools.ZhiWeiTools
;
...
@@ -26,7 +24,7 @@ public class CacheListener {
...
@@ -26,7 +24,7 @@ public class CacheListener {
}
}
ZhiWeiTools
.
sleep
(
500
);
ZhiWeiTools
.
sleep
(
500
);
}
}
}
}
}.
start
();
}.
start
();
}
}
}
}
src/main/java/com/zhiwei/searchhotcrawler/crawler/BaiDuHotSearchCrawler.java
View file @
c51af150
...
@@ -129,7 +129,7 @@ public class BaiDuHotSearchCrawler {
...
@@ -129,7 +129,7 @@ public class BaiDuHotSearchCrawler {
String
everurl
=
element
.
select
(
"td.keyword"
).
select
(
"a.list-title"
).
attr
(
"href"
);
String
everurl
=
element
.
select
(
"td.keyword"
).
select
(
"a.list-title"
).
attr
(
"href"
);
// 获取关键词(String)
// 获取关键词(String)
String
kw
=
element
.
select
(
"td.keyword"
).
select
(
"a.list-title"
).
text
();
String
kw
=
element
.
select
(
"td.keyword"
).
select
(
"a.list-title"
).
text
();
// log
ger
.info("关键词:{}", kw);
// log.info("关键词:{}", kw);
//从连接中获取正确编码关键词
//从连接中获取正确编码关键词
try
{
try
{
if
(!
everurl
.
isEmpty
())
{
if
(!
everurl
.
isEmpty
())
{
...
...
src/main/java/com/zhiwei/searchhotcrawler/crawler/DouyinHotSearchCrawler.java
View file @
c51af150
...
@@ -9,8 +9,7 @@ import lombok.extern.log4j.Log4j2;
...
@@ -9,8 +9,7 @@ import lombok.extern.log4j.Log4j2;
import
okhttp3.Request
;
import
okhttp3.Request
;
import
okhttp3.Response
;
import
okhttp3.Response
;
import
org.apache.commons.lang3.StringUtils
;
import
org.apache.commons.lang3.StringUtils
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
com.alibaba.fastjson.JSONArray
;
import
com.alibaba.fastjson.JSONArray
;
import
com.alibaba.fastjson.JSONObject
;
import
com.alibaba.fastjson.JSONObject
;
...
@@ -71,7 +70,7 @@ public class DouyinHotSearchCrawler {
...
@@ -71,7 +70,7 @@ public class DouyinHotSearchCrawler {
hotValueStr
=
wl
.
getString
(
"hot_value"
);
hotValueStr
=
wl
.
getString
(
"hot_value"
);
Long
hotValue
=
null
;
Long
hotValue
=
null
;
hotValue
=
Long
.
valueOf
(
hotValueStr
);
hotValue
=
Long
.
valueOf
(
hotValueStr
);
// log
ger
.info("热度为:::{}", hot_value);
// log.info("热度为:::{}", hot_value);
HotSearchList
douyin
=
new
HotSearchList
(
null
,
word
,
hotValue
,
position
,
HotSearchType
.
抖音热搜
.
name
(),
date
);
HotSearchList
douyin
=
new
HotSearchList
(
null
,
word
,
hotValue
,
position
,
HotSearchType
.
抖音热搜
.
name
(),
date
);
list
.
add
(
douyin
);
list
.
add
(
douyin
);
}
}
...
...
src/main/java/com/zhiwei/searchhotcrawler/crawler/HotSearch36KrCrawler.java
View file @
c51af150
...
@@ -99,7 +99,7 @@ public class HotSearch36KrCrawler {
...
@@ -99,7 +99,7 @@ public class HotSearch36KrCrawler {
rank
++;
rank
++;
// 获取关键词(String)
// 获取关键词(String)
String
keyWord
=
element
.
select
(
"p.title-wrapper"
).
select
(
"a.article-item-title"
).
text
();
String
keyWord
=
element
.
select
(
"p.title-wrapper"
).
select
(
"a.article-item-title"
).
text
();
// log
ger
.info("关键词:{}", kw);
// log.info("关键词:{}", kw);
// 获取关键词相关链接everurl(String)
// 获取关键词相关链接everurl(String)
String
everurl
=
element
.
select
(
"p.title-wrapper"
).
select
(
"a.article-item-title"
).
attr
(
"href"
);
String
everurl
=
element
.
select
(
"p.title-wrapper"
).
select
(
"a.article-item-title"
).
attr
(
"href"
);
// 获取搜索指数count(int)
// 获取搜索指数count(int)
...
...
src/main/java/com/zhiwei/searchhotcrawler/crawler/SougoHotSearchCrawler.java
View file @
c51af150
...
@@ -13,8 +13,7 @@ import org.jsoup.Jsoup;
...
@@ -13,8 +13,7 @@ import org.jsoup.Jsoup;
import
org.jsoup.nodes.Document
;
import
org.jsoup.nodes.Document
;
import
org.jsoup.nodes.Element
;
import
org.jsoup.nodes.Element
;
import
org.jsoup.select.Elements
;
import
org.jsoup.select.Elements
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
com.zhiwei.crawler.core.HttpBoot
;
import
com.zhiwei.crawler.core.HttpBoot
;
import
com.zhiwei.crawler.core.proxy.ProxyHolder
;
import
com.zhiwei.crawler.core.proxy.ProxyHolder
;
...
@@ -70,7 +69,7 @@ public class SougoHotSearchCrawler {
...
@@ -70,7 +69,7 @@ public class SougoHotSearchCrawler {
// 获取关键词(String)
// 获取关键词(String)
String
kw
=
element
.
select
(
"li"
).
select
(
"a"
).
attr
(
"title"
);
String
kw
=
element
.
select
(
"li"
).
select
(
"a"
).
attr
(
"title"
);
// log
ger
.info("关键词:{}", kw);
// log.info("关键词:{}", kw);
String
everurl
=
element
.
select
(
"li"
).
select
(
"a"
).
attr
(
"href"
);
String
everurl
=
element
.
select
(
"li"
).
select
(
"a"
).
attr
(
"href"
);
...
...
src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiboSuperTopicCrawler.java
View file @
c51af150
...
@@ -13,8 +13,7 @@ import lombok.extern.log4j.Log4j2;
...
@@ -13,8 +13,7 @@ import lombok.extern.log4j.Log4j2;
import
okhttp3.Request
;
import
okhttp3.Request
;
import
okhttp3.Response
;
import
okhttp3.Response
;
import
org.apache.commons.lang3.StringUtils
;
import
org.apache.commons.lang3.StringUtils
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
com.alibaba.fastjson.JSONArray
;
import
com.alibaba.fastjson.JSONArray
;
import
com.alibaba.fastjson.JSONObject
;
import
com.alibaba.fastjson.JSONObject
;
...
...
src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiboTopicCrawler.java
View file @
c51af150
...
@@ -18,8 +18,7 @@ import org.apache.commons.lang3.StringUtils;
...
@@ -18,8 +18,7 @@ import org.apache.commons.lang3.StringUtils;
import
org.jsoup.Jsoup
;
import
org.jsoup.Jsoup
;
import
org.jsoup.nodes.Element
;
import
org.jsoup.nodes.Element
;
import
org.jsoup.select.Elements
;
import
org.jsoup.select.Elements
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
java.util.*
;
import
java.util.*
;
import
java.util.regex.Matcher
;
import
java.util.regex.Matcher
;
import
java.util.regex.Pattern
;
import
java.util.regex.Pattern
;
...
...
src/main/java/com/zhiwei/searchhotcrawler/crawler/ZhihuHotSearchCrawler.java
View file @
c51af150
...
@@ -9,8 +9,7 @@ import java.util.Map;
...
@@ -9,8 +9,7 @@ import java.util.Map;
import
lombok.extern.log4j.Log4j2
;
import
lombok.extern.log4j.Log4j2
;
import
okhttp3.Request
;
import
okhttp3.Request
;
import
okhttp3.Response
;
import
okhttp3.Response
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
com.alibaba.fastjson.JSONArray
;
import
com.alibaba.fastjson.JSONArray
;
import
com.alibaba.fastjson.JSONObject
;
import
com.alibaba.fastjson.JSONObject
;
...
...
src/main/java/com/zhiwei/searchhotcrawler/test/HotSearch36KrCrawlerTest.java
View file @
c51af150
...
@@ -99,7 +99,7 @@ public class HotSearch36KrCrawlerTest {
...
@@ -99,7 +99,7 @@ public class HotSearch36KrCrawlerTest {
rank
++;
rank
++;
// 获取关键词(String)
// 获取关键词(String)
String
keyWord
=
element
.
select
(
"p.title-wrapper"
).
select
(
"a.article-item-title"
).
text
();
String
keyWord
=
element
.
select
(
"p.title-wrapper"
).
select
(
"a.article-item-title"
).
text
();
// log
ger
.info("关键词:{}", kw);
// log.info("关键词:{}", kw);
// 获取关键词相关链接everurl(String)
// 获取关键词相关链接everurl(String)
String
everurl
=
element
.
select
(
"p.title-wrapper"
).
select
(
"a.article-item-title"
).
attr
(
"href"
);
String
everurl
=
element
.
select
(
"p.title-wrapper"
).
select
(
"a.article-item-title"
).
attr
(
"href"
);
// 获取搜索指数count(int)
// 获取搜索指数count(int)
...
...
src/main/java/com/zhiwei/searchhotcrawler/timer/BaiduHotSearchRun.java
View file @
c51af150
...
@@ -12,8 +12,6 @@ import com.zhiwei.searchhotcrawler.dao.HotSearchCacheDAO;
...
@@ -12,8 +12,6 @@ import com.zhiwei.searchhotcrawler.dao.HotSearchCacheDAO;
import
com.zhiwei.searchhotcrawler.util.TipsUtils
;
import
com.zhiwei.searchhotcrawler.util.TipsUtils
;
import
lombok.extern.log4j.Log4j2
;
import
lombok.extern.log4j.Log4j2
;
import
org.bson.Document
;
import
org.bson.Document
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
com.mongodb.BasicDBObject
;
import
com.mongodb.BasicDBObject
;
import
com.mongodb.DBObject
;
import
com.mongodb.DBObject
;
...
@@ -39,8 +37,8 @@ public class BaiduHotSearchRun extends Thread{
...
@@ -39,8 +37,8 @@ public class BaiduHotSearchRun extends Thread{
ZhiWeiTools
.
sleep
(
50
);
ZhiWeiTools
.
sleep
(
50
);
}
}
}
}
private
void
getHotList
()
{
private
void
getHotList
()
{
log
.
info
(
"百度风云榜采集开始........"
);
log
.
info
(
"百度风云榜采集开始........"
);
// HotSearchListDAO hotSearchDAO = new HotSearchListDAO();
// HotSearchListDAO hotSearchDAO = new HotSearchListDAO();
...
@@ -70,4 +68,4 @@ public class BaiduHotSearchRun extends Thread{
...
@@ -70,4 +68,4 @@ public class BaiduHotSearchRun extends Thread{
// log.info("知乎话题采集结束........");
// log.info("知乎话题采集结束........");
}
}
}
}
\ No newline at end of file
src/main/java/com/zhiwei/searchhotcrawler/timer/DouyinHotSearchRun.java
View file @
c51af150
...
@@ -12,8 +12,7 @@ import com.zhiwei.searchhotcrawler.dao.HotSearchCacheDAO;
...
@@ -12,8 +12,7 @@ import com.zhiwei.searchhotcrawler.dao.HotSearchCacheDAO;
import
com.zhiwei.searchhotcrawler.util.TipsUtils
;
import
com.zhiwei.searchhotcrawler.util.TipsUtils
;
import
lombok.extern.log4j.Log4j2
;
import
lombok.extern.log4j.Log4j2
;
import
org.bson.Document
;
import
org.bson.Document
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
com.mongodb.BasicDBObject
;
import
com.mongodb.BasicDBObject
;
import
com.mongodb.DBObject
;
import
com.mongodb.DBObject
;
...
@@ -26,7 +25,7 @@ import com.zhiwei.tools.tools.ZhiWeiTools;
...
@@ -26,7 +25,7 @@ import com.zhiwei.tools.tools.ZhiWeiTools;
public
class
DouyinHotSearchRun
extends
Thread
{
public
class
DouyinHotSearchRun
extends
Thread
{
public
static
List
<
HotSearchList
>
list
=
new
ArrayList
<>();
public
static
List
<
HotSearchList
>
list
=
new
ArrayList
<>();
@Override
@Override
public
void
run
()
{
public
void
run
()
{
boolean
f
=
true
;
boolean
f
=
true
;
...
@@ -41,7 +40,7 @@ public class DouyinHotSearchRun extends Thread{
...
@@ -41,7 +40,7 @@ public class DouyinHotSearchRun extends Thread{
ZhiWeiTools
.
sleep
(
50
);
ZhiWeiTools
.
sleep
(
50
);
}
}
}
}
/**
/**
* 获取热搜列表
* 获取热搜列表
* TODO
* TODO
...
...
src/main/java/com/zhiwei/searchhotcrawler/timer/SougoHotSearchRun.java
View file @
c51af150
...
@@ -10,8 +10,7 @@ import com.zhiwei.searchhotcrawler.dao.HotSearchListDAO;
...
@@ -10,8 +10,7 @@ import com.zhiwei.searchhotcrawler.dao.HotSearchListDAO;
import
com.zhiwei.searchhotcrawler.util.TipsUtils
;
import
com.zhiwei.searchhotcrawler.util.TipsUtils
;
import
lombok.extern.log4j.Log4j2
;
import
lombok.extern.log4j.Log4j2
;
import
org.bson.Document
;
import
org.bson.Document
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
com.mongodb.BasicDBObject
;
import
com.mongodb.BasicDBObject
;
import
com.mongodb.DBObject
;
import
com.mongodb.DBObject
;
...
@@ -36,8 +35,8 @@ public class SougoHotSearchRun extends Thread {
...
@@ -36,8 +35,8 @@ public class SougoHotSearchRun extends Thread {
ZhiWeiTools
.
sleep
(
50
);
ZhiWeiTools
.
sleep
(
50
);
}
}
}
}
private
void
getHotList
()
{
private
void
getHotList
()
{
// HotSearchListDAO hotSearchDAO = new HotSearchListDAO();
// HotSearchListDAO hotSearchDAO = new HotSearchListDAO();
// HotSearchCacheDAO hotSearchCacheDAO = new HotSearchCacheDAO();
// HotSearchCacheDAO hotSearchCacheDAO = new HotSearchCacheDAO();
...
...
src/main/java/com/zhiwei/searchhotcrawler/timer/ZhihuHotSearchRun.java
View file @
c51af150
...
@@ -9,8 +9,7 @@ import com.zhiwei.searchhotcrawler.dao.HotSearchCacheDAO;
...
@@ -9,8 +9,7 @@ import com.zhiwei.searchhotcrawler.dao.HotSearchCacheDAO;
import
com.zhiwei.searchhotcrawler.util.TipsUtils
;
import
com.zhiwei.searchhotcrawler.util.TipsUtils
;
import
lombok.extern.log4j.Log4j2
;
import
lombok.extern.log4j.Log4j2
;
import
org.bson.Document
;
import
org.bson.Document
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
com.mongodb.BasicDBObject
;
import
com.mongodb.BasicDBObject
;
import
com.mongodb.DBObject
;
import
com.mongodb.DBObject
;
...
@@ -21,7 +20,7 @@ import com.zhiwei.tools.tools.ZhiWeiTools;
...
@@ -21,7 +20,7 @@ import com.zhiwei.tools.tools.ZhiWeiTools;
@Log4j2
@Log4j2
public
class
ZhihuHotSearchRun
extends
Thread
{
public
class
ZhihuHotSearchRun
extends
Thread
{
@Override
@Override
public
void
run
()
{
public
void
run
()
{
boolean
f
=
true
;
boolean
f
=
true
;
...
@@ -37,7 +36,7 @@ public class ZhihuHotSearchRun extends Thread{
...
@@ -37,7 +36,7 @@ public class ZhihuHotSearchRun extends Thread{
}
}
}
}
private
void
getHotList
()
{
private
void
getHotList
()
{
// HotSearchListDAO hotSearchDAO = new HotSearchListDAO();
// HotSearchListDAO hotSearchDAO = new HotSearchListDAO();
// HotSearchCacheDAO hotSearchCacheDAO = new HotSearchCacheDAO();
// HotSearchCacheDAO hotSearchCacheDAO = new HotSearchCacheDAO();
...
...
src/main/java/com/zhiwei/searchhotcrawler/timer/quartz/GatherTimer.java
View file @
c51af150
...
@@ -14,9 +14,8 @@ import com.zhiwei.searchhotcrawler.timer.TouTiaoExecutor;
...
@@ -14,9 +14,8 @@ import com.zhiwei.searchhotcrawler.timer.TouTiaoExecutor;
import
com.zhiwei.searchhotcrawler.util.DateUtils
;
import
com.zhiwei.searchhotcrawler.util.DateUtils
;
import
com.zhiwei.searchhotcrawler.util.TipsUtils
;
import
com.zhiwei.searchhotcrawler.util.TipsUtils
;
import
com.zhiwei.tools.tools.ZhiWeiTools
;
import
com.zhiwei.tools.tools.ZhiWeiTools
;
import
lombok.extern.log4j.Log4j2
;
import
org.bson.Document
;
import
org.bson.Document
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
org.springframework.scheduling.annotation.Async
;
import
org.springframework.scheduling.annotation.Async
;
import
org.springframework.scheduling.annotation.EnableAsync
;
import
org.springframework.scheduling.annotation.EnableAsync
;
import
org.springframework.scheduling.annotation.EnableScheduling
;
import
org.springframework.scheduling.annotation.EnableScheduling
;
...
@@ -28,10 +27,9 @@ import java.util.*;
...
@@ -28,10 +27,9 @@ import java.util.*;
@Component
@Component
@EnableScheduling
@EnableScheduling
@EnableAsync
@EnableAsync
@Log4j2
public
class
GatherTimer
{
public
class
GatherTimer
{
private
Logger
logger
=
LoggerFactory
.
getLogger
(
GatherTimer
.
class
);
private
RedisDao
redisDao
=
new
RedisDao
();
private
RedisDao
redisDao
=
new
RedisDao
();
/** 知乎数码子分类 */
/** 知乎数码子分类 */
private
String
DIGITAL
=
"digital"
;
private
String
DIGITAL
=
"digital"
;
...
@@ -47,21 +45,21 @@ public class GatherTimer {
...
@@ -47,21 +45,21 @@ public class GatherTimer {
@Async
(
value
=
"myScheduler"
)
@Async
(
value
=
"myScheduler"
)
@Scheduled
(
cron
=
"0 * * * * ?"
)
@Scheduled
(
cron
=
"0 * * * * ?"
)
public
void
crawlerHuXiu
()
{
public
void
crawlerHuXiu
()
{
log
ger
.
info
(
"虎嗅热文推荐开始采集..."
);
log
.
info
(
"虎嗅热文推荐开始采集..."
);
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
List
<
HotSearchList
>
huXiuList
=
HuXiuHotSearchCrawler
.
HuXiuHotArticleRecommended
(
date
);
List
<
HotSearchList
>
huXiuList
=
HuXiuHotSearchCrawler
.
HuXiuHotArticleRecommended
(
date
);
log
ger
.
info
(
"{}, 虎嗅热文推荐此轮采集到的数据量为:{}"
,
new
Date
(),
Integer
.
valueOf
(
huXiuList
!=
null
?
huXiuList
.
size
()
:
0
));
log
.
info
(
"{}, 虎嗅热文推荐此轮采集到的数据量为:{}"
,
new
Date
(),
Integer
.
valueOf
(
huXiuList
!=
null
?
huXiuList
.
size
()
:
0
));
TipsUtils
.
addHotList
(
HotSearchType
.
虎嗅热文推荐
.
name
(),
huXiuList
);
TipsUtils
.
addHotList
(
HotSearchType
.
虎嗅热文推荐
.
name
(),
huXiuList
);
log
ger
.
info
(
"虎嗅热文推荐采集结束..."
);
log
.
info
(
"虎嗅热文推荐采集结束..."
);
/**
/**
* 36氪人气榜的采集
* 36氪人气榜的采集
*/
*/
log
ger
.
info
(
"36氪人气榜开始采集..."
);
log
.
info
(
"36氪人气榜开始采集..."
);
List
<
HotSearchList
>
list36Kr
=
HotSearch36KrCrawler
.
hotSearch36Kr
(
date
);
List
<
HotSearchList
>
list36Kr
=
HotSearch36KrCrawler
.
hotSearch36Kr
(
date
);
log
ger
.
info
(
"{}, 36氪人气榜此轮采集到的数据量为:{}"
,
new
Date
(),
Integer
.
valueOf
(
list36Kr
!=
null
?
list36Kr
.
size
()
:
0
));
log
.
info
(
"{}, 36氪人气榜此轮采集到的数据量为:{}"
,
new
Date
(),
Integer
.
valueOf
(
list36Kr
!=
null
?
list36Kr
.
size
()
:
0
));
TipsUtils
.
addHotList
(
HotSearchType
.
人气榜
36
氪
.
name
(),
list36Kr
);
TipsUtils
.
addHotList
(
HotSearchType
.
人气榜
36
氪
.
name
(),
list36Kr
);
log
ger
.
info
(
"36氪人气榜采集结束..."
);
log
.
info
(
"36氪人气榜采集结束..."
);
}
}
/**
/**
...
@@ -70,12 +68,12 @@ public class GatherTimer {
...
@@ -70,12 +68,12 @@ public class GatherTimer {
@Async
(
value
=
"myScheduler"
)
@Async
(
value
=
"myScheduler"
)
@Scheduled
(
cron
=
"0 * * * * ? "
)
@Scheduled
(
cron
=
"0 * * * * ? "
)
public
void
crawlerWeiBo
(){
public
void
crawlerWeiBo
(){
log
ger
.
info
(
"微博热搜开始采集..."
);
log
.
info
(
"微博热搜开始采集..."
);
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
List
<
HotSearchList
>
weiboList
=
WeiboHotSearchCrawler
.
weiboHotSearchByPhone
(
date
);
List
<
HotSearchList
>
weiboList
=
WeiboHotSearchCrawler
.
weiboHotSearchByPhone
(
date
);
log
ger
.
info
(
"{}, 微博此轮采集到的数据量为:{}"
,
new
Date
(),
weiboList
!=
null
?
weiboList
.
size
()
:
0
);
log
.
info
(
"{}, 微博此轮采集到的数据量为:{}"
,
new
Date
(),
weiboList
!=
null
?
weiboList
.
size
()
:
0
);
TipsUtils
.
addHotList
(
HotSearchType
.
微博热搜
.
name
(),
weiboList
);
TipsUtils
.
addHotList
(
HotSearchType
.
微博热搜
.
name
(),
weiboList
);
log
ger
.
info
(
"微博热搜采集结束..."
);
log
.
info
(
"微博热搜采集结束..."
);
}
}
/**
/**
...
@@ -84,7 +82,7 @@ public class GatherTimer {
...
@@ -84,7 +82,7 @@ public class GatherTimer {
@Async
(
value
=
"myScheduler"
)
@Async
(
value
=
"myScheduler"
)
@Scheduled
(
cron
=
"45 0/10 * * * ? "
)
@Scheduled
(
cron
=
"45 0/10 * * * ? "
)
public
void
updateWeiBo
(){
public
void
updateWeiBo
(){
log
ger
.
info
(
"微博热搜导语更新..."
);
log
.
info
(
"微博热搜导语更新..."
);
HotSearchCacheDAO
hotSearchCacheDAO
=
new
HotSearchCacheDAO
();
HotSearchCacheDAO
hotSearchCacheDAO
=
new
HotSearchCacheDAO
();
Set
<
String
>
hotSearchIdSet
=
redisDao
.
getRedisSetData
(
RedisConfig
.
WEIBO_HOTSEARCHIDS
);
Set
<
String
>
hotSearchIdSet
=
redisDao
.
getRedisSetData
(
RedisConfig
.
WEIBO_HOTSEARCHIDS
);
redisDao
.
removeRedis
(
RedisConfig
.
WEIBO_HOTSEARCHIDS
);
redisDao
.
removeRedis
(
RedisConfig
.
WEIBO_HOTSEARCHIDS
);
...
@@ -100,7 +98,7 @@ public class GatherTimer {
...
@@ -100,7 +98,7 @@ public class GatherTimer {
ZhiWeiTools
.
sleep
(
3000L
);
ZhiWeiTools
.
sleep
(
3000L
);
}
}
}
}
log
ger
.
info
(
"微博热搜导语更新结束..."
);
log
.
info
(
"微博热搜导语更新结束..."
);
}
}
/**
/**
...
@@ -109,13 +107,13 @@ public class GatherTimer {
...
@@ -109,13 +107,13 @@ public class GatherTimer {
@Async
(
value
=
"myScheduler"
)
@Async
(
value
=
"myScheduler"
)
@Scheduled
(
cron
=
"0 * * * * ? "
)
@Scheduled
(
cron
=
"0 * * * * ? "
)
public
void
crawlerTouTiao
(){
public
void
crawlerTouTiao
(){
log
ger
.
info
(
"今日头条热搜开始采集..."
);
log
.
info
(
"今日头条热搜开始采集..."
);
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
List
<
HotSearchList
>
toutiaoList
=
ToutiaoHotSearchCrawler
.
toutiaoHotSearchByPhone
(
date
);
List
<
HotSearchList
>
toutiaoList
=
ToutiaoHotSearchCrawler
.
toutiaoHotSearchByPhone
(
date
);
log
ger
.
info
(
"{}, 今日头条此轮采集到的数据量为:{}"
,
new
Date
(),
toutiaoList
!=
null
?
toutiaoList
.
size
()
:
0
);
log
.
info
(
"{}, 今日头条此轮采集到的数据量为:{}"
,
new
Date
(),
toutiaoList
!=
null
?
toutiaoList
.
size
()
:
0
);
TipsUtils
.
addHotList
(
HotSearchType
.
今日头条热搜
.
name
(),
toutiaoList
);
TipsUtils
.
addHotList
(
HotSearchType
.
今日头条热搜
.
name
(),
toutiaoList
);
log
ger
.
info
(
"今日头条热搜采集结束..."
);
log
.
info
(
"今日头条热搜采集结束..."
);
log
ger
.
info
(
"今日头条热搜详情趋势阅读量更新..."
);
log
.
info
(
"今日头条热搜详情趋势阅读量更新..."
);
TouTiaoExecutor
.
countTouTiaoReadCount
(
toutiaoList
);
TouTiaoExecutor
.
countTouTiaoReadCount
(
toutiaoList
);
}
}
...
@@ -125,12 +123,12 @@ public class GatherTimer {
...
@@ -125,12 +123,12 @@ public class GatherTimer {
@Async
(
value
=
"myScheduler"
)
@Async
(
value
=
"myScheduler"
)
@Scheduled
(
cron
=
"0 * * * * ? "
)
@Scheduled
(
cron
=
"0 * * * * ? "
)
public
void
crawlerBaiDu
(){
public
void
crawlerBaiDu
(){
log
ger
.
info
(
"百度热搜开始采集..."
);
log
.
info
(
"百度热搜开始采集..."
);
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
List
<
HotSearchList
>
baiduList
=
BaiDuHotSearchCrawler
.
baiduHotSearch
(
date
);
List
<
HotSearchList
>
baiduList
=
BaiDuHotSearchCrawler
.
baiduHotSearch
(
date
);
log
ger
.
info
(
"{}, 百度热搜此轮采集到的数据量为:{}"
,
new
Date
(),
Integer
.
valueOf
(
baiduList
!=
null
?
baiduList
.
size
()
:
0
));
log
.
info
(
"{}, 百度热搜此轮采集到的数据量为:{}"
,
new
Date
(),
Integer
.
valueOf
(
baiduList
!=
null
?
baiduList
.
size
()
:
0
));
TipsUtils
.
addHotList
(
HotSearchType
.
百度热搜
.
name
(),
baiduList
);
TipsUtils
.
addHotList
(
HotSearchType
.
百度热搜
.
name
(),
baiduList
);
log
ger
.
info
(
"百度热搜采集结束..."
);
log
.
info
(
"百度热搜采集结束..."
);
}
}
/**
/**
...
@@ -139,12 +137,12 @@ public class GatherTimer {
...
@@ -139,12 +137,12 @@ public class GatherTimer {
@Async
(
value
=
"myScheduler"
)
@Async
(
value
=
"myScheduler"
)
@Scheduled
(
cron
=
"0 * * * * ? "
)
@Scheduled
(
cron
=
"0 * * * * ? "
)
public
void
crawlerDouYin
(){
public
void
crawlerDouYin
(){
log
ger
.
info
(
"抖音热搜开始采集..."
);
log
.
info
(
"抖音热搜开始采集..."
);
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
List
<
HotSearchList
>
douyinList
=
DouyinHotSearchCrawler
.
getMobileDouyinHotList
(
date
);
List
<
HotSearchList
>
douyinList
=
DouyinHotSearchCrawler
.
getMobileDouyinHotList
(
date
);
log
ger
.
info
(
"{}, 抖音热搜此轮采集到的数据量为:{}"
,
new
Date
(),
Integer
.
valueOf
(
douyinList
!=
null
?
douyinList
.
size
()
:
0
));
log
.
info
(
"{}, 抖音热搜此轮采集到的数据量为:{}"
,
new
Date
(),
Integer
.
valueOf
(
douyinList
!=
null
?
douyinList
.
size
()
:
0
));
TipsUtils
.
addHotList
(
HotSearchType
.
抖音热搜
.
name
(),
douyinList
);
TipsUtils
.
addHotList
(
HotSearchType
.
抖音热搜
.
name
(),
douyinList
);
log
ger
.
info
(
"抖音热搜采集结束..."
);
log
.
info
(
"抖音热搜采集结束..."
);
}
}
/**
/**
...
@@ -153,7 +151,7 @@ public class GatherTimer {
...
@@ -153,7 +151,7 @@ public class GatherTimer {
@Async
(
value
=
"myScheduler"
)
@Async
(
value
=
"myScheduler"
)
@Scheduled
(
cron
=
"0 0/5 * * * ? "
)
@Scheduled
(
cron
=
"0 0/5 * * * ? "
)
public
void
updateDouYinUrl
(){
public
void
updateDouYinUrl
(){
log
ger
.
info
(
"抖音链接更新开始..."
);
log
.
info
(
"抖音链接更新开始..."
);
HotSearchCacheDAO
hotSearchCacheDAO
=
new
HotSearchCacheDAO
();
HotSearchCacheDAO
hotSearchCacheDAO
=
new
HotSearchCacheDAO
();
List
<
HotSearchList
>
douyinList
=
DouyinHotSearchCrawler
.
list
;
List
<
HotSearchList
>
douyinList
=
DouyinHotSearchCrawler
.
list
;
if
(
douyinList
!=
null
&&
douyinList
.
size
()>
0
){
if
(
douyinList
!=
null
&&
douyinList
.
size
()>
0
){
...
@@ -168,9 +166,9 @@ public class GatherTimer {
...
@@ -168,9 +166,9 @@ public class GatherTimer {
hotSearchCacheDAO
.
updateDouyinUrl
(
document
);
hotSearchCacheDAO
.
updateDouyinUrl
(
document
);
}
}
}
}
log
ger
.
info
(
"抖音链接更新结束"
);
log
.
info
(
"抖音链接更新结束"
);
}
else
{
}
else
{
log
ger
.
info
(
"抖音链接更新失败,抖音热搜列表获取为空。"
);
log
.
info
(
"抖音链接更新失败,抖音热搜列表获取为空。"
);
}
}
}
}
...
@@ -180,12 +178,12 @@ public class GatherTimer {
...
@@ -180,12 +178,12 @@ public class GatherTimer {
@Async
(
value
=
"myScheduler"
)
@Async
(
value
=
"myScheduler"
)
@Scheduled
(
cron
=
"0 * * * * ? "
)
@Scheduled
(
cron
=
"0 * * * * ? "
)
public
void
crawlerZhihu
(){
public
void
crawlerZhihu
(){
log
ger
.
info
(
"知乎热搜开始采集..."
);
log
.
info
(
"知乎热搜开始采集..."
);
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
List
<
HotSearchList
>
zhihuList
=
ZhihuHotSearchCrawler
.
getMobileZhihuHotList
(
date
);
List
<
HotSearchList
>
zhihuList
=
ZhihuHotSearchCrawler
.
getMobileZhihuHotList
(
date
);
log
ger
.
info
(
"{}, 知乎热搜此轮采集到的数据量为:{}"
,
new
Date
(),
Integer
.
valueOf
(
zhihuList
!=
null
?
zhihuList
.
size
()
:
0
));
log
.
info
(
"{}, 知乎热搜此轮采集到的数据量为:{}"
,
new
Date
(),
Integer
.
valueOf
(
zhihuList
!=
null
?
zhihuList
.
size
()
:
0
));
TipsUtils
.
addHotList
(
HotSearchType
.
知乎热搜
.
name
(),
zhihuList
);
TipsUtils
.
addHotList
(
HotSearchType
.
知乎热搜
.
name
(),
zhihuList
);
log
ger
.
info
(
"知乎热搜采集结束..."
);
log
.
info
(
"知乎热搜采集结束..."
);
}
}
/**
/**
...
@@ -194,12 +192,12 @@ public class GatherTimer {
...
@@ -194,12 +192,12 @@ public class GatherTimer {
@Async
(
value
=
"myScheduler"
)
@Async
(
value
=
"myScheduler"
)
@Scheduled
(
cron
=
"0 * * * * ? "
)
@Scheduled
(
cron
=
"0 * * * * ? "
)
public
void
crawlerWeChat
(){
public
void
crawlerWeChat
(){
log
ger
.
info
(
"搜狗微信热词开始采集..."
);
log
.
info
(
"搜狗微信热词开始采集..."
);
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
List
<
HotSearchList
>
list
=
SougoHotSearchCrawler
.
sougoHotSearch
(
date
);
List
<
HotSearchList
>
list
=
SougoHotSearchCrawler
.
sougoHotSearch
(
date
);
log
ger
.
info
(
"{}, 搜狗微信热词采集到的数据量为:{}"
,
new
Date
(),
Integer
.
valueOf
(
list
!=
null
?
list
.
size
()
:
0
));
log
.
info
(
"{}, 搜狗微信热词采集到的数据量为:{}"
,
new
Date
(),
Integer
.
valueOf
(
list
!=
null
?
list
.
size
()
:
0
));
TipsUtils
.
addHotList
(
HotSearchType
.
搜狗微信热搜
.
name
(),
list
);
TipsUtils
.
addHotList
(
HotSearchType
.
搜狗微信热搜
.
name
(),
list
);
log
ger
.
info
(
"搜狗微信热词采集结束..."
);
log
.
info
(
"搜狗微信热词采集结束..."
);
}
}
/**
/**
...
@@ -208,12 +206,12 @@ public class GatherTimer {
...
@@ -208,12 +206,12 @@ public class GatherTimer {
//@Async(value = "myScheduler")
//@Async(value = "myScheduler")
//@Scheduled(cron = "20 * * * * ? ")
//@Scheduled(cron = "20 * * * * ? ")
public
void
ceawlerSougouHotData
(){
public
void
ceawlerSougouHotData
(){
log
ger
.
info
(
"搜狗微信热搜开始采集..."
);
log
.
info
(
"搜狗微信热搜开始采集..."
);
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
List
<
HotSearchList
>
list
=
SougoHotSearchCrawler
.
sougouHotDataCrawler
(
date
);
List
<
HotSearchList
>
list
=
SougoHotSearchCrawler
.
sougouHotDataCrawler
(
date
);
log
ger
.
info
(
"{}, 搜狗微信热搜此轮采集到的数据量为:{}"
,
new
Date
(),
list
!=
null
?
list
.
size
()
:
0
);
log
.
info
(
"{}, 搜狗微信热搜此轮采集到的数据量为:{}"
,
new
Date
(),
list
!=
null
?
list
.
size
()
:
0
);
TipsUtils
.
addHotList
(
HotSearchType
.
搜狗微信客户端热搜
.
name
(),
list
);
TipsUtils
.
addHotList
(
HotSearchType
.
搜狗微信客户端热搜
.
name
(),
list
);
log
ger
.
info
(
"搜狗微信热搜采集结束..."
);
log
.
info
(
"搜狗微信热搜采集结束..."
);
}
}
/**
/**
...
@@ -222,12 +220,12 @@ public class GatherTimer {
...
@@ -222,12 +220,12 @@ public class GatherTimer {
@Async
(
value
=
"myScheduler"
)
@Async
(
value
=
"myScheduler"
)
@Scheduled
(
cron
=
"0 * * * * ? "
)
@Scheduled
(
cron
=
"0 * * * * ? "
)
public
void
crawlerWeiBoTopic
(){
public
void
crawlerWeiBoTopic
(){
log
ger
.
info
(
"微博话题开始采集..."
);
log
.
info
(
"微博话题开始采集..."
);
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
List
<
HotSearchList
>
list
=
WeiboTopicCrawler
.
startCrawlerByPhone
(
date
);
List
<
HotSearchList
>
list
=
WeiboTopicCrawler
.
startCrawlerByPhone
(
date
);
log
ger
.
info
(
"{}, 微博话题此轮采集到的数据量为:{}"
,
new
Date
(),
Integer
.
valueOf
(
list
!=
null
?
list
.
size
()
:
0
));
log
.
info
(
"{}, 微博话题此轮采集到的数据量为:{}"
,
new
Date
(),
Integer
.
valueOf
(
list
!=
null
?
list
.
size
()
:
0
));
TipsUtils
.
addHotList
(
HotSearchType
.
微博话题
.
name
(),
list
);
TipsUtils
.
addHotList
(
HotSearchType
.
微博话题
.
name
(),
list
);
log
ger
.
info
(
"微博话题采集结束..."
);
log
.
info
(
"微博话题采集结束..."
);
}
}
/**
/**
...
@@ -313,12 +311,12 @@ public class GatherTimer {
...
@@ -313,12 +311,12 @@ public class GatherTimer {
@Async
(
value
=
"myScheduler"
)
@Async
(
value
=
"myScheduler"
)
@Scheduled
(
cron
=
"10 * * * * ? "
)
@Scheduled
(
cron
=
"10 * * * * ? "
)
public
void
crawlerTengXunVerificationHotSearch
(){
public
void
crawlerTengXunVerificationHotSearch
(){
log
ger
.
info
(
"{},腾讯较真辟谣榜开始采集"
,
new
Date
());
log
.
info
(
"{},腾讯较真辟谣榜开始采集"
,
new
Date
());
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
List
<
HotSearchList
>
list
=
TengXunCrawler
.
getTengXunVerificationList
(
date
);
List
<
HotSearchList
>
list
=
TengXunCrawler
.
getTengXunVerificationList
(
date
);
log
ger
.
info
(
"腾讯较真辟谣榜本轮采集数量:{}"
,
list
.
size
());
log
.
info
(
"腾讯较真辟谣榜本轮采集数量:{}"
,
list
.
size
());
TipsUtils
.
addHotList
(
HotSearchType
.
腾讯较真榜
.
name
(),
list
);
TipsUtils
.
addHotList
(
HotSearchType
.
腾讯较真榜
.
name
(),
list
);
log
ger
.
info
(
"{},腾讯较真辟谣榜采集结束"
,
new
Date
());
log
.
info
(
"{},腾讯较真辟谣榜采集结束"
,
new
Date
());
}
}
/**
/**
...
@@ -338,12 +336,12 @@ public class GatherTimer {
...
@@ -338,12 +336,12 @@ public class GatherTimer {
@Async
(
value
=
"myScheduler"
)
@Async
(
value
=
"myScheduler"
)
@Scheduled
(
cron
=
"20 * * * * ? "
)
@Scheduled
(
cron
=
"20 * * * * ? "
)
public
void
crawlerZhihuHotTopic
(){
public
void
crawlerZhihuHotTopic
(){
log
ger
.
info
(
"知乎热搜话题开始采集..."
);
log
.
info
(
"知乎热搜话题开始采集..."
);
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
List
<
HotSearchList
>
list
=
ZhihuTopicSearchCrawler
.
getZhihuTopicSearch
(
date
);
List
<
HotSearchList
>
list
=
ZhihuTopicSearchCrawler
.
getZhihuTopicSearch
(
date
);
log
ger
.
info
(
"{}, 知乎热搜话题此轮采集到的数据量为:{}"
,
new
Date
(),
Integer
.
valueOf
(
list
!=
null
?
list
.
size
()
:
0
));
log
.
info
(
"{}, 知乎热搜话题此轮采集到的数据量为:{}"
,
new
Date
(),
Integer
.
valueOf
(
list
!=
null
?
list
.
size
()
:
0
));
TipsUtils
.
addHotList
(
HotSearchType
.
知乎热搜榜单
.
name
(),
list
);
TipsUtils
.
addHotList
(
HotSearchType
.
知乎热搜榜单
.
name
(),
list
);
log
ger
.
info
(
"知乎热搜话题采集结束..."
);
log
.
info
(
"知乎热搜话题采集结束..."
);
}
}
/**
/**
...
@@ -352,12 +350,12 @@ public class GatherTimer {
...
@@ -352,12 +350,12 @@ public class GatherTimer {
@Async
(
value
=
"myScheduler"
)
@Async
(
value
=
"myScheduler"
)
@Scheduled
(
cron
=
"20 * * * * ? "
)
@Scheduled
(
cron
=
"20 * * * * ? "
)
public
void
crawlerWeiBoPreheat
(){
public
void
crawlerWeiBoPreheat
(){
log
ger
.
info
(
"微博预热榜开始采集..."
);
log
.
info
(
"微博预热榜开始采集..."
);
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
List
<
HotSearchList
>
list
=
WeiboHotSearchCrawler
.
weiboPreheatSearch
(
date
);
List
<
HotSearchList
>
list
=
WeiboHotSearchCrawler
.
weiboPreheatSearch
(
date
);
log
ger
.
info
(
"{},微博预热榜此轮采集到的数据量为:{}"
,
new
Date
(),
Integer
.
valueOf
(
list
!=
null
?
list
.
size
()
:
0
));
log
.
info
(
"{},微博预热榜此轮采集到的数据量为:{}"
,
new
Date
(),
Integer
.
valueOf
(
list
!=
null
?
list
.
size
()
:
0
));
TipsUtils
.
addHotList
(
HotSearchType
.
微博预热榜
.
name
(),
list
);
TipsUtils
.
addHotList
(
HotSearchType
.
微博预热榜
.
name
(),
list
);
log
ger
.
info
(
"微博预热榜采集结束..."
);
log
.
info
(
"微博预热榜采集结束..."
);
}
}
/**
/**
...
@@ -432,14 +430,14 @@ public class GatherTimer {
...
@@ -432,14 +430,14 @@ public class GatherTimer {
@Async
(
value
=
"myScheduler"
)
@Async
(
value
=
"myScheduler"
)
@Scheduled
(
cron
=
"0 0 0/3 * * ? "
)
@Scheduled
(
cron
=
"0 0 0/3 * * ? "
)
public
void
crawlerWeiBoSuperTopic
(){
public
void
crawlerWeiBoSuperTopic
(){
log
ger
.
info
(
"微博超话采集开始........"
);
log
.
info
(
"微博超话采集开始........"
);
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
WeiboSuperTopicDAO
weiboTopicDAO
=
new
WeiboSuperTopicDAO
();
WeiboSuperTopicDAO
weiboTopicDAO
=
new
WeiboSuperTopicDAO
();
List
<
WeiboSuperTopic
>
list
=
WeiboSuperTopicCrawler
.
startCrawler
();
List
<
WeiboSuperTopic
>
list
=
WeiboSuperTopicCrawler
.
startCrawler
();
log
ger
.
info
(
"{}, 微博超话此轮采集到的数据量为:{}"
,
new
Date
(),
Integer
.
valueOf
(
list
!=
null
?
list
.
size
()
:
0
));
log
.
info
(
"{}, 微博超话此轮采集到的数据量为:{}"
,
new
Date
(),
Integer
.
valueOf
(
list
!=
null
?
list
.
size
()
:
0
));
List
<
Document
>
data
=
new
ArrayList
<>();
List
<
Document
>
data
=
new
ArrayList
<>();
for
(
WeiboSuperTopic
topic
:
list
){
for
(
WeiboSuperTopic
topic
:
list
){
log
ger
.
info
(
"topic::::{}"
,
topic
);
log
.
info
(
"topic::::{}"
,
topic
);
Document
doc
=
new
Document
();
Document
doc
=
new
Document
();
doc
.
put
(
"_id"
,
topic
.
getId
());
doc
.
put
(
"_id"
,
topic
.
getId
());
doc
.
put
(
"name"
,
topic
.
getTopicName
());
doc
.
put
(
"name"
,
topic
.
getTopicName
());
...
@@ -454,7 +452,7 @@ public class GatherTimer {
...
@@ -454,7 +452,7 @@ public class GatherTimer {
data
.
add
(
doc
);
data
.
add
(
doc
);
}
}
weiboTopicDAO
.
addTopicList
(
data
);
weiboTopicDAO
.
addTopicList
(
data
);
log
ger
.
info
(
"微博话题采集结束........"
);
log
.
info
(
"微博话题采集结束........"
);
}
}
...
@@ -471,9 +469,9 @@ public class GatherTimer {
...
@@ -471,9 +469,9 @@ public class GatherTimer {
// ZhiWeiTools.sleep(500L);
// ZhiWeiTools.sleep(500L);
// }
// }
// i++;
// i++;
// log
ger
.info("更新进度:{}",i*100/documentList.size());
// log.info("更新进度:{}",i*100/documentList.size());
// }
// }
// log
ger
.info("更新结束");
// log.info("更新结束");
// }
// }
/**
/**
...
@@ -483,11 +481,11 @@ public class GatherTimer {
...
@@ -483,11 +481,11 @@ public class GatherTimer {
private
void
crawlerZhiHuChild
(
String
type
){
private
void
crawlerZhiHuChild
(
String
type
){
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
String
name
=
this
.
getTypeName
(
type
);
String
name
=
this
.
getTypeName
(
type
);
log
ger
.
info
(
"知乎{}话题热榜采集开始..."
,
name
);
log
.
info
(
"知乎{}话题热榜采集开始..."
,
name
);
List
<
HotSearchList
>
list
=
ZhihuChildHotSearchCrawler
.
getZhihuTopicSearch
(
type
,
name
,
date
);
List
<
HotSearchList
>
list
=
ZhihuChildHotSearchCrawler
.
getZhihuTopicSearch
(
type
,
name
,
date
);
log
ger
.
info
(
"{}, 知乎{}话题此轮采集到的数据量为:{}"
,
new
Date
(),
name
,
Integer
.
valueOf
(
list
!=
null
?
list
.
size
()
:
0
));
log
.
info
(
"{}, 知乎{}话题此轮采集到的数据量为:{}"
,
new
Date
(),
name
,
Integer
.
valueOf
(
list
!=
null
?
list
.
size
()
:
0
));
TipsUtils
.
addHotList
(
name
,
list
);
TipsUtils
.
addHotList
(
name
,
list
);
log
ger
.
info
(
"知乎{}话题热榜采集结束..."
,
name
);
log
.
info
(
"知乎{}话题热榜采集结束..."
,
name
);
}
}
private
String
getTypeName
(
String
type
){
private
String
getTypeName
(
String
type
){
...
@@ -513,12 +511,12 @@ public class GatherTimer {
...
@@ -513,12 +511,12 @@ public class GatherTimer {
@Async
(
value
=
"myScheduler"
)
@Async
(
value
=
"myScheduler"
)
@Scheduled
(
cron
=
"0 * * * * ? "
)
@Scheduled
(
cron
=
"0 * * * * ? "
)
public
void
crawlerKuaiShou
(){
public
void
crawlerKuaiShou
(){
log
ger
.
info
(
"快手热榜开始采集..."
);
log
.
info
(
"快手热榜开始采集..."
);
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
List
<
HotSearchList
>
kuaiShouList
=
KuaiShouHotSearchCrawler
.
KuaiShouHotSearchCrawler
(
date
);
List
<
HotSearchList
>
kuaiShouList
=
KuaiShouHotSearchCrawler
.
KuaiShouHotSearchCrawler
(
date
);
log
ger
.
info
(
"{}, 快手此轮采集到的数据量为:{}"
,
new
Date
(),
kuaiShouList
!=
null
?
kuaiShouList
.
size
()
:
0
);
log
.
info
(
"{}, 快手此轮采集到的数据量为:{}"
,
new
Date
(),
kuaiShouList
!=
null
?
kuaiShouList
.
size
()
:
0
);
TipsUtils
.
addHotList
(
HotSearchType
.
快手热榜
.
name
(),
kuaiShouList
);
TipsUtils
.
addHotList
(
HotSearchType
.
快手热榜
.
name
(),
kuaiShouList
);
log
ger
.
info
(
"快手热榜采集结束..."
);
log
.
info
(
"快手热榜采集结束..."
);
}
}
/**
/**
*淘宝热搜采集
*淘宝热搜采集
...
@@ -526,12 +524,12 @@ public class GatherTimer {
...
@@ -526,12 +524,12 @@ public class GatherTimer {
@Async
(
value
=
"myScheduler"
)
@Async
(
value
=
"myScheduler"
)
@Scheduled
(
cron
=
"0 * * * * ? "
)
@Scheduled
(
cron
=
"0 * * * * ? "
)
public
void
crawlerTaoBao
(){
public
void
crawlerTaoBao
(){
log
ger
.
info
(
"淘宝热搜开始采集..."
);
log
.
info
(
"淘宝热搜开始采集..."
);
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
List
<
HotSearchList
>
taoBaoList
=
TaoBaoHotSearchCrawler
.
taoBaoHotSearch
(
date
);
List
<
HotSearchList
>
taoBaoList
=
TaoBaoHotSearchCrawler
.
taoBaoHotSearch
(
date
);
log
ger
.
info
(
"{}, 淘宝热搜此轮采集到的数据量为:{}"
,
new
Date
(),
taoBaoList
!=
null
?
taoBaoList
.
size
()
:
0
);
log
.
info
(
"{}, 淘宝热搜此轮采集到的数据量为:{}"
,
new
Date
(),
taoBaoList
!=
null
?
taoBaoList
.
size
()
:
0
);
TipsUtils
.
addHotList
(
HotSearchType
.
淘宝热搜
.
name
(),
taoBaoList
);
TipsUtils
.
addHotList
(
HotSearchType
.
淘宝热搜
.
name
(),
taoBaoList
);
log
ger
.
info
(
"淘宝热搜采集结束..."
);
log
.
info
(
"淘宝热搜采集结束..."
);
}
}
// /**
// /**
...
@@ -540,12 +538,12 @@ public class GatherTimer {
...
@@ -540,12 +538,12 @@ public class GatherTimer {
// @Async(value = "myScheduler")
// @Async(value = "myScheduler")
// @Scheduled(cron = "0 * * * * ? ")
// @Scheduled(cron = "0 * * * * ? ")
// public void crawlerDouYinTongCheng(){
// public void crawlerDouYinTongCheng(){
// log
ger
.info("抖音同城榜开始采集...");
// log.info("抖音同城榜开始采集...");
// Date date = DateUtils.getMillSecondTime(new Date());
// Date date = DateUtils.getMillSecondTime(new Date());
// List<HotSearchList> douyinTongChengList = DouYinTongChengCrawlerTest.DouYinTongChengCrawler(date);
// List<HotSearchList> douyinTongChengList = DouYinTongChengCrawlerTest.DouYinTongChengCrawler(date);
// log
ger
.info("{}, 抖音同城榜此轮采集到的数据量为:{}", new Date(), Integer.valueOf(douyinTongChengList != null ? douyinTongChengList.size() : 0));
// log.info("{}, 抖音同城榜此轮采集到的数据量为:{}", new Date(), Integer.valueOf(douyinTongChengList != null ? douyinTongChengList.size() : 0));
// TipsUtils.addHotList(HotSearchType.抖音同城榜.name(),douyinTongChengList);
// TipsUtils.addHotList(HotSearchType.抖音同城榜.name(),douyinTongChengList);
// log
ger
.info("抖音同城榜采集结束...");
// log.info("抖音同城榜采集结束...");
// }
// }
//
//
// /**
// /**
...
@@ -554,7 +552,7 @@ public class GatherTimer {
...
@@ -554,7 +552,7 @@ public class GatherTimer {
// @Async(value = "myScheduler")
// @Async(value = "myScheduler")
// @Scheduled(cron = "0 0/5 * * * ? ")
// @Scheduled(cron = "0 0/5 * * * ? ")
// public void updateDouYinTongChengUrl(){
// public void updateDouYinTongChengUrl(){
// log
ger
.info("抖音同城链接更新开始...");
// log.info("抖音同城链接更新开始...");
// HotSearchCacheDAO hotSearchCacheDAO = new HotSearchCacheDAO();
// HotSearchCacheDAO hotSearchCacheDAO = new HotSearchCacheDAO();
// List<HotSearchList> douyinTongChengList = DouYinTongChengCrawlerTest.list;
// List<HotSearchList> douyinTongChengList = DouYinTongChengCrawlerTest.list;
// if(douyinTongChengList!=null && douyinTongChengList.size()>0){
// if(douyinTongChengList!=null && douyinTongChengList.size()>0){
...
@@ -569,9 +567,9 @@ public class GatherTimer {
...
@@ -569,9 +567,9 @@ public class GatherTimer {
// hotSearchCacheDAO.updateDouyinUrl(document);
// hotSearchCacheDAO.updateDouyinUrl(document);
// }
// }
// }
// }
// log
ger
.info("抖音同城链接更新结束");
// log.info("抖音同城链接更新结束");
// }else{
// }else{
// log
ger
.info("抖音同城链接更新失败,抖音同城榜列表获取为空。");
// log.info("抖音同城链接更新失败,抖音同城榜列表获取为空。");
// }
// }
// }
// }
/**
/**
...
@@ -580,12 +578,12 @@ public class GatherTimer {
...
@@ -580,12 +578,12 @@ public class GatherTimer {
@Async
(
value
=
"myScheduler"
)
@Async
(
value
=
"myScheduler"
)
@Scheduled
(
cron
=
"0 * * * * ? "
)
@Scheduled
(
cron
=
"0 * * * * ? "
)
public
void
crawlerWeiBoEntertainment
(){
public
void
crawlerWeiBoEntertainment
(){
log
ger
.
info
(
"微博娱乐榜开始采集..."
);
log
.
info
(
"微博娱乐榜开始采集..."
);
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
List
<
HotSearchList
>
weiBoEntertainmentList
=
WeiboEntertainmentCrawler
.
weiboEntertainmentByPhone
(
date
);
List
<
HotSearchList
>
weiBoEntertainmentList
=
WeiboEntertainmentCrawler
.
weiboEntertainmentByPhone
(
date
);
log
ger
.
info
(
"{}, 微博娱乐榜此轮采集到的数据量为:{}"
,
new
Date
(),
weiBoEntertainmentList
!=
null
?
weiBoEntertainmentList
.
size
()
:
0
);
log
.
info
(
"{}, 微博娱乐榜此轮采集到的数据量为:{}"
,
new
Date
(),
weiBoEntertainmentList
!=
null
?
weiBoEntertainmentList
.
size
()
:
0
);
TipsUtils
.
addHotList
(
HotSearchType
.
微博娱乐榜
.
name
(),
weiBoEntertainmentList
);
TipsUtils
.
addHotList
(
HotSearchType
.
微博娱乐榜
.
name
(),
weiBoEntertainmentList
);
log
ger
.
info
(
"微博娱乐榜采集结束..."
);
log
.
info
(
"微博娱乐榜采集结束..."
);
}
}
/**
/**
*微博要闻榜采集
*微博要闻榜采集
...
@@ -593,12 +591,12 @@ public class GatherTimer {
...
@@ -593,12 +591,12 @@ public class GatherTimer {
@Async
(
value
=
"myScheduler"
)
@Async
(
value
=
"myScheduler"
)
@Scheduled
(
cron
=
"0 * * * * ? "
)
@Scheduled
(
cron
=
"0 * * * * ? "
)
public
void
crawlerWeiBoNews
(){
public
void
crawlerWeiBoNews
(){
log
ger
.
info
(
"微博要闻榜开始采集..."
);
log
.
info
(
"微博要闻榜开始采集..."
);
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
List
<
HotSearchList
>
WeiboNewsList
=
WeiboNewsCrawler
.
weiboNewsByPhone
(
date
);
List
<
HotSearchList
>
WeiboNewsList
=
WeiboNewsCrawler
.
weiboNewsByPhone
(
date
);
log
ger
.
info
(
"{}, 微博要闻榜此轮采集到的数据量为:{}"
,
new
Date
(),
WeiboNewsList
!=
null
?
WeiboNewsList
.
size
()
:
0
);
log
.
info
(
"{}, 微博要闻榜此轮采集到的数据量为:{}"
,
new
Date
(),
WeiboNewsList
!=
null
?
WeiboNewsList
.
size
()
:
0
);
TipsUtils
.
addHotList
(
HotSearchType
.
微博要闻榜
.
name
(),
WeiboNewsList
);
TipsUtils
.
addHotList
(
HotSearchType
.
微博要闻榜
.
name
(),
WeiboNewsList
);
log
ger
.
info
(
"微博要闻榜采集结束..."
);
log
.
info
(
"微博要闻榜采集结束..."
);
}
}
/**
/**
*微博搜索框热词采集
*微博搜索框热词采集
...
@@ -607,9 +605,9 @@ public class GatherTimer {
...
@@ -607,9 +605,9 @@ public class GatherTimer {
//@Async(value = "myScheduler")
//@Async(value = "myScheduler")
//@Scheduled(cron = "0 * * * * ? ")
//@Scheduled(cron = "0 * * * * ? ")
public
void
crawlerWeiBoSearchBoxHotWords
(){
public
void
crawlerWeiBoSearchBoxHotWords
(){
log
ger
.
info
(
"微博搜索框热词采集开始........"
);
log
.
info
(
"微博搜索框热词采集开始........"
);
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
Date
date
=
DateUtils
.
getMillSecondTime
(
new
Date
());
WeiBoSearchBoxHotWordsCrawler
.
weiBoSearchBoxHotWords
(
date
);
WeiBoSearchBoxHotWordsCrawler
.
weiBoSearchBoxHotWords
(
date
);
log
ger
.
info
(
"微博搜索框热词采集结束........"
);
log
.
info
(
"微博搜索框热词采集结束........"
);
}
}
}
}
src/main/java/com/zhiwei/searchhotcrawler/util/TipsUtils.java
View file @
c51af150
...
@@ -4,9 +4,9 @@ import com.zhiwei.searchhotcrawler.bean.HotSearchList;
...
@@ -4,9 +4,9 @@ import com.zhiwei.searchhotcrawler.bean.HotSearchList;
import
com.zhiwei.searchhotcrawler.dao.HotSearchCacheDAO
;
import
com.zhiwei.searchhotcrawler.dao.HotSearchCacheDAO
;
import
com.zhiwei.searchhotcrawler.dao.HotSearchListDAO
;
import
com.zhiwei.searchhotcrawler.dao.HotSearchListDAO
;
import
org.bson.Document
;
import
org.bson.Document
;
import
org.
checkerframework.checker.units.qual.A
;
import
org.
apache.logging.log4j.LogManager
;
import
org.
slf
4j.Logger
;
import
org.
apache.logging.log
4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
java.util.*
;
import
java.util.*
;
import
java.util.concurrent.ExecutorService
;
import
java.util.concurrent.ExecutorService
;
...
@@ -21,7 +21,7 @@ public class TipsUtils {
...
@@ -21,7 +21,7 @@ public class TipsUtils {
private
static
String
key
=
"a8e26ce3-8aaa-4d3e-bcf6-30b81526050b"
;
private
static
String
key
=
"a8e26ce3-8aaa-4d3e-bcf6-30b81526050b"
;
private
Logger
logger
=
LoggerFactory
.
getLogger
(
TipsUtils
.
class
);
private
static
final
Logger
logger
=
LogManager
.
getLogger
(
TipsUtils
.
class
);
private
static
Map
<
String
,
Date
>
typeTips
=
new
HashMap
<>();
private
static
Map
<
String
,
Date
>
typeTips
=
new
HashMap
<>();
...
...
src/main/java/com/zhiwei/searchhotcrawler/util/WechatCodeUtil.java
View file @
c51af150
...
@@ -8,8 +8,9 @@ import java.util.Map;
...
@@ -8,8 +8,9 @@ import java.util.Map;
import
okhttp3.Request
;
import
okhttp3.Request
;
import
okhttp3.Response
;
import
okhttp3.Response
;
import
org.apache.commons.lang3.StringUtils
;
import
org.apache.commons.lang3.StringUtils
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
org.apache.logging.log4j.LogManager
;
import
org.apache.logging.log4j.Logger
;
import
com.alibaba.fastjson.JSONArray
;
import
com.alibaba.fastjson.JSONArray
;
import
com.alibaba.fastjson.JSONObject
;
import
com.alibaba.fastjson.JSONObject
;
...
@@ -21,7 +22,7 @@ import okhttp3.MediaType;
...
@@ -21,7 +22,7 @@ import okhttp3.MediaType;
import
okhttp3.RequestBody
;
import
okhttp3.RequestBody
;
public
class
WechatCodeUtil
{
public
class
WechatCodeUtil
{
private
static
Logger
log
ger
=
LoggerFactory
.
getLogger
(
WechatCodeUtil
.
class
);
private
static
Logger
log
=
LogManager
.
getLogger
(
WechatCodeUtil
.
class
);
private
static
HttpBoot
httpBoot
=
new
HttpBoot
.
Builder
().
retryTimes
(
3
).
build
();
private
static
HttpBoot
httpBoot
=
new
HttpBoot
.
Builder
().
retryTimes
(
3
).
build
();
/**
/**
* @Title: getToken
* @Title: getToken
...
@@ -43,7 +44,7 @@ public class WechatCodeUtil {
...
@@ -43,7 +44,7 @@ public class WechatCodeUtil {
result
=
response
.
body
().
string
();
result
=
response
.
body
().
string
();
}
catch
(
IOException
e
)
{
}
catch
(
IOException
e
)
{
e
.
printStackTrace
();
e
.
printStackTrace
();
log
ger
.
error
(
"获取微信公众号推送token失败,问题为:::{}"
,
e
.
fillInStackTrace
());
log
.
error
(
"获取微信公众号推送token失败,问题为:::{}"
,
e
.
fillInStackTrace
());
return
null
;
return
null
;
}
}
if
(
result
!=
null
)
{
if
(
result
!=
null
)
{
...
@@ -75,7 +76,7 @@ public class WechatCodeUtil {
...
@@ -75,7 +76,7 @@ public class WechatCodeUtil {
try
(
Response
response
=
httpBoot
.
syncCall
(
request
))
{
try
(
Response
response
=
httpBoot
.
syncCall
(
request
))
{
htmlBody
=
response
.
body
().
string
();
htmlBody
=
response
.
body
().
string
();
}
catch
(
Exception
e
)
{
}
catch
(
Exception
e
)
{
log
ger
.
error
(
"消息推送失败,错误为::{}"
,
e
.
fillInStackTrace
());
log
.
error
(
"消息推送失败,错误为::{}"
,
e
.
fillInStackTrace
());
msgid
=
0
;
msgid
=
0
;
}
}
if
(
StringUtils
.
isNotBlank
(
htmlBody
))
{
if
(
StringUtils
.
isNotBlank
(
htmlBody
))
{
...
@@ -85,7 +86,7 @@ public class WechatCodeUtil {
...
@@ -85,7 +86,7 @@ public class WechatCodeUtil {
msgid
=
jsonObject
.
getIntValue
(
"msgid"
);
msgid
=
jsonObject
.
getIntValue
(
"msgid"
);
}
else
{
}
else
{
msgid
=
0
;
msgid
=
0
;
log
ger
.
info
(
"消息推送失败,错误为::{}"
,
jsonObject
.
toString
());
log
.
info
(
"消息推送失败,错误为::{}"
,
jsonObject
.
toString
());
}
}
}
}
}
}
...
@@ -117,7 +118,7 @@ public class WechatCodeUtil {
...
@@ -117,7 +118,7 @@ public class WechatCodeUtil {
try
(
Response
response
=
httpBoot
.
syncCall
(
request
))
{
try
(
Response
response
=
httpBoot
.
syncCall
(
request
))
{
htmlBody
=
response
.
body
().
string
();
htmlBody
=
response
.
body
().
string
();
}
catch
(
IOException
e
){
}
catch
(
IOException
e
){
log
ger
.
error
(
"页面连接获取失败"
,
e
);
log
.
error
(
"页面连接获取失败"
,
e
);
return
null
;
return
null
;
}
}
if
(
StringUtils
.
isNotBlank
(
htmlBody
)
&&
htmlBody
.
contains
(
"data"
))
{
if
(
StringUtils
.
isNotBlank
(
htmlBody
)
&&
htmlBody
.
contains
(
"data"
))
{
...
@@ -126,12 +127,12 @@ public class WechatCodeUtil {
...
@@ -126,12 +127,12 @@ public class WechatCodeUtil {
if
(
jsonObject
.
containsKey
(
"data"
))
{
if
(
jsonObject
.
containsKey
(
"data"
))
{
return
(
List
<
String
>)
jsonObject
.
getJSONObject
(
"data"
).
getObject
(
"openid"
,
List
.
class
);
return
(
List
<
String
>)
jsonObject
.
getJSONObject
(
"data"
).
getObject
(
"openid"
,
List
.
class
);
}
else
{
}
else
{
log
ger
.
info
(
"拉取用户列表时,出现问题{}"
,
jsonObject
);
log
.
info
(
"拉取用户列表时,出现问题{}"
,
jsonObject
);
}
}
}
}
}
}
}
else
{
}
else
{
log
ger
.
info
(
"token 获取失败"
);
log
.
info
(
"token 获取失败"
);
}
}
}
catch
(
Exception
e
)
{
}
catch
(
Exception
e
)
{
e
.
printStackTrace
();
e
.
printStackTrace
();
...
@@ -156,7 +157,7 @@ public class WechatCodeUtil {
...
@@ -156,7 +157,7 @@ public class WechatCodeUtil {
try
(
Response
response
=
httpBoot
.
syncCall
(
request
)){
try
(
Response
response
=
httpBoot
.
syncCall
(
request
)){
htmlBody
=
response
.
body
().
string
();
htmlBody
=
response
.
body
().
string
();
}
catch
(
IOException
e
){
}
catch
(
IOException
e
){
log
ger
.
error
(
"页面链接获取失败"
,
e
);
log
.
error
(
"页面链接获取失败"
,
e
);
return
null
;
return
null
;
}
}
if
(
StringUtils
.
isNotBlank
(
htmlBody
)
&&
htmlBody
.
contains
(
"data"
))
{
if
(
StringUtils
.
isNotBlank
(
htmlBody
)
&&
htmlBody
.
contains
(
"data"
))
{
...
@@ -165,12 +166,12 @@ public class WechatCodeUtil {
...
@@ -165,12 +166,12 @@ public class WechatCodeUtil {
if
(
jsonObject
.
containsKey
(
"data"
))
{
if
(
jsonObject
.
containsKey
(
"data"
))
{
return
(
List
<
String
>)
jsonObject
.
getJSONObject
(
"data"
).
getObject
(
"openid"
,
List
.
class
);
return
(
List
<
String
>)
jsonObject
.
getJSONObject
(
"data"
).
getObject
(
"openid"
,
List
.
class
);
}
else
{
}
else
{
log
ger
.
info
(
"拉取用户列表时,出现问题{}"
,
jsonObject
);
log
.
info
(
"拉取用户列表时,出现问题{}"
,
jsonObject
);
}
}
}
}
}
}
}
else
{
}
else
{
log
ger
.
info
(
"token 获取失败"
);
log
.
info
(
"token 获取失败"
);
}
}
}
catch
(
Exception
e
)
{
}
catch
(
Exception
e
)
{
e
.
printStackTrace
();
e
.
printStackTrace
();
...
@@ -199,7 +200,7 @@ public class WechatCodeUtil {
...
@@ -199,7 +200,7 @@ public class WechatCodeUtil {
try
(
Response
response
=
httpBoot
.
syncCall
(
request
))
{
try
(
Response
response
=
httpBoot
.
syncCall
(
request
))
{
htmlBody
=
response
.
body
().
string
();
htmlBody
=
response
.
body
().
string
();
}
catch
(
IOException
e
)
{
}
catch
(
IOException
e
)
{
log
ger
.
error
(
"获取分组id时出现错误"
,
e
.
fillInStackTrace
());
log
.
error
(
"获取分组id时出现错误"
,
e
.
fillInStackTrace
());
return
null
;
return
null
;
}
}
if
(
htmlBody
!=
null
)
{
if
(
htmlBody
!=
null
)
{
...
@@ -232,7 +233,7 @@ public class WechatCodeUtil {
...
@@ -232,7 +233,7 @@ public class WechatCodeUtil {
try
(
Response
response
=
httpBoot
.
syncCall
(
request
))
{
try
(
Response
response
=
httpBoot
.
syncCall
(
request
))
{
htmlBody
=
response
.
body
().
string
();
htmlBody
=
response
.
body
().
string
();
}
catch
(
IOException
e
)
{
}
catch
(
IOException
e
)
{
log
ger
.
error
(
"获取分组id时出现错误"
,
e
.
fillInStackTrace
());
log
.
error
(
"获取分组id时出现错误"
,
e
.
fillInStackTrace
());
return
null
;
return
null
;
}
}
if
(
htmlBody
!=
null
)
{
if
(
htmlBody
!=
null
)
{
...
@@ -245,7 +246,7 @@ public class WechatCodeUtil {
...
@@ -245,7 +246,7 @@ public class WechatCodeUtil {
resultMap
.
put
(
name
,
id
);
resultMap
.
put
(
name
,
id
);
}
}
}
else
{
}
else
{
log
ger
.
info
(
"获取分组id时出现错误,数据为:::{}"
,
htmlBody
);
log
.
info
(
"获取分组id时出现错误,数据为:::{}"
,
htmlBody
);
}
}
}
}
return
resultMap
;
return
resultMap
;
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment