Commit c51af150 by chenweitao

修复日志漏洞

parent 2e1ca3af
package com.zhiwei.searchhotcrawler.cache; package com.zhiwei.searchhotcrawler.cache;
import lombok.extern.log4j.Log4j2; import lombok.extern.log4j.Log4j2;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.zhiwei.tools.tools.ZhiWeiTools; import com.zhiwei.tools.tools.ZhiWeiTools;
...@@ -26,7 +24,7 @@ public class CacheListener { ...@@ -26,7 +24,7 @@ public class CacheListener {
} }
ZhiWeiTools.sleep(500); ZhiWeiTools.sleep(500);
} }
} }
}.start(); }.start();
} }
} }
...@@ -129,7 +129,7 @@ public class BaiDuHotSearchCrawler { ...@@ -129,7 +129,7 @@ public class BaiDuHotSearchCrawler {
String everurl = element.select("td.keyword").select("a.list-title").attr("href"); String everurl = element.select("td.keyword").select("a.list-title").attr("href");
// 获取关键词(String) // 获取关键词(String)
String kw = element.select("td.keyword").select("a.list-title").text(); String kw = element.select("td.keyword").select("a.list-title").text();
// logger.info("关键词:{}", kw); // log.info("关键词:{}", kw);
//从连接中获取正确编码关键词 //从连接中获取正确编码关键词
try { try {
if (!everurl.isEmpty()) { if (!everurl.isEmpty()) {
......
...@@ -9,8 +9,7 @@ import lombok.extern.log4j.Log4j2; ...@@ -9,8 +9,7 @@ import lombok.extern.log4j.Log4j2;
import okhttp3.Request; import okhttp3.Request;
import okhttp3.Response; import okhttp3.Response;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
...@@ -71,7 +70,7 @@ public class DouyinHotSearchCrawler { ...@@ -71,7 +70,7 @@ public class DouyinHotSearchCrawler {
hotValueStr = wl.getString("hot_value"); hotValueStr = wl.getString("hot_value");
Long hotValue = null; Long hotValue = null;
hotValue = Long.valueOf(hotValueStr); hotValue = Long.valueOf(hotValueStr);
// logger.info("热度为:::{}", hot_value); // log.info("热度为:::{}", hot_value);
HotSearchList douyin = new HotSearchList(null, word, hotValue, position, HotSearchType.抖音热搜.name(),date); HotSearchList douyin = new HotSearchList(null, word, hotValue, position, HotSearchType.抖音热搜.name(),date);
list.add(douyin); list.add(douyin);
} }
......
...@@ -99,7 +99,7 @@ public class HotSearch36KrCrawler { ...@@ -99,7 +99,7 @@ public class HotSearch36KrCrawler {
rank++; rank++;
// 获取关键词(String) // 获取关键词(String)
String keyWord = element.select("p.title-wrapper").select("a.article-item-title").text(); String keyWord = element.select("p.title-wrapper").select("a.article-item-title").text();
// logger.info("关键词:{}", kw); // log.info("关键词:{}", kw);
// 获取关键词相关链接everurl(String) // 获取关键词相关链接everurl(String)
String everurl = element.select("p.title-wrapper").select("a.article-item-title").attr("href"); String everurl = element.select("p.title-wrapper").select("a.article-item-title").attr("href");
// 获取搜索指数count(int) // 获取搜索指数count(int)
......
...@@ -13,8 +13,7 @@ import org.jsoup.Jsoup; ...@@ -13,8 +13,7 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.jsoup.select.Elements; import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.zhiwei.crawler.core.HttpBoot; import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.core.proxy.ProxyHolder; import com.zhiwei.crawler.core.proxy.ProxyHolder;
...@@ -70,7 +69,7 @@ public class SougoHotSearchCrawler { ...@@ -70,7 +69,7 @@ public class SougoHotSearchCrawler {
// 获取关键词(String) // 获取关键词(String)
String kw = element.select("li").select("a").attr("title"); String kw = element.select("li").select("a").attr("title");
// logger.info("关键词:{}", kw); // log.info("关键词:{}", kw);
String everurl = element.select("li").select("a").attr("href"); String everurl = element.select("li").select("a").attr("href");
......
...@@ -13,8 +13,7 @@ import lombok.extern.log4j.Log4j2; ...@@ -13,8 +13,7 @@ import lombok.extern.log4j.Log4j2;
import okhttp3.Request; import okhttp3.Request;
import okhttp3.Response; import okhttp3.Response;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
......
...@@ -18,8 +18,7 @@ import org.apache.commons.lang3.StringUtils; ...@@ -18,8 +18,7 @@ import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.jsoup.select.Elements; import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*; import java.util.*;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
......
...@@ -9,8 +9,7 @@ import java.util.Map; ...@@ -9,8 +9,7 @@ import java.util.Map;
import lombok.extern.log4j.Log4j2; import lombok.extern.log4j.Log4j2;
import okhttp3.Request; import okhttp3.Request;
import okhttp3.Response; import okhttp3.Response;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
......
...@@ -99,7 +99,7 @@ public class HotSearch36KrCrawlerTest { ...@@ -99,7 +99,7 @@ public class HotSearch36KrCrawlerTest {
rank++; rank++;
// 获取关键词(String) // 获取关键词(String)
String keyWord = element.select("p.title-wrapper").select("a.article-item-title").text(); String keyWord = element.select("p.title-wrapper").select("a.article-item-title").text();
// logger.info("关键词:{}", kw); // log.info("关键词:{}", kw);
// 获取关键词相关链接everurl(String) // 获取关键词相关链接everurl(String)
String everurl = element.select("p.title-wrapper").select("a.article-item-title").attr("href"); String everurl = element.select("p.title-wrapper").select("a.article-item-title").attr("href");
// 获取搜索指数count(int) // 获取搜索指数count(int)
......
...@@ -12,8 +12,6 @@ import com.zhiwei.searchhotcrawler.dao.HotSearchCacheDAO; ...@@ -12,8 +12,6 @@ import com.zhiwei.searchhotcrawler.dao.HotSearchCacheDAO;
import com.zhiwei.searchhotcrawler.util.TipsUtils; import com.zhiwei.searchhotcrawler.util.TipsUtils;
import lombok.extern.log4j.Log4j2; import lombok.extern.log4j.Log4j2;
import org.bson.Document; import org.bson.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.mongodb.BasicDBObject; import com.mongodb.BasicDBObject;
import com.mongodb.DBObject; import com.mongodb.DBObject;
...@@ -39,8 +37,8 @@ public class BaiduHotSearchRun extends Thread{ ...@@ -39,8 +37,8 @@ public class BaiduHotSearchRun extends Thread{
ZhiWeiTools.sleep(50); ZhiWeiTools.sleep(50);
} }
} }
private void getHotList() { private void getHotList() {
log.info("百度风云榜采集开始........"); log.info("百度风云榜采集开始........");
// HotSearchListDAO hotSearchDAO = new HotSearchListDAO(); // HotSearchListDAO hotSearchDAO = new HotSearchListDAO();
...@@ -70,4 +68,4 @@ public class BaiduHotSearchRun extends Thread{ ...@@ -70,4 +68,4 @@ public class BaiduHotSearchRun extends Thread{
// log.info("知乎话题采集结束........"); // log.info("知乎话题采集结束........");
} }
} }
\ No newline at end of file
...@@ -12,8 +12,7 @@ import com.zhiwei.searchhotcrawler.dao.HotSearchCacheDAO; ...@@ -12,8 +12,7 @@ import com.zhiwei.searchhotcrawler.dao.HotSearchCacheDAO;
import com.zhiwei.searchhotcrawler.util.TipsUtils; import com.zhiwei.searchhotcrawler.util.TipsUtils;
import lombok.extern.log4j.Log4j2; import lombok.extern.log4j.Log4j2;
import org.bson.Document; import org.bson.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.mongodb.BasicDBObject; import com.mongodb.BasicDBObject;
import com.mongodb.DBObject; import com.mongodb.DBObject;
...@@ -26,7 +25,7 @@ import com.zhiwei.tools.tools.ZhiWeiTools; ...@@ -26,7 +25,7 @@ import com.zhiwei.tools.tools.ZhiWeiTools;
public class DouyinHotSearchRun extends Thread{ public class DouyinHotSearchRun extends Thread{
public static List<HotSearchList> list = new ArrayList<>(); public static List<HotSearchList> list = new ArrayList<>();
@Override @Override
public void run() { public void run() {
boolean f = true; boolean f = true;
...@@ -41,7 +40,7 @@ public class DouyinHotSearchRun extends Thread{ ...@@ -41,7 +40,7 @@ public class DouyinHotSearchRun extends Thread{
ZhiWeiTools.sleep(50); ZhiWeiTools.sleep(50);
} }
} }
/** /**
* 获取热搜列表 * 获取热搜列表
* TODO * TODO
......
...@@ -10,8 +10,7 @@ import com.zhiwei.searchhotcrawler.dao.HotSearchListDAO; ...@@ -10,8 +10,7 @@ import com.zhiwei.searchhotcrawler.dao.HotSearchListDAO;
import com.zhiwei.searchhotcrawler.util.TipsUtils; import com.zhiwei.searchhotcrawler.util.TipsUtils;
import lombok.extern.log4j.Log4j2; import lombok.extern.log4j.Log4j2;
import org.bson.Document; import org.bson.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.mongodb.BasicDBObject; import com.mongodb.BasicDBObject;
import com.mongodb.DBObject; import com.mongodb.DBObject;
...@@ -36,8 +35,8 @@ public class SougoHotSearchRun extends Thread { ...@@ -36,8 +35,8 @@ public class SougoHotSearchRun extends Thread {
ZhiWeiTools.sleep(50); ZhiWeiTools.sleep(50);
} }
} }
private void getHotList() { private void getHotList() {
// HotSearchListDAO hotSearchDAO = new HotSearchListDAO(); // HotSearchListDAO hotSearchDAO = new HotSearchListDAO();
// HotSearchCacheDAO hotSearchCacheDAO = new HotSearchCacheDAO(); // HotSearchCacheDAO hotSearchCacheDAO = new HotSearchCacheDAO();
......
...@@ -9,8 +9,7 @@ import com.zhiwei.searchhotcrawler.dao.HotSearchCacheDAO; ...@@ -9,8 +9,7 @@ import com.zhiwei.searchhotcrawler.dao.HotSearchCacheDAO;
import com.zhiwei.searchhotcrawler.util.TipsUtils; import com.zhiwei.searchhotcrawler.util.TipsUtils;
import lombok.extern.log4j.Log4j2; import lombok.extern.log4j.Log4j2;
import org.bson.Document; import org.bson.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.mongodb.BasicDBObject; import com.mongodb.BasicDBObject;
import com.mongodb.DBObject; import com.mongodb.DBObject;
...@@ -21,7 +20,7 @@ import com.zhiwei.tools.tools.ZhiWeiTools; ...@@ -21,7 +20,7 @@ import com.zhiwei.tools.tools.ZhiWeiTools;
@Log4j2 @Log4j2
public class ZhihuHotSearchRun extends Thread{ public class ZhihuHotSearchRun extends Thread{
@Override @Override
public void run() { public void run() {
boolean f = true; boolean f = true;
...@@ -37,7 +36,7 @@ public class ZhihuHotSearchRun extends Thread{ ...@@ -37,7 +36,7 @@ public class ZhihuHotSearchRun extends Thread{
} }
} }
private void getHotList() { private void getHotList() {
// HotSearchListDAO hotSearchDAO = new HotSearchListDAO(); // HotSearchListDAO hotSearchDAO = new HotSearchListDAO();
// HotSearchCacheDAO hotSearchCacheDAO = new HotSearchCacheDAO(); // HotSearchCacheDAO hotSearchCacheDAO = new HotSearchCacheDAO();
......
...@@ -14,9 +14,8 @@ import com.zhiwei.searchhotcrawler.timer.TouTiaoExecutor; ...@@ -14,9 +14,8 @@ import com.zhiwei.searchhotcrawler.timer.TouTiaoExecutor;
import com.zhiwei.searchhotcrawler.util.DateUtils; import com.zhiwei.searchhotcrawler.util.DateUtils;
import com.zhiwei.searchhotcrawler.util.TipsUtils; import com.zhiwei.searchhotcrawler.util.TipsUtils;
import com.zhiwei.tools.tools.ZhiWeiTools; import com.zhiwei.tools.tools.ZhiWeiTools;
import lombok.extern.log4j.Log4j2;
import org.bson.Document; import org.bson.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.scheduling.annotation.Async; import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.annotation.EnableScheduling;
...@@ -28,10 +27,9 @@ import java.util.*; ...@@ -28,10 +27,9 @@ import java.util.*;
@Component @Component
@EnableScheduling @EnableScheduling
@EnableAsync @EnableAsync
@Log4j2
public class GatherTimer { public class GatherTimer {
private Logger logger = LoggerFactory.getLogger(GatherTimer.class);
private RedisDao redisDao = new RedisDao(); private RedisDao redisDao = new RedisDao();
/** 知乎数码子分类 */ /** 知乎数码子分类 */
private String DIGITAL = "digital"; private String DIGITAL = "digital";
...@@ -47,21 +45,21 @@ public class GatherTimer { ...@@ -47,21 +45,21 @@ public class GatherTimer {
@Async(value = "myScheduler") @Async(value = "myScheduler")
@Scheduled(cron = "0 * * * * ?") @Scheduled(cron = "0 * * * * ?")
public void crawlerHuXiu() { public void crawlerHuXiu() {
logger.info("虎嗅热文推荐开始采集..."); log.info("虎嗅热文推荐开始采集...");
Date date = DateUtils.getMillSecondTime(new Date()); Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> huXiuList = HuXiuHotSearchCrawler.HuXiuHotArticleRecommended(date); List<HotSearchList> huXiuList = HuXiuHotSearchCrawler.HuXiuHotArticleRecommended(date);
logger.info("{}, 虎嗅热文推荐此轮采集到的数据量为:{}", new Date(), Integer.valueOf(huXiuList != null ? huXiuList.size() : 0)); log.info("{}, 虎嗅热文推荐此轮采集到的数据量为:{}", new Date(), Integer.valueOf(huXiuList != null ? huXiuList.size() : 0));
TipsUtils.addHotList(HotSearchType.虎嗅热文推荐.name(), huXiuList); TipsUtils.addHotList(HotSearchType.虎嗅热文推荐.name(), huXiuList);
logger.info("虎嗅热文推荐采集结束..."); log.info("虎嗅热文推荐采集结束...");
/** /**
* 36氪人气榜的采集 * 36氪人气榜的采集
*/ */
logger.info("36氪人气榜开始采集..."); log.info("36氪人气榜开始采集...");
List<HotSearchList> list36Kr = HotSearch36KrCrawler.hotSearch36Kr(date); List<HotSearchList> list36Kr = HotSearch36KrCrawler.hotSearch36Kr(date);
logger.info("{}, 36氪人气榜此轮采集到的数据量为:{}", new Date(), Integer.valueOf(list36Kr != null ? list36Kr.size() : 0)); log.info("{}, 36氪人气榜此轮采集到的数据量为:{}", new Date(), Integer.valueOf(list36Kr != null ? list36Kr.size() : 0));
TipsUtils.addHotList(HotSearchType.人气榜36.name(), list36Kr); TipsUtils.addHotList(HotSearchType.人气榜36.name(), list36Kr);
logger.info("36氪人气榜采集结束..."); log.info("36氪人气榜采集结束...");
} }
/** /**
...@@ -70,12 +68,12 @@ public class GatherTimer { ...@@ -70,12 +68,12 @@ public class GatherTimer {
@Async(value = "myScheduler") @Async(value = "myScheduler")
@Scheduled(cron = "0 * * * * ? ") @Scheduled(cron = "0 * * * * ? ")
public void crawlerWeiBo(){ public void crawlerWeiBo(){
logger.info("微博热搜开始采集..."); log.info("微博热搜开始采集...");
Date date = DateUtils.getMillSecondTime(new Date()); Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> weiboList = WeiboHotSearchCrawler.weiboHotSearchByPhone(date); List<HotSearchList> weiboList = WeiboHotSearchCrawler.weiboHotSearchByPhone(date);
logger.info("{}, 微博此轮采集到的数据量为:{}", new Date(), weiboList != null ? weiboList.size() : 0); log.info("{}, 微博此轮采集到的数据量为:{}", new Date(), weiboList != null ? weiboList.size() : 0);
TipsUtils.addHotList(HotSearchType.微博热搜.name(),weiboList); TipsUtils.addHotList(HotSearchType.微博热搜.name(),weiboList);
logger.info("微博热搜采集结束..."); log.info("微博热搜采集结束...");
} }
/** /**
...@@ -84,7 +82,7 @@ public class GatherTimer { ...@@ -84,7 +82,7 @@ public class GatherTimer {
@Async(value = "myScheduler") @Async(value = "myScheduler")
@Scheduled(cron = "45 0/10 * * * ? ") @Scheduled(cron = "45 0/10 * * * ? ")
public void updateWeiBo(){ public void updateWeiBo(){
logger.info("微博热搜导语更新..."); log.info("微博热搜导语更新...");
HotSearchCacheDAO hotSearchCacheDAO = new HotSearchCacheDAO(); HotSearchCacheDAO hotSearchCacheDAO = new HotSearchCacheDAO();
Set<String> hotSearchIdSet = redisDao.getRedisSetData(RedisConfig.WEIBO_HOTSEARCHIDS); Set<String> hotSearchIdSet = redisDao.getRedisSetData(RedisConfig.WEIBO_HOTSEARCHIDS);
redisDao.removeRedis(RedisConfig.WEIBO_HOTSEARCHIDS); redisDao.removeRedis(RedisConfig.WEIBO_HOTSEARCHIDS);
...@@ -100,7 +98,7 @@ public class GatherTimer { ...@@ -100,7 +98,7 @@ public class GatherTimer {
ZhiWeiTools.sleep(3000L); ZhiWeiTools.sleep(3000L);
} }
} }
logger.info("微博热搜导语更新结束..."); log.info("微博热搜导语更新结束...");
} }
/** /**
...@@ -109,13 +107,13 @@ public class GatherTimer { ...@@ -109,13 +107,13 @@ public class GatherTimer {
@Async(value = "myScheduler") @Async(value = "myScheduler")
@Scheduled(cron = "0 * * * * ? ") @Scheduled(cron = "0 * * * * ? ")
public void crawlerTouTiao(){ public void crawlerTouTiao(){
logger.info("今日头条热搜开始采集..."); log.info("今日头条热搜开始采集...");
Date date = DateUtils.getMillSecondTime(new Date()); Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> toutiaoList = ToutiaoHotSearchCrawler.toutiaoHotSearchByPhone(date); List<HotSearchList> toutiaoList = ToutiaoHotSearchCrawler.toutiaoHotSearchByPhone(date);
logger.info("{}, 今日头条此轮采集到的数据量为:{}", new Date(), toutiaoList != null ? toutiaoList.size() : 0); log.info("{}, 今日头条此轮采集到的数据量为:{}", new Date(), toutiaoList != null ? toutiaoList.size() : 0);
TipsUtils.addHotList(HotSearchType.今日头条热搜.name(),toutiaoList); TipsUtils.addHotList(HotSearchType.今日头条热搜.name(),toutiaoList);
logger.info("今日头条热搜采集结束..."); log.info("今日头条热搜采集结束...");
logger.info("今日头条热搜详情趋势阅读量更新..."); log.info("今日头条热搜详情趋势阅读量更新...");
TouTiaoExecutor.countTouTiaoReadCount(toutiaoList); TouTiaoExecutor.countTouTiaoReadCount(toutiaoList);
} }
...@@ -125,12 +123,12 @@ public class GatherTimer { ...@@ -125,12 +123,12 @@ public class GatherTimer {
@Async(value = "myScheduler") @Async(value = "myScheduler")
@Scheduled(cron = "0 * * * * ? ") @Scheduled(cron = "0 * * * * ? ")
public void crawlerBaiDu(){ public void crawlerBaiDu(){
logger.info("百度热搜开始采集..."); log.info("百度热搜开始采集...");
Date date = DateUtils.getMillSecondTime(new Date()); Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> baiduList = BaiDuHotSearchCrawler.baiduHotSearch(date); List<HotSearchList> baiduList = BaiDuHotSearchCrawler.baiduHotSearch(date);
logger.info("{}, 百度热搜此轮采集到的数据量为:{}", new Date(), Integer.valueOf(baiduList != null ? baiduList.size() : 0)); log.info("{}, 百度热搜此轮采集到的数据量为:{}", new Date(), Integer.valueOf(baiduList != null ? baiduList.size() : 0));
TipsUtils.addHotList(HotSearchType.百度热搜.name(),baiduList); TipsUtils.addHotList(HotSearchType.百度热搜.name(),baiduList);
logger.info("百度热搜采集结束..."); log.info("百度热搜采集结束...");
} }
/** /**
...@@ -139,12 +137,12 @@ public class GatherTimer { ...@@ -139,12 +137,12 @@ public class GatherTimer {
@Async(value = "myScheduler") @Async(value = "myScheduler")
@Scheduled(cron = "0 * * * * ? ") @Scheduled(cron = "0 * * * * ? ")
public void crawlerDouYin(){ public void crawlerDouYin(){
logger.info("抖音热搜开始采集..."); log.info("抖音热搜开始采集...");
Date date = DateUtils.getMillSecondTime(new Date()); Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> douyinList = DouyinHotSearchCrawler.getMobileDouyinHotList(date); List<HotSearchList> douyinList = DouyinHotSearchCrawler.getMobileDouyinHotList(date);
logger.info("{}, 抖音热搜此轮采集到的数据量为:{}", new Date(), Integer.valueOf(douyinList != null ? douyinList.size() : 0)); log.info("{}, 抖音热搜此轮采集到的数据量为:{}", new Date(), Integer.valueOf(douyinList != null ? douyinList.size() : 0));
TipsUtils.addHotList(HotSearchType.抖音热搜.name(),douyinList); TipsUtils.addHotList(HotSearchType.抖音热搜.name(),douyinList);
logger.info("抖音热搜采集结束..."); log.info("抖音热搜采集结束...");
} }
/** /**
...@@ -153,7 +151,7 @@ public class GatherTimer { ...@@ -153,7 +151,7 @@ public class GatherTimer {
@Async(value = "myScheduler") @Async(value = "myScheduler")
@Scheduled(cron = "0 0/5 * * * ? ") @Scheduled(cron = "0 0/5 * * * ? ")
public void updateDouYinUrl(){ public void updateDouYinUrl(){
logger.info("抖音链接更新开始..."); log.info("抖音链接更新开始...");
HotSearchCacheDAO hotSearchCacheDAO = new HotSearchCacheDAO(); HotSearchCacheDAO hotSearchCacheDAO = new HotSearchCacheDAO();
List<HotSearchList> douyinList = DouyinHotSearchCrawler.list; List<HotSearchList> douyinList = DouyinHotSearchCrawler.list;
if(douyinList!=null && douyinList.size()>0){ if(douyinList!=null && douyinList.size()>0){
...@@ -168,9 +166,9 @@ public class GatherTimer { ...@@ -168,9 +166,9 @@ public class GatherTimer {
hotSearchCacheDAO.updateDouyinUrl(document); hotSearchCacheDAO.updateDouyinUrl(document);
} }
} }
logger.info("抖音链接更新结束"); log.info("抖音链接更新结束");
}else{ }else{
logger.info("抖音链接更新失败,抖音热搜列表获取为空。"); log.info("抖音链接更新失败,抖音热搜列表获取为空。");
} }
} }
...@@ -180,12 +178,12 @@ public class GatherTimer { ...@@ -180,12 +178,12 @@ public class GatherTimer {
@Async(value = "myScheduler") @Async(value = "myScheduler")
@Scheduled(cron = "0 * * * * ? ") @Scheduled(cron = "0 * * * * ? ")
public void crawlerZhihu(){ public void crawlerZhihu(){
logger.info("知乎热搜开始采集..."); log.info("知乎热搜开始采集...");
Date date = DateUtils.getMillSecondTime(new Date()); Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> zhihuList = ZhihuHotSearchCrawler.getMobileZhihuHotList(date); List<HotSearchList> zhihuList = ZhihuHotSearchCrawler.getMobileZhihuHotList(date);
logger.info("{}, 知乎热搜此轮采集到的数据量为:{}", new Date(), Integer.valueOf(zhihuList != null ? zhihuList.size() : 0)); log.info("{}, 知乎热搜此轮采集到的数据量为:{}", new Date(), Integer.valueOf(zhihuList != null ? zhihuList.size() : 0));
TipsUtils.addHotList(HotSearchType.知乎热搜.name(),zhihuList); TipsUtils.addHotList(HotSearchType.知乎热搜.name(),zhihuList);
logger.info("知乎热搜采集结束..."); log.info("知乎热搜采集结束...");
} }
/** /**
...@@ -194,12 +192,12 @@ public class GatherTimer { ...@@ -194,12 +192,12 @@ public class GatherTimer {
@Async(value = "myScheduler") @Async(value = "myScheduler")
@Scheduled(cron = "0 * * * * ? ") @Scheduled(cron = "0 * * * * ? ")
public void crawlerWeChat(){ public void crawlerWeChat(){
logger.info("搜狗微信热词开始采集..."); log.info("搜狗微信热词开始采集...");
Date date = DateUtils.getMillSecondTime(new Date()); Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> list = SougoHotSearchCrawler.sougoHotSearch(date); List<HotSearchList> list = SougoHotSearchCrawler.sougoHotSearch(date);
logger.info("{}, 搜狗微信热词采集到的数据量为:{}", new Date(), Integer.valueOf(list != null ? list.size() : 0)); log.info("{}, 搜狗微信热词采集到的数据量为:{}", new Date(), Integer.valueOf(list != null ? list.size() : 0));
TipsUtils.addHotList(HotSearchType.搜狗微信热搜.name(),list); TipsUtils.addHotList(HotSearchType.搜狗微信热搜.name(),list);
logger.info("搜狗微信热词采集结束..."); log.info("搜狗微信热词采集结束...");
} }
/** /**
...@@ -208,12 +206,12 @@ public class GatherTimer { ...@@ -208,12 +206,12 @@ public class GatherTimer {
//@Async(value = "myScheduler") //@Async(value = "myScheduler")
//@Scheduled(cron = "20 * * * * ? ") //@Scheduled(cron = "20 * * * * ? ")
public void ceawlerSougouHotData(){ public void ceawlerSougouHotData(){
logger.info("搜狗微信热搜开始采集..."); log.info("搜狗微信热搜开始采集...");
Date date = DateUtils.getMillSecondTime(new Date()); Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> list = SougoHotSearchCrawler.sougouHotDataCrawler(date); List<HotSearchList> list = SougoHotSearchCrawler.sougouHotDataCrawler(date);
logger.info("{}, 搜狗微信热搜此轮采集到的数据量为:{}", new Date(), list != null ? list.size() : 0); log.info("{}, 搜狗微信热搜此轮采集到的数据量为:{}", new Date(), list != null ? list.size() : 0);
TipsUtils.addHotList(HotSearchType.搜狗微信客户端热搜.name(),list); TipsUtils.addHotList(HotSearchType.搜狗微信客户端热搜.name(),list);
logger.info("搜狗微信热搜采集结束..."); log.info("搜狗微信热搜采集结束...");
} }
/** /**
...@@ -222,12 +220,12 @@ public class GatherTimer { ...@@ -222,12 +220,12 @@ public class GatherTimer {
@Async(value = "myScheduler") @Async(value = "myScheduler")
@Scheduled(cron = "0 * * * * ? ") @Scheduled(cron = "0 * * * * ? ")
public void crawlerWeiBoTopic(){ public void crawlerWeiBoTopic(){
logger.info("微博话题开始采集..."); log.info("微博话题开始采集...");
Date date = DateUtils.getMillSecondTime(new Date()); Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> list = WeiboTopicCrawler.startCrawlerByPhone(date); List<HotSearchList> list = WeiboTopicCrawler.startCrawlerByPhone(date);
logger.info("{}, 微博话题此轮采集到的数据量为:{}", new Date(), Integer.valueOf(list != null ? list.size() : 0)); log.info("{}, 微博话题此轮采集到的数据量为:{}", new Date(), Integer.valueOf(list != null ? list.size() : 0));
TipsUtils.addHotList(HotSearchType.微博话题.name(),list); TipsUtils.addHotList(HotSearchType.微博话题.name(),list);
logger.info("微博话题采集结束..."); log.info("微博话题采集结束...");
} }
/** /**
...@@ -313,12 +311,12 @@ public class GatherTimer { ...@@ -313,12 +311,12 @@ public class GatherTimer {
@Async(value = "myScheduler") @Async(value = "myScheduler")
@Scheduled(cron = "10 * * * * ? ") @Scheduled(cron = "10 * * * * ? ")
public void crawlerTengXunVerificationHotSearch(){ public void crawlerTengXunVerificationHotSearch(){
logger.info("{},腾讯较真辟谣榜开始采集", new Date()); log.info("{},腾讯较真辟谣榜开始采集", new Date());
Date date = DateUtils.getMillSecondTime(new Date()); Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> list = TengXunCrawler.getTengXunVerificationList(date); List<HotSearchList> list = TengXunCrawler.getTengXunVerificationList(date);
logger.info("腾讯较真辟谣榜本轮采集数量:{}",list.size()); log.info("腾讯较真辟谣榜本轮采集数量:{}",list.size());
TipsUtils.addHotList(HotSearchType.腾讯较真榜.name(), list); TipsUtils.addHotList(HotSearchType.腾讯较真榜.name(), list);
logger.info("{},腾讯较真辟谣榜采集结束", new Date()); log.info("{},腾讯较真辟谣榜采集结束", new Date());
} }
/** /**
...@@ -338,12 +336,12 @@ public class GatherTimer { ...@@ -338,12 +336,12 @@ public class GatherTimer {
@Async(value = "myScheduler") @Async(value = "myScheduler")
@Scheduled(cron = "20 * * * * ? ") @Scheduled(cron = "20 * * * * ? ")
public void crawlerZhihuHotTopic(){ public void crawlerZhihuHotTopic(){
logger.info("知乎热搜话题开始采集..."); log.info("知乎热搜话题开始采集...");
Date date = DateUtils.getMillSecondTime(new Date()); Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> list = ZhihuTopicSearchCrawler.getZhihuTopicSearch(date); List<HotSearchList> list = ZhihuTopicSearchCrawler.getZhihuTopicSearch(date);
logger.info("{}, 知乎热搜话题此轮采集到的数据量为:{}", new Date(), Integer.valueOf(list != null ? list.size() : 0)); log.info("{}, 知乎热搜话题此轮采集到的数据量为:{}", new Date(), Integer.valueOf(list != null ? list.size() : 0));
TipsUtils.addHotList(HotSearchType.知乎热搜榜单.name(),list); TipsUtils.addHotList(HotSearchType.知乎热搜榜单.name(),list);
logger.info("知乎热搜话题采集结束..."); log.info("知乎热搜话题采集结束...");
} }
/** /**
...@@ -352,12 +350,12 @@ public class GatherTimer { ...@@ -352,12 +350,12 @@ public class GatherTimer {
@Async(value = "myScheduler") @Async(value = "myScheduler")
@Scheduled(cron = "20 * * * * ? ") @Scheduled(cron = "20 * * * * ? ")
public void crawlerWeiBoPreheat(){ public void crawlerWeiBoPreheat(){
logger.info("微博预热榜开始采集..."); log.info("微博预热榜开始采集...");
Date date = DateUtils.getMillSecondTime(new Date()); Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> list = WeiboHotSearchCrawler.weiboPreheatSearch(date); List<HotSearchList> list = WeiboHotSearchCrawler.weiboPreheatSearch(date);
logger.info("{},微博预热榜此轮采集到的数据量为:{}", new Date(),Integer.valueOf(list != null ? list.size() : 0)); log.info("{},微博预热榜此轮采集到的数据量为:{}", new Date(),Integer.valueOf(list != null ? list.size() : 0));
TipsUtils.addHotList(HotSearchType.微博预热榜.name(),list); TipsUtils.addHotList(HotSearchType.微博预热榜.name(),list);
logger.info("微博预热榜采集结束..."); log.info("微博预热榜采集结束...");
} }
/** /**
...@@ -432,14 +430,14 @@ public class GatherTimer { ...@@ -432,14 +430,14 @@ public class GatherTimer {
@Async(value = "myScheduler") @Async(value = "myScheduler")
@Scheduled(cron = "0 0 0/3 * * ? ") @Scheduled(cron = "0 0 0/3 * * ? ")
public void crawlerWeiBoSuperTopic(){ public void crawlerWeiBoSuperTopic(){
logger.info("微博超话采集开始........"); log.info("微博超话采集开始........");
Date date = DateUtils.getMillSecondTime(new Date()); Date date = DateUtils.getMillSecondTime(new Date());
WeiboSuperTopicDAO weiboTopicDAO = new WeiboSuperTopicDAO(); WeiboSuperTopicDAO weiboTopicDAO = new WeiboSuperTopicDAO();
List<WeiboSuperTopic> list = WeiboSuperTopicCrawler.startCrawler(); List<WeiboSuperTopic> list = WeiboSuperTopicCrawler.startCrawler();
logger.info("{}, 微博超话此轮采集到的数据量为:{}", new Date(), Integer.valueOf(list != null ? list.size() : 0)); log.info("{}, 微博超话此轮采集到的数据量为:{}", new Date(), Integer.valueOf(list != null ? list.size() : 0));
List<Document> data = new ArrayList<>(); List<Document> data = new ArrayList<>();
for(WeiboSuperTopic topic : list){ for(WeiboSuperTopic topic : list){
logger.info("topic::::{}", topic); log.info("topic::::{}", topic);
Document doc = new Document(); Document doc = new Document();
doc.put("_id", topic.getId()); doc.put("_id", topic.getId());
doc.put("name", topic.getTopicName()); doc.put("name", topic.getTopicName());
...@@ -454,7 +452,7 @@ public class GatherTimer { ...@@ -454,7 +452,7 @@ public class GatherTimer {
data.add(doc); data.add(doc);
} }
weiboTopicDAO.addTopicList(data); weiboTopicDAO.addTopicList(data);
logger.info("微博话题采集结束........"); log.info("微博话题采集结束........");
} }
...@@ -471,9 +469,9 @@ public class GatherTimer { ...@@ -471,9 +469,9 @@ public class GatherTimer {
// ZhiWeiTools.sleep(500L); // ZhiWeiTools.sleep(500L);
// } // }
// i++; // i++;
// logger.info("更新进度:{}",i*100/documentList.size()); // log.info("更新进度:{}",i*100/documentList.size());
// } // }
// logger.info("更新结束"); // log.info("更新结束");
// } // }
/** /**
...@@ -483,11 +481,11 @@ public class GatherTimer { ...@@ -483,11 +481,11 @@ public class GatherTimer {
private void crawlerZhiHuChild(String type){ private void crawlerZhiHuChild(String type){
Date date = DateUtils.getMillSecondTime(new Date()); Date date = DateUtils.getMillSecondTime(new Date());
String name = this.getTypeName(type); String name = this.getTypeName(type);
logger.info("知乎{}话题热榜采集开始...", name); log.info("知乎{}话题热榜采集开始...", name);
List<HotSearchList> list = ZhihuChildHotSearchCrawler.getZhihuTopicSearch(type,name,date); List<HotSearchList> list = ZhihuChildHotSearchCrawler.getZhihuTopicSearch(type,name,date);
logger.info("{}, 知乎{}话题此轮采集到的数据量为:{}", new Date(),name, Integer.valueOf(list != null ? list.size() : 0)); log.info("{}, 知乎{}话题此轮采集到的数据量为:{}", new Date(),name, Integer.valueOf(list != null ? list.size() : 0));
TipsUtils.addHotList(name,list); TipsUtils.addHotList(name,list);
logger.info("知乎{}话题热榜采集结束...", name); log.info("知乎{}话题热榜采集结束...", name);
} }
private String getTypeName(String type){ private String getTypeName(String type){
...@@ -513,12 +511,12 @@ public class GatherTimer { ...@@ -513,12 +511,12 @@ public class GatherTimer {
@Async(value = "myScheduler") @Async(value = "myScheduler")
@Scheduled(cron = "0 * * * * ? ") @Scheduled(cron = "0 * * * * ? ")
public void crawlerKuaiShou(){ public void crawlerKuaiShou(){
logger.info("快手热榜开始采集..."); log.info("快手热榜开始采集...");
Date date = DateUtils.getMillSecondTime(new Date()); Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> kuaiShouList = KuaiShouHotSearchCrawler.KuaiShouHotSearchCrawler(date); List<HotSearchList> kuaiShouList = KuaiShouHotSearchCrawler.KuaiShouHotSearchCrawler(date);
logger.info("{}, 快手此轮采集到的数据量为:{}", new Date(), kuaiShouList != null ? kuaiShouList.size() : 0); log.info("{}, 快手此轮采集到的数据量为:{}", new Date(), kuaiShouList != null ? kuaiShouList.size() : 0);
TipsUtils.addHotList(HotSearchType.快手热榜.name(), kuaiShouList); TipsUtils.addHotList(HotSearchType.快手热榜.name(), kuaiShouList);
logger.info("快手热榜采集结束..."); log.info("快手热榜采集结束...");
} }
/** /**
*淘宝热搜采集 *淘宝热搜采集
...@@ -526,12 +524,12 @@ public class GatherTimer { ...@@ -526,12 +524,12 @@ public class GatherTimer {
@Async(value = "myScheduler") @Async(value = "myScheduler")
@Scheduled(cron = "0 * * * * ? ") @Scheduled(cron = "0 * * * * ? ")
public void crawlerTaoBao(){ public void crawlerTaoBao(){
logger.info("淘宝热搜开始采集..."); log.info("淘宝热搜开始采集...");
Date date = DateUtils.getMillSecondTime(new Date()); Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> taoBaoList = TaoBaoHotSearchCrawler.taoBaoHotSearch(date); List<HotSearchList> taoBaoList = TaoBaoHotSearchCrawler.taoBaoHotSearch(date);
logger.info("{}, 淘宝热搜此轮采集到的数据量为:{}", new Date(), taoBaoList != null ? taoBaoList.size() : 0); log.info("{}, 淘宝热搜此轮采集到的数据量为:{}", new Date(), taoBaoList != null ? taoBaoList.size() : 0);
TipsUtils.addHotList(HotSearchType.淘宝热搜.name(), taoBaoList); TipsUtils.addHotList(HotSearchType.淘宝热搜.name(), taoBaoList);
logger.info("淘宝热搜采集结束..."); log.info("淘宝热搜采集结束...");
} }
// /** // /**
...@@ -540,12 +538,12 @@ public class GatherTimer { ...@@ -540,12 +538,12 @@ public class GatherTimer {
// @Async(value = "myScheduler") // @Async(value = "myScheduler")
// @Scheduled(cron = "0 * * * * ? ") // @Scheduled(cron = "0 * * * * ? ")
// public void crawlerDouYinTongCheng(){ // public void crawlerDouYinTongCheng(){
// logger.info("抖音同城榜开始采集..."); // log.info("抖音同城榜开始采集...");
// Date date = DateUtils.getMillSecondTime(new Date()); // Date date = DateUtils.getMillSecondTime(new Date());
// List<HotSearchList> douyinTongChengList = DouYinTongChengCrawlerTest.DouYinTongChengCrawler(date); // List<HotSearchList> douyinTongChengList = DouYinTongChengCrawlerTest.DouYinTongChengCrawler(date);
// logger.info("{}, 抖音同城榜此轮采集到的数据量为:{}", new Date(), Integer.valueOf(douyinTongChengList != null ? douyinTongChengList.size() : 0)); // log.info("{}, 抖音同城榜此轮采集到的数据量为:{}", new Date(), Integer.valueOf(douyinTongChengList != null ? douyinTongChengList.size() : 0));
// TipsUtils.addHotList(HotSearchType.抖音同城榜.name(),douyinTongChengList); // TipsUtils.addHotList(HotSearchType.抖音同城榜.name(),douyinTongChengList);
// logger.info("抖音同城榜采集结束..."); // log.info("抖音同城榜采集结束...");
// } // }
// //
// /** // /**
...@@ -554,7 +552,7 @@ public class GatherTimer { ...@@ -554,7 +552,7 @@ public class GatherTimer {
// @Async(value = "myScheduler") // @Async(value = "myScheduler")
// @Scheduled(cron = "0 0/5 * * * ? ") // @Scheduled(cron = "0 0/5 * * * ? ")
// public void updateDouYinTongChengUrl(){ // public void updateDouYinTongChengUrl(){
// logger.info("抖音同城链接更新开始..."); // log.info("抖音同城链接更新开始...");
// HotSearchCacheDAO hotSearchCacheDAO = new HotSearchCacheDAO(); // HotSearchCacheDAO hotSearchCacheDAO = new HotSearchCacheDAO();
// List<HotSearchList> douyinTongChengList = DouYinTongChengCrawlerTest.list; // List<HotSearchList> douyinTongChengList = DouYinTongChengCrawlerTest.list;
// if(douyinTongChengList!=null && douyinTongChengList.size()>0){ // if(douyinTongChengList!=null && douyinTongChengList.size()>0){
...@@ -569,9 +567,9 @@ public class GatherTimer { ...@@ -569,9 +567,9 @@ public class GatherTimer {
// hotSearchCacheDAO.updateDouyinUrl(document); // hotSearchCacheDAO.updateDouyinUrl(document);
// } // }
// } // }
// logger.info("抖音同城链接更新结束"); // log.info("抖音同城链接更新结束");
// }else{ // }else{
// logger.info("抖音同城链接更新失败,抖音同城榜列表获取为空。"); // log.info("抖音同城链接更新失败,抖音同城榜列表获取为空。");
// } // }
// } // }
/** /**
...@@ -580,12 +578,12 @@ public class GatherTimer { ...@@ -580,12 +578,12 @@ public class GatherTimer {
@Async(value = "myScheduler") @Async(value = "myScheduler")
@Scheduled(cron = "0 * * * * ? ") @Scheduled(cron = "0 * * * * ? ")
public void crawlerWeiBoEntertainment(){ public void crawlerWeiBoEntertainment(){
logger.info("微博娱乐榜开始采集..."); log.info("微博娱乐榜开始采集...");
Date date = DateUtils.getMillSecondTime(new Date()); Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> weiBoEntertainmentList = WeiboEntertainmentCrawler.weiboEntertainmentByPhone(date); List<HotSearchList> weiBoEntertainmentList = WeiboEntertainmentCrawler.weiboEntertainmentByPhone(date);
logger.info("{}, 微博娱乐榜此轮采集到的数据量为:{}", new Date(), weiBoEntertainmentList != null ? weiBoEntertainmentList.size() : 0); log.info("{}, 微博娱乐榜此轮采集到的数据量为:{}", new Date(), weiBoEntertainmentList != null ? weiBoEntertainmentList.size() : 0);
TipsUtils.addHotList(HotSearchType.微博娱乐榜.name(), weiBoEntertainmentList); TipsUtils.addHotList(HotSearchType.微博娱乐榜.name(), weiBoEntertainmentList);
logger.info("微博娱乐榜采集结束..."); log.info("微博娱乐榜采集结束...");
} }
/** /**
*微博要闻榜采集 *微博要闻榜采集
...@@ -593,12 +591,12 @@ public class GatherTimer { ...@@ -593,12 +591,12 @@ public class GatherTimer {
@Async(value = "myScheduler") @Async(value = "myScheduler")
@Scheduled(cron = "0 * * * * ? ") @Scheduled(cron = "0 * * * * ? ")
public void crawlerWeiBoNews(){ public void crawlerWeiBoNews(){
logger.info("微博要闻榜开始采集..."); log.info("微博要闻榜开始采集...");
Date date = DateUtils.getMillSecondTime(new Date()); Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> WeiboNewsList = WeiboNewsCrawler.weiboNewsByPhone(date); List<HotSearchList> WeiboNewsList = WeiboNewsCrawler.weiboNewsByPhone(date);
logger.info("{}, 微博要闻榜此轮采集到的数据量为:{}", new Date(), WeiboNewsList != null ? WeiboNewsList.size() : 0); log.info("{}, 微博要闻榜此轮采集到的数据量为:{}", new Date(), WeiboNewsList != null ? WeiboNewsList.size() : 0);
TipsUtils.addHotList(HotSearchType.微博要闻榜.name(), WeiboNewsList); TipsUtils.addHotList(HotSearchType.微博要闻榜.name(), WeiboNewsList);
logger.info("微博要闻榜采集结束..."); log.info("微博要闻榜采集结束...");
} }
/** /**
*微博搜索框热词采集 *微博搜索框热词采集
...@@ -607,9 +605,9 @@ public class GatherTimer { ...@@ -607,9 +605,9 @@ public class GatherTimer {
//@Async(value = "myScheduler") //@Async(value = "myScheduler")
//@Scheduled(cron = "0 * * * * ? ") //@Scheduled(cron = "0 * * * * ? ")
public void crawlerWeiBoSearchBoxHotWords(){ public void crawlerWeiBoSearchBoxHotWords(){
logger.info("微博搜索框热词采集开始........"); log.info("微博搜索框热词采集开始........");
Date date = DateUtils.getMillSecondTime(new Date()); Date date = DateUtils.getMillSecondTime(new Date());
WeiBoSearchBoxHotWordsCrawler.weiBoSearchBoxHotWords(date); WeiBoSearchBoxHotWordsCrawler.weiBoSearchBoxHotWords(date);
logger.info("微博搜索框热词采集结束........"); log.info("微博搜索框热词采集结束........");
} }
} }
...@@ -4,9 +4,9 @@ import com.zhiwei.searchhotcrawler.bean.HotSearchList; ...@@ -4,9 +4,9 @@ import com.zhiwei.searchhotcrawler.bean.HotSearchList;
import com.zhiwei.searchhotcrawler.dao.HotSearchCacheDAO; import com.zhiwei.searchhotcrawler.dao.HotSearchCacheDAO;
import com.zhiwei.searchhotcrawler.dao.HotSearchListDAO; import com.zhiwei.searchhotcrawler.dao.HotSearchListDAO;
import org.bson.Document; import org.bson.Document;
import org.checkerframework.checker.units.qual.A; import org.apache.logging.log4j.LogManager;
import org.slf4j.Logger; import org.apache.logging.log4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*; import java.util.*;
import java.util.concurrent.ExecutorService; import java.util.concurrent.ExecutorService;
...@@ -21,7 +21,7 @@ public class TipsUtils { ...@@ -21,7 +21,7 @@ public class TipsUtils {
private static String key = "a8e26ce3-8aaa-4d3e-bcf6-30b81526050b"; private static String key = "a8e26ce3-8aaa-4d3e-bcf6-30b81526050b";
private Logger logger = LoggerFactory.getLogger(TipsUtils.class); private static final Logger logger = LogManager.getLogger(TipsUtils.class);
private static Map<String,Date> typeTips = new HashMap<>(); private static Map<String,Date> typeTips = new HashMap<>();
......
...@@ -8,8 +8,9 @@ import java.util.Map; ...@@ -8,8 +8,9 @@ import java.util.Map;
import okhttp3.Request; import okhttp3.Request;
import okhttp3.Response; import okhttp3.Response;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
...@@ -21,7 +22,7 @@ import okhttp3.MediaType; ...@@ -21,7 +22,7 @@ import okhttp3.MediaType;
import okhttp3.RequestBody; import okhttp3.RequestBody;
public class WechatCodeUtil { public class WechatCodeUtil {
private static Logger logger = LoggerFactory.getLogger(WechatCodeUtil.class); private static Logger log = LogManager.getLogger(WechatCodeUtil.class);
private static HttpBoot httpBoot = new HttpBoot.Builder().retryTimes(3).build(); private static HttpBoot httpBoot = new HttpBoot.Builder().retryTimes(3).build();
/** /**
* @Title: getToken * @Title: getToken
...@@ -43,7 +44,7 @@ public class WechatCodeUtil { ...@@ -43,7 +44,7 @@ public class WechatCodeUtil {
result = response.body().string(); result = response.body().string();
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
logger.error("获取微信公众号推送token失败,问题为:::{}", e.fillInStackTrace()); log.error("获取微信公众号推送token失败,问题为:::{}", e.fillInStackTrace());
return null; return null;
} }
if (result != null) { if (result != null) {
...@@ -75,7 +76,7 @@ public class WechatCodeUtil { ...@@ -75,7 +76,7 @@ public class WechatCodeUtil {
try(Response response = httpBoot.syncCall(request)) { try(Response response = httpBoot.syncCall(request)) {
htmlBody = response.body().string(); htmlBody = response.body().string();
} catch (Exception e) { } catch (Exception e) {
logger.error("消息推送失败,错误为::{}",e.fillInStackTrace()); log.error("消息推送失败,错误为::{}",e.fillInStackTrace());
msgid = 0; msgid = 0;
} }
if (StringUtils.isNotBlank(htmlBody)) { if (StringUtils.isNotBlank(htmlBody)) {
...@@ -85,7 +86,7 @@ public class WechatCodeUtil { ...@@ -85,7 +86,7 @@ public class WechatCodeUtil {
msgid = jsonObject.getIntValue("msgid"); msgid = jsonObject.getIntValue("msgid");
} else { } else {
msgid = 0; msgid = 0;
logger.info("消息推送失败,错误为::{}", jsonObject.toString()); log.info("消息推送失败,错误为::{}", jsonObject.toString());
} }
} }
} }
...@@ -117,7 +118,7 @@ public class WechatCodeUtil { ...@@ -117,7 +118,7 @@ public class WechatCodeUtil {
try(Response response = httpBoot.syncCall(request)) { try(Response response = httpBoot.syncCall(request)) {
htmlBody = response.body().string(); htmlBody = response.body().string();
}catch (IOException e){ }catch (IOException e){
logger.error("页面连接获取失败",e); log.error("页面连接获取失败",e);
return null; return null;
} }
if(StringUtils.isNotBlank(htmlBody) && htmlBody.contains("data")) { if(StringUtils.isNotBlank(htmlBody) && htmlBody.contains("data")) {
...@@ -126,12 +127,12 @@ public class WechatCodeUtil { ...@@ -126,12 +127,12 @@ public class WechatCodeUtil {
if(jsonObject.containsKey("data")) { if(jsonObject.containsKey("data")) {
return (List<String>) jsonObject.getJSONObject("data").getObject("openid", List.class); return (List<String>) jsonObject.getJSONObject("data").getObject("openid", List.class);
}else{ }else{
logger.info("拉取用户列表时,出现问题{}", jsonObject); log.info("拉取用户列表时,出现问题{}", jsonObject);
} }
} }
} }
}else{ }else{
logger.info("token 获取失败"); log.info("token 获取失败");
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
...@@ -156,7 +157,7 @@ public class WechatCodeUtil { ...@@ -156,7 +157,7 @@ public class WechatCodeUtil {
try(Response response = httpBoot.syncCall(request)){ try(Response response = httpBoot.syncCall(request)){
htmlBody = response.body().string(); htmlBody = response.body().string();
}catch (IOException e){ }catch (IOException e){
logger.error("页面链接获取失败",e); log.error("页面链接获取失败",e);
return null; return null;
} }
if(StringUtils.isNotBlank(htmlBody) && htmlBody.contains("data")) { if(StringUtils.isNotBlank(htmlBody) && htmlBody.contains("data")) {
...@@ -165,12 +166,12 @@ public class WechatCodeUtil { ...@@ -165,12 +166,12 @@ public class WechatCodeUtil {
if(jsonObject.containsKey("data")) { if(jsonObject.containsKey("data")) {
return (List<String>) jsonObject.getJSONObject("data").getObject("openid", List.class); return (List<String>) jsonObject.getJSONObject("data").getObject("openid", List.class);
}else{ }else{
logger.info("拉取用户列表时,出现问题{}", jsonObject); log.info("拉取用户列表时,出现问题{}", jsonObject);
} }
} }
} }
}else{ }else{
logger.info("token 获取失败"); log.info("token 获取失败");
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
...@@ -199,7 +200,7 @@ public class WechatCodeUtil { ...@@ -199,7 +200,7 @@ public class WechatCodeUtil {
try(Response response = httpBoot.syncCall(request)) { try(Response response = httpBoot.syncCall(request)) {
htmlBody = response.body().string(); htmlBody = response.body().string();
} catch (IOException e) { } catch (IOException e) {
logger.error("获取分组id时出现错误",e.fillInStackTrace()); log.error("获取分组id时出现错误",e.fillInStackTrace());
return null; return null;
} }
if (htmlBody != null) { if (htmlBody != null) {
...@@ -232,7 +233,7 @@ public class WechatCodeUtil { ...@@ -232,7 +233,7 @@ public class WechatCodeUtil {
try(Response response = httpBoot.syncCall(request)) { try(Response response = httpBoot.syncCall(request)) {
htmlBody = response.body().string(); htmlBody = response.body().string();
} catch (IOException e) { } catch (IOException e) {
logger.error("获取分组id时出现错误",e.fillInStackTrace()); log.error("获取分组id时出现错误",e.fillInStackTrace());
return null; return null;
} }
if (htmlBody != null) { if (htmlBody != null) {
...@@ -245,7 +246,7 @@ public class WechatCodeUtil { ...@@ -245,7 +246,7 @@ public class WechatCodeUtil {
resultMap.put(name, id); resultMap.put(name, id);
} }
} else { } else {
logger.info("获取分组id时出现错误,数据为:::{}", htmlBody); log.info("获取分组id时出现错误,数据为:::{}", htmlBody);
} }
} }
return resultMap; return resultMap;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment