Skip to content

S
searchhotcrawler
Commit 04c6bdac by chenweitao
Options

Merge branch 'working' into 'master' 

修复日志漏洞

See merge request !161
parents 2290d4e2 c51af150
Showing with 119 additions and 133 deletions
src/main/java/com/zhiwei/searchhotcrawler/cache/CacheListener.java
package com.zhiwei.searchhotcrawler.cache;
import lombok.extern.log4j.Log4j2;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.zhiwei.tools.tools.ZhiWeiTools;
......@@ -26,7 +24,7 @@ public class CacheListener {
}
ZhiWeiTools.sleep(500);
}
}
}
}.start();
}
}
src/main/java/com/zhiwei/searchhotcrawler/crawler/BaiDuHotSearchCrawler.java
......@@ -129,7 +129,7 @@ public class BaiDuHotSearchCrawler {
String everurl = element.select("td.keyword").select("a.list-title").attr("href");
// 获取关键词(String)
String kw = element.select("td.keyword").select("a.list-title").text();
// logger.info("关键词:{}", kw);
// log.info("关键词:{}", kw);
//从连接中获取正确编码关键词
try {
if (!everurl.isEmpty()) {
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/DouyinHotSearchCrawler.java
......@@ -9,8 +9,7 @@ import lombok.extern.log4j.Log4j2;
import okhttp3.Request;
import okhttp3.Response;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
......@@ -71,7 +70,7 @@ public class DouyinHotSearchCrawler {
hotValueStr = wl.getString("hot_value");
Long hotValue = null;
hotValue = Long.valueOf(hotValueStr);
// logger.info("热度为:::{}", hot_value);
// log.info("热度为:::{}", hot_value);
HotSearchList douyin = new HotSearchList(null, word, hotValue, position, HotSearchType.抖音热搜.name(),date);
list.add(douyin);
}
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/HotSearch36KrCrawler.java
......@@ -99,7 +99,7 @@ public class HotSearch36KrCrawler {
rank++;
// 获取关键词(String)
String keyWord = element.select("p.title-wrapper").select("a.article-item-title").text();
// logger.info("关键词:{}", kw);
// log.info("关键词:{}", kw);
// 获取关键词相关链接everurl(String)
String everurl = element.select("p.title-wrapper").select("a.article-item-title").attr("href");
// 获取搜索指数count（int）
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/SougoHotSearchCrawler.java
......@@ -13,8 +13,7 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.core.proxy.ProxyHolder;
......@@ -70,7 +69,7 @@ public class SougoHotSearchCrawler {
// 获取关键词(String)
String kw = element.select("li").select("a").attr("title");
// logger.info("关键词:{}", kw);
// log.info("关键词:{}", kw);
String everurl = element.select("li").select("a").attr("href");
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiboSuperTopicCrawler.java
......@@ -13,8 +13,7 @@ import lombok.extern.log4j.Log4j2;
import okhttp3.Request;
import okhttp3.Response;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiboTopicCrawler.java
......@@ -18,8 +18,7 @@ import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/ZhihuHotSearchCrawler.java
......@@ -9,8 +9,7 @@ import java.util.Map;
import lombok.extern.log4j.Log4j2;
import okhttp3.Request;
import okhttp3.Response;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
......
src/main/java/com/zhiwei/searchhotcrawler/test/HotSearch36KrCrawlerTest.java
......@@ -99,7 +99,7 @@ public class HotSearch36KrCrawlerTest {
rank++;
// 获取关键词(String)
String keyWord = element.select("p.title-wrapper").select("a.article-item-title").text();
// logger.info("关键词:{}", kw);
// log.info("关键词:{}", kw);
// 获取关键词相关链接everurl(String)
String everurl = element.select("p.title-wrapper").select("a.article-item-title").attr("href");
// 获取搜索指数count（int）
......
src/main/java/com/zhiwei/searchhotcrawler/timer/BaiduHotSearchRun.java
......@@ -12,8 +12,6 @@ import com.zhiwei.searchhotcrawler.dao.HotSearchCacheDAO;
import com.zhiwei.searchhotcrawler.util.TipsUtils;
import lombok.extern.log4j.Log4j2;
import org.bson.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.mongodb.BasicDBObject;
import com.mongodb.DBObject;
......@@ -39,8 +37,8 @@ public class BaiduHotSearchRun extends Thread{
ZhiWeiTools.sleep(50);
}
}
private void getHotList() {
log.info("百度风云榜采集开始........");
// HotSearchListDAO hotSearchDAO = new HotSearchListDAO();
......@@ -70,4 +68,4 @@ public class BaiduHotSearchRun extends Thread{
// log.info("知乎话题采集结束........");
}
}
\ No newline at end of file
}
src/main/java/com/zhiwei/searchhotcrawler/timer/DouyinHotSearchRun.java
......@@ -12,8 +12,7 @@ import com.zhiwei.searchhotcrawler.dao.HotSearchCacheDAO;
import com.zhiwei.searchhotcrawler.util.TipsUtils;
import lombok.extern.log4j.Log4j2;
import org.bson.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.mongodb.BasicDBObject;
import com.mongodb.DBObject;
......@@ -26,7 +25,7 @@ import com.zhiwei.tools.tools.ZhiWeiTools;
public class DouyinHotSearchRun extends Thread{
public static List<HotSearchList> list = new ArrayList<>();
@Override
public void run() {
boolean f = true;
......@@ -41,7 +40,7 @@ public class DouyinHotSearchRun extends Thread{
ZhiWeiTools.sleep(50);
}
}
/**
* 获取热搜列表
* TODO
......
src/main/java/com/zhiwei/searchhotcrawler/timer/SougoHotSearchRun.java
......@@ -10,8 +10,7 @@ import com.zhiwei.searchhotcrawler.dao.HotSearchListDAO;
import com.zhiwei.searchhotcrawler.util.TipsUtils;
import lombok.extern.log4j.Log4j2;
import org.bson.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.mongodb.BasicDBObject;
import com.mongodb.DBObject;
......@@ -36,8 +35,8 @@ public class SougoHotSearchRun extends Thread {
ZhiWeiTools.sleep(50);
}
}
private void getHotList() {
// HotSearchListDAO hotSearchDAO = new HotSearchListDAO();
// HotSearchCacheDAO hotSearchCacheDAO = new HotSearchCacheDAO();
......
src/main/java/com/zhiwei/searchhotcrawler/timer/ZhihuHotSearchRun.java
......@@ -9,8 +9,7 @@ import com.zhiwei.searchhotcrawler.dao.HotSearchCacheDAO;
import com.zhiwei.searchhotcrawler.util.TipsUtils;
import lombok.extern.log4j.Log4j2;
import org.bson.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.mongodb.BasicDBObject;
import com.mongodb.DBObject;
......@@ -21,7 +20,7 @@ import com.zhiwei.tools.tools.ZhiWeiTools;
@Log4j2
public class ZhihuHotSearchRun extends Thread{
@Override
public void run() {
boolean f = true;
......@@ -37,7 +36,7 @@ public class ZhihuHotSearchRun extends Thread{
}
}
private void getHotList() {
// HotSearchListDAO hotSearchDAO = new HotSearchListDAO();
// HotSearchCacheDAO hotSearchCacheDAO = new HotSearchCacheDAO();
......
src/main/java/com/zhiwei/searchhotcrawler/timer/quartz/GatherTimer.java
......@@ -14,9 +14,8 @@ import com.zhiwei.searchhotcrawler.timer.TouTiaoExecutor;
import com.zhiwei.searchhotcrawler.util.DateUtils;
import com.zhiwei.searchhotcrawler.util.TipsUtils;
import com.zhiwei.tools.tools.ZhiWeiTools;
import lombok.extern.log4j.Log4j2;
import org.bson.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.annotation.EnableScheduling;
......@@ -28,10 +27,9 @@ import java.util.*;
@Component
@EnableScheduling
@EnableAsync
@Log4j2
public class GatherTimer {
private Logger logger = LoggerFactory.getLogger(GatherTimer.class);
private RedisDao redisDao = new RedisDao();
/** 知乎数码子分类 */
private String DIGITAL = "digital";
......@@ -47,21 +45,21 @@ public class GatherTimer {
@Async(value = "myScheduler")
@Scheduled(cron = "0 * * * * ?")
public void crawlerHuXiu() {
logger.info("虎嗅热文推荐开始采集...");
log.info("虎嗅热文推荐开始采集...");
Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> huXiuList = HuXiuHotSearchCrawler.HuXiuHotArticleRecommended(date);
logger.info("{}, 虎嗅热文推荐此轮采集到的数据量为:{}", new Date(), Integer.valueOf(huXiuList != null ? huXiuList.size() : 0));
log.info("{}, 虎嗅热文推荐此轮采集到的数据量为:{}", new Date(), Integer.valueOf(huXiuList != null ? huXiuList.size() : 0));
TipsUtils.addHotList(HotSearchType.虎嗅热文推荐.name(), huXiuList);
logger.info("虎嗅热文推荐采集结束...");
log.info("虎嗅热文推荐采集结束...");
/**
* 36氪人气榜的采集
*/
logger.info("36氪人气榜开始采集...");
log.info("36氪人气榜开始采集...");
List<HotSearchList> list36Kr = HotSearch36KrCrawler.hotSearch36Kr(date);
logger.info("{}, 36氪人气榜此轮采集到的数据量为:{}", new Date(), Integer.valueOf(list36Kr != null ? list36Kr.size() : 0));
log.info("{}, 36氪人气榜此轮采集到的数据量为:{}", new Date(), Integer.valueOf(list36Kr != null ? list36Kr.size() : 0));
TipsUtils.addHotList(HotSearchType.人气榜36氪.name(), list36Kr);
logger.info("36氪人气榜采集结束...");
log.info("36氪人气榜采集结束...");
}
/**
......@@ -70,12 +68,12 @@ public class GatherTimer {
@Async(value = "myScheduler")
@Scheduled(cron = "0 * * * * ? ")
public void crawlerWeiBo(){
logger.info("微博热搜开始采集...");
log.info("微博热搜开始采集...");
Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> weiboList = WeiboHotSearchCrawler.weiboHotSearchByPhone(date);
logger.info("{}, 微博此轮采集到的数据量为:{}", new Date(), weiboList != null ? weiboList.size() : 0);
log.info("{}, 微博此轮采集到的数据量为:{}", new Date(), weiboList != null ? weiboList.size() : 0);
TipsUtils.addHotList(HotSearchType.微博热搜.name(),weiboList);
logger.info("微博热搜采集结束...");
log.info("微博热搜采集结束...");
}
/**
......@@ -84,7 +82,7 @@ public class GatherTimer {
@Async(value = "myScheduler")
@Scheduled(cron = "45 0/10 * * * ? ")
public void updateWeiBo(){
logger.info("微博热搜导语更新...");
log.info("微博热搜导语更新...");
HotSearchCacheDAO hotSearchCacheDAO = new HotSearchCacheDAO();
Set<String> hotSearchIdSet = redisDao.getRedisSetData(RedisConfig.WEIBO_HOTSEARCHIDS);
redisDao.removeRedis(RedisConfig.WEIBO_HOTSEARCHIDS);
......@@ -100,7 +98,7 @@ public class GatherTimer {
ZhiWeiTools.sleep(3000L);
}
}
logger.info("微博热搜导语更新结束...");
log.info("微博热搜导语更新结束...");
}
/**
......@@ -109,13 +107,13 @@ public class GatherTimer {
@Async(value = "myScheduler")
@Scheduled(cron = "0 * * * * ? ")
public void crawlerTouTiao(){
logger.info("今日头条热搜开始采集...");
log.info("今日头条热搜开始采集...");
Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> toutiaoList = ToutiaoHotSearchCrawler.toutiaoHotSearchByPhone(date);
logger.info("{}, 今日头条此轮采集到的数据量为:{}", new Date(), toutiaoList != null ? toutiaoList.size() : 0);
log.info("{}, 今日头条此轮采集到的数据量为:{}", new Date(), toutiaoList != null ? toutiaoList.size() : 0);
TipsUtils.addHotList(HotSearchType.今日头条热搜.name(),toutiaoList);
logger.info("今日头条热搜采集结束...");
logger.info("今日头条热搜详情趋势阅读量更新...");
log.info("今日头条热搜采集结束...");
log.info("今日头条热搜详情趋势阅读量更新...");
TouTiaoExecutor.countTouTiaoReadCount(toutiaoList);
}
......@@ -125,12 +123,12 @@ public class GatherTimer {
@Async(value = "myScheduler")
@Scheduled(cron = "0 * * * * ? ")
public void crawlerBaiDu(){
logger.info("百度热搜开始采集...");
log.info("百度热搜开始采集...");
Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> baiduList = BaiDuHotSearchCrawler.baiduHotSearch(date);
logger.info("{}, 百度热搜此轮采集到的数据量为:{}", new Date(), Integer.valueOf(baiduList != null ? baiduList.size() : 0));
log.info("{}, 百度热搜此轮采集到的数据量为:{}", new Date(), Integer.valueOf(baiduList != null ? baiduList.size() : 0));
TipsUtils.addHotList(HotSearchType.百度热搜.name(),baiduList);
logger.info("百度热搜采集结束...");
log.info("百度热搜采集结束...");
}
/**
......@@ -139,12 +137,12 @@ public class GatherTimer {
@Async(value = "myScheduler")
@Scheduled(cron = "0 * * * * ? ")
public void crawlerDouYin(){
logger.info("抖音热搜开始采集...");
log.info("抖音热搜开始采集...");
Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> douyinList = DouyinHotSearchCrawler.getMobileDouyinHotList(date);
logger.info("{}, 抖音热搜此轮采集到的数据量为:{}", new Date(), Integer.valueOf(douyinList != null ? douyinList.size() : 0));
log.info("{}, 抖音热搜此轮采集到的数据量为:{}", new Date(), Integer.valueOf(douyinList != null ? douyinList.size() : 0));
TipsUtils.addHotList(HotSearchType.抖音热搜.name(),douyinList);
logger.info("抖音热搜采集结束...");
log.info("抖音热搜采集结束...");
}
/**
......@@ -153,7 +151,7 @@ public class GatherTimer {
@Async(value = "myScheduler")
@Scheduled(cron = "0 0/5 * * * ? ")
public void updateDouYinUrl(){
logger.info("抖音链接更新开始...");
log.info("抖音链接更新开始...");
HotSearchCacheDAO hotSearchCacheDAO = new HotSearchCacheDAO();
List<HotSearchList> douyinList = DouyinHotSearchCrawler.list;
if(douyinList!=null && douyinList.size()>0){
......@@ -168,9 +166,9 @@ public class GatherTimer {
hotSearchCacheDAO.updateDouyinUrl(document);
}
}
logger.info("抖音链接更新结束");
log.info("抖音链接更新结束");
}else{
logger.info("抖音链接更新失败,抖音热搜列表获取为空。");
log.info("抖音链接更新失败,抖音热搜列表获取为空。");
}
}
......@@ -180,12 +178,12 @@ public class GatherTimer {
@Async(value = "myScheduler")
@Scheduled(cron = "0 * * * * ? ")
public void crawlerZhihu(){
logger.info("知乎热搜开始采集...");
log.info("知乎热搜开始采集...");
Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> zhihuList = ZhihuHotSearchCrawler.getMobileZhihuHotList(date);
logger.info("{}, 知乎热搜此轮采集到的数据量为:{}", new Date(), Integer.valueOf(zhihuList != null ? zhihuList.size() : 0));
log.info("{}, 知乎热搜此轮采集到的数据量为:{}", new Date(), Integer.valueOf(zhihuList != null ? zhihuList.size() : 0));
TipsUtils.addHotList(HotSearchType.知乎热搜.name(),zhihuList);
logger.info("知乎热搜采集结束...");
log.info("知乎热搜采集结束...");
}
/**
......@@ -194,12 +192,12 @@ public class GatherTimer {
@Async(value = "myScheduler")
@Scheduled(cron = "0 * * * * ? ")
public void crawlerWeChat(){
logger.info("搜狗微信热词开始采集...");
log.info("搜狗微信热词开始采集...");
Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> list = SougoHotSearchCrawler.sougoHotSearch(date);
logger.info("{}, 搜狗微信热词采集到的数据量为:{}", new Date(), Integer.valueOf(list != null ? list.size() : 0));
log.info("{}, 搜狗微信热词采集到的数据量为:{}", new Date(), Integer.valueOf(list != null ? list.size() : 0));
TipsUtils.addHotList(HotSearchType.搜狗微信热搜.name(),list);
logger.info("搜狗微信热词采集结束...");
log.info("搜狗微信热词采集结束...");
}
/**
......@@ -208,12 +206,12 @@ public class GatherTimer {
//@Async(value = "myScheduler")
//@Scheduled(cron = "20 * * * * ? ")
public void ceawlerSougouHotData(){
logger.info("搜狗微信热搜开始采集...");
log.info("搜狗微信热搜开始采集...");
Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> list = SougoHotSearchCrawler.sougouHotDataCrawler(date);
logger.info("{}, 搜狗微信热搜此轮采集到的数据量为:{}", new Date(), list != null ? list.size() : 0);
log.info("{}, 搜狗微信热搜此轮采集到的数据量为:{}", new Date(), list != null ? list.size() : 0);
TipsUtils.addHotList(HotSearchType.搜狗微信客户端热搜.name(),list);
logger.info("搜狗微信热搜采集结束...");
log.info("搜狗微信热搜采集结束...");
}
/**
......@@ -222,12 +220,12 @@ public class GatherTimer {
@Async(value = "myScheduler")
@Scheduled(cron = "0 * * * * ? ")
public void crawlerWeiBoTopic(){
logger.info("微博话题开始采集...");
log.info("微博话题开始采集...");
Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> list = WeiboTopicCrawler.startCrawlerByPhone(date);
logger.info("{}, 微博话题此轮采集到的数据量为:{}", new Date(), Integer.valueOf(list != null ? list.size() : 0));
log.info("{}, 微博话题此轮采集到的数据量为:{}", new Date(), Integer.valueOf(list != null ? list.size() : 0));
TipsUtils.addHotList(HotSearchType.微博话题.name(),list);
logger.info("微博话题采集结束...");
log.info("微博话题采集结束...");
}
/**
......@@ -313,12 +311,12 @@ public class GatherTimer {
@Async(value = "myScheduler")
@Scheduled(cron = "10 * * * * ? ")
public void crawlerTengXunVerificationHotSearch(){
logger.info("{}，腾讯较真辟谣榜开始采集", new Date());
log.info("{}，腾讯较真辟谣榜开始采集", new Date());
Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> list = TengXunCrawler.getTengXunVerificationList(date);
logger.info("腾讯较真辟谣榜本轮采集数量:{}",list.size());
log.info("腾讯较真辟谣榜本轮采集数量:{}",list.size());
TipsUtils.addHotList(HotSearchType.腾讯较真榜.name(), list);
logger.info("{}，腾讯较真辟谣榜采集结束", new Date());
log.info("{}，腾讯较真辟谣榜采集结束", new Date());
}
/**
......@@ -338,12 +336,12 @@ public class GatherTimer {
@Async(value = "myScheduler")
@Scheduled(cron = "20 * * * * ? ")
public void crawlerZhihuHotTopic(){
logger.info("知乎热搜话题开始采集...");
log.info("知乎热搜话题开始采集...");
Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> list = ZhihuTopicSearchCrawler.getZhihuTopicSearch(date);
logger.info("{}, 知乎热搜话题此轮采集到的数据量为:{}", new Date(), Integer.valueOf(list != null ? list.size() : 0));
log.info("{}, 知乎热搜话题此轮采集到的数据量为:{}", new Date(), Integer.valueOf(list != null ? list.size() : 0));
TipsUtils.addHotList(HotSearchType.知乎热搜榜单.name(),list);
logger.info("知乎热搜话题采集结束...");
log.info("知乎热搜话题采集结束...");
}
/**
......@@ -352,12 +350,12 @@ public class GatherTimer {
@Async(value = "myScheduler")
@Scheduled(cron = "20 * * * * ? ")
public void crawlerWeiBoPreheat(){
logger.info("微博预热榜开始采集...");
log.info("微博预热榜开始采集...");
Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> list = WeiboHotSearchCrawler.weiboPreheatSearch(date);
logger.info("{},微博预热榜此轮采集到的数据量为:{}", new Date(),Integer.valueOf(list != null ? list.size() : 0));
log.info("{},微博预热榜此轮采集到的数据量为:{}", new Date(),Integer.valueOf(list != null ? list.size() : 0));
TipsUtils.addHotList(HotSearchType.微博预热榜.name(),list);
logger.info("微博预热榜采集结束...");
log.info("微博预热榜采集结束...");
}
/**
......@@ -432,14 +430,14 @@ public class GatherTimer {
@Async(value = "myScheduler")
@Scheduled(cron = "0 0 0/3 * * ? ")
public void crawlerWeiBoSuperTopic(){
logger.info("微博超话采集开始........");
log.info("微博超话采集开始........");
Date date = DateUtils.getMillSecondTime(new Date());
WeiboSuperTopicDAO weiboTopicDAO = new WeiboSuperTopicDAO();
List<WeiboSuperTopic> list = WeiboSuperTopicCrawler.startCrawler();
logger.info("{}, 微博超话此轮采集到的数据量为:{}", new Date(), Integer.valueOf(list != null ? list.size() : 0));
log.info("{}, 微博超话此轮采集到的数据量为:{}", new Date(), Integer.valueOf(list != null ? list.size() : 0));
List<Document> data = new ArrayList<>();
for(WeiboSuperTopic topic : list){
logger.info("topic::::{}", topic);
log.info("topic::::{}", topic);
Document doc = new Document();
doc.put("_id", topic.getId());
doc.put("name", topic.getTopicName());
......@@ -454,7 +452,7 @@ public class GatherTimer {
data.add(doc);
}
weiboTopicDAO.addTopicList(data);
logger.info("微博话题采集结束........");
log.info("微博话题采集结束........");
}
......@@ -471,9 +469,9 @@ public class GatherTimer {
// ZhiWeiTools.sleep(500L);
// }
// i++;
// logger.info("更新进度:{}",i*100/documentList.size());
// log.info("更新进度:{}",i*100/documentList.size());
// }
// logger.info("更新结束");
// log.info("更新结束");
// }
/**
......@@ -483,11 +481,11 @@ public class GatherTimer {
private void crawlerZhiHuChild(String type){
Date date = DateUtils.getMillSecondTime(new Date());
String name = this.getTypeName(type);
logger.info("知乎{}话题热榜采集开始...", name);
log.info("知乎{}话题热榜采集开始...", name);
List<HotSearchList> list = ZhihuChildHotSearchCrawler.getZhihuTopicSearch(type,name,date);
logger.info("{}, 知乎{}话题此轮采集到的数据量为:{}", new Date(),name, Integer.valueOf(list != null ? list.size() : 0));
log.info("{}, 知乎{}话题此轮采集到的数据量为:{}", new Date(),name, Integer.valueOf(list != null ? list.size() : 0));
TipsUtils.addHotList(name,list);
logger.info("知乎{}话题热榜采集结束...", name);
log.info("知乎{}话题热榜采集结束...", name);
}
private String getTypeName(String type){
......@@ -513,12 +511,12 @@ public class GatherTimer {
@Async(value = "myScheduler")
@Scheduled(cron = "0 * * * * ? ")
public void crawlerKuaiShou(){
logger.info("快手热榜开始采集...");
log.info("快手热榜开始采集...");
Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> kuaiShouList = KuaiShouHotSearchCrawler.KuaiShouHotSearchCrawler(date);
logger.info("{}, 快手此轮采集到的数据量为:{}", new Date(), kuaiShouList != null ? kuaiShouList.size() : 0);
log.info("{}, 快手此轮采集到的数据量为:{}", new Date(), kuaiShouList != null ? kuaiShouList.size() : 0);
TipsUtils.addHotList(HotSearchType.快手热榜.name(), kuaiShouList);
logger.info("快手热榜采集结束...");
log.info("快手热榜采集结束...");
}
/**
*淘宝热搜采集
......@@ -526,12 +524,12 @@ public class GatherTimer {
@Async(value = "myScheduler")
@Scheduled(cron = "0 * * * * ? ")
public void crawlerTaoBao(){
logger.info("淘宝热搜开始采集...");
log.info("淘宝热搜开始采集...");
Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> taoBaoList = TaoBaoHotSearchCrawler.taoBaoHotSearch(date);
logger.info("{}, 淘宝热搜此轮采集到的数据量为:{}", new Date(), taoBaoList != null ? taoBaoList.size() : 0);
log.info("{}, 淘宝热搜此轮采集到的数据量为:{}", new Date(), taoBaoList != null ? taoBaoList.size() : 0);
TipsUtils.addHotList(HotSearchType.淘宝热搜.name(), taoBaoList);
logger.info("淘宝热搜采集结束...");
log.info("淘宝热搜采集结束...");
}
// /**
......@@ -540,12 +538,12 @@ public class GatherTimer {
// @Async(value = "myScheduler")
// @Scheduled(cron = "0 * * * * ? ")
// public void crawlerDouYinTongCheng(){
// logger.info("抖音同城榜开始采集...");
// log.info("抖音同城榜开始采集...");
// Date date = DateUtils.getMillSecondTime(new Date());
// List<HotSearchList> douyinTongChengList = DouYinTongChengCrawlerTest.DouYinTongChengCrawler(date);
// logger.info("{}, 抖音同城榜此轮采集到的数据量为:{}", new Date(), Integer.valueOf(douyinTongChengList != null ? douyinTongChengList.size() : 0));
// log.info("{}, 抖音同城榜此轮采集到的数据量为:{}", new Date(), Integer.valueOf(douyinTongChengList != null ? douyinTongChengList.size() : 0));
// TipsUtils.addHotList(HotSearchType.抖音同城榜.name(),douyinTongChengList);
// logger.info("抖音同城榜采集结束...");
// log.info("抖音同城榜采集结束...");
// }
//
// /**
......@@ -554,7 +552,7 @@ public class GatherTimer {
// @Async(value = "myScheduler")
// @Scheduled(cron = "0 0/5 * * * ? ")
// public void updateDouYinTongChengUrl(){
// logger.info("抖音同城链接更新开始...");
// log.info("抖音同城链接更新开始...");
// HotSearchCacheDAO hotSearchCacheDAO = new HotSearchCacheDAO();
// List<HotSearchList> douyinTongChengList = DouYinTongChengCrawlerTest.list;
// if(douyinTongChengList!=null && douyinTongChengList.size()>0){
......@@ -569,9 +567,9 @@ public class GatherTimer {
// hotSearchCacheDAO.updateDouyinUrl(document);
// }
// }
// logger.info("抖音同城链接更新结束");
// log.info("抖音同城链接更新结束");
// }else{
// logger.info("抖音同城链接更新失败,抖音同城榜列表获取为空。");
// log.info("抖音同城链接更新失败,抖音同城榜列表获取为空。");
// }
// }
/**
......@@ -580,12 +578,12 @@ public class GatherTimer {
@Async(value = "myScheduler")
@Scheduled(cron = "0 * * * * ? ")
public void crawlerWeiBoEntertainment(){
logger.info("微博娱乐榜开始采集...");
log.info("微博娱乐榜开始采集...");
Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> weiBoEntertainmentList = WeiboEntertainmentCrawler.weiboEntertainmentByPhone(date);
logger.info("{}, 微博娱乐榜此轮采集到的数据量为:{}", new Date(), weiBoEntertainmentList != null ? weiBoEntertainmentList.size() : 0);
log.info("{}, 微博娱乐榜此轮采集到的数据量为:{}", new Date(), weiBoEntertainmentList != null ? weiBoEntertainmentList.size() : 0);
TipsUtils.addHotList(HotSearchType.微博娱乐榜.name(), weiBoEntertainmentList);
logger.info("微博娱乐榜采集结束...");
log.info("微博娱乐榜采集结束...");
}
/**
*微博要闻榜采集
......@@ -593,12 +591,12 @@ public class GatherTimer {
@Async(value = "myScheduler")
@Scheduled(cron = "0 * * * * ? ")
public void crawlerWeiBoNews(){
logger.info("微博要闻榜开始采集...");
log.info("微博要闻榜开始采集...");
Date date = DateUtils.getMillSecondTime(new Date());
List<HotSearchList> WeiboNewsList = WeiboNewsCrawler.weiboNewsByPhone(date);
logger.info("{}, 微博要闻榜此轮采集到的数据量为:{}", new Date(), WeiboNewsList != null ? WeiboNewsList.size() : 0);
log.info("{}, 微博要闻榜此轮采集到的数据量为:{}", new Date(), WeiboNewsList != null ? WeiboNewsList.size() : 0);
TipsUtils.addHotList(HotSearchType.微博要闻榜.name(), WeiboNewsList);
logger.info("微博要闻榜采集结束...");
log.info("微博要闻榜采集结束...");
}
/**
*微博搜索框热词采集
......@@ -607,9 +605,9 @@ public class GatherTimer {
//@Async(value = "myScheduler")
//@Scheduled(cron = "0 * * * * ? ")
public void crawlerWeiBoSearchBoxHotWords(){
logger.info("微博搜索框热词采集开始........");
log.info("微博搜索框热词采集开始........");
Date date = DateUtils.getMillSecondTime(new Date());
WeiBoSearchBoxHotWordsCrawler.weiBoSearchBoxHotWords(date);
logger.info("微博搜索框热词采集结束........");
log.info("微博搜索框热词采集结束........");
}
}
src/main/java/com/zhiwei/searchhotcrawler/util/TipsUtils.java
......@@ -4,9 +4,9 @@ import com.zhiwei.searchhotcrawler.bean.HotSearchList;
import com.zhiwei.searchhotcrawler.dao.HotSearchCacheDAO;
import com.zhiwei.searchhotcrawler.dao.HotSearchListDAO;
import org.bson.Document;
import org.checkerframework.checker.units.qual.A;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.*;
import java.util.concurrent.ExecutorService;
......@@ -21,7 +21,7 @@ public class TipsUtils {
private static String key = "a8e26ce3-8aaa-4d3e-bcf6-30b81526050b";
private Logger logger = LoggerFactory.getLogger(TipsUtils.class);
private static final Logger logger = LogManager.getLogger(TipsUtils.class);
private static Map<String,Date> typeTips = new HashMap<>();
......
src/main/java/com/zhiwei/searchhotcrawler/util/WechatCodeUtil.java
......@@ -8,8 +8,9 @@ import java.util.Map;
import okhttp3.Request;
import okhttp3.Response;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
......@@ -21,7 +22,7 @@ import okhttp3.MediaType;
import okhttp3.RequestBody;
public class WechatCodeUtil {
private static Logger logger = LoggerFactory.getLogger(WechatCodeUtil.class);
private static Logger log = LogManager.getLogger(WechatCodeUtil.class);
private static HttpBoot httpBoot = new HttpBoot.Builder().retryTimes(3).build();
/**
* @Title: getToken
......@@ -43,7 +44,7 @@ public class WechatCodeUtil {
result = response.body().string();
} catch (IOException e) {
e.printStackTrace();
logger.error("获取微信公众号推送token失败,问题为:::{}", e.fillInStackTrace());
log.error("获取微信公众号推送token失败,问题为:::{}", e.fillInStackTrace());
return null;
}
if (result != null) {
......@@ -75,7 +76,7 @@ public class WechatCodeUtil {
try(Response response = httpBoot.syncCall(request)) {
htmlBody = response.body().string();
} catch (Exception e) {
logger.error("消息推送失败,错误为::{}",e.fillInStackTrace());
log.error("消息推送失败,错误为::{}",e.fillInStackTrace());
msgid = 0;
}
if (StringUtils.isNotBlank(htmlBody)) {
......@@ -85,7 +86,7 @@ public class WechatCodeUtil {
msgid = jsonObject.getIntValue("msgid");
} else {
msgid = 0;
logger.info("消息推送失败,错误为::{}", jsonObject.toString());
log.info("消息推送失败,错误为::{}", jsonObject.toString());
}
}
}
......@@ -117,7 +118,7 @@ public class WechatCodeUtil {
try(Response response = httpBoot.syncCall(request)) {
htmlBody = response.body().string();
}catch (IOException e){
logger.error("页面连接获取失败",e);
log.error("页面连接获取失败",e);
return null;
}
if(StringUtils.isNotBlank(htmlBody) && htmlBody.contains("data")) {
......@@ -126,12 +127,12 @@ public class WechatCodeUtil {
if(jsonObject.containsKey("data")) {
return (List<String>) jsonObject.getJSONObject("data").getObject("openid", List.class);
}else{
logger.info("拉取用户列表时，出现问题{}", jsonObject);
log.info("拉取用户列表时，出现问题{}", jsonObject);
}
}
}
}else{
logger.info("token 获取失败");
log.info("token 获取失败");
}
} catch (Exception e) {
e.printStackTrace();
......@@ -156,7 +157,7 @@ public class WechatCodeUtil {
try(Response response = httpBoot.syncCall(request)){
htmlBody = response.body().string();
}catch (IOException e){
logger.error("页面链接获取失败",e);
log.error("页面链接获取失败",e);
return null;
}
if(StringUtils.isNotBlank(htmlBody) && htmlBody.contains("data")) {
......@@ -165,12 +166,12 @@ public class WechatCodeUtil {
if(jsonObject.containsKey("data")) {
return (List<String>) jsonObject.getJSONObject("data").getObject("openid", List.class);
}else{
logger.info("拉取用户列表时，出现问题{}", jsonObject);
log.info("拉取用户列表时，出现问题{}", jsonObject);
}
}
}
}else{
logger.info("token 获取失败");
log.info("token 获取失败");
}
} catch (Exception e) {
e.printStackTrace();
......@@ -199,7 +200,7 @@ public class WechatCodeUtil {
try(Response response = httpBoot.syncCall(request)) {
htmlBody = response.body().string();
} catch (IOException e) {
logger.error("获取分组id时出现错误",e.fillInStackTrace());
log.error("获取分组id时出现错误",e.fillInStackTrace());
return null;
}
if (htmlBody != null) {
......@@ -232,7 +233,7 @@ public class WechatCodeUtil {
try(Response response = httpBoot.syncCall(request)) {
htmlBody = response.body().string();
} catch (IOException e) {
logger.error("获取分组id时出现错误",e.fillInStackTrace());
log.error("获取分组id时出现错误",e.fillInStackTrace());
return null;
}
if (htmlBody != null) {
......@@ -245,7 +246,7 @@ public class WechatCodeUtil {
resultMap.put(name, id);
}
} else {
logger.info("获取分组id时出现错误,数据为：：：{}", htmlBody);
log.info("获取分组id时出现错误,数据为：：：{}", htmlBody);
}
}
return resultMap;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment