线程池静态

README.md

@@ -45,6 +45,10 @@
 hot_search_list
 #### zookeeper
 zookeeper://192.168.0.203:2182?backup=192.168.0.104:2182,192.168.0.105:2182
+#### redis
+redis.host = 192.168.0.39
+redis.port = 6379
+redis.database = 1
         
 
 

src/main/java/com/zhiwei/searchhotcrawler/crawler/ToutiaoHotSearchCrawler.java

@@ -157,7 +157,6 @@ public class ToutiaoHotSearchCrawler {
             String url = hotSearchList.getUrl();
             Request request = RequestUtils.wrapGet(url);
             for (int i = 0; i <= 5; i++) {
-                ZhiWeiTools.sleep(1000L);
                 try (Response response = httpBoot.syncCall(request, ProxyHolder.NAT_HEAVY_PROXY)) {
                     htmlBody = response.body().string();
                 } catch (IOException e1) {
@@ -175,6 +174,7 @@ public class ToutiaoHotSearchCrawler {
                         return hotSearchList;
                     }
                 }
+                ZhiWeiTools.sleep(1000L);
             }
         }
         return hotSearchList;

src/main/java/com/zhiwei/searchhotcrawler/timer/TouTiaoExecutor.java

@@ -2,12 +2,11 @@ package com.zhiwei.searchhotcrawler.timer;
 
 import com.zhiwei.searchhotcrawler.bean.HotSearchList;
 import com.zhiwei.searchhotcrawler.crawler.ToutiaoHotSearchCrawler;
+import com.zhiwei.searchhotcrawler.util.TipsUtils;
 import lombok.extern.log4j.Log4j2;
 
 import java.util.ArrayList;
 import java.util.List;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 
 @Log4j2
@@ -38,13 +37,13 @@ public class TouTiaoExecutor extends Thread {
      */
     public static List<HotSearchList> countTouTiaoReadCount(List<HotSearchList> list){
         resultList= new ArrayList<>();
-        ExecutorService service = Executors.newFixedThreadPool(list.size());
+//        ExecutorService service = Executors.newFixedThreadPool(list.size());
         for(int i=0; i<list.size(); i++){
-            service.execute(new TouTiaoExecutor(list.get(i)));
+            TipsUtils.service.execute(new TouTiaoExecutor(list.get(i)));
         }
-        service.shutdown();
+//        TipsUtils.service.shutdown();
         try {
-            if(!service.awaitTermination(1, TimeUnit.MINUTES)){
+            if(!TipsUtils.service.awaitTermination(1, TimeUnit.MINUTES)){
                 log.info("查询今日头条阅读量超时");
             }
         } catch (InterruptedException e) {

src/main/java/com/zhiwei/searchhotcrawler/timer/quartz/GatherTimer.java

@@ -87,7 +87,7 @@ public class GatherTimer {
         List<HotSearchList> toutiaoList = ToutiaoHotSearchCrawler.toutiaoHotSearchByPhone(date);
         List<HotSearchList> toutiaoResult = new ArrayList<>();
         toutiaoResult = TouTiaoExecutor.countTouTiaoReadCount(toutiaoList);
-        logger.info("{}, 今日头条此轮采集到的数据量为:{}", new Date(), Integer.valueOf(toutiaoResult != null ? toutiaoResult.size() : 0));
+        logger.info("{}, 今日头条此轮采集到的数据量为:{}", new Date(), toutiaoResult != null ? toutiaoResult.size() : 0);
         TipsUtils.addHotList(HotSearchType.今日头条热搜.name(),toutiaoResult);
         logger.info("今日头条热搜采集结束...");
     }

src/main/java/com/zhiwei/searchhotcrawler/util/TipsUtils.java

@@ -9,6 +9,8 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
 /**
  * 预警发送
@@ -23,6 +25,8 @@ public class TipsUtils {
 
     private static Map<String,Date> typeTips = new HashMap<>();
 
+    public static ExecutorService service = Executors.newFixedThreadPool(30);
+
     //未采集到数据发送预警信息
     public static void sendTips(String type, Date time){
         HotSearchListDAO hotSearchListDAO = new HotSearchListDAO();