爬虫核心升级

src/main/java/com/zhiwei/searchhotcrawler/crawler/BaiDuHotSearchCrawler.java

@@ -3,6 +3,8 @@ package com.zhiwei.searchhotcrawler.crawler;
 import java.time.Duration;
 import java.util.*;
 
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import lombok.extern.log4j.Log4j2;
 import okhttp3.Request;
 import okhttp3.Response;
@@ -14,8 +16,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
 import com.zhiwei.searchhotcrawler.bean.HotSearchList;
 import com.zhiwei.searchhotcrawler.bean.HotSearchType;
 
@@ -29,7 +29,7 @@ import com.zhiwei.searchhotcrawler.bean.HotSearchType;
 public class BaiDuHotSearchCrawler {
 
 	private static HttpBoot httpBoot = new HttpBoot.Builder().throwException(false).retryTimes(3).connectTimeout(Duration.ofSeconds(60)).build();
-	
+
 	/**
 	 * @Title: BaiDuHotSearchTest
 	 * @author hero
@@ -40,7 +40,7 @@ public class BaiDuHotSearchCrawler {
 		String url = "http://top.baidu.com/buzz?b=1&fr=topindex";
 		String htmlBody = null;
 		Request request = RequestUtils.wrapGet(url);
-		try(Response response = httpBoot.syncCall(request,ProxyHolder.NAT_HEAVY_PROXY)) {
+		try(Response response = httpBoot.syncCall(request, ProxyHolder.NAT_HEAVY_PROXY)) {
 			htmlBody = response.body().string();
 		} catch (Exception e) {
 			log.error("解析百度风云榜时出现解析错误,页面结构有问题", e);
@@ -52,8 +52,8 @@ public class BaiDuHotSearchCrawler {
 		}
 		return Collections.emptyList();
 	}
-	
-	
+
+
 	/**
 	 * 解析数据
 	 * @param htmlBody
@@ -122,4 +122,4 @@ public class BaiDuHotSearchCrawler {
 		return list;
 	}
 
-}
+}
\ No newline at end of file

src/main/java/com/zhiwei/searchhotcrawler/crawler/BililiCrawler.java

@@ -3,8 +3,8 @@ package com.zhiwei.searchhotcrawler.crawler;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.searchhotcrawler.bean.HotSearchList;
 import com.zhiwei.searchhotcrawler.bean.HotSearchType;
 import com.zhiwei.searchhotcrawler.util.TipsUtils;

src/main/java/com/zhiwei/searchhotcrawler/crawler/DouyinHotSearchCrawler.java

@@ -15,8 +15,8 @@ import org.slf4j.LoggerFactory;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.searchhotcrawler.bean.HotSearchList;
 import com.zhiwei.searchhotcrawler.bean.HotSearchType;
 
@@ -30,16 +30,16 @@ import com.zhiwei.searchhotcrawler.bean.HotSearchType;
  */
 @Log4j2
 public class DouyinHotSearchCrawler {
-	
+
 	private static HttpBoot httpBoot = new HttpBoot.Builder().throwException(false).retryTimes(3).build();
 
 	public static List<HotSearchList> list = new ArrayList<>();
-	
+
 	/**
-	 * @Title: getMobileDouyinHotList 
-	 * @author hero 
-	 * @Description: 移动端抖音热搜榜 
-	 * @param @return 设定文件 
+	 * @Title: getMobileDouyinHotList
+	 * @author hero
+	 * @Description: 移动端抖音热搜榜
+	 * @param @return 设定文件
 	 * @return List<ZhihuHotSearch> 返回类型
 	 */
 	public static List<HotSearchList> getMobileDouyinHotList(Date date){

src/main/java/com/zhiwei/searchhotcrawler/crawler/FengHuangSearchCrawler.java

@@ -3,8 +3,8 @@ package com.zhiwei.searchhotcrawler.crawler;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.searchhotcrawler.bean.HotSearchList;
 import com.zhiwei.searchhotcrawler.bean.HotSearchType;
 import com.zhiwei.searchhotcrawler.util.TipsUtils;

src/main/java/com/zhiwei/searchhotcrawler/crawler/MaiMaiHotSearchCrawler.java

@@ -3,8 +3,8 @@ package com.zhiwei.searchhotcrawler.crawler;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.searchhotcrawler.bean.HotSearchList;
 import com.zhiwei.searchhotcrawler.bean.HotSearchType;
 import com.zhiwei.searchhotcrawler.util.TipsUtils;

src/main/java/com/zhiwei/searchhotcrawler/crawler/SougoHotSearchCrawler.java

@@ -17,8 +17,8 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.searchhotcrawler.bean.HotSearchList;
 import com.zhiwei.searchhotcrawler.bean.HotSearchType;
 import com.zhiwei.tools.httpclient.HeaderTool;

src/main/java/com/zhiwei/searchhotcrawler/crawler/SouhuTopicCrawler.java

@@ -3,8 +3,8 @@ package com.zhiwei.searchhotcrawler.crawler;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.searchhotcrawler.bean.HotSearchList;
 import com.zhiwei.searchhotcrawler.bean.HotSearchType;
 import com.zhiwei.searchhotcrawler.util.TipsUtils;

src/main/java/com/zhiwei/searchhotcrawler/crawler/TengXunCrawler.java

@@ -3,8 +3,8 @@ package com.zhiwei.searchhotcrawler.crawler;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.searchhotcrawler.bean.HotSearchList;
 import com.zhiwei.searchhotcrawler.bean.HotSearchType;
 import com.zhiwei.tools.tools.ZhiWeiTools;

src/main/java/com/zhiwei/searchhotcrawler/crawler/ToutiaoHotSearchCrawler.java

@@ -4,8 +4,8 @@ import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.searchhotcrawler.bean.HotSearchList;
 import com.zhiwei.searchhotcrawler.bean.HotSearchType;
 import com.zhiwei.searchhotcrawler.dao.HotSearchListDAO;

src/main/java/com/zhiwei/searchhotcrawler/crawler/WangYiHotSearchCrawler.java

@@ -3,8 +3,8 @@ package com.zhiwei.searchhotcrawler.crawler;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.searchhotcrawler.bean.HotSearchList;
 import com.zhiwei.searchhotcrawler.bean.HotSearchType;
 import com.zhiwei.searchhotcrawler.util.TipsUtils;

src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiboHotSearchCrawler.java

@@ -23,8 +23,8 @@ import org.slf4j.LoggerFactory;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.searchhotcrawler.bean.HotSearchList;
 import com.zhiwei.searchhotcrawler.bean.HotSearchType;
 import com.zhiwei.searchhotcrawler.mail.SendMailWeibo;
@@ -32,9 +32,9 @@ import com.zhiwei.tools.tools.URLCodeUtil;
 import org.springframework.beans.factory.annotation.Autowired;
 
 /**
- * @ClassName: WeiboHotSearch 
+ * @ClassName: WeiboHotSearch
  * @Description: 微博实时热搜采集
- * @author hero 
+ * @author hero
  * @date 2017年9月15日 上午10:54:31
  */
 @Log4j2
@@ -44,9 +44,9 @@ public class WeiboHotSearchCrawler {
 
 	private static RedisDao redisDao = new RedisDao();
 	/**
-	 * @Title: weiboHotSearchTest 
-	 * @author hero 
-	 * @Description: TODO(PC端微博热搜采集) 
+	 * @Title: weiboHotSearchTest
+	 * @author hero
+	 * @Description: TODO(PC端微博热搜采集)
 	 * @return void 返回类型
 	 */
 //	public static List<HotSearchList>  weiboHotSearch(){
@@ -103,14 +103,14 @@ public class WeiboHotSearchCrawler {
 //		}
 //		return list;
 //	}
-	
 
-	
-	
+
+
+
 	/**
-	 * @Title: weiboHotSearchByPhoneTest 
-	 * @author hero 
-	 * @Description: TODO(手机端Iphone 微博热搜采集) 
+	 * @Title: weiboHotSearchByPhoneTest
+	 * @author hero
+	 * @Description: TODO(手机端Iphone 微博热搜采集)
 	 * @return void 返回类型
 	 */
 	public static List<HotSearchList> weiboHotSearchByPhone(Date date){

src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiboSuperTopicCrawler.java

@@ -19,31 +19,31 @@ import org.slf4j.LoggerFactory;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 
 /**
- * 
+ *
  * @ClassName: WeiboSuperTopicCrawler
  * @Description: 微博超话榜单采集（明星）
- * @author Bewilder ZW  
+ * @author Bewilder ZW
  * @date 2019年9月27日 下午3:01:34
  */
 @Log4j2
 public class WeiboSuperTopicCrawler {
-		
+
 	private static HttpBoot httpBoot = new HttpBoot.Builder().retryTimes(3).build();
 	private static Map<String,String> headMap = new HashMap<>();
-	
+
 	static {
 		headMap.put("X-Requested-With", "XMLHttpRequest");
 		headMap.put("Referer", "https://huati.weibo.cn/discovery/super?extparam=ctg1_2%7Cscorll_1&luicode=10000011&lfid=100803_-_super&sourceType=weixin");
 		headMap.put("Host", "huati.weibo.cn");
-		
+
 	}
-	
+
 	/**
-	 * 
+	 *
 	 * 开始采集明星话题
 	 * @return void
 	 */
@@ -52,7 +52,7 @@ public class WeiboSuperTopicCrawler {
 		urlMap.put("明星", "https://huati.weibo.cn/aj/discovery/rank?cate_id=2&topic_to_page=&block_time=0&star_type=star&from=&wm=");
 		urlMap.put("明星潜力", "https://huati.weibo.cn/aj/discovery/rank?cate_id=2&topic_to_page=&block_time=0&star_type=potential&from=&wm=");
 		urlMap.put("明星上升", "https://huati.weibo.cn/aj/discovery/rank?cate_id=2&topic_to_page=&block_time=0&star_type=up&from=&wm=");
-		
+
 		List<WeiboSuperTopic> topicList = new ArrayList<>();
 
 		for(Entry<String,String> entry : urlMap.entrySet()) {
@@ -81,10 +81,10 @@ public class WeiboSuperTopicCrawler {
 		}
 		return topicList;
 	}
-	
-	
+
+
 	/**
-	 * 
+	 *
 	 * 解析话题榜单
 	 * @param htmlBody
 	 * @param type
@@ -95,7 +95,7 @@ public class WeiboSuperTopicCrawler {
 			JSONArray list = JSONObject.parseObject(htmlBody).getJSONObject("data").getJSONArray("list");
 			if(Objects.nonNull(list) && !list.isEmpty()) {
 				page = (page-1)*20;
-				
+
 				List<WeiboSuperTopic> topicList = new ArrayList<>();
 				Integer toprank = null;
 				String topicName = null;
@@ -125,11 +125,11 @@ public class WeiboSuperTopicCrawler {
 		}
 		return Collections.emptyList();
 	}
-	
-	
-	
+
+
+
 	/**
-	 * 
+	 *
 	 * 根据单一话题id获取话题阅读数及发帖数
 	 * @param id
 	 * @param topic
@@ -159,8 +159,8 @@ public class WeiboSuperTopicCrawler {
 		}
 		return topic;
 	}
-	
-	
-	
-	
+
+
+
+
 }

src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiboTopicCrawler.java

@@ -4,8 +4,8 @@ import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.searchhotcrawler.bean.HotSearchList;
 import com.zhiwei.searchhotcrawler.bean.HotSearchType;
 import com.zhiwei.searchhotcrawler.bean.WeiboSuperTopic;

src/main/java/com/zhiwei/searchhotcrawler/crawler/XinLangHotSearchCrawler.java

@@ -3,8 +3,8 @@ package com.zhiwei.searchhotcrawler.crawler;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.searchhotcrawler.bean.HotSearchList;
 import com.zhiwei.searchhotcrawler.bean.HotSearchType;
 import com.zhiwei.searchhotcrawler.util.TipsUtils;

src/main/java/com/zhiwei/searchhotcrawler/crawler/ZhihuChildHotSearchCrawler.java

@@ -5,8 +5,8 @@ package com.zhiwei.searchhotcrawler.crawler;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.searchhotcrawler.bean.HotSearchList;
 import com.zhiwei.searchhotcrawler.bean.HotSearchType;
 import com.zhiwei.searchhotcrawler.util.TipsUtils;

src/main/java/com/zhiwei/searchhotcrawler/crawler/ZhihuHotSearchCrawler.java

@@ -15,26 +15,26 @@ import org.slf4j.LoggerFactory;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.searchhotcrawler.bean.HotSearchList;
 import com.zhiwei.searchhotcrawler.bean.HotSearchType;
 import com.zhiwei.tools.httpclient.HeaderTool;
 import com.zhiwei.tools.tools.URLCodeUtil;
 
 /**
- * @ClassName: ZhihuHotCrawler 
+ * @ClassName: ZhihuHotCrawler
  * @Description: 知乎热搜采集程序
- * @author hero 
+ * @author hero
  * @date 2017年9月15日 上午10:54:31
  */
 @Log4j2
 public class ZhihuHotSearchCrawler {
-	
+
 	private static HttpBoot httpBoot = new HttpBoot.Builder().retryTimes(3).build();
 	/**
-	 * @Title: getZhihuHotList 
-	 * @author hero 
+	 * @Title: getZhihuHotList
+	 * @author hero
 	 * @Description: 知乎热搜采集程序
 	 * @return void 返回类型
 	 */
@@ -74,14 +74,14 @@ public class ZhihuHotSearchCrawler {
 //		}
 //		return list;
 //	}
-	
-	
-	
+
+
+
 	/**
-	 * @Title: getMobileZhihuHotList 
-	 * @author hero 
-	 * @Description: 移動端知乎熱搜榜 
-	 * @param @return 设定文件 
+	 * @Title: getMobileZhihuHotList
+	 * @author hero
+	 * @Description: 移動端知乎熱搜榜
+	 * @param @return 设定文件
 	 * @return List<ZhihuHotSearch> 返回类型
 	 */
 	public static List<HotSearchList> getMobileZhihuHotList(Date date){

src/main/java/com/zhiwei/searchhotcrawler/crawler/ZhihuTopicSearchCrawler.java

@@ -3,8 +3,8 @@ package com.zhiwei.searchhotcrawler.crawler;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.searchhotcrawler.bean.HotSearchList;
 import com.zhiwei.searchhotcrawler.bean.HotSearchType;
 import com.zhiwei.tools.httpclient.HeaderTool;

src/main/java/com/zhiwei/searchhotcrawler/util/HttpClientUtils.java

 package com.zhiwei.searchhotcrawler.util;
 
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import okhttp3.MediaType;
 import okhttp3.Request;
 import okhttp3.RequestBody;

src/main/java/com/zhiwei/searchhotcrawler/util/WechatCodeUtil.java

@@ -14,7 +14,7 @@ import org.slf4j.LoggerFactory;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.tools.httpclient.HeaderTool;
 
 import okhttp3.MediaType;
@@ -139,9 +139,9 @@ public class WechatCodeUtil {
 		}
 		return null;
 	}
-	
-	
-	
+
+
+
 	public static List<String> getUserListByGroupId(Integer groupId) {
 		try {
 			String token = getToken();
@@ -180,7 +180,7 @@ public class WechatCodeUtil {
 	}
 
 	/***
-	 * 
+	 *
 	 * @Title: getGroupIp
 	 * @author hero
 	 * @Description: 根据分组名称获取分组id
@@ -218,7 +218,7 @@ public class WechatCodeUtil {
 		}
 		return groupId;
 	}
-	
+
 	/**
 	 * 查询公众号下的所有分组
 	 * @return