Skip to content

S
searchhotcrawler
Commit 9f6d8158 by chenweitao
Options

Merge branch 'working' into 'master' 

唯一化客户端

See merge request !212
parents 235fffdd 12b6ad6b
Showing with 124 additions and 93 deletions
src/main/java/com/zhiwei/searchhotcrawler/crawler/BaiDuHotSearchCrawler.java
......@@ -9,6 +9,7 @@ import com.zhiwei.http.boot.Response;
import com.zhiwei.http.proxy.ProxyServerSupplier;
import com.zhiwei.http.proxy.ProxySupplier;
import com.zhiwei.http.util.RequestUtils;
import com.zhiwei.searchhotcrawler.util.HttpClientUtils;
import lombok.extern.log4j.Log4j2;
import okhttp3.Request;
import org.apache.commons.lang3.StringUtils;
......@@ -29,7 +30,7 @@ import com.zhiwei.searchhotcrawler.bean.HotSearchType;
public class BaiDuHotSearchCrawler {
//private static HttpBoot httpBoot = new HttpBoot.Builder().throwException(false).retryTimes(3).connectTimeout(Duration.ofSeconds(60)).build();
private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).connectTimeout(Duration.ofSeconds(60)).build();
// private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).connectTimeout(Duration.ofSeconds(60)).build();
/**
* @return void 返回类型
......@@ -41,7 +42,7 @@ public class BaiDuHotSearchCrawler {
String url = "http://top.baidu.com/buzz?b=1&fr=topindex";
String htmlBody = null;
Request request = RequestUtils.wrapGet(url);
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("解析百度风云榜时出现解析错误,页面结构有问题", cause);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/BiliComprehensiveHotCrawler.java
......@@ -9,6 +9,7 @@ import com.zhiwei.http.proxy.ProxySupplier;
import com.zhiwei.http.util.RequestUtils;
import com.zhiwei.searchhotcrawler.bean.HotSearchList;
import com.zhiwei.searchhotcrawler.bean.HotSearchType;
import com.zhiwei.searchhotcrawler.util.HttpClientUtils;
import lombok.extern.log4j.Log4j2;
import okhttp3.Request;
import org.apache.commons.lang3.StringUtils;
......@@ -28,7 +29,7 @@ import java.util.*;
public class BiliComprehensiveHotCrawler {
//private static HttpBoot httpBoot = new HttpBoot.Builder().retryTimes(3).build();
private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
// private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
/**
......@@ -54,7 +55,7 @@ public class BiliComprehensiveHotCrawler {
for (int i = 0; i < urlList.size(); i++) {
Request request = RequestUtils.wrapGet(urlList.get(i));
//发送请求每次获取20条数据
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error(fmt.format(date)+":第"+i+1+"次请求解析B站综合热门时出现连接失败", cause);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/BililiCrawler.java
......@@ -10,6 +10,7 @@ import com.zhiwei.http.proxy.ProxySupplier;
import com.zhiwei.http.util.RequestUtils;
import com.zhiwei.searchhotcrawler.bean.HotSearchList;
import com.zhiwei.searchhotcrawler.bean.HotSearchType;
import com.zhiwei.searchhotcrawler.util.HttpClientUtils;
import com.zhiwei.tools.tools.URLCodeUtil;
import com.zhiwei.tools.tools.ZhiWeiTools;
import lombok.extern.log4j.Log4j2;
......@@ -28,7 +29,7 @@ import java.util.concurrent.Executors;
@Log4j2
public class BililiCrawler {
private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
// private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
/**
* B站排行榜的采集
......@@ -44,7 +45,7 @@ public class BililiCrawler {
String url = "https://api.bilibili.com/x/web-interface/ranking/v2?rid=0&type=all";
Request request = RequestUtils.wrapGet(url);
for(int t=0; t<3 && dataJson==null; t++){
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("B站排行榜页面连接失败",cause.fillInStackTrace());
......@@ -134,7 +135,7 @@ public class BililiCrawler {
Request request = RequestUtils.wrapGet(url);
try {
System.setProperty("https.protocols", "TLSv1,TLSv1.1,TLSv1.2,SSLv3");
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
String htmlBody = response.bodyString();
if (htmlBody != null && htmlBody.contains("v-wrap")) {
Document document = Jsoup.parse(htmlBody);
......@@ -182,7 +183,7 @@ public class BililiCrawler {
String url = "https://app.biliapi.com/x/v2/search/square?build=616050&limit=10";
Request request = RequestUtils.wrapGet(url);
for(int t=0; t<3 && dataJson==null; t++){
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("B站热搜页面连接失败",cause.fillInStackTrace());
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/DouyinHotSearchCrawler.java
......@@ -8,6 +8,7 @@ import com.zhiwei.http.boot.Response;
import com.zhiwei.http.proxy.ProxyServerSupplier;
import com.zhiwei.http.proxy.ProxySupplier;
import com.zhiwei.http.util.RequestUtils;
import com.zhiwei.searchhotcrawler.util.HttpClientUtils;
import lombok.extern.log4j.Log4j2;
import okhttp3.Request;
import org.apache.commons.lang3.StringUtils;
......@@ -29,7 +30,7 @@ import com.zhiwei.searchhotcrawler.bean.HotSearchType;
@Log4j2
public class DouyinHotSearchCrawler {
private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
// private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
public static List<HotSearchList> list = new ArrayList<>();
......@@ -47,7 +48,7 @@ public class DouyinHotSearchCrawler {
String url = "https://api.amemv.com/aweme/v1/hot/search/list/";
String htmlBody = null;
Request request = RequestUtils.wrapGet(url);
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.debug("获取抖音热搜榜时出现问题:{}", cause);
......@@ -91,7 +92,7 @@ public class DouyinHotSearchCrawler {
String htmlBody = null;
Request request = RequestUtils.wrapGet(url);
for (int x = 0; x < 3; x++) {
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.debug("获取抖音热搜榜链接时出现问题:{}", cause);
......@@ -126,7 +127,7 @@ public class DouyinHotSearchCrawler {
String url = "https://api5-normal-c-lq.amemv.com/aweme/v1/hot/search/list/?board_type=2&board_sub_type=2&version_code=140900";
String htmlBody = null;
Request request = RequestUtils.wrapGet(url);
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.debug("获取抖音娱乐榜榜时出现问题:{}", cause);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/FengHuangSearchCrawler.java
......@@ -10,6 +10,7 @@ import com.zhiwei.http.proxy.ProxySupplier;
import com.zhiwei.http.util.RequestUtils;
import com.zhiwei.searchhotcrawler.bean.HotSearchList;
import com.zhiwei.searchhotcrawler.bean.HotSearchType;
import com.zhiwei.searchhotcrawler.util.HttpClientUtils;
import com.zhiwei.searchhotcrawler.util.TipsUtils;
import lombok.extern.log4j.Log4j2;
import okhttp3.Request;
......@@ -22,7 +23,7 @@ import java.util.List;
@Log4j2
public class FengHuangSearchCrawler {
private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
// private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
/**
* 获取凤凰新闻热榜
......@@ -35,7 +36,7 @@ public class FengHuangSearchCrawler {
String url = "https://nine.ifeng.com/hotspotlist?gv=7.9.1&page="+page;
String htmlBody = null;
Request request = RequestUtils.wrapGet(url);
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("凤凰新闻热榜页面连接异常...", cause);
......@@ -75,7 +76,7 @@ public class FengHuangSearchCrawler {
String url = "https://shankapi.ifeng.com/autumn/sogouSearchHotword";
String htmlBody = null;
Request request = RequestUtils.wrapGet(url);
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("凤凰新闻热搜页面连接异常...", cause);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/HotSearch36KrCrawler.java
......@@ -7,6 +7,7 @@ import com.zhiwei.http.proxy.ProxySupplier;
import com.zhiwei.http.util.RequestUtils;
import com.zhiwei.searchhotcrawler.bean.HotSearchList;
import com.zhiwei.searchhotcrawler.bean.HotSearchType;
import com.zhiwei.searchhotcrawler.util.HttpClientUtils;
import lombok.extern.log4j.Log4j2;
import okhttp3.Request;
......@@ -28,7 +29,7 @@ import java.util.*;
@Log4j2
public class HotSearch36KrCrawler {
private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).connectTimeout(Duration.ofSeconds(60)).build();
// private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).connectTimeout(Duration.ofSeconds(60)).build();
/**
* @return void 返回类型
......@@ -47,7 +48,7 @@ public class HotSearch36KrCrawler {
headerMap.put("sec-fetch-dest","empty");
String htmlBody = null;
Request request = RequestUtils.wrapGet(url,headerMap);
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("解析36Kr人气榜时出现解析错误,页面结构有问题", cause);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/HuXiuHotSearchCrawler.java
......@@ -10,6 +10,7 @@ import com.zhiwei.http.proxy.ProxySupplier;
import com.zhiwei.http.util.RequestUtils;
import com.zhiwei.searchhotcrawler.bean.HotSearchList;
import com.zhiwei.searchhotcrawler.bean.HotSearchType;
import com.zhiwei.searchhotcrawler.util.HttpClientUtils;
import lombok.extern.log4j.Log4j2;
import okhttp3.Request;
......@@ -30,7 +31,7 @@ import java.util.*;
*/
@Log4j2
public class HuXiuHotSearchCrawler {
private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).connectTimeout(Duration.ofSeconds(60)).build();
// private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).connectTimeout(Duration.ofSeconds(60)).build();
public static List<HotSearchList> HuXiuHotArticleRecommended(Date date){
......@@ -43,7 +44,7 @@ public class HuXiuHotSearchCrawler {
headerMap.put("sec-ch-ua"," Not A;Brand\";v=\"99\", \"Chromium\";v=\"101\", \"Microsoft Edge\";v=\"101");
String htmlBody = null;
Request request = RequestUtils.wrapGet(url,headerMap);
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("解析虎嗅热文推荐时出现解析错误,页面结构有问题", cause);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/KuaiShouHotSearchCrawler.java
......@@ -9,6 +9,7 @@ import com.zhiwei.http.proxy.ProxySupplier;
import com.zhiwei.http.util.RequestUtils;
import com.zhiwei.searchhotcrawler.bean.HotSearchList;
import com.zhiwei.searchhotcrawler.bean.HotSearchType;
import com.zhiwei.searchhotcrawler.util.HttpClientUtils;
import lombok.extern.log4j.Log4j2;
import okhttp3.Request;
......@@ -25,7 +26,7 @@ import java.util.*;
*/
@Log4j2
public class KuaiShouHotSearchCrawler {
private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).connectTimeout(Duration.ofSeconds(60)).build();
// private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).connectTimeout(Duration.ofSeconds(60)).build();
/**
* @return void 返回类型
......@@ -37,7 +38,7 @@ public class KuaiShouHotSearchCrawler {
String url = "https://video.kuaishou.com/?utm_source=aa&utm_medium=05&utm_campaign=aa_05_pp_yr&plan_id=138090084&unit_id=5205658029&creative_id=43661481717&keyword_id=202928529242&keyword=202928529242&bd_vid=11937382025080724791";
String htmlBody = null;
Request request = RequestUtils.wrapGet(url);
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("解析快手热榜时出现解析错误,页面结构有问题", cause);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/MaiMaiHotSearchCrawler.java
......@@ -10,6 +10,7 @@ import com.zhiwei.http.proxy.ProxySupplier;
import com.zhiwei.http.util.RequestUtils;
import com.zhiwei.searchhotcrawler.bean.HotSearchList;
import com.zhiwei.searchhotcrawler.bean.HotSearchType;
import com.zhiwei.searchhotcrawler.util.HttpClientUtils;
import com.zhiwei.searchhotcrawler.util.TipsUtils;
import lombok.extern.log4j.Log4j2;
import okhttp3.Request;
......@@ -24,7 +25,7 @@ import java.util.List;
@Log4j2
public class MaiMaiHotSearchCrawler {
private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
// private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
/**
* 获取maimai热榜
......@@ -36,7 +37,7 @@ public class MaiMaiHotSearchCrawler {
String url = "https://open.taou.com/maimai/feed/v6/hot_list_entry/feeds?page_version=2&version=6.2.34&u=232258287&access_token=1.4c82e8ad6d6b4e03262a48f334dea336";
String htmlBody = null;
Request request = RequestUtils.wrapGet(url);
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("脉脉热榜页面连接异常...", cause);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/SougoHotSearchCrawler.java
......@@ -10,6 +10,7 @@ import com.zhiwei.http.proxy.ProxyServerSupplier;
import com.zhiwei.http.proxy.ProxySupplier;
import com.zhiwei.http.util.RequestUtils;
import com.zhiwei.searchhotcrawler.util.HeaderTool;
import com.zhiwei.searchhotcrawler.util.HttpClientUtils;
import com.zhiwei.tools.tools.URLCodeUtil;
import lombok.extern.log4j.Log4j2;
import okhttp3.Request;
......@@ -32,7 +33,7 @@ import com.zhiwei.searchhotcrawler.bean.HotSearchType;
@Log4j2
public class SougoHotSearchCrawler {
private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
// private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
/**
* @Title: SougoHotSearchTest
......@@ -47,7 +48,7 @@ public class SougoHotSearchCrawler {
Request request = RequestUtils.wrapGet(url);
for (int i = 0; i < 3; i++) {
String htmlBody = null;
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("解析搜狗微信时出现解析错误,页面结构有问题", cause);
......@@ -91,7 +92,7 @@ public class SougoHotSearchCrawler {
Request request = RequestUtils.wrapGet(url, headMap);
for (int i = 0; i < 3; i++) {
String htmlBody = null;
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("解析搜狗微信时出现解析错误,页面结构有问题", cause);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/SouhuTopicCrawler.java
......@@ -9,6 +9,7 @@ import com.zhiwei.http.proxy.ProxySupplier;
import com.zhiwei.http.util.RequestUtils;
import com.zhiwei.searchhotcrawler.bean.HotSearchList;
import com.zhiwei.searchhotcrawler.bean.HotSearchType;
import com.zhiwei.searchhotcrawler.util.HttpClientUtils;
import com.zhiwei.searchhotcrawler.util.TipsUtils;
import com.zhiwei.tools.tools.ZhiWeiTools;
import lombok.extern.log4j.Log4j2;
......@@ -22,7 +23,7 @@ import java.util.List;
@Log4j2
public class SouhuTopicCrawler {
private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
// private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
public static List<HotSearchList> getSouhuTopic(Date date){
List<HotSearchList> hotSearchLists = new ArrayList<>();
......@@ -32,7 +33,7 @@ public class SouhuTopicCrawler {
String url = "https://api.k.sohu.com/api/news/moment/v2/list.go?pageSize=50&v=6.4.4";
Request request = RequestUtils.wrapGet(url);
for(int t=0; t<3 && dataJson==null; t++){
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("搜狐话题页面连接失败",cause.fillInStackTrace());
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/TaoBaoHotSearchCrawler.java
......@@ -9,6 +9,7 @@ import com.zhiwei.http.proxy.ProxySupplier;
import com.zhiwei.http.util.RequestUtils;
import com.zhiwei.searchhotcrawler.bean.HotSearchList;
import com.zhiwei.searchhotcrawler.bean.HotSearchType;
import com.zhiwei.searchhotcrawler.util.HttpClientUtils;
import com.zhiwei.searchhotcrawler.util.MD5Util;
import lombok.extern.log4j.Log4j2;
import okhttp3.Request;
......@@ -25,7 +26,7 @@ import java.util.*;
*/
@Log4j2
public class TaoBaoHotSearchCrawler {
private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).connectTimeout(Duration.ofSeconds(60)).build();
// private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).connectTimeout(Duration.ofSeconds(60)).build();
public static List<HotSearchList> taoBaoHotSearch(Date date) {
Map<String, String> headerMap = new HashMap<>();
......@@ -36,7 +37,7 @@ public class TaoBaoHotSearchCrawler {
String urls = "https://acs.m.taobao.com/h5/mtop.relationrecommend.wirelessrecommend.recommend/2.0/?appKey=12574478&t=" + time + "&sign=&api=mtop.relationrecommend.WirelessRecommend.recommend&v=2.0&type=jsonp&dataType=jsonp&callback=mtopjsonp1&data=%7B%22appId%22%3A%2210211%22%2C%22params%22%3A%22%7B%5C%22multi_hintq_show%5C%22%3A%5C%22on%5C%22%2C%5C%22src%5C%22%3A%5C%22c2c%5C%22%2C%5C%22area%5C%22%3A%5C%22active_page%5C%22%2C%5C%22sversion%5C%22%3A%5C%227.5%5C%22%2C%5C%22bangdan_src%5C%22%3A%5C%22list%5C%22%7D%22%7D";
Request request1 = RequestUtils.wrapGet(urls);
String token = null;
Response response = httpBoot.syncCall(request1, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request1, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("解析淘宝热搜时出现解析错误,页面结构有问题", cause);
......@@ -55,7 +56,7 @@ public class TaoBaoHotSearchCrawler {
String sign = MD5Util.getMD5(signs).toLowerCase();
String url = "https://acs.m.taobao.com/h5/mtop.relationrecommend.wirelessrecommend.recommend/2.0/?appKey=12574478&t=" + time + "&sign=" + sign + "&api=mtop.relationrecommend.WirelessRecommend.recommend&v=2.0&type=jsonp&dataType=jsonp&callback=mtopjsonp1&data=%7B%22appId%22%3A%2210211%22%2C%22params%22%3A%22%7B%5C%22multi_hintq_show%5C%22%3A%5C%22on%5C%22%2C%5C%22src%5C%22%3A%5C%22c2c%5C%22%2C%5C%22area%5C%22%3A%5C%22active_page%5C%22%2C%5C%22sversion%5C%22%3A%5C%227.5%5C%22%2C%5C%22bangdan_src%5C%22%3A%5C%22list%5C%22%7D%22%7D";
Request request = RequestUtils.wrapGet(url, headerMap);
Response response1 = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response1 = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response1.hasCause()){
Throwable cause = response1.cause();
log.error("解析淘宝热搜时出现解析错误,页面结构有问题", cause);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/TengXunCrawler.java
......@@ -9,6 +9,7 @@ import com.zhiwei.http.proxy.ProxySupplier;
import com.zhiwei.http.util.RequestUtils;
import com.zhiwei.searchhotcrawler.bean.HotSearchList;
import com.zhiwei.searchhotcrawler.bean.HotSearchType;
import com.zhiwei.searchhotcrawler.util.HttpClientUtils;
import com.zhiwei.tools.tools.ZhiWeiTools;
import lombok.extern.log4j.Log4j2;
import okhttp3.Request;
......@@ -21,7 +22,7 @@ import java.util.List;
@Log4j2
public class TengXunCrawler {
private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
// private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
/**
* 腾讯热榜数据采集
......@@ -36,7 +37,7 @@ public class TengXunCrawler {
Request request = RequestUtils.wrapGet(url);
//采集为空最多重试3次
for (int t = 0; t < 3 && dataJson == null; t++) {
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
response.cause().printStackTrace();
......@@ -100,7 +101,7 @@ public class TengXunCrawler {
Request request = RequestUtils.wrapGet(url);
//采集为空最多重试3次
for (int t = 0; t < 3; t++) {
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
response.cause().printStackTrace();
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/ToutiaoHotSearchCrawler.java
......@@ -12,6 +12,7 @@ import com.zhiwei.http.util.RequestUtils;
import com.zhiwei.searchhotcrawler.bean.HotSearchList;
import com.zhiwei.searchhotcrawler.bean.HotSearchType;
import com.zhiwei.searchhotcrawler.dao.HotSearchListDAO;
import com.zhiwei.searchhotcrawler.util.HttpClientUtils;
import com.zhiwei.searchhotcrawler.util.TipsUtils;
import com.zhiwei.tools.tools.URLCodeUtil;
import com.zhiwei.tools.tools.ZhiWeiTools;
......@@ -37,7 +38,7 @@ import java.util.*;
*/
@Log4j2
public class ToutiaoHotSearchCrawler {
private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
// private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
/**
* @return void 返回类型
......@@ -50,7 +51,7 @@ public class ToutiaoHotSearchCrawler {
String jsUrl = "https://s3.pstatp.com/toutiao/feoffline/hot_list/resource/hot_list/js/index.45f50250.chunk.js";
Request jsRequest = RequestUtils.wrapGet(jsUrl);
String jsBody = null;
Response response = httpBoot.syncCall(jsRequest, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(jsRequest, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()) {
Throwable cause = response.cause();
log.error("获取今日头条实时热搜头部信息标识失败", cause);
......@@ -66,7 +67,7 @@ public class ToutiaoHotSearchCrawler {
String htmlBody = null;
Request request = RequestUtils.wrapGet(url);
for (int count = 0; count <= 5; count++) {
Response response1 = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response1 = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response1.hasCause()) {
Throwable cause = response1.cause();
log.error("解析今日头条实时热搜时出现连接失败", cause);
......@@ -166,7 +167,7 @@ public class ToutiaoHotSearchCrawler {
String htmlBody = null;
String url = hotSearchList.getUrl();
Request request = RequestUtils.wrapGet(url);
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()) {
Throwable cause = response.cause();
log.error("解析今日头条热搜详情页面出现连接失败", cause);
......@@ -227,7 +228,7 @@ public class ToutiaoHotSearchCrawler {
headerMap.put("User-Agent", "com.ss.android.article.news/8770 (Linux; U; Android 9; zh_CN; Redmi 8; Build/PKQ1.190319.001; Cronet/TTNetVersion:a867b489 2022-03-11 QuicVersion:b314d107 2021-11-24) Accept-Encoding: gzip, deflate, br");
Request request = RequestUtils.wrapGet(url, headerMap);
String htmlBody = null;
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()) {
Throwable cause = response.cause();
log.error("获取今日头条榜单出错", cause);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/WangYiHotSearchCrawler.java
......@@ -10,6 +10,7 @@ import com.zhiwei.http.proxy.ProxySupplier;
import com.zhiwei.http.util.RequestUtils;
import com.zhiwei.searchhotcrawler.bean.HotSearchList;
import com.zhiwei.searchhotcrawler.bean.HotSearchType;
import com.zhiwei.searchhotcrawler.util.HttpClientUtils;
import com.zhiwei.searchhotcrawler.util.TipsUtils;
import com.zhiwei.tools.tools.ZhiWeiTools;
import lombok.extern.log4j.Log4j2;
......@@ -29,7 +30,7 @@ import java.util.List;
*/
@Log4j2
public class WangYiHotSearchCrawler {
private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
// private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
/**
* 网易新闻实时热榜的采集
......@@ -42,7 +43,7 @@ public class WangYiHotSearchCrawler {
Request request = RequestUtils.wrapGet(url);
String htmlBody = null;
for(int t=0 ;t<3; t++) {
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("网易新闻实时热榜页面连接异常...",cause);
......@@ -85,7 +86,7 @@ public class WangYiHotSearchCrawler {
Request request = RequestUtils.wrapGet(url);
String htmlBody = null;
for(int t=0 ;t<3; t++) {
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("网易新闻跟贴热议页面连接异常...",cause);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiBoBrandCrawler.java
......@@ -9,6 +9,7 @@ import com.zhiwei.http.proxy.ProxySupplier;
import com.zhiwei.http.util.RequestUtils;
import com.zhiwei.searchhotcrawler.bean.HotSearchList;
import com.zhiwei.searchhotcrawler.bean.HotSearchType;
import com.zhiwei.searchhotcrawler.util.HttpClientUtils;
import lombok.extern.log4j.Log4j2;
import okhttp3.Request;
......@@ -27,7 +28,7 @@ import java.util.Objects;
@Log4j2
public class WeiBoBrandCrawler {
private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
// private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
/**
......@@ -43,7 +44,7 @@ public class WeiBoBrandCrawler {
Request request = RequestUtils.wrapGet(url);
//重试两次
for (int x = 0; x < 2; x++) {
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()) {
Throwable cause = response.cause();
log.error("解析微博品牌总榜时出现连接失败", cause);
......@@ -77,7 +78,7 @@ public class WeiBoBrandCrawler {
Request request = RequestUtils.wrapGet(url);
//重试两次
for (int x = 0; x < 2; x++) {
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()) {
Throwable cause = response.cause();
log.error("解析微博品牌汽车榜时出现连接失败", cause);
......@@ -111,7 +112,7 @@ public class WeiBoBrandCrawler {
Request request = RequestUtils.wrapGet(url);
//重试两次
for (int x = 0; x < 2; x++) {
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()) {
Throwable cause = response.cause();
log.error("解析微博品牌手机榜时出现连接失败", cause);
......@@ -145,7 +146,7 @@ public class WeiBoBrandCrawler {
Request request = RequestUtils.wrapGet(url);
//重试两次
for (int x = 0; x < 2; x++) {
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()) {
Throwable cause = response.cause();
log.error("解析微博品牌美妆榜时出现连接失败", cause);
......@@ -179,7 +180,7 @@ public class WeiBoBrandCrawler {
Request request = RequestUtils.wrapGet(url);
//重试两次
for (int x = 0; x < 2; x++) {
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()) {
Throwable cause = response.cause();
log.error("解析微博品牌奢侈品榜时出现连接失败", cause);
......@@ -213,7 +214,7 @@ public class WeiBoBrandCrawler {
Request request = RequestUtils.wrapGet(url);
//重试两次
for (int x = 0; x < 2; x++) {
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()) {
Throwable cause = response.cause();
log.error("解析微博品牌食品饮料榜时出现连接失败", cause);
......@@ -247,7 +248,7 @@ public class WeiBoBrandCrawler {
Request request = RequestUtils.wrapGet(url);
//重试两次
for (int x = 0; x < 2; x++) {
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()) {
Throwable cause = response.cause();
log.error("解析微博品牌家电榜时出现连接失败", cause);
......@@ -281,7 +282,7 @@ public class WeiBoBrandCrawler {
Request request = RequestUtils.wrapGet(url);
//重试两次
for (int x = 0; x < 2; x++) {
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()) {
Throwable cause = response.cause();
log.error("解析微博品牌服装鞋帽榜时出现连接失败", cause);
......@@ -315,7 +316,7 @@ public class WeiBoBrandCrawler {
Request request = RequestUtils.wrapGet(url);
//重试两次
for (int x = 0; x < 2; x++) {
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()) {
Throwable cause = response.cause();
log.error("解析微博品牌服装鞋帽榜时出现连接失败", cause);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiBoSearchBoxHotWordsCrawler.java
......@@ -9,6 +9,7 @@ import com.zhiwei.http.proxy.ProxySupplier;
import com.zhiwei.http.util.RequestUtils;
import com.zhiwei.searchhotcrawler.bean.WeiBoSearchBoxHotWords;
import com.zhiwei.searchhotcrawler.dao.WeiBoSearchBoxHotWordsDao;
import com.zhiwei.searchhotcrawler.util.HttpClientUtils;
import lombok.extern.log4j.Log4j2;
import okhttp3.Request;
......@@ -28,7 +29,7 @@ import java.util.Objects;
*/
@Log4j2
public class WeiBoSearchBoxHotWordsCrawler {
private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).connectTimeout(Duration.ofSeconds(60)).build();
// private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).connectTimeout(Duration.ofSeconds(60)).build();
static WeiBoSearchBoxHotWordsDao weiBoSearchDao = new WeiBoSearchBoxHotWordsDao();
public static void weiBoSearchBoxHotWords(Date date){
......@@ -37,7 +38,7 @@ public class WeiBoSearchBoxHotWordsCrawler {
String htmlBody = null;
Request request = RequestUtils.wrapGet(url);
for (int count = 0; count <= 5; count++) {
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("解析微博搜索框热词时出现解析错误,页面结构有问题",cause);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiBoSearchHotWordsCrawler.java
......@@ -9,6 +9,7 @@ import com.zhiwei.http.proxy.ProxySupplier;
import com.zhiwei.http.util.RequestUtils;
import com.zhiwei.searchhotcrawler.bean.WeiBoSearchBoxHotWords;
import com.zhiwei.searchhotcrawler.dao.WeiBoSearchBoxHotWordsDao;
import com.zhiwei.searchhotcrawler.util.HttpClientUtils;
import lombok.extern.log4j.Log4j2;
import okhttp3.Request;
......@@ -27,7 +28,7 @@ import java.util.Objects;
*/
@Log4j2
public class WeiBoSearchHotWordsCrawler {
private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).connectTimeout(Duration.ofSeconds(60)).build();
// private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).connectTimeout(Duration.ofSeconds(60)).build();
static WeiBoSearchBoxHotWordsDao weiBoSearchDao = new WeiBoSearchBoxHotWordsDao();
public static void weiBoSearchHotWords(Date date){
......@@ -36,7 +37,7 @@ public class WeiBoSearchHotWordsCrawler {
String htmlBody = null;
Request request = RequestUtils.wrapGet(url);
for (int count = 0; count <= 5; count++) {
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiShiHotSearchCrawler.java
......@@ -9,6 +9,7 @@ import com.zhiwei.http.proxy.ProxySupplier;
import com.zhiwei.http.util.RequestUtils;
import com.zhiwei.searchhotcrawler.bean.HotSearchList;
import com.zhiwei.searchhotcrawler.bean.HotSearchType;
import com.zhiwei.searchhotcrawler.util.HttpClientUtils;
import lombok.extern.log4j.Log4j2;
import okhttp3.MediaType;
import okhttp3.Request;
......@@ -27,7 +28,7 @@ import java.util.*;
@Log4j2
public class WeiShiHotSearchCrawler {
private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
// private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
/**
......@@ -45,7 +46,7 @@ public class WeiShiHotSearchCrawler {
headerMap.put("Host","api.weishi.qq.com");
Request request = RequestUtils.wrapPost(url,headerMap,RequestBody.create(MediaType.get("application/json"),"{\"req_body\":{\"hotRankID\":\"\",\"attachInfo\":\"\",\"hotRankType\":1,\"sourceID\":\"WSSearchH5\"}}"));
for (int count = 0; count <=3; count++) {
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("解析微视热榜时出现连接失败", cause);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiboEntertainmentCrawler.java
......@@ -9,6 +9,7 @@ import com.zhiwei.http.proxy.ProxySupplier;
import com.zhiwei.http.util.RequestUtils;
import com.zhiwei.searchhotcrawler.bean.HotSearchList;
import com.zhiwei.searchhotcrawler.bean.HotSearchType;
import com.zhiwei.searchhotcrawler.util.HttpClientUtils;
import lombok.extern.log4j.Log4j2;
import okhttp3.Request;
import org.apache.commons.lang3.StringUtils;
......@@ -24,7 +25,7 @@ import java.util.*;
@Log4j2
public class WeiboEntertainmentCrawler {
private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
// private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
/**
* @return void 返回类型
......@@ -38,7 +39,7 @@ public class WeiboEntertainmentCrawler {
String htmlBody = null;
Request request = RequestUtils.wrapGet(url);
for (int count = 0; count <= 5; count++) {
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("解析微博娱乐榜时出现连接失败",cause);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiboHotSearchCrawler.java
......@@ -20,6 +20,7 @@ import com.zhiwei.searchhotcrawler.config.RedisConfig;
import com.zhiwei.searchhotcrawler.dao.RedisDao;
import com.zhiwei.searchhotcrawler.dao.WeiBoMassageDao;
import com.zhiwei.searchhotcrawler.dao.WeiBoUserDao;
import com.zhiwei.searchhotcrawler.util.HttpClientUtils;
import com.zhiwei.searchhotcrawler.util.TipsUtils;
import lombok.extern.log4j.Log4j2;
import okhttp3.Request;
......@@ -47,7 +48,7 @@ import static java.util.Objects.nonNull;
@Log4j2
public class WeiboHotSearchCrawler {
private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
// private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
private static RedisDao redisDao = new RedisDao();
......@@ -68,7 +69,7 @@ public class WeiboHotSearchCrawler {
for (int i = 0; i < 3; i++) {
String htmlBody = null;
Request request = RequestUtils.wrapGet(url, headerMap);
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
if (i == 2) {
return list;
......@@ -264,7 +265,7 @@ public class WeiboHotSearchCrawler {
String htmlBody = null;
Request request = RequestUtils.wrapGet(url, headerMap);
for (int count = 0; count <= 5; count++) {
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("解析微博时热搜时出现连接失败",cause);
......@@ -362,7 +363,7 @@ public class WeiboHotSearchCrawler {
String url = "https://api.weibo.cn/2/guest/page?c=android&s=3d477777&from=10A8395010&gsid=_2AkMoFNQvf8NhqwJRm_gWy2rkbo1_yA7EieKeSCX0JRM3HRl-wT9kqkIltRV6A-gElEGNj31RgrfclQ31YPAf7UBZPBx2&containerid=106003type%3D25%26t%3D3%26disable_hot%3D1%26filter_type%3Drealtimehot";
String htmlBody = null;
Request request = RequestUtils.wrapGet(url);
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("解析微博热搜时出现连接失败",cause);
......@@ -420,7 +421,7 @@ public class WeiboHotSearchCrawler {
String htmlBody = null;
Request request = RequestUtils.wrapGet(url);
for (int count = 0; count <= 5; count++) {
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("解析微博热搜详情页面时出现连接失败",cause);
......@@ -518,7 +519,7 @@ public class WeiboHotSearchCrawler {
//该cookie有效期一年,微博pc端获取游客cookie链接 https://s.weibo.com/top/summary?cate=realtimehot
headerMap.put("Cookie", "SUB=_2AkMUShJMf8NxqwJRmP0RyWvgb4RwwgnEieKiFuOXJRMxHRl-yT92qlQvtRB6P8o8oso9Ew-s6vf16fdCca-Xz6DwwAMH; SUBP=0033WrSXqPxfM72-Ws9jqgMF55529P9D9WFdAobr6HdAbgQQ9vbUQKDx");
Request request = RequestUtils.wrapGet(url,headerMap);
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("解析微博时热搜时出现连接失败",cause);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiboNewsCrawler.java
......@@ -10,6 +10,7 @@ import com.zhiwei.http.proxy.ProxySupplier;
import com.zhiwei.http.util.RequestUtils;
import com.zhiwei.searchhotcrawler.bean.HotSearchList;
import com.zhiwei.searchhotcrawler.bean.HotSearchType;
import com.zhiwei.searchhotcrawler.util.HttpClientUtils;
import lombok.extern.log4j.Log4j2;
import okhttp3.Request;
......@@ -28,7 +29,7 @@ import java.util.*;
@Log4j2
public class WeiboNewsCrawler {
private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
// private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
/**
......@@ -49,7 +50,7 @@ public class WeiboNewsCrawler {
for (int count = 0; count <= 5; count++) {
List<HotSearchList> result = new ArrayList();
//发送第一次请求获取前20条数据
Response response = httpBoot.syncCall(request1, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request1, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("第一次请求解析微博要闻榜时出现连接失败", cause);
......@@ -71,7 +72,7 @@ public class WeiboNewsCrawler {
continue;
}
//发送第二次请求获取中间20条数据
Response response1 = httpBoot.syncCall(request2, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response1 = HttpClientUtils.httpBoot.syncCall(request2, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response1.hasCause()){
Throwable cause = response1.cause();
log.error("第二次请求解析微博要闻榜时出现连接失败",cause);
......@@ -94,7 +95,7 @@ public class WeiboNewsCrawler {
continue;
}
//发送第三次请求获取最后10条数据
Response response2 = httpBoot.syncCall(request3, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response2 = HttpClientUtils.httpBoot.syncCall(request3, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response2.hasCause()){
Throwable cause = response2.cause();
log.error("第三次请求解析微博要闻榜时出现连接失败",cause);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiboOutCircleCrawler.java
......@@ -7,6 +7,7 @@ import com.zhiwei.http.proxy.ProxySupplier;
import com.zhiwei.http.util.RequestUtils;
import com.zhiwei.searchhotcrawler.bean.HotSearchList;
import com.zhiwei.searchhotcrawler.bean.HotSearchType;
import com.zhiwei.searchhotcrawler.util.HttpClientUtils;
import lombok.extern.log4j.Log4j2;
import okhttp3.Request;
import org.apache.commons.lang3.StringUtils;
......@@ -27,7 +28,7 @@ import java.util.*;
@Log4j2
public class WeiboOutCircleCrawler {
private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
// private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
/**
......@@ -43,7 +44,7 @@ public class WeiboOutCircleCrawler {
String htmlBody = null;
Request request = RequestUtils.wrapGet(url,headerMap);
for (int x = 0; x <= 2; x++) {
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("解析微博出圈榜时出现连接失败", cause);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiboSuperTopicCrawler.java
......@@ -14,6 +14,7 @@ import com.zhiwei.http.proxy.ProxyServerSupplier;
import com.zhiwei.http.proxy.ProxySupplier;
import com.zhiwei.http.util.RequestUtils;
import com.zhiwei.searchhotcrawler.bean.WeiboSuperTopic;
import com.zhiwei.searchhotcrawler.util.HttpClientUtils;
import lombok.extern.log4j.Log4j2;
import okhttp3.Request;
import org.apache.commons.lang3.StringUtils;
......@@ -33,7 +34,7 @@ import com.alibaba.fastjson.JSONObject;
@Log4j2
public class WeiboSuperTopicCrawler {
private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
// private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
private static Map<String,String> headMap = new HashMap<>();
static {
......@@ -65,7 +66,7 @@ public class WeiboSuperTopicCrawler {
String htmlBody = null;
//重试三次
for(int retryTimes = 1; retryTimes<=3; retryTimes++) {
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("获取榜单列表页面时出现错误，错误为:{}",cause);
......@@ -141,7 +142,7 @@ public class WeiboSuperTopicCrawler {
String url = "https://m.weibo.cn/api/container/getIndex?containerid="+ id;
Request request = RequestUtils.wrapGet(url);
String htmlBody = null;
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("解析榜单详情页面时出现错误，错误为:{}",cause);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiboTopicCrawler.java
......@@ -11,6 +11,7 @@ import com.zhiwei.http.util.RequestUtils;
import com.zhiwei.searchhotcrawler.bean.HotSearchList;
import com.zhiwei.searchhotcrawler.bean.HotSearchType;
import com.zhiwei.searchhotcrawler.bean.WeiboSuperTopic;
import com.zhiwei.searchhotcrawler.util.HttpClientUtils;
import com.zhiwei.searchhotcrawler.util.TipsUtils;
import com.zhiwei.tools.tools.URLCodeUtil;
import lombok.extern.log4j.Log4j2;
......@@ -34,7 +35,7 @@ import java.util.regex.Pattern;
*/
@Log4j2
public class WeiboTopicCrawler {
private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
// private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
private static Map<String,String> headMap = new HashMap<>();
static {
......@@ -140,7 +141,7 @@ public class WeiboTopicCrawler {
String htmlBody = null;
//重试三次
for(int retryTimes = 1; retryTimes<=5; retryTimes++) {
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("下载榜单列表页面时出现错误，错误为:{}", cause);
......@@ -229,7 +230,7 @@ public class WeiboTopicCrawler {
String htmlBody = null;
//重试三次
for(int retryTimes = 1; retryTimes<=3; retryTimes++) {
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("下载榜单列表页面时出现错误，错误为:{}", cause);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiboVideoCrawler.java
......@@ -9,6 +9,7 @@ import com.zhiwei.http.proxy.ProxySupplier;
import com.zhiwei.http.util.RequestUtils;
import com.zhiwei.searchhotcrawler.bean.HotSearchList;
import com.zhiwei.searchhotcrawler.bean.HotSearchType;
import com.zhiwei.searchhotcrawler.util.HttpClientUtils;
import lombok.extern.log4j.Log4j2;
import okhttp3.Request;
import org.apache.commons.lang3.StringUtils;
......@@ -28,7 +29,7 @@ import java.util.List;
@Log4j2
public class WeiboVideoCrawler {
private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
// private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
/**
......@@ -42,7 +43,7 @@ public class WeiboVideoCrawler {
String htmlBody = null;
Request request = RequestUtils.wrapGet(url);
for (int x = 0; x <= 2; x++) {
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("解析微博视频榜时出现连接失败", cause);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/XinLangHotSearchCrawler.java
......@@ -9,6 +9,7 @@ import com.zhiwei.http.proxy.ProxySupplier;
import com.zhiwei.http.util.RequestUtils;
import com.zhiwei.searchhotcrawler.bean.HotSearchList;
import com.zhiwei.searchhotcrawler.bean.HotSearchType;
import com.zhiwei.searchhotcrawler.util.HttpClientUtils;
import com.zhiwei.searchhotcrawler.util.TipsUtils;
import com.zhiwei.tools.tools.URLCodeUtil;
import com.zhiwei.tools.tools.ZhiWeiTools;
......@@ -26,7 +27,7 @@ import java.util.*;
@Log4j2
public class XinLangHotSearchCrawler {
private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
// private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
/**
* 新浪热榜的采集
......@@ -40,7 +41,7 @@ public class XinLangHotSearchCrawler {
String htmlBody = null;
JSONObject jsonObject = null;
for(int t=0 ;t<3&&jsonObject==null; t++) {
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("新浪热榜页面连接异常...",cause);
......@@ -116,7 +117,7 @@ public class XinLangHotSearchCrawler {
String htmlBody = null;
JSONArray dataJson = null;
for(int t=0 ;t<3&&dataJson==null; t++) {
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("新浪热点页面连接异常...",cause);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/ZhihuChildHotSearchCrawler.java
......@@ -12,6 +12,7 @@ import com.zhiwei.http.proxy.ProxySupplier;
import com.zhiwei.http.util.RequestUtils;
import com.zhiwei.searchhotcrawler.bean.HotSearchList;
import com.zhiwei.searchhotcrawler.bean.HotSearchType;
import com.zhiwei.searchhotcrawler.util.HttpClientUtils;
import com.zhiwei.searchhotcrawler.util.TipsUtils;
import com.zhiwei.tools.tools.ZhiWeiTools;
import lombok.extern.log4j.Log4j2;
......@@ -24,7 +25,7 @@ import java.util.*;
@Log4j2
public class ZhihuChildHotSearchCrawler {
private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
// private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
/**
* 知乎子级分类数据采集
......@@ -42,7 +43,7 @@ public class ZhihuChildHotSearchCrawler {
Request request = RequestUtils.wrapGet(url, headerMap);
//采集为空最多重试3次
for (int t = 0; t < 3 && dataJson == null; t++) {
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
response.cause().printStackTrace();
}else {
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/ZhihuHotSearchCrawler.java
......@@ -8,6 +8,7 @@ import com.zhiwei.http.boot.Response;
import com.zhiwei.http.proxy.ProxyServerSupplier;
import com.zhiwei.http.proxy.ProxySupplier;
import com.zhiwei.http.util.RequestUtils;
import com.zhiwei.searchhotcrawler.util.HttpClientUtils;
import io.netty.handler.ssl.SslProvider;
import lombok.extern.log4j.Log4j2;
import okhttp3.Request;
......@@ -35,7 +36,7 @@ import static java.util.Objects.nonNull;
@Log4j2
public class ZhihuHotSearchCrawler {
private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
// private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
//private static HttpBoot httpBoot = HttpBoot.newBuilder().sslProvider(SslProvider.CONSCRYPT).retryTimes(3).build();
/**
* @Title: getZhihuHotList
......@@ -100,7 +101,7 @@ public class ZhihuHotSearchCrawler {
String htmlBody = null;
Request request = RequestUtils.wrapGet(url, headerMap);
for (int x = 0; x <= 5; x++) {
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()) {
Throwable cause = response.cause();
log.debug("获取知乎热搜时出现问题:{}", cause);
......@@ -171,7 +172,7 @@ public class ZhihuHotSearchCrawler {
Map.put("cookie", "_xsrf=7NFWM5qBcOutfs8MaW7bhQQH65t3Xia4");
Request request = RequestUtils.wrapGet(url, Map);
try {
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()) {
Throwable cause = response.cause();
log.error("单条知乎热搜数据页面连接失败", cause);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/ZhihuTopicSearchCrawler.java
......@@ -9,6 +9,7 @@ import com.zhiwei.http.proxy.ProxySupplier;
import com.zhiwei.http.util.RequestUtils;
import com.zhiwei.searchhotcrawler.bean.HotSearchList;
import com.zhiwei.searchhotcrawler.bean.HotSearchType;
import com.zhiwei.searchhotcrawler.util.HttpClientUtils;
import lombok.extern.log4j.Log4j2;
import okhttp3.Request;
import org.jsoup.Jsoup;
......@@ -17,7 +18,7 @@ import java.util.*;
@Log4j2
public class ZhihuTopicSearchCrawler {
private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
// private static HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(3).build();
public static List<HotSearchList> getZhihuTopicSearch(Date date){
List<HotSearchList> list = new ArrayList<>();
......@@ -26,7 +27,7 @@ public class ZhihuTopicSearchCrawler {
String htmlBody = null;
Request request = RequestUtils.wrapGet(url);
for (int t = 0; t < 3 && jsonObject == null; t++) {
Response response = httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
Response response = HttpClientUtils.httpBoot.syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){
Throwable cause = response.cause();
log.error("知乎热搜页面连接异常",cause);
......
src/main/java/com/zhiwei/searchhotcrawler/run/HotSearchRun.java
......@@ -7,13 +7,14 @@ import com.zhiwei.network.cynomys.consumer.CynomysConsumerFactory;
import com.zhiwei.searchhotcrawler.config.ProxyConfig;
import com.zhiwei.searchhotcrawler.timer.*;
import com.zhiwei.tools.tools.ZhiWeiTools;
import lombok.extern.log4j.Log4j2;
import org.apache.dubbo.config.ApplicationConfig;
import org.apache.dubbo.config.ConsumerConfig;
import org.apache.dubbo.config.RegistryConfig;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
@Log4j2
public class HotSearchRun {
public static void main(String[] args) {
......@@ -46,7 +47,7 @@ public class HotSearchRun {
// 初始化 http-boot 桥接
CynomysFactory.init(consumer);
log.info("http-boot 桥接 成功");
new UpdateWechatUserRun().start();
ZhiWeiTools.sleep(10000);
......
src/main/java/com/zhiwei/searchhotcrawler/util/HttpClientUtils.java
......@@ -25,7 +25,7 @@ public final class HttpClientUtils {
private static final String URL_QUERY_PARAM_SEPARATOR = "?";
//private static final HttpBoot httpBoot = new HttpBoot.Builder().throwException(false).retryTimes(2).build();
private static final HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(2).build();
public static final HttpBoot httpBoot = HttpBoot.newBuilder().retryTimes(2).build();
public static String sendPost(String url, String jsonParam){
return sendPost(url, jsonParam, null, Charset.forName("UTF-8"));
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment