Commit 55ba4b19 by chenweitao

Merge branch 'working' into 'master'

增加虎嗅采集头部信息

See merge request !193
parents 8da69ed3 c46395f5
...@@ -34,8 +34,14 @@ public class HuXiuHotSearchCrawler { ...@@ -34,8 +34,14 @@ public class HuXiuHotSearchCrawler {
public static List<HotSearchList> HuXiuHotArticleRecommended(Date date){ public static List<HotSearchList> HuXiuHotArticleRecommended(Date date){
String url = "https://www.huxiu.com/"; String url = "https://www.huxiu.com/";
Map<String, String> headerMap = new HashMap<>();
headerMap.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9");
headerMap.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36 Edg/101.0.1210.39");
headerMap.put("Host","www.huxiu.com");
headerMap.put("sec-ch-ua-platform","Windows");
headerMap.put("sec-ch-ua"," Not A;Brand\";v=\"99\", \"Chromium\";v=\"101\", \"Microsoft Edge\";v=\"101");
String htmlBody = null; String htmlBody = null;
Request request = RequestUtils.wrapGet(url); Request request = RequestUtils.wrapGet(url,headerMap);
Response response = httpBoot.syncCall(request, ProxySupplier.NAT_HEAVY_PROXY); Response response = httpBoot.syncCall(request, ProxySupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){ if (response.hasCause()){
Throwable cause = response.cause(); Throwable cause = response.cause();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment