Commit 806b0032 by leiliangliang

增加36氪采集程序头部信息

parent c46395f5
...@@ -37,8 +37,15 @@ public class HotSearch36KrCrawler { ...@@ -37,8 +37,15 @@ public class HotSearch36KrCrawler {
*/ */
public static List<HotSearchList> hotSearch36Kr(Date date) { public static List<HotSearchList> hotSearch36Kr(Date date) {
String url = "https://www.36kr.com/hot-list/catalog"; String url = "https://www.36kr.com/hot-list/catalog";
Map<String, String> headerMap = new HashMap<>();
headerMap.put("origin","https://www.36kr.com");
headerMap.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36 Edg/101.0.1210.53");
headerMap.put("accept","*/*");
headerMap.put("sec-ch-ua"," Not A;Brand\";v=\"99\", \"Chromium\";v=\"101\", \"Microsoft Edge\";v=\"101");
headerMap.put("sec-fetch-site","same-site");
headerMap.put("sec-fetch-dest","empty");
String htmlBody = null; String htmlBody = null;
Request request = RequestUtils.wrapGet(url); Request request = RequestUtils.wrapGet(url,headerMap);
Response response = httpBoot.syncCall(request, ProxySupplier.NAT_HEAVY_PROXY); Response response = httpBoot.syncCall(request, ProxySupplier.NAT_HEAVY_PROXY);
if (response.hasCause()){ if (response.hasCause()){
Throwable cause = response.cause(); Throwable cause = response.cause();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment