Merge branch 'mlbWork' into 'master'

今日头条链接

See merge request !39

src/main/java/com/zhiwei/searchhotcrawler/crawler/ToutiaoHotSearchCrawler.java

@@ -52,11 +52,8 @@ public class ToutiaoHotSearchCrawler {
         }
         //采集头条内容
         String url = "https://i.snssdk.com/hot-event/hot-board/?origin="+origin;
-        Map<String,String> headerMap = new HashMap<>();
-        headerMap.put("User-Agent", "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1");
-        headerMap.put("referer","https://ib.snssdk.com/rogue/aladdin_landingpage/template/aladdin_landingpage/hot_words.html?isBrowser=true&traffic_source=");
         String htmlBody = null;
-        Request request = RequestUtils.wrapGet(url, headerMap);
+        Request request = RequestUtils.wrapGet(url);
         for(int count =0; count<=5; count++){
             try(Response response = httpBoot.syncCall(request)) {
                 htmlBody = response.body().string();
@@ -72,7 +69,8 @@ public class ToutiaoHotSearchCrawler {
                         try {
                             JSONObject word = words.getJSONObject(i);
                             String name = word.getString("Title");
-                            String link = "https://ib.snssdk.com/search/?keyword=" + URLCodeUtil.getURLEncode(name, "utf-8") + "&pd=synthesis&source=trending_list&traffic_source=";
+//                            String link = "https://ib.snssdk.com/search/?keyword=" + URLCodeUtil.getURLEncode(name, "utf-8") + "&pd=synthesis&source=trending_list&traffic_source=";
+                            String link = word.getString("Url");
                             Integer hotCount = word.getInteger("HotValue");
                             String wordsType = word.getString("Label");
                             String icon = getIcon(wordsType);