Merge branch 'working' into 'master'

Working

See merge request !201

src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiboHotSearchCrawler.java

@@ -505,6 +505,7 @@ public class WeiboHotSearchCrawler {
         String url = "https://s.weibo.com/weibo?q=" + encode + "&Refer=top";
         String htmlBody = null;
         Map<String, String> headerMap = new HashMap<>();
+        //该cookie有效期一年,微博pc端获取游客cookie链接 https://s.weibo.com/top/summary?cate=realtimehot
         headerMap.put("Cookie", "SUB=_2AkMUShJMf8NxqwJRmP0RyWvgb4RwwgnEieKiFuOXJRMxHRl-yT92qlQvtRB6P8o8oso9Ew-s6vf16fdCca-Xz6DwwAMH; SUBP=0033WrSXqPxfM72-Ws9jqgMF55529P9D9WFdAobr6HdAbgQQ9vbUQKDx");
         Request request = RequestUtils.wrapGet(url,headerMap);
         Response response = httpBoot.syncCall(request, ProxySupplier.NAT_HEAVY_PROXY);

src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiboTopicCrawler.java

@@ -222,8 +222,8 @@ public class WeiboTopicCrawler {
      */
     public static List<HotSearchList> startCrawlerByPc(Date date){
         List<HotSearchList> topicList = new ArrayList<>();
-        for(int page=1; page<=2; page++){
-            String pageUrl = "https://weibo.com/ajax/statuses/topic_band?sid=v_weibopro&category=all&page="+page+"&count=50";
+        for(int page=1; page<=6; page++){
+            String pageUrl = "https://weibo.com/ajax/statuses/topic_band?sid=v_weibopro&category=all&page="+page+"&count=10";
             Request request = RequestUtils.wrapGet(pageUrl);
             String htmlBody = null;
             //重试三次
@@ -236,8 +236,12 @@ public class WeiboTopicCrawler {
                 }else {
                     htmlBody = response.bodyString();
                 }
-                if (StringUtils.isNotBlank(htmlBody)) {
-                    topicList.addAll(parseTopicPcHtml(htmlBody,date));
+                if (htmlBody.contains("data") && Objects.nonNull(JSONObject.parseObject(htmlBody).get("data"))) {
+                    JSONObject data = JSONObject.parseObject(htmlBody).getJSONObject("data");
+                    JSONArray statuses = data.getJSONArray("statuses");
+                    if (statuses.size()>0) {
+                        topicList.addAll(parseTopicPcHtml(statuses, date));
+                    }
                     break;
                 } else {
                     log.info("下载榜单列表页面时数据格式错误，页面为:{}", htmlBody);
@@ -247,10 +251,8 @@ public class WeiboTopicCrawler {
         return topicList;
     }
 
-    private static List<HotSearchList> parseTopicPcHtml(String htmlBody,Date date) {
+    private static List<HotSearchList> parseTopicPcHtml(JSONArray jsonArray,Date date) {
         try {
-            JSONObject data = JSONObject.parseObject(htmlBody).getJSONObject("data");
-            JSONArray jsonArray = data.getJSONArray("statuses");
             List<HotSearchList> topicList = new ArrayList<>();
                 for (int j=0; j< jsonArray.size(); j++){
                     JSONObject card = jsonArray.getJSONObject(j);