微博话题采集

src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiboTopicCrawler.java

@@ -132,8 +132,8 @@ public class WeiboTopicCrawler {
      */
     public static List<HotSearchList> startCrawlerByPhone(Date date){
         List<HotSearchList> topicList = new ArrayList<>();
-        for(int page=1; page<=6; page++){
-            String pageUrl = "https://m.weibo.cn/api/container/getIndex?containerid=231648_-_2&page=" + page;
+        for(int page=1; page<=3; page++){
+            String pageUrl = "https://api.weibo.cn/2/page?st_bottom_bar_new_style_enable=1&c=android&s=34dc160d&from=10A9295010&gsid=_2A25NH7inDeRxGeNH4lUX9ifIzTWIHXVvjUtvrDV6PUJbkdANLRjfkWpNSk7RXJ9vYwBfAr66TNj0zcFmOBPKZDuI&containerid=231648_-_4&page=" + page;
             Request request = RequestUtils.wrapGet(pageUrl);
             String htmlBody = null;
             //重试三次
@@ -144,7 +144,7 @@ public class WeiboTopicCrawler {
                     log.error("下载榜单列表页面时出现错误，错误为:{}", e);
                     continue;
                 }
-                if (StringUtils.isNotBlank(htmlBody) && htmlBody.contains("data")) {
+                if (StringUtils.isNotBlank(htmlBody)) {
                     topicList.addAll(parseTopicHtml(htmlBody,date));
                     break;
                 } else {
@@ -158,9 +158,12 @@ public class WeiboTopicCrawler {
 
     private static List<HotSearchList> parseTopicHtml(String htmlBody,Date date) {
         try {
-            JSONArray jsonArray = JSONObject.parseObject(htmlBody).getJSONObject("data").getJSONArray("cards");
+            JSONArray jsonArray = JSONObject.parseObject(htmlBody).getJSONArray("cards");
             if(Objects.nonNull(jsonArray) && !jsonArray.isEmpty()) {
-                JSONArray cards = jsonArray.getJSONObject(0).getJSONArray("card_group");
+                for (int j=0; j< jsonArray.size(); j++){
+                    JSONObject card = jsonArray.getJSONObject(j);
+                    if(card.containsKey("card_group")){
+                        JSONArray cards = card.getJSONArray("card_group");
                         List<HotSearchList> topicList = new ArrayList<>();
                         Integer rank = null;
                         String topicName = null;
@@ -184,32 +187,23 @@ public class WeiboTopicCrawler {
                             try {
                                 commentNum = TipsUtils.getHotCount(commentNumStr);
                                 readNum = TipsUtils.getHotCount(readNumStr);
-//                        if(commentNumStr.contains("万")){
-//                            commentNumStr = commentNumStr.replaceAll("万", "");
-//                            commentNum = (int)(Double.parseDouble(commentNumStr)*10000);
-//                        }else if(commentNumStr.contains("亿")){
-//                            commentNumStr = commentNumStr.replaceAll("亿", "");
-//                            commentNum = (int)(Double.parseDouble(commentNumStr)*10000000);
-//                        }else{
-//                            commentNum = Integer.getInteger(commentNumStr);
-//                        }
-//
-//                        if(readNumStr.contains("万")){
-//                            readNumStr = readNumStr.replaceAll("万", "");
-//                            readNum = (int)(Double.parseDouble(readNumStr)*10000);
-//                        }else if(readNumStr.contains("亿")){
-//                            readNumStr = readNumStr.replaceAll("亿", "");
-//                            readNum = (int)(Double.parseDouble(readNumStr)*10000000);
-//                        }else{
-//                            readNum = Integer.getInteger(readNumStr);
-//                        }
                             }catch (Exception e){
                                 e.printStackTrace();
                             }
                             HotSearchList topic = new HotSearchList(url, topicName, readNum, rank, HotSearchType.微博话题.name(), commentNum, description,date);
+                            if(cardGroup.containsKey("title_flag_pic")){
+                                String titlePic = cardGroup.getString("title_flag_pic");
+                                if(titlePic.contains("new")){
+                                    topic.setIcon("新");
+                                }else if(titlePic.contains("hot")){
+                                    topic.setIcon("热");
+                                }
+                            }
                             topicList.add(topic);
                         }
                         return topicList;
+                    }
+                }
             }else{
 //                log.info("html:{}",htmlBody);
             }