脉脉热搜采集程序更新

src/main/java/com/zhiwei/searchhotcrawler/crawler/BililiCrawler.java

@@ -20,10 +20,7 @@ import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-import java.util.Objects;
+import java.util.*;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 
@@ -181,8 +178,10 @@ public class BililiCrawler {
         log.info("B站热搜榜开始采集...");
         JSONArray dataJson = null;
         String htmlBody = null;
-        String url = "https://app.biliapi.com/x/v2/search/square?build=616050&limit=10";
-        Request request = RequestUtils.wrapGet(url);
+        String url = "https://app.bilibili.com/x/v2/search/trending/ranking?limit=30";
+        Map<String, String> headerMap = new HashMap<>();
+        headerMap.put("Host","app.bilibili.com");
+        Request request = RequestUtils.wrapGet(url,headerMap);
         for(int t=0; t<3 && dataJson==null; t++){
             Response response = HttpClientUtils.getHttpBoot().syncCall(request, ProxyServerSupplier.NAT_HEAVY_PROXY);
             if (response.hasCause()){
@@ -192,26 +191,25 @@ public class BililiCrawler {
                 htmlBody = response.bodyString();
             }
             if(StringUtils.isNotBlank(htmlBody) && htmlBody.contains("data")){
-                dataJson = JSONObject.parseObject(htmlBody).getJSONArray("data").getJSONObject(0).getJSONObject("data").getJSONArray("list");
+                dataJson = JSONObject.parseObject(htmlBody).getJSONObject("data").getJSONArray("list");
                 if(dataJson != null) {
                     for (int i=0; i<dataJson.size(); i++) {
                         JSONObject data = dataJson.getJSONObject(i);
-                        int rank = i+1;
+                        int rank = data.getInteger("position");
                         String name = data.getString("show_name");
                         String icon = null;
                         if(data.containsKey("icon")){
                             String iconPicture = data.getString("icon");
-                            if(iconPicture.contains("e9e7a2d8497d4063421b685e72680bf1cfb99a0d")){
+                            if(iconPicture.contains("kcofGEO71W")){
                                 icon = "热";
-                            }else if(iconPicture.contains("4d579fb61f9655316582db193118bba3a721eec0")){
+                            }else if(iconPicture.contains("UF7B1wVKT2")){
                                 icon = "新";
-                            } else {
-                                icon = "未知";
                             }
                         }
                         String keyWord = data.getString("keyword");
                         String biliUrl = "https://search.bilibili.com/all?keyword="+ URLCodeUtil.getURLEncode(keyWord,"utf-8");
                         HotSearchList hotSearchList = new HotSearchList(biliUrl,name,null,rank,HotSearchType.B站热搜.name(),date);
+                        hotSearchList.setIcon(icon);
                         hotSearchLists.add(hotSearchList);
                     }
                 }