Merge branch 'working' into 'master'

更新淘宝采集程序

See merge request !144

src/main/java/com/zhiwei/searchhotcrawler/crawler/TaoBaoHotSearchCrawler.java

@@ -7,7 +7,7 @@ import com.zhiwei.crawler.core.proxy.ProxyHolder;
 import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.searchhotcrawler.bean.HotSearchList;
 import com.zhiwei.searchhotcrawler.bean.HotSearchType;
-import com.zhiwei.searchhotcrawler.util.TaoBaoUtils;
+import com.zhiwei.searchhotcrawler.util.MD5Util;
 import lombok.extern.log4j.Log4j2;
 import okhttp3.Request;
 import okhttp3.Response;
@@ -31,9 +31,7 @@ public class TaoBaoHotSearchCrawler {
         Boolean ht = false;
         long time = new Date().getTime();
         for (int t = 0; t < 3 && ht == false; t++) {
-            String signss = "undefined&" + time + "&12574478&{\"appId\":\"10211\",\"params\":\"{\\\"multi_hintq_show\\\":\\\"on\\\",\\\"src\\\":\\\"c2c\\\",\\\"area\\\":\\\"active_page\\\",\\\"sversion\\\":\\\"7.5\\\",\\\"bangdan_src\\\":\\\"list\\\"}\"}";
-            String sig = TaoBaoUtils.parsJSFunction(signss);
-            String urls = "https://acs.m.taobao.com/h5/mtop.relationrecommend.wirelessrecommend.recommend/2.0/?appKey=12574478&t=" + time + "&sign=" + sig + "&api=mtop.relationrecommend.WirelessRecommend.recommend&v=2.0&type=jsonp&dataType=jsonp&callback=mtopjsonp1&data=%7B%22appId%22%3A%2210211%22%2C%22params%22%3A%22%7B%5C%22multi_hintq_show%5C%22%3A%5C%22on%5C%22%2C%5C%22src%5C%22%3A%5C%22c2c%5C%22%2C%5C%22area%5C%22%3A%5C%22active_page%5C%22%2C%5C%22sversion%5C%22%3A%5C%227.5%5C%22%2C%5C%22bangdan_src%5C%22%3A%5C%22list%5C%22%7D%22%7D";
+            String urls = "https://acs.m.taobao.com/h5/mtop.relationrecommend.wirelessrecommend.recommend/2.0/?appKey=12574478&t=" + time + "&sign=&api=mtop.relationrecommend.WirelessRecommend.recommend&v=2.0&type=jsonp&dataType=jsonp&callback=mtopjsonp1&data=%7B%22appId%22%3A%2210211%22%2C%22params%22%3A%22%7B%5C%22multi_hintq_show%5C%22%3A%5C%22on%5C%22%2C%5C%22src%5C%22%3A%5C%22c2c%5C%22%2C%5C%22area%5C%22%3A%5C%22active_page%5C%22%2C%5C%22sversion%5C%22%3A%5C%227.5%5C%22%2C%5C%22bangdan_src%5C%22%3A%5C%22list%5C%22%7D%22%7D";
             Request request1 = RequestUtils.wrapGet(urls);
             String token = null;
             try (Response response = httpBoot.syncCall(request1, ProxyHolder.NAT_HEAVY_PROXY)) {
@@ -50,7 +48,7 @@ public class TaoBaoHotSearchCrawler {
                 log.error("解析淘宝热搜时出现解析错误,页面结构有问题", e);
             }
             String signs = token + "&" + time + "&12574478&{\"appId\":\"10211\",\"params\":\"{\\\"multi_hintq_show\\\":\\\"on\\\",\\\"src\\\":\\\"c2c\\\",\\\"area\\\":\\\"active_page\\\",\\\"sversion\\\":\\\"7.5\\\",\\\"bangdan_src\\\":\\\"list\\\"}\"}";
-            String sign = TaoBaoUtils.parsJSFunction(signs);
+            String sign = MD5Util.getMD5(signs).toLowerCase();
             String url = "https://acs.m.taobao.com/h5/mtop.relationrecommend.wirelessrecommend.recommend/2.0/?appKey=12574478&t=" + time + "&sign=" + sign + "&api=mtop.relationrecommend.WirelessRecommend.recommend&v=2.0&type=jsonp&dataType=jsonp&callback=mtopjsonp1&data=%7B%22appId%22%3A%2210211%22%2C%22params%22%3A%22%7B%5C%22multi_hintq_show%5C%22%3A%5C%22on%5C%22%2C%5C%22src%5C%22%3A%5C%22c2c%5C%22%2C%5C%22area%5C%22%3A%5C%22active_page%5C%22%2C%5C%22sversion%5C%22%3A%5C%227.5%5C%22%2C%5C%22bangdan_src%5C%22%3A%5C%22list%5C%22%7D%22%7D";
             Request request = RequestUtils.wrapGet(url, headerMap);
             try (Response response = httpBoot.syncCall(request, ProxyHolder.NAT_HEAVY_PROXY)) {
@@ -91,7 +89,6 @@ public class TaoBaoHotSearchCrawler {
                     if ("".equals(tagText)) {
                         tagText = null;
                     }
-                    //String tagText = jsonObject.getString("tagText");
                     Long count = null;
                     HotSearchList hotSearchList = new HotSearchList(url, name, count, true, rank, HotSearchType.淘宝热搜.name(), tagText, date);
                     list.add(hotSearchList);

src/main/java/com/zhiwei/searchhotcrawler/timer/quartz/GatherTimer.java

@@ -520,21 +520,21 @@ public class GatherTimer {
         TipsUtils.addHotList(HotSearchType.快手热榜.name(), kuaiShouList);
         logger.info("快手热榜采集结束...");
     }
+    /**
+     *淘宝热搜采集
+     */
+    @Async(value = "myScheduler")
+    @Scheduled(cron = "0 * * * * ? ")
+    public void crawlerTaoBao(){
+        logger.info("淘宝热搜开始采集...");
+        Date date = DateUtils.getMillSecondTime(new Date());
+        List<HotSearchList> taoBaoList = TaoBaoHotSearchCrawler.taoBaoHotSearch(date);
+        logger.info("{}, 淘宝热搜此轮采集到的数据量为:{}", new Date(),  taoBaoList != null ?  taoBaoList.size() : 0);
+        TipsUtils.addHotList(HotSearchType.淘宝热搜.name(), taoBaoList);
+        logger.info("淘宝热搜采集结束...");
+    }
+
 //    /**
-//     *淘宝热搜采集
-//     */
-//    @Async(value = "myScheduler")
-//    @Scheduled(cron = "0 * * * * ? ")
-//    public void crawlerTaoBao(){
-//        logger.info("淘宝热搜开始采集...");
-//        Date date = DateUtils.getMillSecondTime(new Date());
-//        List<HotSearchList> taoBaoList = TaoBaoHotSearchCrawler.taoBaoHotSearch(date);
-//        logger.info("{}, 淘宝热搜此轮采集到的数据量为:{}", new Date(),  taoBaoList != null ?  taoBaoList.size() : 0);
-//        TipsUtils.addHotList(HotSearchType.淘宝热搜.name(), taoBaoList);
-//        logger.info("淘宝热搜采集结束...");
-//    }
-//
-//        /**
 //     * 抖音同城榜的采集
 //     */
 //    @Async(value = "myScheduler")
@@ -574,7 +574,6 @@ public class GatherTimer {
 //            logger.info("抖音同城链接更新失败,抖音同城榜列表获取为空。");
 //        }
 //    }
-
     /**
      *微博娱乐榜采集
      */
@@ -589,7 +588,7 @@ public class GatherTimer {
         logger.info("微博娱乐榜采集结束...");
     }
     /**
-     *微博娱乐榜采集
+     *微博要闻榜采集
      */
     @Async(value = "myScheduler")
     @Scheduled(cron = "0 * * * * ? ")