Merge branch 'working' into 'master'

Working

See merge request !99

searchhotcrawler.iml

 <?xml version="1.0" encoding="UTF-8"?>
-<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
+<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" version="4">
   <component name="FacetManager">
     <facet type="Spring" name="Spring">
       <configuration />
@@ -10,8 +10,8 @@
     <output-test url="file://$MODULE_DIR$/target/test-classes" />
     <content url="file://$MODULE_DIR$">
       <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
-      <sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" />
       <sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
+      <sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" />
       <excludeFolder url="file://$MODULE_DIR$/target" />
     </content>
     <orderEntry type="inheritedJdk" />
@@ -77,5 +77,8 @@
     <orderEntry type="library" name="Maven: org.apache.commons:commons-pool2:2.4.2" level="project" />
     <orderEntry type="library" scope="TEST" name="Maven: junit:junit:4.12" level="project" />
     <orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.5.6" level="project" />
+    <orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.4.10" level="project" />
+    <orderEntry type="library" name="Maven: commons-codec:commons-codec:1.10" level="project" />
   </component>
 </module>
\ No newline at end of file

src/main/java/com/zhiwei/searchhotcrawler/bean/HotSearchType.java

 package com.zhiwei.searchhotcrawler.bean;
 
 public enum HotSearchType {
-	 百度热搜, 
-	 微博热搜, 
-	 知乎热搜, 
-	 抖音热搜, 
-	 搜狗微信热搜,
-	 搜狗微信客户端热搜,
-	 微博话题,
-	 今日头条热搜,
-	 知乎热搜榜单,
-	 腾讯新闻,
-	 新浪热榜,
-	 新浪热点,
-	 搜狐话题,
-	 凤凰新闻热榜,
-	 凤凰新闻热搜,
-	 网易热榜,
-	 网易跟帖热议,
-	 微博预热榜,
-	 腾讯较真榜,
-	 脉脉热榜,
-	 B站排行榜,
-	 B站热搜,
-	人气榜36氪,
-	虎嗅热文推荐,
-	快手热榜,
+    百度热搜,
+    微博热搜,
+    知乎热搜,
+    抖音热搜,
+    搜狗微信热搜,
+    搜狗微信客户端热搜,
+    微博话题,
+    今日头条热搜,
+    知乎热搜榜单,
+    腾讯新闻,
+    新浪热榜,
+    新浪热点,
+    搜狐话题,
+    凤凰新闻热榜,
+    凤凰新闻热搜,
+    网易热榜,
+    网易跟帖热议,
+    微博预热榜,
+    腾讯较真榜,
+    脉脉热榜,
+    B站排行榜,
+    B站热搜,
+    人气榜36氪,
+    虎嗅热文推荐,
+    快手热榜,
+    淘宝热搜,
 }

src/main/java/com/zhiwei/searchhotcrawler/bean/WeiBoUser.java

@@ -53,6 +53,11 @@ public class WeiBoUser implements Serializable {
      * 头像地址
      */
     private String profileImageUrl;
+
+    /**
+     * 类型
+     */
+    private String type;
     public WeiBoUser() {
     }
 
@@ -66,6 +71,11 @@ public class WeiBoUser implements Serializable {
         this.time=time;
         this.followerCount=followerCount;
         this.profileImageUrl = profileImageUrl;
-
+    }
+    public WeiBoUser(String userId, String userName,String topic,Date time) {
+        this.userId = userId;
+        this.userName = userName;
+        this.topic=topic;
+        this.time=time;
     }
 }

src/main/java/com/zhiwei/searchhotcrawler/crawler/BaiDuHotSearchCrawler.java

@@ -14,8 +14,6 @@ import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 import org.jsoup.select.Elements;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import com.zhiwei.crawler.core.HttpBoot;
 import com.zhiwei.searchhotcrawler.bean.HotSearchList;

src/main/java/com/zhiwei/searchhotcrawler/crawler/MaiMaiHotSearchCrawler.java

@@ -45,7 +45,10 @@ public class MaiMaiHotSearchCrawler {
                     JSONObject jsonObject = jsonArray.getJSONObject(i).getJSONObject("style35");
                     if(jsonObject != null) {
                         String name = jsonObject.getString("text");
-                        log.info(name);
+                        if (name.length()>750){
+                            name = name.substring(0,750);
+                        }
+//                        log.info(name);
                         String maimaiUrl = jsonObject.getString("share_url");
                         String icon = null;
                         if (jsonObject.containsKey("hot_type_card")) {

src/main/java/com/zhiwei/searchhotcrawler/dao/HotSearchCacheDAO.java

@@ -6,12 +6,14 @@ import com.zhiwei.searchhotcrawler.bean.HotSearchList;
 import com.zhiwei.searchhotcrawler.config.DBConfig;
 import com.zhiwei.searchhotcrawler.crawler.WeiboHotSearchCrawler;
 import com.zhiwei.searchhotcrawler.dbtemplate.MongoDBTemplate;
+import com.zhiwei.searchhotcrawler.util.MD5Util;
 import lombok.extern.log4j.Log4j2;
 import org.apache.logging.log4j.util.Strings;
 import org.bson.Document;
 
 import java.util.*;
 
+import static java.util.Objects.isNull;
 import static java.util.Objects.nonNull;
 
 /**
@@ -97,7 +99,12 @@ public class HotSearchCacheDAO  {
            String url = document.getString("url")!=null?document.getString("url"):null;
            String topicResult = document.getString("topic_result")!=null?document.getString("topic_result"):null;
            String pictureUrl = document.getString("pictureUrl")!=null?document.getString("pictureUrl"):null;
-           String id = name + "_" + type;
+
+           String id = document.getString("_id");
+           if (isNull(id)){
+               id = name + "_" + type;
+           }
+
            boolean recommend = false;
 //           Integer readCount = document.getInteger("comment_count");
            if("微博热搜".equals(type)){
@@ -208,6 +215,17 @@ public class HotSearchCacheDAO  {
                    }
                    if("微博热搜".equals(type)){
                        nowDoc = WeiboHotSearchCrawler.weiboUpdate(nowDoc);
+                       //更新微博话题贡献者,关于功能
+                       Document documentPC = WeiboHotSearchCrawler.weiboUpdatePC(nowDoc);
+                       if (documentPC.containsKey("分类")) {
+                           nowDoc.put("classify",documentPC.get("分类"));
+                       }
+                       if (documentPC.containsKey("地区")) {
+                           nowDoc.put("region", documentPC.get("地区"));
+                       }
+                       if (documentPC.containsKey("标签")) {
+                           nowDoc.put("label", documentPC.get("标签"));
+                       }
                        if(nowDoc.containsKey("topicLead")){
                            nowDoc.put("topicLead", nowDoc.getString("topicLead"));
                        }

src/main/java/com/zhiwei/searchhotcrawler/dao/WeiBoUserDao.java

@@ -42,8 +42,17 @@ public class WeiBoUserDao {
             document.put("userName",weiBoUser.getUserName());
             document.put("topic",weiBoUser.getTopic());
             document.put("time",weiBoUser.getTime());
-            document.put("followerCount",weiBoUser.getFollowerCount());
-            document.put("profileImageUrl",weiBoUser.getProfileImageUrl());
+            if (Objects.nonNull(weiBoUser.getType())){
+                document.put("type",weiBoUser.getType());
+            }
+
+            if (Objects.nonNull(weiBoUser.getFollowerCount())){
+                document.put("followerCount",weiBoUser.getFollowerCount());
+            }
+            if (Objects.nonNull(weiBoUser.getProfileImageUrl())){
+                document.put("profileImageUrl",weiBoUser.getProfileImageUrl());
+            }
+
             try {
                 mongoCollection.insertOne(document);
             } catch (Exception e) {

src/main/java/com/zhiwei/searchhotcrawler/run/HotSearchRun.java

@@ -9,6 +9,7 @@ import com.zhiwei.tools.tools.ZhiWeiTools;
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.support.ClassPathXmlApplicationContext;
 
+
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;

src/main/java/com/zhiwei/searchhotcrawler/test/HotSearch36KrCrawlerTest.java

@@ -9,12 +9,6 @@ import lombok.extern.log4j.Log4j2;
 import okhttp3.Request;
 import okhttp3.Response;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.http.HttpEntity;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.util.EntityUtils;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;

src/main/java/com/zhiwei/searchhotcrawler/test/HotSearchRunTest.java

@@ -4,6 +4,9 @@ package com.zhiwei.searchhotcrawler.test;
 import com.zhiwei.crawler.core.proxy.ProxyFactory;
 import com.zhiwei.proxy.config.SimpleConfig;
 import com.zhiwei.searchhotcrawler.config.ProxyConfig;
+import com.zhiwei.searchhotcrawler.timer.BaiduHotSearchRun;
+import com.zhiwei.searchhotcrawler.timer.WeiboHotSearchRun;
+
 import java.text.ParseException;
 
 public class HotSearchRunTest {
@@ -17,5 +20,7 @@ public class HotSearchRunTest {
        // new WeiboHotSearchRun().start();
         //快手热榜开始采集
        // new KuaiShouHotSearchRun().start();
+        //百度热搜
+        new BaiduHotSearchRun().run();
     }
 }

src/main/java/com/zhiwei/searchhotcrawler/test/TaoBaoHotSearchCrawlerTest.java

+package com.zhiwei.searchhotcrawler.test;
+
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.crawler.core.HttpBoot;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
+import com.zhiwei.searchhotcrawler.bean.HotSearchList;
+import com.zhiwei.searchhotcrawler.bean.HotSearchType;
+import com.zhiwei.searchhotcrawler.util.TaoBaoUtils;
+import lombok.extern.log4j.Log4j2;
+import okhttp3.Request;
+import okhttp3.Response;
+
+import java.io.IOException;
+import java.time.Duration;
+import java.util.*;
+
+/**
+ * @author ll
+ * @ClassName:TaoBaoHotSearchCrawler
+ * @Description:
+ * @date 2021年6月18日 下午16:33:31
+ */
+@Log4j2
+public class TaoBaoHotSearchCrawlerTest {
+    private static HttpBoot httpBoot = new HttpBoot.Builder().throwException(false).retryTimes(3).connectTimeout(Duration.ofSeconds(60)).build();
+
+    public static List<HotSearchList> taoBaoHotSearch(Date date) {
+        long time = new Date().getTime();
+        String signs="undefined&"+time+"&12574478&{\"appId\":\"10211\",\"params\":\"{\\\"multi_hintq_show\\\":\\\"on\\\",\\\"src\\\":\\\"c2c\\\",\\\"area\\\":\\\"active_page\\\",\\\"sversion\\\":\\\"7.5\\\",\\\"bangdan_src\\\":\\\"list\\\"}\"}";
+        //String signs="undefined&1624862377708&12574478&{\"appId\":\"10211\",\"params\":\"{\\\"multi_hintq_show\\\":\\\"on\\\",\\\"src\\\":\\\"c2c\\\",\\\"area\\\":\\\"active_page\\\",\\\"sversion\\\":\\\"7.5\\\",\\\"bangdan_src\\\":\\\"list\\\"}\"}";
+        String sign = TaoBaoUtils.parsJSFunction(signs);
+
+        //String url = "https://acs.m.taobao.com/h5/mtop.relationrecommend.wirelessrecommend.recommend/2.0/?appKey=12574478&t="+time+"&sign="+sign+"&api=mtop.relationrecommend.WirelessRecommend.recommend&v=2.0&type=jsonp&dataType=jsonp&callback=mtopjsonp1&data=%7B%22appId%22%3A%2210211%22%2C%22params%22%3A%22%7B%5C%22multi_hintq_show%5C%22%3A%5C%22on%5C%22%2C%5C%22src%5C%22%3A%5C%22c2c%5C%22%2C%5C%22area%5C%22%3A%5C%22active_page%5C%22%2C%5C%22sversion%5C%22%3A%5C%227.5%5C%22%2C%5C%22bangdan_src%5C%22%3A%5C%22list%5C%22%7D%22%7D";
+        String url = "https://acs.m.taobao.com/h5/mtop.relationrecommend.wirelessrecommend.recommend/2.0/?appKey=12574478&t=1624929605260&sign=ada01d783dc9772d2f84124d293bac26&api=mtop.relationrecommend.WirelessRecommend.recommend&v=2.0&type=jsonp&dataType=jsonp&callback=mtopjsonp1&data=%7B%22appId%22%3A%2210211%22%2C%22params%22%3A%22%7B%5C%22multi_hintq_show%5C%22%3A%5C%22on%5C%22%2C%5C%22src%5C%22%3A%5C%22c2c%5C%22%2C%5C%22area%5C%22%3A%5C%22active_page%5C%22%2C%5C%22sversion%5C%22%3A%5C%227.5%5C%22%2C%5C%22bangdan_src%5C%22%3A%5C%22list%5C%22%7D%22%7D";
+        Map<String, String> headerMap = new HashMap<>();
+        headerMap.put("cookie", "_m_h5_tk=975fb07b671f12a689d4ec36cf2e9047_1624937028814; _m_h5_tk_enc=ffb83d60b283eee5992d5e32429c2597;");
+        String htmlBody = null;
+        Request request = RequestUtils.wrapGet(url, headerMap);
+        try (Response response = httpBoot.syncCall(request, ProxyHolder.NAT_HEAVY_PROXY)) {
+            htmlBody = response.body().string();
+            System.out.println(htmlBody);
+        } catch (Exception e) {
+            log.error("解析淘宝热搜时出现解析错误,页面结构有问题", e);
+        }
+        if (htmlBody != null && htmlBody.contains("data")) {
+            return ansysData(htmlBody, date);
+        } else {
+            log.info("解析淘宝热搜时出现解析错误,页面结构有问题");
+        }
+        return Collections.emptyList();
+    }
+
+    private static List<HotSearchList> ansysData(String htmlBody, Date date) {
+        List<HotSearchList> list = new ArrayList<>();
+        try {
+            String sub = htmlBody.substring(htmlBody.indexOf("searchdoor"), htmlBody.indexOf("searchdoorFrom"));
+            String substring = sub.substring(sub.indexOf("showReminder") + 27, sub.indexOf("multi_bangdan_flag") - 2).trim();
+            JSONArray objects = JSONObject.parseArray(substring);
+            JSONArray jsonArray = objects.getJSONObject(0).getJSONObject("result").getJSONArray("text");
+            for (int i = 0; i < jsonArray.size(); i++) {
+                try {
+                    JSONObject jsonObject = jsonArray.getJSONObject(i);
+                    String name = jsonObject.getString("showtext");
+                    String showmark = jsonObject.getString("showmark");
+                    Integer rank = Integer.valueOf(showmark);
+                    String searchtext = jsonObject.getString("searchtext");
+                    String url = "https://s.m.taobao.com/h5?q=" + searchtext;
+                    String tagText = jsonObject.getString("tagText");
+                    Long count = 0L;
+                    HotSearchList hotSearchList = new HotSearchList(url,name,count,true,rank, HotSearchType.淘宝热搜.name(),tagText,date);
+                    list.add(hotSearchList);
+                } catch (Exception e) {
+                    log.error("解析淘宝热搜时出现解析错误",e);
+                }
+            }
+            System.out.println(jsonArray.size());
+        } catch (Exception e) {
+            log.error("解析淘宝热搜时出现解析错误,数据不是json结构", e);
+        }
+
+        return list;
+    }
+}

src/main/java/com/zhiwei/searchhotcrawler/timer/BaiduHotSearchRun.java

@@ -56,18 +56,18 @@ public class BaiduHotSearchRun extends Thread{
 //		}
 		TipsUtils.addHotList("百度热搜",baiduList);
 		log.info("百度风云榜采集结束........");
-		ZhiWeiTools.sleep(2000L);
-		log.info("搜狗微信采集开始........");
-		List<HotSearchList> sougouList = SougoHotSearchCrawler.sougoHotSearch(new Date());
-		log.info("{}, 此轮采集到的数据量为:{}", new Date(), Integer.valueOf(sougouList != null ? sougouList.size() : 0));
-		TipsUtils.addHotList("搜狗微信热搜",sougouList);
-		log.info("搜狗微信采集结束........");
-		ZhiWeiTools.sleep(2000L);
-		log.info("知乎话题采集开始........");
-		List<HotSearchList> zhihuList = ZhihuHotSearchCrawler.getMobileZhihuHotList(new Date());
-		log.info("{}, 知乎此轮采集到的数据量为:{}", new Date(), Integer.valueOf(zhihuList != null ? zhihuList.size() : 0));
-		TipsUtils.addHotList("知乎热搜",zhihuList);
-		log.info("知乎话题采集结束........");
+//		ZhiWeiTools.sleep(2000L);
+//		log.info("搜狗微信采集开始........");
+//		List<HotSearchList> sougouList = SougoHotSearchCrawler.sougoHotSearch(new Date());
+//		log.info("{}, 此轮采集到的数据量为:{}", new Date(), Integer.valueOf(sougouList != null ? sougouList.size() : 0));
+//		TipsUtils.addHotList("搜狗微信热搜",sougouList);
+//		log.info("搜狗微信采集结束........");
+//		ZhiWeiTools.sleep(2000L);
+//		log.info("知乎话题采集开始........");
+//		List<HotSearchList> zhihuList = ZhihuHotSearchCrawler.getMobileZhihuHotList(new Date());
+//		log.info("{}, 知乎此轮采集到的数据量为:{}", new Date(), Integer.valueOf(zhihuList != null ? zhihuList.size() : 0));
+//		TipsUtils.addHotList("知乎热搜",zhihuList);
+//		log.info("知乎话题采集结束........");
 	}
 
 }
\ No newline at end of file

src/main/java/com/zhiwei/searchhotcrawler/timer/WeiboHotSearchRun.java

 package com.zhiwei.searchhotcrawler.timer;
 
-import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
-
 import com.zhiwei.searchhotcrawler.dao.HotSearchCacheDAO;
 import com.zhiwei.searchhotcrawler.util.TipsUtils;
 import lombok.extern.log4j.Log4j2;

src/main/java/com/zhiwei/searchhotcrawler/util/MD5Util.java

+package com.zhiwei.searchhotcrawler.util;
+
+import java.security.MessageDigest;
+
+public class MD5Util {
+
+	private MD5Util() {
+		throw new IllegalStateException("Utility class");
+	}
+
+	/**
+	 * @Title: MD5
+	 * @Description: MD5后的字符串
+	 * @param pwd
+	 * @param @return
+	 *            设定文件
+	 * @return String 返回类型
+	 */
+	public static String getMD5(String pwd) {
+		return getString(pwd);
+	}
+
+	private static String getString(String pwd) {
+		char[] md5String = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' };
+		try {
+			// 使用平台的默认字符集将此 String 编码为 byte序列，并将结果存储到一个新的 byte数组中
+			byte[] btInput = pwd.getBytes();
+
+			// 信息摘要是安全的单向哈希函数，它接收任意大小的数据，并输出固定长度的哈希值。
+			MessageDigest mdInst = MessageDigest.getInstance("MD5");
+
+			// MessageDigest对象通过使用 update方法处理数据， 使用指定的byte数组更新摘要
+			mdInst.update(btInput);
+
+			// 摘要更新之后，通过调用digest（）执行哈希计算，获得密文
+			byte[] md = mdInst.digest();
+
+			// 把密文转换成十六进制的字符串形式
+			int j = md.length;
+			char str[] = new char[j * 2];
+			int k = 0;
+			for (int i = 0; i < j; i++) { // i = 0
+				byte byte0 = md[i]; // 95
+				str[k++] = md5String[byte0 >>> 4 & 0xf]; // 5
+				str[k++] = md5String[byte0 & 0xf]; // F
+			}
+			// 返回经过加密后的字符串
+			return new String(str);
+		} catch (Exception e) {
+			e.printStackTrace();
+			return null;
+		}
+	}
+}

src/main/java/com/zhiwei/searchhotcrawler/util/TaoBaoUtils.java

+package com.zhiwei.searchhotcrawler.util;
+
+import lombok.extern.log4j.Log4j2;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.core.io.Resource;
+
+import javax.script.Invocable;
+import javax.script.ScriptEngine;
+import javax.script.ScriptEngineManager;
+import java.io.FileReader;
+@Log4j2
+public class TaoBaoUtils {
+
+   public static String parsJSFunction(String sign) {
+
+       String scriptResult ="";//脚本的执行结果
+       ScriptEngine engine = new ScriptEngineManager().getEngineByName("JavaScript");//1.得到脚本引擎
+       //ScriptEngine engine = new ScriptEngineManager().getEngineByName("nashorn");//1.得到脚本引擎
+       try {
+           //2.引擎读取 脚本字符串
+           //engine.eval(new StringReader(routeScript));
+           //如果js存在文件里
+           Resource aesJs = new ClassPathResource("taobao.js");
+           engine.eval(new FileReader(aesJs.getFile()));
+
+           //3.将引擎转换为Invocable，这样才可以掉用js的方法
+           Invocable invocable = (Invocable) engine;
+
+           //4.使用 invocable.invokeFunction掉用js脚本里的方法，第一個参数为方法名，后面的参数为被调用的js方法的入参
+           scriptResult = (String) invocable.invokeFunction("h", sign);
+
+       }catch(Exception e){
+           log.error("Error executing script: ",e.getMessage());
+       }
+       return scriptResult;
+   }
+
+}

src/main/resources/redis.properties

 #redis.host=115.236.59.91
 #redis.port=7382
+#redis.host=127.0.0.1
+#redis.port=6379
 #redis.password=
 #ÐÂÏßÉÏredis
 #redis.host = 192.168.0.39

src/test/java/ObjectTest.java

@@ -21,6 +21,6 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
 @RunWith(SpringJUnit4ClassRunner.class)
 @ContextConfiguration(locations =
 { "classpath:applicationContext.xml" })
-public abstract class ObjectTest extends AbstractJUnit4SpringContextTests
-{
+public abstract class ObjectTest extends AbstractJUnit4SpringContextTests {
+
 }

src/test/java/hotSaerchTest/HotSearchTest.java

@@ -14,6 +14,7 @@ import com.zhiwei.searchhotcrawler.crawler.BaiDuHotSearchCrawler;
 import com.zhiwei.searchhotcrawler.dbtemplate.MongoDBTemplate;
 import com.zhiwei.searchhotcrawler.test.KuaiShouHotSearchCrawlerTest;
 import com.zhiwei.searchhotcrawler.test.TaoBaoHotSearchCrawlerTest;
+import com.zhiwei.searchhotcrawler.util.TaoBaoUtils;
 import com.zhiwei.searchhotcrawler.util.TipsUtils;
 import lombok.extern.log4j.Log4j2;
 import okhttp3.Request;
@@ -103,27 +104,28 @@ public class HotSearchTest {
             }
         }
 
-         ad(document);
+        ad(document);
         System.out.println(document);
     }
 
     private void ad(Document nowDoc) {
-         MongoCollection collection = MongoDBTemplate.getCollection(DBConfig.dbName, DBConfig.searchCacheCollName);
-        if(nowDoc.containsKey("topicLead")){
+        MongoCollection collection = MongoDBTemplate.getCollection(DBConfig.dbName, DBConfig.searchCacheCollName);
+        if (nowDoc.containsKey("topicLead")) {
             nowDoc.put("topicLead", nowDoc.getString("topicLead"));
         }
-        if(nowDoc.containsKey("readCount") && nowDoc.containsKey("discussCount")) {
-            nowDoc.put("readCount",  nonNull(nowDoc.get("readCount"))?Long.valueOf(nowDoc.get("readCount").toString()):null);
-            nowDoc.put("discussCount",   nonNull(nowDoc.get("discussCount"))?Long.valueOf(nowDoc.get("discussCount").toString()):null);
+        if (nowDoc.containsKey("readCount") && nowDoc.containsKey("discussCount")) {
+            nowDoc.put("readCount", nonNull(nowDoc.get("readCount")) ? Long.valueOf(nowDoc.get("readCount").toString()) : null);
+            nowDoc.put("discussCount", nonNull(nowDoc.get("discussCount")) ? Long.valueOf(nowDoc.get("discussCount").toString()) : null);
         }
         if (nowDoc.containsKey("pictureUrl")) {
-            nowDoc.put("pictureUrl",nowDoc.getString("pictureUrl"));
+            nowDoc.put("pictureUrl", nowDoc.getString("pictureUrl"));
         }
         if (nowDoc.containsKey("downtext")) {
-            nowDoc.put("downtext",nowDoc.getString("downtext"));
+            nowDoc.put("downtext", nowDoc.getString("downtext"));
         }
         collection.insertOne(nowDoc);
     }
+
     /**
      * 测试淘宝热搜采集
      */
@@ -154,9 +156,20 @@ public class HotSearchTest {
         List<HotSearchList> hotSearchLists = BaiDuHotSearchCrawler.baiduHotSearch(new Date());
         System.out.println(hotSearchLists);
         System.out.println(hotSearchLists.size());
-
     }
 
+    /**
+     * 测试解析淘宝js文件
+     */
+    @Test
+    public void taoBaoJSTest() throws IOException {
+
+        long time = new Date().getTime();
+        String signs="undefined&1625624820156&12574478&{\"appId\":\"10211\",\"params\":\"{\\\"multi_hintq_show\\\":\\\"on\\\",\\\"src\\\":\\\"c2c\\\",\\\"area\\\":\\\"active_page\\\",\\\"sversion\\\":\\\"7.5\\\",\\\"bangdan_src\\\":\\\"list\\\"}\"}";
+//        https://acs.m.taobao.com/h5/mtop.relationrecommend.wirelessrecommend.recommend/2.0/?appKey=12574478&t=1624930984092&sign=acf994dbcee6c0c1d7a8a566a6b8ff0a&api=mtop.relationrecommend.WirelessRecommend.recommend&v=2.0&type=jsonp&dataType=jsonp&callback=mtopjsonp1&data=%7B%22appId%22%3A%2210211%22%2C%22params%22%3A%22%7B%5C%22multi_hintq_show%5C%22%3A%5C%22on%5C%22%2C%5C%22src%5C%22%3A%5C%22c2c%5C%22%2C%5C%22area%5C%22%3A%5C%22active_page%5C%22%2C%5C%22sversion%5C%22%3A%5C%227.5%5C%22%2C%5C%22bangdan_src%5C%22%3A%5C%22list%5C%22%7D%22%7D
+        String s = TaoBaoUtils.parsJSFunction(signs);
+        System.out.println(s);
+    }
 
 }