zhiwei tools 版本提升

pom.xml

@@ -21,10 +21,16 @@
     </developers>
 
     <dependencies>
+      <dependency>
+            <groupId>com.alibaba</groupId>
+            <artifactId>fastjson</artifactId>
+            <version>1.2.62</version>
+            <scope>provided</scope>
+        </dependency>
         <dependency>
             <groupId>com.zhiwei.tools</groupId>
             <artifactId>zhiwei-tools</artifactId>
-            <version>0.1.6-SNAPSHOT</version>
+            <version>0.1.9-SNAPSHOT</version>
         </dependency>
         <dependency>
         	<groupId>com.zhiwei.crawler</groupId>

src/main/java/com/zhiwei/source_forward/crawler/UrlLiveCrawler.java

@@ -71,18 +71,18 @@ public class UrlLiveCrawler {
         Map<String,String> headers = new HashMap<>();
         ProxyHolder ph = null;
         if(url.contains("toutiao.com")){
-        	headers.put("referer", url);
-//        	headers.put("cookie", "csrftoken=6d0e5967684dbb57cea14dc12858d263; WEATHER_CITY=%E5%8C%97%E4%BA%AC; __utmz=24953151.1576744435.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); _ga=GA1.2.1570444666.1576744435; ttcid=d5fa0cec2bca426cbbddeb27a8bc72f629; UM_distinctid=170f5ad3a5b43e-04a6930812e3c5-f313f6d-240000-170f5ad3a5c40b; SLARDAR_WEB_ID=770be4fe-dd54-4701-a4ed-396b861a237d; CNZZDATA1259612802=2091325281-1587691681-%7C1589960650; s_v_web_id=verify_kaqh0jch_kZtSHRI9_GfBj_403X_8i5y_VOyWz4ATIx0d; tt_webid=6833273737980659213; tt_webid=6833273737980659213; __utmc=24953151; __utma=24953151.1570444666.1576744435.1591353562.1591579877.8; tt_scid=a56VD6ALatPbD63MlXw5skpZx9olxW6X.mRiDJBvVfZyQF2lfw8-lNeLPqqPPuCH4c68; __ac_nonce=05ee1cac300c6be6af0fe; __ac_signature=YvewuAAgEBDyCDITpF4SsmL2saAADwawXLDdrzlqO4hucxtXaZyI1l3ZReIsXb1OnF3koe7MdMwhnGPBA-mn5X5ERtMmQrb7RY5NqiBu.g3p0.oY6nNsvIT3NNbIsViZXz3; __tasessionId=48abvzgub1591855812394");
-            headers.put("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9");
-            headers.put("accept-encoding", "gzip, deflate, br");
-            headers.put("accept-language", "zh-CN,zh;q=0.9");
-            headers.put("cache-control", "no-cache");
-            headers.put("sec-fetch-dest", "document");
-            headers.put("sec-fetch-mode", "navigate");
-            headers.put("sec-fetch-site", "same-origin");
-            headers.put("sec-fetch-user", "?1");
-            headers.put("upgrade-insecure-requests", "1");
-            headers.put("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36");
+//        	headers.put("referer", url);
+//        	headers.put("cookie", "csrftoken=6d0e5967684dbb57cea14dc12858d263; WEATHER_CITY=%E5%8C%97%E4%BA%AC; __utmz=24953151.1576744435.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); _ga=GA1.2.1570444666.1576744435; ttcid=d5fa0cec2bca426cbbddeb27a8bc72f629; UM_distinctid=170f5ad3a5b43e-04a6930812e3c5-f313f6d-240000-170f5ad3a5c40b; SLARDAR_WEB_ID=770be4fe-dd54-4701-a4ed-396b861a237d; CNZZDATA1259612802=2091325281-1587691681-%7C1589960650; s_v_web_id=verify_kaqh0jch_kZtSHRI9_GfBj_403X_8i5y_VOyWz4ATIx0d; __utmc=24953151; __utma=24953151.1570444666.1576744435.1591353562.1591579877.8; tt_webid=6837283963338622477; tt_webid=6837283963338622477; __tasessionId=dsei2aty41591951911851; tt_scid=M--AJ-FYwZ0qcYTzQCLyMeS5MlykLS6ktMkvqKKJmq-ghRxX4waEBhJ3YbheuNmi2b8a");
+//            headers.put("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9");
+//            headers.put("accept-encoding", "gzip, deflate, br");
+//            headers.put("accept-language", "zh-CN,zh;q=0.9");
+//            headers.put("cache-control", "no-cache");
+//            headers.put("sec-fetch-dest", "document");
+//            headers.put("sec-fetch-mode", "navigate");
+//            headers.put("sec-fetch-site", "same-origin");
+//            headers.put("sec-fetch-user", "?1");
+//            headers.put("upgrade-insecure-requests", "1");
+//            headers.put("user-agent", "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Mobile Safari/537.36");
             ph = ProxyHolder.NAT_HEAVY_PROXY;
         }else if(url.contains("zhihu.com")) {
         	url = treatZhihuUrl(url);

src/main/java/com/zhiwei/source_forward/run/URLLive.java

@@ -72,7 +72,7 @@ public class URLLive {
 	public static void main(String[] args) {
         ProxyInit.initProxy();
 	    List<String> urlList = new ArrayList<>();
-	    urlList.add("http://www.toutiao.com/a1665677841741827");
+	    urlList.add("http://www.toutiao.com/item/1668646006370318/");
 //	    urlList.add("http://www.yidianzixun.com/article/0PYO4Gbh");	    
 	    
         List<UrlLiveBean> u = URLLive.verificationURLLive(urlList);