Skip to content

S
source_forward
Commit bb108a86 by chenweiyang
Options

zhiwei tools 版本提升

parent 4a661c59
Showing with 20 additions and 14 deletions
pom.xml
......@@ -22,9 +22,15 @@
<dependencies>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.62</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.zhiwei.tools</groupId>
<artifactId>zhiwei-tools</artifactId>
<version>0.1.6-SNAPSHOT</version>
<version>0.1.9-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.zhiwei.crawler</groupId>
......
src/main/java/com/zhiwei/source_forward/crawler/UrlLiveCrawler.java
......@@ -71,18 +71,18 @@ public class UrlLiveCrawler {
Map<String,String> headers = new HashMap<>();
ProxyHolder ph = null;
if(url.contains("toutiao.com")){
headers.put("referer", url);
// headers.put("cookie", "csrftoken=6d0e5967684dbb57cea14dc12858d263; WEATHER_CITY=%E5%8C%97%E4%BA%AC; __utmz=24953151.1576744435.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); _ga=GA1.2.1570444666.1576744435; ttcid=d5fa0cec2bca426cbbddeb27a8bc72f629; UM_distinctid=170f5ad3a5b43e-04a6930812e3c5-f313f6d-240000-170f5ad3a5c40b; SLARDAR_WEB_ID=770be4fe-dd54-4701-a4ed-396b861a237d; CNZZDATA1259612802=2091325281-1587691681-%7C1589960650; s_v_web_id=verify_kaqh0jch_kZtSHRI9_GfBj_403X_8i5y_VOyWz4ATIx0d; tt_webid=6833273737980659213; tt_webid=6833273737980659213; __utmc=24953151; __utma=24953151.1570444666.1576744435.1591353562.1591579877.8; tt_scid=a56VD6ALatPbD63MlXw5skpZx9olxW6X.mRiDJBvVfZyQF2lfw8-lNeLPqqPPuCH4c68; __ac_nonce=05ee1cac300c6be6af0fe; __ac_signature=YvewuAAgEBDyCDITpF4SsmL2saAADwawXLDdrzlqO4hucxtXaZyI1l3ZReIsXb1OnF3koe7MdMwhnGPBA-mn5X5ERtMmQrb7RY5NqiBu.g3p0.oY6nNsvIT3NNbIsViZXz3; __tasessionId=48abvzgub1591855812394");
headers.put("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9");
headers.put("accept-encoding", "gzip, deflate, br");
headers.put("accept-language", "zh-CN,zh;q=0.9");
headers.put("cache-control", "no-cache");
headers.put("sec-fetch-dest", "document");
headers.put("sec-fetch-mode", "navigate");
headers.put("sec-fetch-site", "same-origin");
headers.put("sec-fetch-user", "?1");
headers.put("upgrade-insecure-requests", "1");
headers.put("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36");
// headers.put("referer", url);
// headers.put("cookie", "csrftoken=6d0e5967684dbb57cea14dc12858d263; WEATHER_CITY=%E5%8C%97%E4%BA%AC; __utmz=24953151.1576744435.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); _ga=GA1.2.1570444666.1576744435; ttcid=d5fa0cec2bca426cbbddeb27a8bc72f629; UM_distinctid=170f5ad3a5b43e-04a6930812e3c5-f313f6d-240000-170f5ad3a5c40b; SLARDAR_WEB_ID=770be4fe-dd54-4701-a4ed-396b861a237d; CNZZDATA1259612802=2091325281-1587691681-%7C1589960650; s_v_web_id=verify_kaqh0jch_kZtSHRI9_GfBj_403X_8i5y_VOyWz4ATIx0d; __utmc=24953151; __utma=24953151.1570444666.1576744435.1591353562.1591579877.8; tt_webid=6837283963338622477; tt_webid=6837283963338622477; __tasessionId=dsei2aty41591951911851; tt_scid=M--AJ-FYwZ0qcYTzQCLyMeS5MlykLS6ktMkvqKKJmq-ghRxX4waEBhJ3YbheuNmi2b8a");
// headers.put("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9");
// headers.put("accept-encoding", "gzip, deflate, br");
// headers.put("accept-language", "zh-CN,zh;q=0.9");
// headers.put("cache-control", "no-cache");
// headers.put("sec-fetch-dest", "document");
// headers.put("sec-fetch-mode", "navigate");
// headers.put("sec-fetch-site", "same-origin");
// headers.put("sec-fetch-user", "?1");
// headers.put("upgrade-insecure-requests", "1");
// headers.put("user-agent", "Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Mobile Safari/537.36");
ph = ProxyHolder.NAT_HEAVY_PROXY;
}else if(url.contains("zhihu.com")) {
url = treatZhihuUrl(url);
......
src/main/java/com/zhiwei/source_forward/run/URLLive.java
......@@ -72,7 +72,7 @@ public class URLLive {
public static void main(String[] args) {
ProxyInit.initProxy();
List<String> urlList = new ArrayList<>();
urlList.add("http://www.toutiao.com/a1665677841741827");
urlList.add("http://www.toutiao.com/item/1668646006370318/");
// urlList.add("http://www.yidianzixun.com/article/0PYO4Gbh");
List<UrlLiveBean> u = URLLive.verificationURLLive(urlList);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment