爬虫核心包升级 ， 版本升级 0.2.5

pom.xml

@@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>com.zhiwei</groupId>
 	<artifactId>articlenewscrawler</artifactId>
-	<version>0.2.4-SNAPSHOT</version>
+	<version>0.2.5-SNAPSHOT</version>
 	<name>articlenewscrawler</name>
 	<description>采集凤凰，一点资讯，搜狐历时文章和文章评论</description>
 
@@ -21,7 +21,7 @@
 		<dependency>
 			<groupId>com.zhiwei.crawler</groupId>
 			<artifactId>crawler-core</artifactId>
-			<version>0.6.0.1-SNAPSHOT</version>
+			<version>0.6.6.3-SNAPSHOT</version>
 			<scope>provided</scope>
       </dependency>
 	</dependencies>

src/main/java/com/zhiwei/httpclient/HttpClient.java

@@ -9,8 +9,8 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 
 import okhttp3.FormBody;
 import okhttp3.Headers;

src/main/java/com/zhiwei/parse/Aika.java

@@ -11,8 +11,8 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.parse.analysis.AikaCommentAnalysis;
 import com.zhiwei.tools.tools.ZhiWeiTools;
 

src/main/java/com/zhiwei/parse/Aiqiyi.java

@@ -11,8 +11,8 @@ import org.slf4j.LoggerFactory;
 
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.httpclient.HeadGet;
 import com.zhiwei.httpclient.HttpClient;
 import com.zhiwei.parse.analysis.AiqiyiByWordAnalysis;

src/main/java/com/zhiwei/parse/BTime.java

@@ -13,8 +13,8 @@ import org.slf4j.LoggerFactory;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 
 import okhttp3.Response;
 

src/main/java/com/zhiwei/parse/Baijia.java

@@ -14,12 +14,11 @@ import org.slf4j.LoggerFactory;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.httpclient.HeadGet;
 import com.zhiwei.httpclient.HttpClient;
 import com.zhiwei.parse.analysis.BaijiaAccountAnalysis;
-import com.zhiwei.tools.httpclient.HeaderTool;
 import com.zhiwei.tools.tools.ZhiWeiTools;
 
 public class Baijia {

src/main/java/com/zhiwei/parse/BiliBili.java

@@ -10,8 +10,7 @@ import java.util.Map;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
 import com.zhiwei.httpclient.HttpClient;
 import com.zhiwei.parse.analysis.BilibilikeyWordAnalysis;
 import com.zhiwei.tools.tools.ZhiWeiTools;
@@ -28,6 +27,7 @@ public class BiliBili {
         try {
 //            
             String url = "https://search.bilibili.com/all?keyword="+URLEncoder.encode(word, "utf-8")+"&single_column=1&order=" + type +"&duration=0&tids_1=0";
+//            url = "https://search.bilibili.com/all?keyword="+URLEncoder.encode(word, "utf-8")+"&order=pubdate&duration=0&tids_1=181";
             System.out.println(url);
             Headers header = Headers.of("cookie",cookie,"Referer","https://www.bilibili.com/","Host","search.bilibili.com");
             String result = HttpClient.executeHttpRequestGet(url, ProxyHolder.NAT_HEAVY_PROXY, header);

src/main/java/com/zhiwei/parse/Chejia.java

@@ -15,8 +15,8 @@ import org.slf4j.LoggerFactory;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.tools.timeparse.TimeParse;
 import com.zhiwei.tools.tools.ZhiWeiTools;
 

src/main/java/com/zhiwei/parse/Dayu.java

@@ -11,7 +11,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.alibaba.fastjson.JSONObject;
-import com.zhiwei.crawler.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
 import com.zhiwei.httpclient.HeadGet;
 import com.zhiwei.httpclient.HttpClient;
 import com.zhiwei.parse.analysis.DayuAccountAnalysis;

src/main/java/com/zhiwei/parse/Douban.java

@@ -15,8 +15,8 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.parse.analysis.DoubanCommentAnalysis;
 import com.zhiwei.tools.tools.URLCodeUtil;
 import com.zhiwei.tools.tools.ZhiWeiTools;
@@ -87,7 +87,9 @@ public class Douban {
                 }
                 ZhiWeiTools.sleep(3000);
                 page++;
-                
+                if(page > 40) {
+                    break;
+                }
             } catch (Exception e) {
                 more = false;
                 logger.error("豆瓣 topic 采集出错 {}",e);

src/main/java/com/zhiwei/parse/Fenghuang.java

@@ -10,7 +10,7 @@ import java.util.Map;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.zhiwei.crawler.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
 import com.zhiwei.httpclient.HeadGet;
 import com.zhiwei.httpclient.HttpClient;
 import com.zhiwei.parse.analysis.FenghuangAccountAnalysis;

src/main/java/com/zhiwei/parse/Gftai.java

@@ -10,7 +10,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.parse.analysis.GftaiAnalysis;
 import com.zhiwei.tools.tools.ZhiWeiTools;
 

src/main/java/com/zhiwei/parse/Huxiu.java

@@ -15,8 +15,8 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.tools.timeparse.TimeParse;
 
 import okhttp3.Response;

src/main/java/com/zhiwei/parse/KuaiData.java

@@ -13,8 +13,8 @@ import org.slf4j.LoggerFactory;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 
 import okhttp3.Response;
 

src/main/java/com/zhiwei/parse/KuaiTousu.java

@@ -11,7 +11,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.parse.analysis.KuaiTousuAnalysis;
 import com.zhiwei.tools.tools.ZhiWeiTools;
 

src/main/java/com/zhiwei/parse/Maimai.java

@@ -17,8 +17,8 @@ import org.slf4j.LoggerFactory;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.httpclient.HeadGet;
 import com.zhiwei.httpclient.HttpClient;
 import com.zhiwei.parse.analysis.MaimaiBywordAnalysis;

src/main/java/com/zhiwei/parse/MyZaker.java

@@ -13,8 +13,8 @@ import org.slf4j.LoggerFactory;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 
 import okhttp3.Response;
 

src/main/java/com/zhiwei/parse/Pcauto.java

@@ -13,8 +13,8 @@ import org.slf4j.LoggerFactory;
 
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.parse.analysis.PcautoCommentAnalysis;
 import com.zhiwei.tools.tools.ZhiWeiTools;
 

src/main/java/com/zhiwei/parse/QQKB.java

@@ -12,7 +12,7 @@ import org.slf4j.LoggerFactory;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.bean.QQkbUser;
-import com.zhiwei.crawler.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
 import com.zhiwei.httpclient.HeadGet;
 import com.zhiwei.httpclient.HttpClient;
 import com.zhiwei.parse.analysis.QQKBAccountAnalysis;

src/main/java/com/zhiwei/parse/QQNews.java

@@ -7,8 +7,8 @@ import org.slf4j.LoggerFactory;
 
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 
 import okhttp3.Response;
 

src/main/java/com/zhiwei/parse/QicheHome.java

@@ -9,8 +9,8 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.parse.analysis.QicheHomeKwyWordAnalysis;
 import com.zhiwei.tools.tools.ZhiWeiTools;
 

src/main/java/com/zhiwei/parse/SinaKeji.java

@@ -13,8 +13,8 @@ import org.slf4j.LoggerFactory;
 
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.parse.analysis.SinaKejiCommentAnalysis;
 import com.zhiwei.tools.tools.ZhiWeiTools;
 

src/main/java/com/zhiwei/parse/SinaTousu.java

@@ -11,8 +11,8 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.parse.analysis.SinaTousuAnalysis;
 import com.zhiwei.tools.tools.ZhiWeiTools;
 

src/main/java/com/zhiwei/parse/Souhu.java

@@ -10,7 +10,7 @@ import org.slf4j.LoggerFactory;
 
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
-import com.zhiwei.crawler.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
 import com.zhiwei.httpclient.HeadGet;
 import com.zhiwei.httpclient.HttpClient;
 import com.zhiwei.parse.analysis.SouhuAccountAnalysis;

src/main/java/com/zhiwei/parse/TXNews.java

@@ -13,8 +13,8 @@ import org.slf4j.LoggerFactory;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.httpclient.HeadGet;
 import com.zhiwei.httpclient.HttpClient;
 import com.zhiwei.parse.analysis.TXNewsByWordAnalysis;

src/main/java/com/zhiwei/parse/TechTx.java

@@ -12,8 +12,8 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.parse.analysis.TechTxCommentAnalysis;
 import com.zhiwei.tools.tools.ZhiWeiTools;
 

src/main/java/com/zhiwei/parse/Travel315.java

@@ -16,7 +16,7 @@ import org.slf4j.LoggerFactory;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.tools.tools.ZhiWeiTools;
 
 public class Travel315 {

src/main/java/com/zhiwei/parse/Ts21cn.java

@@ -12,8 +12,8 @@ import org.slf4j.LoggerFactory;
 
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.parse.analysis.Ts21cnAnalysis;
 import com.zhiwei.tools.tools.ZhiWeiTools;
 

src/main/java/com/zhiwei/parse/Wangyi.java

@@ -6,6 +6,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 import org.jsoup.Jsoup;
 import org.slf4j.Logger;
@@ -14,8 +15,8 @@ import org.slf4j.LoggerFactory;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.httpclient.HeadGet;
 import com.zhiwei.httpclient.HttpClient;
 import com.zhiwei.parse.analysis.WangyiCommentAnalysis;
@@ -151,42 +152,51 @@ public class Wangyi {
         while(true) {
             try (Response response = httpBoot.syncCall(RequestUtils.wrapGet("https://c.m.163.com/nc/subscribe/list/"+id+"/all/"+page+"-20.html"), proxy)){
                 String result = response.body().string();
-                JSONObject json = JSONObject.parseObject(result);
-                JSONArray jsonArray = json.getJSONArray("tab_list");
-                for(int i = 0,j = jsonArray.size();i < j;i++) {
-                    JSONObject data = jsonArray.getJSONObject(i);
-                    Map<String,Object> map = new HashMap<>();
-                    String time = data.getString("ptime");
-                    if(endTime != null && endTime.length() > 1) {
-                        System.out.println(time);
-                        if(time.compareTo(endTime) <= 0) {
-                            logger.info("超时时间采集范围 跳出采集");
-                            return dataList;
+                if(Objects.nonNull(result) && result.contains("tab_list")) {
+                    JSONObject json = JSONObject.parseObject(result);
+                    JSONArray jsonArray = json.getJSONArray("tab_list");
+                    for(int i = 0,j = jsonArray.size();i < j;i++) {
+                        JSONObject data = jsonArray.getJSONObject(i);
+                        Map<String,Object> map = new HashMap<>();
+                        String time = data.getString("ptime");
+                        if(endTime != null && endTime.length() > 1) {
+                            if(time.compareTo(endTime) <= 0) {
+                                logger.info("超时时间采集范围 跳出采集");
+                                return dataList;
+                            }
+                        }
+                        map.put("title", data.getString("title"));
+                        map.put("content", data.getString("aheadBody"));
+                        map.put("time", time);
+                        map.put("source", data.getString("source"));
+                        if("video".equals(data.getString("skipType"))) {
+                            map.put("url", "https://c.m.163.com/news/v/" + data.getString("skipID") + ".html");
+                        }else {
+                            map.put("url", "https://c.m.163.com/news/a/" + data.getString("postid") + ".html");
                         }
+                        errorNum = 1;
+                        dataList.add(map);
                     }
-                    map.put("title", data.getString("title"));
-                    map.put("content", data.getString("aheadBody"));
-                    map.put("time", time);
-                    map.put("source", data.getString("source"));
-                    if("video".equals(data.getString("skipType"))) {
-                        map.put("url", "https://c.m.163.com/news/v/" + data.getString("skipID") + ".html");
-                    }else {
-                        map.put("url", "https://c.m.163.com/news/a/" + data.getString("postid") + ".html");
+                    logger.info("id = {} , cralwer count = {}",id,dataList.size() );
+                    page += 20;
+                    if(jsonArray.size() < 10) {
+                        errorNum ++;
+                        page -= 20;
                     }
-//                    System.out.println(map.toString());
-                    dataList.add(map);
-                }
-                logger.info("id = {} , cralwer count = {}",id,dataList.size() );
-                page += 20;
-                if(jsonArray.size() < 10) {
-                    break;
+                    errorNum ++;
+                    ZhiWeiTools.sleep(1000);
+                }else {
+                    errorNum ++;
+                    ZhiWeiTools.sleep(2000);
                 }
             } catch (Exception e) {
                 logger.info("采集数据出错 {}",e);
+                ZhiWeiTools.sleep(1000);
                 errorNum++;
-                if(errorNum > 3) {
-                    break;
-                }
+            }
+            System.out.println(errorNum);
+            if(errorNum > 10) {
+                break;
             }
         }
         return dataList;

src/main/java/com/zhiwei/parse/XinLangCaiJing.java

@@ -12,8 +12,8 @@ import org.slf4j.LoggerFactory;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.tools.timeparse.TimeParse;
 
 import okhttp3.Response;

src/main/java/com/zhiwei/parse/Xueqiu.java

@@ -16,8 +16,8 @@ import org.slf4j.LoggerFactory;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.parse.analysis.XueqiuKeyWordAnalysis;
 import com.zhiwei.tools.timeparse.TimeParse;
 import com.zhiwei.tools.tools.ZhiWeiTools;

src/main/java/com/zhiwei/parse/Yangshi.java

@@ -3,7 +3,6 @@ package com.zhiwei.parse;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 

src/main/java/com/zhiwei/parse/Yiche.java

@@ -14,8 +14,8 @@ import org.slf4j.LoggerFactory;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.tools.tools.ZhiWeiTools;
 
 import okhttp3.Response;

src/main/java/com/zhiwei/parse/Yidianzixun.java

@@ -16,8 +16,8 @@ import org.slf4j.LoggerFactory;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.httpclient.HeadGet;
 import com.zhiwei.httpclient.HttpClient;
 import com.zhiwei.parse.analysis.YidianzixunAccountAnalysis;

src/main/java/com/zhiwei/parse/Youku.java

@@ -14,9 +14,8 @@ import org.slf4j.LoggerFactory;
 
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyFactory;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.tools.tools.URLCodeUtil;
 
 import okhttp3.Response;

src/main/java/com/zhiwei/parse/Zhihu.java

@@ -13,8 +13,8 @@ import org.slf4j.LoggerFactory;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 
 import okhttp3.Response;
 

src/main/java/com/zhiwei/parse/analysis/BaijiaAccountAnalysis.java

@@ -15,7 +15,7 @@ import org.slf4j.LoggerFactory;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.httpclient.HeadGet;
 import com.zhiwei.tools.timeparse.TimeParse;
 import com.zhiwei.tools.tools.ZhiWeiTools;

src/main/java/com/zhiwei/parse/analysis/DayuByWordAnalysis.java

@@ -15,7 +15,7 @@ import org.slf4j.LoggerFactory;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.httpclient.HeadGet;
 import com.zhiwei.tools.timeparse.TimeParse;
 import com.zhiwei.tools.tools.ZhiWeiTools;

src/main/java/com/zhiwei/parse/analysis/FenghuangAccountAnalysis.java

@@ -11,8 +11,8 @@ import org.slf4j.LoggerFactory;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.httpclient.HeadGet;
 
 import okhttp3.Response;

src/main/java/com/zhiwei/parse/analysis/FenghuangCommentAnalysis.java

@@ -13,8 +13,8 @@ import org.slf4j.LoggerFactory;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.httpclient.HeadGet;
 import com.zhiwei.tools.timeparse.TimeParse;
 

src/main/java/com/zhiwei/parse/analysis/SouhuCommentAnalysis.java

@@ -10,8 +10,8 @@ import org.slf4j.LoggerFactory;
 
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.httpclient.HeadGet;
 import com.zhiwei.httpclient.HttpClient;
 

src/main/java/com/zhiwei/parse/shipin/QQTV.java

@@ -17,8 +17,8 @@ import org.slf4j.LoggerFactory;
 
 import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.proxy.ProxyHolder;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.proxy.ProxyHolder;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.tools.tools.ZhiWeiTools;
 
 import okhttp3.Response;

src/main/java/com/zhiwei/parse/shipin/SohuTV.java

@@ -15,7 +15,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 import com.zhiwei.tools.timeparse.TimeParse;
 import com.zhiwei.tools.tools.ZhiWeiTools;
 

src/test/java/com/zhiwei/TestHttpBoot.java

 package com.zhiwei;
 
 import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.crawler.core.utils.RequestUtils;
 
 import okhttp3.Response;
 

src/test/java/com/zhiwei/hsitory/BaijiaAccountExample.java

@@ -6,18 +6,18 @@
 //
 //import org.junit.Test;
 //
+//import com.zhiwei.common.config.GroupType;
 //import com.zhiwei.crawler.proxy.ProxyFactory;
 //import com.zhiwei.crawler.proxy.ProxyHolder;
 //import com.zhiwei.excelpoi.excel.PoiExcelUtil;
 //import com.zhiwei.parse.Baijia;
-//import com.zhiwei.proxy.config.SimpleConfig;
 //
 //public class BaijiaAccountExample {
 // 
 //    @Test
 //    public void test3() {
 //        // , "local", GroupType.PROVIDER , 10000002L
-//        ProxyFactory.init(SimpleConfig.builder().registry("zookeeper://192.168.0.11:2181?backup=192.168.0.30:2181,192.168.0.35:2181").appName("actool").appId(10000002).group("local").build());
+//        ProxyFactory.init("zookeeper://192.168.0.11:2181?backup=192.168.0.30:2181,192.168.0.35:2181", "local", GroupType.PROVIDER , 10000002L);
 //        String path = "D://crawlerdata//自媒体/百家号采集.xlsx";
 //        PoiExcelUtil poi = PoiExcelUtil.getInstance();
 //        String startTime = "2018-05-01 00:00:00";
@@ -27,8 +27,8 @@
 ////        for(Map<String,Object> m : list) {
 ////            try {
 ////                String app_id = m.get("id").toString();
-//                String app_id = "1565848819560927";
-//                String cookie = "BAIDUID=A46414BD701A3738E17E0212A6C2FEEA:FG=1; Hmery-Time=2269711404; BIDUPSID=A46414BD701A3738E17E0212A6C2FEEA; PSTM=1583375258; delPer=0; H_PS_PSSID=30972_1439_21095_30839_30998_30823; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598";
+//                String app_id = "1600247961135097";
+//                String cookie = "Hmery-Time=2853223123; BAIDUID=1BB712AF278D7B04E2A29BEAD50F9731:FG=1; BIDUPSID=1BB712AF278D7B04E2A29BEAD50F9731; PSTM=1589798969; H_PS_PSSID=31622_1431_31670_21083_31592_31270_31661_31463_31322_30823_26350; delPer=0; PSINO=5; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598";
 //                System.out.println(app_id);
 //                List<Map<String,Object>> lists = Baijia.getBaijiaAccountByBaiduData(app_id, "聚富财经", startTime, cookie, ProxyHolder.NAT_HEAVY_PROXY);
 //                if(lists != null) {

src/test/java/com/zhiwei/keyword/DoubanTopicTest.java

 //package com.zhiwei.keyword;
 //
-//import org.testng.annotations.Test;
+//import java.util.Arrays;
+//import java.util.List;
+//import java.util.Map;
 //
+//import org.junit.Test;
+//
+//import com.zhiwei.excelpoi.excel.PoiExcelUtil;
 //import com.zhiwei.parse.Douban;
 //
 //public class DoubanTopicTest {
+//    
 //    @Test
 //    public void f() {
-//        String word = "胡歌";
-//        String cookie = "bid=rymxzs5aojg; ps=y; ll=\"118173\"; __utmc=30149280; dbcl2=\"188038058:9IHyVcSobVc\"; ck=_RvF; push_noty_num=0; push_doumail_num=0; __utmv=30149280.18803; douban-fav-remind=1; __yadk_uid=qLflXyj3R14ro9e0cLoZOQlJoMGVN32j; douban-profile-remind=1; _vwo_uuid_v2=D85F60C118B0AF465035D9CC7BBFDA7A6|4bf255e1e3a2e9aeede3708192f5f1bc; __utmz=30149280.1543564973.3.2.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; _pk_ref.100001.8cb4=%5B%22%22%2C%22%22%2C1543908324%2C%22https%3A%2F%2Fwww.baidu.com%2Flink%3Furl%3DPk19bxnhsVWwfnrcnwT0PquON7D1JpLsbfSu9rRowalyi4pOeM3rMHKFaJo9jJF7%26wd%3D%26eqid%3De7f262650001ef98000000045c00e64f%22%5D; _pk_ses.100001.8cb4=*; ap_v=0,6.0; __utma=30149280.824403997.1543559458.1543885946.1543908324.10; __utmt=1; _pk_id.100001.8cb4=6828fef49f6bcf34.1543559455.9.1543908331.1543885945.; __utmb=30149280.9.7.1543908324";
-//        String time = "2018-11-16 00:00:00";
+//        String word = "龙岭迷窟";
+//        String cookie = "bid=aBxxYnlXock; douban-fav-remind=1; __yadk_uid=EESpKyPIvsPyVjjZ0DHerpLWEN1MLWPy; __gads=ID=9649e00ec2851223:T=1581324126:S=ALNI_MYZ0PbvzjuNKmREEpTjdvrO3x0v7w; __utmv=30149280.18803; gr_user_id=e12baa3e-7aac-44db-94a1-02c271013afd; _vwo_uuid_v2=DB1A0279AE19A1334CC41E68E320575C2|6b1aa2a313e7198665f2fdbffbd75e5c; ll=\"118173\"; _pk_ref.100001.8cb4=%5B%22%22%2C%22%22%2C1591783934%2C%22https%3A%2F%2Fwww.baidu.com%2Flink%3Furl%3Dgjo37j0pVkJmnvxzwAnyq_YEYm3AtwuzNd1W6b0NUckARLYf_cNyMNEgrID_RRaC%26wd%3D%26eqid%3Dcaecc1b00009cacb000000045ee0b1fa%22%5D; _pk_ses.100001.8cb4=*; __utma=30149280.617345388.1581324126.1590475537.1591783934.12; __utmc=30149280; __utmz=30149280.1591783934.12.6.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; ap_v=0,6.0; dbcl2=\"188038058:bRn4JD+366s\"; ck=CqCI; push_noty_num=0; push_doumail_num=0; __utmt=1; _pk_id.100001.8cb4=4ef748b665aff851.1581324124.11.1591785082.1590475562.; __utmb=30149280.19.10.1591783934; ct=y";
+//        String time = "2010-11-16 00:00:00";
+//        String path = "E:\\crawlerdata\\豆瓣.xlsx";
 //        
-//        Douban.doubanTopicGetByWord(word, null, cookie,time);
+//        List<Map<String, Object>> bodyList = Douban.doubanTopicGetByWord(word, null, cookie,time);
+//        PoiExcelUtil poi = PoiExcelUtil.getInstance();
+//        poi.exportExcel(path, "result", Arrays.asList("_id", "title", "group", "time", "reply_count"), bodyList);
 //        
 //    }
 //}

src/test/java/com/zhiwei/shipin/BilibiliTest.java

@@ -22,7 +22,7 @@
 //        for (String word : wordList) {
 //// pubdate 时间 totalrank 综合
 //            List<Map<String, Object>> dataList = BiliBili.getData(word, null, "2001-01-14 00:00:00",
-//                    cookie, "pubdate");
+//                    cookie, "click");
 //            if (dataList != null) {
 //                System.out.println(word + " ----- " + dataList.size());
 //                bodyList.addAll(dataList);
@@ -37,7 +37,7 @@
 //        headlist.add("url");
 //        headlist.add("word");
 //        PoiExcelUtil poi = PoiExcelUtil.getInstance();
-//        poi.exportExcel("E://crawlerdata//视频//bilibili关键词采集数据-腾讯会议-time-20200218.xlsx", "B站数据", headlist, bodyList);
+//        poi.exportExcel("E://crawlerdata//视频//bilibili关键词采集数据-20200515.xlsx", "B站数据", headlist, bodyList);
 //
 //    }
 //}