增加代理

src/main/java/com/zhiwei/toutiao/parse/TouTiaoCommentParse.java

@@ -13,8 +13,6 @@ import org.slf4j.LoggerFactory;
 
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
-import com.zhiwei.crawler.core.HttpBoot;
-import com.zhiwei.crawler.core.RequestUtils;
 import com.zhiwei.tools.httpclient.HttpClientTemplateOK;
 import com.zhiwei.tools.tools.ZhiWeiTools;
 import com.zhiwei.toutiao.bean.TouTiaoComment;
@@ -49,7 +47,7 @@ public class TouTiaoCommentParse {
 		String group_id = getGroupId(url, proxy);
 		//查询评论总页数
 		if(group_id != null){
-			int page = getPage(group_id);
+			int page = getPage(group_id,proxy);
 			if(returnCount>0){
 				int pageMax = (int)Math.ceil((double)returnCount/20.0);
 				if(page>=pageMax){
@@ -66,7 +64,7 @@ public class TouTiaoCommentParse {
 				headerMap.put("Host", "is.snssdk.com");
 				for(int j=1; j<=3; j++){
 					try {
-						String htmlBody = HttpClientTemplateOK.get(urlNew, null,headerMap);
+						String htmlBody = HttpClientTemplateOK.get(urlNew, proxy,headerMap);
 						if(htmlBody!=null)
 						{
 							 List<TouTiaoComment> commentes = analySisComment(htmlBody, url);
@@ -132,12 +130,12 @@ public class TouTiaoCommentParse {
 	 * @return int 返回类型
 	 * @throws Exception 
 	 */
-	private static int getPage(String group_id) throws Exception
+	private static int getPage(String group_id,Proxy proxy) throws Exception
 	{
 		String urlNew = "http://www.toutiao.com/api/comment/list/?group_id="+group_id+"&item_id=0&count=20&offset=0";
 		//设置头信息
 		Map<String,String> headerMap = Tools.getTouTiaoHeader();
-		String htmlBody = HttpClientTemplateOK.get(urlNew, null,headerMap);
+		String htmlBody = HttpClientTemplateOK.get(urlNew, proxy,headerMap);
 		
 		if(htmlBody!=null)
 		{
@@ -174,7 +172,7 @@ public class TouTiaoCommentParse {
 		    try {
     	        //设置头信息
     	        Map<String,String> headerMap = Tools.getTouTiaoHeader();
-    	        String htmlBody = HttpClientTemplateOK.get(url, null,headerMap);
+    	        String htmlBody = HttpClientTemplateOK.get(url, proxy,headerMap);
     	        if(htmlBody!=null && htmlBody.contains("commentInfo"))
     	        {
     	            try {
@@ -205,7 +203,8 @@ public class TouTiaoCommentParse {
     {
         try {
             //设置头信息
-            String htmlBody = HttpBoot.syncCall(RequestUtils.wrapGet(url)).body().string();
+            Map<String,String> headerMap = Tools.getTouTiaoHeader();
+            String htmlBody = HttpClientTemplateOK.get(url, proxy,headerMap);
             if(htmlBody!=null && htmlBody.contains("commentInfo"))
             {
                 try {
@@ -237,7 +236,7 @@ public class TouTiaoCommentParse {
     		    String urlNew = "http://www.toutiao.com/api/comment/list/?group_id="+group_id+"&item_id=0&count=20&offset=0";
     	        //设置头信息
     	        Map<String,String> headerMap = Tools.getTouTiaoHeader();
-    	        String htmlBody = HttpClientTemplateOK.get(urlNew, null,headerMap);
+    	        String htmlBody = HttpClientTemplateOK.get(urlNew, proxy,headerMap);
     	        
     	        if(htmlBody!=null)
     	        {