天天快报历史文章采集修正

src/main/java/com/zhiwei/httpclient/HeadGet.java

 package com.zhiwei.httpclient;
 
 import java.io.IOException;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -652,11 +653,28 @@ public class HeadGet {
         return headerMap;
     }
     
+    public static Map<String,String> getweiboHeaderMap(String cookie) {
+        Map<String, String> headerMap = new HashMap<String, String>();
+        headerMap.put("User-Agent",
+                "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36");
+        headerMap.put("Accept","*/*");
+        headerMap.put("Accept-Language", "zh-CN,zh;q=0.9");
+        headerMap.put("Connection", "keep-alive");
+        headerMap.put("Content-Type", "application/x-www-form-urlencoded");
+        headerMap.put("Host", "d.weibo.com");
+        if(cookie != null) {
+            headerMap.put("Cookie", cookie);
+        }
+        return headerMap;
+    }
+    
+    
+    
     public static void main(String[] args) {
-        String url = "https://view.inews.qq.com/a/NEW2018021000440002";
-        String cookie = "mstuid=1518141097798_2540; Hm_lvt_71558e7b4aa822e282e758f8dc0b88b0=1518141098; lastsource=so.bbs.xiaomi.cn; mstz=||795199218.38||http%3A%2F%2Fso.bbs.xiaomi.cn%2F%3Fq%3D%25e5%25b0%258f%25e7%25b1%25b3%2520%25e7%2594%25b5%25e9%25a5%25ad%25e7%2585%25b2%2520%25e5%25bc%2580%25e8%25a3%2582%7Cp%3D1%7Cfid%3D0%7Ctime%3D31536000%7Corder%3D1|http%3A%2F%2Fso.bbs.xiaomi.cn%2F%3Fq%3D%25e5%25b0%258f%25e7%25b1%25b3%2520%25e7%2594%25b5%25e9%25a5%25ad%25e7%2585%25b2%2520%25e5%25bc%2580%25e8%25a3%2582%7Cp%3D1%7Cfid%3D0%7Ctime%3D63072000%7Corder%3D1; xm_vistor=1518141097798_2540_1518141097798-1518142530797; msttime=http%3A%2F%2Fso.bbs.xiaomi.cn%2F%3Fq%3D%25E5%25B0%258F%25E7%25B1%25B3%2520%25E7%2594%25B5%25E9%25A5%25AD%25E7%2585%25B2%2520%25E5%25BC%2580%25E8%25A3%2582%26p%3D1%26fid%3D0%26time%3D63072000%26order%3D1; msttime1=http%3A%2F%2Fso.bbs.xiaomi.cn%2F%3Fq%3D%25E5%25B0%258F%25E7%25B1%25B3%2520%25E7%2594%25B5%25E9%25A5%25AD%25E7%2585%25B2%2520%25E5%25BC%2580%25E8%25A3%2582%26p%3D1%26fid%3D0%26time%3D63072000%26order%3D1; Hm_lpvt_71558e7b4aa822e282e758f8dc0b88b0=1518142531";
-        Map<String,String> headerMap = HeadGet.getQQkuaiCommentHeaderMap(null);
-        String result = HttpClient.executeHttpRequestGet(url, null);
+        String url = "https://d.weibo.com/1087030002_2975_1003_0?pids=Pl_Core_F4RightUserList__4&page=2&ajaxpagelet=1&__ref=/1087030002_2975_1003_0&_t=FM_151825274677918";
+        String cookie = "SINAGLOBAL=7701198867685.262.1517207017616; _s_tentry=login.sina.com.cn; Apache=6842405326379.926.1517796423994; ULV=1517796424127:3:1:3:6842405326379.926.1517796423994:1517209523882; ULOGIN_IMG=15177972786361; UOR=,,login.sina.com.cn; YF-Page-G0=23b9d9eac864b0d725a27007679967df; SCF=Ag8PQSV7wMV9Lc8UOZupWW2l6wfI5N2imvtjcwFE3ovIEsRCuG5QaKQhPx4ByaNkpC5LpYocPBPnOJT2NSZMkiU.; SUHB=0C1CJFGk8jNm31; SUB=_2AkMtIj0odcPxrABWn_0WzGPhbYhH-jye91TeAn7uJhMyAxgv7lMFqSVutBF-XFWUFIfrHOaUSPWy_1IBv_YbyS5_; SUBP=0033WrSXqPxfM72wWs9jqgMF55529P9D9WWr5b4iYaaqYk4kfrcubkrT5JpVF02ReoMpSo.XeK.f; login_sid_t=10c8fe00b1833b7414093404448d2330; cross_origin_proto=SSL";
+        Map<String,String> headerMap = HeadGet.getweiboHeaderMap(null);
+        String result = HttpClient.executeHttpRequestGet(url, headerMap);
         System.out.println(result);
         System.out.println(result.length());
     }

src/main/java/com/zhiwei/parse/QQKB.java

@@ -33,9 +33,16 @@ public class QQKB {
         Map<String,Object> paramMap = HeadGet.getQQAccountOneParamMap(child);
         List<Map<String,Object>> dataList = new ArrayList<Map<String,Object>>();
         try {
-            String result = HttpClient.executeHttpRequestPost(url, headerMap, paramMap);
-            List<String> idsList = qqAccountAnalysis.getQQAllIds(result);
-            System.out.println(idsList.size());
+            String result = "";
+            List<String> idsList = new ArrayList<String>();
+            for(int i = 0;i < 3;i++) {
+                result = HttpClient.executeHttpRequestPost(url, headerMap, paramMap);
+                idsList = qqAccountAnalysis.getQQAllIds(result);
+                if(idsList.size() > 1) {
+                    break;
+                }
+            }
+            System.out.println("此帐号可采集的历史文章数==============="+idsList.size());
             url = "http://r.cnews.qq.com/getSubNewsListItems";
             String ids = "";
             int i = 0;
@@ -44,6 +51,7 @@ public class QQKB {
                 i++;
                 if(i >= 20) {
                     try {
+                        for(int j = 1; j < 3;j++) {
                             ids = ids.substring(0,ids.length()-1);
                             System.out.println(ids);
                             ZhiWeiTools.sleep(7000);
@@ -51,9 +59,13 @@ public class QQKB {
                             paramMap = HeadGet.getQQAccountOtherParamMap(ids);
                             result = HttpClient.executeHttpRequestPost(url, headerMap, paramMap);
                             List<Map<String,Object>> list = qqAccountAnalysis.analysisQQAccountData(result);
+                            if(list != null) {
                                 dataList.addAll(list);
+                                break;
+                            }
                             ids = "";
                             i = 0;
+                        }
                     } catch (Exception e) {
                         ids = "";
                         paramMap.clear();
@@ -62,23 +74,26 @@ public class QQKB {
                 }
             }
             if(ids.length() > 1) {
+                for(int j = 1; j < 3;j++) {
                     ids = ids.substring(0,ids.length()-1);
                     ZhiWeiTools.sleep(8000);
                     paramMap.clear();
                     paramMap = HeadGet.getQQAccountOtherParamMap(ids);
                     result = HttpClient.executeHttpRequestPost(url, headerMap, paramMap);
                     List<Map<String,Object>> list = qqAccountAnalysis.analysisQQAccountData(result);
+                    if(list != null) {
                         dataList.addAll(list);
+                        break;
+                    }
+                }
             }
             return dataList;
         } catch (Exception e) {
             logger.error("获取企鹅号历史文章未完全成功",e.getMessage());
-            e.printStackTrace();
             return dataList;
         }
     }
     
-    
     /**
      * 
      * @Description 获取天天快报评论

src/test/java/com/zhiwei/crawler/QQAccountExample.java

@@ -8,24 +8,42 @@ import org.junit.Test;
 
 import com.zhiwei.excelpoi.excel.PoiExcelUtil;
 import com.zhiwei.parse.QQKB;
+import com.zhiwei.zhiweiTools.tools.ZhiWeiTools;
 
 public class QQAccountExample {
     
     @Test
     public void qqAccountTest() {
-        String child = "5975325";
-        String cookie = "phone_id=;%20phone_token=;%20luin=o0497332654;%20lskey=000300006218e8444698ebbad28deed8243ef28e87f247b133bf5d19f160bce91d4e3e06ec2003cd4f92aff0;%20uin=o0497332654;%20skey=MQL8ScQBxu;%20sigA2=2EBEFFCC5C22FD27B32E6F21C76CA4A2AD8BF5D626B629A38C50923810CCD19C574CD093CFAD29C845084C8CD3E04B57DD24A69E4418E060C899696A728FB3B8B14C577F1FF93340;%20openid=oijc7uLZNVp85r-MpFBYxwxzdEkg;%20appid=wxe90c9765ad00e2cd;%20access_token=5_IWbsKfygpJ0lnbUnnFnfwmSQ3EqlyzseC9-qGEFY7Tkr0Ypk5vsnSwOaMC-IGxsBeY2K7knHrYstj_5dZpisJd5nihvLNQvCdsFhFwZQcT8;%20refresh_token=5_IWbsKfygpJ0lnbUnnFnfwgDT4pA9HEbY-wuqnqIHWf9AzdmIueZFuzHYfnZPuSNEc0ZjDuHXrtSrRBMMD-7kgj06iF0NdAOi-KRj6-mrmlA;%20unionid=onCs1uNNpjMXeYIHAhacGypamEBk;%20logintype=0";
         
-        List<Map<String,Object>> dataList = QQKB.getQQAccountData(child, cookie);
-        System.out.println(dataList.size());
         PoiExcelUtil poi = PoiExcelUtil.getInstance();
+        Map<String,Object> dataMap = poi.importExcel("D://crawlerdata/天天快报历史文章采集.xlsx", 0);
+        List<Map<String,Object>> dataList = (List<Map<String, Object>>) dataMap.get("body");
+        String cookie = "phone_id=;%20phone_token=;%20luin=o0497332654;%20lskey=000300006218e8444698ebbad28deed8243ef28e87f247b133bf5d19f160bce91d4e3e06ec2003cd4f92aff0;%20uin=o0497332654;%20skey=MQL8ScQBxu;%20sigA2=2EBEFFCC5C22FD27B32E6F21C76CA4A2AD8BF5D626B629A38C50923810CCD19C574CD093CFAD29C845084C8CD3E04B57DD24A69E4418E060C899696A728FB3B8B14C577F1FF93340;%20openid=oijc7uLZNVp85r-MpFBYxwxzdEkg;%20appid=wxe90c9765ad00e2cd;%20access_token=5_IWbsKfygpJ0lnbUnnFnfwmSQ3EqlyzseC9-qGEFY7Tkr0Ypk5vsnSwOaMC-IGxsBeY2K7knHrYstj_5dZpisJd5nihvLNQvCdsFhFwZQcT8;%20refresh_token=5_IWbsKfygpJ0lnbUnnFnfwgDT4pA9HEbY-wuqnqIHWf9AzdmIueZFuzHYfnZPuSNEc0ZjDuHXrtSrRBMMD-7kgj06iF0NdAOi-KRj6-mrmlA;%20unionid=onCs1uNNpjMXeYIHAhacGypamEBk;%20logintype=0";
+        List<Map<String,Object>> bodyList = new ArrayList<Map<String,Object>>();
+        for(Map<String,Object> map : dataList) {
+            String child = map.get("帐号链接")+"";
+            System.out.println(child.split("chlid=")[1]);
+            List<Map<String,Object>> lists = QQKB.getQQAccountData(child.split("chlid=")[1], cookie);
+            if(lists != null) {
+                for(Map<String,Object> map1 : lists) {
+                    map1.put("name", map.get("呢称"));
+                    map1.put("主页地址", map.get("帐号链接"));
+                    bodyList.add(map1);
+                }
+            }
+            System.out.println("采集到的历史文章数总和============="+bodyList.size());
+            ZhiWeiTools.sleep(5000);
+        }
+        System.out.println(dataList.size());
         List<String> headList = new ArrayList<String>();
+        headList.add("name");
+        headList.add("主页地址");
         headList.add("title");
         headList.add("time");
         headList.add("content");
         headList.add("url");
         headList.add("commentid");
-        poi.exportExcel("D://crawlerdata/qq-5975325.xlsx", "asd", headList, dataList);
+        poi.exportExcel("D://crawlerdata/天天快报采集.xlsx", "asd", headList, bodyList);
     }