添加搜狗微信由link获取真实链接（带有效期）方法

pom.xml

@@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>com.zhiwei</groupId>
 	<artifactId>wechat</artifactId>
-	<version>1.2.1-SNAPSHOT</version>
+	<version>1.2.2-SNAPSHOT</version>
 	<description>
 		知微微信采集程序,包含
 		1.微信历史文章采集

src/main/java/com/zhiwei/wechat/search/WechatReal.java

+package com.zhiwei.wechat.search;
+
+import com.zhiwei.crawler.core.HttpBoot;
+import com.zhiwei.crawler.proxy.ProxyHolder;
+import com.zhiwei.crawler.utils.RequestUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.IOException;
+import java.net.Proxy;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class WechatReal {
+
+    private static Logger logger = LogManager.getLogger(WechatReal.class);
+    private static HttpBoot httpBoot = new HttpBoot.Builder().retryTimes(3).useCookieJar(true).build();
+    /**
+     *  获取真实链接
+     * @param originalUrl
+     * @return
+     * @throws IOException
+     */
+    public static String getRealLink(String originalUrl) throws Exception{
+        Proxy proxy = ProxyHolder.NAT_HEAVY_PROXY.getProxy();
+        int b = (int) (Math.floor(100 * Math.random()) + 1);
+        int a = originalUrl.indexOf("url=");
+        int c = originalUrl.indexOf("&k=");
+        String d = null;
+        if (a != -1 && -1 == c) {
+            d = originalUrl.substring(a + 25 + b, a + 26 + b);
+        }
+        originalUrl += "&k=" + b + "&h=" + d;
+        String realUrl = getFinalUrl(originalUrl, proxy);
+        return realUrl;
+    }
+
+    /**
+     * 获取真实链接
+     * @param originalUrl
+     * @return
+     * @throws Exception
+     */
+    private static String getFinalUrl(String originalUrl, Proxy proxy) throws Exception{
+        Map<String,String> headerMap = new HashMap<>();
+        headerMap.put("Sec-Fetch-Mode", "navigate");
+        headerMap.put("Sec-Fetch-User", "?1");
+        headerMap.put("Sec-Fetch-Site", "same-origin");
+        headerMap.put("Host", "weixin.sogou.com");
+        headerMap.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36");
+
+        String word = originalUrl.split("query=")[1];
+        String searchUrl = "https://weixin.sogou.com/weixin?query="+ word;
+
+        for(int i=0; i<3; i++){
+            try{
+                headerMap.put("referer", searchUrl);
+                httpBoot.syncCall(RequestUtils.wrapGet(searchUrl, headerMap), proxy).body().string();
+
+                String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(originalUrl, headerMap), proxy).body().string();
+                headerMap.put("referer", searchUrl);
+
+                if(StringUtils.isNotBlank(htmlBody) && !htmlBody.contains("验证码")){
+                    StringBuilder furl = new StringBuilder();
+                    Pattern pa1 = Pattern.compile("url \\+= \'(.*?)\';");
+                    Matcher ma1 = pa1.matcher(htmlBody);
+                    while (ma1.find()) {
+                        furl.append(ma1.group(1));
+                    }
+                    return furl.toString();
+                }
+            }catch (Exception e){
+                logger.error("获取真实链接时出现错误,错误为:{}", e);
+            }
+        }
+
+        return null;
+    }
+}

src/test/java/com/zhiwei/wechat/example/WechatSearchExample.java

@@ -5,16 +5,17 @@ import java.io.UnsupportedEncodingException;
 import java.net.Proxy;
 import java.net.URLEncoder;
 import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import com.alibaba.fastjson.JSONObject;
 import com.zhiwei.crawler.core.HttpBoot;
 import com.zhiwei.crawler.utils.RequestUtils;
+import com.zhiwei.tools.httpclient.HeaderTool;
+import com.zhiwei.tools.tools.URLCodeUtil;
 import com.zhiwei.tools.tools.ZhiWeiTools;
+import okhttp3.Response;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -36,50 +37,173 @@ public class WechatSearchExample{
 	private Logger logger = LoggerFactory.getLogger(WechatSearchExample.class);
 	private static final String registry = "zookeeper://192.168.0.36:2181";
 	private static final String group = "local";
-	private static HttpBoot httpBoot = new HttpBoot.Builder().retryTimes(3).build();
-    private static Proxy proxy = null;
+	private static HttpBoot httpBoot = new HttpBoot.Builder().retryTimes(3).useCookieJar(true).build();
 
 	public static void main(String[] args) {
 		ProxyFactory.init(registry, group, GroupType.PROVIDER,10000018);
-        proxy = ProxyHolder.SOUGOU_INNER_PROXY.getProxy();
-		String url = "https://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6cwJThYulHEtVjXrGTiVgS8CMDfv9wh9qo5s-_tRRSYjdmlThuLl1UVqXa8Fplpd9fM3bn57YTm93DADHLmE53r3LNleAx90O6EdWFlMiLgABVb5FRuhnTbO_GzJrBhvROBdUYdPJ-HwpjtEvi_VZoCFXsP2Y8IMUHXuJCM5s6KSDiXUXG84daahQ5c0PemEIN_1vJiPn8w7tlTjPfiB-Z3QxFXEDiCN7KNRVfuxuX7N535pjGOOjYg..&type=2&query=%E8%85%BE%E8%AE%AF";
-		try{
-			String cookie = WechatAritcleSearch.getRealUrlCookie(url);
-			if(StringUtils.isNotBlank(cookie)){
-				System.out.println("cookie============="+cookie);
-				boolean f = true;
-				int i = 0;
-				while(f){
-					try{
-						String link = WechatAritcleSearch.getRealLink(url, cookie);
-						if(StringUtils.isNotBlank(link) && link.contains("s?src=")){
-							System.out.println(i+++"=========="+link);
-						}else{
-							System.out.println(i+++"=========="+link);
-						}
-						ZhiWeiTools.sleep(3000);
-					}catch (Exception e){
-						ZhiWeiTools.sleep(50);
-						e.printStackTrace();
+
+		Proxy proxy = ProxyHolder.SOUGOU_INNER_PROXY.getProxy();
+//		String url = "https://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6cwJThYulHEtVjXrGTiVgSzcttWBfUfRyBteZJZKwOQZcZaXkLh7iD1qXa8Fplpd9OIASmEBDDgpc-DopMAxHDRa5rMUETB5W4jcmy1RslCj6dRdWlI71gTiuwjp2qvcTJ8ryfwJWyrd9awnq8kg4J-jH9rgNij43NIxLSEyMEC0OFckdi_fmA1TpUaYEJzIlQ9H-i95UM3h5UwmbSJx95X6FkyXmgknK9g_68U3LLV9hlgeRt7bSzA..&type=2&query=%E4%BA%BA%E6%B0%91%E6%97%A5%E6%8A%A5";
+//		try {
+////			String cookie = getRealUrlCookie(url, proxy);
+////			System.out.println("cookie==================="+cookie);
+//
+//			String realUrl = getRealLink(url, proxy);
+//
+//			System.out.println("realUrl==================="+realUrl);
+//		} catch (Exception e) {
+//			e.printStackTrace();
+//		}
+
+        try{
+            WechatAritcleSearch.wechatKeywordSearch("京东", 5, null, "2019-10-01", "2019-10-01", proxy, 10);
+
+        }catch (Exception e){
+            e.printStackTrace();
+        }
+
+
+
+	}
+
+
+
+
+
+
+
+
+
+	/**
+	 *  获取真实链接
+	 * @param originalUrl
+	 * @return
+	 * @throws IOException
+	 */
+	public static String getRealLink(String originalUrl,Proxy proxy) throws Exception{
+		int b = (int) (Math.floor(100 * Math.random()) + 1);
+		int a = originalUrl.indexOf("url=");
+		int c = originalUrl.indexOf("&k=");
+		String d = null;
+		if (a != -1 && -1 == c) {
+			d = originalUrl.substring(a + 25 + b, a + 26 + b);
+		}
+		originalUrl += "&k=" + b + "&h=" + d;
+		String realUrl = getFinalUrl(originalUrl, proxy);
+		return realUrl;
+	}
+
+
+	/**
+	 * 获取真实链接
+	 * @param originalUrl
+	 * @return
+	 * @throws Exception
+	 */
+	private static String getFinalUrl(String originalUrl, Proxy proxy) throws Exception{
+        Map<String,String> headerMap = new HashMap<>();
+        headerMap.put("Sec-Fetch-Mode", "navigate");
+        headerMap.put("Sec-Fetch-User", "?1");
+        headerMap.put("Sec-Fetch-Site", "same-origin");
+        headerMap.put("Host", "weixin.sogou.com");
+        headerMap.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36");
+
+        String word = originalUrl.split("query=")[1];
+        String searchUrl = "https://weixin.sogou.com/weixin?query="+ word;
+
+        headerMap.put("referer", searchUrl);
+        httpBoot.syncCall(RequestUtils.wrapGet(searchUrl, headerMap), proxy).body().string();
+
+        String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(originalUrl, headerMap), proxy).body().string();
+        headerMap.put("referer", searchUrl);
+
+        if(StringUtils.isNotBlank(htmlBody) && !htmlBody.contains("验证码")){
+            StringBuilder furl = new StringBuilder();
+            Pattern pa1 = Pattern.compile("url \\+= \'(.*?)\';");
+            Matcher ma1 = pa1.matcher(htmlBody);
+            while (ma1.find()) {
+                furl.append(ma1.group(1));
+            }
+            return furl.toString();
+        }
+		return null;
+	}
+
+
+
+	public static String getRealUrlCookie(String originalUrl, Proxy proxy) throws Exception{
+		String word = originalUrl.split("query=")[1];
+		String searchUrl = "https://weixin.sogou.com/weixin?type=2&s_from=input&query="+ word +"&ie=utf8&_sug_=n&_sug_type_=&page=1";
+		Map<String,String> headerMap = HeaderTool.getCommonHead();
+		headerMap.put("Host", "weixin.sogou.com");
+		headerMap.put("Referer", searchUrl);
+        headerMap.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36");
+
+		String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(searchUrl, headerMap), proxy,false).body().string();
+		if(StringUtils.isNotBlank(htmlBody) &&  htmlBody.contains("snuid")){
+			Matcher matcher = Pattern.compile("var uigs_para = [\\s\\S]+?;").matcher(htmlBody);
+			matcher.find();
+			String str = matcher.group().replaceAll("var uigs_para = |;", "");
+			try {
+				str = str.toString().replaceAll(" passportUserId \\? \"1\" \\:", "");
+				JSONObject data = JSONObject.parseObject(str);
+
+				str = str.replaceAll("\\s","").replaceAll("\":\"","=").replaceAll("\",\"","&").replaceAll("\\{\"|\"\\}","");
+				String ac = URLCodeUtil.getURLEncode(str, "utf-8");
+
+				String cookieUrl = "https://pb.sogou.com/cl.gif?uigs_cl=article_title_6&href="+originalUrl+"&uigs_refer=https://weixin.sogou.com/&uigs_t="+ ac +"&right=right0_0&exp_id=null_0-null_1-null_2-null_3-null_4-null_5-null_6-null_7-null_8-null_9";
+
+				headerMap.put("Host", "pb.sogou.com");
+				headerMap.put("Referer", searchUrl);
+                headerMap.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36");
+
+				Response response = httpBoot.syncCall(RequestUtils.wrapGet(cookieUrl, headerMap), proxy,false);
+				if(Objects.nonNull(response)){
+					String cookie = "";
+					List<String> headersResponse = response.headers("Set-Cookie");
+					for (String header : headersResponse) {
+						System.out.println("head======"+header);
+						cookie += header.split(";")[0];
 					}
+					return "uuid=" + data.getString("uuid")+ ";snuid="+ data.getString("snuid")+ ";" +cookie + "IPLOC=CN3302; sct=1;weixinIndexVisited=1";
 				}
-			}else {
-				System.out.println("cookie============="+cookie);
+			}catch (Exception e){
+				e.printStackTrace();
 			}
-		}catch (Exception e){
-			e.printStackTrace();
+		}else{
+			System.out.println("-----------------"+ htmlBody);
 		}
+		return null;
+	}
 
 
-
-//		try {
-//			WechatSearchExample.wechatSearchExample();
-//		} catch (UnknownHostException e) {
-//			e.printStackTrace();
-//		}
+	public static String getCookie(String htmlBody) throws Exception{
+		String cookie = "";
+		if(StringUtils.isNotBlank(htmlBody) &&  htmlBody.contains("snuid")){
+			StringBuilder furl = new StringBuilder();
+			Pattern pa1 = Pattern.compile("\"snuid\"         : \"(.*?)\",");
+			Matcher ma1 = pa1.matcher(htmlBody);
+			while (ma1.find()) {
+				furl.append(ma1.group(1));
+			}
+			return  "SNUID=" + furl.toString();
+		}
+		return null;
 	}
 
 
+
+
+
+
+
+
+
+
+
+
+
+
 	public static void wechatSearchExample() throws UnknownHostException
 	{
 		List<String> wordList = new ArrayList<String>();
@@ -87,7 +211,7 @@ public class WechatSearchExample{
 		for(String word : wordList)
 		{
 			try {
-				List<WechatAricle> list = WechatAritcleSearch.wechatKeywordSearch(word, 5, null,"2019-10-28", "2019-10-28",proxy, 3);
+				List<WechatAricle> list = WechatAritcleSearch.wechatKeywordSearch(word, 5, null,"2019-10-28", "2019-10-28", null, 3);
 				System.out.println("======"+list.size());
 				for(WechatAricle wechat : list){
 					System.out.println(wechat.getId());