Skip to content

W
wechat
Commit ab3e9049 by zhiwei
Options

Merge remote-tracking branch 'origin/master' 

# Conflicts:
#	pom.xml
#	src/main/java/com/zhiwei/wechat/account/WechatAccountFans.java
#	src/main/java/com/zhiwei/wechat/account/WechatAccountInfo.java
#	src/main/java/com/zhiwei/wechat/readAndLike/AriticleContent.java
#	src/main/java/com/zhiwei/wechat/search/WechatAritcleSearch.java
#	src/main/java/com/zhiwei/wechat/search/WechatCount.java
#	src/main/java/com/zhiwei/wechat/search/WechatIndex.java
#	src/main/java/com/zhiwei/wechat/search/WechatReal.java
parents a5057f65 03e0f9fc
Showing with 159 additions and 162 deletions
pom.xml
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>com.zhiwei</groupId> <groupId>com.zhiwei</groupId>
<artifactId>wechat</artifactId> <artifactId>wechat</artifactId>
<version>1.3.2-SNAPSHOT</version> <version>1.3.7-SNAPSHOT</version>
<description> <description>
知微微信采集程序,包含 知微微信采集程序,包含
1.微信历史文章采集 1.微信历史文章采集
......
src/test/java/com/zhiwei/wechat/example/WechatSearchExample.java
package com.zhiwei.wechat.example; //package com.zhiwei.wechat.example;
//
import java.io.IOException; //import java.io.IOException;
import java.io.UnsupportedEncodingException; //import java.io.UnsupportedEncodingException;
import java.net.Proxy; //import java.net.Proxy;
import java.net.URLEncoder; //import java.net.UnknownHostException;
import java.net.UnknownHostException; //import java.util.ArrayList;
import java.util.*; //import java.util.HashMap;
import java.util.regex.Matcher; //import java.util.List;
import java.util.regex.Pattern; //import java.util.Map;
//import java.util.regex.Matcher;
import com.alibaba.fastjson.JSONObject; //import java.util.regex.Pattern;
import com.zhiwei.crawler.core.HttpBoot; //
import com.zhiwei.crawler.utils.RequestUtils; //import org.apache.commons.lang3.StringUtils;
import com.zhiwei.tools.httpclient.HeaderTool; //import org.slf4j.Logger;
import com.zhiwei.tools.tools.URLCodeUtil; //import org.slf4j.LoggerFactory;
import com.zhiwei.tools.tools.ZhiWeiTools; //
import okhttp3.Response; //import com.zhiwei.common.config.GroupType;
import org.apache.commons.lang3.StringUtils; //import com.zhiwei.crawler.core.HttpBoot;
import org.slf4j.Logger; //import com.zhiwei.crawler.core.proxy.ProxyFactory;
import org.slf4j.LoggerFactory; //import com.zhiwei.crawler.core.proxy.ProxyHolder;
//import com.zhiwei.crawler.core.utils.RequestUtils;
import com.zhiwei.common.config.GroupType; //import com.zhiwei.wechat.entity.WechatAricle;
import com.zhiwei.crawler.proxy.ProxyFactory; //import com.zhiwei.wechat.search.WechatAritcleSearch;
import com.zhiwei.crawler.proxy.ProxyHolder; //
import com.zhiwei.wechat.entity.WechatAricle; ///**
import com.zhiwei.wechat.search.WechatAritcleSearch; // * @ClassName: WechatSearchExample
// * @Description: TODO(根据关键词等采集数据)
/** // * @author hero
* @ClassName: WechatSearchExample // * @date 2016年12月16日 上午9:15:42
* @Description: TODO(根据关键词等采集数据) // */
* @author hero //public class WechatSearchExample{
* @date 2016年12月16日 上午9:15:42 //
*/ // private Logger logger = LoggerFactory.getLogger(WechatSearchExample.class);
public class WechatSearchExample{ // private static final String registry = "zookeeper://192.168.0.36:2181";
// private static final String group = "local";
private Logger logger = LoggerFactory.getLogger(WechatSearchExample.class); // private static HttpBoot httpBoot = new HttpBoot.Builder().retryTimes(3).useCookieJar(true).build();
private static final String registry = "zookeeper://192.168.0.36:2181"; //
private static final String group = "local"; // public static void main(String[] args) {
private static HttpBoot httpBoot = new HttpBoot.Builder().retryTimes(3).useCookieJar(true).build(); // ProxyFactory.init(registry, group, GroupType.PROVIDER,10000018);
//
public static void main(String[] args) { // Proxy proxy = ProxyHolder.NAT_HEAVY_PROXY.getProxy();
ProxyFactory.init(registry, group, GroupType.PROVIDER,10000018); // String url = "https://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6cwJThYulHEtVjXrGTiVgSzcttWBfUfRyBteZJZKwOQZcZaXkLh7iD1qXa8Fplpd9OIASmEBDDgpc-DopMAxHDRa5rMUETB5W4jcmy1RslCj6dRdWlI71gTiuwjp2qvcTJ8ryfwJWyrd9awnq8kg4J-jH9rgNij43NIxLSEyMEC0OFckdi_fmA1TpUaYEJzIlQ9H-i95UM3h5UwmbSJx95X6FkyXmgknK9g_68U3LLV9hlgeRt7bSzA..&type=2&query=%E4%BA%BA%E6%B0%91%E6%97%A5%E6%8A%A5";
// try {
Proxy proxy = ProxyHolder.NAT_HEAVY_PROXY.getProxy(); //// String cookie = getRealUrlCookie(url, proxy);
String url = "https://weixin.sogou.com/link?url=dn9a_-gY295K0Rci_xozVXfdMkSQTLW6cwJThYulHEtVjXrGTiVgSzcttWBfUfRyBteZJZKwOQZcZaXkLh7iD1qXa8Fplpd9OIASmEBDDgpc-DopMAxHDRa5rMUETB5W4jcmy1RslCj6dRdWlI71gTiuwjp2qvcTJ8ryfwJWyrd9awnq8kg4J-jH9rgNij43NIxLSEyMEC0OFckdi_fmA1TpUaYEJzIlQ9H-i95UM3h5UwmbSJx95X6FkyXmgknK9g_68U3LLV9hlgeRt7bSzA..&type=2&query=%E4%BA%BA%E6%B0%91%E6%97%A5%E6%8A%A5"; //// System.out.println("cookie==================="+cookie);
try { //
// String cookie = getRealUrlCookie(url, proxy); // String realUrl = getRealLink(url, proxy);
// System.out.println("cookie==================="+cookie); // System.out.println("realUrl==================="+realUrl);
// } catch (Exception e) {
String realUrl = getRealLink(url, proxy); // e.printStackTrace();
System.out.println("realUrl==================="+realUrl); // }
} catch (Exception e) { //
e.printStackTrace(); //// try{
} //// WechatAritcleSearch.wechatKeywordSearch("京东", 5, null, "2019-10-01", "2019-10-01", proxy, 10);
////
// try{ //// }catch (Exception e){
// WechatAritcleSearch.wechatKeywordSearch("京东", 5, null, "2019-10-01", "2019-10-01", proxy, 10); //// e.printStackTrace();
// //// }
// }catch (Exception e){ //
// e.printStackTrace(); //
//
// }
//
//
//
//
//
//
//
//
//
// /**
// * 获取真实链接
// * @param originalUrl
// * @return
// * @throws IOException
// */
// public static String getRealLink(String originalUrl,Proxy proxy) throws Exception{
// int b = (int) (Math.floor(100 * Math.random()) + 1);
// int a = originalUrl.indexOf("url=");
// int c = originalUrl.indexOf("&k=");
// String d = null;
// if (a != -1 && -1 == c) {
// d = originalUrl.substring(a + 25 + b, a + 26 + b);
// }
// originalUrl += "&k=" + b + "&h=" + d;
// String realUrl = getFinalUrl(originalUrl, proxy);
// return realUrl;
// }
//
//
// /**
// * 获取真实链接
// * @param originalUrl
// * @return
// * @throws Exception
// */
// private static String getFinalUrl(String originalUrl, Proxy proxy) throws Exception{
// Map<String,String> headerMap = new HashMap<>();
// headerMap.put("Sec-Fetch-Mode", "navigate");
// headerMap.put("Sec-Fetch-User", "?1");
// headerMap.put("Sec-Fetch-Site", "same-origin");
// headerMap.put("Host", "weixin.sogou.com");
// headerMap.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36");
//
// String word = originalUrl.split("query=")[1];
// String searchUrl = "https://weixin.sogou.com/weixin?query="+ word;
//
// headerMap.put("referer", searchUrl);
// httpBoot.syncCall(RequestUtils.wrapGet(searchUrl, headerMap), proxy).body().string();
//
// String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(originalUrl, headerMap), proxy).body().string();
// headerMap.put("referer", searchUrl);
//
// if(StringUtils.isNotBlank(htmlBody) && !htmlBody.contains("验证码")){
// StringBuilder furl = new StringBuilder();
// Pattern pa1 = Pattern.compile("url \\+= \'(.*?)\';");
// Matcher ma1 = pa1.matcher(htmlBody);
// while (ma1.find()) {
// furl.append(ma1.group(1));
// }
// return furl.toString();
// } // }
// return null;
// }
//
} //
// public static void wechatSearchExample() throws UnknownHostException
// {
// List<String> wordList = new ArrayList<String>();
// wordList.add("京东");
// for(String word : wordList)
// {
// try {
// List<WechatAricle> list = WechatAritcleSearch.wechatKeywordSearch(word, 5, null,"2019-10-28", "2019-10-28", null, 3);
// System.out.println("======"+list.size());
/** // for(WechatAricle wechat : list){
* 获取真实链接 // System.out.println(wechat.getId());
* @param originalUrl
* @return
* @throws IOException
*/
public static String getRealLink(String originalUrl,Proxy proxy) throws Exception{
int b = (int) (Math.floor(100 * Math.random()) + 1);
int a = originalUrl.indexOf("url=");
int c = originalUrl.indexOf("&k=");
String d = null;
if (a != -1 && -1 == c) {
d = originalUrl.substring(a + 25 + b, a + 26 + b);
}
originalUrl += "&k=" + b + "&h=" + d;
String realUrl = getFinalUrl(originalUrl, proxy);
return realUrl;
}
/**
* 获取真实链接
* @param originalUrl
* @return
* @throws Exception
*/
private static String getFinalUrl(String originalUrl, Proxy proxy) throws Exception{
Map<String,String> headerMap = new HashMap<>();
headerMap.put("Sec-Fetch-Mode", "navigate");
headerMap.put("Sec-Fetch-User", "?1");
headerMap.put("Sec-Fetch-Site", "same-origin");
headerMap.put("Host", "weixin.sogou.com");
headerMap.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36");
String word = originalUrl.split("query=")[1];
String searchUrl = "https://weixin.sogou.com/weixin?query="+ word;
headerMap.put("referer", searchUrl);
httpBoot.syncCall(RequestUtils.wrapGet(searchUrl, headerMap), proxy).body().string();
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(originalUrl, headerMap), proxy).body().string();
headerMap.put("referer", searchUrl);
if(StringUtils.isNotBlank(htmlBody) && !htmlBody.contains("验证码")){
StringBuilder furl = new StringBuilder();
Pattern pa1 = Pattern.compile("url \\+= \'(.*?)\';");
Matcher ma1 = pa1.matcher(htmlBody);
while (ma1.find()) {
furl.append(ma1.group(1));
}
return furl.toString();
}
return null;
}
public static void wechatSearchExample() throws UnknownHostException
{
List<String> wordList = new ArrayList<String>();
wordList.add("京东");
for(String word : wordList)
{
try {
List<WechatAricle> list = WechatAritcleSearch.wechatKeywordSearch(word, 5, null,"2019-10-28", "2019-10-28", null, 3);
System.out.println("======"+list.size());
for(WechatAricle wechat : list){
System.out.println(wechat.getId());
}
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
} catch (Exception e) {
e.printStackTrace();
}
// for(String wxId : wechatIds)
// {
// try {
// logger.info("需要采集的wxId:::{}", wxId);
//
// } catch (UnsupportedEncodingException e) {
// e.printStackTrace();
// } catch (Exception e) {
// e.printStackTrace();
// } // }
// } catch (UnsupportedEncodingException e) {
// e.printStackTrace();
// } catch (Exception e) {
// e.printStackTrace();
// } // }
} //// for(String wxId : wechatIds)
} //// {
//// try {
//// logger.info("需要采集的wxId:::{}", wxId);
} ////
//// } catch (UnsupportedEncodingException e) {
//// e.printStackTrace();
//// } catch (Exception e) {
//// e.printStackTrace();
//// }
//// }
// }
// }
//
//
//}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment