Skip to content

W
wechat
Commit a5057f65 by zhiwei
Options

处理搜狗微信搜索链接中出现两次https的问题

parent 09b58307
Showing with 688 additions and 1018 deletions
pom.xml
...@@ -26,15 +26,58 @@ ...@@ -26,15 +26,58 @@
</developer> </developer>
</developers> </developers>
<!-- 分发管理:管理distribution和supporting files -->
<distributionManagement>
<snapshotRepository>
<id>nexus-releases</id>
<name>User Porject Snapshot</name>
<url>http://192.168.0.30:8081/nexus/content/repositories/snapshots/</url>
<uniqueVersion>true</uniqueVersion>
</snapshotRepository>
<repository>
<id>nexus-releases</id>
<name>User Porject Release</name>
<url>http://192.168.0.30:8081/nexus/content/repositories/releases/</url>
</repository>
</distributionManagement>
<dependencies>
<dependency>
<groupId>com.zhiwei.tools</groupId>
<artifactId>zhiwei-tools</artifactId>
<version>0.2.0-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.zhiwei.crawler</groupId>
<artifactId>crawler-core</artifactId>
<version>0.6.6.8-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.zhiwei.middleware</groupId>
<artifactId>automaticmark-client</artifactId>
<version>2.1.7-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.71</version>
<scope>provided</scope>
</dependency>
</dependencies>
<!-- 打包管理 --> <!-- 打包管理 -->
<build> <build>
<plugins> <plugins>
<!-- 发布源码 --> <!-- 发布源码 -->
<plugin> <plugin>
<artifactId>maven-source-plugin</artifactId> <artifactId>maven-source-plugin</artifactId>
<version>2.4</version> <version>2.4</version>
<configuration> <configuration>
<attach>true</attach> <attach>true</attach>
</configuration> </configuration>
<executions> <executions>
<execution> <execution>
...@@ -46,9 +89,9 @@ ...@@ -46,9 +89,9 @@
</executions> </executions>
</plugin> </plugin>
<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId> <artifactId>maven-javadoc-plugin</artifactId>
<version>2.10.4</version> <version>2.10.4</version>
</plugin> </plugin>
<!-- 解决maven test命令时console出现中文乱码乱码 --> <!-- 解决maven test命令时console出现中文乱码乱码 -->
...@@ -63,36 +106,4 @@ ...@@ -63,36 +106,4 @@
</plugin> </plugin>
</plugins> </plugins>
</build> </build>
<!-- 分发管理:管理distribution和supporting files -->
<distributionManagement>
<snapshotRepository>
<id>nexus-releases</id>
<name>User Porject Snapshot</name>
<url>http://192.168.0.30:8081/nexus/content/repositories/snapshots/</url>
<uniqueVersion>true</uniqueVersion>
</snapshotRepository>
<repository>
<id>nexus-releases</id>
<name>User Porject Release</name>
<url>http://192.168.0.30:8081/nexus/content/repositories/releases/</url>
</repository>
</distributionManagement>
<dependencies>
<dependency>
<groupId>com.zhiwei.tools</groupId>
<artifactId>zhiwei-tools</artifactId>
<version>0.1.5-SNAPSHOT</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.zhiwei.crawler</groupId>
<artifactId>crawler-core</artifactId>
<version>0.5.6.3-RELEASE</version>
<scope>provided</scope>
</dependency>
</dependencies>
</project> </project>
\ No newline at end of file
src/main/java/com/zhiwei/wechat/account/WechatAccountFans.java
package com.zhiwei.wechat.account; package com.zhiwei.wechat.account;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import com.zhiwei.crawler.core.HttpBoot; import com.zhiwei.crawler.core.proxy.ProxyHolder;
import com.zhiwei.crawler.utils.RequestUtils; import com.zhiwei.wechat.util.HtmlDownUtil;
import org.apache.commons.lang3.StringUtils;
/**
* @ClassName: WechatAccountFans /**
* @Description: TODO(微信公众号粉丝增量采集程序) * @ClassName: WechatAccountFans
* @author hero * @Description: 微信公众号粉丝增量采集程序
* @date 2017年2月8日 上午11:36:11 * @author hero
*/ * @date 2017年2月8日 上午11:36:11
public class WechatAccountFans { */
public class WechatAccountFans {
// private static Logger logger = LoggerFactory.getLogger(WechatAccountFans.class);
private static HttpBoot httpBoot = new HttpBoot.Builder().retryTimes(3).build(); private Map<String,String> headerMap;
private Map<String,String> headerMap; public WechatAccountFans()
{
public WechatAccountFans() headerMap = new HashMap<String,String>();
{ headerMap.put("Host", "mp.weixin.qq.com");
headerMap = new HashMap<String,String>(); headerMap.put("User-Agent", "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36");
headerMap.put("Host", "mp.weixin.qq.com"); headerMap.put("X-Requested-With", "XMLHttpRequest ");
headerMap.put("User-Agent", "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36"); }
headerMap.put("X-Requested-With", "XMLHttpRequest ");
} /**
* @Title: getWechatAccountFans
/** * @Description: TODO(采集微信公众号的粉丝增量)
* @Title: getWechatAccountFans * @param @param url
* @Description: TODO(采集微信公众号的粉丝增量) * @param @param cookie 设定文件
* @param @param url * @return void 返回类型
* @param @param cookie 设定文件 */
* @return void 返回类型 public JSONArray getWechatAccountFans(String token,String start
*/ ,String end,String cookie) throws Exception
public JSONArray getWechatAccountFans(String token,String start {
,String end,String cookie) throws Exception String url = "https://mp.weixin.qq.com/misc/useranalysis?&begin_date="+start+"&end_date="+end+"&source=99999999,99999999&token="+token+"&lang=zh_CN&f=json&ajax=1";
{ String referer = "https://mp.weixin.qq.com/misc/useranalysis?&token="+token+"&lang=zh_CN";
String url = "https://mp.weixin.qq.com/misc/useranalysis?&begin_date="+start+"&end_date="+end+"&source=99999999,99999999&token="+token+"&lang=zh_CN&f=json&ajax=1"; headerMap.put("Referer", referer);
String referer = "https://mp.weixin.qq.com/misc/useranalysis?&token="+token+"&lang=zh_CN"; headerMap.put("Cookie", cookie);
headerMap.put("Referer", referer);
headerMap.put("Cookie", cookie); String htmlBody = HtmlDownUtil.downloadHtml(url,headerMap, ProxyHolder.NAT_HEAVY_PROXY);
if(StringUtils.isNotBlank(htmlBody))
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(url, headerMap)).body().string(); {
if(htmlBody != null) JSONObject json = JSONObject.parseObject(htmlBody);
{ JSONObject category_list = json.getJSONArray("category_list").getJSONObject(0);
JSONObject json = JSONObject.parseObject(htmlBody); return category_list.getJSONArray("list");
JSONObject category_list = json.getJSONArray("category_list").getJSONObject(0); }else
return category_list.getJSONArray("list"); {
}else return null;
{ }
return null; }
}
}
}
}
src/main/java/com/zhiwei/wechat/comment/WechatCommentList.java deleted 100644 → 0
///**
// * 获取微信文章评论
// * @Title: WechatComment.java
// * @Package com.zhiwei.wechat.comment
// * @Description:获取微信文章评论
// * @author hero
// * @date 2016年6月25日 上午8:17:37
// * @version V1.0
// */ /**
// *
// */
//package com.zhiwei.wechat.comment;
//
//import java.io.IOException;
//import java.util.List;
//import java.util.Map;
//
//import org.slf4j.Logger;
//import org.slf4j.LoggerFactory;
//
//import com.alibaba.fastjson.JSON;
//import com.alibaba.fastjson.JSONObject;
//import com.zhiwei.crawler.core.HttpBoot;
//import com.zhiwei.tools.httpclient.HttpClientTemplateOK;
//import com.zhiwei.wechat.entity.WechatComment;
//import com.zhiwei.wechat.readAndLike.AriticleContent;
//import com.zhiwei.wechat.util.Tools;
//
///**
// * @Description:获取微信文章评论
// * @author hero
// * @date 2016年6月25日 上午8:17:37
// */
//public class WechatCommentList {
//
// private static HttpBoot httpBoot = new HttpBoot.Builder().retryTimes(3).build();
//
// private static WechatComment wc = new WechatComment();
//
// private static Logger logger = LoggerFactory.getLogger(WechatCommentList.class);
// /**
// * 根据文章url获取文章评论列表
// * @Description:
// * @param @param url
// * @param @return
// * @return List<WechatComment> 返回类型
// */
// public static List<WechatComment> getWechatCommentList(String url,String key)
// {
// List<WechatComment> wcList = null;
// /*处理url*/
// String urlcookie = url;
// if(!url.contains("key")){
// urlcookie = Tools.getWechatCookieUrl(url, key);
// }
// // 请求头信息
// Map<String,String> headerMap = Tools.getWechatHeader();
// Map<String, String> cookieMap;
// try {
// cookieMap = HttpClientTemplateOK.getCookie(urlcookie, null, headerMap);
// headerMap.put("Referer", url);
// if(cookieMap.get("cookie").length()>50){
// headerMap.put("Cookie", cookieMap.get("cookie")+"");
// }
// String appmsg_token = Tools.getAppMsgToken(cookieMap.get("htmlBody"));
//
// String biz = url.split("__biz=")[1].split("&")[0];
// String appmsgid = url.split("mid=")[1].split("&")[0];
// String comment_id = AriticleContent.getCommentId(url,key);
// if(comment_id!=null && appmsg_token!=null)
// {
// String comment_url = "https://mp.weixin.qq.com/mp/appmsg_comment?action=getcomment&__biz=" +biz
// + "&appmsgid=" + appmsgid +"&idx=1&comment_id="+comment_id+"&offset=0&limit=100"+key
// + "&appmsg_token=" + appmsg_token;
// /**解析相关数据*/
// System.out.println(comment_url);
// if("0".equals(comment_id))
// {
// logger.info("此条微信文章没有评论");
// }else
// {
// try {
// String htmlBody = HttpClientTemplateOK.get(comment_url, null, headerMap);
// if(htmlBody!=null)
// {
// JSONObject json = JSON.parseObject(htmlBody);
// wcList = wc.constructWechatComment(json.getJSONArray("elected_comment"),url);
// return wcList;
// }
// } catch (Exception e) {
// logger.info("解析微信文章评论列表时出现问题:", e.fillInStackTrace());
// return null;
// }
// }
// }
// } catch (IOException e1) {
// return null;
// } catch (Exception e1) {
// e1.printStackTrace();
// }
//
// return null;
// }
//
//
// /**
// * @Title: getWechatCommentCount
// * @Description: TODO(根据微信文章地址更新微信评论数)
// * @param @param url
// * @param @param key
// * @param @return 设定文件
// * @return int 返回类型
// */
// public static int getWechatCommentCount(String url,String key)
// {
// System.out.println(url);
// /*处理url*/
// String url_new = url;
// if(url.contains("#rd"))
// {
// url_new = url.split("#rd")[0] + key;
// }else if(url.contains("#wechat_redirect"))
// {
// url_new = url.split("#wechat_redirect")[0] + key;
// }
// String biz = url.split("__biz=")[1].split("&")[0];
// String appmsgid = url.split("mid=")[1].split("&")[0];
//
// /**获取网页头信息**/
// Map<String,String> headerMap = Tools.getWechatHeader();
// /*获取评论id*/
// String comment_id = AriticleContent.getCommentId(url,key);
// if(comment_id!=null)
// {
// String comment_url = "http://mp.weixin.qq.com/mp/appmsg_comment?action=getcomment&__biz=" +biz
// + "&appmsgid=" + appmsgid +"&idx=1&comment_id="+comment_id+"&offset=0&limit=100"+key;
// /**解析相关数据*/
//
// if("0".equals(comment_id))
// {
// logger.info("此条微信文章没有评论");
// return 0;
// }else
// {
// try {
// Map<String,String> cookieMap = HttpClientTemplateOK.getCookie(url_new, null,headerMap);
// headerMap.put("Cookie", cookieMap.get("cookie"));
// String htmlBody = HttpClientTemplateOK.get(comment_url, null,headerMap);
// System.out.println(htmlBody);
// if(htmlBody!=null)
// {
// JSONObject json = JSON.parseObject(htmlBody);
// return json.getIntValue("elected_comment_total_cnt");
// }
// } catch (Exception e) {
// logger.debug("更新微信文章评论数时出现问题,问题信息：",e.getMessage());
// return -1;
// }
// }
// }else
// {
// logger.info("获取评论id失败");
// return -1;
// }
// return -1;
// }
//
//
//
//}
src/main/java/com/zhiwei/wechat/entity/WechatAccount.java
package com.zhiwei.wechat.entity; package com.zhiwei.wechat.entity;
import java.io.Serializable; import java.io.Serializable;
/** /**
* @ClassName: WechatAccount * @ClassName: WechatAccount
* @Description: TODO(微信帐号) * @Description: TODO(微信帐号)
* @author Bewilder Z * @author Bewilder Z
* @date 2017年1月16日 上午11:44:21 * @date 2017年1月16日 上午11:44:21
*/ */
public class WechatAccount implements Serializable{ public class WechatAccount implements Serializable{
private static final long serialVersionUID = -5179735277202327683L; private static final long serialVersionUID = -5179735277202327683L;
private String id; //公号id private String id; //公号id
private String name; //公号昵称 private String name; //公号昵称
private String biz; //公号biz private String biz; //公号biz
private String imgurl; //头像地址 private String imgurl; //头像地址
private String descript; //描述 private String descript; //描述
private String verified_reason; //认证原因 private String verifiedReason; //认证原因
private String openid; //认证原因 private String openid; //认证原因
private int article_count_month; //月发文量 private Integer articleCountMonth; //月发文量
private int avg_read_month; //月平均阅读数 private Integer avgReadMonth; //月平均阅读数
public String getId() { public String getId() {
return id; return id;
} }
public void setId(String id) { public void setId(String id) {
this.id = id; this.id = id;
} }
public String getName() { public String getName() {
return name; return name;
} }
public void setName(String name) { public void setName(String name) {
this.name = name; this.name = name;
} }
public String getBiz() { public String getBiz() {
return biz; return biz;
} }
public void setBiz(String biz) { public void setBiz(String biz) {
this.biz = biz; this.biz = biz;
} }
public String getImgurl() { public String getImgurl() {
return imgurl; return imgurl;
} }
public void setImgurl(String imgurl) { public void setImgurl(String imgurl) {
this.imgurl = imgurl; this.imgurl = imgurl;
} }
public String getDescript() { public String getDescript() {
return descript; return descript;
} }
public void setDescript(String descript) { public void setDescript(String descript) {
this.descript = descript; this.descript = descript;
} }
public String getVerified_reason() { public String getVerifiedReason() {
return verified_reason; return verifiedReason;
} }
public void setVerified_reason(String verified_reason) { public void setVerifiedReason(String verifiedReason) {
this.verified_reason = verified_reason; this.verifiedReason = verifiedReason;
} }
public String getOpenid() { public String getOpenid() {
return openid; return openid;
} }
public void setOpenid(String openid) { public void setOpenid(String openid) {
this.openid = openid; this.openid = openid;
} }
public int getArticle_count_month() { public Integer getArticleCountMonth() {
return article_count_month; return articleCountMonth;
} }
public void setArticle_count_month(int article_count_month) { public void setArticleCountMonth(Integer articleCountMonth) {
this.article_count_month = article_count_month; this.articleCountMonth = articleCountMonth;
} }
public int getAvg_read_month() { public Integer getAvgReadMonth() {
return avg_read_month; return avgReadMonth;
} }
public void setAvg_read_month(int avg_read_month) { public void setAvgReadMonth(Integer avgReadMonth) {
this.avg_read_month = avg_read_month; this.avgReadMonth = avgReadMonth;
} }
@Override @Override
public String toString() public String toString()
{ {
return "new WechatAccount[" return "new WechatAccount["
+ "id = " + id + "id = " + id
+ ", name = " + name + ", name = " + name
+ ", biz = " + biz + ", biz = " + biz
+ ", imgurl = " + imgurl + ", imgurl = " + imgurl
+ ", descript = " + descript + ", descript = " + descript
+ ", verified_reason = " + verified_reason + ", verifiedReason = " + verifiedReason
+ ", openid = " + openid + ", openid = " + openid
+ ", article_count_month = " + article_count_month + ", articleCountMonth = " + articleCountMonth
+ ", avg_read_month = " + avg_read_month + ", avgReadMonth = " + avgReadMonth
+ "]"; + "]";
} }
public WechatAccount(){} public WechatAccount(){}
public WechatAccount(String id,String name, String biz, public WechatAccount(String id,String name, String biz,
String imgurl,String descript,String verified_reason, String imgurl,String descript,String verifiedReason,
String openid, int article_count_month,int avg_read_month) String openid, Integer articleCountMonth,Integer avgReadMonth)
{ {
this.id = id; this.id = id;
this.name = name; this.name = name;
this.biz = biz; this.biz = biz;
this.imgurl = imgurl; this.imgurl = imgurl;
this.descript = descript; this.descript = descript;
this.verified_reason = verified_reason; this.verifiedReason = verifiedReason;
this.openid = openid; this.openid = openid;
this.article_count_month = article_count_month; this.articleCountMonth = articleCountMonth;
this.avg_read_month = avg_read_month; this.avgReadMonth = avgReadMonth;
} }
} }
src/main/java/com/zhiwei/wechat/entity/WechatAricle.java
/** /**
* @Title: Wechat.java * @Title: Wechat.java
* @Package com.zhiwei.wechat.bean * @Package com.zhiwei.wechat.bean
* @Description: TODO(用一句话描述该文件做什么) * @Description: TODO(用一句话描述该文件做什么)
* @author zhiweizhang * @author zhiweizhang
* @date 2015年11月5日 下午4:37:10 * @date 2015年11月5日 下午4:37:10
* @version V1.0 * @version V1.0
*/ /** */ /**
* *
*/ */
package com.zhiwei.wechat.entity; package com.zhiwei.wechat.entity;
import java.util.Date; import java.util.Date;
import java.util.List;
/**
* @ClassName: Wechat /**
* @Description: TODO(微信文章) * @ClassName: Wechat
* @author Bewilder Z * @Description: TODO(微信文章)
* @date 2015年11月5日 下午4:37:10 * @author Bewilder Z
*/ * @date 2015年11月5日 下午4:37:10
public class WechatAricle { */
public class WechatAricle {
private String id; //主键
private String id; //主键
private String title; //标题
private String title; //标题
private String source; //来源
private String source; //来源
private String content; //内容
private String content; //内容
private String imgUrl; //图片地址
private Date time; //发布时间
private Date time; //发布时间
private int readNum; //阅读数
private int readNum; //阅读数
private int likeNum; //点赞数
private int likeNum; //点赞数
private String openId;//openid
private String openId;//openid
private String isFirst; //是否为头条文章
private String isFirst; //是否为头条文章
private String biz; //微信公众号唯一标识
private String biz; //微信公众号唯一标识
private String wxId; //微信公众号id
private String wxId; //微信公众号id
private String user_name;//微信公众号初始id
private String user_name;//微信公众号初始id
private String rootSource;
public String getIsFirst() {
return isFirst; private List<String> imgUrls;
}
public void setIsFirst(String isFirst) {
this.isFirst = isFirst; public String getIsFirst() {
} return isFirst;
public String getOpenId() { }
return openId;
} public void setIsFirst(String isFirst) {
public void setOpenId(String openId) { this.isFirst = isFirst;
this.openId = openId; }
}
public String getImgUrl() { public String getOpenId() {
return imgUrl; return openId;
} }
public void setImgUrl(String imgUrl) {
this.imgUrl = imgUrl; public void setOpenId(String openId) {
} this.openId = openId;
}
public String getId() {
return id; public String getId() {
} return id;
public void setId(String id) { }
this.id = id;
} public void setId(String id) {
public String getTitle() { this.id = id;
return title; }
}
public void setTitle(String title) { public String getTitle() {
this.title = title; return title;
} }
public String getSource() {
return source; public void setTitle(String title) {
} this.title = title;
public void setSource(String source) { }
this.source = source;
} public String getSource() {
public String getContent() { return source;
return content; }
}
public void setContent(String content) { public void setSource(String source) {
this.content = content; this.source = source;
} }
public Date getTime() {
return time; public String getContent() {
} return content;
public void setTime(Date time) { }
this.time = time;
} public void setContent(String content) {
public int getReadNum() { this.content = content;
return readNum; }
}
public void setReadNum(int readNum) { public Date getTime() {
this.readNum = readNum; return time;
} }
public int getLikeNum() {
return likeNum; public void setTime(Date time) {
} this.time = time;
public void setLikeNum(int likeNum) { }
this.likeNum = likeNum;
} public int getReadNum() {
return readNum;
}
public String getBiz() {
return biz; public void setReadNum(int readNum) {
} this.readNum = readNum;
public String getWxId() { }
return wxId;
} public int getLikeNum() {
public String getUser_name() { return likeNum;
return user_name; }
}
public void setBiz(String biz) { public void setLikeNum(int likeNum) {
this.biz = biz; this.likeNum = likeNum;
} }
public void setWxId(String wxId) {
this.wxId = wxId;
} public String getBiz() {
public void setUser_name(String user_name) { return biz;
this.user_name = user_name; }
}
public WechatAricle(){} public String getWxId() {
return wxId;
public WechatAricle(String id,String title,String source,String content }
,Date time,int readNum,int likeNum,String openId,String isFirst)
{ public String getUser_name() {
this.id = id.replaceAll("amp;", ""); return user_name;
this.title = title; }
this.source = source;
this.content = content; public void setBiz(String biz) {
this.time = time; this.biz = biz;
this.readNum = readNum; }
this.likeNum = likeNum;
this.openId = openId; public void setWxId(String wxId) {
this.isFirst = isFirst; this.wxId = wxId;
} }
public List<String> getImgUrls() {
@Override return imgUrls;
public String toString() }
{
return "new Wechat[" public void setImgUrls(List<String> imgUrls) {
+ "id = " + id + "," this.imgUrls = imgUrls;
+ "title = " + title + "," }
+ "source = " + source + ","
+ "content = " + content + "," public void setUser_name(String user_name) {
+ "time = " + time + "," this.user_name = user_name;
+ "readNum = " + readNum + "," }
+ "likeNum = " + likeNum + ","
+ "openId = " + openId + "," public String getRootSource() {
+ "isFirst = " + isFirst + "," return rootSource;
+ "]"; }
}
public void setRootSource(String rootSource) {
} this.rootSource = rootSource;
}
public WechatAricle() {
}
public WechatAricle(String id, String title, String source, String content
, Date time, String rootSource, List<String> imgUrls,int readNum, int likeNum, String openId, String isFirst) {
this.id = id.replaceAll("amp;", "");
this.title = title;
this.source = source;
this.content = content;
this.time = time;
this.rootSource = rootSource;
this.imgUrls = imgUrls;
this.readNum = readNum;
this.likeNum = likeNum;
this.openId = openId;
this.isFirst = isFirst;
}
@Override
public String toString() {
return "new Wechat["
+ "id = " + id + ","
+ "title = " + title + ","
+ "source = " + source + ","
+ "content = " + content + ","
+ "time = " + time + ","
+ "rootSource = " + rootSource + ","
+ "imgUrls = " + imgUrls + ","
+ "readNum = " + readNum + ","
+ "likeNum = " + likeNum + ","
+ "openId = " + openId + ","
+ "isFirst = " + isFirst + ","
+ "]";
}
}
src/main/java/com/zhiwei/wechat/readAndLike/AriticleContent.java
/** /**
* 文章具体内容 * 文章具体内容
* @Title: AriticleContent.java * @Title: AriticleContent.java
* @Package com.zhiwei.wechat.readAndLike * @Package com.zhiwei.wechat.readAndLike
* @Description:文章具体内容 * @Description:文章具体内容
* @author Bewilder Z * @author Bewilder Z
* @date 2016年5月20日 下午2:39:46 * @date 2016年5月20日 下午2:39:46
* @version V1.0 * @version V1.0
*/ */
package com.zhiwei.wechat.readAndLike; package com.zhiwei.wechat.readAndLike;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import org.jsoup.Jsoup; import com.zhiwei.crawler.core.proxy.ProxyHolder;
import org.jsoup.nodes.Document; import com.zhiwei.wechat.util.HtmlDownUtil;
import org.slf4j.Logger; import org.apache.commons.lang3.StringUtils;
import org.slf4j.LoggerFactory; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import com.zhiwei.crawler.core.HttpBoot; import org.slf4j.Logger;
import com.zhiwei.crawler.utils.RequestUtils; import org.slf4j.LoggerFactory;
import com.zhiwei.wechat.util.Tools;
import com.zhiwei.crawler.core.HttpBoot;
/** import com.zhiwei.wechat.util.Tools;
* @Description:文章具体内容
* @author hero /**
* @date 2016年5月20日 下午2:39:46 * @Description:文章具体内容
*/ * @author hero
public class AriticleContent{ * @date 2016年5月20日 下午2:39:46
*/
private static Logger logger = LoggerFactory.getLogger(AriticleContent.class); public class AriticleContent{
private static HttpBoot httpBoot = new HttpBoot.Builder().retryTimes(3).build();
private static Logger logger = LoggerFactory.getLogger(AriticleContent.class);
private static HttpBoot httpBoot = new HttpBoot.Builder().retryTimes(3).build();
/**
* @Title: getAriticleContent
* @Description: TODO(根据url获取文章内容) /**
* @param @param url * @Title: getAriticleContent
* @param @return 设定文件 * @Description: TODO(根据url获取文章内容)
* @return Map<String,String> 返回类型 * @param @param url
*/ * @param @return 设定文件
public static Map<String,String> getAriticleContent(String url) * @return Map<String,String> 返回类型
{ */
Map<String,String> result = new HashMap<String,String>(); public static Map<String,String> getAriticleContent(String url)
Map<String,String> headerMap = Tools.getWechatHeader(); {
url = url.replaceAll("amp;", "").replaceAll("amp;", ""); Map<String,String> result = new HashMap<String,String>();
headerMap.put("Referer", url); Map<String,String> headerMap = Tools.getWechatHeader();
String content = null; url = url.replaceAll("amp;", "").replaceAll("amp;", "");
String source = null; headerMap.put("Referer", url);
try { String content = null;
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(url, headerMap)).body().string(); String source = null;
Document document = Jsoup.parse(htmlBody); try {
content = document.select("div.rich_media_content").text(); String htmlBody = HtmlDownUtil.downloadHtml(url, headerMap, ProxyHolder.NAT_HEAVY_PROXY);
if(htmlBody.contains("var nickname = ")){ Document document = Jsoup.parse(htmlBody);
source = htmlBody.split("var nickname = \"")[1].split("\";")[0]; content = document.select("div.rich_media_content").text();
}else{ if(htmlBody.contains("var nickname = ")){
source = document.select("div#meta_content").select("span#profileBt").text(); source = htmlBody.split("var nickname = \"")[1].split("\";")[0];
} }else{
result.put("content", content); source = document.select("div#meta_content").select("span#profileBt").text();
result.put("source", source); }
return result; result.put("content", content);
} catch (Exception e) { result.put("source", source);
logger.debug("获取微信文章内容或来源时出现问题，",e.getMessage()); return result;
return null; } catch (Exception e) {
} logger.debug("获取微信文章内容或来源时出现问题，",e.getMessage());
} return null;
}
/** }
* 获取评论的commentId
* @Description:获取评论的commentId /**
* @param @param url * 获取评论的commentId
* @param @return * @Description:获取评论的commentId
* @return String 返回类型 * @param @param url
*/ * @param @return
public static String getCommentId(String url,String key) * @return String 返回类型
{ */
Map<String,String> headerMap = new HashMap<String,String>(); public static String getCommentId(String url,String key)
{
url = url.replaceAll("amp;", "")+"&key="+key; Map<String,String> headerMap = new HashMap<String,String>();
headerMap.put("Referer", url);
String comment_id = null; url = url.replaceAll("amp;", "")+"&key="+key;
try { headerMap.put("Referer", url);
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(url, headerMap)).body().string(); String comment_id = null;
if(htmlBody!=null) try {
{ String htmlBody = HtmlDownUtil.downloadHtml(url, headerMap, ProxyHolder.NAT_HEAVY_PROXY);
Document document = Jsoup.parse(htmlBody); if(StringUtils.isNotBlank(htmlBody))
String content = document.select("script").html(); {
comment_id = content.split("var comment_id = \"")[1].split("\"")[0]; Document document = Jsoup.parse(htmlBody);
logger.info("comment_id : " + comment_id); String content = document.select("script").html();
} comment_id = content.split("var comment_id = \"")[1].split("\"")[0];
} catch (Exception e) { logger.info("comment_id : " + comment_id);
logger.debug("获取微信文章评论id时出现问题，",e.getMessage()); }
return null; } catch (Exception e) {
} logger.debug("获取微信文章评论id时出现问题，",e.getMessage());
return comment_id; return null;
} }
return comment_id;
public static void main(String[] args) { }
String key = "9ed31d4918c154c810272b09930a4bc0f0cdef34aac4d35a975d8c81fc3261892cf249fedfe7fbacd61a36ecf44d54d1e537ca555379ab6223a63c2c5abf062af1272a2c79a54ee0296ae5d8f22f2092";
String url = "https://mp.weixin.qq.com/s?__biz=MjM5NTE1NTc0MA==&mid=2652457522&idx=1&sn=66496e63dd39097ffb21545a81c45812&scene=0&key="+key+"&ascene=1&uin=MTE4OTQyMDc0MQ%3D%3D"; public static void main(String[] args) {
System.out.println(getCommentId(url,key)); String key = "9ed31d4918c154c810272b09930a4bc0f0cdef34aac4d35a975d8c81fc3261892cf249fedfe7fbacd61a36ecf44d54d1e537ca555379ab6223a63c2c5abf062af1272a2c79a54ee0296ae5d8f22f2092";
} String url = "https://mp.weixin.qq.com/s?__biz=MjM5NTE1NTc0MA==&mid=2652457522&idx=1&sn=66496e63dd39097ffb21545a81c45812&scene=0&key="+key+"&ascene=1&uin=MTE4OTQyMDc0MQ%3D%3D";
System.out.println(getCommentId(url,key));
} }
}
src/main/java/com/zhiwei/wechat/readAndLike/WeChatReadAndLike.java deleted 100644 → 0
///**
// * @Title: WindowsClient.java
// * @Package com.wcral.client
// * @Description: TODO(用一句话描述该文件做什么)
// * @author Bewilder Z
// * @date 2015年8月6日 上午9:13:37
// * @version V1.0
// */
//
//package com.zhiwei.wechat.readAndLike;
//
//import java.net.Proxy;
//import java.net.URLEncoder;
//import java.util.HashMap;
//import java.util.Map;
//
//import org.jsoup.Jsoup;
//import org.jsoup.nodes.Document;
//import org.jsoup.nodes.Element;
//import org.jsoup.select.Elements;
//import org.slf4j.Logger;
//import org.slf4j.LoggerFactory;
//
//import com.alibaba.fastjson.JSONObject;
//import com.zhiwei.tools.httpclient.HttpClientTemplateOK;
//import com.zhiwei.wechat.entity.WechatReadLike;
//import com.zhiwei.wechat.search.WechatAritcleSearch;
//import com.zhiwei.wechat.util.Tools;
//
///**
// * @ClassName: WindowsClient
// * @Description: TODO(利用windows客戶端進行点赞阅读抓取)
// * @author Abner Liu
// * @date 2015年8月6日 上午9:13:37
// */
//public class WeChatReadAndLike {
//
//
// private static Logger logger = LoggerFactory.getLogger(WeChatReadAndLike.class);
//
// /**
// *
// * @Title: getReadAndLike
// * @Description: 利用windows客戶端進行点赞阅读抓取
// * @param url
// * 微信文章链接
// * @return WeChatReadLike 微信文章实体类
// *
// */
// public static WechatReadLike getReadAndLike(String url,String key,Proxy proxy){
// WechatReadLike wLike = new WechatReadLike();
// try {
// String urlcookie = Tools.getWechatCookieUrl(url, key);
// // 请求头信息
// Map<String,String> headerMap = Tools.getWechatHeader();
// Map<String,String> cookieMap = HttpClientTemplateOK.getCookie(urlcookie, proxy, headerMap);
//
// headerMap.put("Referer", urlcookie);
// headerMap.put("Cookie", cookieMap.get("cookie")+"");
// String appmsg_token = Tools.getAppMsgToken(cookieMap.get("htmlBody"));
// System.out.println("appmsg_token==========="+appmsg_token);
// String urlLike = Tools.getWechatLikeUrl(urlcookie,appmsg_token);
// //设置post请求参数
// HashMap<String,Object> postMap = new HashMap<String,Object>();
// postMap.put("is_only_read", "1");
//
// //获取数据
// String htsString = HttpClientTemplateOK.post(urlLike, proxy, headerMap ,postMap);
// System.out.println(htsString);
// JSONObject jsonObject = JSONObject.parseObject(htsString);
// String like_num = jsonObject.getJSONObject("appmsgstat")
// .get("like_num").toString();
//
// String real_read_num = "";
// try {
// real_read_num = jsonObject.getJSONObject("appmsgstat")
// .get("real_read_num").toString();
// if(real_read_num.equals("0"))
// {
// real_read_num = jsonObject.getJSONObject("appmsgstat")
// .get("read_num").toString();
// }
// } catch (Exception e) {
// real_read_num = jsonObject.getJSONObject("appmsgstat")
// .get("read_num").toString();
// }
// wLike.setUrl(url);
// wLike.setRead(Integer.valueOf(real_read_num));
// wLike.setLike(Integer.valueOf(like_num));
// } catch (Exception e) {
// wLike.setUrl(url);
// wLike.setRead(-1);
// wLike.setLike(-1);
// }
// return wLike;
// }
//
//
//
// /**
// * @Title: getReadAndLike
// * @Description: TODO(通过搜狗微信获取阅读数)
// * @param @param word
// * @param @param time
// * @param @param link
// * @param @param wxId
// * @param @return 设定文件
// * @return WeChatReadLike 返回类型
// */
// public static WechatReadLike getReadAndLike(String word,
// String time,String link,String wxId){
//
// WechatReadLike wLike = new WechatReadLike();
//
// Map<String,String> headerMap = new HashMap<String,String>();
// headerMap.put("Upgrade-Insecure-Requests", "1");
// headerMap.put("User-Agent", "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36");
// headerMap.put("Host","weixin.sogou.com");
//
// if(time.contains(" "))
// {
// time = time.split(" ")[0];
// }
//
// String openid = WechatAritcleSearch.getOpenId(wxId,null);
// logger.info("openid is {}", openid);
//
// try {
// String url = "http://weixin.sogou.com/weixin?query=" + URLEncoder.encode(word,"utf-8")
// + "&type=2&ie=utf8&page=1&interation=&tsn=5&ft="+time + "&et="+ time
// + "&wxid="+openid+"&usip="+wxId+"&from=tool";
//
// logger.info("url is {}",url);
//
// String htmlBody = HttpClientTemplateOK.get(url, null, headerMap);
// if(htmlBody!=null)
// {
// try {
// // 解析数据
// Document document = Jsoup.parse(htmlBody);
// Elements elements = document.select("div.news-box")
// .select("ul.news-list").select("li");
// for (Element element : elements)
// {
// try {
// String url_link = element.select("div.txt-box").select("h3 >a").attr("href");
// int readNum = 0;
// try {
// readNum = Integer.valueOf(element.select("div.txt-box")
// .select("div.s-p").select("span.s1").text().trim());
// logger.info("readNum is {}", readNum);
// } catch (Exception e) {
// readNum = 0;
// }
// if(url_link.contains("&chksm="))
// {
// url_link = url_link.split("&chksm=")[0] + "&3rd" + url_link.split("&3rd")[1];
// }
//
// if(link.equals(url_link))
// {
// wLike.setUrl(link);
// wLike.setRead(readNum);
// break;
// }
// } catch (Exception e) {
// continue;
// }
// }
// } catch (Exception e) {
// wLike.setUrl(link);
// wLike.setRead(0);
// return null;
// }
// }
// } catch (Exception e) {
// e.printStackTrace();
// wLike.setUrl(link);
// wLike.setRead(0);
// return null;
// }
// return wLike;
// }
//
//}
src/main/java/com/zhiwei/wechat/search/WechatCount.java
package com.zhiwei.wechat.search; package com.zhiwei.wechat.search;
import java.net.Proxy; import java.net.Proxy;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import org.jsoup.Jsoup; import com.zhiwei.wechat.util.HtmlDownUtil;
import org.jsoup.select.Elements; import org.jsoup.Jsoup;
import org.jsoup.select.Elements;
import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.utils.RequestUtils;
public class WechatCount {
public class WechatCount {
public static int getWechatCountByWord(String word, String cookie,
private static HttpBoot httpBoot = new HttpBoot.Builder().retryTimes(3).build(); String startTime, String endTime, Proxy proxy) {
Map<String, String> headerMap = getWechatCount(cookie);
public static int getWechatCountByWord(String word, String cookie, try {
String startTime, String endTime, Proxy proxy) { String url = "http://weixin.sogou.com/weixin?type=2&ie=utf8&query="
Map<String, String> headerMap = getWechatCount(cookie); + URLEncoder.encode(word, "UTF-8") + "&tsn=5&ft="
try { + startTime.split(" ")[0] + "&et=" + endTime.split(" ")[0]
String url = "http://weixin.sogou.com/weixin?type=2&ie=utf8&query=" + "&interation=&wxid=&usip=";
+ URLEncoder.encode(word, "UTF-8") + "&tsn=5&ft=" headerMap.put("Referer", url);
+ startTime.split(" ")[0] + "&et=" + endTime.split(" ")[0] String result = HtmlDownUtil.downloadHtml(url, headerMap, proxy);
+ "&interation=&wxid=&usip="; String s = "";
headerMap.put("Referer", url); int n = -1;
String result = httpBoot if (result.contains("找到约") && result.contains("条结果")) {
.syncCall(RequestUtils.wrapGet(url, headerMap), proxy) s = result.split("找到约")[1].split("条结果")[0];
.body().string(); n = Integer.valueOf(s.replaceAll(",", ""));
String s = ""; }
int n = -1; Elements elements = Jsoup.parse(result).select("ul.news-list")
if (result.contains("找到约") && result.contains("条结果")) { .select("li");
s = result.split("找到约")[1].split("条结果")[0]; if (n == -1) {
n = Integer.valueOf(s.replaceAll(",", "")); n = elements.size();
} }
Elements elements = Jsoup.parse(result).select("ul.news-list") return n;
.select("li"); } catch (Exception e) {
if (n == -1) { return 0;
n = elements.size(); }
} }
return n;
} catch (Exception e) { public static void main(String[] args) {
return 0; String cookie = "CXID=B4F6BF06873D2EFBC767B5EFF2747322; SUID=EAD6E7735E68860A5BF25BAB000EB028; SUV=0008470373E7D6EA5BF756F2BAF2D602; ABTEST=0|1543028023|v1; IPLOC=CN3302; weixinIndexVisited=1; ad=lFGL3Zllll2bDOmrlllllVZp2JyllllltMKQfkllllwlllllxp2PGK@@@@@@@@@@";
} String startTime = "2018-08-01 00:00:00";
} String endTime = "2018-08-02 00:00:00";
public static void main(String[] args) { int c = getWechatCountByWord("唐嫣", cookie, startTime, endTime, null);
String cookie = "CXID=B4F6BF06873D2EFBC767B5EFF2747322; SUID=EAD6E7735E68860A5BF25BAB000EB028; SUV=0008470373E7D6EA5BF756F2BAF2D602; ABTEST=0|1543028023|v1; IPLOC=CN3302; weixinIndexVisited=1; ad=lFGL3Zllll2bDOmrlllllVZp2JyllllltMKQfkllllwlllllxp2PGK@@@@@@@@@@"; System.out.println(c);
String startTime = "2018-08-01 00:00:00"; }
String endTime = "2018-08-02 00:00:00";
private static Map<String, String> getWechatCount(String cookie) {
int c = getWechatCountByWord("唐嫣", cookie, startTime, endTime, null); Map<String, String> headerMap = new HashMap<>();
System.out.println(c); headerMap.put("User-Agent",
} "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36");
headerMap.put("Accept",
private static Map<String, String> getWechatCount(String cookie) { "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
Map<String, String> headerMap = new HashMap<>(); headerMap.put("Accept-Language", "zh-CN,zh;q=0.9");
headerMap.put("User-Agent", headerMap.put("Connection", "keep-alive");
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36"); headerMap.put("Host", "weixin.sogou.com");
headerMap.put("Accept", if (cookie != null) {
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"); headerMap.put("Cookie", cookie);
headerMap.put("Accept-Language", "zh-CN,zh;q=0.9"); }
headerMap.put("Connection", "keep-alive"); return headerMap;
headerMap.put("Host", "weixin.sogou.com"); }
if (cookie != null) {
headerMap.put("Cookie", cookie); }
}
return headerMap;
}
}
src/main/java/com/zhiwei/wechat/search/WechatIndex.java deleted 100644 → 0
package com.zhiwei.wechat.search;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.utils.RequestUtils;
import com.zhiwei.tools.timeparse.TimeParse;
import com.zhiwei.tools.tools.URLCodeUtil;
/**
* @ClassName: WechatIndex
* @Description: TODO(微信指数搜索)
* @author Bewilder Z
* @date 2017年3月24日 下午2:52:01
*/
public class WechatIndex {
private static HttpBoot httpBoot = new HttpBoot.Builder().retryTimes(3).build();
public static void main(String[] args) throws Exception {
String word = "百度";
String startTime = "2016-01-01 00:00:00";
String endTime = "2017-03-24 00:00:00";
Map<String,String> timeLine = TimeParse.getTimeMap(startTime, endTime, "dd", 7);
for(Entry<String,String> entry: timeLine.entrySet())
{
Date st = TimeParse.stringFormartDate(entry.getKey());
Date et = TimeParse.stringFormartDate(entry.getValue());
getWechatIndex(word,st.getTime()/1000L,et.getTime()/1000L);
}
}
public static void getWechatIndex(String word,long startTime,long endTime) throws Exception
{
String url = "https://search.weixin.qq.com/cgi-bin/searchweb/getwxindex?query="
+URLCodeUtil.getURLEncode(word, "utf-8")+"&start_time="+startTime+"&end_time="+endTime+"&_="+new Date().getTime();
// String urlCookie = "https://search.weixin.qq.com/cgi-bin/searchweb/getjsapiticket?sign_url=https%253A%252F%252Fsearch.weixin.qq.com%252Fcgi-bin%252Fsearchweb%252Fclientjump%253Ftag%253Dwxindex%2526exportkey%253DAStrb5tD4ruSixIDu1cVpTA%25253D%2526pass_ticket%253DbbP7ZT5xEUrYe%25252BoOa6ACUw%25252BmgR05TAGGA1P9xnC7fIyaaOnwkWyNQK8aYtva%25252BGxj&_=1490341301892";
System.out.println(url);
Map<String,String> headerMap = new HashMap<String,String>();
headerMap.put("Host", "search.weixin.qq.com");
headerMap.put("User-Agent", "Mozilla/5.0 (iPhone; CPU iPhone OS 10_2_1 like Mac OS X) AppleWebKit/602.4.6 (KHTML, like Gecko) Mobile/14D27 MicroMessenger/6.5.5 NetType/4G Language/zh_CN");
headerMap.put("Referer", url);
headerMap.put("X-Requested-With","XMLHttpRequest");
headerMap.put("Accept","application/json, text/javascript, */*; q=0.01");
headerMap.put("Cookie","mmsearch_user_key=AStrb5tD4ruSixIDu1cVpTA=; pass_ticket=bbP7ZT5xEUrYe+oOa6ACUw+mgR05TAGGA1P9xnC7fIyaaOnwkWyNQK8aYtva+Gxj; pgv_pvi=4102772736; pgv_si=s1607859200; pgv_pvid=153672700");
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(url, headerMap)).body().string();
System.out.println(htmlBody);
Thread.sleep(3000);
// Document htmlBody = Jsoup.connect(url)
// .header("Host", "search.weixin.qq.com")
// .header("User-Agent", "Mozilla/5.0 (iPhone; CPU iPhone OS 10_2_1 like Mac OS X) AppleWebKit/602.4.6 (KHTML, like Gecko) Mobile/14D27 MicroMessenger/6.5.5 NetType/4G Language/zh_CN")
// .header("Referer", url)
// .header("X-Requested-With","XMLHttpRequest")
// .header("Accept","application/json, text/javascript, */*; q=0.01")
// .header("Cookie", "mmsearch_user_key=AfNSrJx116RWkWvTuVC949k=; pass_ticket=bbP7ZT5xEUrYe+oOa6ACUw+mgR05TAGGA1P9xnC7fIyaaOnwkWyNQK8aYtva+Gxj; pgv_pvi=4102772736; pgv_si=s1607859200; pgv_pvid=153672700")
// .ignoreHttpErrors(false)
// .ignoreContentType(true)
// .timeout(3000)
// .get();
}
}
src/main/java/com/zhiwei/wechat/search/WechatReal.java
package com.zhiwei.wechat.search; package com.zhiwei.wechat.search;
import com.zhiwei.crawler.core.HttpBoot; import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.proxy.ProxyHolder; import com.zhiwei.crawler.core.proxy.ProxyHolder;
import com.zhiwei.crawler.utils.RequestUtils; import com.zhiwei.crawler.core.utils.RequestUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
......
src/main/java/com/zhiwei/wechat/util/HtmlDownUtil.java 0 → 100644
package com.zhiwei.wechat.util;
import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.core.proxy.ProxyHolder;
import com.zhiwei.crawler.core.utils.RequestUtils;
import com.zhiwei.wechat.search.WechatAritcleSearch;
import okhttp3.Response;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.IOException;
import java.net.Proxy;
import java.util.Map;
/**
* @ProjectName: wechat
* @ClassName: HtmlDownUtil
* @Author: admin
* @Description: 网页数据下载
* @Date: 2020/8/3 8:57
* @Version: 1.0
*/
public class HtmlDownUtil {
private static Logger logger = LogManager.getLogger(WechatAritcleSearch.class);
private static HttpBoot httpBoot = new HttpBoot.Builder().retryTimes(3).build();
/**
* 根据链接获取网页数据
* @param url
* @param headMap
* @param proxy
* @return
* @throws IOException
*/
public static String downloadHtml(String url, Map<String,String> headMap, Proxy proxy)throws IOException{
headMap.put("cookie", "com_sohu_websearch_ITEM_PER_PAGE=100;");
try(Response response = httpBoot.syncCall(RequestUtils.wrapGet(url, headMap), proxy, false)) {
return response.body().string();
}catch (IOException e){
throw e;
}
}
public static String downloadHtml(String url, Map<String,String> headMap, ProxyHolder proxyHolder)throws IOException{
headMap.put("cookie", "com_sohu_websearch_ITEM_PER_PAGE=100;");
try(Response response = httpBoot.syncCall(RequestUtils.wrapGet(url, headMap), proxyHolder, false)) {
return response.body().string();
}catch (IOException e){
throw e;
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment