Skip to content

S
searchhotcrawler
Commit aec57a25 by [zhangzhiwei]
Options

修改采集核心包及更新相应方法

parent 62364c66
Showing with 50 additions and 36 deletions
pom.xml
......@@ -21,11 +21,6 @@
</developers>
<dependencies>
<dependency>
<groupId>com.zhiwei</groupId>
<artifactId>zhiweiTools</artifactId>
<version>0.0.6-SNAPSHOT</version>
</dependency>
<!-- 数据解析jar -->
......@@ -40,6 +35,16 @@
<artifactId>sendmail</artifactId>
<version>0.0.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.zhiwei.crawler</groupId>
<artifactId>crawler-core</artifactId>
<version>0.1.1-RELEASE</version>
</dependency>
<dependency>
<groupId>com.zhiwei.tools</groupId>
<artifactId>zhiwei-tools</artifactId>
<version>0.1.1-SNAPSHOT</version>
</dependency>
</dependencies>
<build>
......
src/main/java/com/zhiwei/searchhotcrawler/bean/WeiboHotSearch.java
......@@ -9,7 +9,8 @@ package com.zhiwei.searchhotcrawler.bean;
import java.io.Serializable;
import java.util.Date;
import com.zhiwei.zhiweiTools.timeParse.TimeParse;
import com.zhiwei.tools.timeparse.TimeParse;
public class WeiboHotSearch implements Serializable{
......
src/main/java/com/zhiwei/searchhotcrawler/cache/CacheListener.java
......@@ -3,7 +3,8 @@ package com.zhiwei.searchhotcrawler.cache;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.zhiwei.zhiweiTools.tools.ZhiWeiTools;
import com.zhiwei.tools.tools.ZhiWeiTools;
public class CacheListener {
Logger logger = LoggerFactory.getLogger(CacheListener.class);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiboHotSearchCrawler.java
......@@ -15,10 +15,11 @@ import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.core.RequestUtils;
import com.zhiwei.searchhotcrawler.bean.WeiboHotSearch;
import com.zhiwei.searchhotcrawler.mail.SendMailWeibo;
import com.zhiwei.zhiweiTools.httpClient.HttpClientTemplateOK;
import com.zhiwei.zhiweiTools.tools.URLCodeUtil;
import com.zhiwei.tools.tools.URLCodeUtil;
/**
* @ClassName: WeiboHotSearch
......@@ -29,7 +30,7 @@ import com.zhiwei.zhiweiTools.tools.URLCodeUtil;
public class WeiboHotSearchCrawler {
private static Logger logger = LoggerFactory.getLogger(WeiboHotSearchCrawler.class);
private static HttpBoot httpBoot = new HttpBoot();
/**
* @Title: weiboHotSearchTest
* @author hero
......@@ -51,7 +52,7 @@ public class WeiboHotSearchCrawler {
for(int i =0; i<3; i++){
String htmlBody = null;
try {
htmlBody = HttpClientTemplateOK.get(url, null, headerMap);
htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(url, headerMap)).body().string();
if(htmlBody!=null && htmlBody.contains("pl_top_realtimehot")){
try {
// String script = htmlBody.split("<script>STK && STK.pageletM && STK.pageletM.view")[5].split("<\\/script>")[0];
......@@ -115,7 +116,7 @@ public class WeiboHotSearchCrawler {
List<WeiboHotSearch> result = new ArrayList<WeiboHotSearch>();
String htmlBody;
try {
htmlBody = HttpClientTemplateOK.get(url, null, headerMap);
htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(url, headerMap)).body().string();
if(htmlBody!=null){
try {
JSONObject json = JSONObject.parseObject(htmlBody);
......
src/main/java/com/zhiwei/searchhotcrawler/crawler/ZhihuHotSearchCrawler.java
......@@ -11,10 +11,11 @@ import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.core.RequestUtils;
import com.zhiwei.searchhotcrawler.bean.ZhihuHotSearch;
import com.zhiwei.zhiweiTools.httpClient.HeaderTool;
import com.zhiwei.zhiweiTools.httpClient.HttpClientTemplateOK;
import com.zhiwei.zhiweiTools.tools.URLCodeUtil;
import com.zhiwei.tools.httpclient.HeaderTool;
import com.zhiwei.tools.tools.URLCodeUtil;
/**
* @ClassName: ZhihuHotCrawler
......@@ -25,7 +26,7 @@ import com.zhiwei.zhiweiTools.tools.URLCodeUtil;
public class ZhihuHotSearchCrawler {
private static Logger logger = LoggerFactory.getLogger(ZhihuHotSearchCrawler.class);
private static HttpBoot httpBoot = new HttpBoot();
/**
* @Title: getZhihuHotList
* @author hero
......@@ -46,7 +47,7 @@ public class ZhihuHotSearchCrawler {
headerMap.put("Referer", rerferer);
for(int j=0;j<3;j++){
try {
String htmlBody = HttpClientTemplateOK.get(url, null, headerMap);
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(url, headerMap)).body().string();
if(htmlBody != null){
if(htmlBody.contains("words")){
list = new ArrayList<ZhihuHotSearch>();
......@@ -95,7 +96,7 @@ public class ZhihuHotSearchCrawler {
for(int j=0;j<3;j++){
try {
String htmlBody = HttpClientTemplateOK.get(url, null, headerMap);
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(url, headerMap)).body().string();
if(htmlBody != null){
if(htmlBody.contains("author")){
list = new ArrayList<ZhihuHotSearch>();
......
src/main/java/com/zhiwei/searchhotcrawler/dao/WeiboHotSearchDAO.java
......@@ -4,15 +4,15 @@ package com.zhiwei.searchhotcrawler.dao;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import com.mongodb.BasicDBObject;
import com.mongodb.DBCursor;
import com.mongodb.DBObject;
import com.mongodb.WriteConcern;
import com.zhiwei.searchhotcrawler.bean.WeiboHotSearch;
import com.zhiwei.searchhotcrawler.cache.CacheManager;
import com.zhiwei.searchhotcrawler.config.Config;
import com.zhiwei.searchhotcrawler.dbtemplate.MongoDBTemplate;
import com.zhiwei.zhiweiTools.tools.ZhiWeiTools;
import com.zhiwei.tools.tools.ZhiWeiTools;
public class WeiboHotSearchDAO extends MongoDBTemplate{
......
src/main/java/com/zhiwei/searchhotcrawler/dao/ZhihuHotSearchDAO.java
......@@ -10,7 +10,7 @@ import com.mongodb.DBObject;
import com.mongodb.WriteConcern;
import com.zhiwei.searchhotcrawler.config.Config;
import com.zhiwei.searchhotcrawler.dbtemplate.MongoDBTemplate;
import com.zhiwei.zhiweiTools.tools.ZhiWeiTools;
import com.zhiwei.tools.tools.ZhiWeiTools;
public class ZhihuHotSearchDAO extends MongoDBTemplate{
public ZhihuHotSearchDAO() {
......
src/main/java/com/zhiwei/searchhotcrawler/run/HotSearchRun.java
......@@ -10,7 +10,7 @@ import com.zhiwei.searchhotcrawler.timer.SendZhihuHotSearchRun;
import com.zhiwei.searchhotcrawler.timer.UpdateWechatUserRun;
import com.zhiwei.searchhotcrawler.timer.WeiboHotSearchRun;
import com.zhiwei.searchhotcrawler.timer.ZhihuHotSearchRun;
import com.zhiwei.zhiweiTools.tools.ZhiWeiTools;
import com.zhiwei.tools.tools.ZhiWeiTools;
public class HotSearchRun {
......
src/main/java/com/zhiwei/searchhotcrawler/timer/SendWeiboHotSearchRun.java
......@@ -5,8 +5,10 @@ import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSONObject;
import com.mongodb.DBObject;
import com.zhiwei.searchhotcrawler.dao.WechatUserDao;
......@@ -14,8 +16,8 @@ import com.zhiwei.searchhotcrawler.dao.WeiboHotSearchDAO;
import com.zhiwei.searchhotcrawler.util.Template;
import com.zhiwei.searchhotcrawler.util.WechatCodeUtil;
import com.zhiwei.searchhotcrawler.util.WechatConstant;
import com.zhiwei.zhiweiTools.timeParse.TimeParse;
import com.zhiwei.zhiweiTools.tools.ZhiWeiTools;
import com.zhiwei.tools.timeparse.TimeParse;
import com.zhiwei.tools.tools.ZhiWeiTools;
public class SendWeiboHotSearchRun extends Thread {
private WeiboHotSearchDAO weiboHotSearchDAO = new WeiboHotSearchDAO();
......
src/main/java/com/zhiwei/searchhotcrawler/timer/SendZhihuHotSearchRun.java
......@@ -16,8 +16,8 @@ import com.zhiwei.searchhotcrawler.dao.ZhihuHotSearchDAO;
import com.zhiwei.searchhotcrawler.util.Template;
import com.zhiwei.searchhotcrawler.util.WechatCodeUtil;
import com.zhiwei.searchhotcrawler.util.WechatConstant;
import com.zhiwei.zhiweiTools.timeParse.TimeParse;
import com.zhiwei.zhiweiTools.tools.ZhiWeiTools;
import com.zhiwei.tools.timeparse.TimeParse;
import com.zhiwei.tools.tools.ZhiWeiTools;
public class SendZhihuHotSearchRun extends Thread{
private ZhihuHotSearchDAO zhihuHotSearchDAO = new ZhihuHotSearchDAO();
......
src/main/java/com/zhiwei/searchhotcrawler/timer/UpdateWechatUserRun.java
......@@ -4,11 +4,13 @@ import java.util.Calendar;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.zhiwei.searchhotcrawler.dao.WechatUserDao;
import com.zhiwei.searchhotcrawler.util.WechatCodeUtil;
import com.zhiwei.zhiweiTools.tools.ZhiWeiTools;
import com.zhiwei.tools.tools.ZhiWeiTools;
public class UpdateWechatUserRun extends Thread{
private WechatUserDao wechatUserDao = new WechatUserDao();
......
src/main/java/com/zhiwei/searchhotcrawler/util/WechatCodeUtil.java
......@@ -10,12 +10,13 @@ import org.slf4j.LoggerFactory;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.zhiwei.zhiweiTools.httpClient.HeaderTool;
import com.zhiwei.zhiweiTools.httpClient.HttpClientTemplateOK;
import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.core.RequestUtils;
import com.zhiwei.tools.httpclient.HeaderTool;
public class WechatCodeUtil {
private static Logger logger = LoggerFactory.getLogger(WechatCodeUtil.class);
private static HttpBoot httpBoot = new HttpBoot();
/**
* @Title: getToken
* @author hero
......@@ -28,10 +29,10 @@ public class WechatCodeUtil {
String token = "";
String appId = "wx2f555218d66e5948";
String jmAppId = AESUtils.encrypt("wechat", appId);
String path = "http://yuqing.zhiweidata.com/WechatPublic/common/getToken?appId=" + jmAppId;
String url = "http://yuqing.zhiweidata.com/WechatPublic/common/getToken?appId=" + jmAppId;
Map<String, String> headerMap = HeaderTool.getCommonHead();
try {
String result = HttpClientTemplateOK.get(path, null, headerMap);
String result = httpBoot.syncCall(RequestUtils.wrapGet(url, headerMap)).body().string();
if (result != null) {
JSONObject jsonObject = JSONObject.parseObject(result);
if(jsonObject.containsKey("data")) {
......@@ -160,7 +161,7 @@ public class WechatCodeUtil {
Integer groupId = null;
Map<String, String> headerMap = HeaderTool.getCommonHead();
try {
String htmlBody = HttpClientTemplateOK.get(url, null, headerMap);
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(url, headerMap)).body().string();
if (htmlBody != null) {
if(htmlBody.contains("tags")) {
JSONArray jsonArry = JSONObject.parseObject(htmlBody).getJSONArray("tags");
......@@ -191,7 +192,7 @@ public class WechatCodeUtil {
Map<String,Integer> resultMap = new HashMap<String,Integer>();
Map<String, String> headerMap = HeaderTool.getCommonHead();
try {
String htmlBody = HttpClientTemplateOK.get(url, null, headerMap);
String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(url, headerMap)).body().string();
if (htmlBody != null) {
if(htmlBody.contains("tags")) {
JSONArray jsonArry = JSONObject.parseObject(htmlBody).getJSONArray("tags");
......
src/main/resources/db.properties
#mongoIp=202.107.192.94
mongoIp=192.168.0.101
mongoIp=202.107.192.94
#mongoIp=192.168.0.101
mongoPort=30000
db.username=zzwno
db.paasword=zzwno1q2w3e4r
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment