修改采集核心包及更新相应方法

pom.xml

@@ -21,11 +21,6 @@
 	</developers>
   
   <dependencies>
-  		<dependency>
-			<groupId>com.zhiwei</groupId>
-			<artifactId>zhiweiTools</artifactId>
-			<version>0.0.6-SNAPSHOT</version>
-		</dependency>
 		
 		<!-- 数据解析jar -->
 
@@ -40,6 +35,16 @@
 		  <artifactId>sendmail</artifactId>
 		  <version>0.0.1-SNAPSHOT</version>
 		</dependency>
+  		<dependency>
+  			<groupId>com.zhiwei.crawler</groupId>
+  			<artifactId>crawler-core</artifactId>
+  			<version>0.1.1-RELEASE</version>
+  		</dependency>
+  		<dependency>
+  			<groupId>com.zhiwei.tools</groupId>
+  			<artifactId>zhiwei-tools</artifactId>
+  			<version>0.1.1-SNAPSHOT</version>
+  		</dependency>
   </dependencies>
   
   <build>

src/main/java/com/zhiwei/searchhotcrawler/bean/WeiboHotSearch.java

@@ -9,7 +9,8 @@ package com.zhiwei.searchhotcrawler.bean;
 import java.io.Serializable;
 import java.util.Date;
 
-import com.zhiwei.zhiweiTools.timeParse.TimeParse;
+import com.zhiwei.tools.timeparse.TimeParse;
+
 
 public class WeiboHotSearch implements Serializable{
 

src/main/java/com/zhiwei/searchhotcrawler/cache/CacheListener.java

@@ -3,7 +3,8 @@ package com.zhiwei.searchhotcrawler.cache;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.zhiwei.zhiweiTools.tools.ZhiWeiTools;
+import com.zhiwei.tools.tools.ZhiWeiTools;
+
 
 public class CacheListener {
 	Logger logger = LoggerFactory.getLogger(CacheListener.class);

src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiboHotSearchCrawler.java

@@ -15,10 +15,11 @@ import org.slf4j.LoggerFactory;
 
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.crawler.core.HttpBoot;
+import com.zhiwei.crawler.core.RequestUtils;
 import com.zhiwei.searchhotcrawler.bean.WeiboHotSearch;
 import com.zhiwei.searchhotcrawler.mail.SendMailWeibo;
-import com.zhiwei.zhiweiTools.httpClient.HttpClientTemplateOK;
-import com.zhiwei.zhiweiTools.tools.URLCodeUtil;
+import com.zhiwei.tools.tools.URLCodeUtil;
 
 /**
  * @ClassName: WeiboHotSearch 
@@ -29,7 +30,7 @@ import com.zhiwei.zhiweiTools.tools.URLCodeUtil;
 public class WeiboHotSearchCrawler {
 	
 	private static Logger logger = LoggerFactory.getLogger(WeiboHotSearchCrawler.class);
-	
+	private static HttpBoot httpBoot = new HttpBoot();
 	/**
 	 * @Title: weiboHotSearchTest 
 	 * @author hero 
@@ -51,7 +52,7 @@ public class WeiboHotSearchCrawler {
 		for(int i =0; i<3; i++){
 			String htmlBody = null;
 			try {
-				htmlBody = HttpClientTemplateOK.get(url, null, headerMap);
+				htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(url, headerMap)).body().string();
 				if(htmlBody!=null && htmlBody.contains("pl_top_realtimehot")){
 					try {
 //						String script = htmlBody.split("<script>STK && STK.pageletM && STK.pageletM.view")[5].split("<\\/script>")[0];
@@ -115,7 +116,7 @@ public class WeiboHotSearchCrawler {
 		List<WeiboHotSearch> result = new ArrayList<WeiboHotSearch>();
 		String htmlBody;
 		try {
-			htmlBody = HttpClientTemplateOK.get(url, null, headerMap);
+			htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(url, headerMap)).body().string();
 			if(htmlBody!=null){
 				try {
 					JSONObject json = JSONObject.parseObject(htmlBody);

src/main/java/com/zhiwei/searchhotcrawler/crawler/ZhihuHotSearchCrawler.java

@@ -11,10 +11,11 @@ import org.slf4j.LoggerFactory;
 
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.crawler.core.HttpBoot;
+import com.zhiwei.crawler.core.RequestUtils;
 import com.zhiwei.searchhotcrawler.bean.ZhihuHotSearch;
-import com.zhiwei.zhiweiTools.httpClient.HeaderTool;
-import com.zhiwei.zhiweiTools.httpClient.HttpClientTemplateOK;
-import com.zhiwei.zhiweiTools.tools.URLCodeUtil;
+import com.zhiwei.tools.httpclient.HeaderTool;
+import com.zhiwei.tools.tools.URLCodeUtil;
 
 /**
  * @ClassName: ZhihuHotCrawler 
@@ -25,7 +26,7 @@ import com.zhiwei.zhiweiTools.tools.URLCodeUtil;
 public class ZhihuHotSearchCrawler {
 	
 	private static Logger logger = LoggerFactory.getLogger(ZhihuHotSearchCrawler.class);
-	
+	private static HttpBoot httpBoot = new HttpBoot();
 	/**
 	 * @Title: getZhihuHotList 
 	 * @author hero 
@@ -46,7 +47,7 @@ public class ZhihuHotSearchCrawler {
 		headerMap.put("Referer", rerferer);
 		for(int j=0;j<3;j++){
 			try {
-				String htmlBody = HttpClientTemplateOK.get(url, null, headerMap);
+				String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(url, headerMap)).body().string();
 				if(htmlBody != null){
 					if(htmlBody.contains("words")){
 						list = new ArrayList<ZhihuHotSearch>();
@@ -95,7 +96,7 @@ public class ZhihuHotSearchCrawler {
 		
 		for(int j=0;j<3;j++){
 			try {
-				String htmlBody = HttpClientTemplateOK.get(url, null, headerMap);
+				String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(url, headerMap)).body().string();
 				if(htmlBody != null){
 					if(htmlBody.contains("author")){
 						list = new ArrayList<ZhihuHotSearch>();

src/main/java/com/zhiwei/searchhotcrawler/dao/WeiboHotSearchDAO.java

@@ -4,15 +4,15 @@ package com.zhiwei.searchhotcrawler.dao;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
+
 import com.mongodb.BasicDBObject;
 import com.mongodb.DBCursor;
 import com.mongodb.DBObject;
-import com.mongodb.WriteConcern;
 import com.zhiwei.searchhotcrawler.bean.WeiboHotSearch;
 import com.zhiwei.searchhotcrawler.cache.CacheManager;
 import com.zhiwei.searchhotcrawler.config.Config;
 import com.zhiwei.searchhotcrawler.dbtemplate.MongoDBTemplate;
-import com.zhiwei.zhiweiTools.tools.ZhiWeiTools;
+import com.zhiwei.tools.tools.ZhiWeiTools;
 
 public class WeiboHotSearchDAO extends MongoDBTemplate{
 	

src/main/java/com/zhiwei/searchhotcrawler/dao/ZhihuHotSearchDAO.java

@@ -10,7 +10,7 @@ import com.mongodb.DBObject;
 import com.mongodb.WriteConcern;
 import com.zhiwei.searchhotcrawler.config.Config;
 import com.zhiwei.searchhotcrawler.dbtemplate.MongoDBTemplate;
-import com.zhiwei.zhiweiTools.tools.ZhiWeiTools;
+import com.zhiwei.tools.tools.ZhiWeiTools;
 
 public class ZhihuHotSearchDAO extends MongoDBTemplate{
 	public ZhihuHotSearchDAO() {

src/main/java/com/zhiwei/searchhotcrawler/run/HotSearchRun.java

@@ -10,7 +10,7 @@ import com.zhiwei.searchhotcrawler.timer.SendZhihuHotSearchRun;
 import com.zhiwei.searchhotcrawler.timer.UpdateWechatUserRun;
 import com.zhiwei.searchhotcrawler.timer.WeiboHotSearchRun;
 import com.zhiwei.searchhotcrawler.timer.ZhihuHotSearchRun;
-import com.zhiwei.zhiweiTools.tools.ZhiWeiTools;
+import com.zhiwei.tools.tools.ZhiWeiTools;
 
 
 public class HotSearchRun {

src/main/java/com/zhiwei/searchhotcrawler/timer/SendWeiboHotSearchRun.java

@@ -5,8 +5,10 @@ import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+
 import com.alibaba.fastjson.JSONObject;
 import com.mongodb.DBObject;
 import com.zhiwei.searchhotcrawler.dao.WechatUserDao;
@@ -14,8 +16,8 @@ import com.zhiwei.searchhotcrawler.dao.WeiboHotSearchDAO;
 import com.zhiwei.searchhotcrawler.util.Template;
 import com.zhiwei.searchhotcrawler.util.WechatCodeUtil;
 import com.zhiwei.searchhotcrawler.util.WechatConstant;
-import com.zhiwei.zhiweiTools.timeParse.TimeParse;
-import com.zhiwei.zhiweiTools.tools.ZhiWeiTools;
+import com.zhiwei.tools.timeparse.TimeParse;
+import com.zhiwei.tools.tools.ZhiWeiTools;
 
 public class SendWeiboHotSearchRun extends Thread {
 	private WeiboHotSearchDAO weiboHotSearchDAO = new WeiboHotSearchDAO();

src/main/java/com/zhiwei/searchhotcrawler/timer/SendZhihuHotSearchRun.java

@@ -16,8 +16,8 @@ import com.zhiwei.searchhotcrawler.dao.ZhihuHotSearchDAO;
 import com.zhiwei.searchhotcrawler.util.Template;
 import com.zhiwei.searchhotcrawler.util.WechatCodeUtil;
 import com.zhiwei.searchhotcrawler.util.WechatConstant;
-import com.zhiwei.zhiweiTools.timeParse.TimeParse;
-import com.zhiwei.zhiweiTools.tools.ZhiWeiTools;
+import com.zhiwei.tools.timeparse.TimeParse;
+import com.zhiwei.tools.tools.ZhiWeiTools;
 
 public class SendZhihuHotSearchRun extends Thread{
 	private ZhihuHotSearchDAO zhihuHotSearchDAO = new ZhihuHotSearchDAO();

src/main/java/com/zhiwei/searchhotcrawler/timer/UpdateWechatUserRun.java

@@ -4,11 +4,13 @@ import java.util.Calendar;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+
 import com.zhiwei.searchhotcrawler.dao.WechatUserDao;
 import com.zhiwei.searchhotcrawler.util.WechatCodeUtil;
-import com.zhiwei.zhiweiTools.tools.ZhiWeiTools;
+import com.zhiwei.tools.tools.ZhiWeiTools;
 
 public class UpdateWechatUserRun extends Thread{
 	private WechatUserDao wechatUserDao = new WechatUserDao();

src/main/java/com/zhiwei/searchhotcrawler/util/WechatCodeUtil.java

@@ -10,12 +10,13 @@ import org.slf4j.LoggerFactory;
 
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
-import com.zhiwei.zhiweiTools.httpClient.HeaderTool;
-import com.zhiwei.zhiweiTools.httpClient.HttpClientTemplateOK;
+import com.zhiwei.crawler.core.HttpBoot;
+import com.zhiwei.crawler.core.RequestUtils;
+import com.zhiwei.tools.httpclient.HeaderTool;
 
 public class WechatCodeUtil {
 	private static Logger logger = LoggerFactory.getLogger(WechatCodeUtil.class);
-
+	private static HttpBoot httpBoot = new HttpBoot();
 	/**
 	 * @Title: getToken
 	 * @author hero
@@ -28,10 +29,10 @@ public class WechatCodeUtil {
 		String token = "";
 		String appId = "wx2f555218d66e5948";
 		String jmAppId = AESUtils.encrypt("wechat", appId);
-		String path = "http://yuqing.zhiweidata.com/WechatPublic/common/getToken?appId=" + jmAppId;
+		String url = "http://yuqing.zhiweidata.com/WechatPublic/common/getToken?appId=" + jmAppId;
 		Map<String, String> headerMap = HeaderTool.getCommonHead();
 		try {
-			String result = HttpClientTemplateOK.get(path, null, headerMap);
+			String result = httpBoot.syncCall(RequestUtils.wrapGet(url, headerMap)).body().string();
 			if (result != null) {
 				JSONObject jsonObject = JSONObject.parseObject(result);
 				if(jsonObject.containsKey("data")) {
@@ -160,7 +161,7 @@ public class WechatCodeUtil {
 		Integer groupId = null;
 		Map<String, String> headerMap = HeaderTool.getCommonHead();
 		try {
-			String htmlBody = HttpClientTemplateOK.get(url, null, headerMap);
+			String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(url, headerMap)).body().string();
 			if (htmlBody != null) {
 				if(htmlBody.contains("tags")) {
 					JSONArray jsonArry = JSONObject.parseObject(htmlBody).getJSONArray("tags");
@@ -191,7 +192,7 @@ public class WechatCodeUtil {
 		Map<String,Integer> resultMap = new HashMap<String,Integer>();
 		Map<String, String> headerMap = HeaderTool.getCommonHead();
 		try {
-			String htmlBody = HttpClientTemplateOK.get(url, null, headerMap);
+			String htmlBody = httpBoot.syncCall(RequestUtils.wrapGet(url, headerMap)).body().string();
 			if (htmlBody != null) {
 				if(htmlBody.contains("tags")) {
 					JSONArray jsonArry = JSONObject.parseObject(htmlBody).getJSONArray("tags");

src/main/resources/db.properties

-#mongoIp=202.107.192.94
-mongoIp=192.168.0.101
+mongoIp=202.107.192.94
+#mongoIp=192.168.0.101
 mongoPort=30000
 db.username=zzwno
 db.paasword=zzwno1q2w3e4r