新版本字段采集适配

pom.xml

@@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>com.zhiwei</groupId>
 	<artifactId>weibohotcrawler</artifactId>
-	<version>0.0.3-SNAPSHOT</version>
+	<version>0.0.4-SNAPSHOT</version>
 	<name>weibohotcrawler</name>
 	<description>微博热搜1小时榜单，社会、热点采集程序</description>
 
@@ -22,15 +22,20 @@
 		</dependency>
 		<dependency>
 			<groupId>com.zhiwei.middleware</groupId>
-			<artifactId>cleaner-unified-urlfilter</artifactId>
-			<version>1.0.0.RELEASE</version>
+			<artifactId>cleaner-unified-urlfilterNew</artifactId>
+			<version>0.0.5-SNAPSHOT</version>
 		</dependency>
 		<dependency>
 			<groupId>com.zhiwei.tools</groupId>
 			<artifactId>zhiwei-tools</artifactId>
-			<version>0.0.5-SNAPSHOT</version>
+			<version>0.1.4-SNAPSHOT</version>
 		</dependency>
 
+		<dependency>
+			<groupId>com.zhiwei.crawler</groupId>
+			<artifactId>crawler-core</artifactId>
+			<version>0.5.2-SNAPSHOT</version>
+		</dependency>
 	</dependencies>
 	<!-- 打包管理 -->
 	<build>

src/main/java/com/zhiwei/weibocrawler/httpclient/HttpClientDemo.java

@@ -6,11 +6,15 @@ import java.util.Map;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.zhiwei.tools.httpclient.HttpClientTemplateOK;
+import com.zhiwei.crawler.core.HttpBoot;
+import com.zhiwei.crawler.utils.RequestUtils;
 
 public class HttpClientDemo {
     
     private static Logger logger = LoggerFactory.getLogger(HttpClientDemo.class);
+    private static HttpBoot httpBoot = new HttpBoot.Builder().retryTimes(3).build();
+	
+    
     
     public static String executeHttpRequestGet(String url) {
         String result = null;
@@ -23,11 +27,7 @@ public class HttpClientDemo {
         headerMap.put("Content-Type", "application/x-www-form-urlencoded");
         headerMap.put("Host", "weibo.com");
         try {
-//            Response response = HttpBoot.syncCall(RequestUtils.wrapGet(url, headerMap));
-//            result = response.body().string();
-              result = HttpClientTemplateOK.get(url, null, headerMap);
-
-        	return result;
+        	return httpBoot.syncCall(RequestUtils.wrapGet(url,headerMap)).body().string();
         } catch (Exception e) {
             logger.info("httpClient 获取数据出现问题：{}", e.getMessage());
             e.printStackTrace();

src/main/java/com/zhiwei/weibocrawler/rsidClient/RsidClientDAO.java

@@ -4,8 +4,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.zhiwei.common.config.GroupType;
-import com.zhiwei.middleware.cleaner.ptenum.PTENUM;
-import com.zhiwei.middleware.cleaner.urlfilter.UnifiedUrlFilterClient;
+import com.zhiwei.middleware.cleaner.urlfilter.UnifiedUrlFilterClientNew;
 import com.zhiwei.tools.tools.ZhiWeiTools;
 import com.zhiwei.weibocrawler.config.Config;
 
@@ -18,14 +17,14 @@ import com.zhiwei.weibocrawler.config.Config;
  */
 public class RsidClientDAO {
     
-	private static UnifiedUrlFilterClient client;
+	private static UnifiedUrlFilterClientNew client;
     private static Logger logger = LoggerFactory.getLogger(RsidClientDAO.class);
 	static{
 		if(client==null){
 			synchronized (RsidClientDAO.class) {
 				if(client==null) {
 					try {
-						client = UnifiedUrlFilterClient.getClient(Config.rsidUrl, Config.rsidGroup, GroupType.PROVIDER);
+						client = UnifiedUrlFilterClientNew.getClient(Config.rsidUrl, Config.rsidGroup, GroupType.PROVIDER);
 					} catch (Exception e) {
 						logger.error("链接清洗中间件时出现错误,错误为:::{}", e);
 					}
@@ -43,8 +42,7 @@ public class RsidClientDAO {
         //循环3次避免连接超时引起的验证失效
         for(int i=0; i<3; i++){
             try {
-            	boolean f = client.contains(mid, PTENUM.COMMON);
-                return f;
+            	return client.uniqueContains(mid);
             } catch (Exception e) {
                 logger.error("判断此条微博消息是否存在出现问题",e.fillInStackTrace());
                 ZhiWeiTools.sleep(200);