修改为新版去重

pom.xml

@@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>com.zhiwei</groupId>
 	<artifactId>weibohotcrawler</artifactId>
-	<version>0.0.7-SNAPSHOT</version>
+	<version>0.0.8-SNAPSHOT</version>
 	<name>weibohotcrawler</name>
 	<description>微博热搜1小时榜单，社会、热点采集程序</description>
 
@@ -35,11 +35,11 @@
 			<scope>provided</scope>
 		</dependency>
 		<dependency>
-			<groupId>com.zhiwei.middleware</groupId>
-			<artifactId>cleaner-unified-urlfilterNew</artifactId>
-			<version>0.0.6-SNAPSHOT</version>
-			<scope>provided</scope>
+		  <groupId>com.zhiwei.middleware</groupId>
+		  <artifactId>cleaner-unified-filter</artifactId>
+		  <version>1.0.1.RELEASE</version>
 		</dependency>
+		
 	</dependencies>
 	<!-- 打包管理 -->
 	<build>

src/main/java/com/zhiwei/weibocrawler/config/Config.java

@@ -12,18 +12,16 @@ public class Config {
 			conf = new Properties();
 			conf.load(is);
 			is.close();
-			rsidUrl = conf.getProperty("zookeeper");
-			rsidGroup = conf.getProperty("group");
-			redisWeiboKey = conf.getProperty("rsidType");
+			zookeeper = conf.getProperty("zookeeper");
+			group = conf.getProperty("group");
 			
 		} catch (Exception e) {
 			e.printStackTrace();
 		}
 	}
 	
-	public static String rsidUrl;
-	public static String rsidGroup;
-	public static String redisWeiboKey;
+	public static String zookeeper;
+	public static String group;
 	
 	
 	

src/main/java/com/zhiwei/weibocrawler/crawler/getdata/WeiboCrawlerAnalysis.java

@@ -18,7 +18,7 @@ import com.zhiwei.weibobusiness.weibo4j.model.StatusWapper;
 import com.zhiwei.weibobusiness.weibo4j.model.WeiboException;
 import com.zhiwei.weibocrawler.httpclient.HttpClientDemo;
 import com.zhiwei.weibocrawler.rsidClient.DataQueue;
-import com.zhiwei.weibocrawler.rsidClient.RsidClientDAO;
+import com.zhiwei.weibocrawler.rsidClient.RsidRedisClient;
 
 /**
  * 
@@ -117,7 +117,7 @@ public class WeiboCrawlerAnalysis {
 			
 			if(!midsList.isEmpty()){
 				for(String mid : midsList){
-					boolean f = RsidClientDAO.isWeiboExit(mid);
+					boolean f = RsidRedisClient.isNotDuplicated(mid);
 					if (!f) {
 						DataQueue.offer(mid);
 					}