更新去重条件

pom.xml

@@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>com.zhiwei</groupId>
 	<artifactId>weibohotcrawler</artifactId>
-	<version>0.0.8-SNAPSHOT</version>
+	<version>0.0.9-SNAPSHOT</version>
 	<name>weibohotcrawler</name>
 	<description>微博热搜1小时榜单，社会、热点采集程序</description>
 
@@ -18,7 +18,7 @@
 		<dependency>
 			<groupId>com.zhiwei</groupId>
 			<artifactId>weibobusiness</artifactId>
-			<version>0.0.6-SNAPSHOT</version>
+			<version>0.0.7-SNAPSHOT</version>
 			<scope>provided</scope>
 		</dependency>
 		<dependency>

src/main/java/com/zhiwei/weibocrawler/rsidClient/RsidRedisClient.java

+package com.zhiwei.weibocrawler.rsidClient;
+
+
+import org.apache.log4j.Logger;
+
+import com.zhiwei.base.category.ClassD.TypeD;
+import com.zhiwei.common.config.GroupType;
+import com.zhiwei.middleware.cleaner.filter.UnifiedFilterClient;
+import com.zhiwei.weibocrawler.config.Config;
+
+public class RsidRedisClient
+{
+	
+	private static UnifiedFilterClient client;
+	static Logger log = Logger.getLogger(RsidRedisClient.class.getName());
+	static{
+		if(client==null){
+			synchronized (RsidRedisClient.class) {
+				if(client==null) {
+					try {
+						client = UnifiedFilterClient.getClient(Config.zookeeper,
+								Config.group, GroupType.PROVIDER);
+					} catch (Exception e) {
+						log.error("链接清洗中间件时出现错误,错误为:::{}", e);
+					}
+				}
+			}	
+		}
+	}
+	
+
+
+    public static boolean isNotDuplicated(String mid)
+    {
+        try
+        {
+        	return client.uniqueContains(mid, TypeD.新浪微博);
+        }catch (Exception e)
+        {
+        	try
+        	{
+        		Thread.sleep(3000L);
+        		return isNotDuplicatedTry(mid);
+        	}catch(Exception ex)
+        	{
+        		log.error(mid+" 重复URL过滤错误:"+ex.getMessage());
+        		return true;
+        	}
+        }
+    }
+    
+    private static boolean isNotDuplicatedTry(String mid) throws Exception
+    {
+    	return client.uniqueContains(mid, TypeD.新浪微博);
+    }
+    
+//    private static SendMail mail = new SendMail();
+    
+
+}