修改为新版去重

pom.xml

@@ -38,6 +38,7 @@
 		  <groupId>com.zhiwei.middleware</groupId>
 		  <artifactId>cleaner-unified-filter</artifactId>
 		  <version>1.0.1.RELEASE</version>
+		  <scope>provided</scope>
 		</dependency>
 		
 	</dependencies>

src/main/java/com/zhiwei/weibocrawler/rsidClient/RsidClientDAO.java

-package com.zhiwei.weibocrawler.rsidClient;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.zhiwei.common.config.GroupType;
-import com.zhiwei.middleware.cleaner.urlfilter.UnifiedUrlFilterClientNew;
-import com.zhiwei.tools.tools.ZhiWeiTools;
-import com.zhiwei.weibocrawler.config.Config;
-
-/**
- * @ClassName RsidClientDAO
- * @Description 验证数据重复类
- * @author byte-zbs
- * @Date 2017年12月6日 下午5:21:15
- * @version 1.0.0
- */
-public class RsidClientDAO {
-    
-	private static UnifiedUrlFilterClientNew client;
-    private static Logger logger = LoggerFactory.getLogger(RsidClientDAO.class);
-	static{
-		if(client==null){
-			synchronized (RsidClientDAO.class) {
-				if(client==null) {
-					try {
-						client = UnifiedUrlFilterClientNew.getClient(Config.rsidUrl, Config.rsidGroup, GroupType.PROVIDER);
-					} catch (Exception e) {
-						logger.error("链接清洗中间件时出现错误,错误为:::{}", e);
-					}
-				}
-			}	
-		}
-	}
-
-    /**
-     * @Description 验证微博是否重复
-     * @param mid
-     * @return
-     */
-    public static boolean isWeiboExit(String mid){
-        //循环3次避免连接超时引起的验证失效
-        for(int i=0; i<3; i++){
-            try {
-            	return client.uniqueContains(mid);
-            } catch (Exception e) {
-                logger.error("判断此条微博消息是否存在出现问题",e.fillInStackTrace());
-                ZhiWeiTools.sleep(200);
-                continue;
-            }
-        }
-        return false;
-    }
-
-}