2018/10/12 新版采集消息流线上版1.0.0

pom.xml

@@ -200,6 +200,14 @@
 			<plugin>
 				<groupId>org.springframework.boot</groupId>
 				<artifactId>spring-boot-maven-plugin</artifactId>
+				<version>2.0.4.RELEASE</version>
+				<executions>
+					<execution>
+						<goals>
+							<goal>repackage</goal>
+						</goals>
+					</execution>
+				</executions>
 			</plugin>
 		</plugins>
 	</build>

src/main/java/com/zhiwei/messageflow/es/service/impl/NoiseProcessingServiceImpl.java

@@ -10,12 +10,15 @@ import java.util.Locale;
 import java.util.Map;
 import java.util.TimeZone;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
 import com.alibaba.fastjson.JSONObject;
+import com.zhiwei.messageflow.DirectES4RedisThread;
 import com.zhiwei.messageflow.bean.MediaMessage;
 import com.zhiwei.messageflow.bean.VideoMessage;
 import com.zhiwei.messageflow.bean.WeiboMessage;
@@ -29,6 +32,7 @@ import com.zhiwei.messageflow.util.ESQueryUtil;
 
 @Component
 public class NoiseProcessingServiceImpl implements NoiseProcessingService {
+	private final static Logger log = LogManager.getLogger(NoiseProcessingServiceImpl.class);
 
 	@Autowired
 	private HighLightFillingService highLightFillingService;
@@ -637,7 +641,7 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
 
 	private boolean isTimeOver(String time) {
 		long current=System.currentTimeMillis();//当前时间毫秒数
-        long zero=current/(1000*3600*24)*(1000*3600*24)-TimeZone.getDefault().getRawOffset();//今天零点零分零秒的毫秒数
+        long zero=current-(24*3600*1000L);//前一天毫秒数
 		Date date = new Date();
 		SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
 		// SimpleDateFormat dateFormat = new SimpleDateFormat("EEE MMM d HH:mm:ss 'CST'
@@ -647,7 +651,7 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
 		try {
 			date = dateFormat.parse(time);
 			if ((date.getTime() + (8 * 3600 * 1000L)) < zero) {
-				System.err.println(date.getTime() + (8 * 3600 * 1000L)+"|"+zero);
+				log.info(date.getTime() + (8 * 3600 * 1000L)+"|"+zero);
 				return true;
 			}
 		} catch (ParseException e) {
@@ -677,15 +681,15 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
 
 		String time = map.get("time") != null ? map.get("time").toString() : null;
 		if (isTimeOver(time)) {
-			System.err.println("weibo:time:" + time + map.get("id").toString());
-			return isnoise;
+			log.info("weibo:time:" + time +"\t"+ map.get("id").toString());
+			return true;
 		}
 		String text = map.get("text") != null ? map.get("text").toString() : null;
 		String roottext = map.get("roottext") != null ? map.get("roottext").toString() : null;
 		String username = map.get("username") != null ? map.get("username").toString() : null;
 
 		if (null == username) {
-			return isnoise;
+			return true;
 		}
 		if (noiseRules != null) {
 			for (NoiseRule n : noiseRules) {
@@ -782,8 +786,8 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
 		String platform = platformNew.getPlatformName();
 		String time = map.get("created_at") != null ? map.get("created_at").toString() : null;
 		if (isTimeOver(time)) {
-			System.err.println("zhihu:time:" + time + map.get("id").toString());
-			return isnoise;
+			log.info("zhihu:time:" + time +"\t"+ map.get("id").toString());
+			return true;
 		}
 		String text = map.get("question_title") != null ? map.get("question_title").toString() : null;
 		String roottext = map.get("answer_content") != null ? map.get("answer_content").toString() : null;
@@ -913,8 +917,8 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
 		String platform = platformNew.getPlatformName();
 		String time = map.get("time") != null ? map.get("time").toString() : null;
 		if (isTimeOver(time)) {
-			System.err.println("video:time:" + time + map.get("id").toString());
-			return isnoise;
+			log.info("video:time:" + time +"\t"+ map.get("id").toString());
+			return true;
 		}
 		String text = map.get("title") != null ? map.get("title").toString() : null;
 		String roottext = null;
@@ -1043,15 +1047,15 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
 		String platform = platformNew.getPlatformName();
 		String time = map.get("time") != null ? map.get("time").toString() : null;
 		if (isTimeOver(time)) {
-			System.err.println("media:time:" + time + map.get("id").toString());
-			return isnoise;
+			log.info("media:time:" + time +"\t"+ map.get("id").toString());
+			return true;
 		}
 		String text = map.get("title") != null ? map.get("title").toString() : null;
 		String roottext = map.get("content") != null ? map.get("content").toString() : null;
 		String username = map.get("source") != null ? map.get("source").toString() : null;
 
 		if (null == username) {
-			return isnoise;
+			return true;
 		}
 		if (noiseRules != null) {
 			for (NoiseRule n : noiseRules) {

src/main/java/com/zhiwei/messageflow/redis/RedisPoolAndTools.java

@@ -136,7 +136,7 @@ public class RedisPoolAndTools {
 	}
 
 	/**
-	 * 删除超出上限的数据
+	 * 删除超出上限的数据(弃用
 	 * 
 	 * @param key
 	 * @param removeIndex
@@ -153,9 +153,26 @@ public class RedisPoolAndTools {
 		jedis.zremrangeByRank(key, 0, removeIndex);
 		returnResource(jedis);
 	}
-
 	/**
-	 * 获取有序集合消息数量
+	 * 删除超出上限的数据
+	 * 
+	 * @param key
+	 * @param removeIndex
+	 */
+	public void removeDataByName(String key, int removeIndex) {
+		Jedis jedis = getJedis();
+		while (true) {
+			if (null != jedis) {
+				break;
+			} else {
+				jedis = getJedis();
+			}
+		}
+		jedis.zremrangeByRank(key, 0, removeIndex);
+		returnResource(jedis);
+	}
+	/**
+	 * 获取有序集合消息数量(弃用
 	 * 
 	 * @param key
 	 * @return
@@ -173,6 +190,25 @@ public class RedisPoolAndTools {
 		returnResource(jedis);
 		return nowCount;
 	}
+	/**
+	 * 获取有序集合消息数量
+	 * 
+	 * @param key
+	 * @return
+	 */
+	public Long getNowCount(String key) {
+		Jedis jedis = getJedis();
+		while (true) {
+			if (null != jedis) {
+				break;
+			} else {
+				jedis = getJedis();
+			}
+		}
+		Long nowCount = jedis.zcard(key);
+		returnResource(jedis);
+		return nowCount;
+	}
 
 	/**
 	 * 分页获取redis数据

src/main/java/com/zhiwei/messageflow/redis/service/impl/RedisServiceImpl.java

@@ -61,10 +61,10 @@ public class RedisServiceImpl implements RedisService {
 		/**
 		 * 删除超出存储上限的数据
 		 */
-		long nowCount = redisPoolAndTools.getNowCount(redisKey.getBytes());// 当前数据量
+		long nowCount = redisPoolAndTools.getNowCount(redisKey);// 当前数据量
 		int removeIndex = (int) (nowCount - maxSize);// 需移除数据数量
 		if (maxSize > 0 && nowCount > maxSize) {
-			redisPoolAndTools.removeDataByName(redisKey.getBytes(), removeIndex);
+			redisPoolAndTools.removeDataByName(redisKey, removeIndex);
 		}
 
 	}
@@ -84,10 +84,10 @@ public class RedisServiceImpl implements RedisService {
 		/**
 		 * 删除超出存储上限的数据
 		 */
-		long nowCount = redisPoolAndTools.getNowCount(redisKey.getBytes());// 当前数据量
+		long nowCount = redisPoolAndTools.getNowCount(redisKey);// 当前数据量
 		int removeIndex = (int) (nowCount - maxSize);// 需移除数据数量
 		if (maxSize > 0 && nowCount > maxSize) {
-			redisPoolAndTools.removeDataByName(redisKey.getBytes(), removeIndex);
+			redisPoolAndTools.removeDataByName(redisKey, removeIndex);
 		}
 	}
 
@@ -107,10 +107,10 @@ public class RedisServiceImpl implements RedisService {
 		/**
 		 * 删除超出存储上限的数据
 		 */
-		long nowCount = redisPoolAndTools.getNowCount(redisKey.getBytes());// 当前数据量
+		long nowCount = redisPoolAndTools.getNowCount(redisKey);// 当前数据量
 		int removeIndex = (int) (nowCount - maxSize);// 需移除数据数量
 		if (maxSize > 0 && nowCount > maxSize) {
-			redisPoolAndTools.removeDataByName(redisKey.getBytes(), removeIndex);
+			redisPoolAndTools.removeDataByName(redisKey, removeIndex);
 		}
 
 	}
@@ -130,10 +130,10 @@ public class RedisServiceImpl implements RedisService {
 		/**
 		 * 删除超出存储上限的数据
 		 */
-		long nowCount = redisPoolAndTools.getNowCount(redisKey.getBytes());// 当前数据量
+		long nowCount = redisPoolAndTools.getNowCount(redisKey);// 当前数据量
 		int removeIndex = (int) (nowCount - maxSize);// 需移除数据数量
 		if (maxSize > 0 && nowCount > maxSize) {
-			redisPoolAndTools.removeDataByName(redisKey.getBytes(), removeIndex);
+			redisPoolAndTools.removeDataByName(redisKey, removeIndex);
 		}
 
 	}
@@ -148,10 +148,10 @@ public class RedisServiceImpl implements RedisService {
 		/**
 		 * 删除超出存储上限的数据
 		 */
-		long nowCount = redisPoolAndTools.getNowCount(redisKey.getBytes());// 当前数据量
+		long nowCount = redisPoolAndTools.getNowCount(redisKey);// 当前数据量
 		int removeIndex = (int) (nowCount - maxSize);// 需移除数据数量
 		if (maxSize > 0 && nowCount > maxSize) {
-			redisPoolAndTools.removeDataByName(redisKey.getBytes(), removeIndex);
+			redisPoolAndTools.removeDataByName(redisKey, removeIndex);
 		}
 		
 	}