2018/12/27 定向监测过滤历史数据规则添加

pom.xml

@@ -161,7 +161,7 @@
 		<dependency>
 			<groupId>org.projectlombok</groupId>
 			<artifactId>lombok</artifactId>
-			<version>1.18.0</version>
+			<version>1.18.4</version>
 		</dependency>
 
 		<!-- POI -->

src/main/java/com/zhiwei/messageflow/es/service/impl/NoiseProcessingServiceImpl.java

@@ -634,8 +634,8 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
 	}
 
 	private boolean isTimeOver(String time) {
-		long current=System.currentTimeMillis();//当前时间毫秒数
-        long zero=current-(24*3600*1000L);//前一天毫秒数
+		long current = System.currentTimeMillis();// 当前时间毫秒数
+		long zero = current - (24 * 3600 * 1000L);// 前一天毫秒数
 		Date date = new Date();
 		SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
 		// SimpleDateFormat dateFormat = new SimpleDateFormat("EEE MMM d HH:mm:ss 'CST'
@@ -1179,8 +1179,48 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
 			if (map == null) {
 				continue;
 			}
-			// source为空判断
+			// 噪音判断参数
+			boolean isnoise = false;
+			
 			String type = searchHit.getType();
+			Long rstime = null != searchHit.getSource().get("rstime")
+					? Long.valueOf(searchHit.getSource().get("rstime") + "")
+					: null;
+			if (type.equals("status") && null != rstime) {
+				// 微博
+				String time = map.get("time") != null ? map.get("time").toString() : null;
+				if (isTimeOver(time)) {
+//					log.info("weibo:time:" + time +"\t"+ map.get("id").toString());
+					isnoise = true;
+				}
+			} else if (type.equals("zhihu")) {
+				// zhihu
+				String time = map.get("created_at") != null ? map.get("created_at").toString() : null;
+				if (isTimeOver(time)) {
+//					log.info("zhihu:time:" + time +"\t"+ map.get("id").toString());
+					isnoise = true;
+				}
+			} else if (type.equals("video")) {
+				// 视频
+				String time = map.get("time") != null ? map.get("time").toString() : null;
+				if (isTimeOver(time)) {
+//					log.info("video:time:" + time +"\t"+ map.get("id").toString());
+					isnoise = true;
+				}
+			} else {
+				// 网媒
+				String time = map.get("time") != null ? map.get("time").toString() : null;
+				if (isTimeOver(time)) {
+//					log.info("media:time:" + time +"\t"+ map.get("id").toString());
+					isnoise = true;
+				}
+			}
+
+			if (isnoise) {
+				continue;
+			}
+			
+			// source为空判断
 			if (!type.equals("status")) {
 				// 除微博
 				if (map.get("source") == null) {

src/main/resources/application.properties.local2

-##服务端uri
+##\uFFFD\u009C\u008D\uFFFD\u008A\uFFFD\u7AEFuri
 #spring.data.mongodb.uri=115.236.59.91:27017
-#服务端数据库
+
+#\u5185\u7F6Etomcat\u7AEF\u53E3\u53F7
+server.port=10001
+
+#\uFFFD\u009C\u008D\uFFFD\u008A\uFFFD\u7AEF\uFFFD\u0095\uFFFD\uFFFD\u008D\uFFFD\uFFFD\u0093
 spring.data.mongodb.primary.database=qbjcPhoenix
-#服务ip
+#\uFFFD\u009C\u008D\uFFFD\u008A\uFFFDip
 spring.data.mongodb.primary.host=202.107.192.94
-#服务port
-spring.data.mongodb.primary.port=30000
+#\uFFFD\u009C\u008D\uFFFD\u008A\uFFFDport
+spring.data.mongodb.primary.port=30001
 
 spring.data.mongodb.primary.username=stno
 
@@ -13,11 +17,11 @@ spring.data.mongodb.primary.password=stno1q2w3e4r
 
 spring.data.mongodb.primary.authenticationDatabase=admin
 
-#服务端数据库
+#\uFFFD\u009C\u008D\uFFFD\u008A\uFFFD\u7AEF\uFFFD\u0095\uFFFD\uFFFD\u008D\uFFFD\uFFFD\u0093
 spring.data.mongodb.secondary.database=eventMuseum
-#服务ip
+#\uFFFD\u009C\u008D\uFFFD\u008A\uFFFDip
 spring.data.mongodb.secondary.host=202.107.192.94
-#服务port
+#\uFFFD\u009C\u008D\uFFFD\u008A\uFFFDport
 spring.data.mongodb.secondary.port=30000
 
 spring.data.mongodb.secondary.username=stno
@@ -26,11 +30,11 @@ spring.data.mongodb.secondary.password=stno1q2w3e4r
 
 spring.data.mongodb.secondary.authenticationDatabase=admin
 
-#服务端数据库
+#\uFFFD\u009C\u008D\uFFFD\u008A\uFFFD\u7AEF\uFFFD\u0095\uFFFD\uFFFD\u008D\uFFFD\uFFFD\u0093
 spring.data.mongodb.thirdary.database=WechatPublic
-#服务ip
+#\uFFFD\u009C\u008D\uFFFD\u008A\uFFFDip
 spring.data.mongodb.thirdary.host=202.107.192.94
-#服务port
+#\uFFFD\u009C\u008D\uFFFD\u008A\uFFFDport
 spring.data.mongodb.thirdary.port=30000
 
 spring.data.mongodb.thirdary.username=stno
@@ -40,9 +44,6 @@ spring.data.mongodb.thirdary.password=stno1q2w3e4r
 spring.data.mongodb.thirdary.authenticationDatabase=admin
 
 #kafka
-spring.kafka.bootstrap-servers = kafka1.irybd.com:9092
-spring.kafka.producer.key-serializer = org.apache.kafka.common.serialization.StringSerializer
-spring.kafka.producer.value-serializer = org.apache.kafka.common.serialization.StringSerializer
 #=============== consumer  =======================
 # group id
 #spring.kafka.consumer.group-id=group1
@@ -55,11 +56,12 @@ spring.kafka.producer.value-serializer = org.apache.kafka.common.serialization.S
 #spring.kafka.consumer.key-deserializer=org.apache.kafka.common.serialization.StringDeserializer
 #spring.kafka.consumer.value-deserializer=org.apache.kafka.common.serialization.StringDeserializer
 
+#kafka1.irybd.com:9092,kafka1.irybd.com:9093,kafka1.irybd.com:9094
 kafka.consumer.servers=kafka1.irybd.com:9092
 kafka.consumer.enable.auto.commit=false
 kafka.consumer.session.timeout=15000
 kafka.consumer.auto.commit.interval=100
-kafka.consumer.auto.offset.reset=latest
-kafka.consumer.group.id=group
+kafka.consumer.auto.offset.reset=earliest
+kafka.consumer.group.id=group-test
 kafka.consumer.concurrency=1
 kafka.consumer.maxPollRecordsConfig=100
\ No newline at end of file

src/main/resources/middleware.properties

-#middleware.zookeeperIp=zookeeper://192.168.0.234:2181
+#middleware.zookeeperIp=zookeeper://192.168.0.36:2181
 middleware.zookeeperIp=zookeeper://192.168.0.203:2181;zookeeper://192.168.0.104:2181;zookeeper://192.168.0.105:2181;
\ No newline at end of file

src/main/resources/redis.properties

@@ -4,9 +4,10 @@ redis.maxWaitMillis=1000
 redis.testOnBorrow=true
 redis.testOnReturn=true
 redis.ip = 192.168.0.202
+redis.port=6380
 #redis.ip = 202.107.192.94
 #redis.ip=127.0.0.1
-redis.port=6380
+#redis.port=6379
 #redis.ip=192.168.1.74
 #redis.port=6388
 #redis.password=fjouero&^%^%^$*()*)))*^$$KDFJDKJF9ruorudlfdljfldjf