2018/5/10 1.commonid增加，消息流调整2.关键词规则和噪音规则，调整适应

src/main/java/com/zhiwei/messageflow/bean/MediaMessage.java

@@ -33,4 +33,5 @@ public class MediaMessage {
 	private String rootSourcce;
 	private Double channelIndex;
 	private String crawlerV;
+	private Long commonid;
 }
\ No newline at end of file

src/main/java/com/zhiwei/messageflow/bean/VideoMessage.java

@@ -21,4 +21,5 @@ public class VideoMessage {
 	private String time;
 	private String uper;
 	private String type;
+	private Long commonid;
 }
\ No newline at end of file

src/main/java/com/zhiwei/messageflow/bean/WeiboMessage.java

@@ -29,6 +29,7 @@ public class WeiboMessage {
 	private Long rstime;
 	private String username;
 	private Integer guanzhu;
+	private Long commonid;
 
 	private String rooturl;
 	private String roottext;

src/main/java/com/zhiwei/messageflow/bean/ZhihuMessage.java

@@ -35,4 +35,5 @@ public class ZhihuMessage {
 	private String img;
 	private String follow;
 	private String browse;
+	private Long commonid;
 }
\ No newline at end of file

src/main/java/com/zhiwei/messageflow/es/service/impl/NoiseProcessingServiceImpl.java

@@ -16,6 +16,7 @@ import com.zhiwei.messageflow.bean.ZhihuMessage;
 import com.zhiwei.messageflow.es.service.HighLightFillingService;
 import com.zhiwei.messageflow.es.service.NoiseProcessingService;
 import com.zhiwei.messageflow.mongo.bean.NoiseRule;
+import com.zhiwei.messageflow.mongo.dao.NoiseRuleDao;
 
 @Component
 public class NoiseProcessingServiceImpl implements NoiseProcessingService {
@@ -23,6 +24,9 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
 	@Autowired
 	private HighLightFillingService highLightFillingService;
 	
+	@Autowired
+	private NoiseRuleDao noiseRuleDao;
+
 	@Override
 	public List<WeiboMessage> weiboDenoising(List<NoiseRule> noiseRules, SearchHits searchHits, String platform,
 			String project) {
@@ -49,6 +53,9 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
 						/**
 						 * 关键词去噪
 						 */
+//						if (n.getKwSearch().equals("标题")) {
+//
+//						} else if (n.getKwSearch().equals("全文")) {
 							String kone = n.getKeyWordsInputOne();
 
 							String ktwo = n.getKeyWordsInputTwo();
@@ -58,10 +65,12 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
 										&& (text.indexOf(kone) != -1 || roottext.indexOf(kone) != -1)
 										&& (text.indexOf(ktwo) != -1 || roottext.indexOf(ktwo) != -1)) {
 									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
 									break;
 								} else if (text != null && roottext == null && (text.indexOf(kone) != -1)
 										&& (text.indexOf(ktwo) != -1)) {
 									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
 									break;
 								} else if (text == null && roottext == null) {
 									isnoise = true;
@@ -73,10 +82,12 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
 												|| (text.indexOf(ktwo) != -1 || roottext.indexOf(ktwo) != -1))) {
 
 									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
 									break;
 								} else if (text != null && roottext == null
 										&& ((text.indexOf(kone) != -1) || (text.indexOf(ktwo) != -1))) {
 									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
 									break;
 								} else if (text == null && roottext == null) {
 									isnoise = true;
@@ -86,15 +97,18 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
 								if (text != null && roottext != null
 										&& (text.indexOf(kone) != -1 || roottext.indexOf(kone) != -1)) {
 									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
 									break;
 								} else if (text != null && roottext == null && (text.indexOf(kone) != -1)) {
 									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
 									break;
 								} else if (text == null && roottext == null) {
 									isnoise = true;
 									break;
 								}
 							}
+//						}
 					} else if ((n.getRuleType().equals("channel"))
 							&& ((n.getChannelPt().equals("微博") || n.getChannelPt().equals("全部")))) {
 						/**
@@ -106,6 +120,7 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
 						source = source.replaceAll(" ", "");
 						if (channelQd.equals(source)) {
 							isnoise = true;
+							noiseRuleDao.addEffectNum(n);
 							break;
 						}
 					}
@@ -154,22 +169,57 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
 						 * 关键词去噪
 						 */
 						// searchHit.getSource().get("text").toString().indexOf(str);
+						if (n.getKwSearch().equals("标题")) {
+							String kone = n.getKeyWordsInputOne();
+
+							String ktwo = n.getKeyWordsInputTwo();
+
+							if (n.getAndOr().equals("且")) {
+								if (text != null && (text.indexOf(kone) != -1) && (text.indexOf(ktwo) != -1)) {
+									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
+									break;
+								} else if (text == null && roottext == null) {
+									isnoise = true;
+									break;
+								}
+							} else if (n.getAndOr().equals("或")) {
+								if (text != null && ((text.indexOf(kone) != -1) || (text.indexOf(ktwo) != -1))) {
+									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
+									break;
+								} else if (text == null && roottext == null) {
+									isnoise = true;
+									break;
+								}
+							} else {
+								if (text != null && (text.indexOf(kone) != -1)) {
+									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
+									break;
+								} else if (text == null && roottext == null) {
+									isnoise = true;
+									break;
+								}
+							}
+						} else if (n.getKwSearch().equals("全文")) {
 							String kone = n.getKeyWordsInputOne();
+
 							String ktwo = n.getKeyWordsInputTwo();
+
 							if (n.getAndOr().equals("且")) {
 								if (text != null && roottext != null
 										&& (text.indexOf(kone) != -1 || roottext.indexOf(kone) != -1)
 										&& (text.indexOf(ktwo) != -1 || roottext.indexOf(ktwo) != -1)) {
-								// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
 									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
 									break;
 								} else if (text != null && roottext == null && (text.indexOf(kone) != -1)
 										&& (text.indexOf(ktwo) != -1)) {
-								// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
 									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
 									break;
 								} else if (text == null && roottext == null) {
-								// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
 									isnoise = true;
 									break;
 								}
@@ -177,35 +227,35 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
 								if (text != null && roottext != null
 										&& ((text.indexOf(kone) != -1 || roottext.indexOf(kone) != -1)
 												|| (text.indexOf(ktwo) != -1 || roottext.indexOf(ktwo) != -1))) {
-								// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
+
 									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
 									break;
 								} else if (text != null && roottext == null
 										&& ((text.indexOf(kone) != -1) || (text.indexOf(ktwo) != -1))) {
-								// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
 									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
 									break;
 								} else if (text == null && roottext == null) {
-								// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
 									isnoise = true;
 									break;
 								}
 							} else {
 								if (text != null && roottext != null
 										&& (text.indexOf(kone) != -1 || roottext.indexOf(kone) != -1)) {
-								// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
 									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
 									break;
 								} else if (text != null && roottext == null && (text.indexOf(kone) != -1)) {
-								// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
 									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
 									break;
 								} else if (text == null && roottext == null) {
-								// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
 									isnoise = true;
 									break;
 								}
 							}
+						}
 					} else if ((n.getRuleType().equals("channel"))
 							&& ((n.getChannelPt().equals("知乎") || n.getChannelPt().equals("全部")))) {
 						/**
@@ -217,6 +267,7 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
 						source = source.replaceAll(" ", "");
 						if (channelQd.equals(source)) {
 							isnoise = true;
+							noiseRuleDao.addEffectNum(n);
 							break;
 						}
 					}
@@ -263,17 +314,57 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
 						/**
 						 * 关键词去噪
 						 */
-						// searchHit.getSource().get("text").toString().indexOf(str);
+						if (n.getKwSearch().equals("标题")) {
+							String kone = n.getKeyWordsInputOne();
+
+							String ktwo = n.getKeyWordsInputTwo();
+
+							if (n.getAndOr().equals("且")) {
+								if (text != null && (text.indexOf(kone) != -1) && (text.indexOf(ktwo) != -1)) {
+									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
+									break;
+								} else if (text == null && roottext == null) {
+									isnoise = true;
+									break;
+								}
+							} else if (n.getAndOr().equals("或")) {
+								if (text != null && ((text.indexOf(kone) != -1) || (text.indexOf(ktwo) != -1))) {
+									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
+									break;
+								} else if (text == null && roottext == null) {
+									isnoise = true;
+									break;
+								}
+							} else {
+								if (text != null && (text.indexOf(kone) != -1)) {
+									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
+									break;
+								} else if (text == null && roottext == null) {
+									isnoise = true;
+									break;
+								}
+							}
+						} else if (n.getKwSearch().equals("全文")) {
 							String kone = n.getKeyWordsInputOne();
+
 							String ktwo = n.getKeyWordsInputTwo();
+
 							if (n.getAndOr().equals("且")) {
-							if (text != null && roottext == null && (text.indexOf(kone) != -1)
+								if (text != null && roottext != null
+										&& (text.indexOf(kone) != -1 || roottext.indexOf(kone) != -1)
+										&& (text.indexOf(ktwo) != -1 || roottext.indexOf(ktwo) != -1)) {
+									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
+									break;
+								} else if (text != null && roottext == null && (text.indexOf(kone) != -1)
 										&& (text.indexOf(ktwo) != -1)) {
-								// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
 									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
 									break;
 								} else if (text == null && roottext == null) {
-								// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
 									isnoise = true;
 									break;
 								}
@@ -281,35 +372,35 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
 								if (text != null && roottext != null
 										&& ((text.indexOf(kone) != -1 || roottext.indexOf(kone) != -1)
 												|| (text.indexOf(ktwo) != -1 || roottext.indexOf(ktwo) != -1))) {
-								// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
+
 									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
 									break;
 								} else if (text != null && roottext == null
 										&& ((text.indexOf(kone) != -1) || (text.indexOf(ktwo) != -1))) {
-								// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
 									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
 									break;
 								} else if (text == null && roottext == null) {
-								// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
 									isnoise = true;
 									break;
 								}
 							} else {
 								if (text != null && roottext != null
 										&& (text.indexOf(kone) != -1 || roottext.indexOf(kone) != -1)) {
-								// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
 									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
 									break;
 								} else if (text != null && roottext == null && (text.indexOf(kone) != -1)) {
-								// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
 									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
 									break;
 								} else if (text == null && roottext == null) {
-								// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
 									isnoise = true;
 									break;
 								}
 							}
+						}
 					} else if ((n.getRuleType().equals("channel"))
 							&& ((n.getChannelPt().equals("视频") || n.getChannelPt().equals("全部")))) {
 						/**
@@ -321,6 +412,7 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
 						source = source.replaceAll(" ", "");
 						if (channelQd.equals(source)) {
 							isnoise = true;
+							noiseRuleDao.addEffectNum(n);
 							break;
 						}
 					}
@@ -367,23 +459,57 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
 						/**
 						 * 关键词去噪
 						 */
-						// searchHit.getSource().get("text").toString().indexOf(str);
+						if (n.getKwSearch().equals("标题")) {
+							String kone = n.getKeyWordsInputOne();
+
+							String ktwo = n.getKeyWordsInputTwo();
+
+							if (n.getAndOr().equals("且")) {
+								if (text != null && (text.indexOf(kone) != -1) && (text.indexOf(ktwo) != -1)) {
+									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
+									break;
+								} else if (text == null && roottext == null) {
+									isnoise = true;
+									break;
+								}
+							} else if (n.getAndOr().equals("或")) {
+								if (text != null && ((text.indexOf(kone) != -1) || (text.indexOf(ktwo) != -1))) {
+									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
+									break;
+								} else if (text == null && roottext == null) {
+									isnoise = true;
+									break;
+								}
+							} else {
+								if (text != null && (text.indexOf(kone) != -1)) {
+									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
+									break;
+								} else if (text == null && roottext == null) {
+									isnoise = true;
+									break;
+								}
+							}
+						} else if (n.getKwSearch().equals("全文")) {
 							String kone = n.getKeyWordsInputOne();
+
 							String ktwo = n.getKeyWordsInputTwo();
+
 							if (n.getAndOr().equals("且")) {
 								if (text != null && roottext != null
 										&& (text.indexOf(kone) != -1 || roottext.indexOf(kone) != -1)
 										&& (text.indexOf(ktwo) != -1 || roottext.indexOf(ktwo) != -1)) {
-								// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
 									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
 									break;
 								} else if (text != null && roottext == null && (text.indexOf(kone) != -1)
 										&& (text.indexOf(ktwo) != -1)) {
-								// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
 									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
 									break;
 								} else if (text == null && roottext == null) {
-								// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
 									isnoise = true;
 									break;
 								}
@@ -391,35 +517,35 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
 								if (text != null && roottext != null
 										&& ((text.indexOf(kone) != -1 || roottext.indexOf(kone) != -1)
 												|| (text.indexOf(ktwo) != -1 || roottext.indexOf(ktwo) != -1))) {
-								// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
+
 									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
 									break;
 								} else if (text != null && roottext == null
 										&& ((text.indexOf(kone) != -1) || (text.indexOf(ktwo) != -1))) {
-								// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
 									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
 									break;
 								} else if (text == null && roottext == null) {
-								// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
 									isnoise = true;
 									break;
 								}
 							} else {
 								if (text != null && roottext != null
 										&& (text.indexOf(kone) != -1 || roottext.indexOf(kone) != -1)) {
-								// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
 									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
 									break;
 								} else if (text != null && roottext == null && (text.indexOf(kone) != -1)) {
-								// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
 									isnoise = true;
+									noiseRuleDao.addEffectNum(n);
 									break;
 								} else if (text == null && roottext == null) {
-								// System.out.println(searchHit.getSourceAsString() + kone + " " + ktwo);
 									isnoise = true;
 									break;
 								}
 							}
+						}
 					} else if ((n.getRuleType().equals("channel"))
 							&& ((n.getChannelPt() == platform) || n.getChannelPt().equals("全部"))) {
 						/**
@@ -431,6 +557,7 @@ public class NoiseProcessingServiceImpl implements NoiseProcessingService {
 						source = source.replaceAll(" ", "");
 						if (channelQd.equals(source)) {
 							isnoise = true;
+							noiseRuleDao.addEffectNum(n);
 							break;
 						}
 					}

src/main/java/com/zhiwei/messageflow/mongo/bean/KeywordNew.java

@@ -20,4 +20,5 @@ public class KeywordNew {
 	private String project;// 项目
 	private String submitter;
 	private long createAt;
+	private Boolean isUsed;
 }

src/main/java/com/zhiwei/messageflow/mongo/bean/NoiseRule.java

@@ -24,4 +24,7 @@ public class NoiseRule {
 	private String submitter;
 	private String channelPt;
 	private String channelQd;
+	private String kwSearch;
+	private Boolean isUsed;
+	private Long effectNum;
 }

src/main/java/com/zhiwei/messageflow/mongo/dao/NoiseRuleDao.java

@@ -12,4 +12,12 @@ public interface NoiseRuleDao {
 	 * @return
 	 */
 	List<NoiseRule> getNoiseRuleByProject(String projectName);
+	/**
+	 * 噪音规则加
+	 * @Title: addEffectNum 
+	 * @Description: TODO(这里用一句话描述这个方法的作用) 
+	 * @param @param n 设定文件 
+	 * @return void 返回类型
+	 */
+	void addEffectNum(NoiseRule n);
 }

src/main/java/com/zhiwei/messageflow/mongo/dao/impl/KeywordNewDaoImpl.java

@@ -23,7 +23,7 @@ public class KeywordNewDaoImpl implements KeywordNewDao {
 	public List<KeywordNew> getKeywordNewByProject(String projectName) {
 		Query query = new Query();
 		query.addCriteria(Criteria.where("project").is(projectName));
-
+		query.addCriteria(Criteria.where("isUsed").is(true));
 		return primaryMongoTemplate.find(query, KeywordNew.class);
 	}
 

src/main/java/com/zhiwei/messageflow/mongo/dao/impl/NoiseRuleDaoImpl.java

@@ -7,6 +7,7 @@ import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.data.mongodb.core.MongoTemplate;
 import org.springframework.data.mongodb.core.query.Criteria;
 import org.springframework.data.mongodb.core.query.Query;
+import org.springframework.data.mongodb.core.query.Update;
 import org.springframework.stereotype.Component;
 
 import com.zhiwei.messageflow.mongo.bean.NoiseRule;
@@ -23,8 +24,21 @@ public class NoiseRuleDaoImpl implements NoiseRuleDao {
 	public List<NoiseRule> getNoiseRuleByProject(String projectName) {
 		Query query = new Query();
 		query.addCriteria(Criteria.where("project").is(projectName));
-
+		query.addCriteria(Criteria.where("isUsed").is(true));
 		return primaryMongoTemplate.find(query, NoiseRule.class);
 	}
 
+	@Override
+	public void addEffectNum(NoiseRule n) {
+		Query query = new Query();
+		query.addCriteria(Criteria.where("_id").is(n.get_id()));
+		query.addCriteria(Criteria.where("isUsed").is(true));
+		n = primaryMongoTemplate.findOne(query, NoiseRule.class);
+		if (null != n) {
+			query = Query.query(Criteria.where("_id").is(n.get_id()));
+			Update update = Update.update("isUsed", true).inc("effectNum", 1);
+			primaryMongoTemplate.updateMulti(query, update, NoiseRule.class);
+		}
+	}
+
 }

src/main/java/com/zhiwei/messageflow/mongo/dao/impl/TrackRuleDaoImpl.java

@@ -7,6 +7,7 @@ import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.data.mongodb.core.MongoTemplate;
 import org.springframework.data.mongodb.core.query.Criteria;
 import org.springframework.data.mongodb.core.query.Query;
+import org.springframework.data.mongodb.core.query.Update;
 import org.springframework.stereotype.Component;
 
 import com.zhiwei.messageflow.mongo.bean.TrackRule;
@@ -30,9 +31,9 @@ public class TrackRuleDaoImpl implements TrackRuleDao {
 
 	@Override
 	public void updateTrackrule(Long trackRule_id) {
-		System.out.println("预警状态更新");
-//		primaryMongoTemplate.updateFirst(Query.query(Criteria.where("_id").is(trackRule_id)),
-//				new Update().set("isWarn", true), TrackRule.class);
+//		System.out.println("预警状态更新");
+		primaryMongoTemplate.updateFirst(Query.query(Criteria.where("_id").is(trackRule_id)),
+				new Update().set("isWarn", true), TrackRule.class);
 	}
 
 }

src/test/java/com/zhiwei/messageflow/MessageflowApplicationTests.java

@@ -2,15 +2,25 @@ package com.zhiwei.messageflow;
 
 import org.junit.Test;
 import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.test.context.junit4.SpringRunner;
 
+import com.zhiwei.messageflow.mongo.bean.NoiseRule;
+import com.zhiwei.messageflow.mongo.dao.NoiseRuleDao;
+
 @RunWith(SpringRunner.class)
 @SpringBootTest
 public class MessageflowApplicationTests {
 	
+	@Autowired
+	private NoiseRuleDao noiseRuleDao;
+
 	@Test
 	public void contextLoads() {
+		NoiseRule n =new NoiseRule(); 
+		n.set_id(37);
+		noiseRuleDao.addEffectNum(n);
 	}
 
 }