预警部分的数据修改

src/main/java/com/zhiwei/manage/dao/NewsDao.java

@@ -21,5 +21,5 @@ public interface NewsDao {
 	
 	public List<EasyNews> finAllNews(Date start,Date end);
 	
-	public List<Map<String,Object>> findNews(String host,int port,String dbNamel,String collection,int count,String id,String pt);
+//	public List<Map<String,Object>> findNews(String host,int port,String dbNamel,String collection,int count,String id,String pt);
 }

src/main/java/com/zhiwei/manage/dao/NewsDaoImpl.java

@@ -116,32 +116,32 @@ public class NewsDaoImpl implements NewsDao {
 	/**
 	 * 获取新闻检测
 	 */
-	@SuppressWarnings("unchecked")
-	@Override
-	public List<Map<String, Object>> findNews(String host, int port, String dbNamel, String collection, int count,
-			String id, String pt) {
-		Calendar cal = Calendar.getInstance();
-		cal.add(Calendar.DATE, -1);
-		int month = cal.get(Calendar.MONTH) + 1;
-		String end = cal.get(Calendar.YEAR) + "" + (month > 9 ? month : "0" + month);
-		List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
-		DBCollection con = mongo.getCollection(collection + end);
-		if (id != null && !id.equals("")) {
-			DBCursor cur = con.find(new BasicDBObject("spyderInfoId", id)).sort(new BasicDBObject("rsid", -1))
-					.limit(count);
-			while (cur.hasNext()) {
-				Map<String, Object> map = (Map<String, Object>) cur.next();
-				list.add(map);
-			}
-		} else {
-			DBCursor cur = con.find().sort(new BasicDBObject("rsid", -1))
-					.limit(count);
-			while (cur.hasNext()) {
-				Map<String, Object> map = (Map<String, Object>) cur.next();
-				list.add(map);
-			}
-		}
-		return list;
-	}
+//	@SuppressWarnings("unchecked")
+//	@Override
+//	public List<Map<String, Object>> findNews(String host, int port, String dbNamel, String collection, int count,
+//			String id, String pt) {
+//		Calendar cal = Calendar.getInstance();
+//		cal.add(Calendar.DATE, -1);
+//		int month = cal.get(Calendar.MONTH) + 1;
+//		String end = cal.get(Calendar.YEAR) + "" + (month > 9 ? month : "0" + month);
+//		List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
+//		DBCollection con = mongo.getCollection(collection + end);
+//		if (id != null && !id.equals("")) {
+//			DBCursor cur = con.find(new BasicDBObject("spyderInfoId", id)).sort(new BasicDBObject("rsid", -1))
+//					.limit(count);
+//			while (cur.hasNext()) {
+//				Map<String, Object> map = (Map<String, Object>) cur.next();
+//				list.add(map);
+//			}
+//		} else {
+//			DBCursor cur = con.find().sort(new BasicDBObject("rsid", -1))
+//					.limit(count);
+//			while (cur.hasNext()) {
+//				Map<String, Object> map = (Map<String, Object>) cur.next();
+//				list.add(map);
+//			}
+//		}
+//		return list;
+//	}
 
 }

src/main/java/com/zhiwei/manage/handle/Task.java

@@ -2,12 +2,17 @@ package com.zhiwei.manage.handle;
 
 import java.net.HttpURLConnection;
 import java.net.URL;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Date;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
 import com.zhiwei.manage.bean.FieldIntegerity;
 import com.zhiwei.manage.bean.Message;
 import com.zhiwei.manage.bean.NewsDelayed;
@@ -16,12 +21,18 @@ import com.zhiwei.manage.bean.Template;
 import com.zhiwei.manage.orm.BeanFactory;
 import com.zhiwei.manage.service.MessageService;
 import com.zhiwei.manage.service.NewsService;
-
+/**
+ * 任务执行
+ * @author admin
+ *
+ */
 public class Task implements Runnable {
 
 	private NewsDelayed delayed;
 	private PingUrl ping;
 	private FieldIntegerity field;
+	final static SimpleDateFormat sdf=new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS Z");
+	private final static Log log=LogFactory.getLog(Task.class);
 
 	public Task(Object runMsg) {
 		if (runMsg instanceof NewsDelayed) {
@@ -35,6 +46,7 @@ public class Task implements Runnable {
 
 	@Override
 	public void run() {
+		System.out.println("预警部分启动");
 		MessageService messageService = BeanFactory.getBean(MessageService.class);
 		NewsService newsService = BeanFactory.getBean(NewsService.class);
 		// 连通性测试
@@ -124,68 +136,68 @@ public class Task implements Runnable {
 					}
 			}
 		}
-		if (field != null) {
-			if (MainThread.mainMap.get(field.getTemplateId()) != null) {
-				List<Map<String, Object>> list = newsService.findNews(field.getHost(), field.getPort(),
-						field.getdBName(), field.getCollection(), field.getTestCount(), field.getSpyderInfoId(),
-						field.getPt());
-				if (list.size() ==0) {
-					Template tp = MainThread.mainMap.get(field.getTemplateId());
-					Message msg = new Message();
-					msg.setCreateDate(new Date());
-					msg.setErrorMsg("字段检测发现库内的数据为0");
-					msg.setHandle(false);
-					msg.setPt(field.getPt());
-					msg.setTemplateCreator(tp.getCreator());
-					msg.setTemplateLv(tp.getTemplateLv());
-					msg.setErrorType("count2");
-					msg.setTempName(field.getTempName());
-					messageService.update(msg,field.getPt());
-				} else {
-					Message msg = messageService.findMsg(field.getTempName(), "count2");
-					if (msg != null) {
-						List<String> ids = new ArrayList<>();
-						ids.add(msg.getMessageId());
-						messageService.delete(ids, field.getPt());
-					}
-				}
-				String[] fields = field.getField().split(",");
-				int count = list.size();
-				for (String fid : fields) {
-					String str = "";
-					int num = count;
-					for (Map<String, Object> map : list) {
-						if (map.get(fid) == null) {
-							num--;
-							str = String.valueOf(map.get("_id"));
-						} else if (String.valueOf(map.get(fid)).length() < 1) {
-							num--;
-							str = String.valueOf(map.get("_id"));
-						}
-					}
-					if (num / 1.0 / count < field.getFieldIntegrityRate() / 100.0) {
-						Template tp = MainThread.mainMap.get(field.getTemplateId());
-						Message msg = new Message();
-						msg.setCreateDate(new Date());
-						msg.setErrorMsg(fid + "字段存在缺失,完整率" + num / 1.0 / count + "其中一条：" + str);
-						msg.setHandle(false);
-						msg.setPt(field.getPt());
-						msg.setErrorType("field");
-						msg.setTemplateCreator(tp.getCreator());
-						msg.setTemplateLv(tp.getTemplateLv());
-						msg.setTempName(field.getTempName());
-						messageService.update(msg,field.getPt());
-					} else {
-						Message msg = messageService.findMsg(field.getTempName(), "field");
-						if (msg != null) {
-							List<String> ids = new ArrayList<>();
-							ids.add(msg.getMessageId());
-							messageService.delete(ids, field.getPt());
-						}
-					}
-				}
-			}
-		}
+//		if (field != null) {//字段缺失率，采集部分已经对缺失字段做了处理，没什么必要检测了
+//			if (MainThread.mainMap.get(field.getTemplateId()) != null) {
+//				List<Map<String, Object>> list = newsService.findNews(field.getHost(), field.getPort(),
+//						field.getdBName(), field.getCollection(), field.getTestCount(), field.getSpyderInfoId(),
+//						field.getPt());
+//				if (list.size() ==0) {
+//					Template tp = MainThread.mainMap.get(field.getTemplateId());
+//					Message msg = new Message();
+//					msg.setCreateDate(new Date());
+//					msg.setErrorMsg("字段检测发现库内的数据为0");
+//					msg.setHandle(false);
+//					msg.setPt(field.getPt());
+//					msg.setTemplateCreator(tp.getCreator());
+//					msg.setTemplateLv(tp.getTemplateLv());
+//					msg.setErrorType("count2");
+//					msg.setTempName(field.getTempName());
+//					messageService.update(msg,field.getPt());
+//				} else {
+//					Message msg = messageService.findMsg(field.getTempName(), "count2");
+//					if (msg != null) {
+//						List<String> ids = new ArrayList<>();
+//						ids.add(msg.getMessageId());
+//						messageService.delete(ids, field.getPt());
+//					}
+//				}
+//				String[] fields = field.getField().split(",");
+//				int count = list.size();
+//				for (String fid : fields) {
+//					String str = "";
+//					int num = count;
+//					for (Map<String, Object> map : list) {
+//						if (map.get(fid) == null) {
+//							num--;
+//							str = String.valueOf(map.get("_id"));
+//						} else if (String.valueOf(map.get(fid)).length() < 1) {
+//							num--;
+//							str = String.valueOf(map.get("_id"));
+//						}
+//					}
+//					if (num / 1.0 / count < field.getFieldIntegrityRate() / 100.0) {
+//						Template tp = MainThread.mainMap.get(field.getTemplateId());
+//						Message msg = new Message();
+//						msg.setCreateDate(new Date());
+//						msg.setErrorMsg(fid + "字段存在缺失,完整率" + num / 1.0 / count + "其中一条：" + str);
+//						msg.setHandle(false);
+//						msg.setPt(field.getPt());
+//						msg.setErrorType("field");
+//						msg.setTemplateCreator(tp.getCreator());
+//						msg.setTemplateLv(tp.getTemplateLv());
+//						msg.setTempName(field.getTempName());
+//						messageService.update(msg,field.getPt());
+//					} else {
+//						Message msg = messageService.findMsg(field.getTempName(), "field");
+//						if (msg != null) {
+//							List<String> ids = new ArrayList<>();
+//							ids.add(msg.getMessageId());
+//							messageService.delete(ids, field.getPt());
+//						}
+//					}
+//				}
+//			}
+//		}
 		if (delayed != null) {
 			if (MainThread.mainMap.get(delayed.getTemplateId()) != null) {
 				List<Map<String, Object>> list = newsService.findNews(delayed.getHost(), delayed.getPort(),
@@ -207,7 +219,12 @@ public class Task implements Runnable {
 					Date time = null;
 					Date savetime = null;
 					if (map.get(delayed.getTime()) != null) {
-						time = (Date) map.get(delayed.getTime());
+						try {
+							String date = String.valueOf(map.get(delayed.getTime())).replace("Z", " UTC");
+							time = sdf.parse(String.valueOf(date));
+						} catch (ParseException e) {
+							log.error("时间转换出错【{}】",e);
+						}
 					}
 					if (map.get(delayed.getSavetime()) != null) {
 						Date d = new Date((Long) map.get(delayed.getSavetime()));

src/main/java/com/zhiwei/manage/service/NewsServiceImpl.java

@@ -10,12 +10,13 @@ import org.springframework.stereotype.Service;
 import com.zhiwei.manage.bean.EasyNews;
 import com.zhiwei.manage.bean.News;
 import com.zhiwei.manage.dao.NewsDao;
+import com.zhiwei.manage.util.ESUtil;
 @Service
 public class NewsServiceImpl implements NewsService{
 	
 	@Autowired
 	private NewsDao newsDao;
-	
+	private ESUtil es=new ESUtil();
 
 	@Override
 	public List<News> findByNew(int delayedCount, String type, String source) {
@@ -51,7 +52,7 @@ public class NewsServiceImpl implements NewsService{
 	@Override
 	public List<Map<String, Object>> findNews(String host, int port, String dbNamel, String collection, int count,
 			String id,String pt) {
-		return newsDao.findNews(host, port, dbNamel, collection, count, id, pt);
+		return es.getDatasMap(host, port, dbNamel, collection, count, id, pt);
 	}
 	
 	

src/main/java/com/zhiwei/manage/util/ESUtil.java

@@ -18,9 +18,12 @@ import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.sort.SortBuilder;
+import org.elasticsearch.search.sort.SortOrder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.domain.Sort;
 import org.springframework.stereotype.Component;
 
 import com.zhiwei.manage.bean.Data;
@@ -93,22 +96,67 @@ public class ESUtil {
 		}
 		return 0;
 	}
-
 	public static void main(String[] args) throws ParseException {
-		Client esClient = null;
-		Calendar cal=Calendar.getInstance();
-		cal.add(Calendar.HOUR_OF_DAY, -24);
-		esClient = new ESClient("Media-University","1.119.44.201", 9300).getClient();
-		SearchRequestBuilder srb = esClient.prepareSearch("mediaspider*");
-		BoolQueryBuilder bb = new BoolQueryBuilder();
-		srb.setQuery(bb);
-		QueryBuilder matchQuery = QueryBuilders.matchPhraseQuery("type", "今日头条");
-		bb.should(matchQuery);
-		srb.setSize(1);
-		srb.setPostFilter(QueryBuilders.rangeQuery("time").gte(getNeedTime(cal)).lte(getNeedTime(Calendar.getInstance())));
-		SearchResponse response = srb.execute().actionGet();
-		SearchHits searchHits = response.getHits();
-		System.out.println(searchHits.getTotalHits());
+//		Client esClient = null;
+//		Calendar cal=Calendar.getInstance();
+//		cal.add(Calendar.HOUR_OF_DAY, -24);
+//		esClient = new ESClient("Media-University","1.119.44.201", 9300).getClient();
+//		SearchRequestBuilder srb = esClient.prepareSearch("mediaspider*");
+//		BoolQueryBuilder bb = new BoolQueryBuilder();
+//		srb.setQuery(bb);
+//		QueryBuilder matchQuery = QueryBuilders.matchPhraseQuery("type", "今日头条");
+//		bb.should(matchQuery);
+//		srb.setSize(1);
+//		srb.setPostFilter(QueryBuilders.rangeQuery("time").gte(getNeedTime(cal)).lte(getNeedTime(Calendar.getInstance())));
+//		SearchResponse response = srb.execute().actionGet();
+//		SearchHits searchHits = response.getHits();
+//		System.out.println(searchHits.getTotalHits());
+		
+		SimpleDateFormat sdf=new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS Z");
+		for(Map<String,Object> map:getDatasMap("1.119.44.201", 9300, "Media-University", "mediaspider*", 20, "UrlListDetail74ec93ff-2275-4f22-bb62-bae580f2d43b", "网媒")) {
+			System.out.println(map.get("title"));
+			String date = map.get("time").toString().replace("Z", " UTC");
+//			System.out.println(map.get("time"));
+			System.out.println(sdf.parse(date));
+			System.out.println((long)map.get("savetime"));
+			
+		}
+	}
+	
+	/**
+	 * 数据查询。获取数据
+	 * 
+	 * @param start
+	 * @param end
+	 * @param pt
+	 * @param type
+	 * @param value
+	 * @param spyderInfoId
+	 * @return
+	 */
+	public static List<Map<String, Object>> getDatasMap(String host, int port, String dbNamel, String collection, int count,
+			String id,String pt) {
+		List<Map<String,Object>> list=new ArrayList<>();
+		try {
+			Client esClient = new ESClient(dbNamel, host, port).getClient();
+			// 搜索数据
+			SearchRequestBuilder srb = esClient.prepareSearch(collection);
+			BoolQueryBuilder bb = new BoolQueryBuilder();
+			srb.setQuery(bb);
+			srb.addSort("time", SortOrder.DESC);
+			QueryBuilder matchQuery = QueryBuilders.matchPhraseQuery("spyderInfoId",id);
+			bb.should(matchQuery);
+			srb.setSize(count);
+			SearchResponse response = srb.execute().actionGet();
+			SearchHits searchHits = response.getHits();
+			for(SearchHit result:searchHits) {
+				Map<String,Object> map=result.getSource();
+				list.add(map);
+			}
+		} catch (Exception e) {
+			log.error("es检索出错，错误信息【{}】", e);
+		}
+		return list;
 	}
 
 	public static String getNeedTime(Calendar cal) {