修改数据的获取方式和查询方式

src/main/java/com/zhiwei/manage/dao/DataDao.java

@@ -20,41 +20,5 @@ public interface DataDao {
 
 	public List<Data> findBytempName(Date startTime, Date endTime, String tempName);
 
-	public List<Data> findByMethod(Date startTime, Date endTime, String method, Object value);
-
-	public int findBySourceFromLibrary(Date startTime, Date endTime, String source);
-
-	public int findBySpId(Date startTime, Date endTime, String syId);
-
-	/**
-	 * 根据数据来源来搜索数据
-	 * 
-	 * @param startTime
-	 * @param endTime
-	 * @param source
-	 * @return List
-	 */
-	public List<EasyNews> findByAccurateSource(String startTime, String endTime, String source);
-
-	public int findByAccurateType(Date startTime, Date endTime, String type);
-
-	/**
-	 * 根据可视化模板的id去搜索数据
-	 * 
-	 * @param startTime
-	 * @param endTime
-	 * @param SpId
-	 * @return List
-	 */
-	public List<EasyNews> findByAccurateSpId(String startTime, String endTime, String SpId);
-
-	public List<String> findSourceByLikeParam(String param);
-
 	public List<String> findTmpNameByLikeParam(String param);
-
-	public List<String> findTypeByLikeParam(String param);
-
-	public long finCountByDayData(Date startTime, Date endTime, Template temp);
-
-	public long finCountByDayData(Date startTime, Date endTime, String pt);
 }

src/main/java/com/zhiwei/manage/dao/ServerDao.java

@@ -34,5 +34,7 @@ public interface ServerDao {
 	public List<Channel> findDisChannels();
 
 	public List<String> findDisChan();
+	
+	public ServerBean findServerBean(String pt);
 
 }

src/main/java/com/zhiwei/manage/dao/ServerDaoImpl.java

@@ -26,6 +26,7 @@ public class ServerDaoImpl implements ServerDao {
 
 	@Resource(name = "mongoTemplate")
 	private MongoTemplate mongo;
+
 	@Override
 	public void insert(ServerBean serverBean) {
 		mongo.insert(serverBean);
@@ -74,33 +75,37 @@ public class ServerDaoImpl implements ServerDao {
 
 	@Override
 	public void findPt(String pt) {
-//		List<ServerBean> list = mongo.find(Query.query(Criteria.where("pt").is(pt)), ServerBean.class);
-//		for (ServerBean sb : list) {
-//			if (sb.getDbType().equals("mongo")) {
-//				if (sb.getDbUsername() != null && !sb.getDbUsername().equals("")) {
-//					DBCollection con = new MongoConnect(sb.getDbHost(), sb.getDbPort(), sb.getDbUsername(),
-//							sb.getDbPassword(), sb.getDbName()).getCollection(sb.getCollection());
-//					DbDepot.dbCons.put(sb.getOther(), con);
-//				} else {
-//					DBCollection con = new MongoConnect(sb.getDbHost(), sb.getDbPort(), sb.getDbName())
-//							.getCollection(sb.getCollection());
-//					DbDepot.dbCons.put(sb.getOther(), con);
-//				}
-//			}
-//		}
-//		DbDepot.nowPt = pt;
+		// List<ServerBean> list = mongo.find(Query.query(Criteria.where("pt").is(pt)),
+		// ServerBean.class);
+		// for (ServerBean sb : list) {
+		// if (sb.getDbType().equals("mongo")) {
+		// if (sb.getDbUsername() != null && !sb.getDbUsername().equals("")) {
+		// DBCollection con = new MongoConnect(sb.getDbHost(), sb.getDbPort(),
+		// sb.getDbUsername(),
+		// sb.getDbPassword(), sb.getDbName()).getCollection(sb.getCollection());
+		// DbDepot.dbCons.put(sb.getOther(), con);
+		// } else {
+		// DBCollection con = new MongoConnect(sb.getDbHost(), sb.getDbPort(),
+		// sb.getDbName())
+		// .getCollection(sb.getCollection());
+		// DbDepot.dbCons.put(sb.getOther(), con);
+		// }
+		// }
+		// }
+		// DbDepot.nowPt = pt;
 	}
 
 	@Override
-	public List<Channel> findAllChannels(int pageNo,int pageSize) {
-		Query query=new Query();
-		query.skip((pageNo-1)*pageSize).limit(pageSize);
+	public List<Channel> findAllChannels(int pageNo, int pageSize) {
+		Query query = new Query();
+		query.skip((pageNo - 1) * pageSize).limit(pageSize);
 		List<Channel> list = mongo.find(query, Channel.class);
 		return list;
 	}
+
 	@Override
 	public List<Channel> findChannels() {
-		List<Channel> list = mongo.find(null,Channel.class);
+		List<Channel> list = mongo.find(null, Channel.class);
 		return list;
 	}
 
@@ -133,7 +138,7 @@ public class ServerDaoImpl implements ServerDao {
 	@Override
 	public boolean deleteChannel(List<String> id) {
 		WriteResult result = mongo.remove(Query.query(Criteria.where("_id").in(id)), Channel.class);
-		return result.getN() >0 ? true : false;
+		return result.getN() > 0 ? true : false;
 	}
 
 	@Override
@@ -161,4 +166,10 @@ public class ServerDaoImpl implements ServerDao {
 		return mongo.getCollection("Channel").distinct("pt");
 	}
 
+	@Override
+	public ServerBean findServerBean(String pt) {
+		return mongo.findOne(Query.query(Criteria.where("pt").is(pt).andOperator(Criteria.where("dbType").is("Elasticsearch"))),
+				ServerBean.class);
+	}
+
 }

src/main/java/com/zhiwei/manage/handle/DbDepot.java

@@ -8,22 +8,22 @@ import org.apache.commons.logging.LogFactory;
 import org.springframework.data.mongodb.core.MongoTemplate;
 import org.springframework.stereotype.Service;
 
+import com.zhiwei.manage.bean.ServerBean;
+
 @Service("firstRun")
 public class DbDepot {
 
 	@Resource(name = "mongoTemplate")
 	private MongoTemplate mongo;
-	public static ConcurrentMap<String, Object> dbCons;
-	public static ConcurrentMap<String, Object> newsCol;
-	public static ConcurrentMap<String, Object> dataCol;
+	public static ConcurrentMap<String, Object> esClients;
+	public static ConcurrentMap<String, ServerBean> serverBeans;
 	public static int month = 0;
 	private static final Log log = LogFactory.getLog(DbDepot.class);
 
 	public void init() {
 		log.info("数据连接map初始化");
-		dataCol = new ConcurrentHashMap<String, Object>();
-		dbCons = new ConcurrentHashMap<String, Object>();
-		newsCol = new ConcurrentHashMap<String, Object>();
+		esClients = new ConcurrentHashMap<String, Object>();
+		serverBeans = new ConcurrentHashMap<String, ServerBean>();
 		System.out.println("连接初始化成功");
 	}
 }

src/main/java/com/zhiwei/manage/handle/MainThread.java

@@ -22,6 +22,7 @@ import com.zhiwei.manage.bean.Message;
 import com.zhiwei.manage.bean.Messages;
 import com.zhiwei.manage.bean.NewsDelayed;
 import com.zhiwei.manage.bean.PingUrl;
+import com.zhiwei.manage.bean.ServerBean;
 import com.zhiwei.manage.bean.Template;
 import com.zhiwei.manage.orm.BeanFactory;
 import com.zhiwei.manage.service.CrawTemplateServiceImpl;
@@ -162,8 +163,10 @@ public class MainThread extends Thread {
 		CrawTemplateServiceImpl crawService = BeanFactory.getBean(CrawTemplateServiceImpl.class);
 		DataServiceImpl dataService = BeanFactory.getBean(DataServiceImpl.class);
 		ServerServiceImpl serverService = BeanFactory.getBean(ServerServiceImpl.class);
+		MessageServiceImpl messageService=BeanFactory.getBean(MessageServiceImpl.class);
 		SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH");
 		List<Channel> channel = serverService.findDisChannel();
+		Calendar loop=Calendar.getInstance();
 		while (true) {
 			Calendar start = Calendar.getInstance();
 			if (start.get(Calendar.HOUR_OF_DAY) < 1) {
@@ -179,7 +182,8 @@ public class MainThread extends Thread {
 				for (Channel chan : channel) {
 					map.put(chan.getPt(), chan.getValue());
 					try {
-						count = es.getCountByPt(start.getTime(), end.getTime(), chan.getPt(), chan.getValue());
+						ServerBean ser=serverService.findServer(chan.getPt());
+						count = es.getCountByPt(start.getTime(), end.getTime(), chan.getPt(), chan.getValue(),ser);
 						System.out.println(count);
 						Data data = new Data();
 						data.setCount(count);
@@ -206,7 +210,7 @@ public class MainThread extends Thread {
 						int cwCount = 0;
 						String pt = crawService.findPt(tmp.getSpyderInfoId());
 						cwCount = (int) es.getCounts(start.getTime(), end.getTime(), pt, map.get(pt),
-								tmp.getSpyderInfoId());
+								tmp.getSpyderInfoId(),serverService.findServer(pt));
 						try {
 							Data data = new Data();
 							data.setCount(cwCount);
@@ -214,6 +218,20 @@ public class MainThread extends Thread {
 							data.setTempName(tmp.getTempName());
 							dataService.insert(data);
 							log.info(TimeUtil.yearToDay(start.getTime()) + "|" + tmp.getTempName() + "|入库");
+							if(cwCount==0) {
+								loop.setTime(start.getTime());
+								loop.add(Calendar.DATE, -2);
+								cwCount = (int) es.getCounts(start.getTime(), end.getTime(), pt, map.get(pt),
+										tmp.getSpyderInfoId(),serverService.findServer(pt));
+								if(cwCount<3) {
+									Message msg=new Message();
+									msg.setCreateDate(new Date());
+									msg.setErrorMsg("连续三天数据不足3条，请确认采集是否有问题");
+									msg.setPt(pt);
+									msg.setTemplateLv(1);
+									messageService.insert(msg);
+								}
+							}
 						} catch (Exception e) {
 							log.error(tmp.getTempName() + "入库出错，错误信息{}", e);
 						}

src/main/java/com/zhiwei/manage/service/DataService.java

@@ -15,30 +15,21 @@ public interface DataService {
 
 	public List<Data> findByPtDay(Date startTime, Date endTime, String pt);
 
-	public List<Data> findBySource(Date startTime, Date endTime, String source);
+	public List<Data> findBySource(Date startTime, Date endTime,String source);
 
 	public List<Data> findBytempName(Date startTime, Date endTime, String tempName);
 
-	public List<Data> findByMethod(Date startTime, Date endTime, String method, Object value);
+//	public List<Data> findByMethod(Date startTime, Date endTime, String method, Object value);
 
-	public int findBySourceFromLibrary(Date startTime, Date endTime, String source);
+//	public int findBySourceFromLibrary(Date startTime, Date endTime, String source);
 
-	public List<EasyNews> findByAccurateSource(String startTime, String endTime, String source);
-
-	public List<EasyNews> findByAccurateSpId(String startTime, String endTime, String SpId);
-
-	public int findByAccurateType(Date startTime, Date endTime, String type);
-
-	public int findByTempName(Date startTime, Date endTime, String tempName);
-
-	public List<String> findSourceByLikeParam(String param);
+	
+	public List<Data> findByMethodAndParam(Date startTime, Date endTime,String method,String param);
 
 	public List<String> findTmpNameByLikeParam(String param);
 
-	public List<String> findTypeByLikeParam(String param);
-
 	// 根据配置获取数据
-	public long finCountByDayData(Date startDate, Date endDate, Template temp);
-
-	public long finCountByDayData(Date startDate, Date endDate, String pt);
+//	public long finCountByDayData(Date startDate, Date endDate, Template temp);
+//
+//	public long finCountByDayData(Date startDate, Date endDate, String pt);
 }

src/main/java/com/zhiwei/manage/service/DataServiceImpl.java

@@ -9,15 +9,21 @@ import org.springframework.stereotype.Service;
 import com.zhiwei.manage.bean.Data;
 import com.zhiwei.manage.bean.EasyNews;
 import com.zhiwei.manage.bean.Template;
-import com.zhiwei.manage.dao.CrawTemplateDao;
 import com.zhiwei.manage.dao.DataDao;
+import com.zhiwei.manage.dao.ServerDao;
+import com.zhiwei.manage.util.ESUtil;
 
 @Service
 public class DataServiceImpl implements DataService {
 
 	@Autowired
 	private DataDao dataDao;
-
+	@Autowired
+	private ServerDao serverDao;
+	@Autowired
+	private CrawTemplateService crService;
+	private ESUtil es=new ESUtil();
+	
 	@Override
 	public void insert(Data data) {
 		dataDao.insert(data);
@@ -39,63 +45,18 @@ public class DataServiceImpl implements DataService {
 	}
 
 	@Override
-	public List<EasyNews> findByAccurateSource(String startTime, String endTime, String source) {
-		return dataDao.findByAccurateSource(startTime, endTime, source);
-	}
-
-	@Override
-	public List<EasyNews> findByAccurateSpId(String startTime, String endTime, String SpId) {
-		return dataDao.findByAccurateSpId(startTime, endTime, SpId);
-	}
-
-	@Override
-	public int findBySourceFromLibrary(Date startTime, Date endTime, String source) {
-		return dataDao.findBySourceFromLibrary(startTime, endTime, source);
-	}
-
-	@Override
-	public List<String> findSourceByLikeParam(String param) {
-		return dataDao.findSourceByLikeParam(param);
-	}
-
-	@Override
 	public List<String> findTmpNameByLikeParam(String param) {
 		return dataDao.findTmpNameByLikeParam(param);
 	}
 
 	@Override
-	public int findByAccurateType(Date startTime, Date endTime, String type) {
-		return dataDao.findByAccurateType(startTime, endTime, type);
-	}
-
-	@Override
-	public List<String> findTypeByLikeParam(String param) {
-		return dataDao.findTypeByLikeParam(param);
-	}
-
-	@Override
-	public List<Data> findByMethod(Date startTime, Date endTime, String method, Object value) {
-		return dataDao.findByMethod(startTime, endTime, method, value);
-	}
-
-	@Override
-	public long finCountByDayData(Date startTime, Date endTime, Template temp) {
-		return dataDao.finCountByDayData(startTime, endTime, temp);
-	}
-
-	@Override
-	public int findByTempName(Date startTime, Date endTime, String tempName) {
-		return dataDao.findBySpId(startTime, endTime, tempName);
-	}
-
-	@Override
-	public long finCountByDayData(Date startDate, Date endDate, String pt) {
-		return dataDao.finCountByDayData(startDate, endDate, pt);
+	public List<Data> findByPtDay(Date startTime, Date endTime, String pt) {
+		return dataDao.findByPtDay(startTime, endTime, pt);
 	}
 
 	@Override
-	public List<Data> findByPtDay(Date startTime, Date endTime, String pt) {
-		return dataDao.findByPtDay(startTime, endTime, pt);
+	public List<Data> findByMethodAndParam(Date startTime, Date endTime, String method, String param) {
+		return es.findByMethodAndParam(crService,startTime, endTime, method,param,serverDao.findServerBean("网媒"));
 	}
 
 }

src/main/java/com/zhiwei/manage/service/ServerService.java

@@ -31,5 +31,7 @@ public interface ServerService {
 	public List<Channel> findDisChannel();
 
 	public List<String> findDisChan();
+	
+	public ServerBean findServer(String pt);
 
 }

src/main/java/com/zhiwei/manage/service/ServerServiceImpl.java

@@ -103,4 +103,9 @@ public class ServerServiceImpl implements ServerService {
 		return null;
 	}
 
+	@Override
+	public ServerBean findServer(String pt) {
+		return sd.findServerBean(pt);
+	}
+
 }

src/main/java/com/zhiwei/manage/servlet/DataController.java

 package com.zhiwei.manage.servlet;
 
 import java.io.UnsupportedEncodingException;
-import java.net.URLEncoder;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Date;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
@@ -19,20 +17,14 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.ResponseBody;
-import org.springframework.web.multipart.support.RequestPartServletServerHttpRequest;
 
 import com.alibaba.fastjson.JSON;
 import com.zhiwei.manage.bean.Channel;
 import com.zhiwei.manage.bean.Data;
-import com.zhiwei.manage.bean.EasyNews;
 import com.zhiwei.manage.bean.JsonResult;
-import com.zhiwei.manage.service.CrawTemplateService;
 import com.zhiwei.manage.service.DataService;
 import com.zhiwei.manage.service.ServerService;
-import com.zhiwei.manage.util.Change;
-import com.zhiwei.manage.util.TimeUtil;
 import com.zhiwei.manage.util.Tools;
 
 @Controller
@@ -44,8 +36,6 @@ public class DataController {
 	private DataService dataService;
 	@Autowired
 	private ServerService serverService;
-	@Autowired
-	private CrawTemplateService crService;
 
 	@RequestMapping(value = "/getData", produces = "application/json;charset=utf-8")
 	@ResponseBody
@@ -84,46 +74,20 @@ public class DataController {
 		if (method == null) {
 			JsonResult result = new JsonResult(false, 200, "参数不全", "");
 			return JSON.toJSONString(result);
-		} else if (timeType.equals("hour")) {
-			// 小时数据
+		} else if (timeType.equals("hour")) {// 具体查询数据
 			if (start == 0) {
 				startd.set(Calendar.HOUR, -10);
 			} else {
 				startd.setTimeInMillis(start);
-				endTime.setTimeInMillis(start);
+				startd.add(Calendar.HOUR_OF_DAY, -8);
+				endTime.setTimeInMillis(end);
+				endTime.add(Calendar.HOUR_OF_DAY, -8);
 			}
-			Calendar ds = Calendar.getInstance();
-			ds.setTimeInMillis(end);
-			String source = "";
+			String source = (String) map.get(method);
 			List<Data> list = new ArrayList<Data>();
-			startd.add(Calendar.HOUR, -8);
-			endTime.add(Calendar.HOUR, -7);
-			ds.add(Calendar.HOUR, -6);
-			String spid = crService.getSpIdByConfigName((String) map.get("tempName"));
-			while (endTime.getTime().before(ds.getTime())) {
-				endTime.add(Calendar.HOUR, 1);
-				int count = 0;
-				if (method.equals("type")) {
-					source = (String) map.get("type");
-					count = dataService.findByAccurateType(startd.getTime(), endTime.getTime(), source);
-				} else {
-					if (method.equals("source")) {
-						source = (String) map.get("source");
-						count = dataService.findBySourceFromLibrary(startd.getTime(), endTime.getTime(), source);
-					} else {
-						source = (String) map.get("tempName");
-						count = dataService.findByTempName(startd.getTime(), endTime.getTime(), spid);
-					}
-				}
-				Data data = new Data();
-				data.setCount(count);
-				data.setSource(source);
-				data.setTime(startd.getTime());
-				list.add(data);
-				startd.add(Calendar.HOUR, 1);
-			}
+			list = dataService.findByMethodAndParam(startd.getTime(), endTime.getTime(), method, source);
 			return JSON.toJSONString(Tools.beResult(list, source));
-		} else {
+		} else {// day,数据结果从库里拿
 			if (start == 0) {
 				start = startd.getTimeInMillis() - 1000 * 3600 * 24 * 7;
 				startd.setTimeInMillis(start);
@@ -137,10 +101,12 @@ public class DataController {
 				String source = (String) map.get("source");
 				list = dataService.findBySource(startd.getTime(), endTime.getTime(), source);
 				channel.setValue(source);
-			} else {
+			} else if (method.equals("tempName")) {
 				String tempName = (String) map.get("tempName");
 				list = dataService.findBytempName(startd.getTime(), endTime.getTime(), tempName);
 				channel.setValue(tempName);
+			} else {
+
 			}
 			channel.setList(list);
 			JsonResult result = new JsonResult(true, 200, "返回成功", channel);
@@ -187,6 +153,7 @@ public class DataController {
 				return JSON.toJSONString(result);
 			}
 		} catch (Exception e) {
+			log.error("处理错误，错误信息{}",e);
 			JsonResult result = new JsonResult(false, 500, e.getMessage(), "");
 			return JSON.toJSONString(result);
 		}
@@ -197,9 +164,9 @@ public class DataController {
 	public String getDayDataByChannel(@RequestBody Map<String, Object> map) {
 		long start = (long) map.get("start");
 		long end = (long) map.get("end");
-		String pt = String.valueOf(map.get("pt")) ;
+		String pt = String.valueOf(map.get("pt"));
 		try {
-			Calendar nows=Calendar.getInstance();
+			Calendar nows = Calendar.getInstance();
 			Calendar startTime = Calendar.getInstance();
 			startTime.setTimeInMillis(start);
 			Calendar endTime = Calendar.getInstance();
@@ -212,6 +179,7 @@ public class DataController {
 			JsonResult result = new JsonResult(true, 200, "请求成功", datas);
 			return JSON.toJSONString(result);
 		} catch (Exception e) {
+			log.error("处理错误，错误信息{}",e);
 			JsonResult result = new JsonResult(false, 500, e.getMessage(), "");
 			return JSON.toJSONString(result);
 		}
@@ -219,7 +187,7 @@ public class DataController {
 
 	@RequestMapping(value = "/getSource", produces = "application/json;charset=utf-8")
 	@ResponseBody
-	public String getAllDataCountByTime(@RequestBody Map<String, Object> map) {
+	public String getSource(@RequestBody Map<String, Object> map, HttpServletRequest req) {
 		String value = (String) map.get("value");
 		String method = (String) map.get("method");
 		if (method != null) {
@@ -227,10 +195,6 @@ public class DataController {
 				List<String> list = dataService.findTmpNameByLikeParam(value);
 				JsonResult result = new JsonResult(true, 200, "请求成功", list);
 				return JSON.toJSONString(result);
-			} else if (method.equals("type")) {
-				List<String> list = dataService.findTypeByLikeParam(value);
-				JsonResult result = new JsonResult(true, 200, "请求成功", list);
-				return JSON.toJSONString(result);
 			} else {
 				JsonResult result = new JsonResult(true, 200, "请求成功", new ArrayList<String>());
 				return JSON.toJSONString(result);

src/main/java/com/zhiwei/manage/util/ESClient.java

@@ -15,33 +15,21 @@ import org.elasticsearch.transport.client.PreBuiltTransportClient;
  *
  */
 public class ESClient {
-	private static String esIp;
-	private static int esPort;
-	private static String clusterName;
-
+	static TransportClient client=null;
 	
-	private static class ESClientHolder {
-		
-		static TransportClient client = initESClient();
-		private static TransportClient initESClient() {
-			esIp=Config.getVal("ESIP");
-			clusterName=Config.getVal("clusterName");
-			esPort=Integer.valueOf(Config.getVal("ESPort"));
-			Settings esSettings = Settings.builder().put("cluster.name", clusterName) // 设置ES实例的名称
-					// 自动嗅探整个集群的状态，把集群中其他ES节点的ip添加到本地的客户端列表中
-					.put("client.transport.sniff", false).build();
-			TransportClient client = new PreBuiltTransportClient(esSettings);
-			try {
-				client.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(esIp), esPort));
-			} catch (UnknownHostException e) {
-				e.printStackTrace();
-			}
-			return client;
+	public ESClient(String clusterName,String esIp,int esPort) {
+		Settings esSettings = Settings.builder().put("cluster.name", clusterName) // 设置ES实例的名称
+				.put("client.transport.sniff", false).build();
+		client = new PreBuiltTransportClient(esSettings);
+		try {
+			client.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(esIp), esPort));
+		} catch (UnknownHostException e) {
+			e.printStackTrace();
 		}
 	}
-
-	public static TransportClient getInstance() {
-		return ESClientHolder.client;
+	
+	public TransportClient getClient() {
+		return client;
 	}
 
 }