模板检测修改

src/main/java/com/zhiwei/manage/dao/DataDao.java

@@ -6,6 +6,7 @@ import java.util.List;
 import com.zhiwei.manage.bean.Data;
 import com.zhiwei.manage.bean.EasyNews;
 import com.zhiwei.manage.bean.News;
+import com.zhiwei.manage.bean.Template;
 
 public interface DataDao {
 
@@ -19,8 +20,8 @@ public interface DataDao {
 	
 	public List<Data> findByMethod(Date startTime,Date endTime,String method,Object value);
 	
-	
 	public int findBySourceFromLibrary(Date startTime, Date endTime, String source);
+	
 	public int findBySpId(Date startTime, Date endTime, String syId);
 	/**
 	 * 根据数据来源来搜索数据
@@ -31,9 +32,8 @@ public interface DataDao {
 	 */
 	public List<EasyNews> findByAccurateSource(String startTime, String endTime, String source);
 	
-	
-	
 	public int findByAccurateType(Date startTime, Date endTime, String type);
+	
 	/**
 	 * 根据可视化模板的id去搜索数据
 	 * @param startTime
@@ -43,25 +43,13 @@ public interface DataDao {
 	 */
 	public List<EasyNews> findByAccurateSpId(String startTime, String endTime, String SpId);
 	
-	
-	
-	/**
-	 * 微博数据统计
-	 * @param startTime
-	 * @param endTime
-	 * @return
-	 */
-	public long findWeiboCount(Date startTime,Date endTime);
-	
-	public long findWeixinCount(Date startTime,Date endTime,String pt);
-
 	public List<String> findSourceByLikeParam(String param);
 	
 	public List<String> findTmpNameByLikeParam(String param); 
 	
 	public List<String> findTypeByLikeParam(String param);
 	
-	public long finCountByDayData(Date startTime,Date endTime,String pt,String id);
-
+	public long finCountByDayData(Date startTime,Date endTime,Template temp);
 
+	public long finCountByDayData(Date startTime,Date endTime,String pt);
 }

src/main/java/com/zhiwei/manage/handle/DelayedThread.java

@@ -45,7 +45,6 @@ public class DelayedThread extends Thread {
 					queue.offer(run);
 					cal.add(Calendar.HOUR, run.getDelayedInterval());
 					run.setNextRun(cal.getTime());
-					System.out.println("开始");
 				}
 			}
 			Tools.sleep(1000);

src/main/java/com/zhiwei/manage/handle/MainThread.java

@@ -46,7 +46,6 @@ public class MainThread extends Thread {
 
 	/**
 	 * 加入执行map
-	 * 
 	 * @param template
 	 */
 	public static boolean pushMap(String templateId) {
@@ -148,7 +147,6 @@ public class MainThread extends Thread {
 		ServerServiceImpl serverService = BeanFactory.getBean(ServerServiceImpl.class);
 		MessageServiceImpl msgService = BeanFactory.getBean(MessageServiceImpl.class);
 		SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
-		// int num = 1;
 		while (true) {
 			Calendar countTest = Calendar.getInstance();
 			Calendar start = Calendar.getInstance();
@@ -161,8 +159,6 @@ public class MainThread extends Thread {
 				} catch (ParseException e1) {
 					e1.printStackTrace();
 				}
-				// int num = 1;
-				// for (int i = 0; i < 19; i++) {
 				start.add(Calendar.DATE, -1);
 				countTest.add(Calendar.DATE, -7);
 				try {
@@ -171,53 +167,15 @@ public class MainThread extends Thread {
 					List<Channel> channel = serverService.findDisChannel();
 					for (Channel chan : channel) {
 						try {
-							if (chan.getPt().equals("网媒")) {
-								List<CrawTemplate> list = crawService.findAll();
-								// 每日的数据量入库（对应模板）
-								for (CrawTemplate cr : list) {
-									count = (int) dataService.finCountByDayData(start.getTime(), end.getTime(),
-											cr.getPt(), cr.getId());
-									Data data = new Data();
-									data.setCount(count);
-									data.setTempName(cr.getConfigName());
-									data.setTime(start.getTime());
-									data.setSpyderInfoId(cr.getId());
-									dataService.insert(data);
-									if (count == 0) {
-										int testCount = (int) dataService.finCountByDayData(countTest.getTime(),
-												end.getTime(), cr.getPt(), cr.getId());
-										if (testCount == 0) {
-											Message m = new Message();
-											m.setHandle(false);
-											m.setCreateDate(start.getTime());
-											m.setTemplateLv(1);
-											m.setTempName(cr.getConfigName());
-											m.setPt(cr.getPt());
-											m.setErrorMsg("数据为0检测：" + cr.getConfigName() + "," + cr.getId() + "数据为0");
-											msgService.insert(m);
-										}
-									}
-									log.info(TimeUtil.yearToDay(start.getTime()) + "|" + data.getTempName() + "|入库");
-								}
-								count = (int) dataService.finCountByDayData(start.getTime(), end.getTime(),
-										chan.getPt(), null);
-								Data data = new Data();
-								data.setCount(count);
-								data.setPt(chan.getPt());
-								data.setTime(start.getTime());
-								dataService.insert(data);
-							} else {
-								count = (int) dataService.finCountByDayData(start.getTime(), end.getTime(),
-										chan.getPt(), null);
+							count = (int) dataService.finCountByDayData(start.getTime(), end.getTime(), chan.getPt());
 							Data data = new Data();
 							data.setCount(count);
 							data.setPt(chan.getPt());
 							data.setTime(start.getTime());
 							dataService.insert(data);
 							log.info(TimeUtil.yearToDay(start.getTime()) + "|" + chan.getPt() + "|入库");
-							}
 						} catch (Exception e) {
-
+							log.error(chan.getPt()+"入库出错，错误信息{}",e);
 						}
 					}
 					Set<String> keys = allTmp.keySet();
@@ -227,8 +185,7 @@ public class MainThread extends Thread {
 						if (tmp.getSpyderInfoId() != null) {
 							int cwCount = 0;
 							String pt = crawService.findPt(tmp.getSpyderInfoId());
-							cwCount = (int) dataService.finCountByDayData(start.getTime(), end.getTime(), tmp.getPt(),
-									tmp.getSpyderInfoId());
+							cwCount = (int) dataService.finCountByDayData(start.getTime(), end.getTime(), tmp);
 							if (cwCount < tmp.getNewsNumber()) {
 								Message m = new Message();
 								m.setHandle(false);
@@ -246,9 +203,6 @@ public class MainThread extends Thread {
 				} finally {
 					mainLock.unlock();
 				}
-				// end.add(Calendar.DATE, -1);
-				// num++;
-				// }
 				Tools.sleep(1000 * 60 * 60);
 			}
 			Tools.sleep(1000 * 30);

src/main/java/com/zhiwei/manage/handle/PingThread.java

@@ -46,7 +46,6 @@ public class PingThread extends Thread {
 					PingUrl run = runMap.get(key);
 					Calendar cal = Calendar.getInstance();
 					if (run.getNextRun().before(cal.getTime())) {
-						System.out.println("ping添加一个");
 						queue.offer(run);
 						cal.add(Calendar.HOUR, run.getPingInterval());
 						run.setNextRun(cal.getTime());

src/main/java/com/zhiwei/manage/handle/Task.java

@@ -50,7 +50,7 @@ public class Task implements Runnable {
 						Template tp = MainThread.mainMap.get(ping.getTemplateId());
 						Message msg = new Message();
 						msg.setCreateDate(new Date());
-						msg.setErrorMsg(code == 0 ? ping.getPingUrlOne() + "访问失败,错误代码" + code : "访问超时");
+						msg.setErrorMsg(code == 0 ? "访问超时":ping.getPingUrlOne() + "访问失败,错误代码" + code );
 						msg.setHandle(false);
 						msg.setTemplateCreator(tp.getCreator());
 						msg.setTempName(ping.getTempName());

src/main/java/com/zhiwei/manage/service/DataService.java

@@ -5,6 +5,7 @@ import java.util.List;
 
 import com.zhiwei.manage.bean.Data;
 import com.zhiwei.manage.bean.EasyNews;
+import com.zhiwei.manage.bean.Template;
 
 public interface DataService {
 
@@ -28,15 +29,12 @@ public interface DataService {
 	
 	public int findByTempName(Date startTime, Date endTime, String tempName);
 
-	public long findWeiboCount(Date start, Date end);
-
-	public long findWeixinCount(Date start, Date end,String pt);
-
 	public List<String> findSourceByLikeParam(String param);
 
 	public List<String> findTmpNameByLikeParam(String param);
 	
 	public List<String> findTypeByLikeParam(String param);
 	//根据配置获取数据
-	public long finCountByDayData(Date startDate,Date endDate,String pt,String id);
+	public long finCountByDayData(Date startDate,Date endDate,Template temp);
+	public long finCountByDayData(Date startDate,Date endDate,String pt);
 }

src/main/java/com/zhiwei/manage/service/DataServiceImpl.java

@@ -8,10 +8,12 @@ import org.springframework.stereotype.Service;
 
 import com.zhiwei.manage.bean.Data;
 import com.zhiwei.manage.bean.EasyNews;
+import com.zhiwei.manage.bean.Template;
 import com.zhiwei.manage.dao.CrawTemplateDao;
 import com.zhiwei.manage.dao.DataDao;
+
 @Service
-public class DataServiceImpl implements DataService{
+public class DataServiceImpl implements DataService {
 
 	@Autowired
 	private DataDao dataDao;
@@ -49,16 +51,6 @@ public class DataServiceImpl implements DataService{
 	}
 
 	@Override
-	public long findWeiboCount(Date start, Date end) {		
-		return dataDao.findWeiboCount(start, end);
-	}
-
-	@Override
-	public long findWeixinCount(Date start, Date end,String pt) {
-		return dataDao.findWeixinCount(start, end,pt);
-	}
-
-	@Override
 	public int findBySourceFromLibrary(Date startTime, Date endTime, String source) {
 		return dataDao.findBySourceFromLibrary(startTime, endTime, source);
 	}
@@ -89,16 +81,19 @@ public class DataServiceImpl implements DataService{
 	}
 
 	@Override
-	public long finCountByDayData(Date startTime, Date endTime, String pt, String id) {
-		return dataDao.finCountByDayData(startTime, endTime, pt, id);
+	public long finCountByDayData(Date startTime, Date endTime, Template temp) {
+		return dataDao.finCountByDayData(startTime, endTime, temp);
 	}
 
 	@Override
 	public int findByTempName(Date startTime, Date endTime, String tempName) {
-		String spId=crd.getSpIdByConfigName(tempName);
+		String spId = crd.getSpIdByConfigName(tempName);
 		return dataDao.findBySpId(startTime, endTime, spId);
 	}
 
-	
+	@Override
+	public long finCountByDayData(Date startDate, Date endDate, String pt) {
+		return dataDao.finCountByDayData(startDate, endDate, pt);
+	}
 
 }

src/main/java/com/zhiwei/manage/util/MongoConnect.java

@@ -28,17 +28,9 @@ import com.mongodb.ServerAddress;
 public class MongoConnect {
 	private static final Logger log = LoggerFactory.getLogger(MongoConnect.class);
 	private static MongoClient mongoClient = null;
-	private static String host;
-	private static int port;
-	private static String username;
-	private static String password;
 	private static String dbName;
 
 	public MongoConnect(String host, int port, String username, String password, String dbName) {
-		this.host = host;
-		this.port = port ;
-		this.username = username;
-		this.password = password;
 		this.dbName = dbName;
 		if (mongoClient == null) {
 			MongoClientOptions.Builder optionsBuilder = new MongoClientOptions.Builder();
@@ -46,7 +38,6 @@ public class MongoConnect {
 			optionsBuilder.connectTimeout(1000 * 60 * 40); // 与数据库建立连接的超时时间
 			optionsBuilder.maxWaitTime(1000 * 60 * 5); // 一个线程成功获取到一个可用数据库连接之前的最大等待时间
 			optionsBuilder.threadsAllowedToBlockForConnectionMultiplier(100);// 等待一个连接
-																				// 允许阻塞的线程数
 			optionsBuilder.maxConnectionIdleTime(0);
 			optionsBuilder.maxConnectionLifeTime(0);
 			optionsBuilder.socketKeepAlive(true);
@@ -67,8 +58,6 @@ public class MongoConnect {
 	
 	
 	public MongoConnect(String host, int port, String dbName) {
-		this.host = host;
-		this.port = port ;
 		this.dbName = dbName;
 		if (mongoClient == null) {
 			MongoClientOptions.Builder optionsBuilder = new MongoClientOptions.Builder();