修改微博热搜解析规则

src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiboHotSearchCrawler.java

@@ -38,10 +38,10 @@ public class WeiboHotSearchCrawler {
 	 * @return void 返回类型
 	 */
 	public static List<WeiboHotSearch>  weiboHotSearch(){
-		String url = "http://s.weibo.com/top/summary?cate=realtimehot";
+		String url = "https://s.weibo.com/top/summary?cate=realtimehot";
 		
 		Map<String,String> headerMap = new HashMap<String,String>();
-		headerMap.put("Referer", "http://s.weibo.com/top/summary?cate=realtimehot");
+//		headerMap.put("Referer", "https://s.weibo.com/top/summary?cate=realtimehot");
 		headerMap.put("Host", "s.weibo.com");
 		headerMap.put("User-Agent", "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36");
 		headerMap.put("Upgrade-Insecure-Requests", "1");
@@ -54,12 +54,12 @@ public class WeiboHotSearchCrawler {
 				htmlBody = HttpClientTemplateOK.get(url, null, headerMap);
 				if(htmlBody!=null && htmlBody.contains("pl_top_realtimehot")){
 					try {
-						String script = htmlBody.split("<script>STK && STK.pageletM && STK.pageletM.view")[5].split("<\\/script>")[0];
-						script = script.replace("(", "").replace(")", "");
-						JSONObject json = JSONObject.parseObject(script);
-						String html = json.getString("html");
-						Document document = Jsoup.parse(html);
-						Elements elements = document.select("tbody").select("tr");
+//						String script = htmlBody.split("<script>STK && STK.pageletM && STK.pageletM.view")[5].split("<\\/script>")[0];
+//						script = script.replace("(", "").replace(")", "");
+//						JSONObject json = JSONObject.parseObject(script);
+//						String html = json.getString("html");
+						Document document = Jsoup.parse(htmlBody);
+						Elements elements = document.select("div#pl_top_realtimehot").select("tbody").select("tr");
 						for(Element element : elements){
 							try {
 								String id = "http://s.weibo.com"+element.select("p.star_name").select("a").attr("href");
@@ -97,7 +97,7 @@ public class WeiboHotSearchCrawler {
 		return list;
 	}
 	
-	
+
 	
 	
 	/**

src/main/java/com/zhiwei/searchhotcrawler/run/HotSearchRun.java

@@ -16,18 +16,18 @@ public class HotSearchRun {
 	private ScheduledExecutorService scheduExec;
 
     public HotSearchRun() {
-        this.scheduExec = Executors.newScheduledThreadPool(3);
+        this.scheduExec = Executors.newScheduledThreadPool(2);
     }     
 
     public void showTimer() {
         scheduExec.scheduleAtFixedRate(new WeiboHotSearchRun(), 0, 1, TimeUnit.MINUTES);
         scheduExec.scheduleAtFixedRate(new ZhihuHotSearchRun(), 0, 1 , TimeUnit.MINUTES);
-        scheduExec.scheduleAtFixedRate(new SendZhihuHotSearchRun(), 0, 60 , TimeUnit.MINUTES);
-        scheduExec.scheduleAtFixedRate(new SendWeiboHotSearchRun(), 0,  60 , TimeUnit.MINUTES);
     }
     
     public static void main(String[] args) {
 		new HotSearchRun().showTimer();
 		new CacheListener().startListen();
+		new SendWeiboHotSearchRun().start();
+		new SendZhihuHotSearchRun().start();
 	}
 }

src/main/java/com/zhiwei/searchhotcrawler/timer/SendWeiboHotSearchRun.java

@@ -14,6 +14,7 @@ import com.zhiwei.searchhotcrawler.util.Template;
 import com.zhiwei.searchhotcrawler.util.WechatCodeUtil;
 import com.zhiwei.searchhotcrawler.util.WechatConstant;
 import com.zhiwei.zhiweiTools.timeParse.TimeParse;
+import com.zhiwei.zhiweiTools.tools.ZhiWeiTools;
 
 public class SendWeiboHotSearchRun extends Thread{
 	private WeiboHotSearchDAO weiboHotSearchDAO = new WeiboHotSearchDAO();
@@ -21,20 +22,29 @@ public class SendWeiboHotSearchRun extends Thread{
 	
 	@Override
 	public void run() {
-		Calendar calendar = Calendar.getInstance();
-		int hour = calendar.get(Calendar.HOUR_OF_DAY);
-		logger.info("微博推送,当前系统时间为：" + hour);
-		if(hour > 6 && hour < 23){
-			List<DBObject> list = weiboHotSearchDAO.getWeiboHotOneHour();
-			if(list!=null && list.size()>0){
-				for(DBObject weibo : list){
-					String title = weibo.get("name").toString();
-					String time = TimeParse.dateFormartString((Date)weibo.get("time"), "yyyy-MM-dd HH:mm:ss");
-					String url = weibo.get("url").toString();
-					sendTemplateByUserIds(title, time, url);
+		while(true) {
+			try {
+				Calendar calendar = Calendar.getInstance();
+				int hour = calendar.get(Calendar.HOUR_OF_DAY);
+				logger.info("微博推送,当前系统时间为：" + hour);
+				if(hour > 6 && hour < 23){
+					List<DBObject> list = weiboHotSearchDAO.getWeiboHotOneHour();
+					if(list!=null && list.size()>0){
+						for(DBObject weibo : list){
+							String title = weibo.get("name").toString();
+							String time = TimeParse.dateFormartString((Date)weibo.get("time"), "yyyy-MM-dd HH:mm:ss");
+							String url = weibo.get("url").toString();
+							sendTemplateByUserIds(title, time, url);
+						}
+					}else{
+						sendTemplateByUserIds("最近一小时无数据", TimeParse.dateFormartString(new Date(), "yyyy-MM-dd HH:mm:ss"), null);
+					}
 				}
-			}else{
-				sendTemplateByUserIds("最近一小时无数据", TimeParse.dateFormartString(new Date(), "yyyy-MM-dd HH:mm:ss"), null);
+				ZhiWeiTools.sleep(1*60*60*1000);
+			} catch (Exception e) {
+				logger.debug("微博热搜推送出现问题,问题为:::{}",e.fillInStackTrace());
+				ZhiWeiTools.sleep(1*60*60*1000);
+				continue;
 			}
 		}
 	}

src/main/java/com/zhiwei/searchhotcrawler/timer/SendZhihuHotSearchRun.java

@@ -17,28 +17,39 @@ import com.zhiwei.searchhotcrawler.util.Template;
 import com.zhiwei.searchhotcrawler.util.WechatCodeUtil;
 import com.zhiwei.searchhotcrawler.util.WechatConstant;
 import com.zhiwei.zhiweiTools.timeParse.TimeParse;
+import com.zhiwei.zhiweiTools.tools.ZhiWeiTools;
 
 public class SendZhihuHotSearchRun extends Thread{
 	private ZhihuHotSearchDAO zhihuHotSearchDAO = new ZhihuHotSearchDAO();
 	private static Logger logger = LoggerFactory.getLogger(SendZhihuHotSearchRun.class);
 	@Override
 	public void run() {
-		Calendar calendar = Calendar.getInstance();
-		int hour = calendar.get(Calendar.HOUR_OF_DAY);
-		logger.info("知乎推送,当前系统时间为："+hour);
-		if(hour > 6 && hour <23){
-			List<DBObject> list = zhihuHotSearchDAO.getZhiHuHotSearch();
-			if(list!=null && list.size()>0){
-				for(DBObject zhihu : list){
-					String title = zhihu.get("display_query").toString();
-					String time = TimeParse.dateFormartString((Date)zhihu.get("time"), "yyyy-MM-dd HH:mm:ss");
-					String url = zhihu.get("_id").toString();
-					if(calendar.get(Calendar.HOUR_OF_DAY) > 6 && calendar.get(Calendar.HOUR_OF_DAY) < 23){
-						sendTemplateByUserIds(title, time, url);
+		
+		while(true) {
+			try {
+				Calendar calendar = Calendar.getInstance();
+				int hour = calendar.get(Calendar.HOUR_OF_DAY);
+				logger.info("知乎推送,当前系统时间为："+hour);
+				if(hour > 6 && hour <23){
+					List<DBObject> list = zhihuHotSearchDAO.getZhiHuHotSearch();
+					if(list!=null && list.size()>0){
+						for(DBObject zhihu : list){
+							String title = zhihu.get("display_query").toString();
+							String time = TimeParse.dateFormartString((Date)zhihu.get("time"), "yyyy-MM-dd HH:mm:ss");
+							String url = zhihu.get("_id").toString();
+							if(calendar.get(Calendar.HOUR_OF_DAY) > 6 && calendar.get(Calendar.HOUR_OF_DAY) < 23){
+								sendTemplateByUserIds(title, time, url);
+							}
+						}
+					}else{
+						sendTemplateByUserIds("最近一小时无数据", TimeParse.dateFormartString(new Date(), "yyyy-MM-dd HH:mm:ss"), null);
 					}
 				}
-			}else{
-				sendTemplateByUserIds("最近一小时无数据", TimeParse.dateFormartString(new Date(), "yyyy-MM-dd HH:mm:ss"), null);
+				ZhiWeiTools.sleep(1*60*60*1000);
+			} catch (Exception e) {
+				logger.debug("知乎热搜推送出现问题,问题为:::{}",e.fillInStackTrace());
+				ZhiWeiTools.sleep(1*60*60*1000);
+				continue;
 			}
 		}
 	}