Merge branch 'mlbWork' into 'master'

头条采集入库后更新阅读量

See merge request !51

src/main/java/com/zhiwei/searchhotcrawler/crawler/ToutiaoHotSearchCrawler.java

@@ -8,6 +8,7 @@ import com.zhiwei.crawler.proxy.ProxyHolder;
 import com.zhiwei.crawler.utils.RequestUtils;
 import com.zhiwei.searchhotcrawler.bean.HotSearchList;
 import com.zhiwei.searchhotcrawler.bean.HotSearchType;
+import com.zhiwei.searchhotcrawler.dao.HotSearchListDAO;
 import com.zhiwei.searchhotcrawler.util.TipsUtils;
 import com.zhiwei.tools.tools.URLCodeUtil;
 import com.zhiwei.tools.tools.ZhiWeiTools;
@@ -147,11 +148,12 @@ public class ToutiaoHotSearchCrawler {
 //    }
 
     /**
-     * 获取今日头条热搜阅读量
+     * 更新今日头条热搜阅读量
      * @param hotSearchList
      * @return
      */
     public static HotSearchList toutiaoReadCount(HotSearchList hotSearchList){
+        HotSearchListDAO hotSearchListDAO = new HotSearchListDAO();
         if (hotSearchList.getUrl() != null) {
             String htmlBody = null;
             String url = hotSearchList.getUrl();
@@ -171,6 +173,7 @@ public class ToutiaoHotSearchCrawler {
                         Integer count = TipsUtils.getHotCount(readCount);
                         log.info("{},阅读量:{}", hotSearchList.getName(), count);
                         hotSearchList.setCommentCount(count);
+                        hotSearchListDAO.updateTouTiaoReadCount(hotSearchList);
                         return hotSearchList;
                     }
                 }

src/main/java/com/zhiwei/searchhotcrawler/dao/HotSearchCacheDAO.java

@@ -48,9 +48,9 @@ public class HotSearchCacheDAO  {
                document.put("topic_lead", hotSearch.getTopicLead());
 			   document.put("comment_count", hotSearch.getCommentCount());
            }
-           if("今日头条热搜".equals(hotSearch.getType())){
-               document.put("comment_count", hotSearch.getCommentCount());
-           }
+//           if("今日头条热搜".equals(hotSearch.getType())){
+//               document.put("comment_count", hotSearch.getCommentCount());
+//           }
            if("腾讯较真榜".equals(hotSearch.getType())){
                document.put("topic_result",hotSearch.getTopicResult());
            }
@@ -82,7 +82,7 @@ public class HotSearchCacheDAO  {
            String topicResult = document.getString("topic_result")!=null?document.getString("topic_result"):null;
            String id = name + "_" + type;
            boolean recommend = false;
-           Integer readCount = document.getInteger("comment_count");
+//           Integer readCount = document.getInteger("comment_count");
            if("微博热搜".equals(type)){
                String icon = document.getString("icon");
                if("recom".equals(icon) || "jian".equals(icon)){
@@ -128,9 +128,9 @@ public class HotSearchCacheDAO  {
                    nowDoc.put("preCount", preCount);
                    nowDoc.put("duration", durationNow);
                    nowDoc.put("recommend",recommend);
-                   if(readCount != null){
-                       nowDoc.put("readCount",readCount);
-                   }
+//                   if(readCount != null){
+//                       nowDoc.put("readCount",readCount);
+//                   }
                    if(topicResult != null){
                        nowDoc.put("topicResult",topicResult);
                    }
@@ -154,9 +154,9 @@ public class HotSearchCacheDAO  {
                    nowDoc.put("preRank", null);
                    nowDoc.put("preCount", null);
                    nowDoc.put("recommend",recommend);
-                   if(readCount != null){
-                       nowDoc.put("readCount",readCount);
-                   }
+//                   if(readCount != null){
+//                       nowDoc.put("readCount",readCount);
+//                   }
                    if(topicResult != null){
                        nowDoc.put("topicResult",topicResult);
                    }

src/main/java/com/zhiwei/searchhotcrawler/dao/HotSearchListDAO.java

@@ -78,5 +78,19 @@ public class HotSearchListDAO{
 		}
 		return null;
 	}
+
+	/**
+	 * 今日头条热搜详情趋势添加阅读量
+	 * @param hotSearchList
+	 */
+	public void updateTouTiaoReadCount(HotSearchList hotSearchList){
+		BasicDBObject basicDBObject = new BasicDBObject();
+		basicDBObject.put("_id", hotSearchList.getId());
+		Document document = (Document)mongoCollection.find(basicDBObject).first();
+		if(document != null){
+			document.put("commentCount",hotSearchList.getCommentCount());
+			mongoCollection.replaceOne(basicDBObject,document);
+		}
+	}
 	
 }

src/main/java/com/zhiwei/searchhotcrawler/timer/TouTiaoExecutor.java

@@ -14,8 +14,6 @@ public class TouTiaoExecutor extends Thread {
 
     private HotSearchList hotSearchList;
 
-    private static List<HotSearchList> resultList;
-
     public TouTiaoExecutor(HotSearchList hotSearchList){
         this.hotSearchList = hotSearchList;
     }
@@ -23,8 +21,7 @@ public class TouTiaoExecutor extends Thread {
     @Override
     public void run() {
         try {
-            hotSearchList = ToutiaoHotSearchCrawler.toutiaoReadCount(hotSearchList);
-            resultList.add(hotSearchList);
+           hotSearchList = ToutiaoHotSearchCrawler.toutiaoReadCount(hotSearchList);
         }catch (Exception e){
             e.printStackTrace();
         }
@@ -35,8 +32,7 @@ public class TouTiaoExecutor extends Thread {
      * @param list
      * @return
      */
-    public static List<HotSearchList> countTouTiaoReadCount(List<HotSearchList> list){
-        resultList= new ArrayList<>();
+    public static void countTouTiaoReadCount(List<HotSearchList> list){
 //        ExecutorService service = Executors.newFixedThreadPool(list.size());
         for(int i=0; i<list.size(); i++){
             TipsUtils.service.execute(new TouTiaoExecutor(list.get(i)));
@@ -44,11 +40,10 @@ public class TouTiaoExecutor extends Thread {
 //        TipsUtils.service.shutdown();
         try {
             if(!TipsUtils.service.awaitTermination(1, TimeUnit.MINUTES)){
-                log.info("查询今日头条阅读量超时");
+                log.info("今日头条阅读量更新结束");
             }
         } catch (InterruptedException e) {
             log.info(e.fillInStackTrace());
         }
-        return resultList;
     }
 }

src/main/java/com/zhiwei/searchhotcrawler/timer/quartz/GatherTimer.java

@@ -85,11 +85,11 @@ public class GatherTimer {
         logger.info("今日头条热搜开始采集...");
         Date date = DateUtils.getMillSecondTime(new Date());
         List<HotSearchList> toutiaoList = ToutiaoHotSearchCrawler.toutiaoHotSearchByPhone(date);
-        List<HotSearchList> toutiaoResult = new ArrayList<>();
-        toutiaoResult = TouTiaoExecutor.countTouTiaoReadCount(toutiaoList);
-        logger.info("{}, 今日头条此轮采集到的数据量为:{}", new Date(), toutiaoResult != null ? toutiaoResult.size() : 0);
-        TipsUtils.addHotList(HotSearchType.今日头条热搜.name(),toutiaoResult);
+        logger.info("{}, 今日头条此轮采集到的数据量为:{}", new Date(), toutiaoList != null ? toutiaoList.size() : 0);
+        TipsUtils.addHotList(HotSearchType.今日头条热搜.name(),toutiaoList);
         logger.info("今日头条热搜采集结束...");
+        logger.info("今日头条热搜详情趋势阅读量更新...");
+        TouTiaoExecutor.countTouTiaoReadCount(toutiaoList);
     }
 
     /**