采集调整为每分钟采集一次

src/main/java/com/zhiwei/searchhotcrawler/dao/HotSearchCacheDAO.java

@@ -7,7 +7,6 @@ import com.zhiwei.searchhotcrawler.dbtemplate.MongoDBTemplate;
 import lombok.extern.log4j.Log4j2;
 import org.bson.Document;
 
-import javax.print.Doc;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
@@ -135,31 +134,32 @@ public class HotSearchCacheDAO  {
      * @return
      */
    private int getDuration(String type, int duration){
-        switch (type){
-            case "微博热搜" :
-                duration = duration + 1;
-                break;
-            case "百度热搜" :
-                duration = duration + 5;
-                break;
-            case "知乎热搜" :
-                duration = duration + 10;
-                break;
-            case "抖音热搜" :
-                duration = duration + 10;
-                break;
-            case "搜狗微信热搜" :
-                duration = duration + 5;
-                break;
-            case "微博话题" :
-                duration = duration + 3;
-                break;
-            case "今日头条热搜" :
-                duration = duration + 1;
-                break;
-            default :
-                duration = duration + 1;
-        }
+//        switch (type){
+//            case "微博热搜" :
+//                duration = duration + 1;
+//                break;
+//            case "百度热搜" :
+//                duration = duration + 5;
+//                break;
+//            case "知乎热搜" :
+//                duration = duration + 10;
+//                break;
+//            case "抖音热搜" :
+//                duration = duration + 10;
+//                break;
+//            case "搜狗微信热搜" :
+//                duration = duration + 5;
+//                break;
+//            case "微博话题" :
+//                duration = duration + 3;
+//                break;
+//            case "今日头条热搜" :
+//                duration = duration + 1;
+//                break;
+//            default :
+//                duration = duration + 1;
+//        }
+        duration = duration + 1;
         return duration;
    }
 
@@ -171,32 +171,32 @@ public class HotSearchCacheDAO  {
      * @return
      */
     private Date getEndTime(String type, Date time){
-       long timeLong = time.getTime();
-        switch (type){
-            case "微博热搜" :
-                timeLong = timeLong + 1*60*1000;
-                break;
-            case "百度热搜" :
-                timeLong = timeLong + 5*60*1000;
-                break;
-            case "知乎热搜" :
-                timeLong = timeLong + 10*60*1000;
-                break;
-            case "抖音热搜" :
-                timeLong = timeLong + 10*60*1000;
-                break;
-            case "搜狗微信热搜" :
-                timeLong = timeLong + 5*60*1000;
-                break;
-            case "微博话题" :
-                timeLong = timeLong + 3*60*1000;
-                break;
-            case "今日头条热搜" :
-                timeLong = timeLong + 1*60*1000;
-                break;
-            default :
-                timeLong = timeLong + 1*60*1000;
-        }
+       long timeLong = time.getTime() + 1*60*1000;
+//        switch (type){
+//            case "微博热搜" :
+//                timeLong = timeLong + 1*60*1000;
+//                break;
+//            case "百度热搜" :
+//                timeLong = timeLong + 5*60*1000;
+//                break;
+//            case "知乎热搜" :
+//                timeLong = timeLong + 10*60*1000;
+//                break;
+//            case "抖音热搜" :
+//                timeLong = timeLong + 10*60*1000;
+//                break;
+//            case "搜狗微信热搜" :
+//                timeLong = timeLong + 5*60*1000;
+//                break;
+//            case "微博话题" :
+//                timeLong = timeLong + 3*60*1000;
+//                break;
+//            case "今日头条热搜" :
+//                timeLong = timeLong + 1*60*1000;
+//                break;
+//            default :
+//                timeLong = timeLong + 1*60*1000;
+//        }
         return new Date(timeLong);
     }
 

src/main/java/com/zhiwei/searchhotcrawler/dbtemplate/MongoDBLocalTemplate.java

+package com.zhiwei.searchhotcrawler.dbtemplate;
+
+import com.mongodb.MongoClient;
+import com.mongodb.MongoClientOptions;
+import com.mongodb.MongoClientURI;
+import com.mongodb.WriteConcern;
+import com.mongodb.client.ListIndexesIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.model.IndexOptions;
+import com.zhiwei.searchhotcrawler.config.DBConfig;
+import lombok.extern.log4j.Log4j2;
+import org.bson.Document;
+
+import java.util.Objects;
+
+@Log4j2
+public enum MongoDBLocalTemplate {
+
+	instance;
+	private MongoClient mongoClient;
+
+	static {
+		MongoClientOptions options = new MongoClientOptions.Builder()
+				.connectionsPerHost(300)  //连接池设置为300个连接,默认为100
+				.connectTimeout(15000)    //连接超时，推荐>3000毫秒
+				.maxWaitTime(5000)
+				.socketTimeout(0)         // 套接字超时时间，0无限制
+				.threadsAllowedToBlockForConnectionMultiplier(5000) // 线程队列数，如果连接线程排满了队列就会抛出“Out of semaphores to get db”错误。
+				.writeConcern(WriteConcern.W1) //
+				.build();
+
+		log.info("MongoDBTemplate.static initializer : {}", DBConfig.mongoLocalUri);
+		MongoClientURI mongoClientURI = new MongoClientURI(DBConfig.mongoLocalUri);
+		instance.mongoClient = new MongoClient(mongoClientURI);
+	}
+
+	/**
+	 * 获取DB实例 - 指定DB
+	 *
+	 * @param databaseName
+	 * @return
+	 */
+	public static MongoDatabase getDB(String databaseName) {
+		return instance.mongoClient.getDatabase(databaseName);
+	}
+
+	/**
+	 * 获取collection对象 - 指定Collection
+	 *
+	 * @param databaseName
+	 * @param collectionName
+	 * @return
+	 */
+	public static MongoCollection<Document> getCollection(String databaseName, String collectionName) {
+		MongoDatabase db = instance.mongoClient.getDatabase(databaseName);
+		return db.getCollection(collectionName);
+	}
+
+
+	/**
+	 *  创建索引
+	 * @param databaseName
+	 * @param collectionName
+	 */
+	public static void createIndex(String databaseName, String collectionName){
+		MongoDatabase db = instance.mongoClient.getDatabase(databaseName);
+		MongoCollection mongoCollection = db.getCollection(collectionName);
+		ListIndexesIterable<Document> indexList = mongoCollection.listIndexes();
+		if(Objects.isNull(indexList)){
+			Document countIndexDoc = new Document();
+			if(collectionName.contains("hot_search_list")){
+				countIndexDoc.put("count", -1);
+			}else{
+				countIndexDoc.put("score_num", -1);
+			}
+			Document timeIndexDoc = new Document();
+			timeIndexDoc.put("time", -1);
+			Document rankIndexDoc = new Document();
+			rankIndexDoc.put("rank", -1);
+			Document nameIndexDoc = new Document();
+			nameIndexDoc.put("name", -1);
+			Document typeIndexDoc = new Document();
+			typeIndexDoc.put("type", -1);
+			try {
+				mongoCollection.createIndex(countIndexDoc, new IndexOptions().name("count_desc"));
+				mongoCollection.createIndex(timeIndexDoc, new IndexOptions().name("time_desc"));
+				mongoCollection.createIndex(rankIndexDoc, new IndexOptions().name("rank_desc"));
+				mongoCollection.createIndex(nameIndexDoc, new IndexOptions().name( "name_desc"));
+				mongoCollection.createIndex(typeIndexDoc, new IndexOptions().name( "type_desc"));
+			} catch (Exception e) {
+				e.printStackTrace();
+			}
+		}
+	}
+
+}

src/main/java/com/zhiwei/searchhotcrawler/timer/BaiduHotSearchRun.java

@@ -28,7 +28,7 @@ public class BaiduHotSearchRun extends Thread{
 		while(f) {
 			try {
 				getHotList();
-				TimeUnit.MINUTES.sleep(5);
+				TimeUnit.MINUTES.sleep(1);
 			} catch (Exception e) {
 				e.fillInStackTrace();
 				ZhiWeiTools.sleep(60*60*1000);

src/main/java/com/zhiwei/searchhotcrawler/timer/DouyinHotSearchRun.java

@@ -27,7 +27,7 @@ public class DouyinHotSearchRun extends Thread{
 		while(f) {
 			try {
 				getHotList();
-				TimeUnit.MINUTES.sleep(10);
+				TimeUnit.MINUTES.sleep(1);
 			} catch (Exception e) {
 				e.fillInStackTrace();
 				ZhiWeiTools.sleep(60*60*1000);

src/main/java/com/zhiwei/searchhotcrawler/timer/SougoHotSearchRun.java

@@ -27,7 +27,7 @@ public class SougoHotSearchRun extends Thread {
 		while(f) {
 			try {
 				getHotList();
-				TimeUnit.MINUTES.sleep(5);
+				TimeUnit.MINUTES.sleep(1);
 			} catch (Exception e) {
 				e.fillInStackTrace();
 				ZhiWeiTools.sleep(60*60*1000);

src/main/java/com/zhiwei/searchhotcrawler/timer/WeiboTopicRun.java

@@ -21,7 +21,7 @@ public class WeiboTopicRun extends Thread{
 		while(f) {
 			try {
 				getTopicList();
-				TimeUnit.MINUTES.sleep(3);
+				TimeUnit.MINUTES.sleep(1);
 			} catch (Exception e) {
 				e.fillInStackTrace();
 				ZhiWeiTools.sleep(60*60*1000);

src/main/java/com/zhiwei/searchhotcrawler/timer/ZhihuHotSearchRun.java

@@ -27,7 +27,7 @@ public class ZhihuHotSearchRun extends Thread{
 		while(f) {
 			try {
 				getHotList();
-				TimeUnit.MINUTES.sleep(10);
+				TimeUnit.MINUTES.sleep(1);
 			} catch (Exception e) {
 				e.fillInStackTrace();
 				ZhiWeiTools.sleep(60*60*1000);