backup

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/config/BaseConfig.java

 package com.zhiweidata.automatictest.barragecrawlerserver.config;
 
+import com.mongodb.client.MongoCollection;
+import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
+
+import static com.zhiweidata.automatictest.publics.BeanContainer.MONGO_DATABASE;
+
 /**
  * 基本配置
  *
@@ -9,7 +14,6 @@ package com.zhiweidata.automatictest.barragecrawlerserver.config;
  */
 @SuppressWarnings("JavaDoc")
 public class BaseConfig {
-    public static final String MONGO_LIVE_COLLECTION = "live_url";
     /**
      * 接口的基础url
      */
@@ -30,4 +34,12 @@ public class BaseConfig {
      * 弹幕的excel导出服务器测试，最多延迟 X 天
      */
     public static final int EXPORT_START_MAX_DELAY = 7;
+    /**
+     * 模块名称
+     */
+    public static final String MODULE_NAME = "barrage_crawler_server";
+    /**
+     * 测试采集任务的信息集合
+     */
+    public static final MongoCollection<BarrageCollectionTask> MONGO_TASK_REQUEST_COLLECTION = MONGO_DATABASE.getCollection(MODULE_NAME + "_task_requests", BarrageCollectionTask.class);
 }

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/BiliBiliCreateTaskJob.java

@@ -47,8 +47,6 @@ public class BiliBiliCreateTaskJob extends CreateTaskJob {
             });
         } catch (Exception e) {
             log.error(e.getMessage(), e);
-        } finally {
-            super.finish();
         }
     }
 }

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/CreateTaskJob.java

 package com.zhiweidata.automatictest.barragecrawlerserver.createtasks;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.mongodb.client.MongoCollection;
 import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
 import com.zhiweidata.automatictest.barragecrawlerserver.exporttasks.BarrageExportTaskJob;
 import com.zhiweidata.automatictest.barragecrawlerserver.util.TimeUtil;
@@ -15,7 +14,6 @@ import org.apache.http.HttpResponse;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.entity.ByteArrayEntity;
 import org.apache.http.entity.ContentType;
-import org.bson.Document;
 import org.jetbrains.annotations.NotNull;
 import org.quartz.Job;
 import org.quartz.JobBuilder;
@@ -26,12 +24,11 @@ import org.quartz.TriggerBuilder;
 
 import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.BARRAGE_COLLECTION_TASK_JOB_KEY;
 import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.EXPORT_START_MAX_DELAY;
-import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.MONGO_LIVE_COLLECTION;
+import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.MONGO_TASK_REQUEST_COLLECTION;
 import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.TASK_URL;
 import static com.zhiweidata.automatictest.publics.BeanContainer.HTTP_CLIENT;
-import static com.zhiweidata.automatictest.publics.BeanContainer.MONGO_DATABASE;
-import static com.zhiweidata.automatictest.publics.BeanContainer.STANDARD_DATE_FORMAT;
 import static com.zhiweidata.automatictest.publics.BeanContainer.SCHEDULER;
+import static com.zhiweidata.automatictest.publics.BeanContainer.STANDARD_DATE_FORMAT;
 
 /**
  * @author aszswaz
@@ -41,8 +38,6 @@ import static com.zhiweidata.automatictest.publics.BeanContainer.SCHEDULER;
 @Slf4j
 @SuppressWarnings({"JavaDoc"})
 public abstract class CreateTaskJob implements Job {
-    private final List<Document> liveUrls = new ArrayList<>();
-
     /**
      * 请求弹幕采集服务器创建采集任务
      */
@@ -61,25 +56,18 @@ public abstract class CreateTaskJob implements Job {
         HttpResponse response = HTTP_CLIENT.execute(post);
         Map<?, ?> jsonMap = objectMapper.readValue(response.getEntity().getContent(), Map.class);
 
-        Document document = new Document("liveUrl", liveUrl);
-        jsonMap.forEach((key, value) -> document.put((String) key, value));
-        this.liveUrls.add(document);
+        collectionTask.setCode((Integer) jsonMap.get("code"));
+        collectionTask.setMessage((String) jsonMap.get("message"));
+        MONGO_TASK_REQUEST_COLLECTION.insertOne(collectionTask);
         return collectionTask;
     }
 
-    protected final void finish() {
-        if (!this.liveUrls.isEmpty()) {
-            MongoCollection<Document> liveUrls = MONGO_DATABASE.getCollection(MONGO_LIVE_COLLECTION);
-            liveUrls.insertMany(this.liveUrls);
-        }
-    }
-
     /**
      * 创建定时导出任务
      */
-    protected final <T extends BarrageExportTaskJob> void createExportTask(BarrageCollectionTask collectionTask, Class<T> tClass) throws SchedulerException {
+    protected final <T extends BarrageExportTaskJob> void createExportTask(@NotNull BarrageCollectionTask collectionTask, Class<T> tClass) throws SchedulerException {
         JobBuilder jobBuilder = JobBuilder.newJob(tClass);
-        String name = tClass.getSimpleName(), group = "export";
+        String name = collectionTask.getName(), group = tClass.getSimpleName();
         jobBuilder.withIdentity(name, group);
         jobBuilder.withDescription("弹幕导出，以及导出的excel文件的分析");
         JobDataMap dataMap = new JobDataMap();

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/DouyuCreateTaskJob.java

@@ -59,8 +59,6 @@ public class DouyuCreateTaskJob extends CreateTaskJob {
             });
         } catch (Exception e) {
             log.error(e.getMessage(), e);
-        } finally {
-            super.finish();
         }
     }
 

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/HuyaCreateTaskJob.java

@@ -2,7 +2,6 @@ package com.zhiweidata.automatictest.barragecrawlerserver.createtasks;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.mongodb.client.MongoCollection;
 import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
 import com.zhiweidata.automatictest.barragecrawlerserver.exception.BarrageHomePageException;
 import com.zhiweidata.automatictest.barragecrawlerserver.exporttasks.HuyaExportTaskJob;
@@ -19,16 +18,13 @@ import org.apache.http.HttpStatus;
 import org.apache.http.StatusLine;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.util.EntityUtils;
-import org.bson.Document;
 import org.jsoup.nodes.Element;
 import org.quartz.DisallowConcurrentExecution;
 import org.quartz.JobExecutionContext;
 import org.seimicrawler.xpath.JXDocument;
 import org.seimicrawler.xpath.JXNode;
 
-import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.MONGO_LIVE_COLLECTION;
 import static com.zhiweidata.automatictest.publics.BeanContainer.HTTP_CLIENT;
-import static com.zhiweidata.automatictest.publics.BeanContainer.MONGO_DATABASE;
 
 /**
  * 从虎牙直播的首页获得直播间的地址，并发送到弹幕采集服务器进行测试
@@ -39,16 +35,12 @@ import static com.zhiweidata.automatictest.publics.BeanContainer.MONGO_DATABASE;
  */
 @Slf4j
 @DisallowConcurrentExecution
-@SuppressWarnings({"JavaDoc", "unused", "SpellCheckingInspection", "GrazieInspection"})
+@SuppressWarnings({"JavaDoc", "unused", "SpellCheckingInspection"})
 public class HuyaCreateTaskJob extends CreateTaskJob {
     /**
      * 幻灯片中，直播间的url匹配
      */
     private static final Pattern LIVE_URL = Pattern.compile("var slides= \\[\\{[\\w\\W]+?}];");
-    /**
-     * 直播间地址集合
-     */
-    private final MongoCollection<Document> liveUrls = MONGO_DATABASE.getCollection(MONGO_LIVE_COLLECTION);
 
     @Override
     public void execute(JobExecutionContext jobExecutionContext) {
@@ -64,8 +56,6 @@ public class HuyaCreateTaskJob extends CreateTaskJob {
             this.createTaskByHtml(EntityUtils.toString(response.getEntity()));
         } catch (Exception e) {
             log.error(e.getMessage(), e);
-        } finally {
-            super.finish();
         }
     }
 

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/entity/BarrageCollectionTask.java

 package com.zhiweidata.automatictest.barragecrawlerserver.entity;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import java.util.Calendar;
 import lombok.EqualsAndHashCode;
@@ -40,6 +41,16 @@ public class BarrageCollectionTask {
      */
     @JsonProperty(value = "endTime")
     private Integer endTime;
+    /**
+     * 服务器的响应代码
+     */
+    @JsonIgnore
+    private int code;
+    /**
+     * 响应信息
+     */
+    @JsonIgnore
+    private String message;
 
     public BarrageCollectionTask(String liveUrl, String liveName, long currentTime) {
         this.url = liveUrl;
@@ -47,10 +58,11 @@ public class BarrageCollectionTask {
 
         Calendar calendar = Calendar.getInstance();
         calendar.setTimeInMillis(currentTime);
-        calendar.set(Calendar.MINUTE, 0);
+        // calendar.set(Calendar.MINUTE, 0);
+        calendar.add(Calendar.MINUTE, 1);
         calendar.set(Calendar.SECOND, 0);
         calendar.set(Calendar.MILLISECOND, 0);
-        calendar.add(Calendar.HOUR_OF_DAY, 1);
+        // calendar.add(Calendar.HOUR_OF_DAY, 1);
 
         this.startTime = (int) (calendar.getTimeInMillis() / 1000);
         int minute = (int) (Math.random() * (MAX_COLLECTION_TIME + 1));

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/BiliBiliExportTaskJob.java

@@ -2,6 +2,7 @@ package com.zhiweidata.automatictest.barragecrawlerserver.exporttasks;
 
 import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
 import lombok.extern.slf4j.Slf4j;
+import org.quartz.JobDataMap;
 import org.quartz.JobExecutionContext;
 
 import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.BARRAGE_COLLECTION_TASK_JOB_KEY;
@@ -20,7 +21,8 @@ public class BiliBiliExportTaskJob implements BarrageExportTaskJob {
     @Override
     public void execute(JobExecutionContext context) {
         try {
-            BarrageCollectionTask collectionTask = (BarrageCollectionTask) requireNonNull(context.get(BARRAGE_COLLECTION_TASK_JOB_KEY));
+            JobDataMap dataMap = requireNonNull(context.getMergedJobDataMap());
+            BarrageCollectionTask collectionTask = (BarrageCollectionTask) requireNonNull(dataMap.get(BARRAGE_COLLECTION_TASK_JOB_KEY));
         } catch (Exception e) {
             log.error(e.getMessage(), e);
         }

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/DouyuExportTaskJob.java

@@ -2,6 +2,7 @@ package com.zhiweidata.automatictest.barragecrawlerserver.exporttasks;
 
 import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
 import lombok.extern.slf4j.Slf4j;
+import org.quartz.JobDataMap;
 import org.quartz.JobExecutionContext;
 
 import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.BARRAGE_COLLECTION_TASK_JOB_KEY;
@@ -20,7 +21,8 @@ public class DouyuExportTaskJob implements BarrageExportTaskJob {
     @Override
     public void execute(JobExecutionContext context) {
         try {
-            BarrageCollectionTask collectionTask = (BarrageCollectionTask) requireNonNull(context.get(BARRAGE_COLLECTION_TASK_JOB_KEY));
+            JobDataMap dataMap = requireNonNull(context.getMergedJobDataMap());
+            BarrageCollectionTask collectionTask = (BarrageCollectionTask) requireNonNull(dataMap.get(BARRAGE_COLLECTION_TASK_JOB_KEY));
         } catch (Exception e) {
             log.error(e.getMessage(), e);
         }

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/HuyaExportTaskJob.java

@@ -2,6 +2,7 @@ package com.zhiweidata.automatictest.barragecrawlerserver.exporttasks;
 
 import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
 import lombok.extern.slf4j.Slf4j;
+import org.quartz.JobDataMap;
 import org.quartz.JobExecutionContext;
 
 import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.BARRAGE_COLLECTION_TASK_JOB_KEY;
@@ -20,7 +21,8 @@ public class HuyaExportTaskJob implements BarrageExportTaskJob {
     @Override
     public void execute(JobExecutionContext context) {
         try {
-            BarrageCollectionTask collectionTask = (BarrageCollectionTask) requireNonNull(context.get(BARRAGE_COLLECTION_TASK_JOB_KEY));
+            JobDataMap dataMap = requireNonNull(context.getMergedJobDataMap());
+            BarrageCollectionTask collectionTask = (BarrageCollectionTask) requireNonNull(dataMap.get(BARRAGE_COLLECTION_TASK_JOB_KEY));
         } catch (Exception e) {
             log.error(e.getMessage(), e);
         }

public/src/main/java/com/zhiweidata/automatictest/publics/BeanContainer.java

@@ -9,10 +9,17 @@ import org.apache.http.client.HttpClient;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.impl.client.HttpClients;
+import org.bson.codecs.configuration.CodecProvider;
+import org.bson.codecs.configuration.CodecRegistry;
+import org.bson.codecs.pojo.PojoCodecProvider;
 import org.quartz.Scheduler;
 import org.quartz.SchedulerException;
 import org.quartz.impl.StdSchedulerFactory;
 
+import static com.mongodb.MongoClientSettings.getDefaultCodecRegistry;
+import static org.bson.codecs.configuration.CodecRegistries.fromProviders;
+import static org.bson.codecs.configuration.CodecRegistries.fromRegistries;
+
 /**
  * Bean容器
  *
@@ -66,6 +73,9 @@ public class BeanContainer {
         HTTP_CLIENT = clientBuilder.build();
 
         MONGO_CLIENT = MongoClients.create("mongodb://admin:z199809051593@192.168.0.119");
-        MONGO_DATABASE = MONGO_CLIENT.getDatabase("automatic_test");
+        // 启用 POJO 对象映射
+        CodecProvider pojoCodecProvider = PojoCodecProvider.builder().automatic(true).build();
+        CodecRegistry pojoCodecRegistry = fromRegistries(getDefaultCodecRegistry(), fromProviders(pojoCodecProvider));
+        MONGO_DATABASE = MONGO_CLIENT.getDatabase("automatic_test").withCodecRegistry(pojoCodecRegistry);
     }
 }