建立弹幕导出基本框架

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/config/BaseConfig.java

@@ -18,4 +18,16 @@ public class BaseConfig {
      * 采集任务接口
      */
     public static final String TASK_URL = BASE_URL + "tasks/";
+    /**
+     * 弹幕采集任务的 map 的 key
+     */
+    public static final String BARRAGE_COLLECTION_TASK_JOB_KEY = "BARRAGE_COLLECTION_TASK_JOB_KEY";
+    /**
+     * 最大采集时间，单位：分钟
+     */
+    public static final int MAX_COLLECTION_TIME = 300;
+    /**
+     * 弹幕的excel导出服务器测试，最多延迟 X 天
+     */
+    public static final int EXPORT_START_MAX_DELAY = 7;
 }

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/BiliBiliCreateTaskJob.java

 package com.zhiweidata.automatictest.barragecrawlerserver.createtasks;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
+import com.zhiweidata.automatictest.barragecrawlerserver.exporttasks.BiliBiliExportTaskJob;
 import java.net.URL;
 import java.util.List;
 import java.util.Map;
@@ -36,7 +38,8 @@ public class BiliBiliCreateTaskJob extends CreateTaskJob {
                 roomList.forEach(room -> {
                     try {
                         Map<?, ?> roomMap = (Map<?, ?>) room;
-                        BiliBiliCreateTaskJob.super.createCollectionTask(baseUrl + roomMap.get("link"), currentTime);
+                        BarrageCollectionTask collectionTask = BiliBiliCreateTaskJob.super.createCollectionTask(baseUrl + roomMap.get("link"), currentTime);
+                        BiliBiliCreateTaskJob.super.createExportTask(collectionTask, BiliBiliExportTaskJob.class);
                     } catch (Exception e) {
                         log.error(e.getMessage(), e);
                     }

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/CreateTaskJob.java

 package com.zhiweidata.automatictest.barragecrawlerserver.createtasks;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.mongodb.client.MongoCollection;
 import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
+import com.zhiweidata.automatictest.barragecrawlerserver.exporttasks.BarrageExportTaskJob;
+import com.zhiweidata.automatictest.barragecrawlerserver.util.TimeUtil;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
@@ -17,12 +18,18 @@ import org.apache.http.entity.ContentType;
 import org.bson.Document;
 import org.jetbrains.annotations.NotNull;
 import org.quartz.Job;
+import org.quartz.JobBuilder;
+import org.quartz.SchedulerException;
+import org.quartz.Trigger;
+import org.quartz.TriggerBuilder;
 
+import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.EXPORT_START_MAX_DELAY;
 import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.MONGO_LIVE_COLLECTION;
 import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.TASK_URL;
 import static com.zhiweidata.automatictest.publics.BeanContainer.HTTP_CLIENT;
 import static com.zhiweidata.automatictest.publics.BeanContainer.MONGO_DATABASE;
 import static com.zhiweidata.automatictest.publics.BeanContainer.STANDARD_DATE_FORMAT;
+import static com.zhiweidata.automatictest.publics.BeanContainer.SCHEDULER;
 
 /**
  * @author aszswaz
@@ -64,4 +71,22 @@ public abstract class CreateTaskJob implements Job {
             liveUrls.insertMany(this.liveUrls);
         }
     }
+
+    /**
+     * 创建定时导出任务
+     */
+    protected final <T extends BarrageExportTaskJob> void createExportTask(BarrageCollectionTask collectionTask, Class<T> tClass) throws SchedulerException {
+        JobBuilder jobBuilder = JobBuilder.newJob(tClass);
+        String name = tClass.getSimpleName(), group = "export";
+        jobBuilder.withIdentity(name, group);
+        jobBuilder.withDescription("弹幕导出，以及导出的excel文件的分析");
+
+        TriggerBuilder<Trigger> triggerBuilder = TriggerBuilder.newTrigger();
+        triggerBuilder.withIdentity(name, group);
+        triggerBuilder.withDescription("弹幕导出，以及导出的excel文件的分析");
+        // 在未来 N 天内，随机一个时间启动弹幕的导出测试任务
+        triggerBuilder.startAt(TimeUtil.randmoDate(EXPORT_START_MAX_DELAY));
+
+        SCHEDULER.scheduleJob(jobBuilder.build(), triggerBuilder.build());
+    }
 }

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/DouyuCreateTaskJob.java

@@ -2,7 +2,9 @@ package com.zhiweidata.automatictest.barragecrawlerserver.createtasks;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
 import com.zhiweidata.automatictest.barragecrawlerserver.exception.BarrageHomePageException;
+import com.zhiweidata.automatictest.barragecrawlerserver.exporttasks.DouyuExportTaskJob;
 import java.net.URL;
 import java.util.HashSet;
 import java.util.List;
@@ -49,7 +51,8 @@ public class DouyuCreateTaskJob extends CreateTaskJob {
             final long currentTime = System.currentTimeMillis();
             liveUrls.forEach(liveUrl -> {
                 try {
-                    super.createCollectionTask(liveUrl, currentTime);
+                    BarrageCollectionTask collectionTask = super.createCollectionTask(liveUrl, currentTime);
+                    super.createExportTask(collectionTask, DouyuExportTaskJob.class);
                 } catch (Exception e) {
                     log.error(e.getMessage(), e);
                 }

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/HuyaCreateTaskJob.java

@@ -3,7 +3,9 @@ package com.zhiweidata.automatictest.barragecrawlerserver.createtasks;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.mongodb.client.MongoCollection;
+import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
 import com.zhiweidata.automatictest.barragecrawlerserver.exception.BarrageHomePageException;
+import com.zhiweidata.automatictest.barragecrawlerserver.exporttasks.HuyaExportTaskJob;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -104,7 +106,8 @@ public class HuyaCreateTaskJob extends CreateTaskJob {
         final long currentTime = System.currentTimeMillis();
         liveUrls.forEach(liveUrl -> {
             try {
-                super.createCollectionTask(liveUrl, currentTime);
+                BarrageCollectionTask collectionTask = super.createCollectionTask(liveUrl, currentTime);
+                super.createExportTask(collectionTask, HuyaExportTaskJob.class);
             } catch (Exception e) {
                 log.error(e.getMessage(), e);
             }

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/entity/BarrageCollectionTask.java

@@ -2,12 +2,13 @@ package com.zhiweidata.automatictest.barragecrawlerserver.entity;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import java.util.Calendar;
-import java.util.Random;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
 import lombok.Setter;
 import lombok.ToString;
 
+import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.MAX_COLLECTION_TIME;
+
 /**
  * @author aszswaz
  * @createTime 2021-08-24 15:53:39
@@ -52,12 +53,7 @@ public class BarrageCollectionTask {
         calendar.add(Calendar.HOUR_OF_DAY, 1);
 
         this.startTime = (int) (calendar.getTimeInMillis() / 1000);
-        Random random = new Random();
-        int minute;
-        do {
-            minute = random.nextInt(301);
-        } while (minute <= 0);
-
+        int minute = (int) (Math.random() * (MAX_COLLECTION_TIME + 1));
         calendar.add(Calendar.MINUTE, minute);
         this.endTime = (int) (calendar.getTimeInMillis() / 1000);
     }

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/BarrageExportTaskJob.java

+package com.zhiweidata.automatictest.barragecrawlerserver.exporttasks;
+
+import org.quartz.Job;
+
+/**
+ * 弹幕导出任务
+ *
+ * @author aszswaz
+ * @createTime 2021-08-25 17:29:39
+ * @ide IntelliJ IDEA
+ */
+@SuppressWarnings("JavaDoc")
+public interface BarrageExportTaskJob extends Job {
+}

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/BiliBiliExportTaskJob.java

+package com.zhiweidata.automatictest.barragecrawlerserver.exporttasks;
+
+import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
+import lombok.extern.slf4j.Slf4j;
+import org.quartz.JobExecutionContext;
+
+import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.BARRAGE_COLLECTION_TASK_JOB_KEY;
+import static java.util.Objects.requireNonNull;
+
+/**
+ * bilibili弹幕导出任务
+ *
+ * @author aszswaz
+ * @createTime 2021-08-25 17:30:18
+ * @ide IntelliJ IDEA
+ */
+@Slf4j
+@SuppressWarnings("JavaDoc")
+public class BiliBiliExportTaskJob implements BarrageExportTaskJob {
+    @Override
+    public void execute(JobExecutionContext context) {
+        try {
+            BarrageCollectionTask collectionTask = (BarrageCollectionTask) requireNonNull(context.get(BARRAGE_COLLECTION_TASK_JOB_KEY));
+        } catch (Exception e) {
+            log.error(e.getMessage(), e);
+        }
+    }
+}

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/DouyuExportTaskJob.java

+package com.zhiweidata.automatictest.barragecrawlerserver.exporttasks;
+
+import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
+import lombok.extern.slf4j.Slf4j;
+import org.quartz.JobExecutionContext;
+
+import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.BARRAGE_COLLECTION_TASK_JOB_KEY;
+import static java.util.Objects.requireNonNull;
+
+/**
+ * 斗鱼弹幕导出任务
+ *
+ * @author aszswaz
+ * @createTime 2021-08-25 17:30:18
+ * @ide IntelliJ IDEA
+ */
+@Slf4j
+@SuppressWarnings("JavaDoc")
+public class DouyuExportTaskJob implements BarrageExportTaskJob {
+    @Override
+    public void execute(JobExecutionContext context) {
+        try {
+            BarrageCollectionTask collectionTask = (BarrageCollectionTask) requireNonNull(context.get(BARRAGE_COLLECTION_TASK_JOB_KEY));
+        } catch (Exception e) {
+            log.error(e.getMessage(), e);
+        }
+    }
+}

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/HuyaExportTaskJob.java

+package com.zhiweidata.automatictest.barragecrawlerserver.exporttasks;
+
+import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
+import lombok.extern.slf4j.Slf4j;
+import org.quartz.JobExecutionContext;
+
+import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.BARRAGE_COLLECTION_TASK_JOB_KEY;
+import static java.util.Objects.requireNonNull;
+
+/**
+ * 虎牙弹幕导出任务
+ *
+ * @author aszswaz
+ * @createTime 2021-08-25 17:30:18
+ * @ide IntelliJ IDEA
+ */
+@Slf4j
+@SuppressWarnings("JavaDoc")
+public class HuyaExportTaskJob implements BarrageExportTaskJob {
+    @Override
+    public void execute(JobExecutionContext context) {
+        try {
+            BarrageCollectionTask collectionTask = (BarrageCollectionTask) requireNonNull(context.get(BARRAGE_COLLECTION_TASK_JOB_KEY));
+        } catch (Exception e) {
+            log.error(e.getMessage(), e);
+        }
+    }
+}

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/util/TimeUtil.java

+package com.zhiweidata.automatictest.barragecrawlerserver.util;
+
+import java.util.Date;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * 时间工具类
+ *
+ * @author aszswaz
+ * @createTime 2021-08-25 17:48:14
+ * @ide IntelliJ IDEA
+ */
+@SuppressWarnings("JavaDoc")
+public class TimeUtil {
+    /**
+     * 在未来的 X 天内，随机一个日期
+     *
+     * @param mostDay 未来的 X 天内
+     */
+    @NotNull
+    public static Date randmoDate(int mostDay) {
+        long openInterval = System.currentTimeMillis();
+        long closeInterval = openInterval + mostDay * 24 * 60 * 60 * 1000L;
+        long randmoTime = openInterval + (long) (Math.random() * (closeInterval - openInterval + 1));
+        return new Date(randmoTime);
+    }
+}

barrage-crawler-server/src/test/java/com/zhiweidata/automatictest/barragecrawlerserver/util/TimeUtilTest.java

+package com.zhiweidata.automatictest.barragecrawlerserver.util;
+
+import java.util.Date;
+import lombok.extern.slf4j.Slf4j;
+import org.junit.jupiter.api.Test;
+import static com.zhiweidata.automatictest.publics.BeanContainer.STANDARD_DATE_FORMAT;
+
+/**
+ * @author aszswaz
+ * @createTime 2021-08-25 17:59:09
+ * @ide IntelliJ IDEA
+ */
+@SuppressWarnings("JavaDoc")
+@Slf4j
+class TimeUtilTest {
+
+    @Test
+    void randmoDate() {
+        Date date = TimeUtil.randmoDate(7);
+        log.info(STANDARD_DATE_FORMAT.format(date));
+    }
+}
\ No newline at end of file

barrage-crawler-server/src/test/java/demo/RandmoDemo.java

+package demo;
+
+import org.junit.jupiter.api.Test;
+
+/**
+ * @author aszswaz
+ * @createTime 2021-08-25 18:08:23
+ * @ide IntelliJ IDEA
+ */
+@SuppressWarnings("JavaDoc")
+class RandmoDemo {
+    /**
+     * 随机 0 - 5 之间的数字
+     */
+    @Test
+    void randmo01() {
+        for (int i = 0; i < 100; i++) {
+            double randmo = Math.random();
+            System.out.println(randmo);
+            System.out.println(randmo * 5);
+            System.out.println("============================");
+        }
+    }
+
+    /**
+     * 随机 5 - 10 之间的数字
+     */
+    @Test
+    void randmo02() {
+        for (int i = 0; i < 100; i++) {
+            double randmo = Math.random();
+            System.out.println(randmo);
+            // + 1 表示需要包括 10
+            System.out.println(5 + randmo * (10 - 5 + 1));
+            System.out.println("============================");
+        }
+    }
+}