记录测试信息的主数据库改为mysql

.gitignore

@@ -5,3 +5,6 @@ target
 *.iml
 *.classpath
 *.log
+*.xlsx
+cache
+venv
\ No newline at end of file

barrage-crawler-server/ddl/BarrageCollectionTestTask.sql

+# 弹幕采集测试任务
+CREATE TABLE barrage_collection_test_task
+(
+    name        VARCHAR(255) NOT NULL PRIMARY KEY COMMENT '任务名称，32位UUID',
+    url         VARCHAR(255) NOT NULL COMMENT '直播间地址',
+    start_time  INT          NOT NULL COMMENT '任务开始时间',
+    end_time    INT          NOT NULL COMMENT '任务结束时间',
+    create_time TIMESTAMP(3) DEFAULT CURRENT_TIMESTAMP(3) COMMENT '任务创建时间'
+) CHARSET utf8mb4 COMMENT '弹幕采集服务自动测试任务';
+
+# 服务器响应信息
+CREATE TABLE server_response_message
+(
+    id                                BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
+    barrage_collection_test_task_name VARCHAR(255) NOT NULL COMMENT '任务名称，32位UUD',
+    code                              INT          NOT NULL COMMENT '响应代码',
+    message                           VARCHAR(255) NOT NULL COMMENT '响应信息',
+    create_time                       TIMESTAMP(3) DEFAULT CURRENT_TIMESTAMP(3),
+    FOREIGN KEY (barrage_collection_test_task_name)
+        REFERENCES barrage_collection_test_task (name)
+        ON DELETE CASCADE ON UPDATE CASCADE
+) CHARSET utf8mb4 COMMENT '服务器响应信息';
\ No newline at end of file

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/config/BaseConfig.java

 package com.zhiweidata.automatictest.barragecrawlerserver.config;
 
-import com.mongodb.client.MongoCollection;
-import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
 import java.io.File;
 import lombok.extern.slf4j.Slf4j;
 
-import static com.zhiweidata.automatictest.publics.BeanContainer.MONGO_DATABASE;
-
 /**
  * 基本配置
  *
@@ -38,14 +34,6 @@ public class BaseConfig {
      */
     public static final int EXPORT_START_MAX_DELAY = 7;
     /**
-     * 模块名称
-     */
-    public static final String MODULE_NAME = "barrage_crawler_server";
-    /**
-     * 测试采集任务的信息集合
-     */
-    public static final MongoCollection<BarrageCollectionTask> MONGO_TASK_REQUEST_COLLECTION = MONGO_DATABASE.getCollection(MODULE_NAME + "_task_requests", BarrageCollectionTask.class);
-    /**
      * 文件缓存文件夹
      */
     public static final File CACHE_DIR = new File("cache");

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/BiliBiliCreateTaskJob.java

 package com.zhiweidata.automatictest.barragecrawlerserver.createtasks;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
+import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask;
 import com.zhiweidata.automatictest.barragecrawlerserver.exporttasks.BiliBiliExportTaskJob;
 import java.net.URL;
 import java.util.List;
@@ -38,7 +38,7 @@ public class BiliBiliCreateTaskJob extends CreateTaskJob {
                 roomList.forEach(room -> {
                     try {
                         Map<?, ?> roomMap = (Map<?, ?>) room;
-                        BarrageCollectionTask collectionTask = BiliBiliCreateTaskJob.super.createCollectionTask(baseUrl + roomMap.get("link"), currentTime);
+                        BarrageCollectionTestTask collectionTask = BiliBiliCreateTaskJob.super.createCollectionTask(baseUrl + roomMap.get("link"), currentTime);
                         BiliBiliCreateTaskJob.super.createExportTask(collectionTask, BiliBiliExportTaskJob.class);
                     } catch (Exception e) {
                         log.error(e.getMessage(), e);

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/CreateTaskJob.java

 package com.zhiweidata.automatictest.barragecrawlerserver.createtasks;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
+import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask;
+import com.zhiweidata.automatictest.barragecrawlerserver.entity.ServerResponseMessage;
 import com.zhiweidata.automatictest.barragecrawlerserver.exporttasks.BarrageExportTaskJob;
+import com.zhiweidata.automatictest.barragecrawlerserver.mapper.BarrageCollectionTestTaskMapper;
+import com.zhiweidata.automatictest.barragecrawlerserver.mapper.ServerResponseMessageMapper;
 import com.zhiweidata.automatictest.barragecrawlerserver.util.TimeUtil;
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
 import java.util.UUID;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.entity.ByteArrayEntity;
 import org.apache.http.entity.ContentType;
+import org.apache.ibatis.session.SqlSession;
 import org.jetbrains.annotations.NotNull;
 import org.quartz.Job;
 import org.quartz.JobBuilder;
@@ -24,10 +25,10 @@ import org.quartz.TriggerBuilder;
 
 import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.BARRAGE_COLLECTION_TASK_JOB_KEY;
 import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.EXPORT_START_MAX_DELAY;
-import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.MONGO_TASK_REQUEST_COLLECTION;
 import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.TASK_URL;
 import static com.zhiweidata.automatictest.publics.BeanContainer.HTTP_CLIENT;
 import static com.zhiweidata.automatictest.publics.BeanContainer.SCHEDULER;
+import static com.zhiweidata.automatictest.publics.BeanContainer.SQL_SESSION_FACTORY;
 import static com.zhiweidata.automatictest.publics.BeanContainer.STANDARD_DATE_FORMAT;
 
 /**
@@ -42,9 +43,9 @@ public abstract class CreateTaskJob implements Job {
      * 请求弹幕采集服务器创建采集任务
      */
     @NotNull
-    protected final BarrageCollectionTask createCollectionTask(String liveUrl, long currentTime) throws IOException {
+    protected final BarrageCollectionTestTask createCollectionTask(String liveUrl, long currentTime) throws IOException {
         final String liveName = UUID.randomUUID().toString();
-        BarrageCollectionTask collectionTask = new BarrageCollectionTask(liveUrl, liveName, currentTime);
+        BarrageCollectionTestTask collectionTask = new BarrageCollectionTestTask(liveUrl, liveName, currentTime);
         if (log.isDebugEnabled()) {
             String startTime = STANDARD_DATE_FORMAT.format(collectionTask.getStartTime() * 1000L);
             String endTime = STANDARD_DATE_FORMAT.format(collectionTask.getEndTime() * 1000L);
@@ -54,18 +55,22 @@ public abstract class CreateTaskJob implements Job {
         ObjectMapper objectMapper = new ObjectMapper();
         post.setEntity(new ByteArrayEntity(objectMapper.writeValueAsBytes(collectionTask), ContentType.APPLICATION_JSON));
         HttpResponse response = HTTP_CLIENT.execute(post);
-        Map<?, ?> jsonMap = objectMapper.readValue(response.getEntity().getContent(), Map.class);
+        ServerResponseMessage responseMessage = objectMapper.readValue(response.getEntity().getContent(), ServerResponseMessage.class);
+        responseMessage.setName(collectionTask.getName());
 
-        collectionTask.setCode((Integer) jsonMap.get("code"));
-        collectionTask.setMessage((String) jsonMap.get("message"));
-        MONGO_TASK_REQUEST_COLLECTION.insertOne(collectionTask);
+        try (SqlSession session = SQL_SESSION_FACTORY.openSession(true)) {
+            BarrageCollectionTestTaskMapper testTaskMapper = session.getMapper(BarrageCollectionTestTaskMapper.class);
+            testTaskMapper.insert(collectionTask);
+            ServerResponseMessageMapper messageMapper = session.getMapper(ServerResponseMessageMapper.class);
+            messageMapper.insert(responseMessage);
+        }
         return collectionTask;
     }
 
     /**
      * 创建定时导出任务
      */
-    protected final <T extends BarrageExportTaskJob> void createExportTask(@NotNull BarrageCollectionTask collectionTask, Class<T> tClass) throws SchedulerException {
+    protected final <T extends BarrageExportTaskJob> void createExportTask(@NotNull BarrageCollectionTestTask collectionTask, Class<T> tClass) throws SchedulerException {
         JobBuilder jobBuilder = JobBuilder.newJob(tClass);
         String name = collectionTask.getName(), group = tClass.getSimpleName();
         jobBuilder.withIdentity(name, group);

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/DouyuCreateTaskJob.java

@@ -2,7 +2,7 @@ package com.zhiweidata.automatictest.barragecrawlerserver.createtasks;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
+import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask;
 import com.zhiweidata.automatictest.barragecrawlerserver.exception.BarrageHomePageException;
 import com.zhiweidata.automatictest.barragecrawlerserver.exporttasks.DouyuExportTaskJob;
 import java.util.HashSet;
@@ -44,7 +44,7 @@ public class DouyuCreateTaskJob extends CreateTaskJob {
             final long currentTime = System.currentTimeMillis();
             liveUrls.forEach(liveUrl -> {
                 try {
-                    BarrageCollectionTask collectionTask = super.createCollectionTask(liveUrl, currentTime);
+                    BarrageCollectionTestTask collectionTask = super.createCollectionTask(liveUrl, currentTime);
                     super.createExportTask(collectionTask, DouyuExportTaskJob.class);
                 } catch (Exception e) {
                     log.error(e.getMessage(), e);

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/HuyaCreateTaskJob.java

@@ -2,7 +2,7 @@ package com.zhiweidata.automatictest.barragecrawlerserver.createtasks;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
+import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask;
 import com.zhiweidata.automatictest.barragecrawlerserver.exception.BarrageHomePageException;
 import com.zhiweidata.automatictest.barragecrawlerserver.exporttasks.HuyaExportTaskJob;
 import java.util.HashSet;
@@ -96,7 +96,7 @@ public class HuyaCreateTaskJob extends CreateTaskJob {
         final long currentTime = System.currentTimeMillis();
         liveUrls.forEach(liveUrl -> {
             try {
-                BarrageCollectionTask collectionTask = super.createCollectionTask(liveUrl, currentTime);
+                BarrageCollectionTestTask collectionTask = super.createCollectionTask(liveUrl, currentTime);
                 super.createExportTask(collectionTask, HuyaExportTaskJob.class);
             } catch (Exception e) {
                 log.error(e.getMessage(), e);

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/entity/BarrageCollectionTask.java → barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/entity/BarrageCollectionTestTask.java

 package com.zhiweidata.automatictest.barragecrawlerserver.entity;
 
-import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import java.util.Calendar;
 import lombok.EqualsAndHashCode;
@@ -11,16 +10,18 @@ import lombok.ToString;
 import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.MAX_COLLECTION_TIME;
 
 /**
+ * 弹幕采集测试任务
+ *
  * @author aszswaz
  * @createTime 2021-08-24 15:53:39
  * @ide IntelliJ IDEA
  */
-@SuppressWarnings("JavaDoc")
+@SuppressWarnings({"JavaDoc", "GrazieInspection"})
 @Getter
 @Setter
 @EqualsAndHashCode
 @ToString
-public class BarrageCollectionTask {
+public class BarrageCollectionTestTask {
     /**
      * 任务名称
      */
@@ -41,18 +42,8 @@ public class BarrageCollectionTask {
      */
     @JsonProperty(value = "endTime")
     private Integer endTime;
-    /**
-     * 服务器的响应代码
-     */
-    @JsonIgnore
-    private int code;
-    /**
-     * 响应信息
-     */
-    @JsonIgnore
-    private String message;
 
-    public BarrageCollectionTask(String liveUrl, String name, long currentTime) {
+    public BarrageCollectionTestTask(String liveUrl, String name, long currentTime) {
         this.url = liveUrl;
         this.name = name;
 

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/entity/BarrageTaskResponse.java → barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/entity/BarrageExportInfo.java

@@ -7,33 +7,63 @@ import java.io.Serializable;
 import lombok.Data;
 
 /**
- * 弹幕信息实体类
+ * 弹幕导出信息
  *
  * @author aszswaz
  * @createTime 2021-09-06 11:40:09
  * @ide IntelliJ IDEA
  */
-@SuppressWarnings("JavaDoc")
+@SuppressWarnings({"JavaDoc", "GrazieInspection"})
 @Data
-public class BarrageTaskResponse implements Serializable {
+public class BarrageExportInfo implements Serializable {
+    /**
+     * 任务ID
+     */
     @JsonProperty(value = "id")
     private String id;
+    /**
+     * 任务名称
+     */
     @JsonProperty(value = "name")
     private String name;
+    /**
+     * 直播间地址
+     */
     @JsonProperty(value = "url")
     private String url;
+    /**
+     * 直播间ID
+     */
     @JsonProperty(value = "roomId")
     private String roomId;
+    /**
+     * 任务开始时间
+     */
     @JsonProperty(value = "startTime")
     private int startTime;
+    /**
+     * 任务结束时间
+     */
     @JsonProperty(value = "endTime")
     private int endTime;
+    /**
+     * 任务执行状态
+     */
     @JsonProperty(value = "status")
     private BarrageStatusEnums status;
+    /**
+     * 直播间平台
+     */
     @JsonProperty(value = "platform")
     private PlatformEnum platform;
+    /**
+     * 任务创建时间
+     */
     @JsonProperty(value = "createTime")
     private int createTime;
+    /**
+     * 弹幕采集量
+     */
     @JsonProperty(value = "collectionCount")
     private int collectionCount;
 }

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/entity/ServerResponseMessage.java

+package com.zhiweidata.automatictest.barragecrawlerserver.entity;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Data;
+
+/**
+ * 服务器响应信息
+ *
+ * @author aszswaz
+ * @createTime 2021-09-09 14:18:29
+ * @ide IntelliJ IDEA
+ */
+@SuppressWarnings("JavaDoc")
+@Data
+public class ServerResponseMessage {
+    @JsonIgnore
+    private String name;
+    @JsonProperty(value = "code")
+    private int code;
+    @JsonProperty(value = "message")
+    private String message;
+}

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/BarrageExportTaskJob.java

@@ -14,6 +14,7 @@ import org.slf4j.LoggerFactory;
 
 import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.CACHE_DIR;
 import static java.util.Objects.isNull;
+import static java.util.Objects.nonNull;
 import static java.util.Objects.requireNonNull;
 
 /**
@@ -32,9 +33,8 @@ public interface BarrageExportTaskJob extends Job {
      *
      * @return 解压得到的文件
      */
-    default List<File> write(String taskId, InputStream inputStream) throws IOException {
-        final File outDir = new File(CACHE_DIR, taskId);
-        if (!outDir.exists()) if (!outDir.mkdirs()) log.error("文件夹{}创建失败", outDir.getName());
+    default List<File> write(File cacheDir, InputStream inputStream) throws IOException {
+        if (!cacheDir.exists()) if (!cacheDir.mkdirs()) log.error("文件夹{}创建失败", cacheDir.getName());
         final List<File> outFiles = new ArrayList<>();
 
         try (ZipInputStream zipInputStream = new ZipInputStream(inputStream)) {
@@ -43,7 +43,7 @@ public interface BarrageExportTaskJob extends Job {
             while (true) {
                 ZipEntry zipEntry = zipInputStream.getNextEntry();
                 if (isNull(zipEntry)) break;
-                File execFile = new File(requireNonNull(zipEntry.getName()));
+                File execFile = new File(cacheDir, requireNonNull(zipEntry.getName()));
                 try (FileOutputStream fileOutputStream = new FileOutputStream(execFile)) {
                     while ((len = zipInputStream.read(buff)) != -1) fileOutputStream.write(buff, 0, len);
                 }
@@ -53,4 +53,19 @@ public interface BarrageExportTaskJob extends Job {
         }
         return outFiles;
     }
+
+    /**
+     * 删除文件
+     */
+    default void removeFile(File file) {
+        // 如果是文件夹，先删除文件夹中的文件
+        if (file.isDirectory()) {
+            File[] files = file.listFiles();
+            if (nonNull(files)) {
+                for (File subFile : files) this.removeFile(subFile);
+            }
+        }
+        // 删除文件夹或文件
+        if (!file.delete()) log.error("文件夹：{}删除失败", file.getName());
+    }
 }

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/BiliBiliExportTaskJob.java

@@ -3,8 +3,8 @@ package com.zhiweidata.automatictest.barragecrawlerserver.exporttasks;
 import com.alibaba.excel.EasyExcel;
 import com.alibaba.excel.event.SyncReadListener;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
-import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageTaskResponse;
+import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask;
+import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageExportInfo;
 import java.io.File;
 import java.io.IOException;
 import java.util.List;
@@ -20,7 +20,7 @@ import org.quartz.JobExecutionContext;
 import org.quartz.SchedulerException;
 
 import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.BARRAGE_COLLECTION_TASK_JOB_KEY;
-import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.MONGO_TASK_REQUEST_COLLECTION;
+import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.CACHE_DIR;
 import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.TASK_URL;
 import static com.zhiweidata.automatictest.publics.BeanContainer.HTTP_CLIENT;
 import static java.util.Objects.requireNonNull;
@@ -47,7 +47,7 @@ public class BiliBiliExportTaskJob implements BarrageExportTaskJob {
     public void execute(JobExecutionContext context) {
         try {
             JobDataMap jobDataMap = requireNonNull(context.getMergedJobDataMap());
-            BarrageCollectionTask collectionTask = (BarrageCollectionTask) requireNonNull(jobDataMap.get(BARRAGE_COLLECTION_TASK_JOB_KEY));
+            BarrageCollectionTestTask collectionTask = (BarrageCollectionTestTask) requireNonNull(jobDataMap.get(BARRAGE_COLLECTION_TASK_JOB_KEY));
             this.export(collectionTask);
         } catch (Exception e) {
             log.error(e.getMessage(), e);
@@ -57,7 +57,7 @@ public class BiliBiliExportTaskJob implements BarrageExportTaskJob {
     /**
      * 尝试导出弹幕
      */
-    public void export(@NotNull BarrageCollectionTask collectionTask) throws IOException, SchedulerException {
+    public void export(@NotNull BarrageCollectionTestTask collectionTask) throws IOException, SchedulerException {
         String url = TASK_URL + "?limit=20&page=1&name=" + collectionTask.getName();
         HttpResponse response = HTTP_CLIENT.execute(new HttpGet(url));
 
@@ -71,14 +71,14 @@ public class BiliBiliExportTaskJob implements BarrageExportTaskJob {
 
         Map<?, ?> dataMap = (Map<?, ?>) requireNonNull(jsonMap.get("data"));
         List<?> dataList = (List<?>) requireNonNull(dataMap.get("list"));
-        BarrageTaskResponse taskResponse = jsonMapper.convertValue(dataList.get(0), BarrageTaskResponse.class);
+        BarrageExportInfo taskResponse = jsonMapper.convertValue(dataList.get(0), BarrageExportInfo.class);
         this.checkFile(collectionTask, taskResponse);
     }
 
     /**
      * 下载excel文件，并检查excel文件
      */
-    private void checkFile(@NotNull BarrageCollectionTask collectionTask, @NotNull BarrageTaskResponse taskResponse) throws IOException {
+    private void checkFile(@NotNull BarrageCollectionTestTask collectionTask, @NotNull BarrageExportInfo taskResponse) throws IOException {
         String url = TASK_URL + "/" + taskResponse.getId() + "/barrage";
         HttpGet get = new HttpGet(url);
         HttpResponse response = HTTP_CLIENT.execute(get);
@@ -88,20 +88,20 @@ public class BiliBiliExportTaskJob implements BarrageExportTaskJob {
             Map<?, ?> jsonMap = jsonMapper.readValue(response.getEntity().getContent(), Map.class);
             String message = (String) jsonMap.get("message");
             log.info(message);
-            collectionTask.setCode((Integer) jsonMap.get("code"));
-            collectionTask.setMessage(message);
-            MONGO_TASK_REQUEST_COLLECTION.insertOne(collectionTask);
         } else {
-            List<File> excelFiles = this.write(taskResponse.getId(), response.getEntity().getContent());
+            final File cacheDir = new File(CACHE_DIR, taskResponse.getId());
+            List<File> excelFiles = this.write(cacheDir, response.getEntity().getContent());
             // 检查文件中的数据量
-            excelFiles.forEach(excelFile -> {
+            for (File excelFile : excelFiles) {
                 List<Map<Integer, String>> dataList = EasyExcel.read(excelFile, new SyncReadListener()).doReadAllSync();
                 if (dataList.isEmpty()) {
                     log.error("任务ID：{}，文件：{}为空", taskResponse.getId(), excelFile.getName());
                     return;
                 }
                 log.info("任务ID：{},文件：{}弹幕数量：{}", taskResponse.getId(), excelFile.getName(), dataList.size());
-            });
+            }
+            // 删除文件
+            this.removeFile(cacheDir);
         }
     }
 }

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/DouyuExportTaskJob.java

 package com.zhiweidata.automatictest.barragecrawlerserver.exporttasks;
 
-import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
+import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask;
 import lombok.extern.slf4j.Slf4j;
 import org.quartz.JobDataMap;
 import org.quartz.JobExecutionContext;
@@ -22,7 +22,7 @@ public class DouyuExportTaskJob implements BarrageExportTaskJob {
     public void execute(JobExecutionContext context) {
         try {
             JobDataMap dataMap = requireNonNull(context.getMergedJobDataMap());
-            BarrageCollectionTask collectionTask = (BarrageCollectionTask) requireNonNull(dataMap.get(BARRAGE_COLLECTION_TASK_JOB_KEY));
+            BarrageCollectionTestTask collectionTask = (BarrageCollectionTestTask) requireNonNull(dataMap.get(BARRAGE_COLLECTION_TASK_JOB_KEY));
         } catch (Exception e) {
             log.error(e.getMessage(), e);
         }

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/HuyaExportTaskJob.java

 package com.zhiweidata.automatictest.barragecrawlerserver.exporttasks;
 
-import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
+import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask;
 import lombok.extern.slf4j.Slf4j;
 import org.quartz.JobDataMap;
 import org.quartz.JobExecutionContext;
@@ -22,7 +22,7 @@ public class HuyaExportTaskJob implements BarrageExportTaskJob {
     public void execute(JobExecutionContext context) {
         try {
             JobDataMap dataMap = requireNonNull(context.getMergedJobDataMap());
-            BarrageCollectionTask collectionTask = (BarrageCollectionTask) requireNonNull(dataMap.get(BARRAGE_COLLECTION_TASK_JOB_KEY));
+            BarrageCollectionTestTask collectionTask = (BarrageCollectionTestTask) requireNonNull(dataMap.get(BARRAGE_COLLECTION_TASK_JOB_KEY));
         } catch (Exception e) {
             log.error(e.getMessage(), e);
         }

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/mapper/BarrageCollectionTestTaskMapper.java

+package com.zhiweidata.automatictest.barragecrawlerserver.mapper;
+
+import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask;
+import org.apache.ibatis.annotations.Param;
+
+/**
+ * 测试任务信息Mapper
+ *
+ * @author aszswaz
+ * @createTime 2021-09-09 11:31:16
+ * @ide IntelliJ IDEA
+ */
+@SuppressWarnings("JavaDoc")
+public interface BarrageCollectionTestTaskMapper {
+    void insert(
+            @Param(value = "task") BarrageCollectionTestTask testTask
+    );
+}

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/mapper/ServerResponseMessageMapper.java

+package com.zhiweidata.automatictest.barragecrawlerserver.mapper;
+
+import com.zhiweidata.automatictest.barragecrawlerserver.entity.ServerResponseMessage;
+import org.apache.ibatis.annotations.Param;
+
+/**
+ * 服务器响应信息存储
+ *
+ * @author aszswaz
+ * @createTime 2021-09-09 14:23:25
+ * @ide IntelliJ IDEA
+ */
+@SuppressWarnings("JavaDoc")
+public interface ServerResponseMessageMapper {
+    void insert(@Param(value = "message") ServerResponseMessage responseMessage);
+}

barrage-crawler-server/src/main/resources/mappers/BarrageCollectionTestTaskMapper.xml

+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE mapper
+        PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
+        "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.zhiweidata.automatictest.barragecrawlerserver.mapper.BarrageCollectionTestTaskMapper">
+    <resultMap id="BarrageCollectionTask"
+               type="com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask">
+        <result column="name" property="name"/>
+        <result column="url" property="url"/>
+        <result column="start_time" property="startTime"/>
+        <result column="end_time" property="endTime"/>
+    </resultMap>
+
+    <insert id="insert">
+        INSERT INTO barrage_collection_test_task (name, url, start_time, end_time)
+            VALUE (#{task.name}, #{task.url}, #{task.startTime}, #{task.endTime})
+    </insert>
+</mapper>
\ No newline at end of file

barrage-crawler-server/src/main/resources/mappers/ServerResponseMessageMapper.xml

+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE mapper
+        PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
+        "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.zhiweidata.automatictest.barragecrawlerserver.mapper.ServerResponseMessageMapper">
+    <resultMap id="ServerResponseMessage"
+               type="com.zhiweidata.automatictest.barragecrawlerserver.entity.ServerResponseMessage">
+        <result property="name" column="name"/>
+        <result property="code" column="code"/>
+        <result property="message" column="message"/>
+    </resultMap>
+
+    <insert id="insert">
+        INSERT INTO server_response_message(barrage_collection_test_task_name, code, message)
+            VALUE (#{message.name}, #{message.code}, #{message.message})
+    </insert>
+</mapper>
\ No newline at end of file

barrage-crawler-server/src/test/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/BiliBiliExportTaskJobTest.java

 package com.zhiweidata.automatictest.barragecrawlerserver.exporttasks;
 
-import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
+import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask;
 import java.io.IOException;
 import org.quartz.SchedulerException;
 
+import static com.zhiweidata.automatictest.publics.BeanContainer.SCHEDULER;
+
 /**
  * @author aszswaz
  * @createTime 2021-09-06 10:03:42
@@ -13,11 +15,12 @@ import org.quartz.SchedulerException;
 class BiliBiliExportTaskJobTest {
 
     public static void main(String[] args) throws SchedulerException, IOException {
-        BarrageCollectionTask collectionTask = new BarrageCollectionTask(
-                "https://live.bilibili.com/21544906?hotRank=0", "6783d422-840f-4230-9875-81ee91c06d09", System.currentTimeMillis()
+        BarrageCollectionTestTask collectionTask = new BarrageCollectionTestTask(
+                "https://live.bilibili.com/1440094?hotRank=0", "a9e30479-e231-46f3-991c-a8858400678f", System.currentTimeMillis()
         );
 
         BiliBiliExportTaskJob exportTaskJob = new BiliBiliExportTaskJob();
         exportTaskJob.export(collectionTask);
+        SCHEDULER.shutdown(true);
     }
 }
\ No newline at end of file

public/pom.xml

@@ -43,11 +43,23 @@
             <version>${quartz.version}</version>
         </dependency>
 
-        <!--mongodb client-->
+        <!-- https://mvnrepository.com/artifact/org.mybatis/mybatis -->
         <dependency>
-            <groupId>org.mongodb</groupId>
-            <artifactId>mongodb-driver-sync</artifactId>
-            <version>${mongodb.version}</version>
+            <groupId>org.mybatis</groupId>
+            <artifactId>mybatis</artifactId>
+            <version>3.5.7</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/mysql/mysql-connector-java -->
+        <dependency>
+            <groupId>mysql</groupId>
+            <artifactId>mysql-connector-java</artifactId>
+            <version>8.0.26</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/com.zaxxer/HikariCP -->
+        <dependency>
+            <groupId>com.zaxxer</groupId>
+            <artifactId>HikariCP</artifactId>
+            <version>5.0.0</version>
         </dependency>
     </dependencies>
 

public/src/main/java/com/zhiweidata/automatictest/publics/BeanContainer.java

 package com.zhiweidata.automatictest.publics;
 
-import com.mongodb.client.MongoClient;
-import com.mongodb.client.MongoClients;
-import com.mongodb.client.MongoDatabase;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.time.FastDateFormat;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.impl.client.HttpClients;
-import org.bson.codecs.configuration.CodecProvider;
-import org.bson.codecs.configuration.CodecRegistry;
-import org.bson.codecs.pojo.PojoCodecProvider;
+import org.apache.ibatis.builder.xml.XMLConfigBuilder;
+import org.apache.ibatis.builder.xml.XMLMapperBuilder;
+import org.apache.ibatis.io.Resources;
+import org.apache.ibatis.session.Configuration;
+import org.apache.ibatis.session.SqlSessionFactory;
+import org.apache.ibatis.session.SqlSessionFactoryBuilder;
+import org.jetbrains.annotations.NotNull;
 import org.quartz.Scheduler;
 import org.quartz.SchedulerException;
 import org.quartz.impl.StdSchedulerFactory;
 
-import static com.mongodb.MongoClientSettings.getDefaultCodecRegistry;
-import static org.bson.codecs.configuration.CodecRegistries.fromProviders;
-import static org.bson.codecs.configuration.CodecRegistries.fromRegistries;
+import static java.util.Objects.requireNonNull;
 
 /**
  * Bean容器
@@ -39,17 +43,13 @@ public class BeanContainer {
      */
     public static final Scheduler SCHEDULER;
     /**
-     * mongodb 客户端
-     */
-    public static final MongoClient MONGO_CLIENT;
-    /**
-     * 整个项目专用数据库
-     */
-    public static final MongoDatabase MONGO_DATABASE;
-    /**
      * 标准日期格式化
      */
     public static final FastDateFormat STANDARD_DATE_FORMAT = FastDateFormat.getInstance("yyyy-MM-dd HH:mm:ss.SSS");
+    /**
+     * mybatis session工厂
+     */
+    public static final SqlSessionFactory SQL_SESSION_FACTORY;
 
     static {
 
@@ -71,11 +71,46 @@ public class BeanContainer {
         configBuilder.setRedirectsEnabled(true);
         clientBuilder.setDefaultRequestConfig(configBuilder.build());
         HTTP_CLIENT = clientBuilder.build();
+    }
+
+    static {
+        SqlSessionFactory sqlSessionFactory = null;
+        try (InputStream inputStream = Resources.getResourceAsStream("mybatis.xml")) {
+            SqlSessionFactoryBuilder builder = new SqlSessionFactoryBuilder();
+
+            // 先读取xml中的配置
+            XMLConfigBuilder configBuilder = new XMLConfigBuilder(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
+            configBuilder.parse();
+            Configuration configuration = configBuilder.getConfiguration();
+            // 扫描mapper
+            File mappersDir = Resources.getResourceAsFile("mappers");
+            scanMapper(mappersDir, configuration);
 
-        MONGO_CLIENT = MongoClients.create("mongodb://admin:z199809051593@192.168.0.119");
-        // 启用 POJO 对象映射
-        CodecProvider pojoCodecProvider = PojoCodecProvider.builder().automatic(true).build();
-        CodecRegistry pojoCodecRegistry = fromRegistries(getDefaultCodecRegistry(), fromProviders(pojoCodecProvider));
-        MONGO_DATABASE = MONGO_CLIENT.getDatabase("automatic_test").withCodecRegistry(pojoCodecRegistry);
+            sqlSessionFactory = builder.build(configuration);
+        } catch (Exception e) {
+            log.error(e.getMessage(), e);
+        }
+        SQL_SESSION_FACTORY = sqlSessionFactory;
+    }
+
+    /**
+     * 扫描mapper
+     */
+    private static void scanMapper(@NotNull File file, @NotNull Configuration configuration) throws IOException {
+        if (file.isDirectory()) {
+            File[] mappers = requireNonNull(file.listFiles());
+            for (File mapper : mappers) {
+                scanMapper(mapper, configuration);
+            }
+        } else {
+            String fileName = file.getName();
+            if (".xml".equals(fileName.substring(fileName.lastIndexOf(".")))) {
+                String resource = file.toPath().toUri().toString();
+                try (InputStream inputStream = Resources.getUrlAsStream(resource)) {
+                    XMLMapperBuilder mapperBuilder = new XMLMapperBuilder(inputStream, configuration, resource, configuration.getSqlFragments());
+                    mapperBuilder.parse();
+                }
+            }
+        }
     }
 }

public/src/main/java/com/zhiweidata/automatictest/publics/HikariDataSourceFactory.java

+package com.zhiweidata.automatictest.publics;
+
+import com.zaxxer.hikari.HikariConfig;
+import com.zaxxer.hikari.HikariDataSource;
+import java.util.Properties;
+import javax.sql.DataSource;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.ibatis.datasource.DataSourceFactory;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * mysql的数据库连接池Hikari工厂
+ *
+ * @author aszswaz
+ * @createTime 2021-09-09 10:41:07
+ * @ide IntelliJ IDEA
+ */
+@Slf4j
+@SuppressWarnings("JavaDoc")
+public class HikariDataSourceFactory implements DataSourceFactory {
+    /**
+     * 数据库连接池
+     */
+    private HikariDataSource dataSource;
+
+    @Override
+    public void setProperties(@NotNull Properties props) {
+        final String initSql = "SHOW DATABASES";
+        final long timeout = 60000;
+
+        HikariConfig config = new HikariConfig();
+        config.setAutoCommit(true);
+        config.setDriverClassName(props.getProperty("driver"));
+        config.setUsername(props.getProperty("username"));
+        config.setPassword(props.getProperty("password"));
+        config.setJdbcUrl(props.getProperty("url"));
+        config.setConnectionTestQuery(initSql);
+        config.setConnectionInitSql(initSql);
+        config.setConnectionTimeout(timeout);
+        config.setIdleTimeout(timeout << 1);
+        config.setInitializationFailTimeout(timeout);
+        config.setMaximumPoolSize(10);
+        config.setMinimumIdle(5);
+        config.setMaxLifetime(timeout << 2);
+        this.dataSource = new HikariDataSource(config);
+    }
+
+    @Override
+    public DataSource getDataSource() {
+        return this.dataSource;
+    }
+}

public/src/main/resources/mybatis.xml

+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE configuration
+        PUBLIC "-//mybatis.org//DTD Config 3.0//EN"
+        "http://mybatis.org/dtd/mybatis-3-config.dtd">
+<configuration>
+    <environments default="development">
+        <environment id="development">
+            <transactionManager type="JDBC"/>
+            <dataSource type="com.zhiweidata.automatictest.publics.HikariDataSourceFactory">
+                <property name="driver" value="com.mysql.cj.jdbc.Driver"/>
+                <property name="url" value="jdbc:mysql://192.168.0.119:3306/automatic_test?useSSL=FALSE&amp;serverTimezone=Asia/Shanghai"/>
+                <property name="username" value="root"/>
+                <property name="password" value="z199809051593"/>
+            </dataSource>
+        </environment>
+    </environments>
+</configuration>
\ No newline at end of file