完善BILIBILI弹幕导出测试

barrage-crawler-server/ddl/BarrageCollectionTestTask.sql

 # 弹幕采集测试任务
-CREATE TABLE barrage_collection_test_task
+CREATE TABLE barrage_crawler_server_barrage_collection_test_task
 (
     name        VARCHAR(255) NOT NULL PRIMARY KEY COMMENT '任务名称，32位UUID',
     url         VARCHAR(255) NOT NULL COMMENT '直播间地址',
@@ -9,11 +9,22 @@ CREATE TABLE barrage_collection_test_task
 ) CHARSET utf8mb4 COMMENT '弹幕采集服务自动测试任务';
 
 # 服务器响应信息
-CREATE TABLE server_response_message
+CREATE TABLE barrage_crawler_server_server_response_message
 (
     id                                BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
     barrage_collection_test_task_name VARCHAR(255) NOT NULL COMMENT '任务名称，32位UUD',
     code                              INT          NOT NULL COMMENT '响应代码',
     message                           VARCHAR(255) NOT NULL COMMENT '响应信息',
     create_time                       TIMESTAMP(3) DEFAULT CURRENT_TIMESTAMP(3)
-) CHARSET utf8mb4 COMMENT '服务器响应信息';
+) CHARSET utf8mb4 COMMENT '服务器响应信息';
\ No newline at end of file
+
+# 弹幕导出结果
+CREATE TABLE barrage_crawler_server_barrage_export_result
+(
+    id               BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
+    name             VARCHAR(255)    NOT NULL COMMENT '任务名称',
+    total_collection INT UNSIGNED    NOT NULL COMMENT '弹幕总采集量',
+    export_volume    INT UNSIGNED    NOT NULL COMMENT '导出的弹幕采集量',
+    percentage       DOUBLE UNSIGNED NOT NULL COMMENT '导出的弹幕数量/弹幕总采集量',
+    create_time      TIMESTAMP(3) DEFAULT CURRENT_TIMESTAMP(3)
+) CHARSET utf8mb4 COMMENT '弹幕导出结果';
\ No newline at end of file

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/entity/BarrageExportInfo.java → barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/entity/BarrageCollectionInfo.java

@@ -7,7 +7,7 @@ import java.io.Serializable;
 import lombok.Data;
 
 /**
- * 弹幕导出信息
+ * 弹幕详细信息
  *
  * @author aszswaz
  * @createTime 2021-09-06 11:40:09
@@ -15,7 +15,7 @@ import lombok.Data;
  */
 @SuppressWarnings({"JavaDoc", "GrazieInspection"})
 @Data
-public class BarrageExportInfo implements Serializable {
+public class BarrageCollectionInfo implements Serializable {
     /**
      * 任务ID
      */

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/entity/BarrageExportResult.java

+package com.zhiweidata.automatictest.barragecrawlerserver.entity;
+
+import lombok.Data;
+
+/**
+ * 弹幕导出结果
+ *
+ * @author aszswaz
+ * @createTime 2021-09-09 16:28:51
+ * @ide IntelliJ IDEA
+ */
+@SuppressWarnings("JavaDoc")
+@Data
+public class BarrageExportResult {
+    /**
+     * 任务名称
+     */
+    private String name;
+    /**
+     * 弹幕总采集量
+     */
+    private int totalCollection;
+    /**
+     * 导出的弹幕数量
+     */
+    private int exportVolume;
+    /**
+     * 导出的弹幕数量/弹幕总采集量
+     */
+    private double percentage;
+}

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/BarrageExportTaskJob.java

 package com.zhiweidata.automatictest.barragecrawlerserver.exporttasks;
 
+import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionInfo;
+import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageExportResult;
 import com.zhiweidata.automatictest.barragecrawlerserver.entity.ServerResponseMessage;
+import com.zhiweidata.automatictest.barragecrawlerserver.mapper.BarrageExportResultMapper;
+import com.zhiweidata.automatictest.barragecrawlerserver.mapper.ServerResponseMessageMapper;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -9,11 +13,13 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
+import org.apache.ibatis.session.SqlSession;
+import org.jetbrains.annotations.NotNull;
 import org.quartz.Job;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.CACHE_DIR;
+import static com.zhiweidata.automatictest.publics.BeanContainer.SQL_SESSION_FACTORY;
 import static java.util.Objects.isNull;
 import static java.util.Objects.nonNull;
 import static java.util.Objects.requireNonNull;
@@ -74,6 +80,28 @@ public interface BarrageExportTaskJob extends Job {
      * 保存服务器的响应信息
      */
     default void saveResponseMessage(ServerResponseMessage message) {
+        try (SqlSession session = SQL_SESSION_FACTORY.openSession(true)) {
+            ServerResponseMessageMapper messageMapper = session.getMapper(ServerResponseMessageMapper.class);
+            messageMapper.insert(message);
+        }
+    }
 
+    /**
+     * 保存弹幕的导出信息
+     *
+     * @param collectionInfo 弹幕采集任务信息
+     * @param exportCount    导出的弹幕数量
+     */
+    default void saveExportInfo(@NotNull BarrageCollectionInfo collectionInfo, int exportCount) {
+        try (SqlSession session = SQL_SESSION_FACTORY.openSession(true)) {
+            BarrageExportResult exportResult = new BarrageExportResult();
+            exportResult.setName(collectionInfo.getName());
+            exportResult.setExportVolume(exportCount);
+            exportResult.setTotalCollection(collectionInfo.getCollectionCount());
+            exportResult.setPercentage((double) exportCount / collectionInfo.getCollectionCount());
+
+            BarrageExportResultMapper resultMapper = session.getMapper(BarrageExportResultMapper.class);
+            resultMapper.insert(exportResult);
+        }
     }
 }

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/BiliBiliExportTaskJob.java

@@ -4,7 +4,8 @@ import com.alibaba.excel.EasyExcel;
 import com.alibaba.excel.event.SyncReadListener;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask;
-import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageExportInfo;
+import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionInfo;
+import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageExportResult;
 import com.zhiweidata.automatictest.barragecrawlerserver.entity.ServerResponseMessage;
 import java.io.File;
 import java.io.IOException;
@@ -72,15 +73,15 @@ public class BiliBiliExportTaskJob implements BarrageExportTaskJob {
 
         Map<?, ?> dataMap = (Map<?, ?>) requireNonNull(jsonMap.get("data"));
         List<?> dataList = (List<?>) requireNonNull(dataMap.get("list"));
-        BarrageExportInfo taskResponse = jsonMapper.convertValue(dataList.get(0), BarrageExportInfo.class);
-        this.checkFile(collectionTask, taskResponse);
+        BarrageCollectionInfo taskResponse = jsonMapper.convertValue(dataList.get(0), BarrageCollectionInfo.class);
+        this.checkFile(taskResponse);
     }
 
     /**
      * 下载excel文件，并检查excel文件
      */
-    private void checkFile(@NotNull BarrageCollectionTestTask collectionTask, @NotNull BarrageExportInfo taskResponse) throws IOException {
-        String url = TASK_URL + "/" + taskResponse.getId() + "/barrage";
+    private void checkFile(@NotNull BarrageCollectionInfo collectionInfo) throws IOException {
+        String url = TASK_URL + "/" + collectionInfo.getId() + "/barrage";
         HttpGet get = new HttpGet(url);
         HttpResponse response = HTTP_CLIENT.execute(get);
         String contentType = response.getFirstHeader("Content-Type").getValue();
@@ -88,22 +89,26 @@ public class BiliBiliExportTaskJob implements BarrageExportTaskJob {
             ObjectMapper jsonMapper = new ObjectMapper();
             ServerResponseMessage responseMessage = jsonMapper.readValue(response.getEntity().getContent(), ServerResponseMessage.class);
             log.info(responseMessage.getMessage());
-            responseMessage.setName(collectionTask.getName());
+            responseMessage.setName(collectionInfo.getName());
             this.saveResponseMessage(responseMessage);
         } else {
-            final File cacheDir = new File(CACHE_DIR, taskResponse.getId());
+            final File cacheDir = new File(CACHE_DIR, collectionInfo.getId());
             List<File> excelFiles = this.write(cacheDir, response.getEntity().getContent());
+            int exportCount = 0;
             // 检查文件中的数据量
             for (File excelFile : excelFiles) {
                 List<Map<Integer, String>> dataList = EasyExcel.read(excelFile, new SyncReadListener()).doReadAllSync();
                 if (dataList.isEmpty()) {
-                    log.error("任务ID：{}，文件：{}为空", taskResponse.getId(), excelFile.getName());
+                    log.error("任务ID：{}，文件：{}为空", collectionInfo.getId(), excelFile.getName());
                     return;
                 }
-                log.info("任务ID：{},文件：{}弹幕数量：{}", taskResponse.getId(), excelFile.getName(), dataList.size());
+                exportCount += dataList.size();
+                log.info("任务ID：{},文件：{}弹幕数量：{}", collectionInfo.getId(), excelFile.getName(), dataList.size());
             }
+            log.info("任务：{}，导出弹幕：{}，弹幕总采集量：{}", collectionInfo.getId(), exportCount, collectionInfo.getCollectionCount());
             // 删除文件
             this.removeFile(cacheDir);
+            this.saveExportInfo(collectionInfo, exportCount);
         }
     }
 }

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/mapper/BarrageExportResultMapper.java

+package com.zhiweidata.automatictest.barragecrawlerserver.mapper;
+
+import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageExportResult;
+import org.apache.ibatis.annotations.Param;
+
+/**
+ * 弹幕导出采集量保存
+ *
+ * @author aszswaz
+ * @createTime 2021-09-09 16:39:40
+ * @ide IntelliJ IDEA
+ */
+@SuppressWarnings("JavaDoc")
+public interface BarrageExportResultMapper {
+    void insert(@Param(value = "result") BarrageExportResult result);
+}

barrage-crawler-server/src/main/resources/mappers/BarrageCollectionTestTaskMapper.xml

@@ -12,7 +12,7 @@
     </resultMap>
 
     <insert id="insert">
-        INSERT INTO barrage_collection_test_task (name, url, start_time, end_time)
+        INSERT INTO barrage_crawler_server_barrage_collection_test_task (name, url, start_time, end_time)
             VALUE (#{task.name}, #{task.url}, #{task.startTime}, #{task.endTime})
     </insert>
 </mapper>
\ No newline at end of file

barrage-crawler-server/src/main/resources/mappers/BarrageExportResultMapper.xml

+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE mapper
+        PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
+        "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.zhiweidata.automatictest.barragecrawlerserver.mapper.BarrageExportResultMapper">
+    <insert id="insert">
+        INSERT INTO barrage_crawler_server_barrage_export_result(name, total_collection, export_volume, percentage)
+            VALUE (#{result.name}, #{result.totalCollection}, #{result.exportVolume}, #{result.percentage})
+    </insert>
+</mapper>
\ No newline at end of file

barrage-crawler-server/src/main/resources/mappers/ServerResponseMessageMapper.xml

@@ -11,7 +11,7 @@
     </resultMap>
 
     <insert id="insert">
-        INSERT INTO server_response_message(barrage_collection_test_task_name, code, message)
+        INSERT INTO barrage_crawler_server_server_response_message(barrage_collection_test_task_name, code, message)
             VALUE (#{message.name}, #{message.code}, #{message.message})
     </insert>
 </mapper>
\ No newline at end of file

barrage-crawler-server/src/test/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/BiliBiliExportTaskJobTest.java

@@ -16,7 +16,7 @@ class BiliBiliExportTaskJobTest {
 
     public static void main(String[] args) throws SchedulerException, IOException {
         BarrageCollectionTestTask collectionTask = new BarrageCollectionTestTask(
-                "https://live.bilibili.com/1440094?hotRank=0", "a9e30479-e231-46f3-991c-a8858400678f", System.currentTimeMillis()
+                "https://live.bilibili.com/12265?hotRank=0", "62aae6a3-e563-4f24-9e3a-135c6e74a8ac", System.currentTimeMillis()
         );
 
         BiliBiliExportTaskJob exportTaskJob = new BiliBiliExportTaskJob();

public/src/main/resources/log4j2.xml

@@ -13,6 +13,7 @@
     <Loggers>
         <logger name="org.mongodb" level="ERROR"/>
         <logger name="com.zhiweidata.automatictest.barragecrawlerserver" level="DEBUG"/>
+        <logger name="com.zhiweidata.automatictest.barragecrawlerserver.mapper" level="ERROR"/>
 
         <Root level="INFO" includeLocation="true">
             <AppenderRef ref="console"/>