Skip to content

A
automatic-test
Commit 20dfe82d by 朽木不可雕也
Options

完善BILIBILI弹幕导出测试

parent fbf2f1e3
Showing with 123 additions and 17 deletions
barrage-crawler-server/ddl/BarrageCollectionTestTask.sql
# 弹幕采集测试任务
CREATE TABLE barrage_collection_test_task
CREATE TABLE barrage_crawler_server_barrage_collection_test_task
(
name VARCHAR(255) NOT NULL PRIMARY KEY COMMENT '任务名称，32位UUID',
url VARCHAR(255) NOT NULL COMMENT '直播间地址',
......@@ -9,7 +9,7 @@ CREATE TABLE barrage_collection_test_task
) CHARSET utf8mb4 COMMENT '弹幕采集服务自动测试任务';
# 服务器响应信息
CREATE TABLE server_response_message
CREATE TABLE barrage_crawler_server_server_response_message
(
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
barrage_collection_test_task_name VARCHAR(255) NOT NULL COMMENT '任务名称，32位UUD',
......@@ -17,3 +17,14 @@ CREATE TABLE server_response_message
message VARCHAR(255) NOT NULL COMMENT '响应信息',
create_time TIMESTAMP(3) DEFAULT CURRENT_TIMESTAMP(3)
) CHARSET utf8mb4 COMMENT '服务器响应信息';
# 弹幕导出结果
CREATE TABLE barrage_crawler_server_barrage_export_result
(
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
name VARCHAR(255) NOT NULL COMMENT '任务名称',
total_collection INT UNSIGNED NOT NULL COMMENT '弹幕总采集量',
export_volume INT UNSIGNED NOT NULL COMMENT '导出的弹幕采集量',
percentage DOUBLE UNSIGNED NOT NULL COMMENT '导出的弹幕数量/弹幕总采集量',
create_time TIMESTAMP(3) DEFAULT CURRENT_TIMESTAMP(3)
) CHARSET utf8mb4 COMMENT '弹幕导出结果';
\ No newline at end of file
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/entity/BarrageExportInfo.java → barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/entity/BarrageCollectionInfo.java
......@@ -7,7 +7,7 @@ import java.io.Serializable;
import lombok.Data;
/**
* 弹幕导出信息
* 弹幕详细信息
*
* @author aszswaz
* @createTime 2021-09-06 11:40:09
......@@ -15,7 +15,7 @@ import lombok.Data;
*/
@SuppressWarnings({"JavaDoc", "GrazieInspection"})
@Data
public class BarrageExportInfo implements Serializable {
public class BarrageCollectionInfo implements Serializable {
/**
* 任务ID
*/
......
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/entity/BarrageExportResult.java 0 → 100644
package com.zhiweidata.automatictest.barragecrawlerserver.entity;
import lombok.Data;
/**
* 弹幕导出结果
*
* @author aszswaz
* @createTime 2021-09-09 16:28:51
* @ide IntelliJ IDEA
*/
@SuppressWarnings("JavaDoc")
@Data
public class BarrageExportResult {
/**
* 任务名称
*/
private String name;
/**
* 弹幕总采集量
*/
private int totalCollection;
/**
* 导出的弹幕数量
*/
private int exportVolume;
/**
* 导出的弹幕数量/弹幕总采集量
*/
private double percentage;
}
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/BarrageExportTaskJob.java
package com.zhiweidata.automatictest.barragecrawlerserver.exporttasks;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionInfo;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageExportResult;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.ServerResponseMessage;
import com.zhiweidata.automatictest.barragecrawlerserver.mapper.BarrageExportResultMapper;
import com.zhiweidata.automatictest.barragecrawlerserver.mapper.ServerResponseMessageMapper;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
......@@ -9,11 +13,13 @@ import java.util.ArrayList;
import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import org.apache.ibatis.session.SqlSession;
import org.jetbrains.annotations.NotNull;
import org.quartz.Job;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.CACHE_DIR;
import static com.zhiweidata.automatictest.publics.BeanContainer.SQL_SESSION_FACTORY;
import static java.util.Objects.isNull;
import static java.util.Objects.nonNull;
import static java.util.Objects.requireNonNull;
......@@ -74,6 +80,28 @@ public interface BarrageExportTaskJob extends Job {
* 保存服务器的响应信息
*/
default void saveResponseMessage(ServerResponseMessage message) {
try (SqlSession session = SQL_SESSION_FACTORY.openSession(true)) {
ServerResponseMessageMapper messageMapper = session.getMapper(ServerResponseMessageMapper.class);
messageMapper.insert(message);
}
}
/**
* 保存弹幕的导出信息
*
* @param collectionInfo 弹幕采集任务信息
* @param exportCount 导出的弹幕数量
*/
default void saveExportInfo(@NotNull BarrageCollectionInfo collectionInfo, int exportCount) {
try (SqlSession session = SQL_SESSION_FACTORY.openSession(true)) {
BarrageExportResult exportResult = new BarrageExportResult();
exportResult.setName(collectionInfo.getName());
exportResult.setExportVolume(exportCount);
exportResult.setTotalCollection(collectionInfo.getCollectionCount());
exportResult.setPercentage((double) exportCount / collectionInfo.getCollectionCount());
BarrageExportResultMapper resultMapper = session.getMapper(BarrageExportResultMapper.class);
resultMapper.insert(exportResult);
}
}
}
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/BiliBiliExportTaskJob.java
......@@ -4,7 +4,8 @@ import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.event.SyncReadListener;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageExportInfo;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionInfo;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageExportResult;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.ServerResponseMessage;
import java.io.File;
import java.io.IOException;
......@@ -72,15 +73,15 @@ public class BiliBiliExportTaskJob implements BarrageExportTaskJob {
Map<?, ?> dataMap = (Map<?, ?>) requireNonNull(jsonMap.get("data"));
List<?> dataList = (List<?>) requireNonNull(dataMap.get("list"));
BarrageExportInfo taskResponse = jsonMapper.convertValue(dataList.get(0), BarrageExportInfo.class);
this.checkFile(collectionTask, taskResponse);
BarrageCollectionInfo taskResponse = jsonMapper.convertValue(dataList.get(0), BarrageCollectionInfo.class);
this.checkFile(taskResponse);
}
/**
* 下载excel文件，并检查excel文件
*/
private void checkFile(@NotNull BarrageCollectionTestTask collectionTask, @NotNull BarrageExportInfo taskResponse) throws IOException {
String url = TASK_URL + "/" + taskResponse.getId() + "/barrage";
private void checkFile(@NotNull BarrageCollectionInfo collectionInfo) throws IOException {
String url = TASK_URL + "/" + collectionInfo.getId() + "/barrage";
HttpGet get = new HttpGet(url);
HttpResponse response = HTTP_CLIENT.execute(get);
String contentType = response.getFirstHeader("Content-Type").getValue();
......@@ -88,22 +89,26 @@ public class BiliBiliExportTaskJob implements BarrageExportTaskJob {
ObjectMapper jsonMapper = new ObjectMapper();
ServerResponseMessage responseMessage = jsonMapper.readValue(response.getEntity().getContent(), ServerResponseMessage.class);
log.info(responseMessage.getMessage());
responseMessage.setName(collectionTask.getName());
responseMessage.setName(collectionInfo.getName());
this.saveResponseMessage(responseMessage);
} else {
final File cacheDir = new File(CACHE_DIR, taskResponse.getId());
final File cacheDir = new File(CACHE_DIR, collectionInfo.getId());
List<File> excelFiles = this.write(cacheDir, response.getEntity().getContent());
int exportCount = 0;
// 检查文件中的数据量
for (File excelFile : excelFiles) {
List<Map<Integer, String>> dataList = EasyExcel.read(excelFile, new SyncReadListener()).doReadAllSync();
if (dataList.isEmpty()) {
log.error("任务ID：{}，文件：{}为空", taskResponse.getId(), excelFile.getName());
log.error("任务ID：{}，文件：{}为空", collectionInfo.getId(), excelFile.getName());
return;
}
log.info("任务ID：{},文件：{}弹幕数量：{}", taskResponse.getId(), excelFile.getName(), dataList.size());
exportCount += dataList.size();
log.info("任务ID：{},文件：{}弹幕数量：{}", collectionInfo.getId(), excelFile.getName(), dataList.size());
}
log.info("任务：{}，导出弹幕：{}，弹幕总采集量：{}", collectionInfo.getId(), exportCount, collectionInfo.getCollectionCount());
// 删除文件
this.removeFile(cacheDir);
this.saveExportInfo(collectionInfo, exportCount);
}
}
}
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/mapper/BarrageExportResultMapper.java 0 → 100644
package com.zhiweidata.automatictest.barragecrawlerserver.mapper;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageExportResult;
import org.apache.ibatis.annotations.Param;
/**
* 弹幕导出采集量保存
*
* @author aszswaz
* @createTime 2021-09-09 16:39:40
* @ide IntelliJ IDEA
*/
@SuppressWarnings("JavaDoc")
public interface BarrageExportResultMapper {
void insert(@Param(value = "result") BarrageExportResult result);
}
barrage-crawler-server/src/main/resources/mappers/BarrageCollectionTestTaskMapper.xml
......@@ -12,7 +12,7 @@
</resultMap>
<insert id="insert">
INSERT INTO barrage_collection_test_task (name, url, start_time, end_time)
INSERT INTO barrage_crawler_server_barrage_collection_test_task (name, url, start_time, end_time)
VALUE (#{task.name}, #{task.url}, #{task.startTime}, #{task.endTime})
</insert>
</mapper>
\ No newline at end of file
barrage-crawler-server/src/main/resources/mappers/BarrageExportResultMapper.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper
PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.zhiweidata.automatictest.barragecrawlerserver.mapper.BarrageExportResultMapper">
<insert id="insert">
INSERT INTO barrage_crawler_server_barrage_export_result(name, total_collection, export_volume, percentage)
VALUE (#{result.name}, #{result.totalCollection}, #{result.exportVolume}, #{result.percentage})
</insert>
</mapper>
\ No newline at end of file
barrage-crawler-server/src/main/resources/mappers/ServerResponseMessageMapper.xml
......@@ -11,7 +11,7 @@
</resultMap>
<insert id="insert">
INSERT INTO server_response_message(barrage_collection_test_task_name, code, message)
INSERT INTO barrage_crawler_server_server_response_message(barrage_collection_test_task_name, code, message)
VALUE (#{message.name}, #{message.code}, #{message.message})
</insert>
</mapper>
\ No newline at end of file
barrage-crawler-server/src/test/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/BiliBiliExportTaskJobTest.java
......@@ -16,7 +16,7 @@ class BiliBiliExportTaskJobTest {
public static void main(String[] args) throws SchedulerException, IOException {
BarrageCollectionTestTask collectionTask = new BarrageCollectionTestTask(
"https://live.bilibili.com/1440094?hotRank=0", "a9e30479-e231-46f3-991c-a8858400678f", System.currentTimeMillis()
"https://live.bilibili.com/12265?hotRank=0", "62aae6a3-e563-4f24-9e3a-135c6e74a8ac", System.currentTimeMillis()
);
BiliBiliExportTaskJob exportTaskJob = new BiliBiliExportTaskJob();
......
public/src/main/resources/log4j2.xml
......@@ -13,6 +13,7 @@
<Loggers>
<logger name="org.mongodb" level="ERROR"/>
<logger name="com.zhiweidata.automatictest.barragecrawlerserver" level="DEBUG"/>
<logger name="com.zhiweidata.automatictest.barragecrawlerserver.mapper" level="ERROR"/>
<Root level="INFO" includeLocation="true">
<AppenderRef ref="console"/>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment