Skip to content

A
automatic-test
Commit 77007f74 by 朽木不可雕也
Options

记录测试信息的主数据库改为mysql

parent bb50db18
Showing with 369 additions and 99 deletions
.gitignore
......@@ -4,4 +4,7 @@ target
*.jar
*.iml
*.classpath
*.log
\ No newline at end of file
*.log
*.xlsx
cache
venv
\ No newline at end of file
barrage-crawler-server/ddl/BarrageCollectionTestTask.sql 0 → 100644
# 弹幕采集测试任务
CREATE TABLE barrage_collection_test_task
(
name VARCHAR(255) NOT NULL PRIMARY KEY COMMENT '任务名称，32位UUID',
url VARCHAR(255) NOT NULL COMMENT '直播间地址',
start_time INT NOT NULL COMMENT '任务开始时间',
end_time INT NOT NULL COMMENT '任务结束时间',
create_time TIMESTAMP(3) DEFAULT CURRENT_TIMESTAMP(3) COMMENT '任务创建时间'
) CHARSET utf8mb4 COMMENT '弹幕采集服务自动测试任务';
# 服务器响应信息
CREATE TABLE server_response_message
(
id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
barrage_collection_test_task_name VARCHAR(255) NOT NULL COMMENT '任务名称，32位UUD',
code INT NOT NULL COMMENT '响应代码',
message VARCHAR(255) NOT NULL COMMENT '响应信息',
create_time TIMESTAMP(3) DEFAULT CURRENT_TIMESTAMP(3),
FOREIGN KEY (barrage_collection_test_task_name)
REFERENCES barrage_collection_test_task (name)
ON DELETE CASCADE ON UPDATE CASCADE
) CHARSET utf8mb4 COMMENT '服务器响应信息';
\ No newline at end of file
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/config/BaseConfig.java
package com.zhiweidata.automatictest.barragecrawlerserver.config;
import com.mongodb.client.MongoCollection;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
import java.io.File;
import lombok.extern.slf4j.Slf4j;
import static com.zhiweidata.automatictest.publics.BeanContainer.MONGO_DATABASE;
/**
* 基本配置
*
......@@ -38,14 +34,6 @@ public class BaseConfig {
*/
public static final int EXPORT_START_MAX_DELAY = 7;
/**
* 模块名称
*/
public static final String MODULE_NAME = "barrage_crawler_server";
/**
* 测试采集任务的信息集合
*/
public static final MongoCollection<BarrageCollectionTask> MONGO_TASK_REQUEST_COLLECTION = MONGO_DATABASE.getCollection(MODULE_NAME + "_task_requests", BarrageCollectionTask.class);
/**
* 文件缓存文件夹
*/
public static final File CACHE_DIR = new File("cache");
......
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/BiliBiliCreateTaskJob.java
package com.zhiweidata.automatictest.barragecrawlerserver.createtasks;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask;
import com.zhiweidata.automatictest.barragecrawlerserver.exporttasks.BiliBiliExportTaskJob;
import java.net.URL;
import java.util.List;
......@@ -38,7 +38,7 @@ public class BiliBiliCreateTaskJob extends CreateTaskJob {
roomList.forEach(room -> {
try {
Map<?, ?> roomMap = (Map<?, ?>) room;
BarrageCollectionTask collectionTask = BiliBiliCreateTaskJob.super.createCollectionTask(baseUrl + roomMap.get("link"), currentTime);
BarrageCollectionTestTask collectionTask = BiliBiliCreateTaskJob.super.createCollectionTask(baseUrl + roomMap.get("link"), currentTime);
BiliBiliCreateTaskJob.super.createExportTask(collectionTask, BiliBiliExportTaskJob.class);
} catch (Exception e) {
log.error(e.getMessage(), e);
......
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/CreateTaskJob.java
package com.zhiweidata.automatictest.barragecrawlerserver.createtasks;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.ServerResponseMessage;
import com.zhiweidata.automatictest.barragecrawlerserver.exporttasks.BarrageExportTaskJob;
import com.zhiweidata.automatictest.barragecrawlerserver.mapper.BarrageCollectionTestTaskMapper;
import com.zhiweidata.automatictest.barragecrawlerserver.mapper.ServerResponseMessageMapper;
import com.zhiweidata.automatictest.barragecrawlerserver.util.TimeUtil;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ByteArrayEntity;
import org.apache.http.entity.ContentType;
import org.apache.ibatis.session.SqlSession;
import org.jetbrains.annotations.NotNull;
import org.quartz.Job;
import org.quartz.JobBuilder;
......@@ -24,10 +25,10 @@ import org.quartz.TriggerBuilder;
import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.BARRAGE_COLLECTION_TASK_JOB_KEY;
import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.EXPORT_START_MAX_DELAY;
import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.MONGO_TASK_REQUEST_COLLECTION;
import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.TASK_URL;
import static com.zhiweidata.automatictest.publics.BeanContainer.HTTP_CLIENT;
import static com.zhiweidata.automatictest.publics.BeanContainer.SCHEDULER;
import static com.zhiweidata.automatictest.publics.BeanContainer.SQL_SESSION_FACTORY;
import static com.zhiweidata.automatictest.publics.BeanContainer.STANDARD_DATE_FORMAT;
/**
......@@ -42,9 +43,9 @@ public abstract class CreateTaskJob implements Job {
* 请求弹幕采集服务器创建采集任务
*/
@NotNull
protected final BarrageCollectionTask createCollectionTask(String liveUrl, long currentTime) throws IOException {
protected final BarrageCollectionTestTask createCollectionTask(String liveUrl, long currentTime) throws IOException {
final String liveName = UUID.randomUUID().toString();
BarrageCollectionTask collectionTask = new BarrageCollectionTask(liveUrl, liveName, currentTime);
BarrageCollectionTestTask collectionTask = new BarrageCollectionTestTask(liveUrl, liveName, currentTime);
if (log.isDebugEnabled()) {
String startTime = STANDARD_DATE_FORMAT.format(collectionTask.getStartTime() * 1000L);
String endTime = STANDARD_DATE_FORMAT.format(collectionTask.getEndTime() * 1000L);
......@@ -54,18 +55,22 @@ public abstract class CreateTaskJob implements Job {
ObjectMapper objectMapper = new ObjectMapper();
post.setEntity(new ByteArrayEntity(objectMapper.writeValueAsBytes(collectionTask), ContentType.APPLICATION_JSON));
HttpResponse response = HTTP_CLIENT.execute(post);
Map<?, ?> jsonMap = objectMapper.readValue(response.getEntity().getContent(), Map.class);
ServerResponseMessage responseMessage = objectMapper.readValue(response.getEntity().getContent(), ServerResponseMessage.class);
responseMessage.setName(collectionTask.getName());
collectionTask.setCode((Integer) jsonMap.get("code"));
collectionTask.setMessage((String) jsonMap.get("message"));
MONGO_TASK_REQUEST_COLLECTION.insertOne(collectionTask);
try (SqlSession session = SQL_SESSION_FACTORY.openSession(true)) {
BarrageCollectionTestTaskMapper testTaskMapper = session.getMapper(BarrageCollectionTestTaskMapper.class);
testTaskMapper.insert(collectionTask);
ServerResponseMessageMapper messageMapper = session.getMapper(ServerResponseMessageMapper.class);
messageMapper.insert(responseMessage);
}
return collectionTask;
}
/**
* 创建定时导出任务
*/
protected final <T extends BarrageExportTaskJob> void createExportTask(@NotNull BarrageCollectionTask collectionTask, Class<T> tClass) throws SchedulerException {
protected final <T extends BarrageExportTaskJob> void createExportTask(@NotNull BarrageCollectionTestTask collectionTask, Class<T> tClass) throws SchedulerException {
JobBuilder jobBuilder = JobBuilder.newJob(tClass);
String name = collectionTask.getName(), group = tClass.getSimpleName();
jobBuilder.withIdentity(name, group);
......
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/DouyuCreateTaskJob.java
......@@ -2,7 +2,7 @@ package com.zhiweidata.automatictest.barragecrawlerserver.createtasks;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask;
import com.zhiweidata.automatictest.barragecrawlerserver.exception.BarrageHomePageException;
import com.zhiweidata.automatictest.barragecrawlerserver.exporttasks.DouyuExportTaskJob;
import java.util.HashSet;
......@@ -44,7 +44,7 @@ public class DouyuCreateTaskJob extends CreateTaskJob {
final long currentTime = System.currentTimeMillis();
liveUrls.forEach(liveUrl -> {
try {
BarrageCollectionTask collectionTask = super.createCollectionTask(liveUrl, currentTime);
BarrageCollectionTestTask collectionTask = super.createCollectionTask(liveUrl, currentTime);
super.createExportTask(collectionTask, DouyuExportTaskJob.class);
} catch (Exception e) {
log.error(e.getMessage(), e);
......
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/HuyaCreateTaskJob.java
......@@ -2,7 +2,7 @@ package com.zhiweidata.automatictest.barragecrawlerserver.createtasks;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask;
import com.zhiweidata.automatictest.barragecrawlerserver.exception.BarrageHomePageException;
import com.zhiweidata.automatictest.barragecrawlerserver.exporttasks.HuyaExportTaskJob;
import java.util.HashSet;
......@@ -96,7 +96,7 @@ public class HuyaCreateTaskJob extends CreateTaskJob {
final long currentTime = System.currentTimeMillis();
liveUrls.forEach(liveUrl -> {
try {
BarrageCollectionTask collectionTask = super.createCollectionTask(liveUrl, currentTime);
BarrageCollectionTestTask collectionTask = super.createCollectionTask(liveUrl, currentTime);
super.createExportTask(collectionTask, HuyaExportTaskJob.class);
} catch (Exception e) {
log.error(e.getMessage(), e);
......
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/entity/BarrageCollectionTask.java → barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/entity/BarrageCollectionTestTask.java
package com.zhiweidata.automatictest.barragecrawlerserver.entity;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.Calendar;
import lombok.EqualsAndHashCode;
......@@ -11,16 +10,18 @@ import lombok.ToString;
import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.MAX_COLLECTION_TIME;
/**
* 弹幕采集测试任务
*
* @author aszswaz
* @createTime 2021-08-24 15:53:39
* @ide IntelliJ IDEA
*/
@SuppressWarnings("JavaDoc")
@SuppressWarnings({"JavaDoc", "GrazieInspection"})
@Getter
@Setter
@EqualsAndHashCode
@ToString
public class BarrageCollectionTask {
public class BarrageCollectionTestTask {
/**
* 任务名称
*/
......@@ -41,18 +42,8 @@ public class BarrageCollectionTask {
*/
@JsonProperty(value = "endTime")
private Integer endTime;
/**
* 服务器的响应代码
*/
@JsonIgnore
private int code;
/**
* 响应信息
*/
@JsonIgnore
private String message;
public BarrageCollectionTask(String liveUrl, String name, long currentTime) {
public BarrageCollectionTestTask(String liveUrl, String name, long currentTime) {
this.url = liveUrl;
this.name = name;
......
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/entity/BarrageTaskResponse.java → barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/entity/BarrageExportInfo.java
......@@ -7,33 +7,63 @@ import java.io.Serializable;
import lombok.Data;
/**
* 弹幕信息实体类
* 弹幕导出信息
*
* @author aszswaz
* @createTime 2021-09-06 11:40:09
* @ide IntelliJ IDEA
*/
@SuppressWarnings("JavaDoc")
@SuppressWarnings({"JavaDoc", "GrazieInspection"})
@Data
public class BarrageTaskResponse implements Serializable {
public class BarrageExportInfo implements Serializable {
/**
* 任务ID
*/
@JsonProperty(value = "id")
private String id;
/**
* 任务名称
*/
@JsonProperty(value = "name")
private String name;
/**
* 直播间地址
*/
@JsonProperty(value = "url")
private String url;
/**
* 直播间ID
*/
@JsonProperty(value = "roomId")
private String roomId;
/**
* 任务开始时间
*/
@JsonProperty(value = "startTime")
private int startTime;
/**
* 任务结束时间
*/
@JsonProperty(value = "endTime")
private int endTime;
/**
* 任务执行状态
*/
@JsonProperty(value = "status")
private BarrageStatusEnums status;
/**
* 直播间平台
*/
@JsonProperty(value = "platform")
private PlatformEnum platform;
/**
* 任务创建时间
*/
@JsonProperty(value = "createTime")
private int createTime;
/**
* 弹幕采集量
*/
@JsonProperty(value = "collectionCount")
private int collectionCount;
}
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/entity/ServerResponseMessage.java 0 → 100644
package com.zhiweidata.automatictest.barragecrawlerserver.entity;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
/**
* 服务器响应信息
*
* @author aszswaz
* @createTime 2021-09-09 14:18:29
* @ide IntelliJ IDEA
*/
@SuppressWarnings("JavaDoc")
@Data
public class ServerResponseMessage {
@JsonIgnore
private String name;
@JsonProperty(value = "code")
private int code;
@JsonProperty(value = "message")
private String message;
}
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/BarrageExportTaskJob.java
......@@ -14,6 +14,7 @@ import org.slf4j.LoggerFactory;
import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.CACHE_DIR;
import static java.util.Objects.isNull;
import static java.util.Objects.nonNull;
import static java.util.Objects.requireNonNull;
/**
......@@ -32,9 +33,8 @@ public interface BarrageExportTaskJob extends Job {
*
* @return 解压得到的文件
*/
default List<File> write(String taskId, InputStream inputStream) throws IOException {
final File outDir = new File(CACHE_DIR, taskId);
if (!outDir.exists()) if (!outDir.mkdirs()) log.error("文件夹{}创建失败", outDir.getName());
default List<File> write(File cacheDir, InputStream inputStream) throws IOException {
if (!cacheDir.exists()) if (!cacheDir.mkdirs()) log.error("文件夹{}创建失败", cacheDir.getName());
final List<File> outFiles = new ArrayList<>();
try (ZipInputStream zipInputStream = new ZipInputStream(inputStream)) {
......@@ -43,7 +43,7 @@ public interface BarrageExportTaskJob extends Job {
while (true) {
ZipEntry zipEntry = zipInputStream.getNextEntry();
if (isNull(zipEntry)) break;
File execFile = new File(requireNonNull(zipEntry.getName()));
File execFile = new File(cacheDir, requireNonNull(zipEntry.getName()));
try (FileOutputStream fileOutputStream = new FileOutputStream(execFile)) {
while ((len = zipInputStream.read(buff)) != -1) fileOutputStream.write(buff, 0, len);
}
......@@ -53,4 +53,19 @@ public interface BarrageExportTaskJob extends Job {
}
return outFiles;
}
/**
* 删除文件
*/
default void removeFile(File file) {
// 如果是文件夹，先删除文件夹中的文件
if (file.isDirectory()) {
File[] files = file.listFiles();
if (nonNull(files)) {
for (File subFile : files) this.removeFile(subFile);
}
}
// 删除文件夹或文件
if (!file.delete()) log.error("文件夹：{}删除失败", file.getName());
}
}
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/BiliBiliExportTaskJob.java
......@@ -3,8 +3,8 @@ package com.zhiweidata.automatictest.barragecrawlerserver.exporttasks;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.event.SyncReadListener;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageTaskResponse;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageExportInfo;
import java.io.File;
import java.io.IOException;
import java.util.List;
......@@ -20,7 +20,7 @@ import org.quartz.JobExecutionContext;
import org.quartz.SchedulerException;
import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.BARRAGE_COLLECTION_TASK_JOB_KEY;
import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.MONGO_TASK_REQUEST_COLLECTION;
import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.CACHE_DIR;
import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.TASK_URL;
import static com.zhiweidata.automatictest.publics.BeanContainer.HTTP_CLIENT;
import static java.util.Objects.requireNonNull;
......@@ -47,7 +47,7 @@ public class BiliBiliExportTaskJob implements BarrageExportTaskJob {
public void execute(JobExecutionContext context) {
try {
JobDataMap jobDataMap = requireNonNull(context.getMergedJobDataMap());
BarrageCollectionTask collectionTask = (BarrageCollectionTask) requireNonNull(jobDataMap.get(BARRAGE_COLLECTION_TASK_JOB_KEY));
BarrageCollectionTestTask collectionTask = (BarrageCollectionTestTask) requireNonNull(jobDataMap.get(BARRAGE_COLLECTION_TASK_JOB_KEY));
this.export(collectionTask);
} catch (Exception e) {
log.error(e.getMessage(), e);
......@@ -57,7 +57,7 @@ public class BiliBiliExportTaskJob implements BarrageExportTaskJob {
/**
* 尝试导出弹幕
*/
public void export(@NotNull BarrageCollectionTask collectionTask) throws IOException, SchedulerException {
public void export(@NotNull BarrageCollectionTestTask collectionTask) throws IOException, SchedulerException {
String url = TASK_URL + "?limit=20&page=1&name=" + collectionTask.getName();
HttpResponse response = HTTP_CLIENT.execute(new HttpGet(url));
......@@ -71,14 +71,14 @@ public class BiliBiliExportTaskJob implements BarrageExportTaskJob {
Map<?, ?> dataMap = (Map<?, ?>) requireNonNull(jsonMap.get("data"));
List<?> dataList = (List<?>) requireNonNull(dataMap.get("list"));
BarrageTaskResponse taskResponse = jsonMapper.convertValue(dataList.get(0), BarrageTaskResponse.class);
BarrageExportInfo taskResponse = jsonMapper.convertValue(dataList.get(0), BarrageExportInfo.class);
this.checkFile(collectionTask, taskResponse);
}
/**
* 下载excel文件，并检查excel文件
*/
private void checkFile(@NotNull BarrageCollectionTask collectionTask, @NotNull BarrageTaskResponse taskResponse) throws IOException {
private void checkFile(@NotNull BarrageCollectionTestTask collectionTask, @NotNull BarrageExportInfo taskResponse) throws IOException {
String url = TASK_URL + "/" + taskResponse.getId() + "/barrage";
HttpGet get = new HttpGet(url);
HttpResponse response = HTTP_CLIENT.execute(get);
......@@ -88,20 +88,20 @@ public class BiliBiliExportTaskJob implements BarrageExportTaskJob {
Map<?, ?> jsonMap = jsonMapper.readValue(response.getEntity().getContent(), Map.class);
String message = (String) jsonMap.get("message");
log.info(message);
collectionTask.setCode((Integer) jsonMap.get("code"));
collectionTask.setMessage(message);
MONGO_TASK_REQUEST_COLLECTION.insertOne(collectionTask);
} else {
List<File> excelFiles = this.write(taskResponse.getId(), response.getEntity().getContent());
final File cacheDir = new File(CACHE_DIR, taskResponse.getId());
List<File> excelFiles = this.write(cacheDir, response.getEntity().getContent());
// 检查文件中的数据量
excelFiles.forEach(excelFile -> {
for (File excelFile : excelFiles) {
List<Map<Integer, String>> dataList = EasyExcel.read(excelFile, new SyncReadListener()).doReadAllSync();
if (dataList.isEmpty()) {
log.error("任务ID：{}，文件：{}为空", taskResponse.getId(), excelFile.getName());
return;
}
log.info("任务ID：{},文件：{}弹幕数量：{}", taskResponse.getId(), excelFile.getName(), dataList.size());
});
}
// 删除文件
this.removeFile(cacheDir);
}
}
}
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/DouyuExportTaskJob.java
package com.zhiweidata.automatictest.barragecrawlerserver.exporttasks;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask;
import lombok.extern.slf4j.Slf4j;
import org.quartz.JobDataMap;
import org.quartz.JobExecutionContext;
......@@ -22,7 +22,7 @@ public class DouyuExportTaskJob implements BarrageExportTaskJob {
public void execute(JobExecutionContext context) {
try {
JobDataMap dataMap = requireNonNull(context.getMergedJobDataMap());
BarrageCollectionTask collectionTask = (BarrageCollectionTask) requireNonNull(dataMap.get(BARRAGE_COLLECTION_TASK_JOB_KEY));
BarrageCollectionTestTask collectionTask = (BarrageCollectionTestTask) requireNonNull(dataMap.get(BARRAGE_COLLECTION_TASK_JOB_KEY));
} catch (Exception e) {
log.error(e.getMessage(), e);
}
......
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/HuyaExportTaskJob.java
package com.zhiweidata.automatictest.barragecrawlerserver.exporttasks;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask;
import lombok.extern.slf4j.Slf4j;
import org.quartz.JobDataMap;
import org.quartz.JobExecutionContext;
......@@ -22,7 +22,7 @@ public class HuyaExportTaskJob implements BarrageExportTaskJob {
public void execute(JobExecutionContext context) {
try {
JobDataMap dataMap = requireNonNull(context.getMergedJobDataMap());
BarrageCollectionTask collectionTask = (BarrageCollectionTask) requireNonNull(dataMap.get(BARRAGE_COLLECTION_TASK_JOB_KEY));
BarrageCollectionTestTask collectionTask = (BarrageCollectionTestTask) requireNonNull(dataMap.get(BARRAGE_COLLECTION_TASK_JOB_KEY));
} catch (Exception e) {
log.error(e.getMessage(), e);
}
......
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/mapper/BarrageCollectionTestTaskMapper.java 0 → 100644
package com.zhiweidata.automatictest.barragecrawlerserver.mapper;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask;
import org.apache.ibatis.annotations.Param;
/**
* 测试任务信息Mapper
*
* @author aszswaz
* @createTime 2021-09-09 11:31:16
* @ide IntelliJ IDEA
*/
@SuppressWarnings("JavaDoc")
public interface BarrageCollectionTestTaskMapper {
void insert(
@Param(value = "task") BarrageCollectionTestTask testTask
);
}
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/mapper/ServerResponseMessageMapper.java 0 → 100644
package com.zhiweidata.automatictest.barragecrawlerserver.mapper;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.ServerResponseMessage;
import org.apache.ibatis.annotations.Param;
/**
* 服务器响应信息存储
*
* @author aszswaz
* @createTime 2021-09-09 14:23:25
* @ide IntelliJ IDEA
*/
@SuppressWarnings("JavaDoc")
public interface ServerResponseMessageMapper {
void insert(@Param(value = "message") ServerResponseMessage responseMessage);
}
barrage-crawler-server/src/main/resources/mappers/BarrageCollectionTestTaskMapper.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper
PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.zhiweidata.automatictest.barragecrawlerserver.mapper.BarrageCollectionTestTaskMapper">
<resultMap id="BarrageCollectionTask"
type="com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask">
<result column="name" property="name"/>
<result column="url" property="url"/>
<result column="start_time" property="startTime"/>
<result column="end_time" property="endTime"/>
</resultMap>
<insert id="insert">
INSERT INTO barrage_collection_test_task (name, url, start_time, end_time)
VALUE (#{task.name}, #{task.url}, #{task.startTime}, #{task.endTime})
</insert>
</mapper>
\ No newline at end of file
barrage-crawler-server/src/main/resources/mappers/ServerResponseMessageMapper.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper
PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.zhiweidata.automatictest.barragecrawlerserver.mapper.ServerResponseMessageMapper">
<resultMap id="ServerResponseMessage"
type="com.zhiweidata.automatictest.barragecrawlerserver.entity.ServerResponseMessage">
<result property="name" column="name"/>
<result property="code" column="code"/>
<result property="message" column="message"/>
</resultMap>
<insert id="insert">
INSERT INTO server_response_message(barrage_collection_test_task_name, code, message)
VALUE (#{message.name}, #{message.code}, #{message.message})
</insert>
</mapper>
\ No newline at end of file
barrage-crawler-server/src/test/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/BiliBiliExportTaskJobTest.java
package com.zhiweidata.automatictest.barragecrawlerserver.exporttasks;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask;
import java.io.IOException;
import org.quartz.SchedulerException;
import static com.zhiweidata.automatictest.publics.BeanContainer.SCHEDULER;
/**
* @author aszswaz
* @createTime 2021-09-06 10:03:42
......@@ -13,11 +15,12 @@ import org.quartz.SchedulerException;
class BiliBiliExportTaskJobTest {
public static void main(String[] args) throws SchedulerException, IOException {
BarrageCollectionTask collectionTask = new BarrageCollectionTask(
"https://live.bilibili.com/21544906?hotRank=0", "6783d422-840f-4230-9875-81ee91c06d09", System.currentTimeMillis()
BarrageCollectionTestTask collectionTask = new BarrageCollectionTestTask(
"https://live.bilibili.com/1440094?hotRank=0", "a9e30479-e231-46f3-991c-a8858400678f", System.currentTimeMillis()
);
BiliBiliExportTaskJob exportTaskJob = new BiliBiliExportTaskJob();
exportTaskJob.export(collectionTask);
SCHEDULER.shutdown(true);
}
}
\ No newline at end of file
public/pom.xml
......@@ -43,11 +43,23 @@
<version>${quartz.version}</version>
</dependency>
<!--mongodb client-->
<!-- https://mvnrepository.com/artifact/org.mybatis/mybatis -->
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongodb-driver-sync</artifactId>
<version>${mongodb.version}</version>
<groupId>org.mybatis</groupId>
<artifactId>mybatis</artifactId>
<version>3.5.7</version>
</dependency>
<!-- https://mvnrepository.com/artifact/mysql/mysql-connector-java -->
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>8.0.26</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.zaxxer/HikariCP -->
<dependency>
<groupId>com.zaxxer</groupId>
<artifactId>HikariCP</artifactId>
<version>5.0.0</version>
</dependency>
</dependencies>
......
public/src/main/java/com/zhiweidata/automatictest/publics/BeanContainer.java
package com.zhiweidata.automatictest.publics;
import com.mongodb.client.MongoClient;
import com.mongodb.client.MongoClients;
import com.mongodb.client.MongoDatabase;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.time.FastDateFormat;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.bson.codecs.configuration.CodecProvider;
import org.bson.codecs.configuration.CodecRegistry;
import org.bson.codecs.pojo.PojoCodecProvider;
import org.apache.ibatis.builder.xml.XMLConfigBuilder;
import org.apache.ibatis.builder.xml.XMLMapperBuilder;
import org.apache.ibatis.io.Resources;
import org.apache.ibatis.session.Configuration;
import org.apache.ibatis.session.SqlSessionFactory;
import org.apache.ibatis.session.SqlSessionFactoryBuilder;
import org.jetbrains.annotations.NotNull;
import org.quartz.Scheduler;
import org.quartz.SchedulerException;
import org.quartz.impl.StdSchedulerFactory;
import static com.mongodb.MongoClientSettings.getDefaultCodecRegistry;
import static org.bson.codecs.configuration.CodecRegistries.fromProviders;
import static org.bson.codecs.configuration.CodecRegistries.fromRegistries;
import static java.util.Objects.requireNonNull;
/**
* Bean容器
......@@ -39,17 +43,13 @@ public class BeanContainer {
*/
public static final Scheduler SCHEDULER;
/**
* mongodb 客户端
*/
public static final MongoClient MONGO_CLIENT;
/**
* 整个项目专用数据库
*/
public static final MongoDatabase MONGO_DATABASE;
/**
* 标准日期格式化
*/
public static final FastDateFormat STANDARD_DATE_FORMAT = FastDateFormat.getInstance("yyyy-MM-dd HH:mm:ss.SSS");
/**
* mybatis session工厂
*/
public static final SqlSessionFactory SQL_SESSION_FACTORY;
static {
......@@ -71,11 +71,46 @@ public class BeanContainer {
configBuilder.setRedirectsEnabled(true);
clientBuilder.setDefaultRequestConfig(configBuilder.build());
HTTP_CLIENT = clientBuilder.build();
}
static {
SqlSessionFactory sqlSessionFactory = null;
try (InputStream inputStream = Resources.getResourceAsStream("mybatis.xml")) {
SqlSessionFactoryBuilder builder = new SqlSessionFactoryBuilder();
MONGO_CLIENT = MongoClients.create("mongodb://admin:z199809051593@192.168.0.119");
// 启用 POJO 对象映射
CodecProvider pojoCodecProvider = PojoCodecProvider.builder().automatic(true).build();
CodecRegistry pojoCodecRegistry = fromRegistries(getDefaultCodecRegistry(), fromProviders(pojoCodecProvider));
MONGO_DATABASE = MONGO_CLIENT.getDatabase("automatic_test").withCodecRegistry(pojoCodecRegistry);
// 先读取xml中的配置
XMLConfigBuilder configBuilder = new XMLConfigBuilder(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
configBuilder.parse();
Configuration configuration = configBuilder.getConfiguration();
// 扫描mapper
File mappersDir = Resources.getResourceAsFile("mappers");
scanMapper(mappersDir, configuration);
sqlSessionFactory = builder.build(configuration);
} catch (Exception e) {
log.error(e.getMessage(), e);
}
SQL_SESSION_FACTORY = sqlSessionFactory;
}
/**
* 扫描mapper
*/
private static void scanMapper(@NotNull File file, @NotNull Configuration configuration) throws IOException {
if (file.isDirectory()) {
File[] mappers = requireNonNull(file.listFiles());
for (File mapper : mappers) {
scanMapper(mapper, configuration);
}
} else {
String fileName = file.getName();
if (".xml".equals(fileName.substring(fileName.lastIndexOf(".")))) {
String resource = file.toPath().toUri().toString();
try (InputStream inputStream = Resources.getUrlAsStream(resource)) {
XMLMapperBuilder mapperBuilder = new XMLMapperBuilder(inputStream, configuration, resource, configuration.getSqlFragments());
mapperBuilder.parse();
}
}
}
}
}
public/src/main/java/com/zhiweidata/automatictest/publics/HikariDataSourceFactory.java 0 → 100644
package com.zhiweidata.automatictest.publics;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import java.util.Properties;
import javax.sql.DataSource;
import lombok.extern.slf4j.Slf4j;
import org.apache.ibatis.datasource.DataSourceFactory;
import org.jetbrains.annotations.NotNull;
/**
* mysql的数据库连接池Hikari工厂
*
* @author aszswaz
* @createTime 2021-09-09 10:41:07
* @ide IntelliJ IDEA
*/
@Slf4j
@SuppressWarnings("JavaDoc")
public class HikariDataSourceFactory implements DataSourceFactory {
/**
* 数据库连接池
*/
private HikariDataSource dataSource;
@Override
public void setProperties(@NotNull Properties props) {
final String initSql = "SHOW DATABASES";
final long timeout = 60000;
HikariConfig config = new HikariConfig();
config.setAutoCommit(true);
config.setDriverClassName(props.getProperty("driver"));
config.setUsername(props.getProperty("username"));
config.setPassword(props.getProperty("password"));
config.setJdbcUrl(props.getProperty("url"));
config.setConnectionTestQuery(initSql);
config.setConnectionInitSql(initSql);
config.setConnectionTimeout(timeout);
config.setIdleTimeout(timeout << 1);
config.setInitializationFailTimeout(timeout);
config.setMaximumPoolSize(10);
config.setMinimumIdle(5);
config.setMaxLifetime(timeout << 2);
this.dataSource = new HikariDataSource(config);
}
@Override
public DataSource getDataSource() {
return this.dataSource;
}
}
public/src/main/resources/mybatis.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE configuration
PUBLIC "-//mybatis.org//DTD Config 3.0//EN"
"http://mybatis.org/dtd/mybatis-3-config.dtd">
<configuration>
<environments default="development">
<environment id="development">
<transactionManager type="JDBC"/>
<dataSource type="com.zhiweidata.automatictest.publics.HikariDataSourceFactory">
<property name="driver" value="com.mysql.cj.jdbc.Driver"/>
<property name="url" value="jdbc:mysql://192.168.0.119:3306/automatic_test?useSSL=FALSE&amp;serverTimezone=Asia/Shanghai"/>
<property name="username" value="root"/>
<property name="password" value="z199809051593"/>
</dataSource>
</environment>
</environments>
</configuration>
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment