Skip to content

A
automatic-test
Commit 758fdb39 by 朽木不可雕也
Options

backup

parent 961a0cab
Showing with 55 additions and 41 deletions
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/config/BaseConfig.java
package com.zhiweidata.automatictest.barragecrawlerserver.config;
import com.mongodb.client.MongoCollection;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
import static com.zhiweidata.automatictest.publics.BeanContainer.MONGO_DATABASE;
/**
* 基本配置
*
......@@ -9,7 +14,6 @@ package com.zhiweidata.automatictest.barragecrawlerserver.config;
*/
@SuppressWarnings("JavaDoc")
public class BaseConfig {
public static final String MONGO_LIVE_COLLECTION = "live_url";
/**
* 接口的基础url
*/
......@@ -30,4 +34,12 @@ public class BaseConfig {
* 弹幕的excel导出服务器测试，最多延迟 X 天
*/
public static final int EXPORT_START_MAX_DELAY = 7;
/**
* 模块名称
*/
public static final String MODULE_NAME = "barrage_crawler_server";
/**
* 测试采集任务的信息集合
*/
public static final MongoCollection<BarrageCollectionTask> MONGO_TASK_REQUEST_COLLECTION = MONGO_DATABASE.getCollection(MODULE_NAME + "_task_requests", BarrageCollectionTask.class);
}
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/BiliBiliCreateTaskJob.java
......@@ -47,8 +47,6 @@ public class BiliBiliCreateTaskJob extends CreateTaskJob {
});
} catch (Exception e) {
log.error(e.getMessage(), e);
} finally {
super.finish();
}
}
}
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/CreateTaskJob.java
package com.zhiweidata.automatictest.barragecrawlerserver.createtasks;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.mongodb.client.MongoCollection;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
import com.zhiweidata.automatictest.barragecrawlerserver.exporttasks.BarrageExportTaskJob;
import com.zhiweidata.automatictest.barragecrawlerserver.util.TimeUtil;
......@@ -15,7 +14,6 @@ import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ByteArrayEntity;
import org.apache.http.entity.ContentType;
import org.bson.Document;
import org.jetbrains.annotations.NotNull;
import org.quartz.Job;
import org.quartz.JobBuilder;
......@@ -26,12 +24,11 @@ import org.quartz.TriggerBuilder;
import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.BARRAGE_COLLECTION_TASK_JOB_KEY;
import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.EXPORT_START_MAX_DELAY;
import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.MONGO_LIVE_COLLECTION;
import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.MONGO_TASK_REQUEST_COLLECTION;
import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.TASK_URL;
import static com.zhiweidata.automatictest.publics.BeanContainer.HTTP_CLIENT;
import static com.zhiweidata.automatictest.publics.BeanContainer.MONGO_DATABASE;
import static com.zhiweidata.automatictest.publics.BeanContainer.STANDARD_DATE_FORMAT;
import static com.zhiweidata.automatictest.publics.BeanContainer.SCHEDULER;
import static com.zhiweidata.automatictest.publics.BeanContainer.STANDARD_DATE_FORMAT;
/**
* @author aszswaz
......@@ -41,8 +38,6 @@ import static com.zhiweidata.automatictest.publics.BeanContainer.SCHEDULER;
@Slf4j
@SuppressWarnings({"JavaDoc"})
public abstract class CreateTaskJob implements Job {
private final List<Document> liveUrls = new ArrayList<>();
/**
* 请求弹幕采集服务器创建采集任务
*/
......@@ -61,25 +56,18 @@ public abstract class CreateTaskJob implements Job {
HttpResponse response = HTTP_CLIENT.execute(post);
Map<?, ?> jsonMap = objectMapper.readValue(response.getEntity().getContent(), Map.class);
Document document = new Document("liveUrl", liveUrl);
jsonMap.forEach((key, value) -> document.put((String) key, value));
this.liveUrls.add(document);
collectionTask.setCode((Integer) jsonMap.get("code"));
collectionTask.setMessage((String) jsonMap.get("message"));
MONGO_TASK_REQUEST_COLLECTION.insertOne(collectionTask);
return collectionTask;
}
protected final void finish() {
if (!this.liveUrls.isEmpty()) {
MongoCollection<Document> liveUrls = MONGO_DATABASE.getCollection(MONGO_LIVE_COLLECTION);
liveUrls.insertMany(this.liveUrls);
}
}
/**
* 创建定时导出任务
*/
protected final <T extends BarrageExportTaskJob> void createExportTask(BarrageCollectionTask collectionTask, Class<T> tClass) throws SchedulerException {
protected final <T extends BarrageExportTaskJob> void createExportTask(@NotNull BarrageCollectionTask collectionTask, Class<T> tClass) throws SchedulerException {
JobBuilder jobBuilder = JobBuilder.newJob(tClass);
String name = tClass.getSimpleName(), group = "export";
String name = collectionTask.getName(), group = tClass.getSimpleName();
jobBuilder.withIdentity(name, group);
jobBuilder.withDescription("弹幕导出，以及导出的excel文件的分析");
JobDataMap dataMap = new JobDataMap();
......
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/DouyuCreateTaskJob.java
......@@ -59,8 +59,6 @@ public class DouyuCreateTaskJob extends CreateTaskJob {
});
} catch (Exception e) {
log.error(e.getMessage(), e);
} finally {
super.finish();
}
}
......
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/HuyaCreateTaskJob.java
......@@ -2,7 +2,6 @@ package com.zhiweidata.automatictest.barragecrawlerserver.createtasks;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.mongodb.client.MongoCollection;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
import com.zhiweidata.automatictest.barragecrawlerserver.exception.BarrageHomePageException;
import com.zhiweidata.automatictest.barragecrawlerserver.exporttasks.HuyaExportTaskJob;
......@@ -19,16 +18,13 @@ import org.apache.http.HttpStatus;
import org.apache.http.StatusLine;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.util.EntityUtils;
import org.bson.Document;
import org.jsoup.nodes.Element;
import org.quartz.DisallowConcurrentExecution;
import org.quartz.JobExecutionContext;
import org.seimicrawler.xpath.JXDocument;
import org.seimicrawler.xpath.JXNode;
import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.MONGO_LIVE_COLLECTION;
import static com.zhiweidata.automatictest.publics.BeanContainer.HTTP_CLIENT;
import static com.zhiweidata.automatictest.publics.BeanContainer.MONGO_DATABASE;
/**
* 从虎牙直播的首页获得直播间的地址，并发送到弹幕采集服务器进行测试
......@@ -39,16 +35,12 @@ import static com.zhiweidata.automatictest.publics.BeanContainer.MONGO_DATABASE;
*/
@Slf4j
@DisallowConcurrentExecution
@SuppressWarnings({"JavaDoc", "unused", "SpellCheckingInspection", "GrazieInspection"})
@SuppressWarnings({"JavaDoc", "unused", "SpellCheckingInspection"})
public class HuyaCreateTaskJob extends CreateTaskJob {
/**
* 幻灯片中，直播间的url匹配
*/
private static final Pattern LIVE_URL = Pattern.compile("var slides= \\[\\{[\\w\\W]+?}];");
/**
* 直播间地址集合
*/
private final MongoCollection<Document> liveUrls = MONGO_DATABASE.getCollection(MONGO_LIVE_COLLECTION);
@Override
public void execute(JobExecutionContext jobExecutionContext) {
......@@ -64,8 +56,6 @@ public class HuyaCreateTaskJob extends CreateTaskJob {
this.createTaskByHtml(EntityUtils.toString(response.getEntity()));
} catch (Exception e) {
log.error(e.getMessage(), e);
} finally {
super.finish();
}
}
......
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/entity/BarrageCollectionTask.java
package com.zhiweidata.automatictest.barragecrawlerserver.entity;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.Calendar;
import lombok.EqualsAndHashCode;
......@@ -40,6 +41,16 @@ public class BarrageCollectionTask {
*/
@JsonProperty(value = "endTime")
private Integer endTime;
/**
* 服务器的响应代码
*/
@JsonIgnore
private int code;
/**
* 响应信息
*/
@JsonIgnore
private String message;
public BarrageCollectionTask(String liveUrl, String liveName, long currentTime) {
this.url = liveUrl;
......@@ -47,10 +58,11 @@ public class BarrageCollectionTask {
Calendar calendar = Calendar.getInstance();
calendar.setTimeInMillis(currentTime);
calendar.set(Calendar.MINUTE, 0);
// calendar.set(Calendar.MINUTE, 0);
calendar.add(Calendar.MINUTE, 1);
calendar.set(Calendar.SECOND, 0);
calendar.set(Calendar.MILLISECOND, 0);
calendar.add(Calendar.HOUR_OF_DAY, 1);
// calendar.add(Calendar.HOUR_OF_DAY, 1);
this.startTime = (int) (calendar.getTimeInMillis() / 1000);
int minute = (int) (Math.random() * (MAX_COLLECTION_TIME + 1));
......
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/BiliBiliExportTaskJob.java
......@@ -2,6 +2,7 @@ package com.zhiweidata.automatictest.barragecrawlerserver.exporttasks;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
import lombok.extern.slf4j.Slf4j;
import org.quartz.JobDataMap;
import org.quartz.JobExecutionContext;
import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.BARRAGE_COLLECTION_TASK_JOB_KEY;
......@@ -20,7 +21,8 @@ public class BiliBiliExportTaskJob implements BarrageExportTaskJob {
@Override
public void execute(JobExecutionContext context) {
try {
BarrageCollectionTask collectionTask = (BarrageCollectionTask) requireNonNull(context.get(BARRAGE_COLLECTION_TASK_JOB_KEY));
JobDataMap dataMap = requireNonNull(context.getMergedJobDataMap());
BarrageCollectionTask collectionTask = (BarrageCollectionTask) requireNonNull(dataMap.get(BARRAGE_COLLECTION_TASK_JOB_KEY));
} catch (Exception e) {
log.error(e.getMessage(), e);
}
......
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/DouyuExportTaskJob.java
......@@ -2,6 +2,7 @@ package com.zhiweidata.automatictest.barragecrawlerserver.exporttasks;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
import lombok.extern.slf4j.Slf4j;
import org.quartz.JobDataMap;
import org.quartz.JobExecutionContext;
import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.BARRAGE_COLLECTION_TASK_JOB_KEY;
......@@ -20,7 +21,8 @@ public class DouyuExportTaskJob implements BarrageExportTaskJob {
@Override
public void execute(JobExecutionContext context) {
try {
BarrageCollectionTask collectionTask = (BarrageCollectionTask) requireNonNull(context.get(BARRAGE_COLLECTION_TASK_JOB_KEY));
JobDataMap dataMap = requireNonNull(context.getMergedJobDataMap());
BarrageCollectionTask collectionTask = (BarrageCollectionTask) requireNonNull(dataMap.get(BARRAGE_COLLECTION_TASK_JOB_KEY));
} catch (Exception e) {
log.error(e.getMessage(), e);
}
......
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/HuyaExportTaskJob.java
......@@ -2,6 +2,7 @@ package com.zhiweidata.automatictest.barragecrawlerserver.exporttasks;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
import lombok.extern.slf4j.Slf4j;
import org.quartz.JobDataMap;
import org.quartz.JobExecutionContext;
import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.BARRAGE_COLLECTION_TASK_JOB_KEY;
......@@ -20,7 +21,8 @@ public class HuyaExportTaskJob implements BarrageExportTaskJob {
@Override
public void execute(JobExecutionContext context) {
try {
BarrageCollectionTask collectionTask = (BarrageCollectionTask) requireNonNull(context.get(BARRAGE_COLLECTION_TASK_JOB_KEY));
JobDataMap dataMap = requireNonNull(context.getMergedJobDataMap());
BarrageCollectionTask collectionTask = (BarrageCollectionTask) requireNonNull(dataMap.get(BARRAGE_COLLECTION_TASK_JOB_KEY));
} catch (Exception e) {
log.error(e.getMessage(), e);
}
......
public/src/main/java/com/zhiweidata/automatictest/publics/BeanContainer.java
......@@ -9,10 +9,17 @@ import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.bson.codecs.configuration.CodecProvider;
import org.bson.codecs.configuration.CodecRegistry;
import org.bson.codecs.pojo.PojoCodecProvider;
import org.quartz.Scheduler;
import org.quartz.SchedulerException;
import org.quartz.impl.StdSchedulerFactory;
import static com.mongodb.MongoClientSettings.getDefaultCodecRegistry;
import static org.bson.codecs.configuration.CodecRegistries.fromProviders;
import static org.bson.codecs.configuration.CodecRegistries.fromRegistries;
/**
* Bean容器
*
......@@ -66,6 +73,9 @@ public class BeanContainer {
HTTP_CLIENT = clientBuilder.build();
MONGO_CLIENT = MongoClients.create("mongodb://admin:z199809051593@192.168.0.119");
MONGO_DATABASE = MONGO_CLIENT.getDatabase("automatic_test");
// 启用 POJO 对象映射
CodecProvider pojoCodecProvider = PojoCodecProvider.builder().automatic(true).build();
CodecRegistry pojoCodecRegistry = fromRegistries(getDefaultCodecRegistry(), fromProviders(pojoCodecProvider));
MONGO_DATABASE = MONGO_CLIENT.getDatabase("automatic_test").withCodecRegistry(pojoCodecRegistry);
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment