Skip to content

A
automatic-test
Commit 98fece20 by 朽木不可雕也
Options

backup

parent 20dfe82d
Showing with 132 additions and 213 deletions
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/BiliBiliCreateTaskJob.java
......@@ -2,7 +2,7 @@ package com.zhiweidata.automatictest.barragecrawlerserver.createtasks;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask;
import com.zhiweidata.automatictest.barragecrawlerserver.exporttasks.BiliBiliExportTaskJob;
import com.zhiweidata.automatictest.barragecrawlerserver.exporttasks.ExportTaskJob;
import java.net.URL;
import java.util.List;
import java.util.Map;
......@@ -39,7 +39,7 @@ public class BiliBiliCreateTaskJob extends CreateTaskJob {
try {
Map<?, ?> roomMap = (Map<?, ?>) room;
BarrageCollectionTestTask collectionTask = BiliBiliCreateTaskJob.super.createCollectionTask(baseUrl + roomMap.get("link"), currentTime);
BiliBiliCreateTaskJob.super.createExportTask(collectionTask, BiliBiliExportTaskJob.class);
BiliBiliCreateTaskJob.super.createExportTask(collectionTask);
} catch (Exception e) {
log.error(e.getMessage(), e);
}
......
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/CreateTaskJob.java
......@@ -3,11 +3,12 @@ package com.zhiweidata.automatictest.barragecrawlerserver.createtasks;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.ServerResponseMessage;
import com.zhiweidata.automatictest.barragecrawlerserver.exporttasks.BarrageExportTaskJob;
import com.zhiweidata.automatictest.barragecrawlerserver.exporttasks.ExportTaskJob;
import com.zhiweidata.automatictest.barragecrawlerserver.mapper.BarrageCollectionTestTaskMapper;
import com.zhiweidata.automatictest.barragecrawlerserver.mapper.ServerResponseMessageMapper;
import com.zhiweidata.automatictest.barragecrawlerserver.util.TimeUtil;
import java.io.IOException;
import java.util.Set;
import java.util.UUID;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpResponse;
......@@ -40,6 +41,21 @@ import static com.zhiweidata.automatictest.publics.BeanContainer.STANDARD_DATE_F
@SuppressWarnings({"JavaDoc"})
public abstract class CreateTaskJob implements Job {
/**
* 创建直播间的弹幕采集，和弹幕导出测试任务
*/
protected final void createTasks(@NotNull Set<String> liveUrls) {
final long currentTime = System.currentTimeMillis();
liveUrls.forEach(liveUrl -> {
try {
BarrageCollectionTestTask collectionTask = this.createCollectionTask(liveUrl, currentTime);
this.createExportTask(collectionTask);
} catch (Exception e) {
log.error(e.getMessage(), e);
}
});
}
/**
* 请求弹幕采集服务器创建采集任务
*/
@NotNull
......@@ -70,9 +86,10 @@ public abstract class CreateTaskJob implements Job {
/**
* 创建定时导出任务
*/
protected final <T extends BarrageExportTaskJob> void createExportTask(@NotNull BarrageCollectionTestTask collectionTask, Class<T> tClass) throws SchedulerException {
JobBuilder jobBuilder = JobBuilder.newJob(tClass);
String name = collectionTask.getName(), group = tClass.getSimpleName();
protected final <T extends ExportTaskJob> void createExportTask(@NotNull BarrageCollectionTestTask collectionTask) throws SchedulerException {
Class<ExportTaskJob> taskJobClass = ExportTaskJob.class;
JobBuilder jobBuilder = JobBuilder.newJob(taskJobClass);
String name = collectionTask.getName(), group = taskJobClass.getSimpleName();
jobBuilder.withIdentity(name, group);
jobBuilder.withDescription("弹幕导出，以及导出的excel文件的分析");
JobDataMap dataMap = new JobDataMap();
......
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/DouyuCreateTaskJob.java
......@@ -4,7 +4,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask;
import com.zhiweidata.automatictest.barragecrawlerserver.exception.BarrageHomePageException;
import com.zhiweidata.automatictest.barragecrawlerserver.exporttasks.DouyuExportTaskJob;
import com.zhiweidata.automatictest.barragecrawlerserver.exporttasks.ExportTaskJob;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
......@@ -31,7 +31,7 @@ import static com.zhiweidata.automatictest.publics.BeanContainer.HTTP_CLIENT;
*/
@Slf4j
@DisallowConcurrentExecution
@SuppressWarnings({"JavaDoc", "unused", "SpellCheckingInspection", "GrazieInspection", "DuplicatedCode"})
@SuppressWarnings({"JavaDoc", "unused", "SpellCheckingInspection", "GrazieInspection"})
public class DouyuCreateTaskJob extends CreateTaskJob {
private static final Pattern PATTERN = Pattern.compile("var \\$mainData = \\[[\\w\\W]+?(];)");
......@@ -41,15 +41,7 @@ public class DouyuCreateTaskJob extends CreateTaskJob {
HttpResponse response = HTTP_CLIENT.execute(new HttpGet("https://www.douyu.com/"));
String html = EntityUtils.toString(response.getEntity());
Set<String> liveUrls = this.getLiveUrl(html);
final long currentTime = System.currentTimeMillis();
liveUrls.forEach(liveUrl -> {
try {
BarrageCollectionTestTask collectionTask = super.createCollectionTask(liveUrl, currentTime);
super.createExportTask(collectionTask, DouyuExportTaskJob.class);
} catch (Exception e) {
log.error(e.getMessage(), e);
}
});
super.createTasks(liveUrls);
} catch (Exception e) {
log.error(e.getMessage(), e);
}
......@@ -86,21 +78,13 @@ public class DouyuCreateTaskJob extends CreateTaskJob {
for (Object moduleObj : modules) {
List<?> rl = (List<?>) ((Map<?, ?>) moduleObj).get("rl");
if (Objects.isNull(rl)) continue;
for (Object rlObj : rl) {
String urlString = (String) ((Map<?, ?>) rlObj).get("url");
if (urlString.startsWith("/")) urlString = urlString.replaceFirst("/", "");
liveUrls.add(baseUrl + urlString);
}
this.assemblyPruneUrl(baseUrl, rl, liveUrls);
}
} else if ("recommend".equals(type) || "roomList10".equals(type)) {
// 精彩推荐 和 热门游戏（如 英雄联盟，穿越火线，王者荣耀，CFHD，和平精英，COD手游，云顶模式，天刀手游，竞技游戏，网络游戏，客厅游戏，手机游戏，暴雪专区，科技文化，官方活动 等）
List<?> rl = (List<?>) jsonMap.get("rl");
if (Objects.isNull(rl)) continue;
for (Object rlObj : rl) {
String urlString = (String) ((Map<?, ?>) rlObj).get("url");
if (urlString.startsWith("/")) urlString = urlString.replaceFirst("/", "");
liveUrls.add(baseUrl + urlString);
}
this.assemblyPruneUrl(baseUrl, rl, liveUrls);
} else if ("footerHotroom".equals(type)) {
List<?> ds = (List<?>) jsonMap.get("ds");
for (Object dsObj : ds) {
......@@ -117,4 +101,16 @@ public class DouyuCreateTaskJob extends CreateTaskJob {
throw new BarrageHomePageException("斗鱼直播间弹幕获取失败");
}
}
/**
* 拼接url，以及修建url尾部的“/”
*/
@SuppressWarnings("SameParameterValue")
private void assemblyPruneUrl(String baseUrl, @NotNull List<?> rl, @NotNull Set<String> liveUrls) {
for (Object rlObj : rl) {
String urlString = (String) ((Map<?, ?>) rlObj).get("url");
if (urlString.startsWith("/")) urlString = urlString.replaceFirst("/", "");
liveUrls.add(baseUrl + urlString);
}
}
}
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/HuyaCreateTaskJob.java
......@@ -4,7 +4,6 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask;
import com.zhiweidata.automatictest.barragecrawlerserver.exception.BarrageHomePageException;
import com.zhiweidata.automatictest.barragecrawlerserver.exporttasks.HuyaExportTaskJob;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
......@@ -93,14 +92,6 @@ public class HuyaCreateTaskJob extends CreateTaskJob {
} else {
log.warn("虎牙直播间地址数量：{}", size);
}
final long currentTime = System.currentTimeMillis();
liveUrls.forEach(liveUrl -> {
try {
BarrageCollectionTestTask collectionTask = super.createCollectionTask(liveUrl, currentTime);
super.createExportTask(collectionTask, HuyaExportTaskJob.class);
} catch (Exception e) {
log.error(e.getMessage(), e);
}
});
super.createTasks(liveUrls);
}
}
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/BarrageExportTaskJob.java deleted 100644 → 0
package com.zhiweidata.automatictest.barragecrawlerserver.exporttasks;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionInfo;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageExportResult;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.ServerResponseMessage;
import com.zhiweidata.automatictest.barragecrawlerserver.mapper.BarrageExportResultMapper;
import com.zhiweidata.automatictest.barragecrawlerserver.mapper.ServerResponseMessageMapper;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import org.apache.ibatis.session.SqlSession;
import org.jetbrains.annotations.NotNull;
import org.quartz.Job;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static com.zhiweidata.automatictest.publics.BeanContainer.SQL_SESSION_FACTORY;
import static java.util.Objects.isNull;
import static java.util.Objects.nonNull;
import static java.util.Objects.requireNonNull;
/**
* 弹幕导出任务
*
* @author aszswaz
* @createTime 2021-08-25 17:29:39
* @ide IntelliJ IDEA
*/
@SuppressWarnings("JavaDoc")
public interface BarrageExportTaskJob extends Job {
Logger log = LoggerFactory.getLogger(BarrageExportTaskJob.class);
/**
* 下载并解压zip文件
*
* @return 解压得到的文件
*/
default List<File> write(File cacheDir, InputStream inputStream) throws IOException {
if (!cacheDir.exists()) if (!cacheDir.mkdirs()) log.error("文件夹{}创建失败", cacheDir.getName());
final List<File> outFiles = new ArrayList<>();
try (ZipInputStream zipInputStream = new ZipInputStream(inputStream)) {
byte[] buff = new byte[8192];
int len;
while (true) {
ZipEntry zipEntry = zipInputStream.getNextEntry();
if (isNull(zipEntry)) break;
File execFile = new File(cacheDir, requireNonNull(zipEntry.getName()));
try (FileOutputStream fileOutputStream = new FileOutputStream(execFile)) {
while ((len = zipInputStream.read(buff)) != -1) fileOutputStream.write(buff, 0, len);
}
zipInputStream.closeEntry();
outFiles.add(execFile);
}
}
return outFiles;
}
/**
* 删除文件
*/
default void removeFile(File file) {
// 如果是文件夹，先删除文件夹中的文件
if (file.isDirectory()) {
File[] files = file.listFiles();
if (nonNull(files)) {
for (File subFile : files) this.removeFile(subFile);
}
}
// 删除文件夹或文件
if (!file.delete()) log.error("文件夹：{}删除失败", file.getName());
}
/**
* 保存服务器的响应信息
*/
default void saveResponseMessage(ServerResponseMessage message) {
try (SqlSession session = SQL_SESSION_FACTORY.openSession(true)) {
ServerResponseMessageMapper messageMapper = session.getMapper(ServerResponseMessageMapper.class);
messageMapper.insert(message);
}
}
/**
* 保存弹幕的导出信息
*
* @param collectionInfo 弹幕采集任务信息
* @param exportCount 导出的弹幕数量
*/
default void saveExportInfo(@NotNull BarrageCollectionInfo collectionInfo, int exportCount) {
try (SqlSession session = SQL_SESSION_FACTORY.openSession(true)) {
BarrageExportResult exportResult = new BarrageExportResult();
exportResult.setName(collectionInfo.getName());
exportResult.setExportVolume(exportCount);
exportResult.setTotalCollection(collectionInfo.getCollectionCount());
exportResult.setPercentage((double) exportCount / collectionInfo.getCollectionCount());
BarrageExportResultMapper resultMapper = session.getMapper(BarrageExportResultMapper.class);
resultMapper.insert(exportResult);
}
}
}
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/DouyuExportTaskJob.java deleted 100644 → 0
package com.zhiweidata.automatictest.barragecrawlerserver.exporttasks;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask;
import lombok.extern.slf4j.Slf4j;
import org.quartz.JobDataMap;
import org.quartz.JobExecutionContext;
import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.BARRAGE_COLLECTION_TASK_JOB_KEY;
import static java.util.Objects.requireNonNull;
/**
* 斗鱼弹幕导出任务
*
* @author aszswaz
* @createTime 2021-08-25 17:30:18
* @ide IntelliJ IDEA
*/
@Slf4j
@SuppressWarnings("JavaDoc")
public class DouyuExportTaskJob implements BarrageExportTaskJob {
@Override
public void execute(JobExecutionContext context) {
try {
JobDataMap dataMap = requireNonNull(context.getMergedJobDataMap());
BarrageCollectionTestTask collectionTask = (BarrageCollectionTestTask) requireNonNull(dataMap.get(BARRAGE_COLLECTION_TASK_JOB_KEY));
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
}
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/BiliBiliExportTaskJob.java → barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/ExportTaskJob.java
......@@ -7,16 +7,25 @@ import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectio
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionInfo;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageExportResult;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.ServerResponseMessage;
import com.zhiweidata.automatictest.barragecrawlerserver.mapper.BarrageExportResultMapper;
import com.zhiweidata.automatictest.barragecrawlerserver.mapper.ServerResponseMessageMapper;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.entity.ContentType;
import org.apache.http.util.EntityUtils;
import org.apache.ibatis.session.SqlSession;
import org.jetbrains.annotations.NotNull;
import org.quartz.Job;
import org.quartz.JobDataMap;
import org.quartz.JobExecutionContext;
import org.quartz.SchedulerException;
......@@ -25,6 +34,9 @@ import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfi
import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.CACHE_DIR;
import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.TASK_URL;
import static com.zhiweidata.automatictest.publics.BeanContainer.HTTP_CLIENT;
import static com.zhiweidata.automatictest.publics.BeanContainer.SQL_SESSION_FACTORY;
import static java.util.Objects.isNull;
import static java.util.Objects.nonNull;
import static java.util.Objects.requireNonNull;
/**
......@@ -36,14 +48,13 @@ import static java.util.Objects.requireNonNull;
*/
@Slf4j
@SuppressWarnings("JavaDoc")
public class BiliBiliExportTaskJob implements BarrageExportTaskJob {
public class ExportTaskJob implements Job {
/**
* TODO:
* 1. 使用任务的 name 作为关键词，请求接口，获得对应任务的任务ID
* 2. 根据获得的任务ID，尝试从弹幕导出接口，导出弹幕
* 3. 判断服务的响应是 excel 文件，还是 application/json
* 1. 如果是 application/json，就代表导出失败，内容是服务器响应的失败信息
* 2. 如果是 zip 文件就表示，文件导出成功，分析一下文件中的弹幕
* 1. 使用任务的 name 作为关键词，请求接口，获得对应任务的任务ID
* 2. 根据获得的任务ID，尝试从弹幕导出接口，导出弹幕
* 3. 判断服务的响应是 excel 文件，还是 application/json
* 1. 如果是 application/json，就代表导出失败，内容是服务器响应的失败信息
* 2. 如果是 zip 文件就表示，文件导出成功，分析一下文件中的弹幕
*/
@Override
public void execute(JobExecutionContext context) {
......@@ -111,4 +122,75 @@ public class BiliBiliExportTaskJob implements BarrageExportTaskJob {
this.saveExportInfo(collectionInfo, exportCount);
}
}
/**
* 下载并解压zip文件
*
* @return 解压得到的文件
*/
@NotNull
private List<File> write(@NotNull File cacheDir, InputStream inputStream) throws IOException {
if (!cacheDir.exists()) if (!cacheDir.mkdirs()) log.error("文件夹{}创建失败", cacheDir.getName());
final List<File> outFiles = new ArrayList<>();
try (ZipInputStream zipInputStream = new ZipInputStream(inputStream)) {
byte[] buff = new byte[8192];
int len;
while (true) {
ZipEntry zipEntry = zipInputStream.getNextEntry();
if (isNull(zipEntry)) break;
File execFile = new File(cacheDir, requireNonNull(zipEntry.getName()));
try (FileOutputStream fileOutputStream = new FileOutputStream(execFile)) {
while ((len = zipInputStream.read(buff)) != -1) fileOutputStream.write(buff, 0, len);
}
zipInputStream.closeEntry();
outFiles.add(execFile);
}
}
return outFiles;
}
/**
* 删除文件
*/
private void removeFile(@NotNull File file) {
// 如果是文件夹，先删除文件夹中的文件
if (file.isDirectory()) {
File[] files = file.listFiles();
if (nonNull(files)) {
for (File subFile : files) this.removeFile(subFile);
}
}
// 删除文件夹或文件
if (!file.delete()) log.error("文件夹：{}删除失败", file.getName());
}
/**
* 保存服务器的响应信息
*/
private void saveResponseMessage(ServerResponseMessage message) {
try (SqlSession session = SQL_SESSION_FACTORY.openSession(true)) {
ServerResponseMessageMapper messageMapper = session.getMapper(ServerResponseMessageMapper.class);
messageMapper.insert(message);
}
}
/**
* 保存弹幕的导出信息
*
* @param collectionInfo 弹幕采集任务信息
* @param exportCount 导出的弹幕数量
*/
private void saveExportInfo(@NotNull BarrageCollectionInfo collectionInfo, int exportCount) {
try (SqlSession session = SQL_SESSION_FACTORY.openSession(true)) {
BarrageExportResult exportResult = new BarrageExportResult();
exportResult.setName(collectionInfo.getName());
exportResult.setExportVolume(exportCount);
exportResult.setTotalCollection(collectionInfo.getCollectionCount());
exportResult.setPercentage((double) exportCount / collectionInfo.getCollectionCount());
BarrageExportResultMapper resultMapper = session.getMapper(BarrageExportResultMapper.class);
resultMapper.insert(exportResult);
}
}
}
barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/HuyaExportTaskJob.java deleted 100644 → 0
package com.zhiweidata.automatictest.barragecrawlerserver.exporttasks;
import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTestTask;
import lombok.extern.slf4j.Slf4j;
import org.quartz.JobDataMap;
import org.quartz.JobExecutionContext;
import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.BARRAGE_COLLECTION_TASK_JOB_KEY;
import static java.util.Objects.requireNonNull;
/**
* 虎牙弹幕导出任务
*
* @author aszswaz
* @createTime 2021-08-25 17:30:18
* @ide IntelliJ IDEA
*/
@Slf4j
@SuppressWarnings("JavaDoc")
public class HuyaExportTaskJob implements BarrageExportTaskJob {
@Override
public void execute(JobExecutionContext context) {
try {
JobDataMap dataMap = requireNonNull(context.getMergedJobDataMap());
BarrageCollectionTestTask collectionTask = (BarrageCollectionTestTask) requireNonNull(dataMap.get(BARRAGE_COLLECTION_TASK_JOB_KEY));
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
}
barrage-crawler-server/src/test/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/BiliBiliExportTaskJobTest.java → barrage-crawler-server/src/test/java/com/zhiweidata/automatictest/barragecrawlerserver/exporttasks/ExportTaskJobTest.java
......@@ -12,14 +12,14 @@ import static com.zhiweidata.automatictest.publics.BeanContainer.SCHEDULER;
* @ide IntelliJ IDEA
*/
@SuppressWarnings("JavaDoc")
class BiliBiliExportTaskJobTest {
class ExportTaskJobTest {
public static void main(String[] args) throws SchedulerException, IOException {
BarrageCollectionTestTask collectionTask = new BarrageCollectionTestTask(
"https://live.bilibili.com/12265?hotRank=0", "62aae6a3-e563-4f24-9e3a-135c6e74a8ac", System.currentTimeMillis()
);
BiliBiliExportTaskJob exportTaskJob = new BiliBiliExportTaskJob();
ExportTaskJob exportTaskJob = new ExportTaskJob();
exportTaskJob.export(collectionTask);
SCHEDULER.shutdown(true);
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment