backup

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/BiliBiliCreateTaskJob.java

 package com.zhiweidata.automatictest.barragecrawlerserver.createtasks;
 
+import org.apache.poi.ss.usermodel.CellRange;
 import org.quartz.Job;
 import org.quartz.JobExecutionContext;
 
@@ -11,8 +12,13 @@ import org.quartz.JobExecutionContext;
  * @ide IntelliJ IDEA
  */
 @SuppressWarnings("JavaDoc")
-public class BiliBiliCreateTaskJob implements Job {
+public class BiliBiliCreateTaskJob extends CreateTaskJob {
     @Override
     public void execute(JobExecutionContext jobExecutionContext) {
     }
+
+    @Override
+    public String getLiveName(String liveUrl) {
+        return null;
+    }
 }

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/CreateTaskJob.java

+package com.zhiweidata.automatictest.barragecrawlerserver.createtasks;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.mongodb.client.MongoCollection;
+import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.entity.ContentType;
+import org.bson.Document;
+import org.quartz.Job;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.MONGO_LIVE_COLLECTION;
+import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.TASK_URL;
+import static com.zhiweidata.automatictest.publics.BeanContainer.HTTP_CLIENT;
+import static com.zhiweidata.automatictest.publics.BeanContainer.MONGO_DATABASE;
+import static com.zhiweidata.automatictest.publics.BeanContainer.STANDARD_DATE_FORMAT;
+
+/**
+ * @author aszswaz
+ * @createTime 2021-08-24 18:02:35
+ * @ide IntelliJ IDEA
+ */
+@Slf4j
+@SuppressWarnings({"JavaDoc", "GrazieInspection"})
+public abstract class CreateTaskJob implements Job {
+    private final List<Document> liveUrls = new ArrayList<>();
+
+    /**
+     * 获取直播间名称
+     */
+    protected abstract String getLiveName(String liveUrl) throws IOException;
+
+    /**
+     * 请求弹幕采集服务器创建采集任务
+     */
+    protected final void createCollectionTask(String liveUrl) {
+        try {
+            BarrageCollectionTask collectionTask = new BarrageCollectionTask(liveUrl, this.getLiveName(liveUrl));
+            if (log.isDebugEnabled()) {
+                String startTime = STANDARD_DATE_FORMAT.format(collectionTask.getStartTime() * 1000L);
+                String endTime = STANDARD_DATE_FORMAT.format(collectionTask.getEndTime() * 1000L);
+                log.info("collection start time: {}, collection end time: {}", startTime, endTime);
+            }
+            HttpPost post = new HttpPost(TASK_URL);
+            ObjectMapper objectMapper = new ObjectMapper();
+            post.setEntity(new ByteArrayEntity(objectMapper.writeValueAsBytes(collectionTask), ContentType.APPLICATION_JSON));
+            HttpResponse response = HTTP_CLIENT.execute(post);
+            Map<?, ?> jsonMap = objectMapper.readValue(response.getEntity().getContent(), Map.class);
+            int code = (Integer) jsonMap.get("code");
+            Document document = new Document("liveUrl", liveUrl);
+            jsonMap.forEach((key, value) -> document.put((String) key, value));
+            this.liveUrls.add(document);
+        } catch (Exception e) {
+            log.error(e.getMessage(), e);
+        }
+    }
+
+    protected final void finish() {
+        MongoCollection<Document> liveUrls = MONGO_DATABASE.getCollection(MONGO_LIVE_COLLECTION);
+        liveUrls.insertMany(this.liveUrls);
+    }
+}

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/DouyuCreateTaskJob.java

@@ -11,8 +11,13 @@ import org.quartz.JobExecutionContext;
  * @ide IntelliJ IDEA
  */
 @SuppressWarnings({"JavaDoc", "unused", "SpellCheckingInspection"})
-public class DouyuCreateTaskJob implements Job {
+public class DouyuCreateTaskJob extends CreateTaskJob {
     @Override
     public void execute(JobExecutionContext jobExecutionContext) {
     }
+
+    @Override
+    public String getLiveName(String liveUrl) {
+        return null;
+    }
 }

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/HuyaCreateTaskJob.java

@@ -3,7 +3,6 @@ package com.zhiweidata.automatictest.barragecrawlerserver.createtasks;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.mongodb.client.MongoCollection;
-import com.zhiweidata.automatictest.barragecrawlerserver.entity.BarrageCollectionTask;
 import com.zhiweidata.automatictest.barragecrawlerserver.exception.BarrageHomePageException;
 import java.io.IOException;
 import java.util.HashSet;
@@ -18,23 +17,17 @@ import org.apache.http.HttpResponse;
 import org.apache.http.HttpStatus;
 import org.apache.http.StatusLine;
 import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.entity.ByteArrayEntity;
-import org.apache.http.entity.ContentType;
 import org.apache.http.util.EntityUtils;
 import org.bson.Document;
 import org.jetbrains.annotations.NotNull;
 import org.jsoup.nodes.Element;
-import org.quartz.Job;
 import org.quartz.JobExecutionContext;
 import org.seimicrawler.xpath.JXDocument;
 import org.seimicrawler.xpath.JXNode;
 
 import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.MONGO_LIVE_COLLECTION;
-import static com.zhiweidata.automatictest.barragecrawlerserver.config.BaseConfig.TASK_URL;
 import static com.zhiweidata.automatictest.publics.BeanContainer.HTTP_CLIENT;
 import static com.zhiweidata.automatictest.publics.BeanContainer.MONGO_DATABASE;
-import static com.zhiweidata.automatictest.publics.BeanContainer.STANDARD_DATE_FORMAT;
 
 /**
  * 从虎牙直播的首页获得直播间的地址，并发送到弹幕采集服务器进行测试
@@ -45,7 +38,7 @@ import static com.zhiweidata.automatictest.publics.BeanContainer.STANDARD_DATE_F
  */
 @Slf4j
 @SuppressWarnings({"JavaDoc", "unused", "SpellCheckingInspection", "GrazieInspection"})
-public class HuyaCreateTaskJob implements Job {
+public class HuyaCreateTaskJob extends CreateTaskJob {
     /**
      * 幻灯片中，直播间的url匹配
      */
@@ -106,36 +99,12 @@ public class HuyaCreateTaskJob implements Job {
         } else {
             log.warn("虎牙直播间地址数量：{}", size);
         }
-        liveUrls.forEach(this::createCollectionTask);
-    }
-
-    /**
-     * 请求弹幕采集服务器创建采集任务
-     */
-    private void createCollectionTask(String liveUrl) {
-        try {
-            BarrageCollectionTask collectionTask = new BarrageCollectionTask(liveUrl, this.getLiveName(liveUrl));
-            if (log.isDebugEnabled()) {
-                String startTime = STANDARD_DATE_FORMAT.format(collectionTask.getStartTime() * 1000L);
-                String endTime = STANDARD_DATE_FORMAT.format(collectionTask.getEndTime() * 1000L);
-                log.info("collection start time: {}, collection end time: {}", startTime, endTime);
-            }
-            HttpPost post = new HttpPost(TASK_URL);
-            ObjectMapper objectMapper = new ObjectMapper();
-            post.setEntity(new ByteArrayEntity(objectMapper.writeValueAsBytes(collectionTask), ContentType.APPLICATION_JSON));
-            HttpResponse response = HTTP_CLIENT.execute(post);
-            Map<?, ?> jsonMap = objectMapper.readValue(response.getEntity().getContent(), Map.class);
-            int code = (Integer) jsonMap.get("code");
-            Document document = new Document("liveUrl", liveUrl);
-            jsonMap.forEach((key, value) -> document.put((String) key, value));
-            this.liveUrls.insertOne(document);
-        } catch (Exception e) {
-            log.error(e.getMessage(), e);
-        }
+        liveUrls.forEach(super::createCollectionTask);
+        super.finish();
     }
 
     @NotNull
-    private String getLiveName(String liveUrl) throws IOException {
+    public String getLiveName(String liveUrl) throws IOException {
         HttpResponse response = HTTP_CLIENT.execute(new HttpGet(liveUrl));
         String html = EntityUtils.toString(response.getEntity());
         JXDocument document = JXDocument.create(html);