backup

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/BiliBiliCreateTaskJob.java

@@ -27,17 +27,20 @@ public class BiliBiliCreateTaskJob extends CreateTaskJob {
             Map<?, ?> liveJson = new ObjectMapper().readValue(new URL("https://api.live.bilibili.com/xlive/web-interface/v1/index/getList?platform=web"), Map.class);
             liveJson = (Map<?, ?>) requireNonNull(liveJson.get("data"));
             final List<?> moduleList = (List<?>) liveJson.get("room_list");
+            final String baseUrl = "https://live.bilibili.com";
             moduleList.forEach(liveModule -> {
                 Map<?, ?> liveModuleMap = (Map<?, ?>) liveModule;
                 List<?> roomList = (List<?>) liveModuleMap.get("list");
                 roomList.forEach(room -> {
                     Map<?, ?> roomMap = (Map<?, ?>) room;
                     this.liveTitle = (String) roomMap.get("title");
-                    BiliBiliCreateTaskJob.super.createCollectionTask((String) roomMap.get("link"));
+                    BiliBiliCreateTaskJob.super.createCollectionTask(baseUrl + roomMap.get("link"));
                 });
             });
         } catch (Exception e) {
             log.error(e.getMessage(), e);
+        } finally {
+            super.finish();
         }
     }
 

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/CreateTaskJob.java

@@ -41,11 +41,12 @@ public abstract class CreateTaskJob implements Job {
      */
     protected final void createCollectionTask(String liveUrl) {
         try {
-            BarrageCollectionTask collectionTask = new BarrageCollectionTask(liveUrl, this.getLiveName(liveUrl));
+            String liveName = this.getLiveName(liveUrl);
+            BarrageCollectionTask collectionTask = new BarrageCollectionTask(liveUrl, liveName);
             if (log.isDebugEnabled()) {
                 String startTime = STANDARD_DATE_FORMAT.format(collectionTask.getStartTime() * 1000L);
                 String endTime = STANDARD_DATE_FORMAT.format(collectionTask.getEndTime() * 1000L);
-                log.info("collection start time: {}, collection end time: {}", startTime, endTime);
+                log.debug("collection start time: {}, collection end time: {}", startTime, endTime);
             }
             HttpPost post = new HttpPost(TASK_URL);
             ObjectMapper objectMapper = new ObjectMapper();

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/DouyuCreateTaskJob.java

@@ -3,6 +3,7 @@ package com.zhiweidata.automatictest.barragecrawlerserver.createtasks;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.zhiweidata.automatictest.barragecrawlerserver.exception.BarrageHomePageException;
+import java.net.URL;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -11,6 +12,7 @@ import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.http.HttpException;
 import org.apache.http.HttpResponse;
 import org.apache.http.HttpStatus;
@@ -45,6 +47,8 @@ public class DouyuCreateTaskJob extends CreateTaskJob {
             liveUrls.forEach(super::createCollectionTask);
         } catch (Exception e) {
             log.error(e.getMessage(), e);
+        } finally {
+            super.finish();
         }
     }
 
@@ -122,10 +126,16 @@ public class DouyuCreateTaskJob extends CreateTaskJob {
             String html = EntityUtils.toString(response.getEntity());
             JXDocument document = JXDocument.create(html);
             JXNode node = document.selNOne("//h3[@class='Title-header']");
-            return node.asElement().text();
+            String name = node.asElement().text();
+            if (StringUtils.isBlank(name)) {
+                String roomId = liveUrl.substring(liveUrl.lastIndexOf('/'));
+                Map<?, ?> jsonMap = new ObjectMapper().readValue(new URL("https://www.douyu.com/betard" + roomId), Map.class);
+                jsonMap = (Map<?, ?>) Objects.requireNonNull(jsonMap.get("room"));
+                name = (String) jsonMap.get("room_name");
+            }
+            return name;
         } catch (Exception e) {
-            log.error(e.getMessage(), e);
-            return null;
+            return "autio-create";
         }
     }
 }

barrage-crawler-server/src/main/java/com/zhiweidata/automatictest/barragecrawlerserver/createtasks/HuyaCreateTaskJob.java

@@ -62,6 +62,8 @@ public class HuyaCreateTaskJob extends CreateTaskJob {
             this.createTaskByHtml(EntityUtils.toString(response.getEntity()));
         } catch (Exception e) {
             log.error(e.getMessage(), e);
+        } finally {
+            super.finish();
         }
     }
 
@@ -100,7 +102,6 @@ public class HuyaCreateTaskJob extends CreateTaskJob {
             log.warn("虎牙直播间地址数量：{}", size);
         }
         liveUrls.forEach(super::createCollectionTask);
-        super.finish();
     }
 
     @NotNull

public/src/main/java/com/zhiweidata/automatictest/publics/BeanContainer.java

@@ -61,6 +61,7 @@ public class BeanContainer {
         configBuilder.setConnectionRequestTimeout(timeout);
         configBuilder.setConnectTimeout(timeout);
         configBuilder.setSocketTimeout(timeout);
+        configBuilder.setRedirectsEnabled(true);
         clientBuilder.setDefaultRequestConfig(configBuilder.build());
         HTTP_CLIENT = clientBuilder.build();