Commit 9fcfba2d by zhiwei

各个采集验证添加休眠,避免数据过多导致程序阻塞

parent aa059934
......@@ -2,6 +2,7 @@ package com.zhiwei.source_forward.crawler;
import java.util.Objects;
import com.zhiwei.tools.tools.ZhiWeiTools;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
......@@ -47,6 +48,7 @@ public class ContentCrawler {
ContentDataCallback callback, String... urls) {
if (urls != null && urls.length > 0) {
for (String url : urls) {
ZhiWeiTools.sleep(100);
if (url != null) {
try {
search(counter, url, Attribution.of(url), callback);
......
......@@ -5,6 +5,7 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import com.zhiwei.tools.tools.ZhiWeiTools;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jsoup.Jsoup;
......@@ -64,6 +65,7 @@ public class MediaSelfSourceCrawler {
private void start(GroupSync counter,MediaSelfSourceDataCallBack callback, String... urls) {
if (urls != null && urls.length > 0) {
for (String url : urls) {
ZhiWeiTools.sleep(100);
counter.add();
if (url != null) {
try {
......
......@@ -5,6 +5,7 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import com.zhiwei.tools.tools.ZhiWeiTools;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jsoup.Jsoup;
......@@ -46,6 +47,7 @@ public class SourceForwardCrawler {
if (urls != null && urls.length > 0) {
for (String url : urls) {
counter.add();
ZhiWeiTools.sleep(100);
if (url != null) {
try {
search(counter, url, Attribution.of(url), callback);
......
......@@ -50,6 +50,7 @@ public class UrlLiveCrawler {
if (nonNull(urls) && urls.length > 0) {
for (String url : urls) {
counter.add();
ZhiWeiTools.sleep(100);
if (nonNull(url)) {
try {
// ZhiWeiTools.sleep(3000);
......
......@@ -325,7 +325,6 @@ public class MatchSource {
source = source.replaceAll(".*来源:|)", "");
}
}
if(Objects.nonNull(source) && source.length() != 0) {
return source;
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment