Commit cc074bc0 by yangchen

头条版本升级以及代理使用不加时间间隔

parent 9e857f34
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>com.zhiwei</groupId> <groupId>com.zhiwei</groupId>
<artifactId>toutiao</artifactId> <artifactId>toutiao</artifactId>
<version>0.3.1-SNAPSHOT</version> <version>0.3.2-SNAPSHOT</version>
<dependencies> <dependencies>
<dependency> <dependency>
......
...@@ -6,6 +6,7 @@ import java.util.ArrayList; ...@@ -6,6 +6,7 @@ import java.util.ArrayList;
import java.util.Date; import java.util.Date;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
...@@ -72,11 +73,16 @@ public class TouTiaoCommentParse { ...@@ -72,11 +73,16 @@ public class TouTiaoCommentParse {
{ {
List<TouTiaoComment> commentes = analySisComment(htmlBody, url); List<TouTiaoComment> commentes = analySisComment(htmlBody, url);
ttList.addAll(commentes); ttList.addAll(commentes);
logger.info(" url {} 采集到第 {} 页 采集到 {} 条数据 ",url,page,ttList.size());
}else }else
{ {
logger.info("采集出现问题,地址为:{}", url); logger.info("采集出现问题,地址为:{}", url);
} }
ZhiWeiTools.sleep(4000); if(Objects.nonNull(proxy)) {
ZhiWeiTools.sleep(100);
}else {
ZhiWeiTools.sleep(4000);
}
break; break;
} catch (Exception e) { } catch (Exception e) {
continue; continue;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment