Commit 95487743 by [zhangzhiwei]

因修改采集核心包版本,修改相应的方法

parent f09faf1a
......@@ -25,22 +25,17 @@
<version>3.8.1</version>
</dependency>
<dependency>
<groupId>com.zhiwei.middleware</groupId>
<artifactId>proxy-client</artifactId>
<version>0.0.2-RELEASE</version>
</dependency>
<dependency>
<groupId>com.zhiwei.tools</groupId>
<artifactId>zhiwei-tools</artifactId>
<version>0.0.5-SNAPSHOT</version>
<version>0.0.8-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.zhiwei.middleware</groupId>
<artifactId>cleaner-unified-urlfilter</artifactId>
<version>1.0.0.RELEASE</version>
<version>1.0.6.RELEASE</version>
</dependency>
<dependency>
......
......@@ -4,9 +4,9 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import com.zhiwei.common.config.GroupType;
import com.zhiwei.crawler.config.ProxyConfig;
import com.zhiwei.crawler.proxy.ProxyFactory;
import com.zhiwei.proxy.common.Definition.GroupType;
public class MainRun {
private ScheduledExecutorService scheduExec;
......@@ -21,8 +21,7 @@ public class MainRun {
public static void main(String[] args) {
/** 初始化代理IP **/
ProxyFactory.init(ProxyConfig.registry, ProxyConfig.group, GroupType.PROVIDER,
ProxyFactory::getNatProxy);
ProxyFactory.init(ProxyConfig.registry, ProxyConfig.group, GroupType.PROVIDER);
new MainRun().showTimer();
}
......
package com.zhiwei.crawler.soubao;
import java.net.Proxy;
import java.util.concurrent.BlockingQueue;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import com.zhiwei.crawler.proxy.ProxyFactory;
import com.zhiwei.crawler.proxy.ProxyHolder;
import com.zhiwei.crawler.run.SoubaoCrawlerRun;
import com.zhiwei.tools.tools.ZhiWeiTools;
......@@ -19,7 +18,6 @@ import com.zhiwei.tools.tools.ZhiWeiTools;
public class SouBaoCrawlerThread extends Thread{
private static final Logger logger = LogManager.getLogger(SoubaoCrawlerRun.class);
private BlockingQueue<String> wordsQueue;
public SouBaoCrawlerThread(BlockingQueue<String> wordsQueue) {
this.wordsQueue = wordsQueue;
}
......@@ -30,11 +28,10 @@ public class SouBaoCrawlerThread extends Thread{
while(wordsQueue!=null && wordsQueue.size()>0){
try {
String word = wordsQueue.take();
Proxy proxy = ProxyFactory.proxyCallback().getProxy();
/***开始采集**/
logger.info("开始采集:{}搜报网关键词,目前未采集的关键词为:{}", word, wordsQueue.size());
long s = System.currentTimeMillis();
Crawler.start(1, word, proxy);
Crawler.start(1, word, ProxyHolder.NAT_PROXY.getProxy());
long e = System.currentTimeMillis();
logger.info("采集:::{}搜报网关键词结束,采集所用时间为:{}", word, (e-s));
} catch (Exception e) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment