Commit 95487743 by [zhangzhiwei]

因修改采集核心包版本,修改相应的方法

parent f09faf1a
...@@ -25,22 +25,17 @@ ...@@ -25,22 +25,17 @@
<version>3.8.1</version> <version>3.8.1</version>
</dependency> </dependency>
<dependency>
<groupId>com.zhiwei.middleware</groupId>
<artifactId>proxy-client</artifactId>
<version>0.0.2-RELEASE</version>
</dependency>
<dependency> <dependency>
<groupId>com.zhiwei.tools</groupId> <groupId>com.zhiwei.tools</groupId>
<artifactId>zhiwei-tools</artifactId> <artifactId>zhiwei-tools</artifactId>
<version>0.0.5-SNAPSHOT</version> <version>0.0.8-SNAPSHOT</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.zhiwei.middleware</groupId> <groupId>com.zhiwei.middleware</groupId>
<artifactId>cleaner-unified-urlfilter</artifactId> <artifactId>cleaner-unified-urlfilter</artifactId>
<version>1.0.0.RELEASE</version> <version>1.0.6.RELEASE</version>
</dependency> </dependency>
<dependency> <dependency>
......
...@@ -4,9 +4,9 @@ import java.util.concurrent.Executors; ...@@ -4,9 +4,9 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import com.zhiwei.common.config.GroupType;
import com.zhiwei.crawler.config.ProxyConfig; import com.zhiwei.crawler.config.ProxyConfig;
import com.zhiwei.crawler.proxy.ProxyFactory; import com.zhiwei.crawler.proxy.ProxyFactory;
import com.zhiwei.proxy.common.Definition.GroupType;
public class MainRun { public class MainRun {
private ScheduledExecutorService scheduExec; private ScheduledExecutorService scheduExec;
...@@ -21,8 +21,7 @@ public class MainRun { ...@@ -21,8 +21,7 @@ public class MainRun {
public static void main(String[] args) { public static void main(String[] args) {
/** 初始化代理IP **/ /** 初始化代理IP **/
ProxyFactory.init(ProxyConfig.registry, ProxyConfig.group, GroupType.PROVIDER, ProxyFactory.init(ProxyConfig.registry, ProxyConfig.group, GroupType.PROVIDER);
ProxyFactory::getNatProxy);
new MainRun().showTimer(); new MainRun().showTimer();
} }
......
package com.zhiwei.crawler.soubao; package com.zhiwei.crawler.soubao;
import java.net.Proxy;
import java.util.concurrent.BlockingQueue; import java.util.concurrent.BlockingQueue;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import com.zhiwei.crawler.proxy.ProxyFactory; import com.zhiwei.crawler.proxy.ProxyHolder;
import com.zhiwei.crawler.run.SoubaoCrawlerRun; import com.zhiwei.crawler.run.SoubaoCrawlerRun;
import com.zhiwei.tools.tools.ZhiWeiTools; import com.zhiwei.tools.tools.ZhiWeiTools;
...@@ -19,7 +18,6 @@ import com.zhiwei.tools.tools.ZhiWeiTools; ...@@ -19,7 +18,6 @@ import com.zhiwei.tools.tools.ZhiWeiTools;
public class SouBaoCrawlerThread extends Thread{ public class SouBaoCrawlerThread extends Thread{
private static final Logger logger = LogManager.getLogger(SoubaoCrawlerRun.class); private static final Logger logger = LogManager.getLogger(SoubaoCrawlerRun.class);
private BlockingQueue<String> wordsQueue; private BlockingQueue<String> wordsQueue;
public SouBaoCrawlerThread(BlockingQueue<String> wordsQueue) { public SouBaoCrawlerThread(BlockingQueue<String> wordsQueue) {
this.wordsQueue = wordsQueue; this.wordsQueue = wordsQueue;
} }
...@@ -30,11 +28,10 @@ public class SouBaoCrawlerThread extends Thread{ ...@@ -30,11 +28,10 @@ public class SouBaoCrawlerThread extends Thread{
while(wordsQueue!=null && wordsQueue.size()>0){ while(wordsQueue!=null && wordsQueue.size()>0){
try { try {
String word = wordsQueue.take(); String word = wordsQueue.take();
Proxy proxy = ProxyFactory.proxyCallback().getProxy();
/***开始采集**/ /***开始采集**/
logger.info("开始采集:{}搜报网关键词,目前未采集的关键词为:{}", word, wordsQueue.size()); logger.info("开始采集:{}搜报网关键词,目前未采集的关键词为:{}", word, wordsQueue.size());
long s = System.currentTimeMillis(); long s = System.currentTimeMillis();
Crawler.start(1, word, proxy); Crawler.start(1, word, ProxyHolder.NAT_PROXY.getProxy());
long e = System.currentTimeMillis(); long e = System.currentTimeMillis();
logger.info("采集:::{}搜报网关键词结束,采集所用时间为:{}", word, (e-s)); logger.info("采集:::{}搜报网关键词结束,采集所用时间为:{}", word, (e-s));
} catch (Exception e) { } catch (Exception e) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment