Skip to content

S
soubao_crawler
Commit 78a1b8cf by zhiwei
Options

删除邮件，添加代理

parent b00e3d2d
Showing with 171 additions and 222 deletions
pom.xml
<project xmlns="http://maven.apache.org/POM/4.0.0" <project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>com.zhiwei.crawler</groupId> <groupId>com.zhiwei.crawler</groupId>
<artifactId>soubao-crawlerNew</artifactId> <artifactId>soubao-crawlerNew</artifactId>
<version>0.0.2-SNAPSHOT</version> <version>0.0.2-SNAPSHOT</version>
<name>搜报网采集</name> <name>搜报网采集</name>
<properties> <properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
</properties> </properties>
<developers> <developers>
<developer> <developer>
<id>Bewilder</id> <id>Bewilder</id>
<name>zhiwei zhang</name> <name>zhiwei zhang</name>
<email>zhangzhiwei@zhiweidata.com</email> <email>zhangzhiwei@zhiweidata.com</email>
</developer> </developer>
</developers> </developers>
<dependencies> <dependencies>
<!-- https://mvnrepository.com/artifact/com.squareup.okhttp3/okhttp --> <!-- https://mvnrepository.com/artifact/com.squareup.okhttp3/okhttp -->
<dependency> <dependency>
<groupId>com.squareup.okhttp3</groupId> <groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId> <artifactId>okhttp</artifactId>
<version>3.10.0</version> <version>3.10.0</version>
</dependency> </dependency>
<!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-core --> <!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-core -->
<dependency> <dependency>
<groupId>org.apache.logging.log4j</groupId> <groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId> <artifactId>log4j-core</artifactId>
<version>2.11.0</version> <version>2.11.0</version>
</dependency> </dependency>
<!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-api --> <!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-api -->
<dependency> <dependency>
<groupId>org.apache.logging.log4j</groupId> <groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId> <artifactId>log4j-api</artifactId>
<version>2.11.0</version> <version>2.11.0</version>
</dependency> </dependency>
<!-- https://mvnrepository.com/artifact/org.jsoup/jsoup --> <!-- https://mvnrepository.com/artifact/org.jsoup/jsoup -->
<dependency> <dependency>
<groupId>org.jsoup</groupId> <groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId> <artifactId>jsoup</artifactId>
<version>1.11.3</version> <version>1.11.3</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.mongodb</groupId> <groupId>org.mongodb</groupId>
<artifactId>mongo-java-driver</artifactId> <artifactId>mongo-java-driver</artifactId>
<version>3.6.3</version> <version>3.6.3</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.zhiwei</groupId> <groupId>com.zhiwei</groupId>
<artifactId>mongosave-service-client</artifactId> <artifactId>mongosave-service-client</artifactId>
<version>1.0-SNAPSHOT</version> <version>1.0-SNAPSHOT</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.zhiwei.middleware</groupId> <groupId>com.zhiwei.middleware</groupId>
<artifactId>rsid-client</artifactId> <artifactId>rsid-client</artifactId>
<version>0.0.2-SNAPSHOT</version> <version>0.0.2-SNAPSHOT</version>
</dependency> </dependency>
<dependency>
<groupId>com.zhiwei</groupId> <dependency>
<artifactId>sendmail</artifactId> <groupId>com.zhiwei.middleware</groupId>
<version>0.0.1-SNAPSHOT</version> <artifactId>proxy-client</artifactId>
</dependency> <version>0.0.1-RELEASE</version>
</dependency>
<dependency>
<groupId>com.zhiwei.middleware</groupId> <dependency>
<artifactId>proxy-client</artifactId> <groupId>com.zhiwei</groupId>
<version>0.0.1-RELEASE</version> <artifactId>zhiweiTools</artifactId>
</dependency> <version>0.0.6-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>com.zhiwei</groupId> </dependencies>
<artifactId>zhiweiTools</artifactId>
<version>0.0.6-SNAPSHOT</version>
</dependency>
</dependencies> <build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<build> <version>2.4.2</version>
<plugins> <executions>
<plugin> <execution>
<groupId>org.apache.maven.plugins</groupId> <phase>package</phase>
<artifactId>maven-shade-plugin</artifactId> <goals>
<version>2.4.2</version> <goal>shade</goal>
<executions> </goals>
<execution> <configuration>
<phase>package</phase> <filters>
<goals> <filter>
<goal>shade</goal> <artifact>*:*</artifact>
</goals> <excludes>
<configuration> <exclude>META-INF/*.SF</exclude>
<filters> <exclude>META-INF/*.DSA</exclude>
<filter> <exclude>META-INF/*.RSA</exclude>
<artifact>*:*</artifact> </excludes>
<excludes> </filter>
<exclude>META-INF/*.SF</exclude> </filters>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude> <transformers>
</excludes> <transformer
</filter> implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
</filters> <mainClass>com.zhiwei.crawler.run.MainRun</mainClass>
</transformer>
<transformers> </transformers>
<transformer </configuration>
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> </execution>
<mainClass>com.zhiwei.crawler.run.MainRun</mainClass> </executions>
</transformer> </plugin>
</transformers>
</configuration> <plugin>
</execution> <artifactId>maven-source-plugin</artifactId>
</executions> <version>2.4</version>
</plugin> <configuration>
<attach>true</attach>
<plugin> </configuration>
<artifactId>maven-source-plugin</artifactId> <executions>
<version>2.4</version> <execution>
<configuration> <phase>compile</phase>
<attach>true</attach> <goals>
</configuration> <goal>jar</goal>
<executions> </goals>
<execution> </execution>
<phase>compile</phase> </executions>
<goals> </plugin>
<goal>jar</goal>
</goals> <!-- 解决maven test命令时console出现中文乱码乱码 -->
</execution> <plugin>
</executions> <groupId>org.apache.maven.plugins</groupId>
</plugin> <artifactId>maven-surefire-plugin</artifactId>
<version>2.19.1</version>
<!-- 解决maven test命令时console出现中文乱码乱码 --> <configuration>
<plugin> <forkMode>once</forkMode>
<groupId>org.apache.maven.plugins</groupId> <argLine>-Dfile.encoding=UTF-8</argLine>
<artifactId>maven-surefire-plugin</artifactId> <skipTests>true</skipTests>
<version>2.19.1</version> </configuration>
<configuration> </plugin>
<forkMode>once</forkMode> </plugins>
<argLine>-Dfile.encoding=UTF-8</argLine> </build>
<skipTests>true</skipTests>
</configuration>
</plugin>
</plugins>
</build>
</project> </project>
\ No newline at end of file
src/main/java/com/zhiwei/crawler/soubao/SouBaoCrawlerThread.java
package com.zhiwei.crawler.soubao; package com.zhiwei.crawler.soubao;
import java.net.Proxy; import java.net.Proxy;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.concurrent.BlockingQueue; import java.util.concurrent.BlockingQueue;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
...@@ -33,7 +30,7 @@ public class SouBaoCrawlerThread extends Thread{ ...@@ -33,7 +30,7 @@ public class SouBaoCrawlerThread extends Thread{
while(wordsQueue!=null && wordsQueue.size()>0){ while(wordsQueue!=null && wordsQueue.size()>0){
try { try {
String word = wordsQueue.take(); String word = wordsQueue.take();
Proxy proxy = ProxyClientUtil.getProxy(); Proxy proxy = ProxyClientUtil.getNATProxy();
/***开始采集**/ /***开始采集**/
logger.info("开始采集：{}搜报网关键词,目前未采集的关键词为:{}", word, wordsQueue.size()); logger.info("开始采集：{}搜报网关键词,目前未采集的关键词为:{}", word, wordsQueue.size());
long s = System.currentTimeMillis(); long s = System.currentTimeMillis();
......
src/main/java/com/zhiwei/crawler/util/ProxyClientUtil.java
...@@ -2,21 +2,31 @@ package com.zhiwei.crawler.util; ...@@ -2,21 +2,31 @@ package com.zhiwei.crawler.util;
import java.net.Proxy; import java.net.Proxy;
import com.zhiwei.crawler.config.ProxyConfig;
import com.zhiwei.proxy.common.Definition.GroupType;
import com.zhiwei.proxy.core.ProxyClient; import com.zhiwei.proxy.core.ProxyClient;
import com.zhiwei.proxy.core.ProxyClientFactory; import com.zhiwei.proxy.core.ProxyClientFactory;
import com.zhiwei.rsid.common.Definition.GroupType;
public class ProxyClientUtil { public class ProxyClientUtil {
// private static String registry = "zookeeper://202.107.192.94:2181"; private static volatile ProxyClient client;
// private static String group = "local";
private static String registry = "zookeeper://192.168.0.203:2181";
private static String group = "hangzhou";
private static ProxyClient client = ProxyClientFactory.build(registry, group, GroupType.PROVIDER);
public static Proxy getProxy(){ /**
* @Title: getNATProxy
* @author hero
* @Description: 获取NAT机代理IP
* @param @return 设定文件
* @return Proxy 返回类型
*/
public static Proxy getNATProxy(){
if(client==null) {
synchronized (ProxyClientUtil.class) {
if(client==null) {
client = ProxyClientFactory.build(ProxyConfig.registry, ProxyConfig.group, GroupType.PROVIDER);
}
}
}
return client.getNATProxy(); return client.getNATProxy();
} }
......
src/main/java/com/zhiwei/crawler/util/SendMailTVCloud.java deleted 100644 → 0
package com.zhiwei.crawler.util;
import com.zhiwei.sendmail.SendMail;
import com.zhiwei.sendmail.bean.MailInfo;
public class SendMailTVCloud {
/**
* @Title: sendMail
* @Description: TODO(发送邮件)
* @param @param mailContent
* @param @param email
* @param @return 设定文件
* @return boolean 返回类型
*/
public static boolean sendMail(String mailContent,String email){
//这个类主要是设置邮件
String mailServerHost = "smtp.ym.163.com";
String mailServerPort = "25";
String fromAddress = "zhangzhiwei@zhiweidata.com";
String toAddress = email;
String userName = "zhangzhiwei@zhiweidata.com";
String password = "olp1437z..";
boolean validate = true;
String subject = "监测系统天脉云采集程序";
String content = mailContent;
MailInfo mailInfo = new MailInfo(mailServerHost, mailServerPort, fromAddress, toAddress, userName, password, validate, subject, content,null);
boolean f=false;
//这个类主要来发送邮件
f = SendMail.sendMailByHtml(mailInfo);//发送文体格式
return f;
}
/**
* 测试发送邮件
*/
// public static void main(String[] args) {
// SendMailDaoImpl sendMail=new SendMailDaoImpl();
// String mailSubject="国家核电重点规则追踪提醒";
// String mailContent="消息内容：考或不考，成功的路就在那里，冥冥之中就会有一双手帮你。但如果只是为了一纸文凭，奉劝大家不要将有限的生命投入到无限的为考试而考之中。如果真的发自内心想要学英语，一定是会说会用，才算会英语！<br/><br/>地址：www.baidu.com<br/><br/> 本消息由国家核电舆情系统发送（请勿回复）";
// String[] email={"859548429@qq.com"};
// sendMail.SendMail(mailContent, email);
//
// }
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment