Skip to content

S
source_forward
Commit bde825dd by yangchen
Options

添加今日头条转发原创判断和链接是否删除方法修改

parent 5a79e3d2
Showing with 263 additions and 283 deletions
pom.xml
......@@ -30,6 +30,7 @@
<groupId>com.zhiwei.crawler</groupId>
<artifactId>crawler-core</artifactId>
<version>0.1.1-RELEASE</version>
<scope>provided</scope>
</dependency>
</dependencies>
......
src/main/java/com/zhiwei/source_forward/crawler/SourceForwardCrawler.java
package com.zhiwei.source_forward.crawler;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.apache.logging.log4j.LogManager;
......@@ -11,7 +12,7 @@ import org.jsoup.nodes.Node;
import com.zhiwei.crawler.async.MultiThreadingCounter;
import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.core.HttpRequestBuilder;
import com.zhiwei.crawler.core.RequestUtils;
import com.zhiwei.crawler.proxy.ProxyHolder;
import com.zhiwei.source_forward.bean.SourceForwardBean;
import com.zhiwei.source_forward.bean.SourceForwardBean.Attribution;
......@@ -19,6 +20,7 @@ import com.zhiwei.source_forward.util.MatchChannel;
import com.zhiwei.source_forward.util.MatchSource;
import com.zhiwei.source_forward.util.SourceData;
import com.zhiwei.source_forward.util.SourceForwardDataCallBack;
import com.zhiwei.tools.httpclient.HeaderTool;
import okhttp3.Request;
import okhttp3.Response;
......@@ -31,7 +33,7 @@ public class SourceForwardCrawler {
private static List<String> sourceList = SourceData.getSourceList();
public MultiThreadingCounter submitTask(SourceForwardDataCallBack callback,String... urls) throws Exception {
MultiThreadingCounter counter = new MultiThreadingCounter(5,TimeUnit.MINUTES,false);
MultiThreadingCounter counter = new MultiThreadingCounter(15,TimeUnit.MINUTES,false);
start(counter, callback, urls);
return counter;
}
......@@ -55,7 +57,12 @@ public class SourceForwardCrawler {
private MultiThreadingCounter search(MultiThreadingCounter counter, String url,Attribution attr, SourceForwardDataCallBack callback) {
logger.info("当前处理 URL: {}", url);
Request request = HttpRequestBuilder.newGetRequest(url, null);
Map<String,String> headers = HeaderTool.getCommonHead();
if(url.contains("www.toutiao.com")){
headers.put("referer", url);
}
Request request = RequestUtils.wrapGet(url, headers);
counter.increase();
httpBoot.asyncCall(request, ProxyHolder.NAT_PROXY, true).addListener(future -> {
try {
......@@ -83,7 +90,8 @@ public class SourceForwardCrawler {
String isforward = "未知";
try {
if(response.isSuccessful()){
Document document = Jsoup.parse(response.body().string());
String body = response.body().string();
Document document = Jsoup.parse(body);
if(attr.get().toString().contains("mp.weixin.qq.com")){
isforward = document.select("div#meta_content").select("span#copyright_logo").text();
if(isforward.contains("原创")){
......@@ -91,6 +99,12 @@ public class SourceForwardCrawler {
}else {
isforward = "未知";
}
}else if(attr.get().toString().contains("www.toutiao.com")){
if(body.contains("isOriginal")){
if(body.contains("isOriginal: true")){
isforward = "原创";
}
}
}else{
channel = MatchChannel.verifyChannel(attr.get().toString());
if(channel==null){
......
src/main/java/com/zhiwei/source_forward/crawler/UrlLiveCrawler.java
package com.zhiwei.source_forward.crawler;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Node;
import com.zhiwei.crawler.async.MultiThreadingCounter;
import com.zhiwei.crawler.core.HttpBoot;
import com.zhiwei.crawler.core.HttpRequestBuilder;
import com.zhiwei.crawler.core.RequestUtils;
import com.zhiwei.crawler.proxy.ProxyHolder;
import com.zhiwei.source_forward.bean.UrlLiveBean;
import com.zhiwei.source_forward.bean.UrlLiveBean.Attribution;
import com.zhiwei.source_forward.util.UrlLiveDataCallback;
import com.zhiwei.tools.httpclient.HeaderTool;
import okhttp3.Request;
import okhttp3.Response;
......@@ -57,7 +57,11 @@ public class UrlLiveCrawler {
Attribution attr, UrlLiveDataCallback callback) {
url = dealUrl(url);
logger.info("当前处理 URL: {}", url);
Request request = HttpRequestBuilder.newGetRequest(url, null);
Map<String,String> headers = HeaderTool.getCommonHead();
if(url.contains("www.toutiao.com")){
headers.put("referer", url);
}
Request request = RequestUtils.wrapGet(url, headers);
counter.increase();
httpBoot.asyncCall(request, ProxyHolder.NAT_PROXY, false).addListener(future -> {
try {
......@@ -170,280 +174,236 @@ public class UrlLiveCrawler {
* @return boolean 返回类型
*/
public boolean matchDel(String result,String url){
int step = 1;
Document doc = Jsoup.parse(result);
if(rulerHead(doc)){
logger.info("{}检测规则：第{}步",url,step);
return true;
}
step++;
if (rulerYaoyan(doc))
{
logger.info("{}检测规则：第{}步",url,step);
return true;
}
step++;
if (rulerTousu(doc))
{
logger.info("{}检测规则：第{}步",url,step);
return true;
}
step++;
if (url.contains("huanqiu.com"))
{
logger.info("{}检测规则：第{}步",url,step);
return rulerHuanqiuWuxiao(doc);
}
step++;//7
if (rulerBucunzai(doc))
{
logger.info("{}检测规则：第{}步",url,step);
return true;
}
step++;//8
if (rulerKong(doc))
{
logger.info("{}检测规则：第{}步",url,step);
return true;
}
step++;//9
if (rulerZhaoshang(doc))
{
logger.info("{}检测规则：第{}步",url,step);
return true;
}
step++;//10
if(rulerWeigui(doc)) {
logger.info("{}检测规则：第{}步",url,step);
return true;
}
step++;//11
if (rulerYidian(doc))
{
logger.info("{}检测规则：第{}步",url,step);
return true;
}
step++;
if(rulerWechatWeigui(doc)) {
logger.info("{}检测规则：第{}步",url,step);
return true;
}
return false;
}
/**
*
* ( 微信谣言的无效网址筛选规则)
* @author 陈炜涛
* @param doc
* @return
* @time 2016年6月3日上午9:54:00
* @return boolean
*/
private boolean rulerYaoyan(Document doc)
{
boolean flg = false;
if ("谣言".equals(doc.select(".pic_rumor").text()))
{
flg = true;
}
return flg;
}
/**
*
* ( 微信内容违规的无效网址筛选规则)
* @author 陈炜涛
* @param doc
* @return
* @time 2016年6月3日上午9:59:54
* @return boolean
*/
private boolean rulerWeigui(Document doc)
{
boolean flg = false;
if ((doc.select("p.title").text()).contains("此内容因违规无法查看") || doc.select("p.title").text().contains("此帐号在冻结期,内容无法查看"))
{
flg = true;
}
return flg;
}
private boolean rulerWechatWeigui(Document doc) {
boolean flg = false;
if ((doc.select("h3.msg-title").text()).contains("此内容被投诉且经审核涉嫌侵权，无法查看"))
{
flg = true;
}
return flg;
}
/**
*
* ( 微信内容违规的无效网址筛选规则)
* @author 陈炜涛
* @param doc
* @return
* @time 2016年6月3日上午9:59:54
* @return boolean
*/
private boolean rulerTousu(Document doc)
{
boolean flg = false;
if (0 < doc.select("i[class=\"icon_msg warn\"]").size())
{
flg = true;
}
return flg;
}
/**
*
* ( 环球的无效网址筛选规则)
* @author 陈炜涛
* @param doc
* @return
* @time 2016年6月3日上午9:59:54
* @return boolean
*/
private boolean rulerHuanqiuWuxiao(Document doc)
{
boolean flg = false;
if (0 < doc.select("div[class=\"errMsg\"]").size())
{
flg = true;
}
return flg;
}
/**
*
* ( 空的无效网址筛选规则)
* @author 陈炜涛
* @param doc
* @return
* @time 2016年6月3日上午9:59:54
* @return boolean
*/
private boolean rulerKong(Document doc)
{
boolean flg = false;
if (14 > doc.select("body").toString().length()
&&
14 > doc.select("head").toString().length())
{
flg = true;
}
return flg;
}
/**
*
* ( 内容不存在)
* @author 陈炜涛
* @param doc
* @return
* @time 2016年6月3日上午9:59:54
* @return boolean
*/
private boolean rulerBucunzai(Document doc)
{
boolean flg = false;
if (doc.text().contains("很抱歉，您访问的页面不存在")||doc.text().contains("该内容已被发布者删除"))
{
flg = true;
}
return flg;
}
/**
*
* ( 招商网的无效网址筛选规则)
* @author 陈炜涛
* @param doc
* @return
* @time 2016年6月3日上午9:59:54
* @return boolean
*/
private boolean rulerZhaoshang(Document doc)
{
boolean flg = false;
try
{
if ("<a href=\"\"> </a>".equals(doc.select("div[class=\"paths\"]")
.first().child(2).toString()))
{
flg = true;
}
}
catch (Exception e)
{
e.printStackTrace();
// TODO: handle exception
}
return flg;
}
/**
*
* ( 一点资讯的无效网址筛选规则)
* @author 陈炜涛
* @param doc
* @return
* @time 2016年6月3日上午9:59:54
* @return boolean
*/
private boolean rulerYidian(Document doc)
{
boolean flg = false;
try
{
if (doc.select("div[class=\"content\"]").text().contains("文章没有找到"))
{
flg = true;
}
}
catch (Exception e)
{
e.printStackTrace();
// : handle exception
}
return flg;
String title = null;
if(url.contains("mp.weixin.qq.com") || url.contains("post.mp.qq.com")){
title = doc.select("h2.rich_media_title").text().replaceAll(" ", "");
}else if(url.contains("kuaibao")){
title = doc.select("p.title").text().replaceAll(" ", "");;
}else {
title = doc.select("title").text().replaceAll(" ", "");;
}
/**
* @Title: rulerHead
* @author hero
* @Description: 验证链接头部
* @param @param doc
* @param @return 设定文件
* @return boolean 返回类型
*/
private boolean rulerHead(Document doc)
{
List<Node> nodeList = doc.head().childNodes();
try {
for (Node node : nodeList) {
if (node.outerHtml().contains("<title>")) {
String title = node.toString().split("<title>")[1].split("</title>")[0];
if(title!=null && !title.equals("")){
if(title.contains("未知错误") || title.contains("Object moved") || title.contains("404") || title.contains("页面没有找到") || title.contains("页面未找到") || title.contains("301 Moved Permanently")){
return true;
}
}
if (node.outerHtml().contains("meta")) {
String meta = node.toString();
if(meta.contains("公益404页面")) {
return true;
}
}
}
} catch (Exception e) {
e.printStackTrace();
return false;
}
return false;
}
// /**
// *
// * ( 微信谣言的无效网址筛选规则)
// * @author 陈炜涛
// * @param doc
// * @return
// * @time 2016年6月3日上午9:54:00
// * @return boolean
// */
// private boolean rulerYaoyan(Document doc)
// {
// boolean flg = false;
// if ("谣言".equals(doc.select(".pic_rumor").text()))
// {
// flg = true;
// }
// return flg;
// }
//
// /**
// *
// * ( 微信内容违规的无效网址筛选规则)
// * @author 陈炜涛
// * @param doc
// * @return
// * @time 2016年6月3日上午9:59:54
// * @return boolean
// */
// private boolean rulerWeigui(Document doc)
// {
// boolean flg = false;
// if ((doc.select("p.title").text()).contains("此内容因违规无法查看") || doc.select("p.title").text().contains("此帐号在冻结期,内容无法查看"))
// {
// flg = true;
// }
// return flg;
// }
//
// private boolean rulerWechatWeigui(Document doc) {
// boolean flg = false;
// if ((doc.select("h3.msg-title").text()).contains("此内容被投诉且经审核涉嫌侵权，无法查看"))
// {
// flg = true;
// }
// return flg;
// }
//
// /**
// *
// * ( 微信内容违规的无效网址筛选规则)
// * @author 陈炜涛
// * @param doc
// * @return
// * @time 2016年6月3日上午9:59:54
// * @return boolean
// */
// private boolean rulerTousu(Document doc)
// {
// boolean flg = false;
// if (0 < doc.select("i[class=\"icon_msg warn\"]").size())
// {
// flg = true;
// }
// return flg;
// }
//
// /**
// *
// * ( 环球的无效网址筛选规则)
// * @author 陈炜涛
// * @param doc
// * @return
// * @time 2016年6月3日上午9:59:54
// * @return boolean
// */
// private boolean rulerHuanqiuWuxiao(Document doc)
// {
// boolean flg = false;
// if (0 < doc.select("div[class=\"errMsg\"]").size())
// {
// flg = true;
// }
// return flg;
// }
//
// /**
// *
// * ( 空的无效网址筛选规则)
// * @author 陈炜涛
// * @param doc
// * @return
// * @time 2016年6月3日上午9:59:54
// * @return boolean
// */
// private boolean rulerKong(Document doc)
// {
// boolean flg = false;
// if (14 > doc.select("body").toString().length()
// &&
// 14 > doc.select("head").toString().length())
// {
// flg = true;
// }
// return flg;
// }
//
// /**
// *
// * ( 内容不存在)
// * @author 陈炜涛
// * @param doc
// * @return
// * @time 2016年6月3日上午9:59:54
// * @return boolean
// */
// private boolean rulerBucunzai(Document doc)
// {
// boolean flg = false;
// if (doc.text().contains("很抱歉，您访问的页面不存在")||doc.text().contains("该内容已被发布者删除"))
// {
// flg = true;
// }
// return flg;
// }
//
// /**
// *
// * ( 招商网的无效网址筛选规则)
// * @author 陈炜涛
// * @param doc
// * @return
// * @time 2016年6月3日上午9:59:54
// * @return boolean
// */
// private boolean rulerZhaoshang(Document doc)
// {
// boolean flg = false;
// try
// {
// if ("<a href=\"\"> </a>".equals(doc.select("div[class=\"paths\"]")
// .first().child(2).toString()))
// {
// flg = true;
// }
// }
// catch (Exception e)
// {
// e.printStackTrace();
// // TODO: handle exception
// }
//
// return flg;
// }
//
//
// /**
// *
// * ( 一点资讯的无效网址筛选规则)
// * @author 陈炜涛
// * @param doc
// * @return
// * @time 2016年6月3日上午9:59:54
// * @return boolean
// */
// private boolean rulerYidian(Document doc)
// {
// boolean flg = false;
// try
// {
// if (doc.select("div[class=\"content\"]").text().contains("文章没有找到"))
// {
// flg = true;
// }
// }
// catch (Exception e)
// {
// e.printStackTrace();
// // : handle exception
// }
// return flg;
// }
//
// /**
// * @Title: rulerHead
// * @author hero
// * @Description: 验证链接头部
// * @param @param doc
// * @param @return 设定文件
// * @return boolean 返回类型
// */
// private boolean rulerHead(Document doc)
// {
// List<Node> nodeList = doc.head().childNodes();
// try {
// for (Node node : nodeList) {
// if (node.outerHtml().contains("<title>")) {
// String title = node.toString().split("<title>")[1].split("</title>")[0];
// if(title.contains("未知错误") || title.contains("Object moved") || title.contains("404") || title.contains("页面没有找到") || title.contains("页面未找到") || title.contains("301 Moved Permanently")){
// return true;
// }
// }
// if (node.outerHtml().contains("meta")) {
// String meta = node.toString();
// if(meta.contains("公益404页面")) {
// return true;
// }
// }
// }
// } catch (Exception e) {
// e.printStackTrace();
// return false;
// }
// return false;
// }
}
src/main/java/com/zhiwei/source_forward/run/MediaSelfSource.java
......@@ -7,6 +7,8 @@ import java.util.List;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import com.zhiwei.common.config.GroupType;
import com.zhiwei.crawler.proxy.ProxyFactory;
import com.zhiwei.source_forward.bean.MediaSelfSourceBean;
import com.zhiwei.source_forward.bean.MediaSelfSourceBean.Attribution;
import com.zhiwei.source_forward.crawler.MediaSelfSourceCrawler;
......@@ -22,13 +24,13 @@ public class MediaSelfSource {
}
public static void main(String[] args) {
// ProxyFactory.init("zookeeper://192.168.0.36:2181","local",GroupType.PROVIDER);
// List<String> urlList = new ArrayList<>();
// urlList.add("https://www.toutiao.com/a6452936157751968013/");
// List<MediaSelfSourceBean> u = MediaSelfSource.getMediaSelfSource(urlList);
// for(MediaSelfSourceBean b : u) {
// System.out.println(b.toString());
// }
ProxyFactory.init("zookeeper://192.168.0.36:2181","local",GroupType.PROVIDER);
List<String> urlList = new ArrayList<>();
urlList.add("http://sh.qihoo.com/pc/91d1d565fe552fa1e?sign=360_e39369d1");
List<MediaSelfSourceBean> u = MediaSelfSource.getMediaSelfSource(urlList);
for(MediaSelfSourceBean b : u) {
System.out.println(b.toString());
}
}
static class MediaSelfSourceCrawlerThread extends Thread{
......
src/main/java/com/zhiwei/source_forward/util/MatchSource.java
......@@ -149,7 +149,10 @@ public class MatchSource {
}
}else if(url.contains("sh.qihoo.com")){
//今日报点解析
source = document.select("p.info").select("span.source").text().trim();
source = document.select("span.source").text().trim();
if(source.length() < 2) {
source = document.select("p.article-info").select("a").text().trim();
}
if(source!=null && !source.equals("")){
source = "快资讯-" + source;
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment