Merge branch 'source-forward-chen' of…

Merge branch 'source-forward-chen' of http://git.zhiweidata.top/zhangzhiwei/source_forward.git into source-forward-chen

pom.xml

@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.zhiwei</groupId>
     <artifactId>source-forward</artifactId>
-    <version>0.0.7-SNAPSHOT</version>
+    <version>0.0.9-SNAPSHOT</version>
     <name>source-forward</name>
     <description>验证网媒的转发关系及链接的有效性（转发验证微信及自媒体匹配率不高）</description>
 

src/main/java/com/zhiwei/source_forward/bean/UrlLiveBean.java

@@ -6,6 +6,8 @@ public class UrlLiveBean {
     
     private Integer isLive;
     
+    private String title;
+    
     public UrlLiveBean() {
         super();
     }
@@ -16,15 +18,31 @@ public class UrlLiveBean {
         this.isLive = isLive;
     }
     
-    public UrlLiveBean(String url, boolean isLive) {
+    public UrlLiveBean(String url, Integer isLive,String title) {
+        super();
+        this.url = url;
+        this.isLive = isLive;
+        this.title = title;
+    }
+    
+    public UrlLiveBean(String url, boolean isLive,String title) {
         super();
         this.url = url;
+        this.title = title;
         if(isLive) {
             this.isLive = 1;      //已删除
         }else {
             this.isLive = 0;
         }
     }
+    
+    public String getTitle() {
+        return title;
+    }
+
+    public void setTitle(String title) {
+        this.title = title;
+    }
 
     public String getUrl() {
         return url;
@@ -44,7 +62,8 @@ public class UrlLiveBean {
 
     @Override
     public String toString() {
-        return "UrlLiveBean [url=" + url + ", isLive=" + isLive + "]";
+        return "UrlLiveBean [url=" + url + ", isLive=" + isLive + ", title="
+                + title + "]";
     }
     
     /** 

src/main/java/com/zhiwei/source_forward/crawler/ContentCrawler.java

@@ -32,9 +32,14 @@ public class ContentCrawler {
      */
     public MultiThreadingCounter submitTask(ContentDataCallback callback,
             String... urls) {
-        MultiThreadingCounter counter = new MultiThreadingCounter(15,TimeUnit.MINUTES,false);
-        start(counter, callback, urls);
-        return counter;
+        try {
+            MultiThreadingCounter counter = new MultiThreadingCounter(15,TimeUnit.MINUTES,false);
+            start(counter, callback, urls);
+            return counter;
+        } catch (Exception e) {
+            logger.error(" exception {}",e);
+            return null;
+        }
     }
 
     /**

src/main/java/com/zhiwei/source_forward/crawler/MediaSelfSourceCrawler.java

@@ -44,10 +44,15 @@ public class MediaSelfSourceCrawler {
      * @return
      * @throws Exception
      */
-    public MultiThreadingCounter submitTask(MediaSelfSourceDataCallBack callback,String... urls) throws Exception {
-        MultiThreadingCounter counter = new MultiThreadingCounter("任务======= ", 15,TimeUnit.SECONDS,true);
-        start(counter, callback, urls);
-        return counter;
+    public MultiThreadingCounter submitTask(MediaSelfSourceDataCallBack callback,String... urls) {
+        try {
+            MultiThreadingCounter counter = new MultiThreadingCounter("任务======= ", 15,TimeUnit.MINUTES,true);
+            start(counter, callback, urls);
+            return counter;
+        } catch (Exception e) {
+            logger.error(" exception  {}",e);
+            return null;
+        }
     }
     
     /**
@@ -60,16 +65,15 @@ public class MediaSelfSourceCrawler {
     private void start(MultiThreadingCounter counter,MediaSelfSourceDataCallBack callback, String... urls) {
         if (urls != null && urls.length > 0) {
             for (String url : urls) {
+                counter.increase();
                 if (url != null) {
                     try {
-                        counter.increase();
                         search(counter, url, Attribution.of(url), callback);
                     } catch (Exception e) {
                         logger.error("搜索创建出错", e);
-                    } finally {
-                        counter.reduce();
                     }
                 }
+                counter.reduce();
             }
         }
     }

src/main/java/com/zhiwei/source_forward/crawler/SourceForwardCrawler.java

@@ -32,25 +32,29 @@ public class SourceForwardCrawler {
     private static HttpBoot httpBoot = new HttpBoot();
     private static List<String> sourceList = SourceData.getSourceList();
     
-    public MultiThreadingCounter submitTask(SourceForwardDataCallBack callback,String... urls) throws Exception {
-        MultiThreadingCounter counter = new MultiThreadingCounter(15,TimeUnit.MINUTES,false);
-        start(counter, callback, urls);
-        return counter;
+    public MultiThreadingCounter submitTask(SourceForwardDataCallBack callback,String... urls) {
+        try {
+            MultiThreadingCounter counter = new MultiThreadingCounter(15,TimeUnit.MINUTES,false);
+            start(counter, callback, urls);
+            return counter;
+        } catch (Exception e) {
+            logger.error(" exception  ", e);
+            return null;
+        }
     }
 
     private void start(MultiThreadingCounter counter,SourceForwardDataCallBack callback, String... urls) {
         if (urls != null && urls.length > 0) {
             for (String url : urls) {
+                counter.increase();
                 if (url != null) {
                     try {
-                        counter.increase();
                         search(counter, url, Attribution.of(url), callback);
                     } catch (Exception e) {
                         logger.error("搜索创建出错", e);
-                    } finally {
-                        counter.reduce();
-                    }
+                    } 
                 }
+                counter.reduce();
             }
         }
     }
@@ -100,10 +104,8 @@ public class SourceForwardCrawler {
                         isforward = "未知";
                     }
                 }else if(attr.get().toString().contains("www.toutiao.com")){
-                	if(body.contains("isOriginal")){
-    					if(body.contains("isOriginal: true")){
-    						isforward = "原创";
-    					}
+                	if(body.contains("isOriginal") && body.contains("isOriginal: true")){
+                	    isforward = "原创";
     				}
                 }else{
                     channel = MatchChannel.verifyChannel(attr.get().toString());

src/main/java/com/zhiwei/source_forward/run/MediaSelfSource.java

@@ -7,8 +7,6 @@ import java.util.List;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
-import com.zhiwei.common.config.GroupType;
-import com.zhiwei.crawler.proxy.ProxyFactory;
 import com.zhiwei.source_forward.bean.MediaSelfSourceBean;
 import com.zhiwei.source_forward.bean.MediaSelfSourceBean.Attribution;
 import com.zhiwei.source_forward.crawler.MediaSelfSourceCrawler;
@@ -19,18 +17,17 @@ public class MediaSelfSource {
     private static Logger logger = LogManager.getLogger(MediaSelfSource.class);
     
     public static List<MediaSelfSourceBean> getMediaSelfSource(List<String> urlList) {
-        List<MediaSelfSourceBean> list = MediaSelfSourceCrawlerThread.getMediaSelfSource(urlList);
-        return list;
+        return MediaSelfSourceCrawlerThread.getMediaSelfSource(urlList);
     }
     
     public static void main(String[] args) {
-        ProxyFactory.init("zookeeper://192.168.0.36:2181","local",GroupType.PROVIDER);
-        List<String> urlList = new ArrayList<>();
-        urlList.add("http://sh.qihoo.com/pc/91d1d565fe552fa1e?sign=360_e39369d1");
-        List<MediaSelfSourceBean> u = MediaSelfSource.getMediaSelfSource(urlList);
-        for(MediaSelfSourceBean b : u) {
-            System.out.println(b.toString());
-        }
+//        ProxyFactory.init("zookeeper://192.168.0.36:2181","local",GroupType.PROVIDER);
+//        List<String> urlList = new ArrayList<>();
+//        urlList.add("http://sh.qihoo.com/pc/91d1d565fe552fa1e?sign=360_e39369d1");
+//        List<MediaSelfSourceBean> u = MediaSelfSource.getMediaSelfSource(urlList);
+//        for(MediaSelfSourceBean b : u) {
+//            System.out.println(b.toString());
+//        }
     }
     
     static class MediaSelfSourceCrawlerThread extends Thread{
@@ -50,7 +47,7 @@ public class MediaSelfSource {
                 };
                 crawler.submitTask(callback,urlList.toArray(new String[urlList.size()])).await();
             }catch (Exception e){
-                e.printStackTrace();
+                logger.error(" 网媒自媒体号 判断 {} ",e);
             }
             return list;
         }

src/main/java/com/zhiwei/source_forward/run/SourceForward.java

@@ -2,7 +2,6 @@ package com.zhiwei.source_forward.run;
 
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -10,11 +9,9 @@ import java.util.Map.Entry;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
-import com.zhiwei.source_forward.bean.MediaSelfSourceBean;
 import com.zhiwei.source_forward.bean.SourceForwardBean;
 import com.zhiwei.source_forward.bean.SourceForwardBean.Attribution;
 import com.zhiwei.source_forward.crawler.SourceForwardCrawler;
-import com.zhiwei.source_forward.run.MediaSelfSource.MediaSelfSourceCrawlerThread;
 import com.zhiwei.source_forward.util.SourceForwardDataCallBack;
 
 /**
@@ -28,82 +25,6 @@ public class SourceForward {
     private static Logger logger = LogManager.getLogger(SourceForward.class);
     
     /**
-     * @Title: getMediaSelfSource 
-     * @author hero 
-     * @Description: 根据链接匹配自媒体号名称 
-     * @param @param dataMap
-     * @param @return 设定文件 
-     * @return Map<String,Map<String,Object>> 返回类型
-     */
-    public static Map<String,Map<String,Object>> getMediaSelfSource(Map<String,Map<String,Object>> dataMap){
-        //启动验证来源程序
-        List<String> urlList = new ArrayList<>();
-        for(Entry<String,Map<String,Object>> entry : dataMap.entrySet()){
-            urlList.add(entry.getKey());
-        }
-        List<MediaSelfSourceBean> sourceForwardList = MediaSelfSourceCrawlerThread.getMediaSelfSource(urlList);
-        for(MediaSelfSourceBean msfb : sourceForwardList){
-            String url = msfb.getUrl();
-            //整合数据及验证转发原创
-            if(dataMap.containsKey(url)){
-                Map<String,Object> data = dataMap.get(url);
-                data.put("自媒体号", msfb.getMediaself());
-                data.put("频道", msfb.getChannel());
-                dataMap.put(url, data);
-            }
-        }
-        return dataMap;
-    }
-    
-    /**
-     * @Title: getMediaSelfSource 
-     * @author hero 
-     * @Description: 根据链接匹配自媒体账号
-     * @param @param urlList
-     * @param @return 设定文件 
-     * @return Map<String,String> 返回类型
-     */
-    public static Map<String,String> getMediaSelfSource(List<String> urlList){
-        //启动验证来源程序
-        Map<String,String> dataMap = new HashMap<>();
-        for(String url : urlList){
-            dataMap.put(url, null);
-        }
-        
-        List<MediaSelfSourceBean> sourceForwardList = MediaSelfSourceCrawlerThread.getMediaSelfSource(urlList);
-        for(MediaSelfSourceBean mssb : sourceForwardList){
-            String url = mssb.getUrl();
-            //整合数据及验证转发原创
-            if(dataMap.containsKey(url)){
-                dataMap.put(url, mssb.getMediaself());
-            }
-        }
-        return dataMap;
-    }
-
-    /**
-     * 
-     * @Title: getMediaSelfSource 
-     * @author hero 
-     * @Description: 根据链接匹配自媒体账号
-     * @param @param url
-     * @param @return 设定文件 
-     * @return String 返回类型
-     */
-    public static String getMediaSelfSource(String url){
-        //启动验证来源程序
-        List<String> urlList = new ArrayList<>();
-        urlList.add(url);
-        
-        List<MediaSelfSourceBean> sourceForwardList = MediaSelfSourceCrawlerThread.getMediaSelfSource(urlList);
-        for(MediaSelfSourceBean sourceMap : sourceForwardList){
-            return sourceMap.getMediaself();
-        }
-        return null;
-    }
-
-    
-    /**
      * @Title: getSourceForward 
      * @author hero 
      * @Description: 验证文章是否转发
@@ -117,9 +38,7 @@ public class SourceForward {
         for(Entry<String,Map<String,Object>> entry : dataMap.entrySet()){
             urlList.add(entry.getKey());
         }
-        System.out.println(urlList.size());
         List<SourceForwardBean> dataList = SourceForwardCrawlerThread.getSourceForward(urlList);
-        System.out.println(dataList.size());
         for(SourceForwardBean sfb : dataList){
             String url = sfb.getUrl();
             String root_source = sfb.getRoot_source();
@@ -143,7 +62,6 @@ public class SourceForward {
                 dataMap.put(url, data);
             }
         }
-        System.out.println("success");
         return dataMap;
     } 
 
@@ -161,7 +79,7 @@ public class SourceForward {
 	public static void main(String[] args) {
 //	    ProxyFactory.init("zookeeper://192.168.0.36:2181","local",GroupType.PROVIDER);
 //        List<String> urlList = new ArrayList<>();
-//        urlList.add("http://www.toutiao.com/a6452936157751968013/");
+//        urlList.add("https://www.toutiao.com/a6634320415839748621");
 //        List<SourceForwardBean> da = SourceForward.getSourceForward(urlList);
 //        for(SourceForwardBean sfb : da) {
 //            System.out.println(sfb.toString());
@@ -185,7 +103,7 @@ public class SourceForward {
               };
               crawler.submitTask(callback,urlList.toArray(new String[urlList.size()])).await();
           }catch (Exception e){
-              e.printStackTrace();
+              logger.error(" 来源判断 出错 {} ",e);
           }
           return list;
       }

src/main/java/com/zhiwei/source_forward/run/URLLive.java

@@ -6,6 +6,9 @@ import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
 import com.zhiwei.source_forward.bean.UrlLiveBean;
 import com.zhiwei.source_forward.bean.UrlLiveBean.Attribution;
 import com.zhiwei.source_forward.crawler.UrlLiveCrawler;
@@ -19,6 +22,8 @@ import com.zhiwei.source_forward.util.UrlLiveDataCallback;
  */
 public class URLLive {
     
+    private static Logger logger = LogManager.getLogger(URLLive.class);
+    
     /**
      * @Title: verificationURLLive 
      * @author hero 
@@ -46,6 +51,7 @@ public class URLLive {
                 }else if(i == 0) {
                     map.put("是否删除", false);
                 }
+                map.put("title", ub.getTitle());
                 dataMap.put(url, map);
             }
         }
@@ -60,18 +66,20 @@ public class URLLive {
      */
 	public static List<UrlLiveBean> verificationURLLive(List<String> urlList){
         //启动验证链接是否有效程序程序
-	    List<UrlLiveBean> dataList = UrlLiveCrawlerThread.getUrlLiveCrawle(urlList);
-	    return dataList;
+	    return UrlLiveCrawlerThread.getUrlLiveCrawle(urlList);
     }
 	
-	public static void main(String[] args) {
-	    List<String> urlList = new ArrayList<>();
-	    urlList.add("http://www.teso.cc/html/zixun/201606/233848.html");
-        List<UrlLiveBean> u = URLLive.verificationURLLive(urlList);
-        for(UrlLiveBean b : u) {
-            System.out.println(b.toString());
-        }
-    }
+//	public static void main(String[] args) {
+//	    ProxyFactory.init("zookeeper://192.168.0.36:2181","local",GroupType.PROVIDER);
+//	    List<String> urlList = new ArrayList<>();
+//	    urlList.add("http://sh.qihoo.com/mob/transcoding?sign=360_e39369d1&n=10&pg=41&u=84c80ad777cd9a41152b4fd9c44f96e2&gzh=3093075895&news_sdk_version=&sqid=&_=1545026725607&callback=jsonp75&url=http%3A%2F%2Fzm.news.so.com%2F708e22872ce43ca08eec2a1fc57c6897&check=e0fae47326e7916f&ucheck=75e961d9583cfebe81a39e2dd972b0aa&uid=84c80ad777cd9a41152b4fd9c44f96e2&360newsdetail=1&c=detail&apiflag=detail&articlety=zmt");
+//	    
+//	    
+//        List<UrlLiveBean> u = URLLive.verificationURLLive(urlList);
+//        for(UrlLiveBean b : u) {
+//            System.out.println(b.toString());
+//        }
+//    }
 	
 	static class UrlLiveCrawlerThread extends Thread{
 
@@ -90,7 +98,7 @@ public class URLLive {
               };
               crawler.submitTask(callback,urlList.toArray(new String[urlList.size()])).await();
           }catch (Exception e){
-              e.printStackTrace();
+              logger.error(" 数据采集运行有问题 {} ", e);
           }
           return list;
       }

src/main/java/com/zhiwei/source_forward/util/MatchSource.java

@@ -130,22 +130,23 @@ public class MatchSource {
 				}else if(html.contains("screen_name:")){
 					source = html.split("screen_name:'")[1].split("',")[0].trim();
 				}
-				if(source!=null && !source.equals("")){
+				if(source!=null && source.length()>1){
 					source =  "今日头条-" + source;
 				}
 			}else if(url.contains("sohu.com")){ 
 				//搜狐自媒体号
 				if(html.contains("<meta name=\"mediaid\"")){
 					source = html.split("<meta name=\"mediaid\" content=\"")[1].split("\"")[0].trim();
-					if(source!=null && !source.equals("")){
+					if(source!=null && source.length()>1){
 						source =  "搜狐-" + source;
 					}
 				}
 			}else if(url.contains("a.mini.eastday.com")){  
 				//处理东方头条网-自媒体号匹配
 				source = document.select("[class=\"share_cnt_p clearfix\"]").select("div.fl").select("i").get(1).text().trim();
-				if(source!=null && !source.equals("")){
+				if(source!=null && source.length()>1){
 					source =  "东方头条-" + source;
+					
 				}
 			}else if(url.contains("sh.qihoo.com")){
 				//今日报点解析
@@ -153,45 +154,83 @@ public class MatchSource {
 				if(source.length() < 2) {
 				    source = document.select("p.article-info").select("a").text().trim();
 				}
-				if(source!=null && !source.equals("")){
+				if(source!=null && source.length()>1){
 					source =  "快资讯-" + source;
 				}
 			}else if(url.contains("cj.sina.com.cn")){
 				//新浪财经头条号
 				if(html.contains("<meta name=\"mediaid\"")){
 					source = html.split("<meta name=\"mediaid\" content=\"")[1].split("\"")[0].trim();
-					if(source!=null && !source.equals("")){
+					if(source!=null && source.length()>1){
 						source = "财经头条-" + source;
 					}
 				}
 			}else if(url.contains("baijiahao.baidu.com")){
 				//百度百家
-				source = document.select("div.author").select("div.author-txt").select("p.author-name").text().trim();
-				if(source!=null && !source.equals("")){
+			    source = document.select("p.author-name").first().text().trim();
+				if(source!=null && source.length()>1){
 					source = "百度百家-" + source;
 				}
 			}else if(url.contains("yidianzixun.com")){
 				//一点资讯
 				if(html.contains("related_wemedia")){
 					source = html.split("media_name\":\"")[1].split("\",\"")[0].trim();
-					if(source!=null && !source.equals("")){
+					if(source!=null && source.length()>1){
 						source = "一点资讯-" + source;
 					}
 				}else{
 					source = html.split("source\":\"")[1].split("\",\"")[0];
 				}
 			}else if(url.contains("news.bitauto.com")){
+				//易车网
 				source = document.select("[class=\"gz-box clearfix\"]").select("div.txt-box")
 						.select("p.p-n").select("a").text();
-				if(source!=null && !source.equals("")){
+				if(source!=null && source.length()>1){
 					source = "易车网-" + source;
 				}
 			}else if(url.contains("chejiahao.autohome.com.cn")){
+				//汽车之家
 				source = document.select("div.authorMes").select("[class=\"name text-overflow\"]")
 						.select("a").text();
-				if(source!=null && !source.equals("")){
+				if(source!=null && source.length()>1){
 					source = "汽车之家-" + source;
 				}
+			}else if(url.contains("item.btime.com")){
+				//北京时间
+				source = document.select("a.author").text();
+				if(source!=null && source.length()>1){
+					source = "北京时间-" + source;
+				}
+			}else if(url.contains("new.qq.com/omn/")){
+				//腾讯网-企鹅号
+				source = html.split("media\":\"")[1].split("\",\"")[0];
+				if(source!=null && source.length()>1){
+					source = "企鹅号-" + source;
+				}
+			}else if(url.contains("feng.ifeng.com")){
+				//凤凰网-大风号
+				source = html.split("source\":\"")[1].split("\",\"")[0];
+				if(source!=null && source.length()>1){
+					source = "大风号-" + source;
+				}
+			}else if(url.contains("dy.163.com")){
+				//网易订阅-网易号
+				source = document.select("div.colum_info>h4").text();
+				if(source!=null && source.length()>1){
+					source = "网易号-" + source;
+				}
+			}else if(url.contains("qctt.cn")){
+				//汽车头条
+				source = document.select("div.part2>a").text();
+				if(source!=null && source.length()>1){
+					source = "汽车头条-" + source;
+				}
+			}else if(url.contains("maiche.com")){
+				//买车网
+				source = document.select("div.info-left > div:nth-child(2) > span > a").text();
+				if(source!=null && source.length()>1){
+					source = "买车网-" + source;
+				}
 			}else if(url.contains("3g.163.com")){
 				source = document.select("div.info").select("[class=\"source js-source\"]")
 						.text();
@@ -201,7 +240,6 @@ public class MatchSource {
 			}
 			return source;
 		} catch (Exception e) {
-		    e.printStackTrace();
 			return null;
 		}
 	}