提交修改后版本

pom.xml

@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>com.zhiwei</groupId>
     <artifactId>source-forward</artifactId>
-    <version>0.0.7-SNAPSHOT</version>
+    <version>0.0.8-SNAPSHOT</version>
     <name>source-forward</name>
     <description>验证网媒的转发关系及链接的有效性（转发验证微信及自媒体匹配率不高）</description>
 

src/main/java/com/zhiwei/source_forward/crawler/MediaSelfSourceCrawler.java

@@ -45,7 +45,7 @@ public class MediaSelfSourceCrawler {
      * @throws Exception
      */
     public MultiThreadingCounter submitTask(MediaSelfSourceDataCallBack callback,String... urls) throws Exception {
-        MultiThreadingCounter counter = new MultiThreadingCounter("任务======= ", 15,TimeUnit.SECONDS,true);
+        MultiThreadingCounter counter = new MultiThreadingCounter("任务======= ", 15,TimeUnit.MINUTES,true);
         start(counter, callback, urls);
         return counter;
     }

src/main/java/com/zhiwei/source_forward/crawler/SourceForwardCrawler.java

@@ -100,11 +100,9 @@ public class SourceForwardCrawler {
                         isforward = "未知";
                     }
                 }else if(attr.get().toString().contains("www.toutiao.com")){
-                	if(body.contains("isOriginal")){
-    					if(body.contains("isOriginal: true")){
+                	if(body.contains("isOriginal") && body.contains("isOriginal: true")){
                 	    isforward = "原创";
     				}
-    				}
                 }else{
                     channel = MatchChannel.verifyChannel(attr.get().toString());
                     if(channel==null){

src/main/java/com/zhiwei/source_forward/run/MediaSelfSource.java

@@ -7,8 +7,6 @@ import java.util.List;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
-import com.zhiwei.common.config.GroupType;
-import com.zhiwei.crawler.proxy.ProxyFactory;
 import com.zhiwei.source_forward.bean.MediaSelfSourceBean;
 import com.zhiwei.source_forward.bean.MediaSelfSourceBean.Attribution;
 import com.zhiwei.source_forward.crawler.MediaSelfSourceCrawler;
@@ -19,18 +17,17 @@ public class MediaSelfSource {
     private static Logger logger = LogManager.getLogger(MediaSelfSource.class);
     
     public static List<MediaSelfSourceBean> getMediaSelfSource(List<String> urlList) {
-        List<MediaSelfSourceBean> list = MediaSelfSourceCrawlerThread.getMediaSelfSource(urlList);
-        return list;
+        return MediaSelfSourceCrawlerThread.getMediaSelfSource(urlList);
     }
     
     public static void main(String[] args) {
-        ProxyFactory.init("zookeeper://192.168.0.36:2181","local",GroupType.PROVIDER);
-        List<String> urlList = new ArrayList<>();
-        urlList.add("http://sh.qihoo.com/pc/91d1d565fe552fa1e?sign=360_e39369d1");
-        List<MediaSelfSourceBean> u = MediaSelfSource.getMediaSelfSource(urlList);
-        for(MediaSelfSourceBean b : u) {
-            System.out.println(b.toString());
-        }
+//        ProxyFactory.init("zookeeper://192.168.0.36:2181","local",GroupType.PROVIDER);
+//        List<String> urlList = new ArrayList<>();
+//        urlList.add("http://sh.qihoo.com/pc/91d1d565fe552fa1e?sign=360_e39369d1");
+//        List<MediaSelfSourceBean> u = MediaSelfSource.getMediaSelfSource(urlList);
+//        for(MediaSelfSourceBean b : u) {
+//            System.out.println(b.toString());
+//        }
     }
     
     static class MediaSelfSourceCrawlerThread extends Thread{

src/main/java/com/zhiwei/source_forward/run/SourceForward.java

@@ -2,7 +2,6 @@ package com.zhiwei.source_forward.run;
 
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -10,11 +9,9 @@ import java.util.Map.Entry;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
-import com.zhiwei.source_forward.bean.MediaSelfSourceBean;
 import com.zhiwei.source_forward.bean.SourceForwardBean;
 import com.zhiwei.source_forward.bean.SourceForwardBean.Attribution;
 import com.zhiwei.source_forward.crawler.SourceForwardCrawler;
-import com.zhiwei.source_forward.run.MediaSelfSource.MediaSelfSourceCrawlerThread;
 import com.zhiwei.source_forward.util.SourceForwardDataCallBack;
 
 /**
@@ -28,82 +25,6 @@ public class SourceForward {
     private static Logger logger = LogManager.getLogger(SourceForward.class);
     
     /**
-     * @Title: getMediaSelfSource 
-     * @author hero 
-     * @Description: 根据链接匹配自媒体号名称 
-     * @param @param dataMap
-     * @param @return 设定文件 
-     * @return Map<String,Map<String,Object>> 返回类型
-     */
-    public static Map<String,Map<String,Object>> getMediaSelfSource(Map<String,Map<String,Object>> dataMap){
-        //启动验证来源程序
-        List<String> urlList = new ArrayList<>();
-        for(Entry<String,Map<String,Object>> entry : dataMap.entrySet()){
-            urlList.add(entry.getKey());
-        }
-        List<MediaSelfSourceBean> sourceForwardList = MediaSelfSourceCrawlerThread.getMediaSelfSource(urlList);
-        for(MediaSelfSourceBean msfb : sourceForwardList){
-            String url = msfb.getUrl();
-            //整合数据及验证转发原创
-            if(dataMap.containsKey(url)){
-                Map<String,Object> data = dataMap.get(url);
-                data.put("自媒体号", msfb.getMediaself());
-                data.put("频道", msfb.getChannel());
-                dataMap.put(url, data);
-            }
-        }
-        return dataMap;
-    }
-    
-    /**
-     * @Title: getMediaSelfSource 
-     * @author hero 
-     * @Description: 根据链接匹配自媒体账号
-     * @param @param urlList
-     * @param @return 设定文件 
-     * @return Map<String,String> 返回类型
-     */
-    public static Map<String,String> getMediaSelfSource(List<String> urlList){
-        //启动验证来源程序
-        Map<String,String> dataMap = new HashMap<>();
-        for(String url : urlList){
-            dataMap.put(url, null);
-        }
-        
-        List<MediaSelfSourceBean> sourceForwardList = MediaSelfSourceCrawlerThread.getMediaSelfSource(urlList);
-        for(MediaSelfSourceBean mssb : sourceForwardList){
-            String url = mssb.getUrl();
-            //整合数据及验证转发原创
-            if(dataMap.containsKey(url)){
-                dataMap.put(url, mssb.getMediaself());
-            }
-        }
-        return dataMap;
-    }
-
-    /**
-     * 
-     * @Title: getMediaSelfSource 
-     * @author hero 
-     * @Description: 根据链接匹配自媒体账号
-     * @param @param url
-     * @param @return 设定文件 
-     * @return String 返回类型
-     */
-    public static String getMediaSelfSource(String url){
-        //启动验证来源程序
-        List<String> urlList = new ArrayList<>();
-        urlList.add(url);
-        
-        List<MediaSelfSourceBean> sourceForwardList = MediaSelfSourceCrawlerThread.getMediaSelfSource(urlList);
-        for(MediaSelfSourceBean sourceMap : sourceForwardList){
-            return sourceMap.getMediaself();
-        }
-        return null;
-    }
-
-    
-    /**
      * @Title: getSourceForward 
      * @author hero 
      * @Description: 验证文章是否转发
@@ -117,9 +38,7 @@ public class SourceForward {
         for(Entry<String,Map<String,Object>> entry : dataMap.entrySet()){
             urlList.add(entry.getKey());
         }
-        System.out.println(urlList.size());
         List<SourceForwardBean> dataList = SourceForwardCrawlerThread.getSourceForward(urlList);
-        System.out.println(dataList.size());
         for(SourceForwardBean sfb : dataList){
             String url = sfb.getUrl();
             String root_source = sfb.getRoot_source();
@@ -161,7 +80,7 @@ public class SourceForward {
 	public static void main(String[] args) {
 //	    ProxyFactory.init("zookeeper://192.168.0.36:2181","local",GroupType.PROVIDER);
 //        List<String> urlList = new ArrayList<>();
-//        urlList.add("http://www.toutiao.com/a6452936157751968013/");
+//        urlList.add("https://www.toutiao.com/a6634320415839748621");
 //        List<SourceForwardBean> da = SourceForward.getSourceForward(urlList);
 //        for(SourceForwardBean sfb : da) {
 //            System.out.println(sfb.toString());