Skip to content

M
media_data_crawler
Commit 0537cf5a by zhiwei
Options

111

parent 1b78ab01
Showing with 106 additions and 62 deletions
src/main/java/com/zhiwei/media_data_crawler/data/DataCrawler.java
...@@ -18,23 +18,33 @@ import com.zhiwei.media_data_crawler.entity.ZhiHuData; ...@@ -18,23 +18,33 @@ import com.zhiwei.media_data_crawler.entity.ZhiHuData;
public class DataCrawler { public class DataCrawler {
public static Long sleepTime;
public void setSleepTime(Long sleepTime) {
DataCrawler.sleepTime = sleepTime;
}
/** /**
* *
* @Title: getBaiduNewsData * @Title: getBaiduNewsData
* @author hero * @author hero
* @Description: 根据关键词和时间，全文匹配百度新闻数据 * @Description: 根据关键词和时间，全文匹配百度新闻数据
* @param @param word * @param @param
* @param @param startTime * word
* @param @param endTime * @param @param
* @param @param proxy * startTime
* @param @return 设定文件 * @param @param
* endTime
* @param @param
* proxy
* @param @return
* 设定文件
* @return List<NewsData> 返回类型 * @return List<NewsData> 返回类型
*/ */
public static List<NewsData> getBaiduNewsData(String word, String startTime, String endTime, Proxy proxy,Long sleepTime){ public static List<NewsData> getBaiduNewsData(String word, String startTime, String endTime, Proxy proxy
) {
try { try {
return BaiduNewsCrawlerParse.getBaiduNewsData(word, startTime, endTime, proxy, sleepTime); return BaiduNewsCrawlerParse.getBaiduNewsData(word, startTime, endTime, proxy);
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
return null; return null;
...@@ -46,16 +56,21 @@ public class DataCrawler { ...@@ -46,16 +56,21 @@ public class DataCrawler {
* @Title: getBaiduNewsDataByTitle * @Title: getBaiduNewsDataByTitle
* @author hero * @author hero
* @Description: 根据关键词和时间，标题匹配百度新闻数据 * @Description: 根据关键词和时间，标题匹配百度新闻数据
* @param @param word * @param @param
* @param @param startTime * word
* @param @param endTime * @param @param
* @param @param proxy * startTime
* @param @return 设定文件 * @param @param
* endTime
* @param @param
* proxy
* @param @return
* 设定文件
* @return List<NewsData> 返回类型 * @return List<NewsData> 返回类型
*/ */
public static List<NewsData> getBaiduNewsDataByTitle(String word, String startTime, String endTime, Proxy proxy,Long sleepTime){ public static List<NewsData> getBaiduNewsDataByTitle(String word, String startTime, String endTime, Proxy proxy) {
try { try {
return BaiduNewsCrawlerParse.getBaiduNewsDataByTitle(word, startTime, endTime, proxy,sleepTime); return BaiduNewsCrawlerParse.getBaiduNewsDataByTitle(word, startTime, endTime, proxy);
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
return null; return null;
...@@ -67,12 +82,15 @@ public class DataCrawler { ...@@ -67,12 +82,15 @@ public class DataCrawler {
* @Title: getSoNewsData * @Title: getSoNewsData
* @author hero * @author hero
* @Description: 采集360新闻数据,按照全文匹配 * @Description: 采集360新闻数据,按照全文匹配
* @param @param word * @param @param
* @param @param proxy * word
* @param @return 设定文件 * @param @param
* proxy
* @param @return
* 设定文件
* @return List<NewsData> 返回类型 * @return List<NewsData> 返回类型
*/ */
public static List<NewsData> getSoNewsData(String word, Proxy proxy){ public static List<NewsData> getSoNewsData(String word, Proxy proxy) {
try { try {
return SoNewsCrawlerParse.getSoNewsData(word, proxy); return SoNewsCrawlerParse.getSoNewsData(word, proxy);
} catch (Exception e) { } catch (Exception e) {
...@@ -86,12 +104,15 @@ public class DataCrawler { ...@@ -86,12 +104,15 @@ public class DataCrawler {
* @Title: getSoNewsDataByTitle * @Title: getSoNewsDataByTitle
* @author hero * @author hero
* @Description: 采集360新闻数据 ,按照标题匹配 * @Description: 采集360新闻数据 ,按照标题匹配
* @param @param word * @param @param
* @param @param proxy * word
* @param @return 设定文件 * @param @param
* proxy
* @param @return
* 设定文件
* @return List<NewsData> 返回类型 * @return List<NewsData> 返回类型
*/ */
public static List<NewsData> getSoNewsDataByTitle(String word, Proxy proxy){ public static List<NewsData> getSoNewsDataByTitle(String word, Proxy proxy) {
try { try {
return SoNewsCrawlerParse.getSoNewsDataByTitle(word, proxy); return SoNewsCrawlerParse.getSoNewsDataByTitle(word, proxy);
} catch (Exception e) { } catch (Exception e) {
...@@ -100,21 +121,22 @@ public class DataCrawler { ...@@ -100,21 +121,22 @@ public class DataCrawler {
} }
} }
/** /**
* *
* @Title: getSougouNewsData * @Title: getSougouNewsData
* @author hero * @author hero
* @Description: 搜狗新闻采集，全文匹配 * @Description: 搜狗新闻采集，全文匹配
* @param @param word * @param @param
* @param @param proxy * word
* @param @return 设定文件 * @param @param
* proxy
* @param @return
* 设定文件
* @return List<NewsData> 返回类型 * @return List<NewsData> 返回类型
*/ */
public static List<NewsData> getSougouNewsData(String word, Proxy proxy,Long sleepTime){ public static List<NewsData> getSougouNewsData(String word, Proxy proxy) {
try { try {
return SougouNewsCrawlerParse.getSougouNewsData(word, proxy,sleepTime); return SougouNewsCrawlerParse.getSougouNewsData(word, proxy);
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
return null; return null;
...@@ -126,14 +148,17 @@ public class DataCrawler { ...@@ -126,14 +148,17 @@ public class DataCrawler {
* @Title: getSougouNewsDataByTitle * @Title: getSougouNewsDataByTitle
* @author hero * @author hero
* @Description: 搜狗新闻采集，标题匹配 * @Description: 搜狗新闻采集，标题匹配
* @param @param word * @param @param
* @param @param proxy * word
* @param @return 设定文件 * @param @param
* proxy
* @param @return
* 设定文件
* @return List<NewsData> 返回类型 * @return List<NewsData> 返回类型
*/ */
public static List<NewsData> getSougouNewsDataByTitle(String word, Proxy proxy,Long sleepTime){ public static List<NewsData> getSougouNewsDataByTitle(String word, Proxy proxy) {
try { try {
return SougouNewsCrawlerParse.getSougouNewsDataByTitle(word, proxy, sleepTime); return SougouNewsCrawlerParse.getSougouNewsDataByTitle(word, proxy);
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
return null; return null;
...@@ -144,14 +169,17 @@ public class DataCrawler { ...@@ -144,14 +169,17 @@ public class DataCrawler {
* @Title: getSougouZhihuData * @Title: getSougouZhihuData
* @author hero * @author hero
* @Description: 根据关键词在搜狗知乎采集相应的知乎数据 * @Description: 根据关键词在搜狗知乎采集相应的知乎数据
* @param @param word * @param @param
* @param @param proxy * word
* @param @return 设定文件 * @param @param
* proxy
* @param @return
* 设定文件
* @return List<ZhiHuData> 返回类型 * @return List<ZhiHuData> 返回类型
*/ */
public static List<ZhiHuData> getSougouZhihuData(String word, Proxy proxy,Long sleepTime){ public static List<ZhiHuData> getSougouZhihuData(String word, Proxy proxy) {
try { try {
return SougouZhihuCrawlerParse.getSougouZhihuData(word, proxy, sleepTime); return SougouZhihuCrawlerParse.getSougouZhihuData(word, proxy);
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
return null; return null;
...@@ -162,14 +190,17 @@ public class DataCrawler { ...@@ -162,14 +190,17 @@ public class DataCrawler {
* @Title: getBaiduTiebaData * @Title: getBaiduTiebaData
* @author hero * @author hero
* @Description: 根据关键词采集贴吧数据 * @Description: 根据关键词采集贴吧数据
* @param @param word * @param @param
* @param @param proxy * word
* @param @return 设定文件 * @param @param
* proxy
* @param @return
* 设定文件
* @return List<TiebaData> 返回类型 * @return List<TiebaData> 返回类型
*/ */
public static List<TiebaData> getBaiduTiebaData(String word, Proxy proxy,Long sleepTime){ public static List<TiebaData> getBaiduTiebaData(String word, Proxy proxy) {
try { try {
return BaiduTiebaCrawlerParse.getBaiduTiebaData(word, proxy, null, sleepTime); return BaiduTiebaCrawlerParse.getBaiduTiebaData(word, proxy, null);
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
return null; return null;
...@@ -180,15 +211,19 @@ public class DataCrawler { ...@@ -180,15 +211,19 @@ public class DataCrawler {
* @Title: getBaiduTiebaData * @Title: getBaiduTiebaData
* @author hero * @author hero
* @Description: 根据关键词采集指定贴吧内数据 * @Description: 根据关键词采集指定贴吧内数据
* @param @param word * @param @param
* @param @param proxy * word
* @param @param tiebaName * @param @param
* @param @return 设定文件 * proxy
* @param @param
* tiebaName
* @param @return
* 设定文件
* @return List<TiebaData> 返回类型 * @return List<TiebaData> 返回类型
*/ */
public static List<TiebaData> getBaiduTiebaData(String word, Proxy proxy, String tiebaName,Long sleepTime){ public static List<TiebaData> getBaiduTiebaData(String word, Proxy proxy, String tiebaName) {
try { try {
return BaiduTiebaCrawlerParse.getBaiduTiebaData(word, proxy, tiebaName,sleepTime); return BaiduTiebaCrawlerParse.getBaiduTiebaData(word, proxy, tiebaName);
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
return null; return null;
...@@ -199,15 +234,19 @@ public class DataCrawler { ...@@ -199,15 +234,19 @@ public class DataCrawler {
* @Title: getLunTanData * @Title: getLunTanData
* @author hero * @author hero
* @Description: 根据关键词采集天涯论坛数据 * @Description: 根据关键词采集天涯论坛数据
* @param @param word * @param @param
* @param @param proxy * word
* @param @param endTime * @param @param
* @param @return 设定文件 * proxy
* @param @param
* endTime
* @param @return
* 设定文件
* @return List<LunTanData> 返回类型 * @return List<LunTanData> 返回类型
*/ */
public static List<LunTanData> getLunTanData(String word, Proxy proxy, String endTime,Long sleepTime){ public static List<LunTanData> getLunTanData(String word, Proxy proxy, String endTime) {
try { try {
return TianYaCrawlerParse.getLunTanData(word, proxy, endTime,sleepTime); return TianYaCrawlerParse.getLunTanData(word, proxy, endTime);
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
return null; return null;
...@@ -218,14 +257,19 @@ public class DataCrawler { ...@@ -218,14 +257,19 @@ public class DataCrawler {
* @Title: getDouBanData * @Title: getDouBanData
* @author hero * @author hero
* @Description: 根据关键词采集豆瓣数据 * @Description: 根据关键词采集豆瓣数据
* @param @param word * @param @param
* @param @param type type=topic,type=note * word
* @param @param proxy * @param @param
* @param @param endTime * type type=topic,type=note
* @param @return 设定文件 * @param @param
* proxy
* @param @param
* endTime
* @param @return
* 设定文件
* @return List<DouBanData> 返回类型 * @return List<DouBanData> 返回类型
*/ */
public static List<DouBanData> getDouBanData(String word, String type, Proxy proxy){ public static List<DouBanData> getDouBanData(String word, String type, Proxy proxy) {
try { try {
return DoubanCrawlerParse.getDoubanData(word, type, proxy); return DoubanCrawlerParse.getDoubanData(word, type, proxy);
} catch (Exception e) { } catch (Exception e) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment