Skip to content

M
media_data_crawler
Commit 0537cf5a by zhiwei
Options

111

parent 1b78ab01
Showing with 106 additions and 62 deletions
src/main/java/com/zhiwei/media_data_crawler/data/DataCrawler.java
......@@ -18,23 +18,33 @@ import com.zhiwei.media_data_crawler.entity.ZhiHuData;
public class DataCrawler {
public static Long sleepTime;
public void setSleepTime(Long sleepTime) {
DataCrawler.sleepTime = sleepTime;
}
/**
*
* @Title: getBaiduNewsData
* @author hero
* @Description: 根据关键词和时间，全文匹配百度新闻数据
* @param @param word
* @param @param startTime
* @param @param endTime
* @param @param proxy
* @param @return 设定文件
* @param @param
* word
* @param @param
* startTime
* @param @param
* endTime
* @param @param
* proxy
* @param @return
* 设定文件
* @return List<NewsData> 返回类型
*/
public static List<NewsData> getBaiduNewsData(String word, String startTime, String endTime, Proxy proxy,Long sleepTime){
public static List<NewsData> getBaiduNewsData(String word, String startTime, String endTime, Proxy proxy
) {
try {
return BaiduNewsCrawlerParse.getBaiduNewsData(word, startTime, endTime, proxy, sleepTime);
return BaiduNewsCrawlerParse.getBaiduNewsData(word, startTime, endTime, proxy);
} catch (Exception e) {
e.printStackTrace();
return null;
......@@ -46,16 +56,21 @@ public class DataCrawler {
* @Title: getBaiduNewsDataByTitle
* @author hero
* @Description: 根据关键词和时间，标题匹配百度新闻数据
* @param @param word
* @param @param startTime
* @param @param endTime
* @param @param proxy
* @param @return 设定文件
* @param @param
* word
* @param @param
* startTime
* @param @param
* endTime
* @param @param
* proxy
* @param @return
* 设定文件
* @return List<NewsData> 返回类型
*/
public static List<NewsData> getBaiduNewsDataByTitle(String word, String startTime, String endTime, Proxy proxy,Long sleepTime){
public static List<NewsData> getBaiduNewsDataByTitle(String word, String startTime, String endTime, Proxy proxy) {
try {
return BaiduNewsCrawlerParse.getBaiduNewsDataByTitle(word, startTime, endTime, proxy,sleepTime);
return BaiduNewsCrawlerParse.getBaiduNewsDataByTitle(word, startTime, endTime, proxy);
} catch (Exception e) {
e.printStackTrace();
return null;
......@@ -67,12 +82,15 @@ public class DataCrawler {
* @Title: getSoNewsData
* @author hero
* @Description: 采集360新闻数据,按照全文匹配
* @param @param word
* @param @param proxy
* @param @return 设定文件
* @param @param
* word
* @param @param
* proxy
* @param @return
* 设定文件
* @return List<NewsData> 返回类型
*/
public static List<NewsData> getSoNewsData(String word, Proxy proxy){
public static List<NewsData> getSoNewsData(String word, Proxy proxy) {
try {
return SoNewsCrawlerParse.getSoNewsData(word, proxy);
} catch (Exception e) {
......@@ -86,12 +104,15 @@ public class DataCrawler {
* @Title: getSoNewsDataByTitle
* @author hero
* @Description: 采集360新闻数据 ,按照标题匹配
* @param @param word
* @param @param proxy
* @param @return 设定文件
* @param @param
* word
* @param @param
* proxy
* @param @return
* 设定文件
* @return List<NewsData> 返回类型
*/
public static List<NewsData> getSoNewsDataByTitle(String word, Proxy proxy){
public static List<NewsData> getSoNewsDataByTitle(String word, Proxy proxy) {
try {
return SoNewsCrawlerParse.getSoNewsDataByTitle(word, proxy);
} catch (Exception e) {
......@@ -100,21 +121,22 @@ public class DataCrawler {
}
}
/**
*
* @Title: getSougouNewsData
* @author hero
* @Description: 搜狗新闻采集，全文匹配
* @param @param word
* @param @param proxy
* @param @return 设定文件
* @param @param
* word
* @param @param
* proxy
* @param @return
* 设定文件
* @return List<NewsData> 返回类型
*/
public static List<NewsData> getSougouNewsData(String word, Proxy proxy,Long sleepTime){
public static List<NewsData> getSougouNewsData(String word, Proxy proxy) {
try {
return SougouNewsCrawlerParse.getSougouNewsData(word, proxy,sleepTime);
return SougouNewsCrawlerParse.getSougouNewsData(word, proxy);
} catch (Exception e) {
e.printStackTrace();
return null;
......@@ -126,14 +148,17 @@ public class DataCrawler {
* @Title: getSougouNewsDataByTitle
* @author hero
* @Description: 搜狗新闻采集，标题匹配
* @param @param word
* @param @param proxy
* @param @return 设定文件
* @param @param
* word
* @param @param
* proxy
* @param @return
* 设定文件
* @return List<NewsData> 返回类型
*/
public static List<NewsData> getSougouNewsDataByTitle(String word, Proxy proxy,Long sleepTime){
public static List<NewsData> getSougouNewsDataByTitle(String word, Proxy proxy) {
try {
return SougouNewsCrawlerParse.getSougouNewsDataByTitle(word, proxy, sleepTime);
return SougouNewsCrawlerParse.getSougouNewsDataByTitle(word, proxy);
} catch (Exception e) {
e.printStackTrace();
return null;
......@@ -144,14 +169,17 @@ public class DataCrawler {
* @Title: getSougouZhihuData
* @author hero
* @Description: 根据关键词在搜狗知乎采集相应的知乎数据
* @param @param word
* @param @param proxy
* @param @return 设定文件
* @param @param
* word
* @param @param
* proxy
* @param @return
* 设定文件
* @return List<ZhiHuData> 返回类型
*/
public static List<ZhiHuData> getSougouZhihuData(String word, Proxy proxy,Long sleepTime){
public static List<ZhiHuData> getSougouZhihuData(String word, Proxy proxy) {
try {
return SougouZhihuCrawlerParse.getSougouZhihuData(word, proxy, sleepTime);
return SougouZhihuCrawlerParse.getSougouZhihuData(word, proxy);
} catch (Exception e) {
e.printStackTrace();
return null;
......@@ -162,14 +190,17 @@ public class DataCrawler {
* @Title: getBaiduTiebaData
* @author hero
* @Description: 根据关键词采集贴吧数据
* @param @param word
* @param @param proxy
* @param @return 设定文件
* @param @param
* word
* @param @param
* proxy
* @param @return
* 设定文件
* @return List<TiebaData> 返回类型
*/
public static List<TiebaData> getBaiduTiebaData(String word, Proxy proxy,Long sleepTime){
public static List<TiebaData> getBaiduTiebaData(String word, Proxy proxy) {
try {
return BaiduTiebaCrawlerParse.getBaiduTiebaData(word, proxy, null, sleepTime);
return BaiduTiebaCrawlerParse.getBaiduTiebaData(word, proxy, null);
} catch (Exception e) {
e.printStackTrace();
return null;
......@@ -180,15 +211,19 @@ public class DataCrawler {
* @Title: getBaiduTiebaData
* @author hero
* @Description: 根据关键词采集指定贴吧内数据
* @param @param word
* @param @param proxy
* @param @param tiebaName
* @param @return 设定文件
* @param @param
* word
* @param @param
* proxy
* @param @param
* tiebaName
* @param @return
* 设定文件
* @return List<TiebaData> 返回类型
*/
public static List<TiebaData> getBaiduTiebaData(String word, Proxy proxy, String tiebaName,Long sleepTime){
public static List<TiebaData> getBaiduTiebaData(String word, Proxy proxy, String tiebaName) {
try {
return BaiduTiebaCrawlerParse.getBaiduTiebaData(word, proxy, tiebaName,sleepTime);
return BaiduTiebaCrawlerParse.getBaiduTiebaData(word, proxy, tiebaName);
} catch (Exception e) {
e.printStackTrace();
return null;
......@@ -199,15 +234,19 @@ public class DataCrawler {
* @Title: getLunTanData
* @author hero
* @Description: 根据关键词采集天涯论坛数据
* @param @param word
* @param @param proxy
* @param @param endTime
* @param @return 设定文件
* @param @param
* word
* @param @param
* proxy
* @param @param
* endTime
* @param @return
* 设定文件
* @return List<LunTanData> 返回类型
*/
public static List<LunTanData> getLunTanData(String word, Proxy proxy, String endTime,Long sleepTime){
public static List<LunTanData> getLunTanData(String word, Proxy proxy, String endTime) {
try {
return TianYaCrawlerParse.getLunTanData(word, proxy, endTime,sleepTime);
return TianYaCrawlerParse.getLunTanData(word, proxy, endTime);
} catch (Exception e) {
e.printStackTrace();
return null;
......@@ -218,14 +257,19 @@ public class DataCrawler {
* @Title: getDouBanData
* @author hero
* @Description: 根据关键词采集豆瓣数据
* @param @param word
* @param @param type type=topic,type=note
* @param @param proxy
* @param @param endTime
* @param @return 设定文件
* @param @param
* word
* @param @param
* type type=topic,type=note
* @param @param
* proxy
* @param @param
* endTime
* @param @return
* 设定文件
* @return List<DouBanData> 返回类型
*/
public static List<DouBanData> getDouBanData(String word, String type, Proxy proxy){
public static List<DouBanData> getDouBanData(String word, String type, Proxy proxy) {
try {
return DoubanCrawlerParse.getDoubanData(word, type, proxy);
} catch (Exception e) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment