Skip to content

S
searchhotcrawler
Commit 6f72ce80 by zhiwei
Options

修复按月分库失败bug及添加索引

parent db96247a
Showing with 82 additions and 45 deletions
src/main/java/com/zhiwei/searchhotcrawler/crawler/WeiboHuatiCrawler.java
...@@ -108,7 +108,7 @@ public class WeiboHuatiCrawler { ...@@ -108,7 +108,7 @@ public class WeiboHuatiCrawler {
id = data.getString("page_id"); id = data.getString("page_id");
score = data.getString("score"); score = data.getString("score");
desc1 = data.getString("desc1"); desc1 = data.getString("desc1");
fensi = desc1.replaceAll("影响力.*", ""); fensi = desc1.replaceAll(".*影响力|粉丝", "").trim();
url = data.getString("link"); url = data.getString("link");
WeiboTopic topic = new WeiboTopic(url, topicName, toprank, score, fensi, type); WeiboTopic topic = new WeiboTopic(url, topicName, toprank, score, fensi, type);
......
src/main/java/com/zhiwei/searchhotcrawler/dao/HotSearchListDAO.java
...@@ -28,6 +28,25 @@ public class HotSearchListDAO extends MongoDBTemplate{ ...@@ -28,6 +28,25 @@ public class HotSearchListDAO extends MongoDBTemplate{
String month = time.substring(5,7); String month = time.substring(5,7);
String collName = Config.searchCollName + year + "_" + month; String collName = Config.searchCollName + year + "_" + month;
super.setCollName(collName); super.setCollName(collName);
DBObject countIndexDoc = new BasicDBObject();
countIndexDoc.put("count", -1);
DBObject timeIndexDoc = new BasicDBObject();
timeIndexDoc.put("time", -1);
DBObject rankIndexDoc = new BasicDBObject();
rankIndexDoc.put("rank", -1);
DBObject nameIndexDoc = new BasicDBObject();
nameIndexDoc.put("name", -1);
DBObject typeIndexDoc = new BasicDBObject();
typeIndexDoc.put("type", -1);
try {
super.getReadColl().createIndex(countIndexDoc, new BasicDBObject("name", "count_desc"));
super.getReadColl().createIndex(timeIndexDoc, new BasicDBObject("name", "time_desc"));
super.getReadColl().createIndex(rankIndexDoc, new BasicDBObject("name", "rank_desc"));
super.getReadColl().createIndex(nameIndexDoc, new BasicDBObject("name", "name_desc"));
super.getReadColl().createIndex(typeIndexDoc, new BasicDBObject("name", "type_desc"));
} catch (Exception e) {
e.printStackTrace();
}
} }
/** /**
......
src/main/java/com/zhiwei/searchhotcrawler/dao/WeiboTopicDAO.java
...@@ -7,6 +7,7 @@ import java.util.List; ...@@ -7,6 +7,7 @@ import java.util.List;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import com.mongodb.BasicDBObject;
import com.mongodb.DBObject; import com.mongodb.DBObject;
import com.zhiwei.searchhotcrawler.config.Config; import com.zhiwei.searchhotcrawler.config.Config;
import com.zhiwei.searchhotcrawler.dbtemplate.MongoDBTemplate; import com.zhiwei.searchhotcrawler.dbtemplate.MongoDBTemplate;
...@@ -23,6 +24,26 @@ public class WeiboTopicDAO extends MongoDBTemplate{ ...@@ -23,6 +24,26 @@ public class WeiboTopicDAO extends MongoDBTemplate{
String month = time.substring(5,7); String month = time.substring(5,7);
String collName = Config.topicCollName + year + "_" + month; String collName = Config.topicCollName + year + "_" + month;
super.setCollName(collName); super.setCollName(collName);
DBObject countIndexDoc = new BasicDBObject();
countIndexDoc.put("score_num", -1);
DBObject timeIndexDoc = new BasicDBObject();
timeIndexDoc.put("time", -1);
DBObject rankIndexDoc = new BasicDBObject();
rankIndexDoc.put("rank", -1);
DBObject nameIndexDoc = new BasicDBObject();
nameIndexDoc.put("name", -1);
DBObject typeIndexDoc = new BasicDBObject();
typeIndexDoc.put("type", -1);
try {
super.getReadColl().createIndex(countIndexDoc, new BasicDBObject("name", "score_desc"));
super.getReadColl().createIndex(timeIndexDoc, new BasicDBObject("name", "time_desc"));
super.getReadColl().createIndex(rankIndexDoc, new BasicDBObject("name", "rank_desc"));
super.getReadColl().createIndex(nameIndexDoc, new BasicDBObject("name", "name_desc"));
super.getReadColl().createIndex(typeIndexDoc, new BasicDBObject("name", "type_desc"));
} catch (Exception e) {
e.printStackTrace();
}
} }
/** /**
......
src/main/java/com/zhiwei/searchhotcrawler/test/HotSearchListTest.java
...@@ -28,15 +28,15 @@ public class HotSearchListTest{ ...@@ -28,15 +28,15 @@ public class HotSearchListTest{
ServerAddress address = new ServerAddress(Config.mongoIp, Config.mongoPort); ServerAddress address = new ServerAddress(Config.mongoIp, Config.mongoPort);
Mongo mongo = new MongoClient(address, Arrays.asList(credential)); Mongo mongo = new MongoClient(address, Arrays.asList(credential));
DB db = mongo.getDB("NetWork"); DB db = mongo.getDB("hot_search_list");
DBCollection coll = db.getCollection("weibo_hotsearch2018_10"); DBCollection coll = db.getCollection("hot_search_list2019_09");
MongoCredential credentialNew = MongoCredential.createCredential("datapush", "admin", "4d8ce5c42073c".toCharArray()); MongoCredential credentialNew = MongoCredential.createCredential("datapush", "admin", "4d8ce5c42073c".toCharArray());
ServerAddress addressNew = new ServerAddress(Config.mongoIp, Config.mongoPort); ServerAddress addressNew = new ServerAddress(Config.mongoIp, Config.mongoPort);
Mongo mongoNew = new MongoClient(address, Arrays.asList(credentialNew)); Mongo mongoNew = new MongoClient(address, Arrays.asList(credentialNew));
DB dbNew = mongoNew.getDB("hot_search_list"); DB dbNew = mongoNew.getDB("hot_search_list");
Map<String,String> timLine = TimeParse.getTimeMap("2018-02-01 00:00:00", "2019-04-30 23:59:59", "MM", 1); Map<String,String> timLine = TimeParse.getTimeMap("2019-10-02 00:00:00", "2019-10-09 23:59:59", "dd", 1);
timLine.forEach((start, end) ->{ timLine.forEach((start, end) ->{
...@@ -70,37 +70,34 @@ public class HotSearchListTest{ ...@@ -70,37 +70,34 @@ public class HotSearchListTest{
e.printStackTrace(); e.printStackTrace();
} }
DBObject query = new BasicDBObject(new BasicDBObject("time",
new BasicDBObject("$gte",startDate).append("$lte", endDate)));
// DBObject query = new BasicDBObject(new BasicDBObject("time", System.out.println(query);
// new BasicDBObject("$gte",startDate).append("$lte", endDate))); int i = 0;
// DBCursor cur = coll.find(query); DBCursor cur = coll.find(query).skip(i);
// System.out.println(query +"======="+ cur.count()); System.out.println(query +"======="+ cur.count());
// List<DBObject> dataList = new ArrayList<>(); List<DBObject> dataList = new ArrayList<>();
// int i = 0; while(cur.hasNext()) {
// while(cur.hasNext()) { DBObject doc = cur.next();
// DBObject doc = cur.next(); try {
// DBObject zhihu = new BasicDBObject(); System.out.println(i+"====");
// zhihu.put("_id", doc.get("_id")); collNew.save(doc);
// zhihu.put("name", doc.get("name")); i++;
// zhihu.put("url", doc.get("url")); // coll.remove(doc);
// zhihu.put("count", doc.get("count")); } catch (Exception e2) {
// zhihu.put("hot", doc.get("hot")); e2.printStackTrace();
// zhihu.put("day", doc.get("day")); }
// zhihu.put("time", doc.get("time")); dataList.add(doc);
// zhihu.put("changeCount", doc.get("changeCount")); }
// zhihu.put("rank", doc.get("rank")); System.out.println(collName +"数据量大小" +dataList.size());
// zhihu.put("type", HotSearchType.微博热搜.name()); cur.close();
//
// collNew.save(zhihu);
// dataList.add(zhihu);
// }
// System.out.println(collName +"数据量大小" +dataList.size());
// cur.close();
// if(!dataList.isEmpty()) { // if(!dataList.isEmpty()) {
// try {
// collNew.insert(dataList); // collNew.insert(dataList);
// } catch (Exception e) {
// e.printStackTrace();
// }
// } // }
}); });
mongo.close(); mongo.close();
} }
......
src/main/java/com/zhiwei/searchhotcrawler/timer/BaiduHotSearchRun.java
...@@ -19,7 +19,6 @@ import com.zhiwei.tools.tools.ZhiWeiTools; ...@@ -19,7 +19,6 @@ import com.zhiwei.tools.tools.ZhiWeiTools;
public class BaiduHotSearchRun extends Thread{ public class BaiduHotSearchRun extends Thread{
private static Logger logger = LoggerFactory.getLogger(BaiduHotSearchRun.class); private static Logger logger = LoggerFactory.getLogger(BaiduHotSearchRun.class);
private HotSearchListDAO hotSearchDAO = new HotSearchListDAO();
@Override @Override
public void run() { public void run() {
...@@ -38,6 +37,7 @@ public class BaiduHotSearchRun extends Thread{ ...@@ -38,6 +37,7 @@ public class BaiduHotSearchRun extends Thread{
private void getHotList() { private void getHotList() {
logger.info("百度风云榜采集开始........"); logger.info("百度风云榜采集开始........");
HotSearchListDAO hotSearchDAO = new HotSearchListDAO();
List<HotSearchList> list = BaiDuHotSearchCrawler.baiduHotSearch(); List<HotSearchList> list = BaiDuHotSearchCrawler.baiduHotSearch();
logger.info("{}, 此轮百度风云榜采集到的数据量为:{}", new Date(), Integer.valueOf(list != null ? list.size() : 0)); logger.info("{}, 此轮百度风云榜采集到的数据量为:{}", new Date(), Integer.valueOf(list != null ? list.size() : 0));
List<DBObject> saveDataList = new ArrayList<>(); List<DBObject> saveDataList = new ArrayList<>();
......
src/main/java/com/zhiwei/searchhotcrawler/timer/DouyinHotSearchRun.java
...@@ -18,8 +18,6 @@ import com.zhiwei.tools.tools.ZhiWeiTools; ...@@ -18,8 +18,6 @@ import com.zhiwei.tools.tools.ZhiWeiTools;
public class DouyinHotSearchRun extends Thread{ public class DouyinHotSearchRun extends Thread{
private static Logger logger = LoggerFactory.getLogger(DouyinHotSearchRun.class); private static Logger logger = LoggerFactory.getLogger(DouyinHotSearchRun.class);
private HotSearchListDAO hotSearchDAO = new HotSearchListDAO();
@Override @Override
public void run() { public void run() {
...@@ -42,6 +40,7 @@ public class DouyinHotSearchRun extends Thread{ ...@@ -42,6 +40,7 @@ public class DouyinHotSearchRun extends Thread{
*/ */
private void getHotList() { private void getHotList() {
logger.info("抖音热搜榜采集开始........"); logger.info("抖音热搜榜采集开始........");
HotSearchListDAO hotSearchDAO = new HotSearchListDAO();
List<HotSearchList> list = DouyinHotSearchCrawler.getMobileDouyinHotList(); List<HotSearchList> list = DouyinHotSearchCrawler.getMobileDouyinHotList();
logger.info("{}, 抖音热搜榜此轮采集到的数据量为:{}", new Date(), Integer.valueOf(list != null ? list.size() : 0)); logger.info("{}, 抖音热搜榜此轮采集到的数据量为:{}", new Date(), Integer.valueOf(list != null ? list.size() : 0));
List<DBObject> data = new ArrayList<>(); List<DBObject> data = new ArrayList<>();
......
src/main/java/com/zhiwei/searchhotcrawler/timer/SougoHotSearchRun.java
...@@ -17,8 +17,6 @@ import com.zhiwei.tools.tools.ZhiWeiTools; ...@@ -17,8 +17,6 @@ import com.zhiwei.tools.tools.ZhiWeiTools;
public class SougoHotSearchRun extends Thread { public class SougoHotSearchRun extends Thread {
private static Logger logger = LoggerFactory.getLogger(SougoHotSearchRun.class); private static Logger logger = LoggerFactory.getLogger(SougoHotSearchRun.class);
private HotSearchListDAO hotSearchDAO = new HotSearchListDAO();
@Override @Override
public void run() { public void run() {
...@@ -36,6 +34,7 @@ public class SougoHotSearchRun extends Thread { ...@@ -36,6 +34,7 @@ public class SougoHotSearchRun extends Thread {
private void getHotList() { private void getHotList() {
HotSearchListDAO hotSearchDAO = new HotSearchListDAO();
logger.info("搜狗微信采集开始........"); logger.info("搜狗微信采集开始........");
List<HotSearchList> list = SougoHotSearchCrawler.sougoHotSearch(); List<HotSearchList> list = SougoHotSearchCrawler.sougoHotSearch();
logger.info("{}, 此轮采集到的数据量为:{}", new Date(), Integer.valueOf(list != null ? list.size() : 0)); logger.info("{}, 此轮采集到的数据量为:{}", new Date(), Integer.valueOf(list != null ? list.size() : 0));
......
src/main/java/com/zhiwei/searchhotcrawler/timer/WeiboHotSearchRun.java
...@@ -18,7 +18,6 @@ import com.zhiwei.tools.tools.ZhiWeiTools; ...@@ -18,7 +18,6 @@ import com.zhiwei.tools.tools.ZhiWeiTools;
public class WeiboHotSearchRun extends Thread{ public class WeiboHotSearchRun extends Thread{
private static Logger logger = LoggerFactory.getLogger(WeiboHotSearchRun.class); private static Logger logger = LoggerFactory.getLogger(WeiboHotSearchRun.class);
private HotSearchListDAO weiboHotSearchDAO = new HotSearchListDAO();
@Override @Override
public void run() { public void run() {
...@@ -37,6 +36,7 @@ public class WeiboHotSearchRun extends Thread{ ...@@ -37,6 +36,7 @@ public class WeiboHotSearchRun extends Thread{
private void getHotList() { private void getHotList() {
logger.info("微博话题采集开始........"); logger.info("微博话题采集开始........");
HotSearchListDAO weiboHotSearchDAO = new HotSearchListDAO();
// List<HotSearchList> list = WeiboHotSearchCrawler.weiboHotSearch(); // List<HotSearchList> list = WeiboHotSearchCrawler.weiboHotSearch();
List<HotSearchList> list = WeiboHotSearchCrawler.weiboHotSearchByPhone(); List<HotSearchList> list = WeiboHotSearchCrawler.weiboHotSearchByPhone();
logger.info("{}, 微博此轮采集到的数据量为:{}", new Date(), Integer.valueOf(list != null ? list.size() : 0)); logger.info("{}, 微博此轮采集到的数据量为:{}", new Date(), Integer.valueOf(list != null ? list.size() : 0));
......
src/main/java/com/zhiwei/searchhotcrawler/timer/WeiboTopicRun.java
...@@ -18,7 +18,6 @@ import com.zhiwei.tools.tools.ZhiWeiTools; ...@@ -18,7 +18,6 @@ import com.zhiwei.tools.tools.ZhiWeiTools;
public class WeiboTopicRun extends Thread{ public class WeiboTopicRun extends Thread{
private static Logger logger = LoggerFactory.getLogger(WeiboTopicRun.class); private static Logger logger = LoggerFactory.getLogger(WeiboTopicRun.class);
private WeiboTopicDAO weiboTopicDAO = new WeiboTopicDAO();
@Override @Override
public void run() { public void run() {
...@@ -36,11 +35,13 @@ public class WeiboTopicRun extends Thread{ ...@@ -36,11 +35,13 @@ public class WeiboTopicRun extends Thread{
private void getTopicList() { private void getTopicList() {
WeiboTopicDAO weiboTopicDAO = new WeiboTopicDAO();
logger.info("微博超话采集开始........"); logger.info("微博超话采集开始........");
List<WeiboTopic> list = WeiboHuatiCrawler.startCrawler(); List<WeiboTopic> list = WeiboHuatiCrawler.startCrawler();
logger.info("{}, 微博超话此轮采集到的数据量为:{}", new Date(), Integer.valueOf(list != null ? list.size() : 0)); logger.info("{}, 微博超话此轮采集到的数据量为:{}", new Date(), Integer.valueOf(list != null ? list.size() : 0));
List<DBObject> data = new ArrayList<>(); List<DBObject> data = new ArrayList<>();
for(WeiboTopic topic : list){ for(WeiboTopic topic : list){
System.out.println("topic::::"+topic);
DBObject doc = new BasicDBObject(); DBObject doc = new BasicDBObject();
doc.put("_id", topic.getId()); doc.put("_id", topic.getId());
doc.put("name", topic.getTopicName()); doc.put("name", topic.getTopicName());
......
src/main/java/com/zhiwei/searchhotcrawler/timer/ZhihuHotSearchRun.java
...@@ -17,7 +17,6 @@ import com.zhiwei.tools.tools.ZhiWeiTools; ...@@ -17,7 +17,6 @@ import com.zhiwei.tools.tools.ZhiWeiTools;
public class ZhihuHotSearchRun extends Thread{ public class ZhihuHotSearchRun extends Thread{
private static Logger logger = LoggerFactory.getLogger(ZhihuHotSearchRun.class); private static Logger logger = LoggerFactory.getLogger(ZhihuHotSearchRun.class);
private HotSearchListDAO hotSearchDAO = new HotSearchListDAO();
@Override @Override
public void run() { public void run() {
...@@ -36,6 +35,8 @@ public class ZhihuHotSearchRun extends Thread{ ...@@ -36,6 +35,8 @@ public class ZhihuHotSearchRun extends Thread{
private void getHotList() { private void getHotList() {
HotSearchListDAO hotSearchDAO = new HotSearchListDAO();
logger.info("知乎话题采集开始...,当前线程名字:{}", Thread.currentThread().getName()); logger.info("知乎话题采集开始...,当前线程名字:{}", Thread.currentThread().getName());
List<HotSearchList> list = ZhihuHotSearchCrawler.getZhihuHotList(); List<HotSearchList> list = ZhihuHotSearchCrawler.getZhihuHotList();
List<HotSearchList> mobilelist = ZhihuHotSearchCrawler.getMobileZhihuHotList(); List<HotSearchList> mobilelist = ZhihuHotSearchCrawler.getMobileZhihuHotList();
......
src/main/resources/proxyip.properties
registry=zookeeper://192.168.0.203:2182?backup=192.168.0.104:2182,192.168.0.105:2182 #registry=zookeeper://192.168.0.203:2182?backup=192.168.0.104:2182,192.168.0.105:2182
group=hangzhou #group=hangzhou
######################################################## ########################################################
#registry=zookeeper://192.168.0.36:2181 registry=zookeeper://192.168.0.36:2181
#group=local group=local
\ No newline at end of file \ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment