Commit 3f8e90d7 by zhiwei

微博热搜推送添加缓存机制,避免推送重复的数据

parent a1ef24dc
...@@ -9,6 +9,7 @@ import com.mongodb.DBCursor; ...@@ -9,6 +9,7 @@ import com.mongodb.DBCursor;
import com.mongodb.DBObject; import com.mongodb.DBObject;
import com.mongodb.WriteConcern; import com.mongodb.WriteConcern;
import com.zhiwei.searchhotcrawler.bean.WeiboHotSearch; import com.zhiwei.searchhotcrawler.bean.WeiboHotSearch;
import com.zhiwei.searchhotcrawler.cache.CacheManager;
import com.zhiwei.searchhotcrawler.config.Config; import com.zhiwei.searchhotcrawler.config.Config;
import com.zhiwei.searchhotcrawler.dbtemplate.MongoDBTemplate; import com.zhiwei.searchhotcrawler.dbtemplate.MongoDBTemplate;
import com.zhiwei.zhiweiTools.tools.ZhiWeiTools; import com.zhiwei.zhiweiTools.tools.ZhiWeiTools;
...@@ -87,18 +88,17 @@ public class WeiboHotSearchDAO extends MongoDBTemplate{ ...@@ -87,18 +88,17 @@ public class WeiboHotSearchDAO extends MongoDBTemplate{
DBObject query = new BasicDBObject(); DBObject query = new BasicDBObject();
query.put("time", new BasicDBObject("$gte", date)); query.put("time", new BasicDBObject("$gte", date));
query.put("changeCount", 0); query.put("changeCount", 0);
List<String> nameList = new ArrayList<String>();
try { try {
DBCursor cur = this.getReadColl().find(query); DBCursor cur = this.getReadColl().find(query);
while(cur.hasNext()){ while(cur.hasNext()){
DBObject doc = cur.next(); DBObject doc = cur.next();
String name = doc.get("name").toString(); String name = doc.get("name").toString();
if(!nameList.contains(name)){ if(CacheManager.getCacheByKey(name)==null){
nameList.add(name); CacheManager.putCache(name, doc, 48*60*60*1000);
list.add(doc); list.add(doc);
} }
} }
nameList.clear();
cur.close(); cur.close();
} catch (Exception e) { } catch (Exception e) {
return null; return null;
......
...@@ -4,6 +4,7 @@ import java.util.concurrent.Executors; ...@@ -4,6 +4,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import com.zhiwei.searchhotcrawler.cache.CacheListener;
import com.zhiwei.searchhotcrawler.timer.SendWeiboHotSearchRun; import com.zhiwei.searchhotcrawler.timer.SendWeiboHotSearchRun;
import com.zhiwei.searchhotcrawler.timer.SendZhihuHotSearchRun; import com.zhiwei.searchhotcrawler.timer.SendZhihuHotSearchRun;
import com.zhiwei.searchhotcrawler.timer.WeiboHotSearchRun; import com.zhiwei.searchhotcrawler.timer.WeiboHotSearchRun;
...@@ -15,17 +16,18 @@ public class HotSearchRun { ...@@ -15,17 +16,18 @@ public class HotSearchRun {
private ScheduledExecutorService scheduExec; private ScheduledExecutorService scheduExec;
public HotSearchRun() { public HotSearchRun() {
this.scheduExec = Executors.newScheduledThreadPool(4); this.scheduExec = Executors.newScheduledThreadPool(1);
} }
public void showTimer() { public void showTimer() {
scheduExec.scheduleAtFixedRate(new WeiboHotSearchRun(), 1000, 60 * 1000, TimeUnit.MILLISECONDS); scheduExec.scheduleAtFixedRate(new WeiboHotSearchRun(), 1000, 60 * 1000, TimeUnit.MILLISECONDS);
scheduExec.scheduleAtFixedRate(new ZhihuHotSearchRun(), 1000, 60 * 1000 , TimeUnit.MILLISECONDS); scheduExec.scheduleAtFixedRate(new ZhihuHotSearchRun(), 1000, 60 * 1000 , TimeUnit.MILLISECONDS);
scheduExec.scheduleAtFixedRate(new SendZhihuHotSearchRun(), 1000, 60 * 60 * 1000 , TimeUnit.MILLISECONDS); scheduExec.scheduleAtFixedRate(new SendZhihuHotSearchRun(), 1000, 60 * 60 * 1000 , TimeUnit.MILLISECONDS);
scheduExec.scheduleAtFixedRate(new SendWeiboHotSearchRun(), 1000, 60 * 60 * 1000 , TimeUnit.MILLISECONDS); scheduExec.scheduleAtFixedRate(new SendWeiboHotSearchRun(), 1000, 60*60 * 1000 , TimeUnit.MILLISECONDS);
} }
public static void main(String[] args) { public static void main(String[] args) {
new HotSearchRun().showTimer(); new HotSearchRun().showTimer();
new CacheListener().startListen();
} }
} }
...@@ -16,6 +16,7 @@ import com.zhiwei.zhiweiTools.timeParse.TimeParse; ...@@ -16,6 +16,7 @@ import com.zhiwei.zhiweiTools.timeParse.TimeParse;
public class SendWeiboHotSearchRun extends Thread{ public class SendWeiboHotSearchRun extends Thread{
private WeiboHotSearchDAO weiboHotSearchDAO = new WeiboHotSearchDAO(); private WeiboHotSearchDAO weiboHotSearchDAO = new WeiboHotSearchDAO();
@Override @Override
public void run() { public void run() {
List<DBObject> list = weiboHotSearchDAO.getWeiboHotOneHour(); List<DBObject> list = weiboHotSearchDAO.getWeiboHotOneHour();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment