提交数据

src/main/java/com/zhiwei/crawler/dao/WordsDao.java

@@ -33,8 +33,7 @@ private static Logger logger = LoggerFactory.getLogger(WordsDao.class);
 		try {
 			BlockingQueue<String> list = new LinkedBlockingQueue<String>();
 			DBObject query = new BasicDBObject();
-			query.put("company", "美赞臣");
-			DBCursor cur = this.getReadColl().find(query);
+			DBCursor cur = this.getReadColl().find();
 			while(cur.hasNext()){
 				DBObject doc = cur.next();
 				list.add(doc.get("word").toString());
@@ -46,9 +45,6 @@ private static Logger logger = LoggerFactory.getLogger(WordsDao.class);
 		}
 	}
 	
-	
-	
-	
 	/**
 	 * @Title: getWordList 
 	 * @author hero 
@@ -59,7 +55,13 @@ private static Logger logger = LoggerFactory.getLogger(WordsDao.class);
 	 */
 	public BlockingQueue<String> getWordList(String company){
 		DBObject query = new BasicDBObject();
+		
+		if(company.contains("-")) {
+			company = company.replace("-", "");
+			query.put("company", new BasicDBObject("$ne", company));
+		}else {
 			query.put("company", company);
+		}
 		
 		try {
 			BlockingQueue<String> list = new LinkedBlockingQueue<String>();

src/main/java/com/zhiwei/crawler/run/MainRun.java

@@ -12,7 +12,7 @@ public class MainRun {
     }     
 
     public void showTimer() {
-        scheduExec.scheduleAtFixedRate(new SoubaoCrawlerRun(), 1000, 60 * 60 * 1000, TimeUnit.MILLISECONDS);
+        scheduExec.scheduleAtFixedRate(new SoubaoCrawlerRun(), 1, 60*3, TimeUnit.MINUTES);
     }
     
     public static void main(String[] args) {

src/main/java/com/zhiwei/crawler/run/SoubaoCrawlerRun.java

@@ -37,7 +37,12 @@ public class SoubaoCrawlerRun implements Runnable{
 		logger.info("采集开始.....");
 		long s = System.currentTimeMillis();
 		int thread = 5;
-		BlockingQueue<String> wordesQueue = wordsDao.getAllWordList();
+		//美赞臣数据优先
+		BlockingQueue<String> wordesQueue = wordsDao.getWordList("美赞臣");
+		//其他组数据采集关键词
+		BlockingQueue<String> otherWordQueue = wordsDao.getWordList("-美赞臣");
+		wordesQueue.addAll(otherWordQueue);
+		
 		SouBaoCrawlerThread[] souBaoCrawlerThread = new SouBaoCrawlerThread[thread];
         ExecutorService service = Executors.newFixedThreadPool(2);
         for (int i = 0; i < thread; i++) {

src/main/java/com/zhiwei/crawler/soubao/SouBaoCrawlerThread.java

@@ -35,7 +35,7 @@ public class SouBaoCrawlerThread extends Thread{
 				String word = wordsQueue.take();
 				Proxy proxy = ProxyClientUtil.getProxy();
 				/***开始采集**/
-				logger.info("开始采集：：：{}搜报网关键词", word);
+				logger.info("开始采集：{}搜报网关键词,目前未采集的关键词为:{}", word, wordsQueue.size());
 				long s = System.currentTimeMillis();
 				Crawler.start(1, word, proxy);
 				long e = System.currentTimeMillis();