添加字段部分

src/main/java/com/zhiwei/manage/bean/CrawTemplate.java

 package com.zhiwei.manage.bean;
 
+import java.util.List;
+import java.util.Map;
+
 import org.springframework.data.mongodb.core.mapping.Document;
 import org.springframework.data.mongodb.core.mapping.Field;
 
+
 @Document(collection = "spyderInfo")
 public class CrawTemplate {
 
 	@Field("configName")
 	private String configName;
 
+	@Field("staticFieldList")
+	private List<Map<String,Object>> staticFieldList;
+	
 	@Field("_id")
 	private String id;
 	
+	private String pt;
+	
+	public List<Map<String, Object>> getStaticFieldList() {
+		return staticFieldList;
+	}
+
+	public void setStaticFieldList(List<Map<String, Object>> staticFieldList) {
+		this.staticFieldList = staticFieldList;
+	}
+
 	public String getId() {
 		return id;
 	}
@@ -28,9 +45,18 @@ public class CrawTemplate {
 		this.configName = configName;
 	}
 
+	public String getPt() {
+		return pt;
+	}
+
+	public void setPt(String pt) {
+		this.pt = pt;
+	}
+
 	@Override
 	public String toString() {
-		return "CrawTemplate [configName=" + configName + ", id=" + id + "]";
+		return "CrawTemplate [configName=" + configName + ", staticFieldList=" + staticFieldList + ", id=" + id
+				+ ", pt=" + pt + "]";
 	}
 
 	

src/main/java/com/zhiwei/manage/dao/CrawTemplateDao.java

@@ -10,4 +10,6 @@ public interface CrawTemplateDao {
 	
 	public String getSpIdByConfigName(String tempName);
 	
+	public CrawTemplate getPt(String sypId);
+	
 }

src/main/java/com/zhiwei/manage/dao/CrawTemplateDaoImpl.java

@@ -6,6 +6,7 @@ import javax.annotation.Resource;
 
 import org.springframework.data.mongodb.core.MongoTemplate;
 import org.springframework.data.mongodb.core.query.Criteria;
+import org.springframework.data.mongodb.core.query.CriteriaDefinition;
 import org.springframework.data.mongodb.core.query.Query;
 import org.springframework.stereotype.Component;
 
@@ -18,6 +19,7 @@ public class CrawTemplateDaoImpl implements CrawTemplateDao{
 	
 	@Override
 	public List<CrawTemplate> findAll() {
+//		mongo.find(Query.query(Criteria.where("websitetype").is("main")), CrawTemplate.class);
 		return mongo.findAll(CrawTemplate.class);
 	}
 
@@ -26,4 +28,9 @@ public class CrawTemplateDaoImpl implements CrawTemplateDao{
 		return mongo.findOne(Query.query(Criteria.where("configName").is(tempName)), CrawTemplate.class).getId();
 	}
 
+	@Override
+	public CrawTemplate getPt(String spyId) {
+		return mongo.findOne(Query.query(Criteria.where("_id").is(spyId)), CrawTemplate.class);
+	}
+
 }

src/main/java/com/zhiwei/manage/dao/DataDaoImpl.java

@@ -116,6 +116,7 @@ public class DataDaoImpl implements DataDao {
 	public long finCountByDayData(Date startTime, Date endTime, String pt, String id) {
 		String end = "";
 		Calendar cal = Calendar.getInstance();
+		cal.setTime(startTime);
 		MongoCollection<Document> col = null;
 		if (pt.equals("知乎")) {
 			if (DbDepot.dataCol != null) {

src/main/java/com/zhiwei/manage/dao/MessageDaoImpl.java

@@ -33,7 +33,7 @@ public class MessageDaoImpl implements MessageDao {
 	@Override
 	public String insert(Message message) {
 		mongo.insert(message);
-		return null;
+		return "添加成功";
 	}
 
 	@Override
@@ -160,9 +160,6 @@ public class MessageDaoImpl implements MessageDao {
 			return 0;
 		}
 		return (int) con.count(new BasicDBObject("tempName", new BasicDBObject("$regex", templateName)).append("pt",pt ));
-		// return (int)
-		// mongo.count(Query.query(Criteria.where("templateName").regex(templateName)),
-		// Message.class);
 	}
 
 	@Override

src/main/java/com/zhiwei/manage/dao/ServerDaoImpl.java

@@ -26,8 +26,6 @@ public class ServerDaoImpl implements ServerDao {
 
 	@Resource(name = "mongoTemplate")
 	private MongoTemplate mongo;
-	@Autowired
-	private DbDepot dbDepot;
 	@Override
 	public void insert(ServerBean serverBean) {
 		mongo.insert(serverBean);
@@ -57,7 +55,6 @@ public class ServerDaoImpl implements ServerDao {
 					continue;
 				}
 				if (fields[j].get(serverBean) != null)
-					// 字段名，字段值
 					update.set(fields[j].getName(), fields[j].get(serverBean));
 			} catch (Exception e) {
 				e.printStackTrace();
@@ -78,7 +75,6 @@ public class ServerDaoImpl implements ServerDao {
 	@Override
 	public void findPt(String pt) {
 		List<ServerBean> list = mongo.find(Query.query(Criteria.where("pt").is(pt)), ServerBean.class);
-//		DbDepot.dbCons = new ConcurrentHashMap<String, Object>();
 		for (ServerBean sb : list) {
 			System.out.println(sb);
 			if (sb.getDbType().equals("mongo")) {
@@ -120,7 +116,6 @@ public class ServerDaoImpl implements ServerDao {
 					continue;
 				}
 				if (fields[j].get(channel) != null)
-					// 字段名，字段值
 					update.set(fields[j].getName(), fields[j].get(channel));
 			} catch (Exception e) {
 				e.printStackTrace();

src/main/java/com/zhiwei/manage/handle/FieldThread.java

@@ -45,7 +45,6 @@ public class FieldThread extends Thread {
 					String key = it.next();
 					FieldIntegerity run = runMap.get(key);
 					Calendar cal = Calendar.getInstance();
-					
 					if (run.getNextRun().before(cal.getTime()) && !run.isStartNow()) {
 						System.out.println("字段添加一个");
 						queue.offer(run);

src/main/java/com/zhiwei/manage/handle/MainThread.java

@@ -4,8 +4,10 @@ import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Calendar;
 import java.util.Date;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
@@ -162,7 +164,7 @@ public class MainThread extends Thread {
 				// int num = 1;
 				// for (int i = 0; i < 19; i++) {
 				start.add(Calendar.DATE, -1);
-				countTest.add(Calendar.DATE, -3);
+				countTest.add(Calendar.DATE, -7);
 				try {
 					mainLock.lock();
 					int count = 0;
@@ -182,7 +184,6 @@ public class MainThread extends Thread {
 								data.setSpyderInfoId(cr.getId());
 								dataService.insert(data);
 								if (count == 0) {
-									// if (num == 1) {
 									int testCount = (int) dataService.finCountByDayData(countTest.getTime(),
 											end.getTime(), chan.getPt(), cr.getId());
 									if (testCount == 0) {
@@ -190,12 +191,11 @@ public class MainThread extends Thread {
 										m.setHandle(false);
 										m.setCreateDate(start.getTime());
 										m.setTemplateLv(1);
-										m.setTempName("未定义");
-										m.setPt("网媒");
-										m.setErrorMsg("未定义，" + cr.getConfigName() + "," + chan.getId() + "数据为0");
+										m.setTempName(cr.getConfigName());
+										m.setPt(cr.getPt());
+										m.setErrorMsg("数据为0检测：" + cr.getConfigName() + "," + chan.getId() + "数据为0");
 										msgService.insert(m);
 									}
-									// }
 								}
 								log.info(TimeUtil.yearToDay(start.getTime()) + "|" + data.getTempName() + "|入库");
 							}
@@ -217,6 +217,27 @@ public class MainThread extends Thread {
 							log.info(TimeUtil.yearToDay(start.getTime()) + "|" + chan.getPt() + "|入库");
 						}
 					}
+					Set<String> keys = allTmp.keySet();
+					for (Iterator<String> it = keys.iterator(); it.hasNext();) {
+						String key = it.next();
+						Template tmp = allTmp.get(key);
+						if (tmp.getSpyderInfoId() != null) {
+							int cwCount = 0;
+							String pt = crawService.findPt(tmp.getSpyderInfoId());
+							cwCount = (int) dataService.finCountByDayData(start.getTime(), end.getTime(), tmp.getPt(),
+									tmp.getSpyderInfoId());
+							if (cwCount < tmp.getNewsNumber()) {
+								Message m = new Message();
+								m.setHandle(false);
+								m.setCreateDate(start.getTime());
+								m.setTemplateLv(tmp.getTemplateLv());
+								m.setTempName(tmp.getTempName());
+								m.setPt(pt);
+								m.setErrorMsg("数据量检测：" + tmp.getTempName() + "的数据为小于" + tmp.getNewsNumber());
+								msgService.insert(m);
+							}
+						}
+					}
 				} catch (Exception e) {
 					e.printStackTrace();
 				} finally {

src/main/java/com/zhiwei/manage/service/CrawTemplateService.java

@@ -7,4 +7,6 @@ import com.zhiwei.manage.bean.CrawTemplate;
 public interface CrawTemplateService {
 	
 	public List<CrawTemplate> findAll();
+	
+	public String findPt(String spyId);
 }

src/main/java/com/zhiwei/manage/service/CrawTemplateServiceImpl.java

 package com.zhiwei.manage.service;
 
+import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
-import org.springframework.stereotype.Service;
 
 import com.zhiwei.manage.bean.CrawTemplate;
 import com.zhiwei.manage.dao.CrawTemplateDao;
@@ -16,7 +18,33 @@ public class CrawTemplateServiceImpl implements CrawTemplateService {
 	
 	@Override
 	public List<CrawTemplate> findAll() {
-		return crawDao.findAll();
+		List<CrawTemplate> list=crawDao.findAll();
+		for(CrawTemplate ct:list) {
+			ct.setPt(getPtByCt(ct));
+		}
+		return list;
+	}
+
+	@Override
+	public String findPt(String spyId) {
+		CrawTemplate ct=crawDao.getPt(spyId);
+		return getPtByCt(ct);
+	}
+	
+	
+	public String getPtByCt(CrawTemplate ct) {
+		String pt="";
+		List<Map<String,Object>> list=ct.getStaticFieldList();
+		for(Map<String,Object> map:list) {
+			Set<String> keys=map.keySet();
+			for(Iterator<String> it=keys.iterator();it.hasNext();) {
+				String key=it.next();
+				if(String.valueOf(map.get(key)).equals("pt")) {
+					pt=String.valueOf(map.get("value"));
+				}
+			}
+		}
+		return pt;
 	}
 
 }