Skip to content

D
datamanage
Commit 3f7225fc by shenjinzhu
Options

更新检测部分，修改显示bug

parent fba731d6
Showing with 42 additions and 25 deletions
src/main/java/com/zhiwei/manage/dao/DataDaoImpl.java
...@@ -86,7 +86,7 @@ public class DataDaoImpl implements DataDao { ...@@ -86,7 +86,7 @@ public class DataDaoImpl implements DataDao {
Criteria criteria = new Criteria(); Criteria criteria = new Criteria();
criteria.and("source").is(source); criteria.and("source").is(source);
criteria.and("time").gte(startTime).lt(endTime); criteria.and("time").gte(startTime).lt(endTime);
return mongo2.find(Query.query(criteria), EasyNews.class, "net_media_" + year + (month>10?""+month:"0"+month)); return mongo2.find(Query.query(criteria), EasyNews.class, "net_media_" + year + (month>9?""+month:"0"+month));
} }
@Override @Override
...@@ -97,7 +97,7 @@ public class DataDaoImpl implements DataDao { ...@@ -97,7 +97,7 @@ public class DataDaoImpl implements DataDao {
Criteria criteria = new Criteria(); Criteria criteria = new Criteria();
criteria.and("spyderInfoId").is(SpId); criteria.and("spyderInfoId").is(SpId);
criteria.and("time").gt(startTime).lt(endTime); criteria.and("time").gt(startTime).lt(endTime);
return mongo2.find(Query.query(criteria), EasyNews.class, "net_media_" + year + (month>10?""+month:"0"+month)); return mongo2.find(Query.query(criteria), EasyNews.class, "net_media_" + year + (month>9?""+month:"0"+month));
} }
@Override @Override
...@@ -108,7 +108,7 @@ public class DataDaoImpl implements DataDao { ...@@ -108,7 +108,7 @@ public class DataDaoImpl implements DataDao {
int month = c.get(Calendar.MONTH) + 1; int month = c.get(Calendar.MONTH) + 1;
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm"); SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm");
return weibo.count(Query.query(Criteria.where("time").gte(sdf.format(startTime)).lt(sdf.format(endTime))), return weibo.count(Query.query(Criteria.where("time").gte(sdf.format(startTime)).lt(sdf.format(endTime))),
Weibo.class, "status_" + year + (month>10?""+month:"0"+month)); Weibo.class, "status_" + year + (month>9?""+month:"0"+month));
} }
@Override @Override
...@@ -149,7 +149,7 @@ public class DataDaoImpl implements DataDao { ...@@ -149,7 +149,7 @@ public class DataDaoImpl implements DataDao {
ServerBean.class); ServerBean.class);
if (ser.isTable()) { if (ser.isTable()) {
int month=cal.get(Calendar.MONTH)+1; int month=cal.get(Calendar.MONTH)+1;
end = cal.get(Calendar.YEAR) +""+ (month>10?""+month:"0"+month); end = cal.get(Calendar.YEAR) +""+ (month>9?""+month:"0"+month);
System.out.println(end); System.out.println(end);
} }
MongoClient mongoClient = new MongoClient(ser.getDbHost(), ser.getDbPort()); MongoClient mongoClient = new MongoClient(ser.getDbHost(), ser.getDbPort());
...@@ -166,7 +166,7 @@ public class DataDaoImpl implements DataDao { ...@@ -166,7 +166,7 @@ public class DataDaoImpl implements DataDao {
ServerBean.class); ServerBean.class);
if (ser.isTable()) { if (ser.isTable()) {
int month=cal.get(Calendar.MONTH)+1; int month=cal.get(Calendar.MONTH)+1;
end = cal.get(Calendar.YEAR) +""+ (month>10?""+month:"0"+month); end = cal.get(Calendar.YEAR) +""+ (month>9?""+month:"0"+month);
} }
MongoClient mongoClient = new MongoClient(ser.getDbHost(), ser.getDbPort()); MongoClient mongoClient = new MongoClient(ser.getDbHost(), ser.getDbPort());
// 连接到数据库 // 连接到数据库
...@@ -247,7 +247,7 @@ public class DataDaoImpl implements DataDao { ...@@ -247,7 +247,7 @@ public class DataDaoImpl implements DataDao {
int year = c.get(Calendar.YEAR); int year = c.get(Calendar.YEAR);
int month = c.get(Calendar.MONTH) + 1; int month = c.get(Calendar.MONTH) + 1;
return mongo2.find(Query.query(Criteria.where("time").gte(startTime).lt(endTime).and("type").is(source)), return mongo2.find(Query.query(Criteria.where("time").gte(startTime).lt(endTime).and("type").is(source)),
EasyNews.class, "net_media_" + year + (month>10?""+month:"0"+month)); EasyNews.class, "net_media_" + year + (month>9?""+month:"0"+month));
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
...@@ -257,10 +257,10 @@ public class DataDaoImpl implements DataDao { ...@@ -257,10 +257,10 @@ public class DataDaoImpl implements DataDao {
int year = c.get(Calendar.YEAR); int year = c.get(Calendar.YEAR);
int month = c.get(Calendar.MONTH) + 1; int month = c.get(Calendar.MONTH) + 1;
try { try {
return mongo2.getCollection("net_media_" + year + (month>10?""+month:"0"+month)).distinct("source", return mongo2.getCollection("net_media_" + year + (month>9?""+month:"0"+month)).distinct("source",
new BasicDBObject("source", new BasicDBObject("$regex", param))); new BasicDBObject("source", new BasicDBObject("$regex", param)));
} catch (Exception e) { } catch (Exception e) {
return mongo2.getCollection("net_media_" + year + (month>10?""+month:"0"+month)).distinct("source", return mongo2.getCollection("net_media_" + year + (month>9?""+month:"0"+month)).distinct("source",
new BasicDBObject("source", new BasicDBObject("$regex", param))); new BasicDBObject("source", new BasicDBObject("$regex", param)));
} }
} }
...@@ -284,7 +284,7 @@ public class DataDaoImpl implements DataDao { ...@@ -284,7 +284,7 @@ public class DataDaoImpl implements DataDao {
int year = c.get(Calendar.YEAR); int year = c.get(Calendar.YEAR);
int month = c.get(Calendar.MONTH) + 1; int month = c.get(Calendar.MONTH) + 1;
return mongo2.find(Query.query(Criteria.where("time").gte(startTime).lt(endTime).and("type").is(type)), return mongo2.find(Query.query(Criteria.where("time").gte(startTime).lt(endTime).and("type").is(type)),
EasyNews.class, "net_media_" + year + (month>10?""+month:"0"+month)); EasyNews.class, "net_media_" + year + (month>9?""+month:"0"+month));
} }
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
...@@ -297,7 +297,7 @@ public class DataDaoImpl implements DataDao { ...@@ -297,7 +297,7 @@ public class DataDaoImpl implements DataDao {
DBObject obj = new BasicDBObject(); DBObject obj = new BasicDBObject();
obj.put("pt", new BasicDBObject("$ne", "微信")); obj.put("pt", new BasicDBObject("$ne", "微信"));
obj.put("type", new BasicDBObject("$regex", param)); obj.put("type", new BasicDBObject("$regex", param));
return mongo2.getCollection("net_media_" + year + (month>10?""+month:"0"+month)).distinct("type", obj); return mongo2.getCollection("net_media_" + year + (month>9?""+month:"0"+month)).distinct("type", obj);
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
...@@ -360,7 +360,7 @@ public class DataDaoImpl implements DataDao { ...@@ -360,7 +360,7 @@ public class DataDaoImpl implements DataDao {
int year = c.get(Calendar.YEAR); int year = c.get(Calendar.YEAR);
int month = c.get(Calendar.MONTH) + 1; int month = c.get(Calendar.MONTH) + 1;
return mongo2.find(Query.query(Criteria.where("time").gte(startTime).lt(endTime).and("spyderInfoId").is(syId)), return mongo2.find(Query.query(Criteria.where("time").gte(startTime).lt(endTime).and("spyderInfoId").is(syId)),
EasyNews.class, "net_media_" + year + (month>10?""+month:"0"+month)); EasyNews.class, "net_media_" + year + (month>9?""+month:"0"+month));
} }
} }
src/main/java/com/zhiwei/manage/dao/NewsDaoImpl.java
...@@ -52,13 +52,13 @@ public class NewsDaoImpl implements NewsDao { ...@@ -52,13 +52,13 @@ public class NewsDaoImpl implements NewsDao {
query.addCriteria(Criteria.where("source").is(source)); query.addCriteria(Criteria.where("source").is(source));
query.with(new Sort(new Order(Direction.DESC, "time"))); query.with(new Sort(new Order(Direction.DESC, "time")));
query.limit(delayedCount); query.limit(delayedCount);
return mongo.find(query, News.class, "net_media_" + year + (month>10?""+month:"0"+month)); return mongo.find(query, News.class, "net_media_" + year + (month>9?""+month:"0"+month));
} else { } else {
Query query = new Query(); Query query = new Query();
query.addCriteria(Criteria.where("type").is(type)); query.addCriteria(Criteria.where("type").is(type));
query.with(new Sort(new Order(Direction.DESC, "time"))); query.with(new Sort(new Order(Direction.DESC, "time")));
query.limit(delayedCount); query.limit(delayedCount);
return mongo.find(query, News.class, "net_media_" + year + (month>10?""+month:"0"+month)); return mongo.find(query, News.class, "net_media_" + year + (month>9?""+month:"0"+month));
} }
} }
...@@ -72,13 +72,13 @@ public class NewsDaoImpl implements NewsDao { ...@@ -72,13 +72,13 @@ public class NewsDaoImpl implements NewsDao {
query.addCriteria(Criteria.where("source").is(source)); query.addCriteria(Criteria.where("source").is(source));
query.with(new Sort(new Order(Direction.DESC, "time"))); query.with(new Sort(new Order(Direction.DESC, "time")));
query.limit(filedCount); query.limit(filedCount);
return mongo.find(query, News.class, "net_media_" + year + (month>10?""+month:"0"+month)); return mongo.find(query, News.class, "net_media_" + year + (month>9?""+month:"0"+month));
} else { } else {
Query query = new Query(); Query query = new Query();
query.addCriteria(Criteria.where("type").is(type)); query.addCriteria(Criteria.where("type").is(type));
query.with(new Sort(new Order(Direction.DESC, "time"))); query.with(new Sort(new Order(Direction.DESC, "time")));
query.limit(filedCount); query.limit(filedCount);
return mongo.find(query, News.class, "net_media_" + year + (month>10?""+month:"0"+month)); return mongo.find(query, News.class, "net_media_" + year + (month>9?""+month:"0"+month));
} }
} }
...@@ -90,7 +90,7 @@ public class NewsDaoImpl implements NewsDao { ...@@ -90,7 +90,7 @@ public class NewsDaoImpl implements NewsDao {
Criteria criteria = new Criteria(); Criteria criteria = new Criteria();
criteria.and("spyderInfoId").is(spyId); criteria.and("spyderInfoId").is(spyId);
criteria.and("time").gte(start).lt(end); criteria.and("time").gte(start).lt(end);
return mongo.count(Query.query(criteria), News.class, "net_media_" + year + (month>10?""+month:"0"+month)); return mongo.count(Query.query(criteria), News.class, "net_media_" + year + (month>9?""+month:"0"+month));
} }
@Override @Override
...@@ -109,7 +109,7 @@ public class NewsDaoImpl implements NewsDao { ...@@ -109,7 +109,7 @@ public class NewsDaoImpl implements NewsDao {
Criteria criteria = new Criteria(); Criteria criteria = new Criteria();
criteria.and("source").is(source); criteria.and("source").is(source);
criteria.and("time").gte(start).lt(end); criteria.and("time").gte(start).lt(end);
return (int) mongo.count(Query.query(criteria), News.class, "net_media_" + year + (month>10?""+month:"0"+month)); return (int) mongo.count(Query.query(criteria), News.class, "net_media_" + year + (month>9?""+month:"0"+month));
} }
@Override @Override
...@@ -118,7 +118,7 @@ public class NewsDaoImpl implements NewsDao { ...@@ -118,7 +118,7 @@ public class NewsDaoImpl implements NewsDao {
int month = c.get(Calendar.MONTH) + 1; int month = c.get(Calendar.MONTH) + 1;
int year = c.get(Calendar.YEAR); int year = c.get(Calendar.YEAR);
return mongo.find(Query.query(Criteria.where("time").gte(start).lt(end)), EasyNews.class, return mongo.find(Query.query(Criteria.where("time").gte(start).lt(end)), EasyNews.class,
"net_media_" + year + (month>10?""+month:"0"+month)); "net_media_" + year + (month>9?""+month:"0"+month));
} }
/** /**
...@@ -128,6 +128,10 @@ public class NewsDaoImpl implements NewsDao { ...@@ -128,6 +128,10 @@ public class NewsDaoImpl implements NewsDao {
@Override @Override
public List<Map<String, Object>> findNews(String host, int port, String dbNamel, String collection, int count, public List<Map<String, Object>> findNews(String host, int port, String dbNamel, String collection, int count,
String id, String pt) { String id, String pt) {
Calendar cal=Calendar.getInstance();
cal.add(Calendar.DATE, -1);
int month=cal.get(Calendar.MONTH)+1;
String end=cal.get(Calendar.YEAR)+""+(month>9?month:"0"+month);
List<Map<String, Object>> list = new ArrayList<Map<String, Object>>(); List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
MongoCollection<Document> col = null; MongoCollection<Document> col = null;
if (DbDepot.newsCol != null) { if (DbDepot.newsCol != null) {
...@@ -138,14 +142,14 @@ public class NewsDaoImpl implements NewsDao { ...@@ -138,14 +142,14 @@ public class NewsDaoImpl implements NewsDao {
MongoClient mongoClient = new MongoClient(host, port); MongoClient mongoClient = new MongoClient(host, port);
// 连接到数据库 // 连接到数据库
MongoDatabase mongo = mongoClient.getDatabase(dbNamel); MongoDatabase mongo = mongoClient.getDatabase(dbNamel);
col = mongo.getCollection(collection); col = mongo.getCollection(collection+end);
DbDepot.newsCol.put(pt, col); DbDepot.newsCol.put(pt, col);
} }
} else { } else {
DbDepot.newsCol = new ConcurrentHashMap<String, Object>(); DbDepot.newsCol = new ConcurrentHashMap<String, Object>();
MongoClient mongoClient = new MongoClient(host, port); MongoClient mongoClient = new MongoClient(host, port);
MongoDatabase mongo = mongoClient.getDatabase(dbNamel); MongoDatabase mongo = mongoClient.getDatabase(dbNamel);
col = mongo.getCollection(collection); col = mongo.getCollection(collection+end);
DbDepot.newsCol.put(pt, col); DbDepot.newsCol.put(pt, col);
} }
if (id != null & id.length() > 5) { if (id != null & id.length() > 5) {
...@@ -167,5 +171,10 @@ public class NewsDaoImpl implements NewsDao { ...@@ -167,5 +171,10 @@ public class NewsDaoImpl implements NewsDao {
} }
return list; return list;
} }
public static void main(String[] args) {
MongoClient mongoClient = new MongoClient("115.236.59.91", 27017);
MongoDatabase mongo = mongoClient.getDatabase("mediaspider");
MongoCollection<Document> col = mongo.getCollection("");
}
} }
src/main/java/com/zhiwei/manage/handle/DbDepot.java
...@@ -44,7 +44,7 @@ public class DbDepot { ...@@ -44,7 +44,7 @@ public class DbDepot {
if (sb.isTable()) { if (sb.isTable()) {
if (sb.getTableKind().equals("month")) { if (sb.getTableKind().equals("month")) {
int month=cal.get(Calendar.MONTH) + 1; int month=cal.get(Calendar.MONTH) + 1;
end = cal.get(Calendar.YEAR) + (month>10?""+month:"0"+month); end = cal.get(Calendar.YEAR) + (month>9?""+month:"0"+month);
} else { } else {
} }
...@@ -79,7 +79,7 @@ public class DbDepot { ...@@ -79,7 +79,7 @@ public class DbDepot {
if(sb.isTable()) { if(sb.isTable()) {
if(sb.getTableKind().equals("month")) { if(sb.getTableKind().equals("month")) {
int month=cal.get(Calendar.MONTH) + 1; int month=cal.get(Calendar.MONTH) + 1;
end = cal.get(Calendar.YEAR) + (month>10?""+month:"0"+month); end = cal.get(Calendar.YEAR) + (month>9?""+month:"0"+month);
}else { }else {
} }
......
src/main/java/com/zhiwei/manage/handle/Task.java
...@@ -108,7 +108,7 @@ public class Task implements Runnable { ...@@ -108,7 +108,7 @@ public class Task implements Runnable {
Template tp = MainThread.mainMap.get(field.getTemplateId()); Template tp = MainThread.mainMap.get(field.getTemplateId());
Message msg = new Message(); Message msg = new Message();
msg.setCreateDate(new Date()); msg.setCreateDate(new Date());
msg.setErrorMsg("字段检测时发现库内数据小于"+field.getTestCount()+"数据量为"+list); msg.setErrorMsg("字段检测发现库内的数据小于预设检测数量："+delayed.getTestCount()+"条,实际数据量为"+list.size());
msg.setHandle(false); msg.setHandle(false);
msg.setPt(field.getPt()); msg.setPt(field.getPt());
msg.setTemplateCreator(tp.getCreator()); msg.setTemplateCreator(tp.getCreator());
...@@ -154,7 +154,7 @@ public class Task implements Runnable { ...@@ -154,7 +154,7 @@ public class Task implements Runnable {
Template tp = MainThread.mainMap.get(delayed.getTemplateId()); Template tp = MainThread.mainMap.get(delayed.getTemplateId());
Message msg = new Message(); Message msg = new Message();
msg.setCreateDate(new Date()); msg.setCreateDate(new Date());
msg.setErrorMsg("延时检测发现库内的数据小于"+delayed.getTestCount()+"条"+"数据量为"+list); msg.setErrorMsg("延时检测发现库内的数据小于预设检测数量："+delayed.getTestCount()+"条,实际数据量为"+list.size());
msg.setHandle(false); msg.setHandle(false);
msg.setPt(delayed.getPt()); msg.setPt(delayed.getPt());
msg.setTemplateCreator(tp.getCreator()); msg.setTemplateCreator(tp.getCreator());
...@@ -210,5 +210,5 @@ public class Task implements Runnable { ...@@ -210,5 +210,5 @@ public class Task implements Runnable {
} }
} }
} }
} }
src/main/java/com/zhiwei/manage/service/TemplateServiceImpl.java
...@@ -78,6 +78,14 @@ public class TemplateServiceImpl implements TemplateService { ...@@ -78,6 +78,14 @@ public class TemplateServiceImpl implements TemplateService {
try { try {
int count=tempDao.findByUserCount(templateName,pt); int count=tempDao.findByUserCount(templateName,pt);
List<Template> data=tempDao.findByTmpName(templateName, pageNo, pageSize,pt); List<Template> data=tempDao.findByTmpName(templateName, pageNo, pageSize,pt);
for(Template tp:data){
if(MainThread.allTmp.get(tp.getTemplateId())==null){
MainThread.allTmp.put(tp.getTemplateId(), tp);
}
if(MainThread.mainMap.get(tp.getTemplateId())!=null){
tp.setOpen(true);
}
}
PageEty page=new PageEty(); PageEty page=new PageEty();
if(data.size()==0){ if(data.size()==0){
page.setData(new ArrayList<>()); page.setData(new ArrayList<>());
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment