Skip to content

D
datamanage
Commit 187b5ec5 by shenjinzhu
Options

更新，时更新

parent b496c9ce
Showing with 94 additions and 62 deletions
src/main/java/com/zhiwei/manage/dao/DataDaoImpl.java
package com.zhiwei.manage.dao;
import java.lang.reflect.Field;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
......@@ -15,6 +16,7 @@ import org.bson.conversions.Bson;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;
import org.springframework.stereotype.Component;
import com.mongodb.BasicDBObject;
......@@ -26,6 +28,7 @@ import com.mongodb.Mongo;
import com.mongodb.MongoClient;
import com.mongodb.MongoCredential;
import com.mongodb.ServerAddress;
import com.mongodb.WriteResult;
import com.mongodb.client.FindIterable;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoCursor;
......@@ -77,7 +80,36 @@ public class DataDaoImpl implements DataDao {
@Override
public void insert(Data data) {
mongo.insert(data, "Data");
Data d = null;
if (data != null) {
d = mongo.findOne(Query.query(Criteria.where("time").is(data.getTime())
.andOperator(Criteria.where("tempName").is(data.getTempName()))), Data.class);
} else {
d = mongo.findOne(Query.query(
Criteria.where("time").is(data.getTime()).andOperator(Criteria.where("pt").is(data.getPt()))),
Data.class);
}
if (d == null) {
mongo.insert(data, "Data");
}else {
d.setCount(data.getCount() + d.getCount());
Update update = new Update();
Field[] fields = d.getClass().getDeclaredFields();
for (int j = 0; j < fields.length; j++) {
fields[j].setAccessible(true);
try {
if (fields[j].getName().equals("dataId")) {
continue;
}
if (fields[j].get(d) != null)
update.set(fields[j].getName(), fields[j].get(d));
} catch (Exception e) {
e.printStackTrace();
}
}
mongo.updateFirst(Query.query(Criteria.where("_id").is(d.getDataId())), update,
Data.class);
}
}
@Override
......@@ -180,7 +212,7 @@ public class DataDaoImpl implements DataDao {
System.out.println(end);
}
MongoClient mongoClient = null;
if (ser.getDbUsername() != null&&ser.getDbUsername().length()>2) {
if (ser.getDbUsername() != null && ser.getDbUsername().length() > 2) {
ServerAddress sa = new ServerAddress(ser.getDbHost(), ser.getDbPort());
List<MongoCredential> mongoCredentialList = new ArrayList<MongoCredential>();
mongoCredentialList.add(MongoCredential.createMongoCRCredential(ser.getDbUsername(),
......
src/main/java/com/zhiwei/manage/handle/MainThread.java
......@@ -46,6 +46,7 @@ public class MainThread extends Thread {
/**
* 加入执行map
*
* @param template
*/
public static boolean pushMap(String templateId) {
......@@ -147,77 +148,76 @@ public class MainThread extends Thread {
ServerServiceImpl serverService = BeanFactory.getBean(ServerServiceImpl.class);
MessageServiceImpl msgService = BeanFactory.getBean(MessageServiceImpl.class);
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
SimpleDateFormat sdf2 = new SimpleDateFormat("yyyy-MM-dd HH");
while (true) {
Calendar countTest = Calendar.getInstance();
String today = sdf.format(new Date());
Calendar start = Calendar.getInstance();
Calendar end = Calendar.getInstance();
// start.set(Calendar.HOUR_OF_DAY, 0);
// end.set(Calendar.HOUR_OF_DAY, 0);
if (start.get(Calendar.HOUR_OF_DAY) < 1) {
try {
countTest.setTime(sdf.parse(sdf.format(countTest.getTime())));
start.setTime(sdf.parse(sdf.format(countTest.getTime())));
end.setTime(sdf.parse(sdf.format(countTest.getTime())));
} catch (ParseException e1) {
e1.printStackTrace();
try {
start.setTime(sdf2.parse(sdf2.format(start.getTime())));
if (start.get(Calendar.HOUR_OF_DAY) > 1) {
start.add(Calendar.HOUR_OF_DAY, -1);
} else {
start.setTime(sdf.parse(today));
}
start.add(Calendar.DATE, -1);
countTest.add(Calendar.DATE, -7);
try {
mainLock.lock();
int count = 0;
List<Channel> channel = serverService.findDisChannel();
for (Channel chan : channel) {
end.setTime(sdf2.parse(sdf2.format(end.getTime())));
} catch (ParseException e2) {
e2.printStackTrace();
}
try {
mainLock.lock();
int count = 0;
List<Channel> channel = serverService.findDisChannel();
for (Channel chan : channel) {
try {
count = (int) dataService.finCountByDayData(start.getTime(), end.getTime(), chan.getPt());
Data data = new Data();
data.setCount(count);
data.setPt(chan.getPt());
data.setTime(start.getTime());
dataService.insert(data);
log.info(TimeUtil.yearToDay(start.getTime()) + "|" + chan.getPt() + "|入库");
} catch (Exception e) {
log.error(chan.getPt() + "入库出错，错误信息{}", e);
}
}
Set<String> keys = allTmp.keySet();
for (Iterator<String> it = keys.iterator(); it.hasNext();) {
String key = it.next();
Template tmp = allTmp.get(key);
if (tmp.getSpyderInfoId() != null) {
int cwCount = 0;
String pt = crawService.findPt(tmp.getSpyderInfoId());
cwCount = (int) dataService.finCountByDayData(start.getTime(), end.getTime(), tmp);
try {
count = (int) dataService.finCountByDayData(start.getTime(), end.getTime(), chan.getPt());
Data data = new Data();
data.setCount(count);
data.setPt(chan.getPt());
data.setCount(cwCount);
data.setTime(start.getTime());
data.setTempName(tmp.getTempName());
dataService.insert(data);
log.info(TimeUtil.yearToDay(start.getTime()) + "|" + chan.getPt() + "|入库");
log.info(TimeUtil.yearToDay(start.getTime()) + "|" + tmp.getTempName() + "|入库");
} catch (Exception e) {
log.error(chan.getPt()+"入库出错，错误信息{}",e);
log.error(tmp.getTempName() + "入库出错，错误信息{}", e);
}
}
Set<String> keys = allTmp.keySet();
for (Iterator<String> it = keys.iterator(); it.hasNext();) {
String key = it.next();
Template tmp = allTmp.get(key);
if (tmp.getSpyderInfoId() != null) {
int cwCount = 0;
String pt = crawService.findPt(tmp.getSpyderInfoId());
cwCount = (int) dataService.finCountByDayData(start.getTime(), end.getTime(), tmp);
try {
Data data = new Data();
data.setCount(cwCount);
data.setTime(start.getTime());
data.setTempName(tmp.getTempName());
dataService.insert(data);
log.info(TimeUtil.yearToDay(start.getTime()) + "|" + tmp.getTempName() + "|入库");
} catch (Exception e) {
log.error(tmp.getTempName()+"入库出错，错误信息{}",e);
}
if (cwCount < tmp.getNewsNumber()) {
Message m = new Message();
m.setHandle(false);
m.setCreateDate(start.getTime());
m.setTemplateLv(tmp.getTemplateLv());
m.setTempName(tmp.getTempName());
m.setPt(pt);
m.setErrorMsg("数据量检测：" + tmp.getTempName() + "的数据为小于" + tmp.getNewsNumber());
msgService.insert(m);
}
if (cwCount < tmp.getNewsNumber()) {
Message m = new Message();
m.setHandle(false);
m.setCreateDate(start.getTime());
m.setTemplateLv(tmp.getTemplateLv());
m.setTempName(tmp.getTempName());
m.setErrorType("count");
m.setPt(pt);
m.setErrorMsg("数据量检测：" + tmp.getTempName() + "的数据为小于" + tmp.getNewsNumber());
msgService.insert(m);
}
}
} catch (Exception e) {
e.printStackTrace();
} finally {
mainLock.unlock();
}
Tools.sleep(1000 * 60 * 60);
} catch (Exception e) {
e.printStackTrace();
} finally {
mainLock.unlock();
}
Tools.sleep(1000 * 30);
Tools.sleep(1000 * 60 * 60);
}
}
......
src/main/java/com/zhiwei/manage/handle/Task.java
......@@ -37,7 +37,7 @@ public class Task implements Runnable {
public void run() {
MessageService messageService = BeanFactory.getBean(MessageService.class);
NewsService newsService = BeanFactory.getBean(NewsService.class);
// 连通性测试
// 连通性测试
if (ping != null) {
if (MainThread.mainMap.get(ping.getTemplateId()) != null) {
URL url;
......@@ -119,7 +119,7 @@ public class Task implements Runnable {
msg.setPt(field.getPt());
msg.setTemplateCreator(tp.getCreator());
msg.setTemplateLv(tp.getTemplateLv());
msg.setErrorType("field");
msg.setErrorType("count");
msg.setTempName(field.getTempName());
messageService.insert(msg);
}
......@@ -166,7 +166,7 @@ public class Task implements Runnable {
msg.setErrorMsg("延时检测发现库内的数据小于预设检测数量：" + delayed.getTestCount() + "条,实际数据量为" + list.size());
msg.setHandle(false);
msg.setPt(delayed.getPt());
msg.setErrorType("delayed");
msg.setErrorType("count");
msg.setTemplateCreator(tp.getCreator());
msg.setTemplateLv(tp.getTemplateLv());
msg.setTempName(delayed.getTempName());
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment