Skip to content

D
datamanage
Commit 19a369e9 by shenjinzhu
Options

数据搜索优化

parent 46071e6c
Showing with 159 additions and 147 deletions
src/main/java/com/zhiwei/manage/dao/DataDao.java
...@@ -20,8 +20,8 @@ public interface DataDao { ...@@ -20,8 +20,8 @@ public interface DataDao {
public List<Data> findByMethod(Date startTime,Date endTime,String method,Object value); public List<Data> findByMethod(Date startTime,Date endTime,String method,Object value);
public List<EasyNews> findBySourceFromLibrary(Date startTime, Date endTime, String source); public int findBySourceFromLibrary(Date startTime, Date endTime, String source);
public List<EasyNews> findBySpId(Date startTime, Date endTime, String syId); public int findBySpId(Date startTime, Date endTime, String syId);
/** /**
* 根据数据来源来搜索数据 * 根据数据来源来搜索数据
* @param startTime * @param startTime
...@@ -33,7 +33,7 @@ public interface DataDao { ...@@ -33,7 +33,7 @@ public interface DataDao {
public List<EasyNews> findByAccurateType(Date startTime, Date endTime, String type); public int findByAccurateType(Date startTime, Date endTime, String type);
/** /**
* 根据可视化模板的id去搜索数据 * 根据可视化模板的id去搜索数据
* @param startTime * @param startTime
......
src/main/java/com/zhiwei/manage/dao/DataDaoImpl.java
...@@ -240,12 +240,12 @@ public class DataDaoImpl implements DataDao { ...@@ -240,12 +240,12 @@ public class DataDaoImpl implements DataDao {
} }
@Override @Override
public List<EasyNews> findBySourceFromLibrary(Date startTime, Date endTime, String source) { public int findBySourceFromLibrary(Date startTime, Date endTime, String source) {
Calendar c = Calendar.getInstance(); Calendar c = Calendar.getInstance();
c.setTime(startTime); c.setTime(startTime);
int year = c.get(Calendar.YEAR); int year = c.get(Calendar.YEAR);
int month = c.get(Calendar.MONTH) + 1; int month = c.get(Calendar.MONTH) + 1;
return mongo2.find(Query.query(Criteria.where("time").gte(startTime).lt(endTime).and("type").is(source)), return (int) mongo2.count(Query.query(Criteria.where("time").gte(startTime).lt(endTime).and("type").is(source)),
EasyNews.class, "net_media_" + year + (month>9?""+month:"0"+month)); EasyNews.class, "net_media_" + year + (month>9?""+month:"0"+month));
} }
...@@ -277,12 +277,12 @@ public class DataDaoImpl implements DataDao { ...@@ -277,12 +277,12 @@ public class DataDaoImpl implements DataDao {
} }
@Override @Override
public List<EasyNews> findByAccurateType(Date startTime, Date endTime, String type) { public int findByAccurateType(Date startTime, Date endTime, String type) {
Calendar c = Calendar.getInstance(); Calendar c = Calendar.getInstance();
c.setTime(startTime); c.setTime(startTime);
int year = c.get(Calendar.YEAR); int year = c.get(Calendar.YEAR);
int month = c.get(Calendar.MONTH) + 1; int month = c.get(Calendar.MONTH) + 1;
return mongo2.find(Query.query(Criteria.where("time").gte(startTime).lt(endTime).and("type").is(type)), return (int) mongo2.count(Query.query(Criteria.where("time").gte(startTime).lt(endTime).and("type").is(type)),
EasyNews.class, "net_media_" + year + (month>9?""+month:"0"+month)); EasyNews.class, "net_media_" + year + (month>9?""+month:"0"+month));
} }
...@@ -353,12 +353,12 @@ public class DataDaoImpl implements DataDao { ...@@ -353,12 +353,12 @@ public class DataDaoImpl implements DataDao {
} }
@Override @Override
public List<EasyNews> findBySpId(Date startTime, Date endTime, String syId) { public int findBySpId(Date startTime, Date endTime, String syId) {
Calendar c = Calendar.getInstance(); Calendar c = Calendar.getInstance();
c.setTime(startTime); c.setTime(startTime);
int year = c.get(Calendar.YEAR); int year = c.get(Calendar.YEAR);
int month = c.get(Calendar.MONTH) + 1; int month = c.get(Calendar.MONTH) + 1;
return mongo2.find(Query.query(Criteria.where("time").gte(startTime).lt(endTime).and("spyderInfoId").is(syId)), return (int) mongo2.count(Query.query(Criteria.where("time").gte(startTime).lt(endTime).and("spyderInfoId").is(syId)),
EasyNews.class, "net_media_" + year + (month>9?""+month:"0"+month)); EasyNews.class, "net_media_" + year + (month>9?""+month:"0"+month));
} }
......
src/main/java/com/zhiwei/manage/handle/Task.java
...@@ -35,46 +35,14 @@ public class Task implements Runnable { ...@@ -35,46 +35,14 @@ public class Task implements Runnable {
@Override @Override
public void run() { public void run() {
System.out.println("开始做检测");
MessageService messageService = BeanFactory.getBean(MessageService.class); MessageService messageService = BeanFactory.getBean(MessageService.class);
NewsService newsService = BeanFactory.getBean(NewsService.class); NewsService newsService = BeanFactory.getBean(NewsService.class);
// 连通性测试 // 连通性测试
if (ping != null) { if (ping != null) {
URL url; if (MainThread.mainMap.get(ping.getTemplateId()) != null) {
try { URL url;
url = new URL(ping.getPingUrlOne());
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setConnectTimeout(1000 * 10);
int code = conn.getResponseCode();
if (code != 200) {
Template tp = MainThread.mainMap.get(ping.getTemplateId());
Message msg = new Message();
msg.setCreateDate(new Date());
msg.setErrorMsg(code == 0 ? ping.getPingUrlOne() + "访问失败,错误代码" + code : "访问超时");
msg.setHandle(false);
msg.setTemplateCreator(tp.getCreator());
msg.setTempName(ping.getTempName());
msg.setTemplateLv(tp.getTemplateLv());
msg.setErrorType("url");
messageService.insert(msg);
}
} catch (Exception e) {
Template tp = MainThread.mainMap.get(ping.getTemplateId());
Message msg = new Message();
msg.setCreateDate(new Date());
msg.setErrorMsg("访问出错,url不可访问");
msg.setHandle(false);
msg.setTemplateCreator(tp.getCreator());
msg.setTempName(ping.getTempName());
msg.setTemplateLv(1);
msg.setErrorType("url");
msg.setPt(ping.getPt());
messageService.insert(msg);
}
if (null!=ping.getPingUrlTwo()&& !ping.getPingUrlTwo().equals(""))
try { try {
url = new URL(ping.getPingUrlTwo()); url = new URL(ping.getPingUrlOne());
HttpURLConnection conn = (HttpURLConnection) url.openConnection(); HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setConnectTimeout(1000 * 10); conn.setConnectTimeout(1000 * 10);
int code = conn.getResponseCode(); int code = conn.getResponseCode();
...@@ -88,7 +56,6 @@ public class Task implements Runnable { ...@@ -88,7 +56,6 @@ public class Task implements Runnable {
msg.setTempName(ping.getTempName()); msg.setTempName(ping.getTempName());
msg.setTemplateLv(tp.getTemplateLv()); msg.setTemplateLv(tp.getTemplateLv());
msg.setErrorType("url"); msg.setErrorType("url");
msg.setPt(ping.getPt());
messageService.insert(msg); messageService.insert(msg);
} }
} catch (Exception e) { } catch (Exception e) {
...@@ -99,127 +66,165 @@ public class Task implements Runnable { ...@@ -99,127 +66,165 @@ public class Task implements Runnable {
msg.setHandle(false); msg.setHandle(false);
msg.setTemplateCreator(tp.getCreator()); msg.setTemplateCreator(tp.getCreator());
msg.setTempName(ping.getTempName()); msg.setTempName(ping.getTempName());
msg.setPt(ping.getPt()); msg.setTemplateLv(1);
msg.setErrorType("url"); msg.setErrorType("url");
msg.setTemplateLv(tp.getTemplateLv()); msg.setPt(ping.getPt());
messageService.insert(msg); messageService.insert(msg);
} }
if (null != ping.getPingUrlTwo() && !ping.getPingUrlTwo().equals(""))
try {
url = new URL(ping.getPingUrlTwo());
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setConnectTimeout(1000 * 10);
int code = conn.getResponseCode();
if (code != 200) {
Template tp = MainThread.mainMap.get(ping.getTemplateId());
Message msg = new Message();
msg.setCreateDate(new Date());
msg.setErrorMsg(code == 0 ? ping.getPingUrlOne() + "访问失败,错误代码" + code : "访问超时");
msg.setHandle(false);
msg.setTemplateCreator(tp.getCreator());
msg.setTempName(ping.getTempName());
msg.setTemplateLv(tp.getTemplateLv());
msg.setErrorType("url");
msg.setPt(ping.getPt());
messageService.insert(msg);
}
} catch (Exception e) {
Template tp = MainThread.mainMap.get(ping.getTemplateId());
Message msg = new Message();
msg.setCreateDate(new Date());
msg.setErrorMsg("访问出错,url不可访问");
msg.setHandle(false);
msg.setTemplateCreator(tp.getCreator());
msg.setTempName(ping.getTempName());
msg.setPt(ping.getPt());
msg.setErrorType("url");
msg.setTemplateLv(tp.getTemplateLv());
messageService.insert(msg);
}
}
} }
if (field != null) { if (field != null) {
List<Map<String, Object>> list = newsService.findNews(field.getHost(), field.getPort(), field.getdBName(), field.getCollection(), if (MainThread.mainMap.get(field.getTemplateId()) != null) {
field.getTestCount(), field.getSpyderInfoId(), field.getPt()); List<Map<String, Object>> list = newsService.findNews(field.getHost(), field.getPort(),
System.out.println("field检测数据长度"+list.size()); field.getdBName(), field.getCollection(), field.getTestCount(), field.getSpyderInfoId(),
if(list.size()<field.getTestCount()){ field.getPt());
Template tp = MainThread.mainMap.get(field.getTemplateId()); System.out.println("field检测数据长度" + list.size());
Message msg = new Message(); if (list.size() < field.getTestCount()) {
msg.setCreateDate(new Date());
msg.setErrorMsg("字段检测发现库内的数据小于预设检测数量："+field.getTestCount()+"条,实际数据量为"+list.size());
msg.setHandle(false);
msg.setPt(field.getPt());
msg.setTemplateCreator(tp.getCreator());
msg.setTemplateLv(tp.getTemplateLv());
msg.setErrorType("field");
msg.setTempName(field.getTempName());
messageService.insert(msg);
}
String[] fields = field.getField().split(",");
int count = list.size();
for (String fid : fields) {
String str="";
int num = count;
for (Map<String, Object> map : list) {
if (map.get(fid) == null) {
num--;
str=String.valueOf(map.get("_id"));
}else if(String.valueOf(map.get(fid)).length()<1){
num--;
str=String.valueOf(map.get("_id"));
}
}
if (num / 1.0 / count < field.getFieldIntegrityRate()/100.0) {
Template tp = MainThread.mainMap.get(field.getTemplateId()); Template tp = MainThread.mainMap.get(field.getTemplateId());
Message msg = new Message(); Message msg = new Message();
msg.setCreateDate(new Date()); msg.setCreateDate(new Date());
msg.setErrorMsg(fid + "字段存在缺失,完整率"+num / 1.0 / count+"其中一条："+str); msg.setErrorMsg("字段检测发现库内的数据小于预设检测数量：" + field.getTestCount() + "条,实际数据量为" + list.size());
msg.setHandle(false); msg.setHandle(false);
msg.setPt(field.getPt()); msg.setPt(field.getPt());
msg.setErrorType("field");
msg.setTemplateCreator(tp.getCreator()); msg.setTemplateCreator(tp.getCreator());
msg.setTemplateLv(tp.getTemplateLv()); msg.setTemplateLv(tp.getTemplateLv());
msg.setErrorType("field");
msg.setTempName(field.getTempName()); msg.setTempName(field.getTempName());
messageService.insert(msg); messageService.insert(msg);
} }
String[] fields = field.getField().split(",");
int count = list.size();
for (String fid : fields) {
String str = "";
int num = count;
for (Map<String, Object> map : list) {
if (map.get(fid) == null) {
num--;
str = String.valueOf(map.get("_id"));
} else if (String.valueOf(map.get(fid)).length() < 1) {
num--;
str = String.valueOf(map.get("_id"));
}
}
if (num / 1.0 / count < field.getFieldIntegrityRate() / 100.0) {
Template tp = MainThread.mainMap.get(field.getTemplateId());
Message msg = new Message();
msg.setCreateDate(new Date());
msg.setErrorMsg(fid + "字段存在缺失,完整率" + num / 1.0 / count + "其中一条：" + str);
msg.setHandle(false);
msg.setPt(field.getPt());
msg.setErrorType("field");
msg.setTemplateCreator(tp.getCreator());
msg.setTemplateLv(tp.getTemplateLv());
msg.setTempName(field.getTempName());
messageService.insert(msg);
}
}
} }
} }
if (delayed != null) { if (delayed != null) {
List<Map<String, Object>> list = newsService.findNews(delayed.getHost(), delayed.getPort(), delayed.getdBName(), if (MainThread.mainMap.get(delayed.getTemplateId()) != null) {
delayed.getCollection(), delayed.getTestCount(), delayed.getSpyderInfoId(), delayed.getPt()); List<Map<String, Object>> list = newsService.findNews(delayed.getHost(), delayed.getPort(),
System.out.println("delay检测数据长度"+list.size()); delayed.getdBName(), delayed.getCollection(), delayed.getTestCount(), delayed.getSpyderInfoId(),
long sum = 0; delayed.getPt());
if(list.size()<delayed.getTestCount()){ System.out.println("delay检测数据长度" + list.size());
Template tp = MainThread.mainMap.get(delayed.getTemplateId()); long sum = 0;
Message msg = new Message(); if (list.size() < delayed.getTestCount()) {
msg.setCreateDate(new Date()); Template tp = MainThread.mainMap.get(delayed.getTemplateId());
msg.setErrorMsg("延时检测发现库内的数据小于预设检测数量："+delayed.getTestCount()+"条,实际数据量为"+list.size()); Message msg = new Message();
msg.setHandle(false); msg.setCreateDate(new Date());
msg.setPt(delayed.getPt()); msg.setErrorMsg("延时检测发现库内的数据小于预设检测数量：" + delayed.getTestCount() + "条,实际数据量为" + list.size());
msg.setErrorType("delayed"); msg.setHandle(false);
msg.setTemplateCreator(tp.getCreator()); msg.setPt(delayed.getPt());
msg.setTemplateLv(tp.getTemplateLv()); msg.setErrorType("delayed");
msg.setTempName(delayed.getTempName()); msg.setTemplateCreator(tp.getCreator());
messageService.insert(msg); msg.setTemplateLv(tp.getTemplateLv());
} msg.setTempName(delayed.getTempName());
double delay=0; messageService.insert(msg);
List<Double> dNum=new ArrayList<Double>();
for (Map<String, Object> map : list) {
Date time = null;
Date savetime = null;
if (map.get(delayed.getTime()) != null) {
time = (Date) map.get(delayed.getTime());
} }
if (map.get(delayed.getSavetime()) != null) { double delay = 0;
Date d=new Date((Long)map.get(delayed.getSavetime())); List<Double> dNum = new ArrayList<Double>();
savetime = d; for (Map<String, Object> map : list) {
Date time = null;
Date savetime = null;
if (map.get(delayed.getTime()) != null) {
time = (Date) map.get(delayed.getTime());
}
if (map.get(delayed.getSavetime()) != null) {
Date d = new Date((Long) map.get(delayed.getSavetime()));
savetime = d;
}
if (time != null && savetime != null) {
delay = savetime.getTime() - time.getTime();
dNum.add(delay);
sum += delay;
} else {
Template tp = MainThread.mainMap.get(delayed.getTemplateId());
Message msg = new Message();
msg.setCreateDate(new Date());
msg.setErrorMsg("延时检测发现时间字段缺失");
msg.setHandle(false);
msg.setPt(delayed.getPt());
msg.setErrorType("delayed");
msg.setTemplateCreator(tp.getCreator());
msg.setTemplateLv(tp.getTemplateLv());
msg.setTempName(delayed.getTempName());
messageService.insert(msg);
}
} }
if (time != null && savetime != null) { Collections.sort(dNum);
delay= savetime.getTime() - time.getTime(); System.out.println(sum);
dNum.add(delay); int cutSize = (int) (dNum.size() * 0.8 / 1);
sum +=delay; for (int i = cutSize; i < dNum.size(); i++) {
} else { sum -= dNum.get(i);
}
System.out.println(sum);
if (sum / 1.0 / 1000 / 60 / cutSize > delayed.getNewsDelayed()) {
Template tp = MainThread.mainMap.get(delayed.getTemplateId()); Template tp = MainThread.mainMap.get(delayed.getTemplateId());
Message msg = new Message(); Message msg = new Message();
msg.setCreateDate(new Date()); msg.setCreateDate(new Date());
msg.setErrorMsg("延时检测发现时间字段缺失"); msg.setErrorMsg("延时过高，延时检测为" + sum / 1.0 / 1000 / 60 / cutSize + "分");
msg.setHandle(false); msg.setHandle(false);
msg.setPt(delayed.getPt());
msg.setErrorType("delayed"); msg.setErrorType("delayed");
msg.setTemplateCreator(tp.getCreator()); msg.setTemplateCreator(tp.getCreator());
msg.setTemplateLv(tp.getTemplateLv()); msg.setTemplateLv(tp.getTemplateLv());
msg.setPt(delayed.getPt());
msg.setTempName(delayed.getTempName()); msg.setTempName(delayed.getTempName());
messageService.insert(msg); messageService.insert(msg);
} }
} }
Collections.sort(dNum);
System.out.println(sum);
int cutSize=(int) (dNum.size()*0.8/1);
for(int i=cutSize;i<dNum.size();i++) {
sum-=dNum.get(i);
}
System.out.println(sum);
if (sum / 1.0 / 1000 / 60/cutSize > delayed.getNewsDelayed()) {
Template tp = MainThread.mainMap.get(delayed.getTemplateId());
Message msg = new Message();
msg.setCreateDate(new Date());
msg.setErrorMsg("延时过高，延时检测为" + sum / 1.0 / 1000 / 60/cutSize + "分");
msg.setHandle(false);
msg.setErrorType("delayed");
msg.setTemplateCreator(tp.getCreator());
msg.setTemplateLv(tp.getTemplateLv());
msg.setPt(delayed.getPt());
msg.setTempName(delayed.getTempName());
messageService.insert(msg);
}
} }
} }
} }
src/main/java/com/zhiwei/manage/service/DataService.java
...@@ -18,15 +18,15 @@ public interface DataService { ...@@ -18,15 +18,15 @@ public interface DataService {
public List<Data> findByMethod(Date startTime, Date endTime,String method,Object value); public List<Data> findByMethod(Date startTime, Date endTime,String method,Object value);
public List<EasyNews> findBySourceFromLibrary(Date startTime, Date endTime, String source); public int findBySourceFromLibrary(Date startTime, Date endTime, String source);
public List<EasyNews> findByAccurateSource(String startTime, String endTime, String source); public List<EasyNews> findByAccurateSource(String startTime, String endTime, String source);
public List<EasyNews> findByAccurateSpId(String startTime, String endTime, String SpId); public List<EasyNews> findByAccurateSpId(String startTime, String endTime, String SpId);
public List<EasyNews> findByAccurateType(Date startTime, Date endTime, String type); public int findByAccurateType(Date startTime, Date endTime, String type);
public List<EasyNews> findByTempName(Date startTime, Date endTime, String tempName); public int findByTempName(Date startTime, Date endTime, String tempName);
public long findWeiboCount(Date start, Date end); public long findWeiboCount(Date start, Date end);
......
src/main/java/com/zhiwei/manage/service/DataServiceImpl.java
...@@ -59,7 +59,7 @@ public class DataServiceImpl implements DataService{ ...@@ -59,7 +59,7 @@ public class DataServiceImpl implements DataService{
} }
@Override @Override
public List<EasyNews> findBySourceFromLibrary(Date startTime, Date endTime, String source) { public int findBySourceFromLibrary(Date startTime, Date endTime, String source) {
return dataDao.findBySourceFromLibrary(startTime, endTime, source); return dataDao.findBySourceFromLibrary(startTime, endTime, source);
} }
...@@ -74,7 +74,7 @@ public class DataServiceImpl implements DataService{ ...@@ -74,7 +74,7 @@ public class DataServiceImpl implements DataService{
} }
@Override @Override
public List<EasyNews> findByAccurateType(Date startTime, Date endTime, String type) { public int findByAccurateType(Date startTime, Date endTime, String type) {
return dataDao.findByAccurateType(startTime, endTime, type); return dataDao.findByAccurateType(startTime, endTime, type);
} }
...@@ -94,7 +94,7 @@ public class DataServiceImpl implements DataService{ ...@@ -94,7 +94,7 @@ public class DataServiceImpl implements DataService{
} }
@Override @Override
public List<EasyNews> findByTempName(Date startTime, Date endTime, String tempName) { public int findByTempName(Date startTime, Date endTime, String tempName) {
String spId=crd.getSpIdByConfigName(tempName); String spId=crd.getSpIdByConfigName(tempName);
return dataDao.findBySpId(startTime, endTime, spId); return dataDao.findBySpId(startTime, endTime, spId);
} }
......
src/main/java/com/zhiwei/manage/servlet/DataController.java
...@@ -89,25 +89,32 @@ public class DataController { ...@@ -89,25 +89,32 @@ public class DataController {
startd.setTimeInMillis(start); startd.setTimeInMillis(start);
endTime.setTimeInMillis(start); endTime.setTimeInMillis(start);
} }
Calendar ds=Calendar.getInstance();
ds.setTimeInMillis(end);
System.out.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(startd.getTime()));
System.out.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(ds.getTime()));
String source = ""; String source = "";
List<Data> list = new ArrayList<Data>(); List<Data> list = new ArrayList<Data>();
startd.add(Calendar.HOUR, -8); startd.add(Calendar.HOUR, -8);
while (endTime.getTimeInMillis() < end) { while (endTime.getTime().before(ds.getTime())) {
endTime.add(Calendar.HOUR, 1); endTime.add(Calendar.HOUR, 1);
List<EasyNews> newsList = new ArrayList<>(); int count=0;
if(method.equals("type")){ if(method.equals("type")){
source=(String) map.get("type"); source=(String) map.get("type");
newsList=dataService.findByAccurateType(startd.getTime(), endTime.getTime(), source); count=dataService.findByAccurateType(startd.getTime(), endTime.getTime(), source);
}else{ }else{
if (method.equals("source")) { if (method.equals("source")) {
source = (String) map.get("source"); source = (String) map.get("source");
newsList = dataService.findBySourceFromLibrary(startd.getTime(), endTime.getTime(), source); count = dataService.findBySourceFromLibrary(startd.getTime(), endTime.getTime(), source);
} else { } else {
source = (String) map.get("tempName"); source = (String) map.get("tempName");
newsList = dataService.findByTempName(startd.getTime(), endTime.getTime(), source); count = dataService.findByTempName(startd.getTime(), endTime.getTime(), source);
} }
} }
Data data = Change.changeNewsToDate(newsList, startd.getTime()); Data data = new Data();
data.setCount(count);
data.setSource(source);
data.setTime(startd.getTime());
list.add(data); list.add(data);
startd.add(Calendar.HOUR, 1); startd.add(Calendar.HOUR, 1);
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment