Skip to content

D
datamanage
Commit 19a369e9 by shenjinzhu
Options

数据搜索优化

parent 46071e6c
Showing with 159 additions and 147 deletions
src/main/java/com/zhiwei/manage/dao/DataDao.java
......@@ -20,8 +20,8 @@ public interface DataDao {
public List<Data> findByMethod(Date startTime,Date endTime,String method,Object value);
public List<EasyNews> findBySourceFromLibrary(Date startTime, Date endTime, String source);
public List<EasyNews> findBySpId(Date startTime, Date endTime, String syId);
public int findBySourceFromLibrary(Date startTime, Date endTime, String source);
public int findBySpId(Date startTime, Date endTime, String syId);
/**
* 根据数据来源来搜索数据
* @param startTime
......@@ -33,7 +33,7 @@ public interface DataDao {
public List<EasyNews> findByAccurateType(Date startTime, Date endTime, String type);
public int findByAccurateType(Date startTime, Date endTime, String type);
/**
* 根据可视化模板的id去搜索数据
* @param startTime
......
src/main/java/com/zhiwei/manage/dao/DataDaoImpl.java
......@@ -240,12 +240,12 @@ public class DataDaoImpl implements DataDao {
}
@Override
public List<EasyNews> findBySourceFromLibrary(Date startTime, Date endTime, String source) {
public int findBySourceFromLibrary(Date startTime, Date endTime, String source) {
Calendar c = Calendar.getInstance();
c.setTime(startTime);
int year = c.get(Calendar.YEAR);
int month = c.get(Calendar.MONTH) + 1;
return mongo2.find(Query.query(Criteria.where("time").gte(startTime).lt(endTime).and("type").is(source)),
return (int) mongo2.count(Query.query(Criteria.where("time").gte(startTime).lt(endTime).and("type").is(source)),
EasyNews.class, "net_media_" + year + (month>9?""+month:"0"+month));
}
......@@ -277,12 +277,12 @@ public class DataDaoImpl implements DataDao {
}
@Override
public List<EasyNews> findByAccurateType(Date startTime, Date endTime, String type) {
public int findByAccurateType(Date startTime, Date endTime, String type) {
Calendar c = Calendar.getInstance();
c.setTime(startTime);
int year = c.get(Calendar.YEAR);
int month = c.get(Calendar.MONTH) + 1;
return mongo2.find(Query.query(Criteria.where("time").gte(startTime).lt(endTime).and("type").is(type)),
return (int) mongo2.count(Query.query(Criteria.where("time").gte(startTime).lt(endTime).and("type").is(type)),
EasyNews.class, "net_media_" + year + (month>9?""+month:"0"+month));
}
......@@ -353,12 +353,12 @@ public class DataDaoImpl implements DataDao {
}
@Override
public List<EasyNews> findBySpId(Date startTime, Date endTime, String syId) {
public int findBySpId(Date startTime, Date endTime, String syId) {
Calendar c = Calendar.getInstance();
c.setTime(startTime);
int year = c.get(Calendar.YEAR);
int month = c.get(Calendar.MONTH) + 1;
return mongo2.find(Query.query(Criteria.where("time").gte(startTime).lt(endTime).and("spyderInfoId").is(syId)),
return (int) mongo2.count(Query.query(Criteria.where("time").gte(startTime).lt(endTime).and("spyderInfoId").is(syId)),
EasyNews.class, "net_media_" + year + (month>9?""+month:"0"+month));
}
......
src/main/java/com/zhiwei/manage/handle/Task.java
......@@ -35,46 +35,14 @@ public class Task implements Runnable {
@Override
public void run() {
System.out.println("开始做检测");
MessageService messageService = BeanFactory.getBean(MessageService.class);
NewsService newsService = BeanFactory.getBean(NewsService.class);
// 连通性测试
if (ping != null) {
URL url;
try {
url = new URL(ping.getPingUrlOne());
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setConnectTimeout(1000 * 10);
int code = conn.getResponseCode();
if (code != 200) {
Template tp = MainThread.mainMap.get(ping.getTemplateId());
Message msg = new Message();
msg.setCreateDate(new Date());
msg.setErrorMsg(code == 0 ? ping.getPingUrlOne() + "访问失败,错误代码" + code : "访问超时");
msg.setHandle(false);
msg.setTemplateCreator(tp.getCreator());
msg.setTempName(ping.getTempName());
msg.setTemplateLv(tp.getTemplateLv());
msg.setErrorType("url");
messageService.insert(msg);
}
} catch (Exception e) {
Template tp = MainThread.mainMap.get(ping.getTemplateId());
Message msg = new Message();
msg.setCreateDate(new Date());
msg.setErrorMsg("访问出错,url不可访问");
msg.setHandle(false);
msg.setTemplateCreator(tp.getCreator());
msg.setTempName(ping.getTempName());
msg.setTemplateLv(1);
msg.setErrorType("url");
msg.setPt(ping.getPt());
messageService.insert(msg);
}
if (null!=ping.getPingUrlTwo()&& !ping.getPingUrlTwo().equals(""))
if (MainThread.mainMap.get(ping.getTemplateId()) != null) {
URL url;
try {
url = new URL(ping.getPingUrlTwo());
url = new URL(ping.getPingUrlOne());
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setConnectTimeout(1000 * 10);
int code = conn.getResponseCode();
......@@ -88,7 +56,6 @@ public class Task implements Runnable {
msg.setTempName(ping.getTempName());
msg.setTemplateLv(tp.getTemplateLv());
msg.setErrorType("url");
msg.setPt(ping.getPt());
messageService.insert(msg);
}
} catch (Exception e) {
......@@ -99,127 +66,165 @@ public class Task implements Runnable {
msg.setHandle(false);
msg.setTemplateCreator(tp.getCreator());
msg.setTempName(ping.getTempName());
msg.setPt(ping.getPt());
msg.setTemplateLv(1);
msg.setErrorType("url");
msg.setTemplateLv(tp.getTemplateLv());
msg.setPt(ping.getPt());
messageService.insert(msg);
}
if (null != ping.getPingUrlTwo() && !ping.getPingUrlTwo().equals(""))
try {
url = new URL(ping.getPingUrlTwo());
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setConnectTimeout(1000 * 10);
int code = conn.getResponseCode();
if (code != 200) {
Template tp = MainThread.mainMap.get(ping.getTemplateId());
Message msg = new Message();
msg.setCreateDate(new Date());
msg.setErrorMsg(code == 0 ? ping.getPingUrlOne() + "访问失败,错误代码" + code : "访问超时");
msg.setHandle(false);
msg.setTemplateCreator(tp.getCreator());
msg.setTempName(ping.getTempName());
msg.setTemplateLv(tp.getTemplateLv());
msg.setErrorType("url");
msg.setPt(ping.getPt());
messageService.insert(msg);
}
} catch (Exception e) {
Template tp = MainThread.mainMap.get(ping.getTemplateId());
Message msg = new Message();
msg.setCreateDate(new Date());
msg.setErrorMsg("访问出错,url不可访问");
msg.setHandle(false);
msg.setTemplateCreator(tp.getCreator());
msg.setTempName(ping.getTempName());
msg.setPt(ping.getPt());
msg.setErrorType("url");
msg.setTemplateLv(tp.getTemplateLv());
messageService.insert(msg);
}
}
}
if (field != null) {
List<Map<String, Object>> list = newsService.findNews(field.getHost(), field.getPort(), field.getdBName(), field.getCollection(),
field.getTestCount(), field.getSpyderInfoId(), field.getPt());
System.out.println("field检测数据长度"+list.size());
if(list.size()<field.getTestCount()){
Template tp = MainThread.mainMap.get(field.getTemplateId());
Message msg = new Message();
msg.setCreateDate(new Date());
msg.setErrorMsg("字段检测发现库内的数据小于预设检测数量："+field.getTestCount()+"条,实际数据量为"+list.size());
msg.setHandle(false);
msg.setPt(field.getPt());
msg.setTemplateCreator(tp.getCreator());
msg.setTemplateLv(tp.getTemplateLv());
msg.setErrorType("field");
msg.setTempName(field.getTempName());
messageService.insert(msg);
}
String[] fields = field.getField().split(",");
int count = list.size();
for (String fid : fields) {
String str="";
int num = count;
for (Map<String, Object> map : list) {
if (map.get(fid) == null) {
num--;
str=String.valueOf(map.get("_id"));
}else if(String.valueOf(map.get(fid)).length()<1){
num--;
str=String.valueOf(map.get("_id"));
}
}
if (num / 1.0 / count < field.getFieldIntegrityRate()/100.0) {
if (MainThread.mainMap.get(field.getTemplateId()) != null) {
List<Map<String, Object>> list = newsService.findNews(field.getHost(), field.getPort(),
field.getdBName(), field.getCollection(), field.getTestCount(), field.getSpyderInfoId(),
field.getPt());
System.out.println("field检测数据长度" + list.size());
if (list.size() < field.getTestCount()) {
Template tp = MainThread.mainMap.get(field.getTemplateId());
Message msg = new Message();
msg.setCreateDate(new Date());
msg.setErrorMsg(fid + "字段存在缺失,完整率"+num / 1.0 / count+"其中一条："+str);
msg.setErrorMsg("字段检测发现库内的数据小于预设检测数量：" + field.getTestCount() + "条,实际数据量为" + list.size());
msg.setHandle(false);
msg.setPt(field.getPt());
msg.setErrorType("field");
msg.setTemplateCreator(tp.getCreator());
msg.setTemplateLv(tp.getTemplateLv());
msg.setErrorType("field");
msg.setTempName(field.getTempName());
messageService.insert(msg);
}
String[] fields = field.getField().split(",");
int count = list.size();
for (String fid : fields) {
String str = "";
int num = count;
for (Map<String, Object> map : list) {
if (map.get(fid) == null) {
num--;
str = String.valueOf(map.get("_id"));
} else if (String.valueOf(map.get(fid)).length() < 1) {
num--;
str = String.valueOf(map.get("_id"));
}
}
if (num / 1.0 / count < field.getFieldIntegrityRate() / 100.0) {
Template tp = MainThread.mainMap.get(field.getTemplateId());
Message msg = new Message();
msg.setCreateDate(new Date());
msg.setErrorMsg(fid + "字段存在缺失,完整率" + num / 1.0 / count + "其中一条：" + str);
msg.setHandle(false);
msg.setPt(field.getPt());
msg.setErrorType("field");
msg.setTemplateCreator(tp.getCreator());
msg.setTemplateLv(tp.getTemplateLv());
msg.setTempName(field.getTempName());
messageService.insert(msg);
}
}
}
}
if (delayed != null) {
List<Map<String, Object>> list = newsService.findNews(delayed.getHost(), delayed.getPort(), delayed.getdBName(),
delayed.getCollection(), delayed.getTestCount(), delayed.getSpyderInfoId(), delayed.getPt());
System.out.println("delay检测数据长度"+list.size());
long sum = 0;
if(list.size()<delayed.getTestCount()){
Template tp = MainThread.mainMap.get(delayed.getTemplateId());
Message msg = new Message();
msg.setCreateDate(new Date());
msg.setErrorMsg("延时检测发现库内的数据小于预设检测数量："+delayed.getTestCount()+"条,实际数据量为"+list.size());
msg.setHandle(false);
msg.setPt(delayed.getPt());
msg.setErrorType("delayed");
msg.setTemplateCreator(tp.getCreator());
msg.setTemplateLv(tp.getTemplateLv());
msg.setTempName(delayed.getTempName());
messageService.insert(msg);
}
double delay=0;
List<Double> dNum=new ArrayList<Double>();
for (Map<String, Object> map : list) {
Date time = null;
Date savetime = null;
if (map.get(delayed.getTime()) != null) {
time = (Date) map.get(delayed.getTime());
if (MainThread.mainMap.get(delayed.getTemplateId()) != null) {
List<Map<String, Object>> list = newsService.findNews(delayed.getHost(), delayed.getPort(),
delayed.getdBName(), delayed.getCollection(), delayed.getTestCount(), delayed.getSpyderInfoId(),
delayed.getPt());
System.out.println("delay检测数据长度" + list.size());
long sum = 0;
if (list.size() < delayed.getTestCount()) {
Template tp = MainThread.mainMap.get(delayed.getTemplateId());
Message msg = new Message();
msg.setCreateDate(new Date());
msg.setErrorMsg("延时检测发现库内的数据小于预设检测数量：" + delayed.getTestCount() + "条,实际数据量为" + list.size());
msg.setHandle(false);
msg.setPt(delayed.getPt());
msg.setErrorType("delayed");
msg.setTemplateCreator(tp.getCreator());
msg.setTemplateLv(tp.getTemplateLv());
msg.setTempName(delayed.getTempName());
messageService.insert(msg);
}
if (map.get(delayed.getSavetime()) != null) {
Date d=new Date((Long)map.get(delayed.getSavetime()));
savetime = d;
double delay = 0;
List<Double> dNum = new ArrayList<Double>();
for (Map<String, Object> map : list) {
Date time = null;
Date savetime = null;
if (map.get(delayed.getTime()) != null) {
time = (Date) map.get(delayed.getTime());
}
if (map.get(delayed.getSavetime()) != null) {
Date d = new Date((Long) map.get(delayed.getSavetime()));
savetime = d;
}
if (time != null && savetime != null) {
delay = savetime.getTime() - time.getTime();
dNum.add(delay);
sum += delay;
} else {
Template tp = MainThread.mainMap.get(delayed.getTemplateId());
Message msg = new Message();
msg.setCreateDate(new Date());
msg.setErrorMsg("延时检测发现时间字段缺失");
msg.setHandle(false);
msg.setPt(delayed.getPt());
msg.setErrorType("delayed");
msg.setTemplateCreator(tp.getCreator());
msg.setTemplateLv(tp.getTemplateLv());
msg.setTempName(delayed.getTempName());
messageService.insert(msg);
}
}
if (time != null && savetime != null) {
delay= savetime.getTime() - time.getTime();
dNum.add(delay);
sum +=delay;
} else {
Collections.sort(dNum);
System.out.println(sum);
int cutSize = (int) (dNum.size() * 0.8 / 1);
for (int i = cutSize; i < dNum.size(); i++) {
sum -= dNum.get(i);
}
System.out.println(sum);
if (sum / 1.0 / 1000 / 60 / cutSize > delayed.getNewsDelayed()) {
Template tp = MainThread.mainMap.get(delayed.getTemplateId());
Message msg = new Message();
msg.setCreateDate(new Date());
msg.setErrorMsg("延时检测发现时间字段缺失");
msg.setErrorMsg("延时过高，延时检测为" + sum / 1.0 / 1000 / 60 / cutSize + "分");
msg.setHandle(false);
msg.setPt(delayed.getPt());
msg.setErrorType("delayed");
msg.setTemplateCreator(tp.getCreator());
msg.setTemplateLv(tp.getTemplateLv());
msg.setPt(delayed.getPt());
msg.setTempName(delayed.getTempName());
messageService.insert(msg);
}
}
Collections.sort(dNum);
System.out.println(sum);
int cutSize=(int) (dNum.size()*0.8/1);
for(int i=cutSize;i<dNum.size();i++) {
sum-=dNum.get(i);
}
System.out.println(sum);
if (sum / 1.0 / 1000 / 60/cutSize > delayed.getNewsDelayed()) {
Template tp = MainThread.mainMap.get(delayed.getTemplateId());
Message msg = new Message();
msg.setCreateDate(new Date());
msg.setErrorMsg("延时过高，延时检测为" + sum / 1.0 / 1000 / 60/cutSize + "分");
msg.setHandle(false);
msg.setErrorType("delayed");
msg.setTemplateCreator(tp.getCreator());
msg.setTemplateLv(tp.getTemplateLv());
msg.setPt(delayed.getPt());
msg.setTempName(delayed.getTempName());
messageService.insert(msg);
}
}
}
}
src/main/java/com/zhiwei/manage/service/DataService.java
......@@ -18,15 +18,15 @@ public interface DataService {
public List<Data> findByMethod(Date startTime, Date endTime,String method,Object value);
public List<EasyNews> findBySourceFromLibrary(Date startTime, Date endTime, String source);
public int findBySourceFromLibrary(Date startTime, Date endTime, String source);
public List<EasyNews> findByAccurateSource(String startTime, String endTime, String source);
public List<EasyNews> findByAccurateSpId(String startTime, String endTime, String SpId);
public List<EasyNews> findByAccurateType(Date startTime, Date endTime, String type);
public int findByAccurateType(Date startTime, Date endTime, String type);
public List<EasyNews> findByTempName(Date startTime, Date endTime, String tempName);
public int findByTempName(Date startTime, Date endTime, String tempName);
public long findWeiboCount(Date start, Date end);
......
src/main/java/com/zhiwei/manage/service/DataServiceImpl.java
......@@ -59,7 +59,7 @@ public class DataServiceImpl implements DataService{
}
@Override
public List<EasyNews> findBySourceFromLibrary(Date startTime, Date endTime, String source) {
public int findBySourceFromLibrary(Date startTime, Date endTime, String source) {
return dataDao.findBySourceFromLibrary(startTime, endTime, source);
}
......@@ -74,7 +74,7 @@ public class DataServiceImpl implements DataService{
}
@Override
public List<EasyNews> findByAccurateType(Date startTime, Date endTime, String type) {
public int findByAccurateType(Date startTime, Date endTime, String type) {
return dataDao.findByAccurateType(startTime, endTime, type);
}
......@@ -94,7 +94,7 @@ public class DataServiceImpl implements DataService{
}
@Override
public List<EasyNews> findByTempName(Date startTime, Date endTime, String tempName) {
public int findByTempName(Date startTime, Date endTime, String tempName) {
String spId=crd.getSpIdByConfigName(tempName);
return dataDao.findBySpId(startTime, endTime, spId);
}
......
src/main/java/com/zhiwei/manage/servlet/DataController.java
......@@ -89,25 +89,32 @@ public class DataController {
startd.setTimeInMillis(start);
endTime.setTimeInMillis(start);
}
Calendar ds=Calendar.getInstance();
ds.setTimeInMillis(end);
System.out.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(startd.getTime()));
System.out.println(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(ds.getTime()));
String source = "";
List<Data> list = new ArrayList<Data>();
startd.add(Calendar.HOUR, -8);
while (endTime.getTimeInMillis() < end) {
while (endTime.getTime().before(ds.getTime())) {
endTime.add(Calendar.HOUR, 1);
List<EasyNews> newsList = new ArrayList<>();
int count=0;
if(method.equals("type")){
source=(String) map.get("type");
newsList=dataService.findByAccurateType(startd.getTime(), endTime.getTime(), source);
count=dataService.findByAccurateType(startd.getTime(), endTime.getTime(), source);
}else{
if (method.equals("source")) {
source = (String) map.get("source");
newsList = dataService.findBySourceFromLibrary(startd.getTime(), endTime.getTime(), source);
count = dataService.findBySourceFromLibrary(startd.getTime(), endTime.getTime(), source);
} else {
source = (String) map.get("tempName");
newsList = dataService.findByTempName(startd.getTime(), endTime.getTime(), source);
count = dataService.findByTempName(startd.getTime(), endTime.getTime(), source);
}
}
Data data = Change.changeNewsToDate(newsList, startd.getTime());
Data data = new Data();
data.setCount(count);
data.setSource(source);
data.setTime(startd.getTime());
list.add(data);
startd.add(Calendar.HOUR, 1);
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment