Skip to content

D
datamanage
Commit 1f779ff3 by admin
Options

修改了检测的规则，并且把其他媒体加上了预警

parent 103ea399
Showing with 308 additions and 146 deletions
.gitignore
......@@ -5,4 +5,5 @@
.project
.svn
.settings/
target/
\ No newline at end of file
target/
/Jenkinsfile/
\ No newline at end of file
log/error.log
......@@ -14,3 +14,147 @@ java.lang.NullPointerException: index must not be null
at org.elasticsearch.client.support.AbstractClient.prepareSearch(AbstractClient.java:535)
at com.zhiwei.manage.util.ESUtil.getCountByPt(ESUtil.java:138)
at com.zhiwei.manage.util.ESUtil.main(ESUtil.java:88)
2018-05-18 12:03:27,954 [main] [com.zhiwei.manage.util.ESUtil] [ERROR] - es检索出错，错误信息【{}】
java.lang.NullPointerException
at com.zhiwei.manage.util.ESUtil.getDatasMap(ESUtil.java:150)
at com.zhiwei.manage.util.ESUtil.main(ESUtil.java:122)
2018-05-18 12:04:01,225 [main] [com.zhiwei.manage.util.ESUtil] [ERROR] - es检索出错，错误信息【{}】
java.lang.NullPointerException
at com.zhiwei.manage.util.ESUtil.getDatasMap(ESUtil.java:150)
at com.zhiwei.manage.util.ESUtil.main(ESUtil.java:122)
2018-05-18 12:04:51,859 [main] [com.zhiwei.manage.util.ESUtil] [ERROR] - es检索出错，错误信息【{}】
java.lang.NullPointerException
at com.zhiwei.manage.util.ESUtil.getDatasMap(ESUtil.java:150)
at com.zhiwei.manage.util.ESUtil.main(ESUtil.java:122)
2018-05-18 12:05:20,555 [main] [com.zhiwei.manage.util.ESUtil] [ERROR] - es检索出错，错误信息【{}】
java.lang.NullPointerException
at com.zhiwei.manage.util.ESUtil.getDatasMap(ESUtil.java:154)
at com.zhiwei.manage.util.ESUtil.main(ESUtil.java:122)
2018-05-18 12:05:40,438 [main] [com.zhiwei.manage.util.ESUtil] [ERROR] - es检索出错，错误信息【{}】
java.lang.NullPointerException
at com.zhiwei.manage.util.ESUtil.getDatasMap(ESUtil.java:152)
at com.zhiwei.manage.util.ESUtil.main(ESUtil.java:122)
2018-05-18 12:05:54,767 [main] [com.zhiwei.manage.util.ESUtil] [ERROR] - es检索出错，错误信息【{}】
UncategorizedExecutionException[Failed execution]; nested: IOException[can not write type [class java.util.GregorianCalendar]];
at org.elasticsearch.action.support.AdapterActionFuture.rethrowExecutionException(AdapterActionFuture.java:87)
at org.elasticsearch.action.support.AdapterActionFuture.actionGet(AdapterActionFuture.java:46)
at com.zhiwei.manage.util.ESUtil.getDatasMap(ESUtil.java:164)
at com.zhiwei.manage.util.ESUtil.main(ESUtil.java:122)
Caused by: java.io.IOException: can not write type [class java.util.GregorianCalendar]
at org.elasticsearch.common.io.stream.StreamOutput.writeGenericValue(StreamOutput.java:655)
at org.elasticsearch.index.query.RangeQueryBuilder.doWriteTo(RangeQueryBuilder.java:129)
at org.elasticsearch.index.query.AbstractQueryBuilder.writeTo(AbstractQueryBuilder.java:73)
at org.elasticsearch.common.io.stream.StreamOutput.writeNamedWriteable(StreamOutput.java:857)
at org.elasticsearch.common.io.stream.StreamOutput.writeOptionalNamedWriteable(StreamOutput.java:868)
at org.elasticsearch.search.builder.SearchSourceBuilder.writeTo(SearchSourceBuilder.java:237)
at org.elasticsearch.common.io.stream.StreamOutput.writeOptionalWriteable(StreamOutput.java:732)
at org.elasticsearch.action.search.SearchRequest.writeTo(SearchRequest.java:337)
at org.elasticsearch.transport.TcpTransport.buildMessage(TcpTransport.java:1179)
at org.elasticsearch.transport.TcpTransport.sendRequestToChannel(TcpTransport.java:1031)
at org.elasticsearch.transport.TcpTransport.access$1100(TcpTransport.java:113)
at org.elasticsearch.transport.TcpTransport$NodeChannels.sendRequest(TcpTransport.java:435)
at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:554)
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:485)
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:473)
at org.elasticsearch.action.TransportActionNodeProxy.execute(TransportActionNodeProxy.java:51)
at org.elasticsearch.client.transport.TransportProxyClient.lambda$execute$0(TransportProxyClient.java:59)
at org.elasticsearch.client.transport.TransportClientNodesService.execute(TransportClientNodesService.java:247)
at org.elasticsearch.client.transport.TransportProxyClient.execute(TransportProxyClient.java:59)
at org.elasticsearch.client.transport.TransportClient.doExecute(TransportClient.java:356)
at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:403)
at org.elasticsearch.action.ActionRequestBuilder.execute(ActionRequestBuilder.java:80)
at org.elasticsearch.action.ActionRequestBuilder.execute(ActionRequestBuilder.java:54)
... 2 more
2018-05-18 12:07:03,923 [main] [com.zhiwei.manage.util.ESUtil] [ERROR] - es检索出错，错误信息【{}】
UncategorizedExecutionException[Failed execution]; nested: IOException[can not write type [class java.util.GregorianCalendar]];
at org.elasticsearch.action.support.AdapterActionFuture.rethrowExecutionException(AdapterActionFuture.java:87)
at org.elasticsearch.action.support.AdapterActionFuture.actionGet(AdapterActionFuture.java:46)
at com.zhiwei.manage.util.ESUtil.getDatasMap(ESUtil.java:165)
at com.zhiwei.manage.util.ESUtil.main(ESUtil.java:122)
Caused by: java.io.IOException: can not write type [class java.util.GregorianCalendar]
at org.elasticsearch.common.io.stream.StreamOutput.writeGenericValue(StreamOutput.java:655)
at org.elasticsearch.index.query.RangeQueryBuilder.doWriteTo(RangeQueryBuilder.java:129)
at org.elasticsearch.index.query.AbstractQueryBuilder.writeTo(AbstractQueryBuilder.java:73)
at org.elasticsearch.common.io.stream.StreamOutput.writeNamedWriteable(StreamOutput.java:857)
at org.elasticsearch.common.io.stream.StreamOutput.writeOptionalNamedWriteable(StreamOutput.java:868)
at org.elasticsearch.search.builder.SearchSourceBuilder.writeTo(SearchSourceBuilder.java:237)
at org.elasticsearch.common.io.stream.StreamOutput.writeOptionalWriteable(StreamOutput.java:732)
at org.elasticsearch.action.search.SearchRequest.writeTo(SearchRequest.java:337)
at org.elasticsearch.transport.TcpTransport.buildMessage(TcpTransport.java:1179)
at org.elasticsearch.transport.TcpTransport.sendRequestToChannel(TcpTransport.java:1031)
at org.elasticsearch.transport.TcpTransport.access$1100(TcpTransport.java:113)
at org.elasticsearch.transport.TcpTransport$NodeChannels.sendRequest(TcpTransport.java:435)
at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:554)
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:485)
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:473)
at org.elasticsearch.action.TransportActionNodeProxy.execute(TransportActionNodeProxy.java:51)
at org.elasticsearch.client.transport.TransportProxyClient.lambda$execute$0(TransportProxyClient.java:59)
at org.elasticsearch.client.transport.TransportClientNodesService.execute(TransportClientNodesService.java:247)
at org.elasticsearch.client.transport.TransportProxyClient.execute(TransportProxyClient.java:59)
at org.elasticsearch.client.transport.TransportClient.doExecute(TransportClient.java:356)
at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:403)
at org.elasticsearch.action.ActionRequestBuilder.execute(ActionRequestBuilder.java:80)
at org.elasticsearch.action.ActionRequestBuilder.execute(ActionRequestBuilder.java:54)
... 2 more
2018-05-18 12:08:16,688 [main] [com.zhiwei.manage.util.ESUtil] [ERROR] - es检索出错，错误信息【{}】
UncategorizedExecutionException[Failed execution]; nested: IOException[can not write type [class java.util.GregorianCalendar]];
at org.elasticsearch.action.support.AdapterActionFuture.rethrowExecutionException(AdapterActionFuture.java:87)
at org.elasticsearch.action.support.AdapterActionFuture.actionGet(AdapterActionFuture.java:46)
at com.zhiwei.manage.util.ESUtil.getDatasMap(ESUtil.java:164)
at com.zhiwei.manage.util.ESUtil.main(ESUtil.java:122)
Caused by: java.io.IOException: can not write type [class java.util.GregorianCalendar]
at org.elasticsearch.common.io.stream.StreamOutput.writeGenericValue(StreamOutput.java:655)
at org.elasticsearch.index.query.RangeQueryBuilder.doWriteTo(RangeQueryBuilder.java:129)
at org.elasticsearch.index.query.AbstractQueryBuilder.writeTo(AbstractQueryBuilder.java:73)
at org.elasticsearch.common.io.stream.StreamOutput.writeNamedWriteable(StreamOutput.java:857)
at org.elasticsearch.common.io.stream.StreamOutput.writeOptionalNamedWriteable(StreamOutput.java:868)
at org.elasticsearch.search.builder.SearchSourceBuilder.writeTo(SearchSourceBuilder.java:237)
at org.elasticsearch.common.io.stream.StreamOutput.writeOptionalWriteable(StreamOutput.java:732)
at org.elasticsearch.action.search.SearchRequest.writeTo(SearchRequest.java:337)
at org.elasticsearch.transport.TcpTransport.buildMessage(TcpTransport.java:1179)
at org.elasticsearch.transport.TcpTransport.sendRequestToChannel(TcpTransport.java:1031)
at org.elasticsearch.transport.TcpTransport.access$1100(TcpTransport.java:113)
at org.elasticsearch.transport.TcpTransport$NodeChannels.sendRequest(TcpTransport.java:435)
at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:554)
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:485)
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:473)
at org.elasticsearch.action.TransportActionNodeProxy.execute(TransportActionNodeProxy.java:51)
at org.elasticsearch.client.transport.TransportProxyClient.lambda$execute$0(TransportProxyClient.java:59)
at org.elasticsearch.client.transport.TransportClientNodesService.execute(TransportClientNodesService.java:247)
at org.elasticsearch.client.transport.TransportProxyClient.execute(TransportProxyClient.java:59)
at org.elasticsearch.client.transport.TransportClient.doExecute(TransportClient.java:356)
at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:403)
at org.elasticsearch.action.ActionRequestBuilder.execute(ActionRequestBuilder.java:80)
at org.elasticsearch.action.ActionRequestBuilder.execute(ActionRequestBuilder.java:54)
... 2 more
2018-05-18 12:09:45,001 [main] [com.zhiwei.manage.util.ESUtil] [ERROR] - es检索出错，错误信息【{}】
UncategorizedExecutionException[Failed execution]; nested: IOException[can not write type [class java.util.GregorianCalendar]];
at org.elasticsearch.action.support.AdapterActionFuture.rethrowExecutionException(AdapterActionFuture.java:87)
at org.elasticsearch.action.support.AdapterActionFuture.actionGet(AdapterActionFuture.java:46)
at com.zhiwei.manage.util.ESUtil.getDatasMap(ESUtil.java:164)
at com.zhiwei.manage.util.ESUtil.main(ESUtil.java:122)
Caused by: java.io.IOException: can not write type [class java.util.GregorianCalendar]
at org.elasticsearch.common.io.stream.StreamOutput.writeGenericValue(StreamOutput.java:655)
at org.elasticsearch.index.query.RangeQueryBuilder.doWriteTo(RangeQueryBuilder.java:129)
at org.elasticsearch.index.query.AbstractQueryBuilder.writeTo(AbstractQueryBuilder.java:73)
at org.elasticsearch.common.io.stream.StreamOutput.writeNamedWriteable(StreamOutput.java:857)
at org.elasticsearch.common.io.stream.StreamOutput.writeOptionalNamedWriteable(StreamOutput.java:868)
at org.elasticsearch.search.builder.SearchSourceBuilder.writeTo(SearchSourceBuilder.java:237)
at org.elasticsearch.common.io.stream.StreamOutput.writeOptionalWriteable(StreamOutput.java:732)
at org.elasticsearch.action.search.SearchRequest.writeTo(SearchRequest.java:337)
at org.elasticsearch.transport.TcpTransport.buildMessage(TcpTransport.java:1179)
at org.elasticsearch.transport.TcpTransport.sendRequestToChannel(TcpTransport.java:1031)
at org.elasticsearch.transport.TcpTransport.access$1100(TcpTransport.java:113)
at org.elasticsearch.transport.TcpTransport$NodeChannels.sendRequest(TcpTransport.java:435)
at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:554)
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:485)
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:473)
at org.elasticsearch.action.TransportActionNodeProxy.execute(TransportActionNodeProxy.java:51)
at org.elasticsearch.client.transport.TransportProxyClient.lambda$execute$0(TransportProxyClient.java:59)
at org.elasticsearch.client.transport.TransportClientNodesService.execute(TransportClientNodesService.java:247)
at org.elasticsearch.client.transport.TransportProxyClient.execute(TransportProxyClient.java:59)
at org.elasticsearch.client.transport.TransportClient.doExecute(TransportClient.java:356)
at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:403)
at org.elasticsearch.action.ActionRequestBuilder.execute(ActionRequestBuilder.java:80)
at org.elasticsearch.action.ActionRequestBuilder.execute(ActionRequestBuilder.java:54)
... 2 more
src/main/java/com/zhiwei/manage/bean/NewsDelayed.java
......@@ -156,7 +156,9 @@ public class NewsDelayed implements Serializable {
return "NewsDelayed [templateId=" + templateId + ", tempName=" + tempName + ", newsDelayed=" + newsDelayed
+ ", delayedInterval=" + delayedInterval + ", testCount=" + testCount + ", time=" + time + ", savetime="
+ savetime + ", host=" + host + ", port=" + port + ", collection=" + collection + ", dBName=" + dBName
+ ", pt=" + pt + ", startNow=" + startNow + ", nextRun=" + nextRun + "]";
+ ", pt=" + pt + ", startNow=" + startNow + ", nextRun=" + nextRun + ", spyderInfoId=" + spyderInfoId
+ "]";
}
}
src/main/java/com/zhiwei/manage/dao/MessageDaoImpl.java
......@@ -59,7 +59,7 @@ public class MessageDaoImpl implements MessageDao {
for (int j = 0; j < fields.length; j++) {
fields[j].setAccessible(true);
try {
if (fields[j].getName().equals("templateId")) {
if (fields[j].getName().equals("templateId")||fields[j].getName().equals("tempName")) {
continue;
}
if (fields[j].get(message) != null)
......@@ -69,7 +69,7 @@ public class MessageDaoImpl implements MessageDao {
e.printStackTrace();
}
}
WriteResult result = mongo.upsert(Query.query(Criteria.where("_id").is(message.getMessageId())), update,
WriteResult result = mongo.upsert(Query.query(Criteria.where("tempName").is(message.getTempName())), update,
Message.class);
if (result.getN() == 1) {
return "修改成功";
......
src/main/java/com/zhiwei/manage/handle/MainThread.java
......@@ -48,7 +48,10 @@ public class MainThread extends Thread {
counts.put("微博", 2000);
counts.put("微信", 3500);
counts.put("知乎", 300);
// counts.put("平媒", 300);
counts.put("平媒", 300);
counts.put("今日头条", 1000);
counts.put("贴吧", 250);
counts.put("论坛", 30);
}
public MainThread() {
......@@ -183,7 +186,7 @@ public class MainThread extends Thread {
map.put(chan.getPt(), chan.getValue());
try {
ServerBean ser = serverService.findServer(chan.getPt());
count = es.getCountByPt(start.getTime(), end.getTime(), chan.getPt(), chan.getValue(), ser);
count = es.getCountByPt(start.getTime(), end.getTime(), chan.getPt(), chan.getValue(), ser);//当日的数据量
System.out.println(count);
Data data = new Data();
data.setCount(count);
......@@ -191,11 +194,15 @@ public class MainThread extends Thread {
data.setTime(start.getTime());
dataService.insert(data);
log.info(sdf.format(start.getTime()) + "|" + chan.getPt() + "|入库");
start.setTime(end.getTime());
start.add(Calendar.HOUR, -1);
count = es.getCountByPt(start.getTime(), end.getTime(), chan.getPt(), chan.getValue(), ser);//小时的数据量
if (counts.get(chan.getPt()) != null)
if (count < counts.get(chan.getPt())) {
Messages m = new Messages();
m.setTitle("渠道低数据量预警{" + chan.getPt() + "}");
m.setContent("1小时内的数据量为：" + count);
m.setDate(sdf.format(start.getTime()));
for (String mail : Config.getVal("count_mail").split(",")) {
SendMailUtil.sendMessage(mail, m);
}
......
src/main/java/com/zhiwei/manage/handle/Task.java
......@@ -21,8 +21,10 @@ import com.zhiwei.manage.bean.Template;
import com.zhiwei.manage.orm.BeanFactory;
import com.zhiwei.manage.service.MessageService;
import com.zhiwei.manage.service.NewsService;
/**
* 任务执行
*
* @author admin
*
*/
......@@ -31,8 +33,8 @@ public class Task implements Runnable {
private NewsDelayed delayed;
private PingUrl ping;
private FieldIntegerity field;
final static SimpleDateFormat sdf=new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS Z");
private final static Log log=LogFactory.getLog(Task.class);
final static SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS Z");
private final static Log log = LogFactory.getLog(Task.class);
public Task(Object runMsg) {
if (runMsg instanceof NewsDelayed) {
......@@ -70,7 +72,7 @@ public class Task implements Runnable {
msg.setTemplateLv(tp.getTemplateLv());
msg.setErrorType("url");
msg.setPt(ping.getPt());
messageService.update(msg,ping.getPt());
messageService.update(msg, ping.getPt());
} else {
Message msg = messageService.findMsg(ping.getTempName(), "url");
if (msg != null) {
......@@ -90,7 +92,7 @@ public class Task implements Runnable {
msg.setTemplateLv(1);
msg.setErrorType("url");
msg.setPt(ping.getPt());
messageService.update(msg,ping.getPt());
messageService.update(msg, ping.getPt());
}
if (null != ping.getPingUrlTwo() && !ping.getPingUrlTwo().equals(""))
try {
......@@ -111,7 +113,7 @@ public class Task implements Runnable {
msg.setTemplateLv(tp.getTemplateLv());
msg.setErrorType("url");
msg.setPt(ping.getPt());
messageService.update(msg,ping.getPt());
messageService.update(msg, ping.getPt());
} else {
Message msg = messageService.findMsg(ping.getTempName(), "url");
if (msg != null) {
......@@ -131,153 +133,157 @@ public class Task implements Runnable {
msg.setPt(ping.getPt());
msg.setErrorType("url");
msg.setTemplateLv(tp.getTemplateLv());
messageService.update(msg,ping.getPt());
messageService.update(msg, ping.getPt());
}
}
}
// if (field != null) {//字段缺失率，采集部分已经对缺失字段做了处理，没什么必要检测了
// if (MainThread.mainMap.get(field.getTemplateId()) != null) {
// List<Map<String, Object>> list = newsService.findNews(field.getHost(), field.getPort(),
// field.getdBName(), field.getCollection(), field.getTestCount(), field.getSpyderInfoId(),
// field.getPt());
// if (list.size() ==0) {
// Template tp = MainThread.mainMap.get(field.getTemplateId());
// Message msg = new Message();
// msg.setCreateDate(new Date());
// msg.setErrorMsg("字段检测发现库内的数据为0");
// msg.setHandle(false);
// msg.setPt(field.getPt());
// msg.setTemplateCreator(tp.getCreator());
// msg.setTemplateLv(tp.getTemplateLv());
// msg.setErrorType("count2");
// msg.setTempName(field.getTempName());
// messageService.update(msg,field.getPt());
// } else {
// Message msg = messageService.findMsg(field.getTempName(), "count2");
// if (msg != null) {
// List<String> ids = new ArrayList<>();
// ids.add(msg.getMessageId());
// messageService.delete(ids, field.getPt());
// }
// }
// String[] fields = field.getField().split(",");
// int count = list.size();
// for (String fid : fields) {
// String str = "";
// int num = count;
// for (Map<String, Object> map : list) {
// if (map.get(fid) == null) {
// num--;
// str = String.valueOf(map.get("_id"));
// } else if (String.valueOf(map.get(fid)).length() < 1) {
// num--;
// str = String.valueOf(map.get("_id"));
// }
// }
// if (num / 1.0 / count < field.getFieldIntegrityRate() / 100.0) {
// Template tp = MainThread.mainMap.get(field.getTemplateId());
// Message msg = new Message();
// msg.setCreateDate(new Date());
// msg.setErrorMsg(fid + "字段存在缺失,完整率" + num / 1.0 / count + "其中一条：" + str);
// msg.setHandle(false);
// msg.setPt(field.getPt());
// msg.setErrorType("field");
// msg.setTemplateCreator(tp.getCreator());
// msg.setTemplateLv(tp.getTemplateLv());
// msg.setTempName(field.getTempName());
// messageService.update(msg,field.getPt());
// } else {
// Message msg = messageService.findMsg(field.getTempName(), "field");
// if (msg != null) {
// List<String> ids = new ArrayList<>();
// ids.add(msg.getMessageId());
// messageService.delete(ids, field.getPt());
// }
// }
// }
// }
// }
// if (field != null) {//字段缺失率，采集部分已经对缺失字段做了处理，没什么必要检测了
// if (MainThread.mainMap.get(field.getTemplateId()) != null) {
// List<Map<String, Object>> list = newsService.findNews(field.getHost(),
// field.getPort(),
// field.getdBName(), field.getCollection(), field.getTestCount(),
// field.getSpyderInfoId(),
// field.getPt());
// if (list.size() ==0) {
// Template tp = MainThread.mainMap.get(field.getTemplateId());
// Message msg = new Message();
// msg.setCreateDate(new Date());
// msg.setErrorMsg("字段检测发现库内的数据为0");
// msg.setHandle(false);
// msg.setPt(field.getPt());
// msg.setTemplateCreator(tp.getCreator());
// msg.setTemplateLv(tp.getTemplateLv());
// msg.setErrorType("count2");
// msg.setTempName(field.getTempName());
// messageService.update(msg,field.getPt());
// } else {
// Message msg = messageService.findMsg(field.getTempName(), "count2");
// if (msg != null) {
// List<String> ids = new ArrayList<>();
// ids.add(msg.getMessageId());
// messageService.delete(ids, field.getPt());
// }
// }
// String[] fields = field.getField().split(",");
// int count = list.size();
// for (String fid : fields) {
// String str = "";
// int num = count;
// for (Map<String, Object> map : list) {
// if (map.get(fid) == null) {
// num--;
// str = String.valueOf(map.get("_id"));
// } else if (String.valueOf(map.get(fid)).length() < 1) {
// num--;
// str = String.valueOf(map.get("_id"));
// }
// }
// if (num / 1.0 / count < field.getFieldIntegrityRate() / 100.0) {
// Template tp = MainThread.mainMap.get(field.getTemplateId());
// Message msg = new Message();
// msg.setCreateDate(new Date());
// msg.setErrorMsg(fid + "字段存在缺失,完整率" + num / 1.0 / count + "其中一条：" + str);
// msg.setHandle(false);
// msg.setPt(field.getPt());
// msg.setErrorType("field");
// msg.setTemplateCreator(tp.getCreator());
// msg.setTemplateLv(tp.getTemplateLv());
// msg.setTempName(field.getTempName());
// messageService.update(msg,field.getPt());
// } else {
// Message msg = messageService.findMsg(field.getTempName(), "field");
// if (msg != null) {
// List<String> ids = new ArrayList<>();
// ids.add(msg.getMessageId());
// messageService.delete(ids, field.getPt());
// }
// }
// }
// }
// }
if (delayed != null) {
log.info(delayed.getTempName() + "开始检测");
if (MainThread.mainMap.get(delayed.getTemplateId()) != null) {
List<Map<String, Object>> list = newsService.findNews(delayed.getHost(), delayed.getPort(),
delayed.getdBName(), delayed.getCollection(), delayed.getTestCount(), delayed.getSpyderInfoId(),
delayed.getPt());
log.info("size"+list.size());
long sum = 0;
if (list.size() ==0) {
if (list.size() == 0) {
Template tp = MainThread.mainMap.get(delayed.getTemplateId());
Message msg = new Message();
msg.setCreateDate(new Date());
msg.setErrorMsg("查不到该数据");
msg.setHandle(false);
msg.setPt(delayed.getPt());
msg.setErrorType("count2");
msg.setTemplateCreator(tp.getCreator());
msg.setTemplateLv(1);
msg.setTempName(delayed.getTempName());
messageService.update(msg, delayed.getPt());
} else {
Message msg = messageService.findMsg(delayed.getTempName(), "count2");
if (msg != null) {
List<String> ids = new ArrayList<>();
ids.add(msg.getMessageId());
messageService.delete(ids, delayed.getPt());
}
}
double delay = 0;
List<Double> dNum = new ArrayList<Double>();
for (Map<String, Object> map : list) {
Date time = null;
Date savetime = null;
if (map.get(delayed.getTime()) != null) {
String date=null;
try {
date = String.valueOf(map.get(delayed.getTime())).replace("Z", " UTC");
time = sdf.parse(String.valueOf(date));
} catch (ParseException e) {
log.info("时间:"+date);
log.error("时间转换出错【{}】",e);
Message msg = null;
double delay = 0;
List<Double> dNum = new ArrayList<Double>();
for (Map<String, Object> map : list) {
Date time = null;
Date savetime = null;
if (map.get(delayed.getTime()) != null) {
String date = null;
try {
date = String.valueOf(map.get(delayed.getTime())).replace("Z", " UTC");
time = sdf.parse(String.valueOf(date));
} catch (ParseException e) {
log.info("时间:" + date);
log.error("时间转换出错【{}】", e);
}
}
if (map.get(delayed.getSavetime()) != null) {
Date d = new Date((Long) map.get(delayed.getSavetime()));
savetime = d;
}
if (time != null && savetime != null) {
delay = savetime.getTime() - time.getTime();
dNum.add(delay);
sum += delay;
} else {
Template tp = MainThread.mainMap.get(delayed.getTemplateId());
msg = new Message();
msg.setCreateDate(new Date());
msg.setErrorMsg("延时检测发现时间字段缺失");
msg.setHandle(false);
msg.setPt(delayed.getPt());
msg.setErrorType("timeIsNull");
msg.setTemplateCreator(tp.getCreator());
msg.setTemplateLv(tp.getTemplateLv());
msg.setTempName(delayed.getTempName());
messageService.update(msg, delayed.getPt());
}
}
if (map.get(delayed.getSavetime()) != null) {
Date d = new Date((Long) map.get(delayed.getSavetime()));
savetime = d;
Collections.sort(dNum);
int cutSize = (int) (dNum.size() * 0.8 / 1);
for (int i = cutSize; i < dNum.size(); i++) {
sum -= dNum.get(i);
}
if (time != null && savetime != null) {
delay = savetime.getTime() - time.getTime();
dNum.add(delay);
sum += delay;
Message msg = messageService.findMsg(delayed.getTempName(), "delayed");
if (msg != null) {
List<String> ids = new ArrayList<>();
ids.add(msg.getMessageId());
messageService.delete(ids, delayed.getPt());
}
} else {
if (sum / 1.0 / 1000 / 60 / cutSize > delayed.getNewsDelayed()) {
Template tp = MainThread.mainMap.get(delayed.getTemplateId());
Message msg = new Message();
msg = new Message();
msg.setCreateDate(new Date());
msg.setErrorMsg("延时检测发现时间字段缺失");
msg.setErrorMsg("延时过高，延时检测为" + sum / 1.0 / 1000 / 60 / cutSize + "分");
msg.setHandle(false);
msg.setPt(delayed.getPt());
msg.setErrorType("delayed");
msg.setTemplateCreator(tp.getCreator());
msg.setTemplateLv(tp.getTemplateLv());
msg.setPt(delayed.getPt());
msg.setTempName(delayed.getTempName());
messageService.update(msg,delayed.getPt());
}
}
Collections.sort(dNum);
int cutSize = (int) (dNum.size() * 0.8 / 1);
for (int i = cutSize; i < dNum.size(); i++) {
sum -= dNum.get(i);
}
if (sum / 1.0 / 1000 / 60 / cutSize > delayed.getNewsDelayed()) {
Template tp = MainThread.mainMap.get(delayed.getTemplateId());
Message msg = new Message();
msg.setCreateDate(new Date());
msg.setErrorMsg("延时过高，延时检测为" + sum / 1.0 / 1000 / 60 / cutSize + "分");
msg.setHandle(false);
msg.setErrorType("delayed");
msg.setTemplateCreator(tp.getCreator());
msg.setTemplateLv(tp.getTemplateLv());
msg.setPt(delayed.getPt());
msg.setTempName(delayed.getTempName());
messageService.update(msg,delayed.getPt());
} else {
Message msg = messageService.findMsg(delayed.getTempName(), "delayed");
if (msg != null) {
List<String> ids = new ArrayList<>();
ids.add(msg.getMessageId());
messageService.delete(ids, delayed.getPt());
messageService.update(msg, delayed.getPt());
} else {
msg = messageService.findMsg(delayed.getTempName(), "delayed");
if (msg != null) {
List<String> ids = new ArrayList<>();
ids.add(msg.getMessageId());
messageService.delete(ids, delayed.getPt());
}
}
}
}
......
src/main/java/com/zhiwei/manage/service/NewsServiceImpl.java
......@@ -16,7 +16,6 @@ public class NewsServiceImpl implements NewsService{
@Autowired
private NewsDao newsDao;
private ESUtil es=new ESUtil();
@Override
public List<News> findByNew(int delayedCount, String type, String source) {
......@@ -52,7 +51,7 @@ public class NewsServiceImpl implements NewsService{
@Override
public List<Map<String, Object>> findNews(String host, int port, String dbNamel, String collection, int count,
String id,String pt) {
return es.getDatasMap(host, port, dbNamel, collection, count, id, pt);
return ESUtil.getDatasMap(host, port, dbNamel, collection, count, id, pt);
}
......
src/main/java/com/zhiwei/manage/util/ESUtil.java
......@@ -119,7 +119,8 @@ public class ESUtil {
// System.out.println(searchHits.getTotalHits());
SimpleDateFormat sdf=new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS Z");
for(Map<String,Object> map:getDatasMap("1.119.44.201", 9300, "Media-University", "mediaspider*", 20, "UrlListDetail74ec93ff-2275-4f22-bb62-bae580f2d43b", "网媒")) {
for(Map<String,Object> map: getDatasMap("1.119.44.201", 9300, "Media-University", "mediaspider*"
, 20, "UrlListDetaildb4b9042-006f-4d09-a643-de379ac07e15", "平媒")) {
System.out.println(map.get("title"));
String date = map.get("time").toString().replace("Z", " UTC");
// System.out.println(map.get("time"));
......@@ -144,20 +145,22 @@ public class ESUtil {
String id,String pt) {
List<Map<String,Object>> list=new ArrayList<>();
Client esClient=null;
Calendar cal=Calendar.getInstance();
cal.add(Calendar.DATE, -7);
try {
if (DbDepot.esClients.get(pt) == null) {
if (DbDepot.esClients==null||DbDepot.esClients.get(pt) == null) {
esClient = new ESClient(dbNamel, host, port).getClient();
DbDepot.esClients.put(pt, esClient);
// DbDepot.esClients.put(pt, esClient);
} else {
esClient = (Client) DbDepot.esClients.get(pt);
}
// 搜索数据
SearchRequestBuilder srb = esClient.prepareSearch(collection);
srb.setPostFilter(QueryBuilders.rangeQuery("time").gte(getNeedTime(cal)).lte(getNeedTime(Calendar.getInstance())));
BoolQueryBuilder bb = new BoolQueryBuilder();
srb.setQuery(bb);
srb.addSort("time", SortOrder.DESC);
QueryBuilder matchQuery = QueryBuilders.matchPhraseQuery("spyderInfoId",id);
bb.should(matchQuery);
srb.setQuery(bb);
srb.setSize(count);
SearchResponse response = srb.execute().actionGet();
SearchHits searchHits = response.getHits();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment