Skip to content

D
datamanage
Commit 1f779ff3 by admin
Options

修改了检测的规则，并且把其他媒体加上了预警

parent 103ea399
Showing with 266 additions and 103 deletions
.gitignore
...@@ -6,3 +6,4 @@ ...@@ -6,3 +6,4 @@
.svn .svn
.settings/ .settings/
target/ target/
/Jenkinsfile/
\ No newline at end of file
log/error.log
...@@ -14,3 +14,147 @@ java.lang.NullPointerException: index must not be null ...@@ -14,3 +14,147 @@ java.lang.NullPointerException: index must not be null
at org.elasticsearch.client.support.AbstractClient.prepareSearch(AbstractClient.java:535) at org.elasticsearch.client.support.AbstractClient.prepareSearch(AbstractClient.java:535)
at com.zhiwei.manage.util.ESUtil.getCountByPt(ESUtil.java:138) at com.zhiwei.manage.util.ESUtil.getCountByPt(ESUtil.java:138)
at com.zhiwei.manage.util.ESUtil.main(ESUtil.java:88) at com.zhiwei.manage.util.ESUtil.main(ESUtil.java:88)
2018-05-18 12:03:27,954 [main] [com.zhiwei.manage.util.ESUtil] [ERROR] - es检索出错，错误信息【{}】
java.lang.NullPointerException
at com.zhiwei.manage.util.ESUtil.getDatasMap(ESUtil.java:150)
at com.zhiwei.manage.util.ESUtil.main(ESUtil.java:122)
2018-05-18 12:04:01,225 [main] [com.zhiwei.manage.util.ESUtil] [ERROR] - es检索出错，错误信息【{}】
java.lang.NullPointerException
at com.zhiwei.manage.util.ESUtil.getDatasMap(ESUtil.java:150)
at com.zhiwei.manage.util.ESUtil.main(ESUtil.java:122)
2018-05-18 12:04:51,859 [main] [com.zhiwei.manage.util.ESUtil] [ERROR] - es检索出错，错误信息【{}】
java.lang.NullPointerException
at com.zhiwei.manage.util.ESUtil.getDatasMap(ESUtil.java:150)
at com.zhiwei.manage.util.ESUtil.main(ESUtil.java:122)
2018-05-18 12:05:20,555 [main] [com.zhiwei.manage.util.ESUtil] [ERROR] - es检索出错，错误信息【{}】
java.lang.NullPointerException
at com.zhiwei.manage.util.ESUtil.getDatasMap(ESUtil.java:154)
at com.zhiwei.manage.util.ESUtil.main(ESUtil.java:122)
2018-05-18 12:05:40,438 [main] [com.zhiwei.manage.util.ESUtil] [ERROR] - es检索出错，错误信息【{}】
java.lang.NullPointerException
at com.zhiwei.manage.util.ESUtil.getDatasMap(ESUtil.java:152)
at com.zhiwei.manage.util.ESUtil.main(ESUtil.java:122)
2018-05-18 12:05:54,767 [main] [com.zhiwei.manage.util.ESUtil] [ERROR] - es检索出错，错误信息【{}】
UncategorizedExecutionException[Failed execution]; nested: IOException[can not write type [class java.util.GregorianCalendar]];
at org.elasticsearch.action.support.AdapterActionFuture.rethrowExecutionException(AdapterActionFuture.java:87)
at org.elasticsearch.action.support.AdapterActionFuture.actionGet(AdapterActionFuture.java:46)
at com.zhiwei.manage.util.ESUtil.getDatasMap(ESUtil.java:164)
at com.zhiwei.manage.util.ESUtil.main(ESUtil.java:122)
Caused by: java.io.IOException: can not write type [class java.util.GregorianCalendar]
at org.elasticsearch.common.io.stream.StreamOutput.writeGenericValue(StreamOutput.java:655)
at org.elasticsearch.index.query.RangeQueryBuilder.doWriteTo(RangeQueryBuilder.java:129)
at org.elasticsearch.index.query.AbstractQueryBuilder.writeTo(AbstractQueryBuilder.java:73)
at org.elasticsearch.common.io.stream.StreamOutput.writeNamedWriteable(StreamOutput.java:857)
at org.elasticsearch.common.io.stream.StreamOutput.writeOptionalNamedWriteable(StreamOutput.java:868)
at org.elasticsearch.search.builder.SearchSourceBuilder.writeTo(SearchSourceBuilder.java:237)
at org.elasticsearch.common.io.stream.StreamOutput.writeOptionalWriteable(StreamOutput.java:732)
at org.elasticsearch.action.search.SearchRequest.writeTo(SearchRequest.java:337)
at org.elasticsearch.transport.TcpTransport.buildMessage(TcpTransport.java:1179)
at org.elasticsearch.transport.TcpTransport.sendRequestToChannel(TcpTransport.java:1031)
at org.elasticsearch.transport.TcpTransport.access$1100(TcpTransport.java:113)
at org.elasticsearch.transport.TcpTransport$NodeChannels.sendRequest(TcpTransport.java:435)
at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:554)
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:485)
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:473)
at org.elasticsearch.action.TransportActionNodeProxy.execute(TransportActionNodeProxy.java:51)
at org.elasticsearch.client.transport.TransportProxyClient.lambda$execute$0(TransportProxyClient.java:59)
at org.elasticsearch.client.transport.TransportClientNodesService.execute(TransportClientNodesService.java:247)
at org.elasticsearch.client.transport.TransportProxyClient.execute(TransportProxyClient.java:59)
at org.elasticsearch.client.transport.TransportClient.doExecute(TransportClient.java:356)
at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:403)
at org.elasticsearch.action.ActionRequestBuilder.execute(ActionRequestBuilder.java:80)
at org.elasticsearch.action.ActionRequestBuilder.execute(ActionRequestBuilder.java:54)
... 2 more
2018-05-18 12:07:03,923 [main] [com.zhiwei.manage.util.ESUtil] [ERROR] - es检索出错，错误信息【{}】
UncategorizedExecutionException[Failed execution]; nested: IOException[can not write type [class java.util.GregorianCalendar]];
at org.elasticsearch.action.support.AdapterActionFuture.rethrowExecutionException(AdapterActionFuture.java:87)
at org.elasticsearch.action.support.AdapterActionFuture.actionGet(AdapterActionFuture.java:46)
at com.zhiwei.manage.util.ESUtil.getDatasMap(ESUtil.java:165)
at com.zhiwei.manage.util.ESUtil.main(ESUtil.java:122)
Caused by: java.io.IOException: can not write type [class java.util.GregorianCalendar]
at org.elasticsearch.common.io.stream.StreamOutput.writeGenericValue(StreamOutput.java:655)
at org.elasticsearch.index.query.RangeQueryBuilder.doWriteTo(RangeQueryBuilder.java:129)
at org.elasticsearch.index.query.AbstractQueryBuilder.writeTo(AbstractQueryBuilder.java:73)
at org.elasticsearch.common.io.stream.StreamOutput.writeNamedWriteable(StreamOutput.java:857)
at org.elasticsearch.common.io.stream.StreamOutput.writeOptionalNamedWriteable(StreamOutput.java:868)
at org.elasticsearch.search.builder.SearchSourceBuilder.writeTo(SearchSourceBuilder.java:237)
at org.elasticsearch.common.io.stream.StreamOutput.writeOptionalWriteable(StreamOutput.java:732)
at org.elasticsearch.action.search.SearchRequest.writeTo(SearchRequest.java:337)
at org.elasticsearch.transport.TcpTransport.buildMessage(TcpTransport.java:1179)
at org.elasticsearch.transport.TcpTransport.sendRequestToChannel(TcpTransport.java:1031)
at org.elasticsearch.transport.TcpTransport.access$1100(TcpTransport.java:113)
at org.elasticsearch.transport.TcpTransport$NodeChannels.sendRequest(TcpTransport.java:435)
at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:554)
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:485)
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:473)
at org.elasticsearch.action.TransportActionNodeProxy.execute(TransportActionNodeProxy.java:51)
at org.elasticsearch.client.transport.TransportProxyClient.lambda$execute$0(TransportProxyClient.java:59)
at org.elasticsearch.client.transport.TransportClientNodesService.execute(TransportClientNodesService.java:247)
at org.elasticsearch.client.transport.TransportProxyClient.execute(TransportProxyClient.java:59)
at org.elasticsearch.client.transport.TransportClient.doExecute(TransportClient.java:356)
at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:403)
at org.elasticsearch.action.ActionRequestBuilder.execute(ActionRequestBuilder.java:80)
at org.elasticsearch.action.ActionRequestBuilder.execute(ActionRequestBuilder.java:54)
... 2 more
2018-05-18 12:08:16,688 [main] [com.zhiwei.manage.util.ESUtil] [ERROR] - es检索出错，错误信息【{}】
UncategorizedExecutionException[Failed execution]; nested: IOException[can not write type [class java.util.GregorianCalendar]];
at org.elasticsearch.action.support.AdapterActionFuture.rethrowExecutionException(AdapterActionFuture.java:87)
at org.elasticsearch.action.support.AdapterActionFuture.actionGet(AdapterActionFuture.java:46)
at com.zhiwei.manage.util.ESUtil.getDatasMap(ESUtil.java:164)
at com.zhiwei.manage.util.ESUtil.main(ESUtil.java:122)
Caused by: java.io.IOException: can not write type [class java.util.GregorianCalendar]
at org.elasticsearch.common.io.stream.StreamOutput.writeGenericValue(StreamOutput.java:655)
at org.elasticsearch.index.query.RangeQueryBuilder.doWriteTo(RangeQueryBuilder.java:129)
at org.elasticsearch.index.query.AbstractQueryBuilder.writeTo(AbstractQueryBuilder.java:73)
at org.elasticsearch.common.io.stream.StreamOutput.writeNamedWriteable(StreamOutput.java:857)
at org.elasticsearch.common.io.stream.StreamOutput.writeOptionalNamedWriteable(StreamOutput.java:868)
at org.elasticsearch.search.builder.SearchSourceBuilder.writeTo(SearchSourceBuilder.java:237)
at org.elasticsearch.common.io.stream.StreamOutput.writeOptionalWriteable(StreamOutput.java:732)
at org.elasticsearch.action.search.SearchRequest.writeTo(SearchRequest.java:337)
at org.elasticsearch.transport.TcpTransport.buildMessage(TcpTransport.java:1179)
at org.elasticsearch.transport.TcpTransport.sendRequestToChannel(TcpTransport.java:1031)
at org.elasticsearch.transport.TcpTransport.access$1100(TcpTransport.java:113)
at org.elasticsearch.transport.TcpTransport$NodeChannels.sendRequest(TcpTransport.java:435)
at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:554)
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:485)
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:473)
at org.elasticsearch.action.TransportActionNodeProxy.execute(TransportActionNodeProxy.java:51)
at org.elasticsearch.client.transport.TransportProxyClient.lambda$execute$0(TransportProxyClient.java:59)
at org.elasticsearch.client.transport.TransportClientNodesService.execute(TransportClientNodesService.java:247)
at org.elasticsearch.client.transport.TransportProxyClient.execute(TransportProxyClient.java:59)
at org.elasticsearch.client.transport.TransportClient.doExecute(TransportClient.java:356)
at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:403)
at org.elasticsearch.action.ActionRequestBuilder.execute(ActionRequestBuilder.java:80)
at org.elasticsearch.action.ActionRequestBuilder.execute(ActionRequestBuilder.java:54)
... 2 more
2018-05-18 12:09:45,001 [main] [com.zhiwei.manage.util.ESUtil] [ERROR] - es检索出错，错误信息【{}】
UncategorizedExecutionException[Failed execution]; nested: IOException[can not write type [class java.util.GregorianCalendar]];
at org.elasticsearch.action.support.AdapterActionFuture.rethrowExecutionException(AdapterActionFuture.java:87)
at org.elasticsearch.action.support.AdapterActionFuture.actionGet(AdapterActionFuture.java:46)
at com.zhiwei.manage.util.ESUtil.getDatasMap(ESUtil.java:164)
at com.zhiwei.manage.util.ESUtil.main(ESUtil.java:122)
Caused by: java.io.IOException: can not write type [class java.util.GregorianCalendar]
at org.elasticsearch.common.io.stream.StreamOutput.writeGenericValue(StreamOutput.java:655)
at org.elasticsearch.index.query.RangeQueryBuilder.doWriteTo(RangeQueryBuilder.java:129)
at org.elasticsearch.index.query.AbstractQueryBuilder.writeTo(AbstractQueryBuilder.java:73)
at org.elasticsearch.common.io.stream.StreamOutput.writeNamedWriteable(StreamOutput.java:857)
at org.elasticsearch.common.io.stream.StreamOutput.writeOptionalNamedWriteable(StreamOutput.java:868)
at org.elasticsearch.search.builder.SearchSourceBuilder.writeTo(SearchSourceBuilder.java:237)
at org.elasticsearch.common.io.stream.StreamOutput.writeOptionalWriteable(StreamOutput.java:732)
at org.elasticsearch.action.search.SearchRequest.writeTo(SearchRequest.java:337)
at org.elasticsearch.transport.TcpTransport.buildMessage(TcpTransport.java:1179)
at org.elasticsearch.transport.TcpTransport.sendRequestToChannel(TcpTransport.java:1031)
at org.elasticsearch.transport.TcpTransport.access$1100(TcpTransport.java:113)
at org.elasticsearch.transport.TcpTransport$NodeChannels.sendRequest(TcpTransport.java:435)
at org.elasticsearch.transport.TransportService.sendRequestInternal(TransportService.java:554)
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:485)
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:473)
at org.elasticsearch.action.TransportActionNodeProxy.execute(TransportActionNodeProxy.java:51)
at org.elasticsearch.client.transport.TransportProxyClient.lambda$execute$0(TransportProxyClient.java:59)
at org.elasticsearch.client.transport.TransportClientNodesService.execute(TransportClientNodesService.java:247)
at org.elasticsearch.client.transport.TransportProxyClient.execute(TransportProxyClient.java:59)
at org.elasticsearch.client.transport.TransportClient.doExecute(TransportClient.java:356)
at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:403)
at org.elasticsearch.action.ActionRequestBuilder.execute(ActionRequestBuilder.java:80)
at org.elasticsearch.action.ActionRequestBuilder.execute(ActionRequestBuilder.java:54)
... 2 more
src/main/java/com/zhiwei/manage/bean/NewsDelayed.java
...@@ -156,7 +156,9 @@ public class NewsDelayed implements Serializable { ...@@ -156,7 +156,9 @@ public class NewsDelayed implements Serializable {
return "NewsDelayed [templateId=" + templateId + ", tempName=" + tempName + ", newsDelayed=" + newsDelayed return "NewsDelayed [templateId=" + templateId + ", tempName=" + tempName + ", newsDelayed=" + newsDelayed
+ ", delayedInterval=" + delayedInterval + ", testCount=" + testCount + ", time=" + time + ", savetime=" + ", delayedInterval=" + delayedInterval + ", testCount=" + testCount + ", time=" + time + ", savetime="
+ savetime + ", host=" + host + ", port=" + port + ", collection=" + collection + ", dBName=" + dBName + savetime + ", host=" + host + ", port=" + port + ", collection=" + collection + ", dBName=" + dBName
+ ", pt=" + pt + ", startNow=" + startNow + ", nextRun=" + nextRun + "]"; + ", pt=" + pt + ", startNow=" + startNow + ", nextRun=" + nextRun + ", spyderInfoId=" + spyderInfoId
+ "]";
} }
} }
src/main/java/com/zhiwei/manage/dao/MessageDaoImpl.java
...@@ -59,7 +59,7 @@ public class MessageDaoImpl implements MessageDao { ...@@ -59,7 +59,7 @@ public class MessageDaoImpl implements MessageDao {
for (int j = 0; j < fields.length; j++) { for (int j = 0; j < fields.length; j++) {
fields[j].setAccessible(true); fields[j].setAccessible(true);
try { try {
if (fields[j].getName().equals("templateId")) { if (fields[j].getName().equals("templateId")||fields[j].getName().equals("tempName")) {
continue; continue;
} }
if (fields[j].get(message) != null) if (fields[j].get(message) != null)
...@@ -69,7 +69,7 @@ public class MessageDaoImpl implements MessageDao { ...@@ -69,7 +69,7 @@ public class MessageDaoImpl implements MessageDao {
e.printStackTrace(); e.printStackTrace();
} }
} }
WriteResult result = mongo.upsert(Query.query(Criteria.where("_id").is(message.getMessageId())), update, WriteResult result = mongo.upsert(Query.query(Criteria.where("tempName").is(message.getTempName())), update,
Message.class); Message.class);
if (result.getN() == 1) { if (result.getN() == 1) {
return "修改成功"; return "修改成功";
......
src/main/java/com/zhiwei/manage/handle/MainThread.java
...@@ -48,7 +48,10 @@ public class MainThread extends Thread { ...@@ -48,7 +48,10 @@ public class MainThread extends Thread {
counts.put("微博", 2000); counts.put("微博", 2000);
counts.put("微信", 3500); counts.put("微信", 3500);
counts.put("知乎", 300); counts.put("知乎", 300);
// counts.put("平媒", 300); counts.put("平媒", 300);
counts.put("今日头条", 1000);
counts.put("贴吧", 250);
counts.put("论坛", 30);
} }
public MainThread() { public MainThread() {
...@@ -183,7 +186,7 @@ public class MainThread extends Thread { ...@@ -183,7 +186,7 @@ public class MainThread extends Thread {
map.put(chan.getPt(), chan.getValue()); map.put(chan.getPt(), chan.getValue());
try { try {
ServerBean ser = serverService.findServer(chan.getPt()); ServerBean ser = serverService.findServer(chan.getPt());
count = es.getCountByPt(start.getTime(), end.getTime(), chan.getPt(), chan.getValue(), ser); count = es.getCountByPt(start.getTime(), end.getTime(), chan.getPt(), chan.getValue(), ser);//当日的数据量
System.out.println(count); System.out.println(count);
Data data = new Data(); Data data = new Data();
data.setCount(count); data.setCount(count);
...@@ -191,11 +194,15 @@ public class MainThread extends Thread { ...@@ -191,11 +194,15 @@ public class MainThread extends Thread {
data.setTime(start.getTime()); data.setTime(start.getTime());
dataService.insert(data); dataService.insert(data);
log.info(sdf.format(start.getTime()) + "|" + chan.getPt() + "|入库"); log.info(sdf.format(start.getTime()) + "|" + chan.getPt() + "|入库");
start.setTime(end.getTime());
start.add(Calendar.HOUR, -1);
count = es.getCountByPt(start.getTime(), end.getTime(), chan.getPt(), chan.getValue(), ser);//小时的数据量
if (counts.get(chan.getPt()) != null) if (counts.get(chan.getPt()) != null)
if (count < counts.get(chan.getPt())) { if (count < counts.get(chan.getPt())) {
Messages m = new Messages(); Messages m = new Messages();
m.setTitle("渠道低数据量预警{" + chan.getPt() + "}"); m.setTitle("渠道低数据量预警{" + chan.getPt() + "}");
m.setContent("1小时内的数据量为：" + count); m.setContent("1小时内的数据量为：" + count);
m.setDate(sdf.format(start.getTime()));
for (String mail : Config.getVal("count_mail").split(",")) { for (String mail : Config.getVal("count_mail").split(",")) {
SendMailUtil.sendMessage(mail, m); SendMailUtil.sendMessage(mail, m);
} }
......
src/main/java/com/zhiwei/manage/handle/Task.java
...@@ -21,8 +21,10 @@ import com.zhiwei.manage.bean.Template; ...@@ -21,8 +21,10 @@ import com.zhiwei.manage.bean.Template;
import com.zhiwei.manage.orm.BeanFactory; import com.zhiwei.manage.orm.BeanFactory;
import com.zhiwei.manage.service.MessageService; import com.zhiwei.manage.service.MessageService;
import com.zhiwei.manage.service.NewsService; import com.zhiwei.manage.service.NewsService;
/** /**
* 任务执行 * 任务执行
*
* @author admin * @author admin
* *
*/ */
...@@ -31,8 +33,8 @@ public class Task implements Runnable { ...@@ -31,8 +33,8 @@ public class Task implements Runnable {
private NewsDelayed delayed; private NewsDelayed delayed;
private PingUrl ping; private PingUrl ping;
private FieldIntegerity field; private FieldIntegerity field;
final static SimpleDateFormat sdf=new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS Z"); final static SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS Z");
private final static Log log=LogFactory.getLog(Task.class); private final static Log log = LogFactory.getLog(Task.class);
public Task(Object runMsg) { public Task(Object runMsg) {
if (runMsg instanceof NewsDelayed) { if (runMsg instanceof NewsDelayed) {
...@@ -70,7 +72,7 @@ public class Task implements Runnable { ...@@ -70,7 +72,7 @@ public class Task implements Runnable {
msg.setTemplateLv(tp.getTemplateLv()); msg.setTemplateLv(tp.getTemplateLv());
msg.setErrorType("url"); msg.setErrorType("url");
msg.setPt(ping.getPt()); msg.setPt(ping.getPt());
messageService.update(msg,ping.getPt()); messageService.update(msg, ping.getPt());
} else { } else {
Message msg = messageService.findMsg(ping.getTempName(), "url"); Message msg = messageService.findMsg(ping.getTempName(), "url");
if (msg != null) { if (msg != null) {
...@@ -90,7 +92,7 @@ public class Task implements Runnable { ...@@ -90,7 +92,7 @@ public class Task implements Runnable {
msg.setTemplateLv(1); msg.setTemplateLv(1);
msg.setErrorType("url"); msg.setErrorType("url");
msg.setPt(ping.getPt()); msg.setPt(ping.getPt());
messageService.update(msg,ping.getPt()); messageService.update(msg, ping.getPt());
} }
if (null != ping.getPingUrlTwo() && !ping.getPingUrlTwo().equals("")) if (null != ping.getPingUrlTwo() && !ping.getPingUrlTwo().equals(""))
try { try {
...@@ -111,7 +113,7 @@ public class Task implements Runnable { ...@@ -111,7 +113,7 @@ public class Task implements Runnable {
msg.setTemplateLv(tp.getTemplateLv()); msg.setTemplateLv(tp.getTemplateLv());
msg.setErrorType("url"); msg.setErrorType("url");
msg.setPt(ping.getPt()); msg.setPt(ping.getPt());
messageService.update(msg,ping.getPt()); messageService.update(msg, ping.getPt());
} else { } else {
Message msg = messageService.findMsg(ping.getTempName(), "url"); Message msg = messageService.findMsg(ping.getTempName(), "url");
if (msg != null) { if (msg != null) {
...@@ -131,100 +133,109 @@ public class Task implements Runnable { ...@@ -131,100 +133,109 @@ public class Task implements Runnable {
msg.setPt(ping.getPt()); msg.setPt(ping.getPt());
msg.setErrorType("url"); msg.setErrorType("url");
msg.setTemplateLv(tp.getTemplateLv()); msg.setTemplateLv(tp.getTemplateLv());
messageService.update(msg,ping.getPt()); messageService.update(msg, ping.getPt());
} }
} }
} }
// if (field != null) {//字段缺失率，采集部分已经对缺失字段做了处理，没什么必要检测了 // if (field != null) {//字段缺失率，采集部分已经对缺失字段做了处理，没什么必要检测了
// if (MainThread.mainMap.get(field.getTemplateId()) != null) { // if (MainThread.mainMap.get(field.getTemplateId()) != null) {
// List<Map<String, Object>> list = newsService.findNews(field.getHost(), field.getPort(), // List<Map<String, Object>> list = newsService.findNews(field.getHost(),
// field.getdBName(), field.getCollection(), field.getTestCount(), field.getSpyderInfoId(), // field.getPort(),
// field.getPt()); // field.getdBName(), field.getCollection(), field.getTestCount(),
// if (list.size() ==0) { // field.getSpyderInfoId(),
// Template tp = MainThread.mainMap.get(field.getTemplateId()); // field.getPt());
// Message msg = new Message(); // if (list.size() ==0) {
// msg.setCreateDate(new Date()); // Template tp = MainThread.mainMap.get(field.getTemplateId());
// msg.setErrorMsg("字段检测发现库内的数据为0"); // Message msg = new Message();
// msg.setHandle(false); // msg.setCreateDate(new Date());
// msg.setPt(field.getPt()); // msg.setErrorMsg("字段检测发现库内的数据为0");
// msg.setTemplateCreator(tp.getCreator()); // msg.setHandle(false);
// msg.setTemplateLv(tp.getTemplateLv()); // msg.setPt(field.getPt());
// msg.setErrorType("count2"); // msg.setTemplateCreator(tp.getCreator());
// msg.setTempName(field.getTempName()); // msg.setTemplateLv(tp.getTemplateLv());
// messageService.update(msg,field.getPt()); // msg.setErrorType("count2");
// } else { // msg.setTempName(field.getTempName());
// Message msg = messageService.findMsg(field.getTempName(), "count2"); // messageService.update(msg,field.getPt());
// if (msg != null) { // } else {
// List<String> ids = new ArrayList<>(); // Message msg = messageService.findMsg(field.getTempName(), "count2");
// ids.add(msg.getMessageId()); // if (msg != null) {
// messageService.delete(ids, field.getPt()); // List<String> ids = new ArrayList<>();
// } // ids.add(msg.getMessageId());
// } // messageService.delete(ids, field.getPt());
// String[] fields = field.getField().split(","); // }
// int count = list.size(); // }
// for (String fid : fields) { // String[] fields = field.getField().split(",");
// String str = ""; // int count = list.size();
// int num = count; // for (String fid : fields) {
// for (Map<String, Object> map : list) { // String str = "";
// if (map.get(fid) == null) { // int num = count;
// num--; // for (Map<String, Object> map : list) {
// str = String.valueOf(map.get("_id")); // if (map.get(fid) == null) {
// } else if (String.valueOf(map.get(fid)).length() < 1) { // num--;
// num--; // str = String.valueOf(map.get("_id"));
// str = String.valueOf(map.get("_id")); // } else if (String.valueOf(map.get(fid)).length() < 1) {
// } // num--;
// } // str = String.valueOf(map.get("_id"));
// if (num / 1.0 / count < field.getFieldIntegrityRate() / 100.0) { // }
// Template tp = MainThread.mainMap.get(field.getTemplateId()); // }
// Message msg = new Message(); // if (num / 1.0 / count < field.getFieldIntegrityRate() / 100.0) {
// msg.setCreateDate(new Date()); // Template tp = MainThread.mainMap.get(field.getTemplateId());
// msg.setErrorMsg(fid + "字段存在缺失,完整率" + num / 1.0 / count + "其中一条：" + str); // Message msg = new Message();
// msg.setHandle(false); // msg.setCreateDate(new Date());
// msg.setPt(field.getPt()); // msg.setErrorMsg(fid + "字段存在缺失,完整率" + num / 1.0 / count + "其中一条：" + str);
// msg.setErrorType("field"); // msg.setHandle(false);
// msg.setTemplateCreator(tp.getCreator()); // msg.setPt(field.getPt());
// msg.setTemplateLv(tp.getTemplateLv()); // msg.setErrorType("field");
// msg.setTempName(field.getTempName()); // msg.setTemplateCreator(tp.getCreator());
// messageService.update(msg,field.getPt()); // msg.setTemplateLv(tp.getTemplateLv());
// } else { // msg.setTempName(field.getTempName());
// Message msg = messageService.findMsg(field.getTempName(), "field"); // messageService.update(msg,field.getPt());
// if (msg != null) { // } else {
// List<String> ids = new ArrayList<>(); // Message msg = messageService.findMsg(field.getTempName(), "field");
// ids.add(msg.getMessageId()); // if (msg != null) {
// messageService.delete(ids, field.getPt()); // List<String> ids = new ArrayList<>();
// } // ids.add(msg.getMessageId());
// } // messageService.delete(ids, field.getPt());
// } // }
// } // }
// } // }
// }
// }
if (delayed != null) { if (delayed != null) {
log.info(delayed.getTempName() + "开始检测");
if (MainThread.mainMap.get(delayed.getTemplateId()) != null) { if (MainThread.mainMap.get(delayed.getTemplateId()) != null) {
List<Map<String, Object>> list = newsService.findNews(delayed.getHost(), delayed.getPort(), List<Map<String, Object>> list = newsService.findNews(delayed.getHost(), delayed.getPort(),
delayed.getdBName(), delayed.getCollection(), delayed.getTestCount(), delayed.getSpyderInfoId(), delayed.getdBName(), delayed.getCollection(), delayed.getTestCount(), delayed.getSpyderInfoId(),
delayed.getPt()); delayed.getPt());
log.info("size"+list.size());
long sum = 0; long sum = 0;
if (list.size() ==0) { if (list.size() == 0) {
Template tp = MainThread.mainMap.get(delayed.getTemplateId());
Message msg = new Message();
msg.setCreateDate(new Date());
msg.setErrorMsg("查不到该数据");
msg.setHandle(false);
msg.setPt(delayed.getPt());
msg.setErrorType("count2");
msg.setTemplateCreator(tp.getCreator());
msg.setTemplateLv(1);
msg.setTempName(delayed.getTempName());
messageService.update(msg, delayed.getPt());
} else { } else {
Message msg = messageService.findMsg(delayed.getTempName(), "count2"); Message msg = null;
if (msg != null) {
List<String> ids = new ArrayList<>();
ids.add(msg.getMessageId());
messageService.delete(ids, delayed.getPt());
}
}
double delay = 0; double delay = 0;
List<Double> dNum = new ArrayList<Double>(); List<Double> dNum = new ArrayList<Double>();
for (Map<String, Object> map : list) { for (Map<String, Object> map : list) {
Date time = null; Date time = null;
Date savetime = null; Date savetime = null;
if (map.get(delayed.getTime()) != null) { if (map.get(delayed.getTime()) != null) {
String date=null; String date = null;
try { try {
date = String.valueOf(map.get(delayed.getTime())).replace("Z", " UTC"); date = String.valueOf(map.get(delayed.getTime())).replace("Z", " UTC");
time = sdf.parse(String.valueOf(date)); time = sdf.parse(String.valueOf(date));
} catch (ParseException e) { } catch (ParseException e) {
log.info("时间:"+date); log.info("时间:" + date);
log.error("时间转换出错【{}】",e); log.error("时间转换出错【{}】", e);
} }
} }
if (map.get(delayed.getSavetime()) != null) { if (map.get(delayed.getSavetime()) != null) {
...@@ -235,24 +246,18 @@ public class Task implements Runnable { ...@@ -235,24 +246,18 @@ public class Task implements Runnable {
delay = savetime.getTime() - time.getTime(); delay = savetime.getTime() - time.getTime();
dNum.add(delay); dNum.add(delay);
sum += delay; sum += delay;
Message msg = messageService.findMsg(delayed.getTempName(), "delayed");
if (msg != null) {
List<String> ids = new ArrayList<>();
ids.add(msg.getMessageId());
messageService.delete(ids, delayed.getPt());
}
} else { } else {
Template tp = MainThread.mainMap.get(delayed.getTemplateId()); Template tp = MainThread.mainMap.get(delayed.getTemplateId());
Message msg = new Message(); msg = new Message();
msg.setCreateDate(new Date()); msg.setCreateDate(new Date());
msg.setErrorMsg("延时检测发现时间字段缺失"); msg.setErrorMsg("延时检测发现时间字段缺失");
msg.setHandle(false); msg.setHandle(false);
msg.setPt(delayed.getPt()); msg.setPt(delayed.getPt());
msg.setErrorType("delayed"); msg.setErrorType("timeIsNull");
msg.setTemplateCreator(tp.getCreator()); msg.setTemplateCreator(tp.getCreator());
msg.setTemplateLv(tp.getTemplateLv()); msg.setTemplateLv(tp.getTemplateLv());
msg.setTempName(delayed.getTempName()); msg.setTempName(delayed.getTempName());
messageService.update(msg,delayed.getPt()); messageService.update(msg, delayed.getPt());
} }
} }
Collections.sort(dNum); Collections.sort(dNum);
...@@ -262,7 +267,7 @@ public class Task implements Runnable { ...@@ -262,7 +267,7 @@ public class Task implements Runnable {
} }
if (sum / 1.0 / 1000 / 60 / cutSize > delayed.getNewsDelayed()) { if (sum / 1.0 / 1000 / 60 / cutSize > delayed.getNewsDelayed()) {
Template tp = MainThread.mainMap.get(delayed.getTemplateId()); Template tp = MainThread.mainMap.get(delayed.getTemplateId());
Message msg = new Message(); msg = new Message();
msg.setCreateDate(new Date()); msg.setCreateDate(new Date());
msg.setErrorMsg("延时过高，延时检测为" + sum / 1.0 / 1000 / 60 / cutSize + "分"); msg.setErrorMsg("延时过高，延时检测为" + sum / 1.0 / 1000 / 60 / cutSize + "分");
msg.setHandle(false); msg.setHandle(false);
...@@ -271,9 +276,9 @@ public class Task implements Runnable { ...@@ -271,9 +276,9 @@ public class Task implements Runnable {
msg.setTemplateLv(tp.getTemplateLv()); msg.setTemplateLv(tp.getTemplateLv());
msg.setPt(delayed.getPt()); msg.setPt(delayed.getPt());
msg.setTempName(delayed.getTempName()); msg.setTempName(delayed.getTempName());
messageService.update(msg,delayed.getPt()); messageService.update(msg, delayed.getPt());
} else { } else {
Message msg = messageService.findMsg(delayed.getTempName(), "delayed"); msg = messageService.findMsg(delayed.getTempName(), "delayed");
if (msg != null) { if (msg != null) {
List<String> ids = new ArrayList<>(); List<String> ids = new ArrayList<>();
ids.add(msg.getMessageId()); ids.add(msg.getMessageId());
...@@ -283,4 +288,5 @@ public class Task implements Runnable { ...@@ -283,4 +288,5 @@ public class Task implements Runnable {
} }
} }
} }
}
} }
src/main/java/com/zhiwei/manage/service/NewsServiceImpl.java
...@@ -16,7 +16,6 @@ public class NewsServiceImpl implements NewsService{ ...@@ -16,7 +16,6 @@ public class NewsServiceImpl implements NewsService{
@Autowired @Autowired
private NewsDao newsDao; private NewsDao newsDao;
private ESUtil es=new ESUtil();
@Override @Override
public List<News> findByNew(int delayedCount, String type, String source) { public List<News> findByNew(int delayedCount, String type, String source) {
...@@ -52,7 +51,7 @@ public class NewsServiceImpl implements NewsService{ ...@@ -52,7 +51,7 @@ public class NewsServiceImpl implements NewsService{
@Override @Override
public List<Map<String, Object>> findNews(String host, int port, String dbNamel, String collection, int count, public List<Map<String, Object>> findNews(String host, int port, String dbNamel, String collection, int count,
String id,String pt) { String id,String pt) {
return es.getDatasMap(host, port, dbNamel, collection, count, id, pt); return ESUtil.getDatasMap(host, port, dbNamel, collection, count, id, pt);
} }
......
src/main/java/com/zhiwei/manage/util/ESUtil.java
...@@ -119,7 +119,8 @@ public class ESUtil { ...@@ -119,7 +119,8 @@ public class ESUtil {
// System.out.println(searchHits.getTotalHits()); // System.out.println(searchHits.getTotalHits());
SimpleDateFormat sdf=new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS Z"); SimpleDateFormat sdf=new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS Z");
for(Map<String,Object> map:getDatasMap("1.119.44.201", 9300, "Media-University", "mediaspider*", 20, "UrlListDetail74ec93ff-2275-4f22-bb62-bae580f2d43b", "网媒")) { for(Map<String,Object> map: getDatasMap("1.119.44.201", 9300, "Media-University", "mediaspider*"
, 20, "UrlListDetaildb4b9042-006f-4d09-a643-de379ac07e15", "平媒")) {
System.out.println(map.get("title")); System.out.println(map.get("title"));
String date = map.get("time").toString().replace("Z", " UTC"); String date = map.get("time").toString().replace("Z", " UTC");
// System.out.println(map.get("time")); // System.out.println(map.get("time"));
...@@ -144,20 +145,22 @@ public class ESUtil { ...@@ -144,20 +145,22 @@ public class ESUtil {
String id,String pt) { String id,String pt) {
List<Map<String,Object>> list=new ArrayList<>(); List<Map<String,Object>> list=new ArrayList<>();
Client esClient=null; Client esClient=null;
Calendar cal=Calendar.getInstance();
cal.add(Calendar.DATE, -7);
try { try {
if (DbDepot.esClients.get(pt) == null) { if (DbDepot.esClients==null||DbDepot.esClients.get(pt) == null) {
esClient = new ESClient(dbNamel, host, port).getClient(); esClient = new ESClient(dbNamel, host, port).getClient();
DbDepot.esClients.put(pt, esClient); // DbDepot.esClients.put(pt, esClient);
} else { } else {
esClient = (Client) DbDepot.esClients.get(pt); esClient = (Client) DbDepot.esClients.get(pt);
} }
// 搜索数据 // 搜索数据
SearchRequestBuilder srb = esClient.prepareSearch(collection); SearchRequestBuilder srb = esClient.prepareSearch(collection);
srb.setPostFilter(QueryBuilders.rangeQuery("time").gte(getNeedTime(cal)).lte(getNeedTime(Calendar.getInstance())));
BoolQueryBuilder bb = new BoolQueryBuilder(); BoolQueryBuilder bb = new BoolQueryBuilder();
srb.setQuery(bb);
srb.addSort("time", SortOrder.DESC);
QueryBuilder matchQuery = QueryBuilders.matchPhraseQuery("spyderInfoId",id); QueryBuilder matchQuery = QueryBuilders.matchPhraseQuery("spyderInfoId",id);
bb.should(matchQuery); bb.should(matchQuery);
srb.setQuery(bb);
srb.setSize(count); srb.setSize(count);
SearchResponse response = srb.execute().actionGet(); SearchResponse response = srb.execute().actionGet();
SearchHits searchHits = response.getHits(); SearchHits searchHits = response.getHits();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment