Skip to content

D
datamanage
Commit 7af303ba by admin
Options

预警修改，平媒取消

parent c6fb8c18
Showing with 44 additions and 49 deletions
src/main/java/com/zhiwei/manage/handle/MainThread.java
......@@ -12,6 +12,7 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.stereotype.Component;
......@@ -20,19 +21,15 @@ import com.zhiwei.manage.bean.Channel;
import com.zhiwei.manage.bean.Data;
import com.zhiwei.manage.bean.FieldIntegerity;
import com.zhiwei.manage.bean.Message;
import com.zhiwei.manage.bean.Messages;
import com.zhiwei.manage.bean.NewsDelayed;
import com.zhiwei.manage.bean.PingUrl;
import com.zhiwei.manage.bean.ServerBean;
import com.zhiwei.manage.bean.Template;
import com.zhiwei.manage.orm.BeanFactory;
import com.zhiwei.manage.service.CrawTemplateServiceImpl;
import com.zhiwei.manage.service.DataServiceImpl;
import com.zhiwei.manage.service.MessageServiceImpl;
import com.zhiwei.manage.service.ServerServiceImpl;
import com.zhiwei.manage.util.Config;
import com.zhiwei.manage.util.ESUtil;
import com.zhiwei.manage.util.SendMailUtil;
import com.zhiwei.manage.util.TimeUtil;
/**
......@@ -55,7 +52,7 @@ public class MainThread extends Thread {
counts.put("微博", 2000);
counts.put("微信", 3500);
counts.put("知乎", 300);
counts.put("平媒", 300);
// counts.put("平媒", 300);
counts.put("今日头条", 1000);
counts.put("贴吧", 250);
counts.put("论坛", 30);
......@@ -191,37 +188,39 @@ public class MainThread extends Thread {
end.setTime(sdf.parse(sdf.format(end.getTime())));
int count = 0;
channel = serverService.findDisChannel();
for (Channel chan : channel) {
map.put(chan.getPt(), chan.getValue());
try {
ServerBean ser = serverService.findServer(chan.getPt());
count = es.getCountByPt(start.getTime(), end.getTime(), chan.getPt(), chan.getValue(), ser);// 当日的数据量
System.out.println(count);
Data data = new Data();
data.setCount(count);
data.setPt(chan.getPt());
data.setTime(start.getTime());
dataService.insert(data);
log.info(sdf.format(start.getTime()) + "|" + chan.getPt() + "|入库");
count = es.getCountByPt(inTime.getTime(), end.getTime(), chan.getPt(), chan.getValue(), ser);// 小时的数据量
if (inTime.get(Calendar.HOUR_OF_DAY) > 6)
if (counts.get(chan.getPt()) != null)
if (count < counts.get(chan.getPt())) {
Messages m = new Messages();
m.setTitle("渠道低数据量预警{" + chan.getPt() + "}");
m.setContent("1小时内的数据量为：" + count);
m.setDate(sdf.format(start.getTime()));
for (String mail : Config.getVal("count_mail").split(",")) {
SendMailUtil.sendMessage(mail, m);
}
}
} catch (Exception e) {
log.error(chan.getPt() + "入库出错，错误信息{}", e);
}
}
// for (Channel chan : channel) {
// map.put(chan.getPt(), chan.getValue());
// try {
// ServerBean ser = serverService.findServer(chan.getPt());
// count = es.getCountByPt(start.getTime(), end.getTime(), chan.getPt(),
// chan.getValue(), ser);// 当日的数据量
// System.out.println(count);
// Data data = new Data();
// data.setCount(count);
// data.setPt(chan.getPt());
// data.setTime(start.getTime());
// dataService.insert(data);
// log.info(sdf.format(start.getTime()) + "|" + chan.getPt() + "|入库");
// count = es.getCountByPt(inTime.getTime(), end.getTime(), chan.getPt(),
// chan.getValue(), ser);// 小时的数据量
// if (inTime.get(Calendar.HOUR_OF_DAY) > 6)
// if (counts.get(chan.getPt()) != null)
// if (count < counts.get(chan.getPt())) {
// Messages m = new Messages();
// m.setTitle("渠道低数据量预警{" + chan.getPt() + "}");
// m.setContent("1小时内的数据量为：" + count);
// m.setDate(sdf.format(start.getTime()));
// for (String mail : Config.getVal("count_mail").split(",")) {
// SendMailUtil.sendMessage(mail, m);
// }
// }
// } catch (Exception e) {
// log.error(chan.getPt() + "入库出错，错误信息{}", e);
// }
// }
Thread.sleep(1000 * 120);
Set<String> keys = allTmp.keySet();
List<String> needRemoveWm = new ArrayList<>();
List<String> needRemovePm = new ArrayList<>();
List<String> needRemove = new ArrayList<>();
for (Iterator<String> it = keys.iterator(); it.hasNext();) {
String key = it.next();
Template tmp = allTmp.get(key);
......@@ -243,7 +242,11 @@ public class MainThread extends Thread {
cwCount = es.getCounts(start.getTime(), end.getTime(), pt, map.get(pt),
tmp.getSpyderInfoId(), serverService.findServer(pt));
if (cwCount < 3) {
if (StringUtils.isBlank(tmp.getTempName())) {
continue;
}
Message msg = new Message();
msg.setTempName(tmp.getTempName());
msg.setCreateDate(new Date());
msg.setErrorMsg("连续三天数据不足3条，请确认采集是否有问题");
msg.setPt(pt);
......@@ -252,31 +255,23 @@ public class MainThread extends Thread {
} else {
Message msg = messageService.findMsg(tmp.getTempName(), null);
if (msg != null) {
if (msg.getPt().equals("网媒")) {
needRemoveWm.add(msg.getMessageId());
} else {
needRemovePm.add(msg.getMessageId());
}
needRemove.add(msg.getMessageId());
}
}
} else {
Message msg = messageService.findMsg(tmp.getTempName(), null);
if (msg != null) {
if (msg.getPt().equals("网媒")) {
needRemoveWm.add(msg.getMessageId());
} else {
needRemovePm.add(msg.getMessageId());
}
needRemove.add(msg.getMessageId());
}
}
} catch (Exception e) {
log.error(tmp.getTempName() + "入库出错，错误信息{}", e);
}
}
sleep(1000);
}
// 有数据了，移除错误信息
messageService.delete(needRemovePm);
messageService.delete(needRemoveWm);
messageService.delete(needRemove);
} catch (Exception e) {
e.printStackTrace();
}
......
src/main/resources/mongo.properties
......@@ -32,4 +32,4 @@ wechat.indexName=mediaspider*
mobile.indexName=mediaspider*
toutiao.indexName=mediaspider*
###########################################
count_mail=shenjinzhu999@163.com,zhangzhiwei@zhiweidata.com
\ No newline at end of file
count_mail=zhangzhiwei@zhiweidata.com
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment