Skip to content

D
datamanage
Commit 63fcf6fa by admin
Options

预警部分的数据修改

parent 8943d8e5
Showing with 170 additions and 104 deletions
src/main/java/com/zhiwei/manage/dao/NewsDao.java
......@@ -21,5 +21,5 @@ public interface NewsDao {
public List<EasyNews> finAllNews(Date start,Date end);
public List<Map<String,Object>> findNews(String host,int port,String dbNamel,String collection,int count,String id,String pt);
// public List<Map<String,Object>> findNews(String host,int port,String dbNamel,String collection,int count,String id,String pt);
}
src/main/java/com/zhiwei/manage/dao/NewsDaoImpl.java
......@@ -116,32 +116,32 @@ public class NewsDaoImpl implements NewsDao {
/**
* 获取新闻检测
*/
@SuppressWarnings("unchecked")
@Override
public List<Map<String, Object>> findNews(String host, int port, String dbNamel, String collection, int count,
String id, String pt) {
Calendar cal = Calendar.getInstance();
cal.add(Calendar.DATE, -1);
int month = cal.get(Calendar.MONTH) + 1;
String end = cal.get(Calendar.YEAR) + "" + (month > 9 ? month : "0" + month);
List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
DBCollection con = mongo.getCollection(collection + end);
if (id != null && !id.equals("")) {
DBCursor cur = con.find(new BasicDBObject("spyderInfoId", id)).sort(new BasicDBObject("rsid", -1))
.limit(count);
while (cur.hasNext()) {
Map<String, Object> map = (Map<String, Object>) cur.next();
list.add(map);
}
} else {
DBCursor cur = con.find().sort(new BasicDBObject("rsid", -1))
.limit(count);
while (cur.hasNext()) {
Map<String, Object> map = (Map<String, Object>) cur.next();
list.add(map);
}
}
return list;
}
// @SuppressWarnings("unchecked")
// @Override
// public List<Map<String, Object>> findNews(String host, int port, String dbNamel, String collection, int count,
// String id, String pt) {
// Calendar cal = Calendar.getInstance();
// cal.add(Calendar.DATE, -1);
// int month = cal.get(Calendar.MONTH) + 1;
// String end = cal.get(Calendar.YEAR) + "" + (month > 9 ? month : "0" + month);
// List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
// DBCollection con = mongo.getCollection(collection + end);
// if (id != null && !id.equals("")) {
// DBCursor cur = con.find(new BasicDBObject("spyderInfoId", id)).sort(new BasicDBObject("rsid", -1))
// .limit(count);
// while (cur.hasNext()) {
// Map<String, Object> map = (Map<String, Object>) cur.next();
// list.add(map);
// }
// } else {
// DBCursor cur = con.find().sort(new BasicDBObject("rsid", -1))
// .limit(count);
// while (cur.hasNext()) {
// Map<String, Object> map = (Map<String, Object>) cur.next();
// list.add(map);
// }
// }
// return list;
// }
}
src/main/java/com/zhiwei/manage/handle/Task.java
......@@ -2,12 +2,17 @@ package com.zhiwei.manage.handle;
import java.net.HttpURLConnection;
import java.net.URL;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.zhiwei.manage.bean.FieldIntegerity;
import com.zhiwei.manage.bean.Message;
import com.zhiwei.manage.bean.NewsDelayed;
......@@ -16,12 +21,18 @@ import com.zhiwei.manage.bean.Template;
import com.zhiwei.manage.orm.BeanFactory;
import com.zhiwei.manage.service.MessageService;
import com.zhiwei.manage.service.NewsService;
/**
* 任务执行
* @author admin
*
*/
public class Task implements Runnable {
private NewsDelayed delayed;
private PingUrl ping;
private FieldIntegerity field;
final static SimpleDateFormat sdf=new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS Z");
private final static Log log=LogFactory.getLog(Task.class);
public Task(Object runMsg) {
if (runMsg instanceof NewsDelayed) {
......@@ -35,6 +46,7 @@ public class Task implements Runnable {
@Override
public void run() {
System.out.println("预警部分启动");
MessageService messageService = BeanFactory.getBean(MessageService.class);
NewsService newsService = BeanFactory.getBean(NewsService.class);
// 连通性测试
......@@ -124,68 +136,68 @@ public class Task implements Runnable {
}
}
}
if (field != null) {
if (MainThread.mainMap.get(field.getTemplateId()) != null) {
List<Map<String, Object>> list = newsService.findNews(field.getHost(), field.getPort(),
field.getdBName(), field.getCollection(), field.getTestCount(), field.getSpyderInfoId(),
field.getPt());
if (list.size() ==0) {
Template tp = MainThread.mainMap.get(field.getTemplateId());
Message msg = new Message();
msg.setCreateDate(new Date());
msg.setErrorMsg("字段检测发现库内的数据为0");
msg.setHandle(false);
msg.setPt(field.getPt());
msg.setTemplateCreator(tp.getCreator());
msg.setTemplateLv(tp.getTemplateLv());
msg.setErrorType("count2");
msg.setTempName(field.getTempName());
messageService.update(msg,field.getPt());
} else {
Message msg = messageService.findMsg(field.getTempName(), "count2");
if (msg != null) {
List<String> ids = new ArrayList<>();
ids.add(msg.getMessageId());
messageService.delete(ids, field.getPt());
}
}
String[] fields = field.getField().split(",");
int count = list.size();
for (String fid : fields) {
String str = "";
int num = count;
for (Map<String, Object> map : list) {
if (map.get(fid) == null) {
num--;
str = String.valueOf(map.get("_id"));
} else if (String.valueOf(map.get(fid)).length() < 1) {
num--;
str = String.valueOf(map.get("_id"));
}
}
if (num / 1.0 / count < field.getFieldIntegrityRate() / 100.0) {
Template tp = MainThread.mainMap.get(field.getTemplateId());
Message msg = new Message();
msg.setCreateDate(new Date());
msg.setErrorMsg(fid + "字段存在缺失,完整率" + num / 1.0 / count + "其中一条：" + str);
msg.setHandle(false);
msg.setPt(field.getPt());
msg.setErrorType("field");
msg.setTemplateCreator(tp.getCreator());
msg.setTemplateLv(tp.getTemplateLv());
msg.setTempName(field.getTempName());
messageService.update(msg,field.getPt());
} else {
Message msg = messageService.findMsg(field.getTempName(), "field");
if (msg != null) {
List<String> ids = new ArrayList<>();
ids.add(msg.getMessageId());
messageService.delete(ids, field.getPt());
}
}
}
}
}
// if (field != null) {//字段缺失率，采集部分已经对缺失字段做了处理，没什么必要检测了
// if (MainThread.mainMap.get(field.getTemplateId()) != null) {
// List<Map<String, Object>> list = newsService.findNews(field.getHost(), field.getPort(),
// field.getdBName(), field.getCollection(), field.getTestCount(), field.getSpyderInfoId(),
// field.getPt());
// if (list.size() ==0) {
// Template tp = MainThread.mainMap.get(field.getTemplateId());
// Message msg = new Message();
// msg.setCreateDate(new Date());
// msg.setErrorMsg("字段检测发现库内的数据为0");
// msg.setHandle(false);
// msg.setPt(field.getPt());
// msg.setTemplateCreator(tp.getCreator());
// msg.setTemplateLv(tp.getTemplateLv());
// msg.setErrorType("count2");
// msg.setTempName(field.getTempName());
// messageService.update(msg,field.getPt());
// } else {
// Message msg = messageService.findMsg(field.getTempName(), "count2");
// if (msg != null) {
// List<String> ids = new ArrayList<>();
// ids.add(msg.getMessageId());
// messageService.delete(ids, field.getPt());
// }
// }
// String[] fields = field.getField().split(",");
// int count = list.size();
// for (String fid : fields) {
// String str = "";
// int num = count;
// for (Map<String, Object> map : list) {
// if (map.get(fid) == null) {
// num--;
// str = String.valueOf(map.get("_id"));
// } else if (String.valueOf(map.get(fid)).length() < 1) {
// num--;
// str = String.valueOf(map.get("_id"));
// }
// }
// if (num / 1.0 / count < field.getFieldIntegrityRate() / 100.0) {
// Template tp = MainThread.mainMap.get(field.getTemplateId());
// Message msg = new Message();
// msg.setCreateDate(new Date());
// msg.setErrorMsg(fid + "字段存在缺失,完整率" + num / 1.0 / count + "其中一条：" + str);
// msg.setHandle(false);
// msg.setPt(field.getPt());
// msg.setErrorType("field");
// msg.setTemplateCreator(tp.getCreator());
// msg.setTemplateLv(tp.getTemplateLv());
// msg.setTempName(field.getTempName());
// messageService.update(msg,field.getPt());
// } else {
// Message msg = messageService.findMsg(field.getTempName(), "field");
// if (msg != null) {
// List<String> ids = new ArrayList<>();
// ids.add(msg.getMessageId());
// messageService.delete(ids, field.getPt());
// }
// }
// }
// }
// }
if (delayed != null) {
if (MainThread.mainMap.get(delayed.getTemplateId()) != null) {
List<Map<String, Object>> list = newsService.findNews(delayed.getHost(), delayed.getPort(),
......@@ -207,7 +219,12 @@ public class Task implements Runnable {
Date time = null;
Date savetime = null;
if (map.get(delayed.getTime()) != null) {
time = (Date) map.get(delayed.getTime());
try {
String date = String.valueOf(map.get(delayed.getTime())).replace("Z", " UTC");
time = sdf.parse(String.valueOf(date));
} catch (ParseException e) {
log.error("时间转换出错【{}】",e);
}
}
if (map.get(delayed.getSavetime()) != null) {
Date d = new Date((Long) map.get(delayed.getSavetime()));
......
src/main/java/com/zhiwei/manage/service/NewsServiceImpl.java
......@@ -10,12 +10,13 @@ import org.springframework.stereotype.Service;
import com.zhiwei.manage.bean.EasyNews;
import com.zhiwei.manage.bean.News;
import com.zhiwei.manage.dao.NewsDao;
import com.zhiwei.manage.util.ESUtil;
@Service
public class NewsServiceImpl implements NewsService{
@Autowired
private NewsDao newsDao;
private ESUtil es=new ESUtil();
@Override
public List<News> findByNew(int delayedCount, String type, String source) {
......@@ -51,7 +52,7 @@ public class NewsServiceImpl implements NewsService{
@Override
public List<Map<String, Object>> findNews(String host, int port, String dbNamel, String collection, int count,
String id,String pt) {
return newsDao.findNews(host, port, dbNamel, collection, count, id, pt);
return es.getDatasMap(host, port, dbNamel, collection, count, id, pt);
}
......
src/main/java/com/zhiwei/manage/util/ESUtil.java
......@@ -18,9 +18,12 @@ import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.sort.SortBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Sort;
import org.springframework.stereotype.Component;
import com.zhiwei.manage.bean.Data;
......@@ -93,22 +96,67 @@ public class ESUtil {
}
return 0;
}
public static void main(String[] args) throws ParseException {
Client esClient = null;
Calendar cal=Calendar.getInstance();
cal.add(Calendar.HOUR_OF_DAY, -24);
esClient = new ESClient("Media-University","1.119.44.201", 9300).getClient();
SearchRequestBuilder srb = esClient.prepareSearch("mediaspider*");
// Client esClient = null;
// Calendar cal=Calendar.getInstance();
// cal.add(Calendar.HOUR_OF_DAY, -24);
// esClient = new ESClient("Media-University","1.119.44.201", 9300).getClient();
// SearchRequestBuilder srb = esClient.prepareSearch("mediaspider*");
// BoolQueryBuilder bb = new BoolQueryBuilder();
// srb.setQuery(bb);
// QueryBuilder matchQuery = QueryBuilders.matchPhraseQuery("type", "今日头条");
// bb.should(matchQuery);
// srb.setSize(1);
// srb.setPostFilter(QueryBuilders.rangeQuery("time").gte(getNeedTime(cal)).lte(getNeedTime(Calendar.getInstance())));
// SearchResponse response = srb.execute().actionGet();
// SearchHits searchHits = response.getHits();
// System.out.println(searchHits.getTotalHits());
SimpleDateFormat sdf=new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS Z");
for(Map<String,Object> map:getDatasMap("1.119.44.201", 9300, "Media-University", "mediaspider*", 20, "UrlListDetail74ec93ff-2275-4f22-bb62-bae580f2d43b", "网媒")) {
System.out.println(map.get("title"));
String date = map.get("time").toString().replace("Z", " UTC");
// System.out.println(map.get("time"));
System.out.println(sdf.parse(date));
System.out.println((long)map.get("savetime"));
}
}
/**
* 数据查询。获取数据
*
* @param start
* @param end
* @param pt
* @param type
* @param value
* @param spyderInfoId
* @return
*/
public static List<Map<String, Object>> getDatasMap(String host, int port, String dbNamel, String collection, int count,
String id,String pt) {
List<Map<String,Object>> list=new ArrayList<>();
try {
Client esClient = new ESClient(dbNamel, host, port).getClient();
// 搜索数据
SearchRequestBuilder srb = esClient.prepareSearch(collection);
BoolQueryBuilder bb = new BoolQueryBuilder();
srb.setQuery(bb);
QueryBuilder matchQuery = QueryBuilders.matchPhraseQuery("type", "今日头条");
srb.addSort("time", SortOrder.DESC);
QueryBuilder matchQuery = QueryBuilders.matchPhraseQuery("spyderInfoId",id);
bb.should(matchQuery);
srb.setSize(1);
srb.setPostFilter(QueryBuilders.rangeQuery("time").gte(getNeedTime(cal)).lte(getNeedTime(Calendar.getInstance())));
srb.setSize(count);
SearchResponse response = srb.execute().actionGet();
SearchHits searchHits = response.getHits();
System.out.println(searchHits.getTotalHits());
for(SearchHit result:searchHits) {
Map<String,Object> map=result.getSource();
list.add(map);
}
} catch (Exception e) {
log.error("es检索出错，错误信息【{}】", e);
}
return list;
}
public static String getNeedTime(Calendar cal) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment